Better behaviour in the presence of 429s #786
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| /* | ||
| * Copyright 2018 Palantir Technologies, Inc. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.palantir.remoting3.okhttp; | ||
|
Contributor
There was a problem hiding this comment. @iamdanfox what's the deal with remoting-vs-conjure in PRs? |
||
|
|
||
| import com.google.common.annotations.VisibleForTesting; | ||
| import com.netflix.concurrency.limits.Limiter; | ||
| import com.netflix.concurrency.limits.limit.VegasLimit; | ||
| import com.netflix.concurrency.limits.limiter.SimpleLimiter; | ||
| import com.palantir.logsafe.SafeArg; | ||
| import com.palantir.remoting3.tracing.okhttp3.OkhttpTraceInterceptor; | ||
| import java.time.Duration; | ||
| import java.util.Optional; | ||
| import java.util.concurrent.ConcurrentHashMap; | ||
| import java.util.concurrent.ConcurrentMap; | ||
| import okhttp3.Request; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| @SuppressWarnings("DesignForExtension") | ||
| class ConcurrencyLimiters { | ||
| private static final Logger log = LoggerFactory.getLogger(ConcurrencyLimiters.class); | ||
| private static final Duration DEFAULT_TIMEOUT = Duration.ofMinutes(1); | ||
| private static final Void NO_CONTEXT = null; | ||
| private static final String FALLBACK = ""; | ||
|
|
||
| private final ConcurrentMap<String, Limiter<Void>> limiters = new ConcurrentHashMap<>(); | ||
| private final Duration timeout; | ||
|
|
||
| @VisibleForTesting | ||
| ConcurrencyLimiters(Duration timeout) { | ||
| this.timeout = timeout; | ||
| } | ||
|
|
||
| ConcurrencyLimiters() { | ||
| this(DEFAULT_TIMEOUT); | ||
| } | ||
|
|
||
| Limiter.Listener limiter(Request request) { | ||
| return limiter(limiterKey(request)); | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| Limiter.Listener limiter(String name) { | ||
| Limiter<Void> limiter = limiters.computeIfAbsent(name, key -> newLimiter()); | ||
| Optional<Limiter.Listener> listener = limiter.acquire(NO_CONTEXT); | ||
| return listener.orElseGet(() -> { | ||
| if (Thread.currentThread().isInterrupted()) { | ||
| throw new RuntimeException("Thread was interrupted"); | ||
| } | ||
| log.warn("Timed out waiting to get permits for concurrency. In most cases this would indicate " | ||
| + "some kind of deadlock. We expect that either this is caused by not closing response " | ||
| + "bodies (there should be OkHttp log lines indicating this), or service overloading.", | ||
| SafeArg.of("timeout", timeout)); | ||
| limiters.replace(name, limiter, newLimiter()); | ||
| return limiter(name); | ||
| }); | ||
| } | ||
|
|
||
| private Limiter<Void> newLimiter() { | ||
| Limiter<Void> limiter = SimpleLimiter.newBuilder() | ||
| .limit(new RemotingWindowedLimit(VegasLimit.newDefault())) | ||
| .build(); | ||
| return RemotingBlockingLimiter.wrap(limiter, timeout); | ||
| } | ||
|
|
||
| private static String limiterKey(Request request) { | ||
| String pathTemplate = request.header(OkhttpTraceInterceptor.PATH_TEMPLATE_HEADER); | ||
| if (pathTemplate == null) { | ||
| return FALLBACK; | ||
| } else { | ||
| return request.method() + " " + pathTemplate; | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| /* | ||
| * Copyright 2018 Palantir Technologies, Inc. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.palantir.remoting3.okhttp; | ||
|
|
||
| import com.google.common.annotations.VisibleForTesting; | ||
| import com.google.common.collect.ImmutableSet; | ||
| import com.netflix.concurrency.limits.Limiter; | ||
| import java.io.IOException; | ||
| import java.lang.reflect.InvocationHandler; | ||
| import java.lang.reflect.InvocationTargetException; | ||
| import java.lang.reflect.Method; | ||
| import java.lang.reflect.Proxy; | ||
| import okhttp3.Interceptor; | ||
| import okhttp3.Response; | ||
| import okhttp3.ResponseBody; | ||
| import okio.BufferedSource; | ||
|
|
||
| /** | ||
| * Flow control in Conjure is a collaborative effort between servers and clients. Servers advertise an overloaded state | ||
| * via 429/503 responses, and clients throttle the number of requests that they send concurrently as a response to this. | ||
| * The latter is implemented as a combination of two techniques, yielding a mechanism similar to flow control in TCP/IP. | ||
| * <ol> | ||
| * <li> | ||
| * Clients use the frequency of 429/503 responses (as well as the request latency) to determine an estimate | ||
| * for the number of permissible concurrent requests | ||
| * </li> | ||
| * <li> | ||
| * Each such request gets scheduled according to an exponential backoff algorithm. | ||
| * </li> | ||
| * </ol> | ||
| * <p> | ||
| * This class provides an asynchronous implementation of Netflix's | ||
| * <a href="https://github.com/Netflix/concurrency-limits/">concurrency-limits</a> library for determining the | ||
| * above mentioned concurrency estimates. | ||
| * <p> | ||
| * In order to use this class, one should acquire a Limiter for their request, which returns a future. once the Future | ||
| * is completed, the caller can assume that the request is schedulable. After the request completes, the caller | ||
| * <b>must</b> call one of the methods on {@link Limiter.Listener} in order to provide feedback about the request's | ||
| * success. If this is not done, a deadlock could result. | ||
| */ | ||
| final class ConcurrencyLimitingInterceptor implements Interceptor { | ||
| private static final ImmutableSet<Integer> DROPPED_CODES = ImmutableSet.of(429, 503); | ||
|
|
||
| private final ConcurrencyLimiters limiters; | ||
|
|
||
| @VisibleForTesting | ||
| ConcurrencyLimitingInterceptor(ConcurrencyLimiters limiters) { | ||
| this.limiters = limiters; | ||
| } | ||
|
|
||
| ConcurrencyLimitingInterceptor() { | ||
| this(new ConcurrencyLimiters()); | ||
| } | ||
|
|
||
| @Override | ||
| public Response intercept(Chain chain) throws IOException { | ||
| Limiter.Listener listener = limiters.limiter(chain.request()); | ||
| try { | ||
| Response response = chain.proceed(chain.request()); | ||
| if (DROPPED_CODES.contains(response.code())) { | ||
| listener.onDropped(); | ||
| return response; | ||
| } else if (!response.isSuccessful() || response.isRedirect()) { | ||
| listener.onIgnore(); | ||
| return response; | ||
| } else { | ||
| return wrapResponse(listener, response); | ||
| } | ||
| } catch (IOException e) { | ||
| listener.onIgnore(); | ||
| throw e; | ||
| } | ||
| } | ||
|
|
||
| private static Response wrapResponse(Limiter.Listener listener, Response response) { | ||
| if (response.body() == null) { | ||
| return response; | ||
| } | ||
| ResponseBody currentBody = response.body(); | ||
| ResponseBody newResponseBody = | ||
| ResponseBody.create( | ||
| currentBody.contentType(), | ||
| currentBody.contentLength(), | ||
| wrapSource(currentBody.source(), listener)); | ||
| return response.newBuilder() | ||
| .body(newResponseBody) | ||
| .build(); | ||
| } | ||
|
|
||
| private static BufferedSource wrapSource(BufferedSource currentSource, Limiter.Listener listener) { | ||
| return (BufferedSource) Proxy.newProxyInstance( | ||
| BufferedSource.class.getClassLoader(), | ||
| new Class<?>[] { BufferedSource.class }, | ||
| new ReleaseConcurrencyLimitProxy(currentSource, listener)); | ||
| } | ||
|
|
||
| /** | ||
| * This proxy enables e.g. Okio to make additive additions to their API without breaking us. | ||
| */ | ||
| private static final class ReleaseConcurrencyLimitProxy implements InvocationHandler { | ||
| private final BufferedSource delegate; | ||
| private final Limiter.Listener listener; | ||
| private boolean closed = false; | ||
|
|
||
| private ReleaseConcurrencyLimitProxy(BufferedSource delegate, Limiter.Listener listener) { | ||
| this.delegate = delegate; | ||
| this.listener = listener; | ||
| } | ||
|
|
||
| @Override | ||
| public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { | ||
| if (method.getName().equals("close") && !closed) { | ||
| closed = true; | ||
| listener.onSuccess(); | ||
|
Contributor
There was a problem hiding this comment. So the closing mechanism is now implicit, in that as long as the response body is closed (either by reading it fully during json object mapping, or via Is it viable at some point to have more control over this? One thing that occasionally happens when streaming is your stream ends too early because an error was encountered once some data (esp. headers) was already sent. We would preferably mark those as failed. |
||
| } | ||
|
|
||
| try { | ||
| return method.invoke(delegate, args); | ||
| } catch (InvocationTargetException e) { | ||
| throw e.getCause(); | ||
| } | ||
| } | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@samrogerson fysa
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
uhm, I thought we had merged such a change already?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
yes, but we did it in only one of the two places (see above in this class)