Metric

okhttp-clients/src/main/java/com/palantir/remoting3/okhttp/ConcurrencyLimiters.java

@@ -16,16 +16,20 @@
 
 package com.palantir.remoting3.okhttp;
 
+import com.codahale.metrics.Timer;
 import com.google.common.annotations.VisibleForTesting;
 import com.netflix.concurrency.limits.Limiter;
 import com.netflix.concurrency.limits.limit.VegasLimit;
 import com.netflix.concurrency.limits.limiter.SimpleLimiter;
 import com.palantir.logsafe.SafeArg;
 import com.palantir.remoting3.tracing.okhttp3.OkhttpTraceInterceptor;
+import com.palantir.tritium.metrics.registry.MetricName;
+import com.palantir.tritium.metrics.registry.TaggedMetricRegistry;
 import java.time.Duration;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
 import okhttp3.Request;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,17 +40,21 @@ class ConcurrencyLimiters {
     private static final Duration DEFAULT_TIMEOUT = Duration.ofMinutes(1);
     private static final Void NO_CONTEXT = null;
     private static final String FALLBACK = "";
+    private static final MetricName SLOW_ACQUIRE =
+            MetricName.builder().safeName("conjure-java-client.qos.request-permit.slow-acquire").build();
 
+    private final Timer slowAcquire;
     private final ConcurrentMap<String, Limiter<Void>> limiters = new ConcurrentHashMap<>();
     private final Duration timeout;
 
     @VisibleForTesting
-    ConcurrencyLimiters(Duration timeout) {
+    ConcurrencyLimiters(TaggedMetricRegistry taggedMetricRegistry, Duration timeout) {
+        this.slowAcquire = taggedMetricRegistry.timer(SLOW_ACQUIRE);
         this.timeout = timeout;
     }
 
-    ConcurrencyLimiters() {
-        this(DEFAULT_TIMEOUT);
+    ConcurrencyLimiters(TaggedMetricRegistry taggedMetricRegistry) {
+        this(taggedMetricRegistry, DEFAULT_TIMEOUT);
     }
 
     Limiter.Listener limiter(Request request) {
@@ -71,9 +79,9 @@ Limiter.Listener limiter(String name) {
     }
 
     private Limiter<Void> newLimiter() {
-        Limiter<Void> limiter = SimpleLimiter.newBuilder()
+        Limiter<Void> limiter = new InstrumentedLimiter(SimpleLimiter.newBuilder()
                 .limit(new RemotingWindowedLimit(VegasLimit.newDefault()))
-                .build();
+                .build(), slowAcquire);
         return RemotingBlockingLimiter.wrap(limiter, timeout);
     }
 
@@ -85,4 +93,28 @@ private static String limiterKey(Request request) {
             return request.method() + " " + pathTemplate;
         }
     }
+
+    private static final class InstrumentedLimiter implements Limiter<Void> {
+        private final Limiter<Void> delegate;
+        private final Timer timer;
+
+        private InstrumentedLimiter(Limiter<Void> delegate, Timer timer) {
+            this.delegate = delegate;
+            this.timer = timer;
+        }
+
+        @Override
+        public Optional<Listener> acquire(Void context) {
+            long start = System.nanoTime();
+            try {
+                return delegate.acquire(context);
+            } finally {
+                long end = System.nanoTime();
+                long durationNanos = end - start;
+                if (TimeUnit.NANOSECONDS.toMillis(durationNanos) > 1) {
+                    timer.update(durationNanos, TimeUnit.NANOSECONDS);
+                }
+            }
+        }
+    }
 }

okhttp-clients/src/main/java/com/palantir/remoting3/okhttp/ConcurrencyLimitingInterceptor.java

@@ -19,6 +19,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 import com.netflix.concurrency.limits.Limiter;
+import com.palantir.tritium.metrics.registry.TaggedMetricRegistry;
 import java.io.IOException;
 import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.InvocationTargetException;
@@ -62,8 +63,8 @@ final class ConcurrencyLimitingInterceptor implements Interceptor {
         this.limiters = limiters;
     }
 
-    ConcurrencyLimitingInterceptor() {
-        this(new ConcurrencyLimiters());
+    ConcurrencyLimitingInterceptor(TaggedMetricRegistry taggedMetricRegistry) {
+        this(new ConcurrencyLimiters(taggedMetricRegistry));
     }
 
     @Override

okhttp-clients/src/main/java/com/palantir/remoting3/okhttp/OkHttpClients.java

@@ -190,7 +190,7 @@ private static RemotingOkHttpClient createInternal(
         client.sslSocketFactory(config.sslSocketFactory(), config.trustManager());
 
         // Intercept calls to augment request meta data
-        client.addInterceptor(new ConcurrencyLimitingInterceptor());
+        client.addInterceptor(new ConcurrencyLimitingInterceptor(registry));
         client.addInterceptor(InstrumentedInterceptor.create(registry, hostMetrics, serviceClass));
         client.addInterceptor(OkhttpTraceInterceptor.INSTANCE);
         client.addInterceptor(UserAgentInterceptor.of(augmentUserAgent(userAgent, serviceClass)));

okhttp-clients/src/test/java/com/palantir/remoting3/okhttp/ConcurrencyLimitersTest.java

@@ -18,14 +18,15 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+import com.palantir.tritium.metrics.registry.DefaultTaggedMetricRegistry;
 import java.time.Duration;
 import java.time.Instant;
 import org.junit.Test;
 
 public final class ConcurrencyLimitersTest {
     private static final String KEY = "";
     private static final Duration TIMEOUT = Duration.ofSeconds(1);
-    private final ConcurrencyLimiters limiters = new ConcurrencyLimiters(TIMEOUT);
+    private final ConcurrencyLimiters limiters = new ConcurrencyLimiters(new DefaultTaggedMetricRegistry(), TIMEOUT);
 
     @Test
     public void testTimeout() {

okhttp-clients/src/test/java/com/palantir/remoting3/okhttp/FlowControlTest.java

@@ -27,6 +27,7 @@
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.RateLimiter;
 import com.netflix.concurrency.limits.Limiter;
+import com.palantir.tritium.metrics.registry.DefaultTaggedMetricRegistry;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
@@ -58,7 +59,7 @@ public final class FlowControlTest {
     private static final int REQUESTS_PER_THREAD = 5;
     private static ListeningExecutorService executorService;
 
-    private final ConcurrencyLimiters limiters = new ConcurrencyLimiters();
+    private final ConcurrencyLimiters limiters = new ConcurrencyLimiters(new DefaultTaggedMetricRegistry());
 
     @BeforeClass
     public static void beforeClass() {