From 584f7a23f62b4bdc5a9bd83259e2416624627c0c Mon Sep 17 00:00:00 2001 From: alextrofymenko Date: Fri, 25 Jan 2019 16:20:53 +0000 Subject: [PATCH 001/107] Do not specify charset for multipart requests Details in the following issue: https://github.com/dakrone/clj-http/issues/472 --- src/clj_http/multipart.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clj_http/multipart.clj b/src/clj_http/multipart.clj index e2691556..dcb5c16c 100644 --- a/src/clj_http/multipart.clj +++ b/src/clj_http/multipart.clj @@ -131,7 +131,6 @@ [multipart {:keys [mime-subtype]}] (let [mp-entity (doto (MultipartEntityBuilder/create) (.setStrictMode) - (.setCharset (encoding-to-charset "UTF-8")) (.setMimeSubtype (or mime-subtype "form-data")))] (doseq [m multipart] (let [name (or (:part-name m) (:name m)) From 8679b55ef0e4b5d1c9467378e54a778aff4bec0f Mon Sep 17 00:00:00 2001 From: Alex Trofymenko Date: Sat, 26 Jan 2019 14:15:45 +0000 Subject: [PATCH 002/107] Add test to verify charset is nil for multipart requests --- test/clj_http/test/multipart_test.clj | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/clj_http/test/multipart_test.clj b/test/clj_http/test/multipart_test.clj index 7151c12e..a0fa4958 100644 --- a/test/clj_http/test/multipart_test.clj +++ b/test/clj_http/test/multipart_test.clj @@ -2,9 +2,10 @@ (:require [clj-http.multipart :refer :all] [clojure.test :refer :all]) (:import (java.io File ByteArrayOutputStream ByteArrayInputStream) + (java.nio.charset Charset) (org.apache.http.entity.mime.content FileBody StringBody ContentBody ByteArrayBody InputStreamBody) - (java.nio.charset Charset))) + (org.apache.http.util EntityUtils))) (defn body-str [^StringBody body] (-> body .getReader slurp)) @@ -173,3 +174,8 @@ (is (= (Charset/forName "ascii") (body-charset body))) (is (= test-file (.getFile body) )) (is (= "testname" (.getFilename body))))))) + +(deftest test-multipart-content-charset + (testing "charset is nil for all multipart requests" + (let [mp-entity (create-multipart-entity [] nil)] + (is (nil? (EntityUtils/getContentCharSet mp-entity)))))) From 79716b45f72e7bb97b9546b59c64b8534a7cd423 Mon Sep 17 00:00:00 2001 From: Anatoly Smolyaninov Date: Tue, 5 Feb 2019 00:20:13 +0100 Subject: [PATCH 003/107] Fix NPE for `unexceptional-status-for-request?` (#474) Fixes `(unexceptional-status-for-request? {:unexceptional-status nil} 200) => NPE`. I.e. if you have no `:unexceptional-status` key, this will work fine, but if this key is present then `nil` will be used as fn --- src/clj_http/client.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index f3b30af0..85f5193f 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -206,7 +206,8 @@ (defn unexceptional-status-for-request? [req status] - ((:unexceptional-status req unexceptional-status?) status)) + ((or (:unexceptional-status req) unexceptional-status?) + status)) ;; helper methods to determine realm of a response (defn success? From 3083aa529a35337d37998330e48f223b35bea008 Mon Sep 17 00:00:00 2001 From: Pauli Jaakkola Date: Tue, 5 Feb 2019 01:30:12 +0200 Subject: [PATCH 004/107] Reduce body allocation and copying (#475) Removes various intermediate byte arrays and strings in body decoding that were unnecessary. JSON and Transit decoding get the most benefit. I didn't have time to figure out every format and it seems that some of them would need changes to other libraries like `ring.util.codec`. https://github.com/dakrone/cheshire/pull/139 would also enable a bit more gains for JSON decoding. --- src/clj_http/client.clj | 134 +++++++++++++++++++--------------------- src/clj_http/util.clj | 51 ++++++++++----- 2 files changed, 99 insertions(+), 86 deletions(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index 85f5193f..b17f22cb 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -6,11 +6,12 @@ [clj-http.headers :refer [wrap-header-map]] [clj-http.links :refer [wrap-links]] [clj-http.util :refer [opt] :as util] + [clojure.java.io :as io] [clojure.stacktrace :refer [root-cause]] [clojure.string :as str] [clojure.walk :refer [keywordize-keys prewalk]] [slingshot.slingshot :refer [throw+]]) - (:import (java.io InputStream File ByteArrayOutputStream ByteArrayInputStream) + (:import (java.io InputStream File ByteArrayOutputStream ByteArrayInputStream EOFException BufferedReader) (java.net URL UnknownHostException) (org.apache.http.entity BufferedHttpEntity ByteArrayEntity InputStreamEntity FileEntity StringEntity) @@ -136,6 +137,12 @@ {:pre [json-enabled?]} (apply (ns-resolve (symbol "cheshire.core") (symbol "decode-strict")) args)) +(defn ^:dynamic json-decode-stream + "Resolve and apply cheshire's json stream decoding dynamically." + [& args] + {:pre [json-enabled?]} + (apply (ns-resolve (symbol "cheshire.core") (symbol "decode-stream")) args)) + (defn ^:dynamic form-decode "Resolve and apply ring-codec's form decoding dynamically." [& args] @@ -428,83 +435,75 @@ (defmulti coerce-response-body (fn [req _] (:as req))) (defmethod coerce-response-body :byte-array [_ resp] - (assoc resp :body (util/force-byte-array (:body resp)))) + (update resp :body util/force-byte-array)) (defmethod coerce-response-body :stream [_ resp] - (let [body (:body resp)] - (cond (instance? InputStream body) resp - ;; This shouldn't happen, but we plan for it anyway - (instance? (Class/forName "[B") body) - (assoc resp :body (ByteArrayInputStream. body))))) + (update resp :body util/force-stream)) + +(defn- response-charset [response] + (or (-> response :content-type-params :charset) + "UTF-8")) + +(defn- can-parse-body? [{:keys [coerce] :as request} {:keys [status] :as _response}] + (or (= coerce :always) + (and (unexceptional-status-for-request? request status) + (or (nil? coerce) + (= coerce :unexceptional))) + (and (not (unexceptional-status-for-request? request status)) + (= coerce :exceptional)))) + +(defn- decode-json-body [body keyword? strict? charset] + (if strict? + ;; OPTIMIZE: When/if Cheshire gets a parse-stream-strict this won't need to go through String: + (json-decode-strict (util/force-string body charset) keyword?) + (let [^BufferedReader br (io/reader (util/force-stream body))] + (try + (.mark br 1) + (let [^int first-char (try (.read br) (catch EOFException _ -1))] + (case first-char + -1 nil + (do (.reset br) + (json-decode-stream br keyword?)))) + (finally (.close br)))))) (defn coerce-json-body - [{:keys [coerce] :as request} - {:keys [body status] :as resp} keyword? strict? & [charset]] - (let [^String charset (or charset (-> resp :content-type-params :charset) - "UTF-8") - body (util/force-byte-array body) - decode-func (if strict? json-decode-strict json-decode)] - (if json-enabled? - (cond - (= coerce :always) - (assoc resp :body (decode-func (String. ^"[B" body charset) keyword?)) - - (and (unexceptional-status-for-request? request status) - (or (nil? coerce) (= coerce :unexceptional))) - (assoc resp :body (decode-func (String. ^"[B" body charset) keyword?)) - - (and (not (unexceptional-status-for-request? request status)) - (= coerce :exceptional)) - (assoc resp :body (decode-func (String. ^"[B" body charset) keyword?)) - - :else (assoc resp :body (String. ^"[B" body charset))) - (assoc resp :body (String. ^"[B" body charset))))) + [request {:keys [body] :as resp} keyword? strict? & [charset]] + (let [charset (or charset (response-charset resp)) + body (if json-enabled? + (if (can-parse-body? request resp) + (decode-json-body body keyword? strict? charset) + (util/force-string body charset)) + (util/force-string body charset))] + (assoc resp :body body))) (defn coerce-clojure-body - [request {:keys [body] :as resp}] - (let [^String charset (or (-> resp :content-type-params :charset) "UTF-8") - body (util/force-byte-array body)] + [_request {:keys [body] :as resp}] + (let [charset (response-charset resp) + body (util/force-string body charset)] (assoc resp :body (cond (empty? body) nil - edn-enabled? (parse-edn (String. ^"[B" body charset)) + edn-enabled? (parse-edn body) :else (binding [*read-eval* false] - (read-string (String. ^"[B" body charset))))))) + (read-string body)))))) (defn coerce-transit-body - [{:keys [transit-opts coerce] :as request} - {:keys [body status] :as resp} type & [charset]] - (let [^String charset (or charset (-> resp :content-type-params :charset) - "UTF-8") - body (util/force-byte-array body)] - (if-not (empty? body) - (if transit-enabled? - (cond - (= coerce :always) - (assoc resp :body (parse-transit - (ByteArrayInputStream. body) type transit-opts)) - - (and (unexceptional-status-for-request? request status) - (or (nil? coerce) (= coerce :unexceptional))) - (assoc resp :body (parse-transit - (ByteArrayInputStream. body) type transit-opts)) - - (and (not (unexceptional-status-for-request? request status)) - (= coerce :exceptional)) - (assoc resp :body (parse-transit - (ByteArrayInputStream. body) type transit-opts)) - - :else (assoc resp :body (String. ^"[B" body charset))) - (assoc resp :body (String. ^"[B" body charset))) - (assoc resp :body nil)))) + [{:keys [transit-opts] :as request} + {:keys [body] :as resp} type & [charset]] + (let [charset (or charset (response-charset resp)) + body (if transit-enabled? + (if (can-parse-body? request resp) + (parse-transit (util/force-stream body) type transit-opts) + (util/force-string body charset)) + nil)] + (assoc resp :body body))) (defn coerce-form-urlencoded-body - [request {:keys [body] :as resp}] - (let [^String charset (or (-> resp :content-type-params :charset) "UTF-8") - body-bytes (util/force-byte-array body)] - (if ring-codec-enabled? - (assoc resp :body (-> (String. ^"[B" body-bytes charset) - form-decode keywordize-keys)) - (assoc resp :body (String. ^"[B" body-bytes charset))))) + [_request {:keys [body] :as resp}] + (let [charset (response-charset resp) + body (util/force-string body charset)] + (assoc resp :body (if ring-codec-enabled? + (-> body form-decode keywordize-keys) + body)))) (defmulti coerce-content-type (fn [req resp] (:content-type resp))) @@ -562,10 +561,7 @@ (defmethod coerce-response-body :default [{:keys [as]} {:keys [body] :as resp}] - (let [body-bytes (util/force-byte-array body)] - (cond - (string? as) (assoc resp :body (String. ^"[B" body-bytes ^String as)) - :else (assoc resp :body (String. ^"[B" body-bytes "UTF-8"))))) + (assoc resp :body (util/force-string body (if (string? as) as "UTF-8")))) (defn- output-coercion-response [req {:keys [body] :as resp}] diff --git a/src/clj_http/util.clj b/src/clj_http/util.clj index b6d6ec94..a125fed4 100644 --- a/src/clj_http/util.clj +++ b/src/clj_http/util.clj @@ -4,8 +4,8 @@ [clojure.walk :refer [postwalk]]) (:import (org.apache.commons.codec.binary Base64) (org.apache.commons.io IOUtils) - (java.io BufferedInputStream ByteArrayInputStream - ByteArrayOutputStream EOFException) + (java.io InputStream BufferedInputStream ByteArrayInputStream + ByteArrayOutputStream EOFException PushbackInputStream) (java.net URLEncoder URLDecoder) (java.util.zip InflaterInputStream DeflaterInputStream GZIPInputStream GZIPOutputStream))) @@ -43,7 +43,7 @@ [b] (when b (cond - (instance? java.io.InputStream b) + (instance? InputStream b) (GZIPInputStream. b) :else (IOUtils/toByteArray (GZIPInputStream. (ByteArrayInputStream. b)))))) @@ -58,24 +58,41 @@ (.close gos) (.toByteArray baos)))) +(defn force-stream + "Force b as InputStream if it is a ByteArray." + ^InputStream [b] + (if (instance? InputStream b) + b + (ByteArrayInputStream. b))) + (defn force-byte-array "force b as byte array if it is an InputStream, also close the stream" ^bytes [b] - (if (instance? java.io.InputStream b) - (try - (let [^int first-byte (try - (.read ^java.io.InputStream b) - (catch EOFException e -1))] - (if (= -1 first-byte) - (byte-array 0) - (let [rest-bytes (IOUtils/toByteArray ^java.io.InputStream b) - barray (byte-array (inc (count rest-bytes)))] - (aset-byte barray 0 (unchecked-byte first-byte)) - (System/arraycopy rest-bytes 0 barray 1 (count rest-bytes)) - barray))) - (finally (.close ^java.io.InputStream b))) + (if (instance? InputStream b) + (let [^PushbackInputStream bs (PushbackInputStream. b)] + (try + (let [^int first-byte (try (.read bs) (catch EOFException _ -1))] + (case first-byte + -1 (byte-array 0) + (do (.unread bs first-byte) + (IOUtils/toByteArray bs)))) + (finally (.close bs)))) b)) +(defn force-string + "Convert s (a ByteArray or InputStream) to String." + ^String [s ^String charset] + (if (instance? InputStream s) + (let [^PushbackInputStream bs (PushbackInputStream. s)] + (try + (let [^int first-byte (try (.read bs) (catch EOFException _ -1))] + (case first-byte + -1 "" + (do (.unread bs first-byte) + (IOUtils/toString bs charset)))) + (finally (.close bs)))) + (IOUtils/toString ^"[B" s charset))) + (defn inflate "Returns a zlib inflate'd version of the given byte array or InputStream." [b] @@ -83,7 +100,7 @@ ;; This weirdness is because HTTP servers lie about what kind of deflation ;; they're using, so we try one way, then if that doesn't work, reset and ;; try the other way - (let [stream (BufferedInputStream. (if (instance? java.io.InputStream b) + (let [stream (BufferedInputStream. (if (instance? InputStream b) b (ByteArrayInputStream. b))) _ (.mark stream 512) From 1e77ba2c73b15534522c18eed6e159fc6ef3d977 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Thu, 7 Feb 2019 18:50:46 +0100 Subject: [PATCH 005/107] Use consistent connection option names This is backward-compatible because it still supports the current `:conn-` -prefixed keys. Fixes #477. --- README.org | 6 +++--- src/clj_http/core.clj | 19 ++++++++++++------- src/clj_http/core_old.clj | 10 +++++++--- test/clj_http/test/conn_mgr_test.clj | 2 +- test/clj_http/test/core_test.clj | 8 ++++---- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/README.org b/README.org index 67a344bb..2f2f08bd 100644 --- a/README.org +++ b/README.org @@ -268,7 +268,7 @@ Example requests: (client/get "http://example.com/redirects-somewhere" {:max-redirects 5 :redirect-strategy :graceful}) ;; Throw an exception if the get takes too long. Timeouts in milliseconds. -(client/get "http://example.com/redirects-somewhere" {:socket-timeout 1000 :conn-timeout 1000}) +(client/get "http://example.com/redirects-somewhere" {:socket-timeout 1000 :connection-timeout 1000}) ;; Query parameters (client/get "http://example.com/search" {:query-params {"q" "foo, bar"}}) @@ -325,8 +325,8 @@ content encodings. :body "{\"json\": \"input\"}" :headers {"X-Api-Version" "2"} :content-type :json - :socket-timeout 1000 ;; in milliseconds - :conn-timeout 1000 ;; in milliseconds + :socket-timeout 1000 ;; in milliseconds + :connection-timeout 1000 ;; in milliseconds :accept :json}) ;; Send form params as a urlencoded body (POST or PUT) diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index c4e1f18e..b0ef57ec 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -179,17 +179,20 @@ (defmethod get-cookie-policy :stardard-strict standard-strict-cookie-policy [_] CookieSpecs/STANDARD_STRICT) -(defn request-config [{:keys [conn-timeout +(defn request-config [{:keys [connection-timeout + connection-request-timeout socket-timeout - conn-request-timeout max-redirects - cookie-spec] + cookie-spec + ; deprecated + conn-request-timeout + conn-timeout] :as req}] (let [config (-> (RequestConfig/custom) - (.setConnectTimeout (or conn-timeout -1)) + (.setConnectTimeout (or connection-timeout conn-timeout -1)) (.setSocketTimeout (or socket-timeout -1)) (.setConnectionRequestTimeout - (or conn-request-timeout -1)) + (or connection-request-timeout conn-request-timeout -1)) (.setRedirectsEnabled true) (.setCircularRedirectsAllowed (boolean (opt req :allow-circular-redirects))) @@ -553,13 +556,15 @@ (defn request ([req] (request req nil nil)) - ([{:keys [body conn-timeout conn-request-timeout connection-manager + ([{:keys [body connection-timeout connection-request-timeout connection-manager cookie-store cookie-policy headers multipart query-string redirect-strategy max-redirects retry-handler request-method scheme server-name server-port socket-timeout uri response-interceptor proxy-host proxy-port http-client-context http-request-config http-client - proxy-ignore-hosts proxy-user proxy-pass digest-auth ntlm-auth] + proxy-ignore-hosts proxy-user proxy-pass digest-auth ntlm-auth + ; deprecated + conn-timeout conn-request-timeout] :as req} respond raise] (let [async? (opt req :async) cache? (opt req :cache) diff --git a/src/clj_http/core_old.clj b/src/clj_http/core_old.clj index d6a7e8c1..8695d643 100644 --- a/src/clj_http/core_old.clj +++ b/src/clj_http/core_old.clj @@ -212,10 +212,13 @@ Note that where Ring uses InputStreams for the request and response bodies, the clj-http uses ByteArrays for the bodies." [{:keys [request-method scheme server-name server-port uri query-string - headers body multipart socket-timeout conn-timeout proxy-host + headers body multipart socket-timeout connection-timeout proxy-host proxy-ignore-hosts proxy-port proxy-user proxy-pass as cookie-store retry-handler response-interceptor digest-auth ntlm-auth - connection-manager client-params] + connection-manager client-params + ; deprecated + conn-timeout + ] :as req}] (let [^ClientConnectionManager conn-mgr (or connection-manager @@ -237,7 +240,8 @@ ;; merge in map of specified timeouts, to ;; support backward compatibility. (merge {CoreConnectionPNames/SO_TIMEOUT socket-timeout - CoreConnectionPNames/CONNECTION_TIMEOUT conn-timeout} + CoreConnectionPNames/CONNECTION_TIMEOUT (or connection-timeout + conn-timeout)} client-params)) (when-let [[user pass] digest-auth] diff --git a/test/clj_http/test/conn_mgr_test.clj b/test/clj_http/test/conn_mgr_test.clj index 1333b1e3..05391fcd 100644 --- a/test/clj_http/test/conn_mgr_test.clj +++ b/test/clj_http/test/conn_mgr_test.clj @@ -156,7 +156,7 @@ :server-name "localhost" ;; timeouts forces an exception being thrown :socket-timeout 1 - :conn-timeout 1 + :connection-timeout 1 :connection-manager cm :as :stream}) (is false "request should have thrown an exception") diff --git a/test/clj_http/test/core_test.clj b/test/clj_http/test/core_test.clj index 582aec15..b1b35aba 100644 --- a/test/clj_http/test/core_test.clj +++ b/test/clj_http/test/core_test.clj @@ -605,7 +605,7 @@ ;; This relies on connections to writequit.org being slower than 10ms, if this ;; fails, you must have very nice internet. -(deftest ^:integration sets-conn-timeout +(deftest ^:integration sets-connection-timeout (run-server) (try (is (thrown? SocketTimeoutException @@ -613,7 +613,7 @@ :server-name "writequit.org" :server-port 80 :request-method :get :uri "/" - :conn-timeout 10}))))) + :connection-timeout 10}))))) (deftest ^:integration connection-pool-timeout (run-server) @@ -622,8 +622,8 @@ :server-name "localhost" :server-port 18080 :request-method :get - :conn-timeout 1 - :conn-request-timeout 1 + :connection-timeout 1 + :connection-request-timeout 1 :uri "/timeout"})) is-pool-timeout-error? (fn [req-fut] From 6ee9439dcbe0e8d0f3f372e152de99c36e6f2f52 Mon Sep 17 00:00:00 2001 From: Christoph Burgmer Date: Mon, 11 Mar 2019 23:58:08 +0100 Subject: [PATCH 006/107] Remove client-params example (#481) 3.0.0 seems to have dropped support for this. --- README.org | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.org b/README.org index 67a344bb..125af249 100644 --- a/README.org +++ b/README.org @@ -187,12 +187,6 @@ Example requests: (client/get "http://example.com" {:headers {:foo ["bar" "baz"], :eggplant "quux"}}) -;; Set any specific client parameters manually: -(client/post "http://example.com" - {:client-params {"http.protocol.allow-circular-redirects" false - "http.protocol.version" HttpVersion/HTTP_1_0 - "http.useragent" "clj-http"}}) - ;; Completely ignore cookies: (client/post "http://example.com" {:cookie-policy :none}) ;; There are also multiple ways to handle cookies From b88084a9888adaeff9771908403b705b500e8693 Mon Sep 17 00:00:00 2001 From: Patrick Tuckey Date: Fri, 12 Apr 2019 15:46:28 -0700 Subject: [PATCH 007/107] Fixes #482: async mgr "reuseable -> reusable" (#483) This addresses a naming inconsistency in conn mgr functions. --- src/clj_http/conn_mgr.clj | 7 ++++++- test/clj_http/test/conn_mgr_test.clj | 6 ++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/clj_http/conn_mgr.clj b/src/clj_http/conn_mgr.clj index ad59237a..96e9a227 100644 --- a/src/clj_http/conn_mgr.clj +++ b/src/clj_http/conn_mgr.clj @@ -385,7 +385,7 @@ [io-reactor nil registry nil nil timeout java.util.concurrent.TimeUnit/SECONDS]))) -(defn ^PoolingNHttpClientConnectionManager make-reuseable-async-conn-manager +(defn ^PoolingNHttpClientConnectionManager make-reusable-async-conn-manager "Creates a default pooling async connection manager with the specified options. Handles the same options as make-reusable-conn-manager plus :io-config which should be a map containing some of the following keys: @@ -429,6 +429,11 @@ (.setDefaultMaxPerRoute conn-man default-per-route)) conn-man)) +(defn ^PoolingNHttpClientConnectionManager make-reuseable-async-conn-manager + "Wraps correctly-spelled version - keeping for backwards compatibility." + [opts] + (make-reusable-async-conn-manager opts)) + (defmulti shutdown-manager "Shut down the given connection manager, if it is not nil" class) diff --git a/test/clj_http/test/conn_mgr_test.clj b/test/clj_http/test/conn_mgr_test.clj index 05391fcd..e8b446aa 100644 --- a/test/clj_http/test/conn_mgr_test.clj +++ b/test/clj_http/test/conn_mgr_test.clj @@ -181,8 +181,10 @@ (let [regular (conn-mgr/make-regular-conn-manager {}) regular-reusable (conn-mgr/make-reusable-conn-manager {}) async (conn-mgr/make-regular-async-conn-manager {}) - async-reusable (conn-mgr/make-reuseable-async-conn-manager {})] + async-reusable (conn-mgr/make-reusable-async-conn-manager {}) + async-reuseable (conn-mgr/make-reuseable-async-conn-manager {})] (is (false? (conn-mgr/reusable? regular))) (is (true? (conn-mgr/reusable? regular-reusable))) (is (false? (conn-mgr/reusable? async))) - (is (true? (conn-mgr/reusable? async-reusable))))) + (is (true? (conn-mgr/reusable? async-reusable))) + (is (true? (conn-mgr/reusable? async-reuseable))))) From d6ae196af7bd603aa4dfe54c302830b31e1928ba Mon Sep 17 00:00:00 2001 From: Ryan Smith Date: Thu, 18 Apr 2019 09:22:50 -0700 Subject: [PATCH 008/107] Add integration tests to check dynamic variable propagation --- src/clj_http/core.clj | 6 +++++- test/clj_http/test/client_test.clj | 33 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index 214b840e..777f42e3 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -630,7 +630,8 @@ (conn/shutdown-manager conn-mgr)) (throw t)))) (let [^CloseableHttpAsyncClient client - (build-async-http-client req conn-mgr http-url proxy-ignore-hosts)] + (build-async-http-client req conn-mgr http-url proxy-ignore-hosts) + original-thread-bindings (clojure.lang.Var/getThreadBindingFrame)] (when cache? (throw (IllegalArgumentException. "caching is not yet supported for async clients"))) @@ -638,12 +639,14 @@ (.execute client http-req context (reify org.apache.http.concurrent.FutureCallback (failed [this ex] + (clojure.lang.Var/resetThreadBindingFrame original-thread-bindings) (when-not (conn/reusable? conn-mgr) (conn/shutdown-manager conn-mgr)) (if (opt req :ignore-unknown-host) ((:unknown-host-respond req) nil) (raise ex))) (completed [this resp] + (clojure.lang.Var/resetThreadBindingFrame original-thread-bindings) (try (respond (build-response-map resp req http-req http-url @@ -653,6 +656,7 @@ (conn/shutdown-manager conn-mgr)) (raise t)))) (cancelled [this] + (clojure.lang.Var/resetThreadBindingFrame original-thread-bindings) ;; Run the :oncancel function if available (when-let [oncancel (:oncancel req)] (oncancel)) diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index 58b56f0c..7c04b3ec 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -111,6 +111,39 @@ (is (= params (read-fn (:body @resp)))) (is (not (realized? exception))))))) +(def ^:dynamic *test-dynamic-var* nil) + +(deftest ^:integration async-preserves-dynamic-variable-bindings + (run-server) + (let [expected-var "cat"] + (binding [*test-dynamic-var* expected-var] + (let [test-fn (fn [uri success-p fail-p] + (request {:uri uri + :method :get + :scheme "http" + :async? true} + (fn [_] + (deliver success-p *test-dynamic-var*) + (deliver fail-p :success)) + (fn [_] + (deliver success-p :fail) + (deliver fail-p *test-dynamic-var*))))] + (testing "dynamic variables on success responses" + (let [success-p (promise) + fail-p (promise)] + (test-fn "/get" success-p fail-p) + (is (= @success-p expected-var *test-dynamic-var*)) + (is (= @fail-p :success) + "Verify that we went through the success path, not the failure"))) + + (testing "dynamic variables on fail responses" + (let [success-p (promise) + fail-p (promise)] + (test-fn "/json-bad" success-p fail-p) + (is (= @success-p :fail) + "Verify that we went through the failure path, not the success") + (is (= @fail-p expected-var *test-dynamic-var*)))))))) + (deftest ^:integration multipart-async (run-server) (let [resp (promise) From 537738ddf3a58c99643d7d0d335f317dd4266835 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 30 Apr 2019 09:27:05 -0600 Subject: [PATCH 009/107] Bump dependencies --- project.clj | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/project.clj b/project.clj index c4377346..1b33fbe8 100644 --- a/project.clj +++ b/project.clj @@ -7,42 +7,43 @@ :global-vars {*warn-on-reflection* false} :min-lein-version "2.0.0" :exclusions [org.clojure/clojure] - :dependencies [[org.apache.httpcomponents/httpcore "4.4.9"] - [org.apache.httpcomponents/httpclient "4.5.5"] - [org.apache.httpcomponents/httpclient-cache "4.5.5"] - [org.apache.httpcomponents/httpasyncclient "4.1.3"] - [org.apache.httpcomponents/httpmime "4.5.5"] - [commons-codec "1.11"] + :dependencies [[org.apache.httpcomponents/httpcore "4.4.11"] + [org.apache.httpcomponents/httpclient "4.5.8"] + [org.apache.httpcomponents/httpclient-cache "4.5.8"] + [org.apache.httpcomponents/httpasyncclient "4.1.4"] + [org.apache.httpcomponents/httpmime "4.5.8"] + [commons-codec "1.12"] [commons-io "2.6"] [slingshot "0.12.2"] [potemkin "0.4.5"]] :resource-paths ["resources"] :profiles {:dev {:dependencies [;; optional deps - [cheshire "5.8.0"] + [cheshire "5.8.1"] [crouton "0.1.2" :exclusions [[org.jsoup/jsoup]]] [org.jsoup/jsoup "1.11.3"] - [org.clojure/tools.reader "1.2.2"] - [com.cognitect/transit-clj "0.8.300"] + [org.clojure/tools.reader "1.3.2"] + [com.cognitect/transit-clj "0.8.313"] [ring/ring-codec "1.1.1"] ;; other (testing) deps - [org.clojure/clojure "1.9.0"] + [org.clojure/clojure "1.10.0"] [org.clojure/tools.logging "0.4.0"] - [ring/ring-jetty-adapter "1.6.3"] - [ring/ring-devel "1.6.3"] + [ring/ring-jetty-adapter "1.7.1"] + [ring/ring-devel "1.7.1"] ;; caching example deps - [org.clojure/core.cache "0.7.1"] + [org.clojure/core.cache "0.7.2"] ;; logging - [org.apache.logging.log4j/log4j-api "2.11.0"] - [org.apache.logging.log4j/log4j-core "2.11.0"] - [org.apache.logging.log4j/log4j-1.2-api "2.11.0"]] + [org.apache.logging.log4j/log4j-api "2.11.2"] + [org.apache.logging.log4j/log4j-core "2.11.2"] + [org.apache.logging.log4j/log4j-1.2-api "2.11.2"]] :plugins [[lein-ancient "0.6.15"] [jonase/eastwood "0.2.5"] [lein-kibit "0.1.5"] [lein-nvd "0.5.2"]]} :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} :1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} - :1.8 {:dependencies [[org.clojure/clojure "1.8.0"]]}} - :aliases {"all" ["with-profile" "dev,1.6:dev,1.7:dev,1.8:dev"]} + :1.8 {:dependencies [[org.clojure/clojure "1.8.0"]]} + :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]}} + :aliases {"all" ["with-profile" "dev,1.6:dev,1.7:dev,1.8:dev,1.9:dev"]} :plugins [[codox "0.6.4"]] :test-selectors {:default #(not (:integration %)) :integration :integration From 9ec411156154f8036501f6f2762958675fac600a Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 30 Apr 2019 09:27:15 -0600 Subject: [PATCH 010/107] Release 3.10.0 --- README.org | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index cf4a187b..13a69ab5 100644 --- a/README.org +++ b/README.org @@ -117,7 +117,7 @@ function. With Leiningen/Boot: #+BEGIN_SRC clojure -[clj-http "3.9.1"] +[clj-http "3.10.0"] #+END_SRC If you need an older version, a 2.x release is also available. diff --git a/project.clj b/project.clj index 1b33fbe8..8e8c2edc 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.10.0-SNAPSHOT" +(defproject clj-http "3.10.0" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From c57169864422fe1e142aa3ac8a63e8c87676c4cd Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 30 Apr 2019 09:28:39 -0600 Subject: [PATCH 011/107] Bump to 3.10.1-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 8e8c2edc..a39bd126 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.10.0" +(defproject clj-http "3.10.1-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 917e6f7ebe930cac21b4e481c2d756faac02b66f Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 30 Apr 2019 10:04:47 -0600 Subject: [PATCH 012/107] Don't test on Travis with JDK 7 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index acc3a1d8..d8d2b0ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ branches: # 'master' is the new apache 5 client, and not passing yet # - master jdk: - - openjdk7 - oraclejdk8 - openjdk9 - openjdk10 From dba100d2c05baa39b76a02c86c493d21ed4be431 Mon Sep 17 00:00:00 2001 From: Alex ter Weele Date: Sun, 9 Jun 2019 21:08:17 -0700 Subject: [PATCH 013/107] Clarify under what conditions JSON parsing is applied. (#492) Re https://github.com/dakrone/clj-http/issues/488. --- README.org | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.org b/README.org index 13a69ab5..bea69239 100644 --- a/README.org +++ b/README.org @@ -527,15 +527,19 @@ it's received (output coercion) from the server. Output coercion with =:as :json=, =:as :json-strict=, =:as :json-strict-string-keys=, =:as :json-string-keys= or =:as :x-www-form-urlencoded= will only work with an optional dependency, see [[#optional-dependencies][Optional Dependencies]]. -JSON coercion defaults to only an "unexceptional" statuses, meaning status codes -in the #{200 201 202 203 204 205 206 207 300 301 302 303 304 307} range. If you -would like to change this, you can send the =:coerce= option, which can be set -to: +By default, JSON coercion is only applied when the response's status +is considered "unexceptional". If the =:unexeceptional-status= option +is provided, then its value is a function which specifies what status +codes are unexceptional. =:unexceptional-status= defaults to +=clj-http.client/unexceptional-status?=. + +If you would like to change under what conditions coercion is applied, +you can send the =:coerce= option, which can be set to: #+BEGIN_SRC clojure :always ;; always json decode the body -:unexceptional ;; only json decode when not an HTTP error response -:exceptional ;; only json decode when it IS an HTTP error response +:unexceptional ;; json decode when an HTTP response is considered unexceptional +:exceptional ;; json decode when an HTTP response is considered exceptional #+END_SRC The =:coerce= setting defaults to =:unexceptional=. From 1933bcf51d4b66582abb95f36038c718a03a1e45 Mon Sep 17 00:00:00 2001 From: Jason Whitlark Date: Sun, 9 Jun 2019 21:12:27 -0700 Subject: [PATCH 014/107] Add Kubernetes API (from inside pod) Example (#487) --- examples/kubernetes_pod.clj | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 examples/kubernetes_pod.clj diff --git a/examples/kubernetes_pod.clj b/examples/kubernetes_pod.clj new file mode 100644 index 00000000..4add03bf --- /dev/null +++ b/examples/kubernetes_pod.clj @@ -0,0 +1,20 @@ +(:ns clj-http.examples.kubernetes-pod + "This is an example of calling the Kubernetes API from inside a pod. K8s uses a + custom CA so that you can authenticate the API server, and provides a token per pod + so that each pod can authenticate itself with the APi server. + + If you are still having 401/403 errors, look carefully at the message, if it includes + a ServiceAccount name, this part worked, and your problem is likely at the Role/RoleBinding level." + (:require [clj-http.client :as http] + [less.awful.ssl :refer [trust-store]])) + +;; Note that this is not a working example, you'll need to figure out your K8s API path. +(let [k8s-trust-store (trust-store (clojure.java.io/file "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")) + bearer-token (format "Bearer %s" (slurp "/var/run/secrets/kubernetes.io/serviceaccount/token")) + kube-api-host (System/getenv "KUBERNETES_SERVICE_HOST") + kube-api-port (System/getenv "KUBERNETES_SERVICE_PORT")] + (http/get + (format "https://%s:%s/apis/" kube-api-host kube-api-port) + {:trust-store k8s-trust-store + :headers {:authorization bearer-token}})) + From 97ec651b31d5da9fd73665c21ab5bcb3e17c2002 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 15 Jul 2019 07:48:22 -0700 Subject: [PATCH 015/107] Update changelog up to 3.10.0 (#498) * Update changelog up to 3.10.0 Fixes #496 * Fixup changelog --- changelog.org | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/changelog.org b/changelog.org index c14a14ce..35db4b19 100644 --- a/changelog.org +++ b/changelog.org @@ -10,10 +10,23 @@ * Changelog List of user-visible changes that have gone into each release -** 3.10.0 (Unreleased) -- Add trust-manager and key-managers support to the client - +** 3.10.1 (Unreleased) +- TBD +** 3.10.0 +- Add trust-manager and key-managers support to the client https://github.com/dakrone/clj-http/pull/469 - +- Improving consistency of connection option names + https://github.com/dakrone/clj-http/pull/483 + https://github.com/dakrone/clj-http/issues/477 +- Ensure Socket Timeout is set for BasicHttpClientConnectionManager + https://github.com/dakrone/clj-http/pull/463 +- Reduce body allocation and copying + https://github.com/dakrone/clj-http/pull/475 +** 3.9.1 +- Fix body parsing when first byte value is 255 + https://github.com/dakrone/clj-http/pull/449 +- Add custom =:unexceptional-status= option + https://github.com/dakrone/clj-http/pull/451 ** 3.9.0 - Add support for reusable http clients, returning the client in =:http-client= and allowing one to be specified (with the same setting) - https://github.com/dakrone/clj-http/issues/441 From 66f04aecb67d78253918d636a96efdb8fb3ea365 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 15 Jul 2019 09:33:59 -0700 Subject: [PATCH 016/107] Add example for creating a custom RedirectStrategy (#500) Closes #456 --- README.org | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.org b/README.org index bea69239..783b6833 100644 --- a/README.org +++ b/README.org @@ -37,6 +37,7 @@ - [[#meta-tag-headers][Meta Tag Headers]] - [[#link-headers][Link Headers]] - [[#redirects][Redirects]] + - [[#how-to-create-a-custom-redirectstrategy][How to create a custom RedirectStrategy]] - [[#cookies][Cookies]] - [[#cookiestores][Cookiestores]] - [[#keystores-trust-stores][Keystores, Trust-stores]] @@ -710,6 +711,34 @@ this by setting =:validate-redirects false= in the request (the default is true) NOTE: The options =:force-redirects= and =:follow-redirects= (present in clj-http 2.x are no longer used). You can use =:graceful= to mostly emulate the old redirect behaviour. +*** How to create a custom RedirectStrategy +As mentioned earlier, it's possible to pass a custom instance of RedirectStrategy. The snippet below shows how to create a custom =RedirectStrategy= by wrapping the default strategy. + +#+begin_src clojure + (def default-strategy org.apache.http.impl.client.DefaultRedirectStrategy/INSTANCE) + + (def logging-redirect-strategy + (reify org.apache.http.client.RedirectStrategy + (getRedirect [this request response context] + (println "attempting redirect...") + (.getRedirect default-strategy request response context)) + (isRedirected [this request response context] + (println "checking isRedirected") + (.isRedirected default-strategy request response context)))) + + (client/get "https://httpbin.org/absolute-redirect/3" {:redirect-strategy logging-redirect-strategy}) + ;; this will output the following: + ;; + ;; checking isRedirected + ;; attempting redirect... + ;; checking isRedirected + ;; attempting redirect... + ;; checking isRedirected + ;; attempting redirect... + ;; checking isRedirected +#+end_src + + ** Cookies :PROPERTIES: :CUSTOM_ID: h-3bb89b16-4be3-455e-98ec-c5ca5830ddb9 From 64f212c58318cd4c7b4594af79d320b745ac07ea Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 15 Jul 2019 09:35:20 -0700 Subject: [PATCH 017/107] Add documentation for full request (#499) * Fixing ToC based on orgmode's suggestions * Add additional example for specifying multiple get options Closes #485 --- README.org | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.org b/README.org index 783b6833..8f160f3f 100644 --- a/README.org +++ b/README.org @@ -17,6 +17,7 @@ #+ATTR_HTML: title="Join the chat at https://gitter.im/clj-http/Lobby" [[https://gitter.im/clj-http/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge][file:https://badges.gitter.im/clj-http/Lobby.svg]] +- [[#branches][Branches]] - [[#introduction][Introduction]] - [[#overview][Overview]] - [[#philosophy][Philosophy]] @@ -44,6 +45,7 @@ - [[#exceptions][Exceptions]] - [[#decompression][Decompression]] - [[#debugging][Debugging]] + - [[#logging][Logging]] - [[#caching][Caching]] - [[#authentication][Authentication]] - [[#basic-auth][Basic Auth]] @@ -54,8 +56,10 @@ - [[#raw-request][Raw Request]] - [[#boolean-options][Boolean options]] - [[#persistent-connections][Persistent Connections]] + - [[#re-using-httpclient-between-requests][Re-using =HttpClient= between requests]] - [[#proxies][Proxies]] - [[#custom-middleware][Custom Middleware]] + - [[#modifying-apache-specific-features-of-the-httpclientbuilder-and-httpasyncclientbuilder][Modifying Apache-specific features of the =HttpClientBuilder= and =HttpAsyncClientBuilder=]] - [[#development][Development]] - [[#faking-responses][Faking Responses]] - [[#optional-dependencies][Optional Dependencies]] @@ -65,7 +69,7 @@ - [[#nohttpresponseexception--due-to-stale-connections][NoHttpResponseException ... due to stale connections**]] - [[#tests][Tests]] - [[#testimonials][Testimonials]] -- [[#other-middleware][Other Middleware]] +- [[#other-libraries-providing-middleware][Other Libraries Providing Middleware]] - [[#license][License]] @@ -178,7 +182,9 @@ Example requests: (client/get "http://example.com/resources/id") +;; Setting options (client/get "http://example.com/resources/3" {:accept :json}) +(client/get "http://example.com/resources/3" {:accept :json :query-params {"q" "foo, bar"}}) ;; Specifying headers as either a string or collection: (client/get "http://example.com" @@ -423,7 +429,7 @@ start an async request is easy, for example: All exceptions thrown during the request will be passed to the raise callback. -*** Cancelling requests +*** Cancelling Requests :PROPERTIES: :CUSTOM_ID: cancelling-requests :END: @@ -1498,7 +1504,7 @@ Libraries inspired by clj-http: - [[https://github.com/mpenet/jet][jet]] - [[https://github.com/hiredman/clj-http-lite][clj-http-lite]] -* Other libraries providing middleware +* Other Libraries Providing Middleware :PROPERTIES: :CUSTOM_ID: other-middleware :END: From fa075ccb66e7a1e77c4c094b35bfc21d270fcc76 Mon Sep 17 00:00:00 2001 From: Ben Bader Date: Wed, 31 Jul 2019 14:18:04 -0700 Subject: [PATCH 018/107] Add {:as :reader} response-body coercion (#503) --- README.org | 6 ++++++ src/clj_http/client.clj | 7 +++++++ test/clj_http/test/client_test.clj | 15 +++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/README.org b/README.org index 8f160f3f..833fc282 100644 --- a/README.org +++ b/README.org @@ -530,6 +530,12 @@ it's received (output coercion) from the server. (client/get "http://example.com/bigrequest.html" {:as :stream}) ;; Note that the connection to the server will NOT be closed until the ;; stream has been read + +;; Return the body as a java.io.BufferedReader +(client/get "http://example.com/bigrequest.html" {:as :reader}) +;; As above, the connection will remain open until the stream has been +;; read. The reader will attempt to respect the server-specified charset, +;; if any, defaulting to UTF-8. #+END_SRC Output coercion with =:as :json=, =:as :json-strict=, =:as :json-strict-string-keys=, =:as :json-string-keys= or =:as :x-www-form-urlencoded= will only work with an optional dependency, see [[#optional-dependencies][Optional Dependencies]]. diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index b17f22cb..64f21c79 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -444,6 +444,13 @@ (or (-> response :content-type-params :charset) "UTF-8")) +(defmethod coerce-response-body :reader + [_ {:keys [body] :as resp}] + (let [header (get-in resp [:headers "content-type"]) + parsed-values (util/parse-content-type header) + charset (response-charset parsed-values)] + (assoc resp :body (io/reader body :encoding charset)))) + (defn- can-parse-body? [{:keys [coerce] :as request} {:keys [status] :as _response}] (or (= coerce :always) (and (unexceptional-status-for-request? request status) diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index 7c04b3ec..8c0a1373 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -1556,6 +1556,21 @@ (:body (client/coerce-response-body {:as :x-www-form-urlencoded} www-form-urlencoded-resp)))))) +(deftest t-reader-coercion + (let [read-lines (fn [reader] (vec (take-while not-empty (repeatedly #(.readLine reader))))) + reader-body (ByteArrayInputStream. (.getBytes "foo\nbar\n")) + reader-resp {:body reader-body :status 200 :headers {"content-type" "text/plain; charset=utf-8"}} + encoded-body (ByteArrayInputStream. (byte-array [0xA9])) + encoded-resp {:body encoded-body :status 200 :headers {"content-type" "text/plain; charset=iso-8859-1"}} + utf8-body (ByteArrayInputStream. (byte-array [0xC2 0xA9])) + utf8-resp {:body utf8-body :status 200 :headers {"content-type" "text/plain; charset=utf-8"}}] + (is (= ["foo" "bar"] + (read-lines (:body (client/coerce-response-body {:as :reader} reader-resp))))) + + (is (= "©" + (.readLine (:body (client/coerce-response-body {:as :reader} encoded-resp))) + (.readLine (:body (client/coerce-response-body {:as :reader} utf8-resp))))))) + (deftest ^:integration t-with-middleware (run-server) (is (:request-time (request {:uri "/get" :method :get}))) From fe91ea53b9a311f5121f31f8a6da5c4b3a956232 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Tue, 6 Aug 2019 12:48:36 -0700 Subject: [PATCH 019/107] Switch oraclejdk8 to openjdk8 (#506) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d8d2b0ea..fd2e94e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ branches: # 'master' is the new apache 5 client, and not passing yet # - master jdk: - - oraclejdk8 + - openjdk8 - openjdk9 - openjdk10 - openjdk11 From 0aee7dc5ba14c33b3184e08494b5da0d4966893a Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Tue, 6 Aug 2019 21:59:02 +0200 Subject: [PATCH 020/107] Throw on missing coercing dependency (#493) * Throw on missing coercing dependency This ensures coercing functions that depend on optional dependencies fail if that dependency is missing, instead of silently failing. See: * https://github.com/dakrone/clj-http/issues/479 * https://github.com/dakrone/clj-http/issues/281 * https://github.com/dakrone/clj-http/issues/282 * Add tests for json/transit disabled state * Add tests for x-www-form-urlencoded disabled state --- src/clj_http/client.clj | 21 +++++++++------------ test/clj_http/test/client_test.clj | 13 ++++++++++++- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index 64f21c79..12c96174 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -475,11 +475,10 @@ (defn coerce-json-body [request {:keys [body] :as resp} keyword? strict? & [charset]] + {:pre [json-enabled?]} (let [charset (or charset (response-charset resp)) - body (if json-enabled? - (if (can-parse-body? request resp) - (decode-json-body body keyword? strict? charset) - (util/force-string body charset)) + body (if (can-parse-body? request resp) + (decode-json-body body keyword? strict? charset) (util/force-string body charset))] (assoc resp :body body))) @@ -496,21 +495,19 @@ (defn coerce-transit-body [{:keys [transit-opts] :as request} {:keys [body] :as resp} type & [charset]] + {:pre [transit-enabled?]} (let [charset (or charset (response-charset resp)) - body (if transit-enabled? - (if (can-parse-body? request resp) - (parse-transit (util/force-stream body) type transit-opts) - (util/force-string body charset)) - nil)] + body (if (can-parse-body? request resp) + (parse-transit (util/force-stream body) type transit-opts) + (util/force-string body charset))] (assoc resp :body body))) (defn coerce-form-urlencoded-body [_request {:keys [body] :as resp}] + {:pre [ring-codec-enabled?]} (let [charset (response-charset resp) body (util/force-string body charset)] - (assoc resp :body (if ring-codec-enabled? - (-> body form-decode keywordize-keys) - body)))) + (assoc resp :body (-> body form-decode keywordize-keys)))) (defmulti coerce-content-type (fn [req resp] (:content-type resp))) diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index 8c0a1373..c15d4146 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -1554,7 +1554,18 @@ (:body (client/coerce-response-body {:as :auto} auto-www-form-urlencoded-resp)) (:body (client/coerce-response-body {:as :x-www-form-urlencoded} - www-form-urlencoded-resp)))))) + www-form-urlencoded-resp)))) + + (testing "throws AssertionError when optional libraries are not loaded" + (with-redefs [client/json-enabled? false] + (is (thrown? AssertionError (client/coerce-response-body {:as :json} json-resp))) + (is (thrown? AssertionError (client/coerce-response-body {:as :auto} json-resp)))) + (with-redefs [client/transit-enabled? false] + (is (thrown? AssertionError (client/coerce-response-body {:as :transit+json} transit-json-resp))) + (is (thrown? AssertionError (client/coerce-response-body {:as :transit+msgpack} transit-msgpack-resp)))) + (with-redefs [client/ring-codec-enabled? false] + (is (thrown? AssertionError (client/coerce-response-body {:as :x-www-form-urlencoded} www-form-urlencoded-resp))) + (is (thrown? AssertionError (client/coerce-response-body {:as :auto} auto-www-form-urlencoded-resp))))))) (deftest t-reader-coercion (let [read-lines (fn [reader] (vec (take-while not-empty (repeatedly #(.readLine reader))))) From 7120fc8d8981bb6261c46002296b24fec524430c Mon Sep 17 00:00:00 2001 From: Stanislav Yurin Date: Wed, 21 Aug 2019 20:48:11 -0400 Subject: [PATCH 021/107] add multipart-mode request option --- src/clj_http/core.clj | 1 + src/clj_http/multipart.clj | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index 1acc835d..5f2a7ea7 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -563,6 +563,7 @@ uri response-interceptor proxy-host proxy-port http-client-context http-request-config http-client proxy-ignore-hosts proxy-user proxy-pass digest-auth ntlm-auth + multipart-mode ; deprecated conn-timeout conn-request-timeout] :as req} respond raise] diff --git a/src/clj_http/multipart.clj b/src/clj_http/multipart.clj index dcb5c16c..cb11637f 100644 --- a/src/clj_http/multipart.clj +++ b/src/clj_http/multipart.clj @@ -128,10 +128,12 @@ (defn create-multipart-entity "Takes a multipart vector of maps and creates a MultipartEntity with each map added as a part, depending on the type of content." - [multipart {:keys [mime-subtype]}] + [multipart {:keys [mime-subtype multipart-mode] + :or {mime-subtype "form-data" + multipart-mode HttpMultipartMode/STRICT}}] (let [mp-entity (doto (MultipartEntityBuilder/create) - (.setStrictMode) - (.setMimeSubtype (or mime-subtype "form-data")))] + (.setMode multipart-mode) + (.setMimeSubtype mime-subtype))] (doseq [m multipart] (let [name (or (:part-name m) (:name m)) part (make-multipart-body m)] From 709693da98cdccbe1900c0eecb769c801ac99731 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 7 Oct 2019 23:42:38 -0700 Subject: [PATCH 022/107] Implement Associative.EntryAt for header-map --- src/clj_http/headers.clj | 5 +++++ test/clj_http/test/headers_test.clj | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/clj_http/headers.clj b/src/clj_http/headers.clj index 9b1e068e..59743008 100644 --- a/src/clj_http/headers.clj +++ b/src/clj_http/headers.clj @@ -119,9 +119,14 @@ mta) (with-meta [_ mta] (HeaderMap. m mta)) + clojure.lang.Associative (containsKey [_ k] (contains? m (normalize k))) + (entryAt [_ k] + (if (contains? m (normalize k)) + (clojure.lang.MapEntry. k (get _ k)))) + (empty [_] (HeaderMap. {} nil))) diff --git a/test/clj_http/test/headers_test.clj b/test/clj_http/test/headers_test.clj index 74dc0e69..1470017d 100644 --- a/test/clj_http/test/headers_test.clj +++ b/test/clj_http/test/headers_test.clj @@ -66,7 +66,13 @@ (is (= "baz" (:foo (merge (header-map :foo "bar") {"Foo" "baz"})))) (let [m-with-meta (with-meta m {:withmeta-test true})] - (is (= (:withmeta-test (meta m-with-meta)) true))))) + (is (= (:withmeta-test (meta m-with-meta)) true))) + + (testing "select-keys" + (are [expected keyset] (= expected (select-keys m keyset)) + {"foo" "bar"} ["foo"] + {"foo" "bar"} ["foo" "non-existent-key"] + {"foo" "bar" "Foo" "bar" :foo "bar"} ["foo" "Foo" :foo])))) (deftest test-empty (testing "an empty header-map is a header-map" From 7c9a6d095252a59aae4f41fde40485838e487bef Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Thu, 10 Oct 2019 20:30:45 -0700 Subject: [PATCH 023/107] Set .disableContentCompression in build-http-client The HttpAsyncClientBuilder does not have this option, which leads to inconsistency of which options to set and runtime behaviour. By disabling it in the HttpClientBuilder and relying on wrap-decompression, the experience for end-users is consistent. --- src/clj_http/core.clj | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index 5f2a7ea7..4e526df7 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -323,6 +323,11 @@ (.setRedirectStrategy (get-redirect-strategy req)) (add-retry-handler retry-handler) + + ;; prefer using clj-http.client/wrap-decompression + ;; for consistency between sync/async client options + (.disableContentCompression) + ;; By default, get the proxy settings ;; from the jvm or system properties (.setRoutePlanner From ce9a02b04ec93278cc1444fa3c92e8ba7ca34a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juha=20Syrj=C3=A4l=C3=A4?= Date: Mon, 14 Oct 2019 23:23:23 +0300 Subject: [PATCH 024/107] Update httpcomponents to latest versions (#517) org.apache.httpcomponents/httpcore 4.4.12 org.apache.httpcomponents/httpclient 4.5.10 org.apache.httpcomponents/httpclient-cache 4.5.10 org.apache.httpcomponents/httpmime 4.5.10 Fixes #514 --- project.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project.clj b/project.clj index a39bd126..7c6cbb9f 100644 --- a/project.clj +++ b/project.clj @@ -7,11 +7,11 @@ :global-vars {*warn-on-reflection* false} :min-lein-version "2.0.0" :exclusions [org.clojure/clojure] - :dependencies [[org.apache.httpcomponents/httpcore "4.4.11"] - [org.apache.httpcomponents/httpclient "4.5.8"] - [org.apache.httpcomponents/httpclient-cache "4.5.8"] + :dependencies [[org.apache.httpcomponents/httpcore "4.4.12"] + [org.apache.httpcomponents/httpclient "4.5.10"] + [org.apache.httpcomponents/httpclient-cache "4.5.10"] [org.apache.httpcomponents/httpasyncclient "4.1.4"] - [org.apache.httpcomponents/httpmime "4.5.8"] + [org.apache.httpcomponents/httpmime "4.5.10"] [commons-codec "1.12"] [commons-io "2.6"] [slingshot "0.12.2"] From ff91bc61684db7254ba25912fb79dca64fb20555 Mon Sep 17 00:00:00 2001 From: Alex ter Weele <30635088+atw-gr@users.noreply.github.com> Date: Thu, 24 Oct 2019 15:52:49 -0500 Subject: [PATCH 025/107] Add :http-client-builder option. (#490) This adds an advanced option for specifying the `HttpClientBuilder`. --- README.org | 8 ++++++++ src/clj_http/core.clj | 8 ++++++-- test/clj_http/test/core_test.clj | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index 833fc282..349245b1 100644 --- a/README.org +++ b/README.org @@ -1384,6 +1384,14 @@ Each of these variables is a sequence of functions of two arguments, the http bu The functions are run in the order they are passed in (inside a =doseq=). +By specifying =:http-client-builder=, your own instance of +=HttpClientBuilder= will be used. A supplied =HttpClientBuilder= which +sets the connection manager, redirect strategy, retry handler, route +planner, cache, or cookie spec registry may find these overridden by +clj-http's =:connection-manager=, =:redirect-strategy=, +=:retry-handler=, =:cache=, or =:cookie-policy-registry= or +=:cookie-spec=, respectively. + * Development :PROPERTIES: :CUSTOM_ID: h-65bbf017-2e8b-4c43-824b-24b89cc27a70 diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index 4e526df7..4af7ab3d 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -309,15 +309,19 @@ [{:keys [retry-handler request-interceptor response-interceptor proxy-host proxy-port http-builder-fns cookie-spec - cookie-policy-registry] + cookie-policy-registry + ^HttpClientBuilder http-client-builder] :as req} caching? conn-mgr & [http-url proxy-ignore-hosts]] ;; have to let first, otherwise we get a reflection warning on (.build) (let [cache? (opt req :cache) - builder (-> (if caching? + builder (-> (cond + http-client-builder http-client-builder + caching? ^HttpClientBuilder (CachingHttpClientBuilder/create) + :else ^HttpClientBuilder (HttpClients/custom)) (.setConnectionManager conn-mgr) (.setRedirectStrategy diff --git a/test/clj_http/test/core_test.clj b/test/clj_http/test/core_test.clj index b1b35aba..468fcf23 100644 --- a/test/clj_http/test/core_test.clj +++ b/test/clj_http/test/core_test.clj @@ -740,6 +740,23 @@ "Headers should have included the new default headers") (is (not (realized? error))))) +(deftest ^:integration test-custom-http-client-builder + (run-server) + (let [methods (atom nil) + resp (client/get + (localhost "/get") + {:http-client-builder + (-> (org.apache.http.impl.client.HttpClientBuilder/create) + (.setRequestExecutor + (proxy [org.apache.http.protocol.HttpRequestExecutor] [] + (execute [request connection context] + (->> request + .getRequestLine + .getMethod + (swap! methods conj)) + (proxy-super execute request connection context)))))})] + (is (= ["GET"] @methods)))) + (deftest ^:integration test-bad-redirects (run-server) (try From 217393258e7863514debece4eb7b23a7a3fa8bd9 Mon Sep 17 00:00:00 2001 From: saitouena <40712240+saitouena@users.noreply.github.com> Date: Fri, 8 Nov 2019 01:16:15 +0900 Subject: [PATCH 026/107] Add option to add multipart charset (#522) * add option to add multipart charset see also: https://stackoverflow.com/questions/3393445/international-characters-in-filename-in-mutipart-formdata To handle non-ascii characters, we need a way to set .setCharset and .setMode BROWSER_COMPATIBLE --- README.org | 9 +++++++++ src/clj_http/core.clj | 2 +- src/clj_http/multipart.clj | 4 +++- test/clj_http/test/multipart_test.clj | 7 +++++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/README.org b/README.org index 349245b1..f05b44ec 100644 --- a/README.org +++ b/README.org @@ -383,6 +383,15 @@ content encodings. :retry-handler (fn [ex try-count http-context] (println "Got:" ex) (if (> try-count 4) false true))}) + +;; to handle a file with non-ascii filename, try :multipart-charset "UTF-8" and :multipart-mode BROWSER_COMPATIBLE +;; see also: https://stackoverflow.com/questions/3393445/international-characters-in-filename-in-mutipart-formdata +(import (org.apache.http.entity.mime HttpMultipartMode)) + +(client/post "http://example.org" {:multipart [{:content (clojure.java.io/file "日本語.txt")}] + :multipart-mode HttpMultipartMode/BROWSER_COMPATIBLE + :multipart-charset "UTF-8"} ) + #+END_SRC A word about flattening nested =:query-params= and =:form-params= maps. There are essentially three diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index 4af7ab3d..527caf68 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -572,7 +572,7 @@ uri response-interceptor proxy-host proxy-port http-client-context http-request-config http-client proxy-ignore-hosts proxy-user proxy-pass digest-auth ntlm-auth - multipart-mode + multipart-mode multipart-charset ; deprecated conn-timeout conn-request-timeout] :as req} respond raise] diff --git a/src/clj_http/multipart.clj b/src/clj_http/multipart.clj index cb11637f..ce72b5fa 100644 --- a/src/clj_http/multipart.clj +++ b/src/clj_http/multipart.clj @@ -128,12 +128,14 @@ (defn create-multipart-entity "Takes a multipart vector of maps and creates a MultipartEntity with each map added as a part, depending on the type of content." - [multipart {:keys [mime-subtype multipart-mode] + [multipart {:keys [mime-subtype multipart-mode multipart-charset] :or {mime-subtype "form-data" multipart-mode HttpMultipartMode/STRICT}}] (let [mp-entity (doto (MultipartEntityBuilder/create) (.setMode multipart-mode) (.setMimeSubtype mime-subtype))] + (when multipart-charset + (.setCharset mp-entity (encoding-to-charset multipart-charset))) (doseq [m multipart] (let [name (or (:part-name m) (:name m)) part (make-multipart-body m)] diff --git a/test/clj_http/test/multipart_test.clj b/test/clj_http/test/multipart_test.clj index a0fa4958..eebb69a3 100644 --- a/test/clj_http/test/multipart_test.clj +++ b/test/clj_http/test/multipart_test.clj @@ -176,6 +176,9 @@ (is (= "testname" (.getFilename body))))))) (deftest test-multipart-content-charset - (testing "charset is nil for all multipart requests" + (testing "charset is nil if no multipart-charset is supplied" (let [mp-entity (create-multipart-entity [] nil)] - (is (nil? (EntityUtils/getContentCharSet mp-entity)))))) + (is (nil? (EntityUtils/getContentCharSet mp-entity))))) + (testing "charset is set if a multipart-charset is supplied" + (let [mp-entity (create-multipart-entity [] {:multipart-charset "UTF-8"})] + (is (= "UTF-8" (EntityUtils/getContentCharSet mp-entity)))))) From 39935e5e9637067a8aae2e4bffb0b8b312c0f0fd Mon Sep 17 00:00:00 2001 From: Ambrose Bonnaire-Sergeant Date: Mon, 3 Feb 2020 18:20:36 -0500 Subject: [PATCH 027/107] Fix #528: resolve performance warnings (#532) --- src/clj_http/client.clj | 2 +- src/clj_http/util.clj | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index 12c96174..f11bb740 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -466,7 +466,7 @@ (let [^BufferedReader br (io/reader (util/force-stream body))] (try (.mark br 1) - (let [^int first-char (try (.read br) (catch EOFException _ -1))] + (let [first-char (int (try (.read br) (catch EOFException _ -1)))] (case first-char -1 nil (do (.reset br) diff --git a/src/clj_http/util.clj b/src/clj_http/util.clj index a125fed4..201e7391 100644 --- a/src/clj_http/util.clj +++ b/src/clj_http/util.clj @@ -71,7 +71,7 @@ (if (instance? InputStream b) (let [^PushbackInputStream bs (PushbackInputStream. b)] (try - (let [^int first-byte (try (.read bs) (catch EOFException _ -1))] + (let [first-byte (int (try (.read bs) (catch EOFException _ -1)))] (case first-byte -1 (byte-array 0) (do (.unread bs first-byte) @@ -85,7 +85,7 @@ (if (instance? InputStream s) (let [^PushbackInputStream bs (PushbackInputStream. s)] (try - (let [^int first-byte (try (.read bs) (catch EOFException _ -1))] + (let [first-byte (int (try (.read bs) (catch EOFException _ -1)))] (case first-byte -1 "" (do (.unread bs first-byte) From 25193b1f4e80c0eacc3aab51aa406a1ce84d6f32 Mon Sep 17 00:00:00 2001 From: wmatson Date: Mon, 3 Feb 2020 16:21:13 -0700 Subject: [PATCH 028/107] #530 fixing build-http-client calls in README (#531) --- README.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index f05b44ec..c16b7b4c 100644 --- a/README.org +++ b/README.org @@ -1265,7 +1265,7 @@ manager must be used. ;; You can also build your own, using clj-http's helper or manually building it: (let [cm (conn/make-reusable-conn-manager {}) - hclient (core/build-http-client {} cm "https://example.com" false)] + hclient (core/build-http-client {} false cm)] (client/get "http://example.com/1" {:connection-manager cm :http-client hclient}) (client/get "http://example.com/2" @@ -1275,7 +1275,7 @@ manager must be used. ;; Async http clients may also be created and re-used: (let [acm (conn/make-reuseable-async-conn-manager {}) - ahclient (core/build-async-http-client {} acm "https://example.com" false)] + ahclient (core/build-async-http-client {} acm)] (client/get "http://example.com/1" {:connection-manager cm :http-client ahclient} handle-response handle-failure) From d68242caa72f80ce24288d3e579508d6523d917d Mon Sep 17 00:00:00 2001 From: "Jose V. Trigueros" Date: Tue, 4 Feb 2020 08:58:48 -0800 Subject: [PATCH 029/107] Fix typo in "unexceptional" docs (#533) The word "unexceptional" was misspelled and the description of usage for the `:unexceptional-status` was missing *not*. --- README.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index c16b7b4c..5eba4cd5 100644 --- a/README.org +++ b/README.org @@ -550,7 +550,7 @@ it's received (output coercion) from the server. Output coercion with =:as :json=, =:as :json-strict=, =:as :json-strict-string-keys=, =:as :json-string-keys= or =:as :x-www-form-urlencoded= will only work with an optional dependency, see [[#optional-dependencies][Optional Dependencies]]. By default, JSON coercion is only applied when the response's status -is considered "unexceptional". If the =:unexeceptional-status= option +is considered "unexceptional". If the =:unexceptional-status= option is provided, then its value is a function which specifies what status codes are unexceptional. =:unexceptional-status= defaults to =clj-http.client/unexceptional-status?=. @@ -876,7 +876,7 @@ HTTP responses other than =#{200 201 202 203 204 205 206 207 300 301 302 303 304 ;; Or ignore an unknown host (methods return 'nil' if this is set to ;; true and the host does not exist: (client/get "http://example.invalid" {:ignore-unknown-host? true}) -;; Or customize the http statuses that will throw: +;; Or customize the http statuses that will not throw: (client/get "http://example.com/broken" {:unexceptional-status #(<= 200 % 299)}) #+END_SRC From 5154fcc6b24a9ce4bbff6ba45c971c19976491a3 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 13 Apr 2020 16:01:00 -0700 Subject: [PATCH 030/107] Always parse json in strict mode (#507) * Make #'handler reloadable during test iteration * Add test-case for lazily parsing large json array that throws IOException * Respect org file fill-column * Add instructions for how to incrementally/lazily parse json * Remove lazy json parsing from coercions * Fix tests -- should always return a vector now --- README.org | 42 ++++++++++-- changelog.org | 2 +- project.clj | 2 +- src/clj_http/client.clj | 58 +++++++--------- test-resources/big_array_json.json | 102 +++++++++++++++++++++++++++++ test/clj_http/test/core_test.clj | 20 +++++- 6 files changed, 182 insertions(+), 44 deletions(-) create mode 100644 test-resources/big_array_json.json diff --git a/README.org b/README.org index 5eba4cd5..4d8b0dc9 100644 --- a/README.org +++ b/README.org @@ -60,6 +60,7 @@ - [[#proxies][Proxies]] - [[#custom-middleware][Custom Middleware]] - [[#modifying-apache-specific-features-of-the-httpclientbuilder-and-httpasyncclientbuilder][Modifying Apache-specific features of the =HttpClientBuilder= and =HttpAsyncClientBuilder=]] + - [[#incrementally-json-parsing][Incrementally JSON Parsing]] - [[#development][Development]] - [[#faking-responses][Faking Responses]] - [[#optional-dependencies][Optional Dependencies]] @@ -72,7 +73,6 @@ - [[#other-libraries-providing-middleware][Other Libraries Providing Middleware]] - [[#license][License]] - * Branches :PROPERTIES: :CUSTOM_ID: h-e390585c-cbd8-4e94-b36b-4e9c27c16720 @@ -514,9 +514,7 @@ it's received (output coercion) from the server. ;; Coerce as json (client/get "http://example.com/foo.json" {:as :json}) -(client/get "http://example.com/foo.json" {:as :json-strict}) (client/get "http://example.com/foo.json" {:as :json-string-keys}) -(client/get "http://example.com/foo.json" {:as :json-strict-string-keys}) ;; Coerce as Transit encoded JSON or MessagePack (client/get "http://example.com/foo" {:as :transit+json}) @@ -547,7 +545,7 @@ it's received (output coercion) from the server. ;; if any, defaulting to UTF-8. #+END_SRC -Output coercion with =:as :json=, =:as :json-strict=, =:as :json-strict-string-keys=, =:as :json-string-keys= or =:as :x-www-form-urlencoded= will only work with an optional dependency, see [[#optional-dependencies][Optional Dependencies]]. +Output coercion with =:as :json=, =:as :json-string-keys= or =:as :x-www-form-urlencoded=, will only work with an optional dependency, see [[#optional-dependencies][Optional Dependencies]]. By default, JSON coercion is only applied when the response's status is considered "unexceptional". If the =:unexceptional-status= option @@ -1401,6 +1399,38 @@ clj-http's =:connection-manager=, =:redirect-strategy=, =:retry-handler=, =:cache=, or =:cookie-policy-registry= or =:cookie-spec=, respectively. +** Incrementally JSON Parsing +[[https://github.com/dakrone/cheshire][cheshire]] supports incrementally parsing JSON using lazy sequences. This approach can useful for +processing large top-level JSON arrays because it doesn't require upfront work consuming the entire stream. + +#+begin_src clojure + (require '[cheshire.core :as json]) + + (defn print-all-pokemon-names [pokemons] + (for [pokemon pokemons] + (println (get-in pokemon [:name :english])))) + + (let [url "https://raw.githubusercontent.com/fanzeyi/pokemon.json/master/pokedex.json" + response (get url {:as :reader})] + (with-open [reader (:body response)] ; closes the underlying connection when we're done + (let [pokemons (json/parse-stream reader true)] + ; You must perform all reads from the stream inside `with-open`, + ; any , any lazy + (doall (print-all-pokemon-names pokemons))))) +#+end_src + +Keep in mind that the =reader= object wraps a HTTP connection. The user needs to be aware of two +things: + +1. The user should close the reader after processing the stream, otherwise the underlying HTTP + Connection may leak and create subtle bugs. Clojure's [[https://clojuredocs.org/clojure.core/with-open][with-open]] is useful here. + +2. You should realize any lazy sequences before closing the connection. Use [[https://clojuredocs.org/clojure.core/doall][doall]] or [[https://clojure.org/reference/transducers][transducers]] to + prevent bugs from lazy IO. See [[https://stuartsierra.com/2015/08/25/clojure-donts-lazy-effects][Clojure Don'ts: Lazy Effects]]. + +In previous versions of =clj-http= (<= 3.10.0), =clj-http= defaulted to lazily parsing JSON, but this +was slow and also confused users who didn't expect laziness. + * Development :PROPERTIES: :CUSTOM_ID: h-65bbf017-2e8b-4c43-824b-24b89cc27a70 @@ -1543,3 +1573,7 @@ Libraries inspired by clj-http: Released under the MIT License: + +# Local Variables: +# fill-column: 100 +# End: diff --git a/changelog.org b/changelog.org index 35db4b19..d861be76 100644 --- a/changelog.org +++ b/changelog.org @@ -11,7 +11,7 @@ List of user-visible changes that have gone into each release ** 3.10.1 (Unreleased) -- TBD +- JSON parsing is always strict. See [[file:README.org::*Incrementally%20JSON%20Parsing][README#Incrementally JSON Parsing]]. Requires cheshire > 5.9.0. ** 3.10.0 - Add trust-manager and key-managers support to the client https://github.com/dakrone/clj-http/pull/469 diff --git a/project.clj b/project.clj index 7c6cbb9f..6142c45b 100644 --- a/project.clj +++ b/project.clj @@ -18,7 +18,7 @@ [potemkin "0.4.5"]] :resource-paths ["resources"] :profiles {:dev {:dependencies [;; optional deps - [cheshire "5.8.1"] + [cheshire "5.9.0"] [crouton "0.1.2" :exclusions [[org.jsoup/jsoup]]] [org.jsoup/jsoup "1.11.3"] [org.clojure/tools.reader "1.3.2"] diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index f11bb740..e058c9b4 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -128,20 +128,7 @@ "Resolve and apply cheshire's json decoding dynamically." [& args] {:pre [json-enabled?]} - (apply (ns-resolve (symbol "cheshire.core") (symbol "decode")) args)) - -(defn ^:dynamic json-decode-strict - "Resolve and apply cheshire's json decoding dynamically (with lazy parsing - disabled)." - [& args] - {:pre [json-enabled?]} - (apply (ns-resolve (symbol "cheshire.core") (symbol "decode-strict")) args)) - -(defn ^:dynamic json-decode-stream - "Resolve and apply cheshire's json stream decoding dynamically." - [& args] - {:pre [json-enabled?]} - (apply (ns-resolve (symbol "cheshire.core") (symbol "decode-stream")) args)) + (apply (ns-resolve (symbol "cheshire.core") (symbol "parse-stream-strict")) args)) (defn ^:dynamic form-decode "Resolve and apply ring-codec's form decoding dynamically." @@ -459,26 +446,23 @@ (and (not (unexceptional-status-for-request? request status)) (= coerce :exceptional)))) -(defn- decode-json-body [body keyword? strict? charset] - (if strict? - ;; OPTIMIZE: When/if Cheshire gets a parse-stream-strict this won't need to go through String: - (json-decode-strict (util/force-string body charset) keyword?) - (let [^BufferedReader br (io/reader (util/force-stream body))] - (try - (.mark br 1) - (let [first-char (int (try (.read br) (catch EOFException _ -1)))] - (case first-char - -1 nil - (do (.reset br) - (json-decode-stream br keyword?)))) - (finally (.close br)))))) +(defn- decode-json-body [body keyword? charset] + (let [^BufferedReader br (io/reader (util/force-stream body))] + (try + (.mark br 1) + (let [^int first-char (try (.read br) (catch EOFException _ -1))] + (case first-char + -1 nil + (do (.reset br) + (json-decode br keyword?)))) + (finally (.close br))))) (defn coerce-json-body - [request {:keys [body] :as resp} keyword? strict? & [charset]] + [request {:keys [body] :as resp} keyword? & [charset]] {:pre [json-enabled?]} (let [charset (or charset (response-charset resp)) body (if (can-parse-body? request resp) - (decode-json-body body keyword? strict? charset) + (decode-json-body body keyword? charset) (util/force-string body charset))] (assoc resp :body body))) @@ -540,16 +524,20 @@ (coerce-content-type request)))) (defmethod coerce-response-body :json [req resp] - (coerce-json-body req resp true false)) + (coerce-json-body req resp true)) + +(defmethod coerce-response-body :json-string-keys [req resp] + (coerce-json-body req resp false)) +;; There is no longer any distinction between strict and non-strict JSON parsing +;; options. +;; +;; `:json-strict` and `:json-strict-string-keys` will be removed in a future version (defmethod coerce-response-body :json-strict [req resp] - (coerce-json-body req resp true true)) + (coerce-json-body req resp true)) (defmethod coerce-response-body :json-strict-string-keys [req resp] - (coerce-json-body req resp false true)) - -(defmethod coerce-response-body :json-string-keys [req resp] - (coerce-json-body req resp false false)) + (coerce-json-body req resp false)) (defmethod coerce-response-body :clojure [req resp] (coerce-clojure-body req resp)) diff --git a/test-resources/big_array_json.json b/test-resources/big_array_json.json new file mode 100644 index 00000000..51ccef7d --- /dev/null +++ b/test-resources/big_array_json.json @@ -0,0 +1,102 @@ +[ + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]} +] diff --git a/test/clj_http/test/core_test.clj b/test/clj_http/test/core_test.clj index 468fcf23..84c48250 100644 --- a/test/clj_http/test/core_test.clj +++ b/test/clj_http/test/core_test.clj @@ -58,6 +58,9 @@ [:get "/json-array"] {:status 200 :body "[\"foo\", \"bar\"]" :headers {"content-type" "application/json"}} + [:get "/json-large-array"] + {:status 200 :body (file "test-resources/big_array_json.json") + :headers {"content-type" "application/json"}} [:get "/json-bad"] {:status 400 :body "{\"foo\":\"bar\"}"} [:get "/redirect"] @@ -147,7 +150,7 @@ (defn run-server [] (defonce server - (ring/run-jetty (add-headers-if-requested handler) {:port 18080 :join? false}))) + (ring/run-jetty (add-headers-if-requested #'handler) {:port 18080 :join? false}))) (defn localhost [path] (str "http://localhost:18080" path)) @@ -427,7 +430,10 @@ (deftest ^:integration t-json-output-coercion (run-server) (let [resp (client/get (localhost "/json") {:as :json}) - resp-array (client/get (localhost "/json-array") {:as :json-strict}) + resp-array (client/get (localhost "/json-array") {:as :json}) + resp-array-strict (client/get (localhost "/json-array") {:as :json-strict}) + resp-large-array (client/get (localhost "/json-large-array") {:as :json}) + resp-large-array-strict (client/get (localhost "/json-large-array") {:as :json-strict}) resp-str (client/get (localhost "/json") {:as :json :coerce :exceptional}) resp-str-keys (client/get (localhost "/json") {:as :json-string-keys}) @@ -445,6 +451,9 @@ (is (= 200 (:status resp) (:status resp-array) + (:status resp-array-strict) + (:status resp-large-array) + (:status resp-large-array-strict) (:status resp-str) (:status resp-str-keys) (:status resp-strict-str-keys) @@ -459,6 +468,7 @@ (:body resp-str-keys))) ;; '("foo" "bar") and ["foo" "bar"] compare as equal with =. (is (vector? (:body resp-array))) + (is (vector? (:body resp-array-strict))) (is (= "{\"foo\":\"bar\"}" (:body resp-str))) (is (= 400 (:status bad-resp) @@ -467,7 +477,11 @@ (is (= "{\"foo\":\"bar\"}" (:body bad-resp)) "don't coerce on bad response status by default") (is (= {:foo "bar"} (:body bad-resp-json))) - (is (= "{\"foo\":\"bar\"}" (:body bad-resp-json2))))) + (is (= "{\"foo\":\"bar\"}" (:body bad-resp-json2))) + + (testing "lazily parsed stream completes parsing." + (is (= 100 (count (:body resp-large-array))))) + (is (= 100 (count (:body resp-large-array-strict)))))) (deftest ^:integration t-ipv6 (run-server) From 54699dd925b82ede520f9a1b76cee6583dec6b02 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Mon, 13 Apr 2020 17:03:28 -0600 Subject: [PATCH 031/107] Release 3.10.1 --- README.org | 8 +++++++- project.clj | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index 4d8b0dc9..cfcaf7e4 100644 --- a/README.org +++ b/README.org @@ -122,7 +122,7 @@ function. With Leiningen/Boot: #+BEGIN_SRC clojure -[clj-http "3.10.0"] +[clj-http "3.10.1"] #+END_SRC If you need an older version, a 2.x release is also available. @@ -731,6 +731,9 @@ NOTE: The options =:force-redirects= and =:follow-redirects= (present in clj-htt used). You can use =:graceful= to mostly emulate the old redirect behaviour. *** How to create a custom RedirectStrategy +:PROPERTIES: +:CUSTOM_ID: h:a3b8b124-411f-4c4c-ac4b-777624e76bf1 +:END: As mentioned earlier, it's possible to pass a custom instance of RedirectStrategy. The snippet below shows how to create a custom =RedirectStrategy= by wrapping the default strategy. #+begin_src clojure @@ -1400,6 +1403,9 @@ clj-http's =:connection-manager=, =:redirect-strategy=, =:cookie-spec=, respectively. ** Incrementally JSON Parsing +:PROPERTIES: +:CUSTOM_ID: h:b01c16e8-7179-468e-8890-316939ec0e38 +:END: [[https://github.com/dakrone/cheshire][cheshire]] supports incrementally parsing JSON using lazy sequences. This approach can useful for processing large top-level JSON arrays because it doesn't require upfront work consuming the entire stream. diff --git a/project.clj b/project.clj index 6142c45b..4652f68b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.10.1-SNAPSHOT" +(defproject clj-http "3.10.1" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From ab3a85d00f5effc6850a1f885e079f92bb83e044 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Mon, 13 Apr 2020 17:05:05 -0600 Subject: [PATCH 032/107] Bump to 3.10.2-SNAPSHOT --- changelog.org | 3 ++- project.clj | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog.org b/changelog.org index d861be76..2a4808d1 100644 --- a/changelog.org +++ b/changelog.org @@ -10,7 +10,8 @@ * Changelog List of user-visible changes that have gone into each release -** 3.10.1 (Unreleased) +** 3.10.2 (Unreleased) +** 3.10.1 - JSON parsing is always strict. See [[file:README.org::*Incrementally%20JSON%20Parsing][README#Incrementally JSON Parsing]]. Requires cheshire > 5.9.0. ** 3.10.0 - Add trust-manager and key-managers support to the client diff --git a/project.clj b/project.clj index 4652f68b..5bc7e8f9 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.10.1" +(defproject clj-http "3.10.2-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 911f4807f28a323515f1ee433a9d362b653b1768 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Thu, 14 May 2020 20:49:25 -0700 Subject: [PATCH 033/107] Add clojure 1.10 to test aliases (#541) --- project.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 5bc7e8f9..4d549b8c 100644 --- a/project.clj +++ b/project.clj @@ -42,8 +42,9 @@ :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} :1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} :1.8 {:dependencies [[org.clojure/clojure "1.8.0"]]} - :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]}} - :aliases {"all" ["with-profile" "dev,1.6:dev,1.7:dev,1.8:dev,1.9:dev"]} + :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]} + :1.10 {:dependencies [[org.clojure/clojure "1.10.1"]]}} + :aliases {"all" ["with-profile" "dev,1.6:dev,1.7:dev,1.8:dev,1.9:dev,1.10:dev"]} :plugins [[codox "0.6.4"]] :test-selectors {:default #(not (:integration %)) :integration :integration From 6fab3e190698450a584a4ff24ad843ade9efd69c Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Wed, 24 Jun 2020 22:20:14 -0700 Subject: [PATCH 034/107] Buffer debug output to avoid mangling output (#544) Fixes #536 --- src/clj_http/core.clj | 44 ++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index 527caf68..dacc77bf 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -485,27 +485,29 @@ (defn- print-debug! "Print out debugging information to *out* for a given request." [{:keys [debug-body body] :as req} http-req] - (println "Request:" (type body)) - (clojure.pprint/pprint - (assoc req - :body (if (opt req :debug-body) - (cond - (isa? (type body) String) - body - - (isa? (type body) HttpEntity) - (let [baos (ByteArrayOutputStream.)] - (.writeTo ^HttpEntity body baos) - (.toString baos "UTF-8")) - - :else nil) - (if (isa? (type body) String) - (format "... %s bytes ..." - (count body)) - (and body (bean body)))) - :body-type (type body))) - (println "HttpRequest:") - (clojure.pprint/pprint (bean http-req))) + (println + (with-out-str + (println "Request:" (type body)) + (clojure.pprint/pprint + (assoc req + :body (if (opt req :debug-body) + (cond + (isa? (type body) String) + body + + (isa? (type body) HttpEntity) + (let [baos (ByteArrayOutputStream.)] + (.writeTo ^HttpEntity body baos) + (.toString baos "UTF-8")) + + :else nil) + (if (isa? (type body) String) + (format "... %s bytes ..." + (count body)) + (and body (bean body)))) + :body-type (type body))) + (println "HttpRequest:") + (clojure.pprint/pprint (bean http-req))))) (defn- build-response-map [^HttpResponse response req ^HttpUriRequest http-req http-url From 8f84a39591276ff41ea64e3bbbcc1c1301f9ad20 Mon Sep 17 00:00:00 2001 From: Omar Hughes Date: Thu, 25 Jun 2020 09:20:34 +0400 Subject: [PATCH 035/107] Make compatible with graalvm (#543) --- src/clj_http/conn_mgr.clj | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/clj_http/conn_mgr.clj b/src/clj_http/conn_mgr.clj index 96e9a227..0585d1bb 100644 --- a/src/clj_http/conn_mgr.clj +++ b/src/clj_http/conn_mgr.clj @@ -49,10 +49,10 @@ (SSLIOSessionStrategy. ^SSLContext context ^HostnameVerifier verifier)))) (def ^:private ^SSLConnectionSocketFactory secure-ssl-socket-factory - (SSLConnectionSocketFactory/getSocketFactory)) + (delay (SSLConnectionSocketFactory/getSocketFactory))) (def ^:private ^SSLIOSessionStrategy secure-strategy - (SSLIOSessionStrategy/getDefaultStrategy)) + (delay (SSLIOSessionStrategy/getDefaultStrategy))) (defn ^SSLConnectionSocketFactory SSLGenericSocketFactory "Given a function that returns a new socket, create an @@ -178,16 +178,16 @@ (.build)))) (def ^:private regular-scheme-registry - (-> (RegistryBuilder/create) - (.register "http" (PlainConnectionSocketFactory/getSocketFactory)) - (.register "https" secure-ssl-socket-factory) - (.build))) + (delay (-> (RegistryBuilder/create) + (.register "http" (PlainConnectionSocketFactory/getSocketFactory)) + (.register "https" @secure-ssl-socket-factory) + (.build)))) (def ^:private regular-strategy-registry - (-> (RegistryBuilder/create) - (.register "http" NoopIOSessionStrategy/INSTANCE) - (.register "https" secure-strategy) - (.build))) + (delay (-> (RegistryBuilder/create) + (.register "http" NoopIOSessionStrategy/INSTANCE) + (.register "https" @secure-strategy) + (.build)))) (defn ^Registry get-custom-scheme-registry [{:keys [context verifier]}] @@ -245,7 +245,7 @@ (opt req :insecure) (BasicHttpClientConnectionManager. @insecure-scheme-registry) - :else (BasicHttpClientConnectionManager. regular-scheme-registry))] + :else (BasicHttpClientConnectionManager. @regular-scheme-registry))] (when socket-timeout (.setSocketConfig conn-manager (-> (.getSocketConfig conn-manager) @@ -286,7 +286,7 @@ (opt req :insecure) @insecure-strategy-registry - :else regular-strategy-registry) + :else @regular-strategy-registry) io-reactor (make-ioreactor {:shutdown-grace-period 1})] (doto (PoolingNHttpClientConnectionManager. io-reactor registry) (.setMaxTotal 1)))) @@ -312,7 +312,7 @@ (or keystore trust-store) (get-keystore-scheme-registry config) - :else regular-scheme-registry)] + :else @regular-scheme-registry)] (PoolingHttpClientConnectionManager. registry nil nil nil timeout java.util.concurrent.TimeUnit/SECONDS))) @@ -375,7 +375,7 @@ (or keystore trust-store) (get-keystore-scheme-registry config) - :else regular-strategy-registry) + :else @regular-strategy-registry) io-reactor (make-ioreactor io-config) protocol-handler (HttpAsyncRequestExecutor.) io-event-dispatch (DefaultHttpClientIODispatch. protocol-handler From 4b22f0f0476174af788e3c385beb7d0ad10eee1c Mon Sep 17 00:00:00 2001 From: Ambrose Bonnaire-Sergeant Date: Mon, 6 Jul 2020 20:22:33 -0400 Subject: [PATCH 036/107] Fix performance regression related to #528 (#546) --- src/clj_http/client.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index e058c9b4..4fdfb365 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -450,7 +450,7 @@ (let [^BufferedReader br (io/reader (util/force-stream body))] (try (.mark br 1) - (let [^int first-char (try (.read br) (catch EOFException _ -1))] + (let [first-char (int (try (.read br) (catch EOFException _ -1)))] (case first-char -1 nil (do (.reset br) From e27b1b5dae4b58940f7ad144948bbe6f822fd28d Mon Sep 17 00:00:00 2001 From: Ambrose Bonnaire-Sergeant Date: Fri, 17 Jul 2020 17:00:45 -0400 Subject: [PATCH 037/107] Fix suggested cheshire version in changelog (#548) Now agrees with project.clj. --- changelog.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.org b/changelog.org index 2a4808d1..56b3bf16 100644 --- a/changelog.org +++ b/changelog.org @@ -12,7 +12,7 @@ List of user-visible changes that have gone into each release ** 3.10.2 (Unreleased) ** 3.10.1 -- JSON parsing is always strict. See [[file:README.org::*Incrementally%20JSON%20Parsing][README#Incrementally JSON Parsing]]. Requires cheshire > 5.9.0. +- JSON parsing is always strict. See [[file:README.org::*Incrementally%20JSON%20Parsing][README#Incrementally JSON Parsing]]. Requires cheshire >= 5.9.0. ** 3.10.0 - Add trust-manager and key-managers support to the client https://github.com/dakrone/clj-http/pull/469 From 244f3b64aa2cdb623d179f94f4e9f8d2dbc2f1e2 Mon Sep 17 00:00:00 2001 From: fhitchen Date: Fri, 24 Jul 2020 17:59:23 -0500 Subject: [PATCH 038/107] Add ability to override the use of the OS DNS Resolver. (#545) * Added dns-resolver to provide custom dns resolution capability. * Update README.org * Fixed review issues. * Removed unused Java imports. * Added a unknown host exception to prove the dns-resolver does not call the system resolver. --- README.org | 29 +++++++++++++++++++ src/clj_http/conn_mgr.clj | 30 +++++++++++++------ test/clj_http/test/core_test.clj | 49 +++++++++++++++++++++++++++++++- 3 files changed, 98 insertions(+), 10 deletions(-) diff --git a/README.org b/README.org index cfcaf7e4..d6ae74ee 100644 --- a/README.org +++ b/README.org @@ -1437,6 +1437,35 @@ things: In previous versions of =clj-http= (<= 3.10.0), =clj-http= defaulted to lazily parsing JSON, but this was slow and also confused users who didn't expect laziness. +** DNS Resolution + +Users may add their own DNS resolver function to override the default DNS Resolver. This is useful in situations where you are unable to change the name to IP Address mapping. It is analogous to the =--resolve= flag present in =curl=. This example uses =org.apache.http.impl.conn.InMemoryDnsResolver= to resolve =example.com= to IP Address =127.0.0.1=. + +#+BEGIN_SRC clojure +(client/get "https://example.com" {:dns-resolver (doto (InMemoryDnsResolver.) + (.add "example.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))])))}) +#+END_SRC + +This option is supported for all of the connection managers. + +The =dns-resolver= can be any instance of =DnsResolver=. Here is an example of a custom implementation that attempts to look up the hostname in the supplied map and falls back to the default SystemDnsResolver if not found. Note how IPV6 addresses are specified. + +#+BEGIN_SRC clojure +(defn custom-dns-resolver + [host-map] + (let [system-dns-resolver (org.apache.http.impl.conn.SystemDefaultDnsResolver.)] + (reify + org.apache.http.conn.DnsResolver + (^"[Ljava.net.InetAddress;" resolve [this ^String host] + (if-let [address (get host-map host)] + (into-array [(java.net.InetAddress/getByAddress host (byte-array address))]) + (.resolve system-dns-resolver host)))))) + +(client/get "https://example.com" {:dns-resolver (custom-dns-resolver {"example.com" [127 0 0 1] + "www.google.com" [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1]})}) +#+END_SRC + + * Development :PROPERTIES: :CUSTOM_ID: h-65bbf017-2e8b-4c43-824b-24b89cc27a70 diff --git a/src/clj_http/conn_mgr.clj b/src/clj_http/conn_mgr.clj index 0585d1bb..f9f646eb 100644 --- a/src/clj_http/conn_mgr.clj +++ b/src/clj_http/conn_mgr.clj @@ -232,20 +232,28 @@ get-custom-strategy-registry)) (defn ^BasicHttpClientConnectionManager make-regular-conn-manager - [{:keys [keystore trust-store + [{:keys [dns-resolver + keystore trust-store key-managers trust-managers socket-timeout] :as req}] + (let [conn-manager (cond (or key-managers trust-managers) - (BasicHttpClientConnectionManager. (get-managers-scheme-registry req)) + (BasicHttpClientConnectionManager. (get-managers-scheme-registry req) + nil nil + dns-resolver) (or keystore trust-store) - (BasicHttpClientConnectionManager. (get-keystore-scheme-registry req)) + (BasicHttpClientConnectionManager. (get-keystore-scheme-registry req) + nil nil + dns-resolver) (opt req :insecure) (BasicHttpClientConnectionManager. - @insecure-scheme-registry) + @insecure-scheme-registry nil nil + dns-resolver) - :else (BasicHttpClientConnectionManager. @regular-scheme-registry))] + :else (BasicHttpClientConnectionManager. @regular-scheme-registry nil nil + dns-resolver))] (when socket-timeout (.setSocketConfig conn-manager (-> (.getSocketConfig conn-manager) @@ -300,7 +308,8 @@ "Given an timeout and optional insecure? flag, create a PoolingHttpClientConnectionManager with seconds set as the timeout value." - [{:keys [timeout + [{:keys [dns-resolver + timeout keystore trust-store key-managers trust-managers] :as config}] (let [registry (cond @@ -314,7 +323,7 @@ :else @regular-scheme-registry)] (PoolingHttpClientConnectionManager. - registry nil nil nil timeout java.util.concurrent.TimeUnit/SECONDS))) + registry nil nil dns-resolver timeout java.util.concurrent.TimeUnit/SECONDS))) (defn reusable? [conn-mgr] (or (instance? PoolingHttpClientConnectionManager conn-mgr) @@ -342,6 +351,8 @@ :key-managers - KeyManager objects to be used for connection manager :trust-managers - TrustManager objects to be used for connection manager + :dns-resolver - Use a custom DNS resolver instead of the default DNS resolver. + Note that :insecure? and :keystore/:trust-store/:key-managers/:trust-managers options are mutually exclusive Note that :key-managers/:trust-managers have precedence over :keystore/:trust-store options @@ -364,7 +375,8 @@ conn-man)) (defn- ^PoolingNHttpClientConnectionManager make-reusable-async-conn-manager* - [{:keys [timeout keystore trust-store io-config + [{:keys [dns-resolver + timeout keystore trust-store io-config key-managers trust-managers] :as config}] (let [registry (cond (opt config :insecure) @insecure-strategy-registry @@ -382,7 +394,7 @@ ConnectionConfig/DEFAULT)] (future (.execute io-reactor io-event-dispatch)) (proxy [PoolingNHttpClientConnectionManager ReuseableAsyncConnectionManager] - [io-reactor nil registry nil nil timeout + [io-reactor nil registry nil dns-resolver timeout java.util.concurrent.TimeUnit/SECONDS]))) (defn ^PoolingNHttpClientConnectionManager make-reusable-async-conn-manager diff --git a/test/clj_http/test/core_test.clj b/test/clj_http/test/core_test.clj index 84c48250..f551c804 100644 --- a/test/clj_http/test/core_test.clj +++ b/test/clj_http/test/core_test.clj @@ -9,7 +9,7 @@ [clojure.test :refer :all] [ring.adapter.jetty :as ring]) (:import (java.io ByteArrayInputStream) - (java.net SocketTimeoutException) + (java.net InetAddress SocketTimeoutException) (java.util.concurrent TimeoutException TimeUnit) (org.apache.http.params CoreConnectionPNames CoreProtocolPNames) (org.apache.http.message BasicHeader BasicHeaderIterator) @@ -25,6 +25,7 @@ (org.apache.http.impl.client DefaultHttpClient) (org.apache.http.impl.cookie RFC6265CookieSpec RFC6265CookieSpecProvider RFC6265CookieSpecProvider$CompatibilityLevel) + (org.apache.http.impl.conn InMemoryDnsResolver) (org.apache.http.client.params ClientPNames) (org.apache.logging.log4j LogManager) (sun.security.provider.certpath SunCertPathBuilderException))) @@ -172,6 +173,52 @@ (is (= 200 (:status resp))) (is (= "get" (slurp-body resp))))) +(deftest ^:integration dns-resolver + (run-server) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "foo.bar.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))]))) + resp (request {:request-method :get :uri "/get" + :server-name "foo.bar.com" + :dns-resolver custom-dns-resolver})] + (is (= 200 (:status resp))) + (is (= "get" (slurp-body resp))))) + +(deftest ^:integration dns-resolver-unknown-host + (run-server) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "foo.bar.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))])))] + (is (thrown? java.net.UnknownHostException (request {:request-method :get :uri "/get" + :server-name "www.google.com" + :dns-resolver custom-dns-resolver}))))) + +(deftest ^:integration dns-resolver-reusable-connection-manager + (run-server) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "totallynonexistant.google.com" + (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))]))) + cm (conn/make-reuseable-async-conn-manager {:dns-resolver custom-dns-resolver}) + hc (core/build-async-http-client {} cm)] + (client/get "http://totallynonexistant.google.com:18080/json" + {:connection-manager cm + :http-client hc + :as :json + :async true} + (fn [resp] + (is (= 200 (:status resp))) + (is (= {:foo "bar"} (:body resp)))) + (fn [e] (is false (str "failed with " e))))) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "nonexistant.google.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))]))) + cm (conn/make-reusable-conn-manager {:dns-resolver custom-dns-resolver}) + hc (:http-client (client/get "http://nonexistant.google.com:18080/get" + {:connection-manager cm})) + resp (client/get "http://nonexistant.google.com:18080/json" + {:connection-manager cm + :http-client hc + :as :json})] + (is (= 200 (:status resp))) + (is (= {:foo "bar"} (:body resp))))) + (deftest ^:integration save-request-option (run-server) (let [resp (request {:request-method :post From ff7dacb02c4de9095ff4d8647824f88b8bf10bf0 Mon Sep 17 00:00:00 2001 From: Raymond Huang <1694040+rymndhng@users.noreply.github.com> Date: Tue, 28 Jul 2020 22:20:31 -0700 Subject: [PATCH 039/107] Check first byte before gunzip (#549) This is a regression in the :integration suite. The tests have broken since #521. --- src/clj_http/util.clj | 7 ++++++- test/clj_http/test/util_test.clj | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/clj_http/util.clj b/src/clj_http/util.clj index 201e7391..0528703b 100644 --- a/src/clj_http/util.clj +++ b/src/clj_http/util.clj @@ -44,7 +44,12 @@ (when b (cond (instance? InputStream b) - (GZIPInputStream. b) + (let [^PushbackInputStream b (PushbackInputStream. b) + first-byte (int (try (.read b) (catch EOFException _ -1)))] + (case first-byte + -1 b + (do (.unread b first-byte) + (GZIPInputStream. b)))) :else (IOUtils/toByteArray (GZIPInputStream. (ByteArrayInputStream. b)))))) diff --git a/test/clj_http/test/util_test.clj b/test/clj_http/test/util_test.clj index 781a9541..249ddcef 100644 --- a/test/clj_http/test/util_test.clj +++ b/test/clj_http/test/util_test.clj @@ -53,3 +53,12 @@ ;; coerce to seq to force byte-by-byte comparison (is (= (seq (IOUtils/toByteArray (io/input-stream jpg-path))) (seq (force-byte-array (io/input-stream jpg-path)))))))) + +(deftest test-gunzip + (testing "with input streams" + (testing "with empty stream, does not apply gunzip stream" + (is (= "" (slurp (gunzip (force-stream (byte-array 0))))))) + (testing "with non-empty stream, gunzip decompresses data" + (let [data "hello world"] + (is (= data + (slurp (gunzip (force-stream (gzip (.getBytes data))))))))))) From 072c4f9dc0e603d5ae73d8277947a7b8afbc1c01 Mon Sep 17 00:00:00 2001 From: Raymond Huang <1694040+rymndhng@users.noreply.github.com> Date: Tue, 28 Jul 2020 22:27:53 -0700 Subject: [PATCH 040/107] Migrate CI to Github Actions (#550) The motivation for moving to Github Actions is to be able to run the integration suite. * Migrate to Github Actions * Fix conn-mgr tests using Mutual TLS on Java 11 The certificates in keystore & client-keystore no longer work under Java 11. They are regenerated with this sequence of commands: ``` # Step 1: Create server keystore with private key keytool -genkeypair -keyalg RSA -keysize 2048 -alias server -dname "CN=Hakan,OU=Amsterdam,O=Thunderberry,C=NL" -ext "SAN:c=DNS:localhost,IP:127.0.0.1" -validity 3650 -keystore keystore -storepass keykey -keypass keykey # Step 2: Export server certificate from keystore keytool -exportcert -keystore keystore -storepass keykey -alias server -rfc -file server.cer # Step 3: Create client keystore and import server certificate keytool -importcert -keystore client-keystore -file server.cer -alias server-storepass keykey # Step 4: Create client private key and store in client-keystore keytool -genkeypair -keystore client-keystore -keyalg RSA -keysize 2048 -alias client -dname "CN=Suleyman,OU=Altindag,O=Altindag,C=NL" -validity 3650 -storepass keykey -keypass keykey # Step 5: Export client certificate from client-keystore keytool -exportcert -keystore client-keystore -storepass keykey -alias client-rfc -file client.cer # Step 6: Import client certificate into server keystore keytool -keystore keystore -importcert -file client.cer -alias client -storepass keykey ``` --- .github/workflows/clojure.yml | 36 +++++++++++++++++++++++++++++++++ .travis.yml | 13 ------------ test-resources/client-keystore | Bin 1919 -> 3463 bytes test-resources/keystore | Bin 2218 -> 3463 bytes 4 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/clojure.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/clojure.yml b/.github/workflows/clojure.yml new file mode 100644 index 00000000..865c5b04 --- /dev/null +++ b/.github/workflows/clojure.yml @@ -0,0 +1,36 @@ +name: Clojure CI + +on: + push: + branches: [ 3.x ] + pull_request: + branches: [ 3.x ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + java: ["8", "11", "14"] + clojure: ["1.6", "1.7", "1.8", "1.9", "1.10"] + + name: Java ${{ matrix.java }} Clojure ${{ matrix.clojure }} + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-lein-${{ hashFiles('**/project.clj') }} + restore-keys: | + ${{ runner.os }}-lein- + + - name: Setup java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Install dependencies + run: lein deps + + - name: Run tests + run: lein with-profile dev,${{matrix.clojure}} test :all diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index fd2e94e1..00000000 --- a/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ -language: clojure -lein: lein -script: lein all do clean, test -branches: - only: - - 3.x - # 'master' is the new apache 5 client, and not passing yet - # - master -jdk: - - openjdk8 - - openjdk9 - - openjdk10 - - openjdk11 diff --git a/test-resources/client-keystore b/test-resources/client-keystore index d30af3db6bf8872c3e6777b065a5114113e60885..55bdc0c42df8e0ae1f5fa7c247558e5b62f5e9dd 100644 GIT binary patch literal 3463 zcmY+EWmFRm+lL2?jSw72N;fDmq>=Cs5)vb%VMxeuq?91&gaOi0(zVe@OG=3Jq`L$` zVuW-f10LV^yzldz=fi#ObFSa-zCT`2G*vJ$5eXDcr2rzo_g?e;85I$j2!o~)0ivmd z{$f5TniTn85vc_bO=|KN8~z;_i1PogZcz{sW6;1$C>nSM6#`NGKmPli1xP0!Ct{$B zg4>&s>0x>t+%hw!^bLtg0G;|kG!Ul?&|Jv6PV%D`FUq;{;mmTYu*|&zu(o_Qj1H3Ar$O{hp4++yma1#cqqem6H9Rk} z%GkdK6=@<^{<&%lB65(TkDrMrRin^}99mE_?rMhR=jnvW7AY{=nW*mxK-`ckNLqPVtfXT^e@O}fjv&Lpi z+}x*5{c3qpxC)L8-ekt|SL_ogozukZ@@|*0g4$=CJrTd@uGzf@t2H^6A3BV!G@Xu3 z98NOE8-@X+2@nUs)A69f8-~T)^CfFuP}^QI$m}2POjF?)RtYqf|s7QkRlhy)<6?{=&(+Fw? z%T6OnsxvU*1c{jr7K-5A4Q)QQYr(+B+dnwRIk(tD_xbRFb>Hg~dI6Lc{Y=%i@iTH2 z@Ctxm$w(UY)l{B$)U0pS;`FPOgdEfI zMk84(b?Xl2yC*Onj8JFZK?|7!kzK~gH8QG|g&aF0-zjDc?0)*sji2FzzrPw^Ja4`A zkwaY8`>lrE>_6?!SdPyXnQ3zAc|!B1j_o@K@O-bju3X!Iuvb-s3C{1Ff|mJ2?klww z|I)i8YnB0@@#~C?Kuj{S$H-=H7P?cIvJW?X${$E4JJTjeY}qc;r*pfK3=Nz#{>}y_ z@mT(uXw|LO`*4;euFcAI#$w%G0w&(%QaKh$)Mirw_5xBlb+OVL)uQ&E)KMJaeldgc zEpj-81?lYmqEc$??oF1-oL}0LekM~`~I2WoJ5BCtF3OYs%=dnW#!vkG>T@r(!(&P`NHSw8kR!)nl(>Oiz1;4ktjdVzCJ~Dj5;$bo*fCi>aW3 zJ-YtQ{Nttx1B0VS8#IJVr4WAz;DGOVlaUO}Kf~kRx*y-E*lhQN z70n&s&*z`SPbY&AK9qf(;c%W^s@2O|4o`RD9TKJ*9neexl+p_26Hi~=Y- z)S190p6U@>g3LNEv{+;HvZzC8)~}K|HNLmqQ?r8WDrG(L4`m8FQAK3p?e1bqmMFy^ zL0SH-N^&}JYW7f732*?o0lcAH{|V{E>43CGNLL3=aVa?|DH$mVX=y1bNhlgz z`|lDMgh7J~{vs?f5#aAi|4%~nUxo$!k702+jnrZLccPbt;~zH_q>!yCdy)LFVFS@% z8`2T1Kqc8L*e-c-Eej^eA|h3b0bwL~n*yQObW>42UdDmc{!u_yS)bq33QK4wd(^|^ z=masi7&BL^?KAmV7l{iS(*3G$XVw<%`-(UCW(y*#mm*lLVjc_mEc0A8QudDWlilOk z)@ei+AE2eCmVRz!PuCvslUDwJzT z!CDkOk2|R6q1PuVokpjRsS-tU@lBN?gC6uV`xj0RCvTO5SeT!j|Q0|@J_XWC ze9^X0`C2a-H+(FHpEpwI+wqx6C%>S>7JKf0dR7Ws#2=Eq+-b6<~MasBM!R#(ZF6Yf%MRk>Yd{6rR! zTC`B&^pZv@@lhYLfx{XGY+UqI2d}|)Y3F(A4;7IAL-6a|^3SQyFy;gMhq@e>c(~+H z22I+r-e(DVvslD8YeZ6PgZQYdEyoJG-~hzR^9A+~z+geaFNfGVL@LZ@w7jIkjPa&@ zoW~Li-IeJzq|jIf^l8gppGOC)z?w^lS55)HzC4=~pNI-3~FEa2h4qA1=-nK5sA;ZuSQC9jxUTw5jDMex<9&F=w;bo^34H(6zCT!P!q) z{;`g>q5ayW*k^qQ%DUUP@0-a{@ek&q}9+b0U;%Wgc6 zE<5N&@Zq4EWZERjsp$`K-ja@->Z+v=nl;D-#zZ7azia4hbJzDCQlp! zvb!iN7|c?VoFJC(_5u>^PwQzHv678AVt_!p&RtHF%g~T_Vy}LT;GMRquYWru%j>ef0n(M2d{WMup zxaAEq_}y~t>;ARij|U34nR>5pRaN~V^5b`hpTR9rYVJ?#=7eC=0+LE0B)fg0i(mzr z42t5oD;MRL@=JJId*@l4t6$X5f=o+cX?5=+qe@G{#R`!xu2;2u;pW8*tvL}yaDzc( zk-ve(rK+x7L72z6jYO{F7aA;*!ce}wai1&NZN7GebV;R9EdQW%{6o|vPA67Y8=@of zQ%80hD&?=M8LFm6$QrDVdHSJh$I*%2)b0V)_wGsajO@hKoKt>l=J09&$6^g7L$K>3 z$*fo?a3ak{Vw?z(Y zJYX~}YP;yr%Ih8V+`qolvb5gazj->TIt?|}OFYLsE3O?qJUrUToGJ(2%j!l3uUf&6 zbEn&8%2T`V>|dteYPHX5Rt+KtC8_VlMO*?0We|-$4KLRVTxzcy=V=+*CYRNYUn2X= z&PNo>%L%=a&iMmcvl4cQq|THd*&Mu(fwlxANJ9$KvTjt=IGbSLuCByBCV&T3av7yl z2;?V?vPKNjP~Knd=Qog7q2Oo6YtGv=dzukNuljVa??7I+yT39=QZeK*_Qzz0=giEZ za63j^s%ISvj7`8zV}he7n?3h(cIvtLfDuOp-i@iCWqxa3|DEP@X!;c6Ug?&0SpDb5 zOH$^JD}ok_Ivp#jVNbEUCeMQtBFScWycnlw@tbk;;2*g0afqyNs`c1SdKHa7J zh%m!N4CKUl4Gj$qj4TWcjEqbSqkvo^14AP-D3{KzX`GK78jP$A%#FPa292Fejg1Ub zdpeBc)r}Xe&9>Y4O}uz!AMd(Ybxz)|w)@zYm4r-`SF5Nj*j;;d_D0WHxm~f(WqInq z{7aKOy7tq7We0s_LM2+5RSZ(ZuWc~jBlf0MSEb8OwYk{aQIggF!OEMQ$6F4B?~#bK z4Xy6`%crZ$6`=GYD{kekyBo_nBK*EK=RZr~XJTe#U_^EvFtV6|?n=p!b_;U2ebde} zw5jVylZ$L+_4$Z69ap}F-nftC|pZkO<(h8fI4kTUsVWj0|kaIhm<>CEyg-c3lZb_XCqzH!vY_X|pl1 zFlsRg0#g?QOA}*xoEP&^k==80Sr}G5N&HIk7UpM&Ar>q_3^7s({jE) zdvfO;{##oWc8$e)X7H*e-KVJu=XQLW{>aJQc;Uel+_i_9t6tPx>-4$CC2cSFTIz_* z&BSLm3>>Sc+1LnuTA?`Shn)J^-wELx!X75Q*2roL?N!rG|63gEE2^F$F8S5F=%`hU zbf);;l?%ek_PujlR(CD=)pj3?sXf7|>RnrN?S$`EZTPuwVpX4YerLbr;!Wr3RI8k? zonOWJ@72DCI?Ctz0u!!SZ7f@O$6Yn+`?E6)toOg@KH0wBEqi@``rPTEcednB6MEUQ z?&>^QtH|?ZRmz-Z@h{gz?=}BlzT=C&jM4SKIx>nUduyz#Z=YF}yx1oxTv+PKgGs?G zQzu6p{b(WTaAU=@TUS?n>skKo*n-Dt@80m*-+cHuX_F9(8&_lvSKBUdG{Fi2W($KR zW)omx@CjNF1Sjqhq{7-2K+p}i|EL}2x{tK3B0i{F!yH{TSxBZy(59{#f z$5t#8wMkM+Ovo?6U}EeuSJ z4e`Q8Av+8ASRB0ZX?f|coYT{uN&CoMDx7b~XV=NDKjZU(4OP{5jT+uEohtS?a8M_* zMzV?T)Sua^TmEyOT6E5D|3yB|$MWSo3DT=4Fn`!`dqs&Y@1F}7#2agp48%9>H9!1W z@rb}z-O47tJsBJH*mtpQi+=U5;gi&tqf5CHB0oLtU94yzZy*b-y=3`V#8^aX%%@m- z#J)Cuz$wWa>O23}vQzi}pd@og=#MVa17By diff --git a/test-resources/keystore b/test-resources/keystore index 13d78bf94806e8e2802e9f83e4f97f2236c96817..2944ca21c6d5bbbe54c991353ed18cedb237520b 100644 GIT binary patch literal 3463 zcmY+GcQhLg*T)k{#7xcD)LuoR6s1;;XpPpUR?JcqX@zPlqEv0QS8CLzN>O_?trb*l zQVFe9d)2Q-`}TRy`##Tk{;|Ca@^2jd~ue`WK(VM@#Re^X2hK$-$P_!NN$A0wn`8UA1XyPOZq z@kqRFxYZrD2&YFr&!3C`t*3QYmj(p*a0`qF_Z#Lo>&X=f*E5u426wW>rsbncj*m1e z)7`j-JK}rbiGjn!jR*Pxbo|X{#q&!rPyLYUt&|rbbT?s7E>#&h*0FXK)V_}Q<|q@x z7UZ}*nZdBW_9czpy<7NLby3?LGDS98P<7lHldqiC^nzOPw=*3b8iyBa-m%5RcbCj8 zdOc=w0;*q6VBt)Iz%;lod?IQ1YNY%aL27xOmc}3V%Me+C zGh(_Y!oU9!MhpUDA7b{vG7%Zk6ez5F0-GSP2kMpQCGl^bcLNKZ$zCwv-JRQYumTcY5AF3;Up~^9qfGKO%sK zS*)ci>nV%saSUG_Bo9>Qq;q*sg?KTTp)QfBf-=dl-XLe^VF=;4{49SPYw6@sq3am- z;OZKr&z^UwM*VgIa!_OEpS286!}fL+1)H+e?sbFH>b_Ua4cgZnKOO2}MVjcXd^l_< z(~|M^!p}E|+mlTR3Xc{=dYxF^OfblYQeiIN7iIdiUUn@Utc`Xb-!%CntTI6Q`9LvT z+G8sL+oW)V=3UEHb?iR5aKGA^{%|@&m!d;fX8L}PzTjA|DYx-6j1)~fdrYJf{$HYxl zS|jenvq8th13#zr81`GZe|{WNuzJ_;+tXPOaxV5S+TZf8qN_0chlWHj{#q+?+b^%K z-q)~qH`+^LXr%;e$@7?W>sv?pPi~ak;Xib~()S1jg=^9Cnl3fdKEOmSrxcB^H<&MK zDpd?E(3goDJELzBOjez?5`kJW`HxDcts3Iu%FJf;8{~zW6z8_?hL-TyHS>=X#p7+4 zgE`)Rlx?a_T4LPFJqiO`9HhuQ;cm7zatLH#7&Gzs&aG|8#l+V0bFKHa{DOHCpyXV7 z0>-{;g+y39(7|9y2RPZHKcfwWiO0$PTT0kMboY?)rvtQ(;!_s zdcSk*klF~of1{F~0|^sk0$>0hfFMBdzcNDjKPD%V1I%XW?dNe3sjQ}~tfGumQc_h! zBJj|Le>XvC3-HjAzfvI$5b(F;{Ko0}nBIR}BA` zusA%_#jDCid`Z}7+JsuiVGss>NNWCZ>D%e>G611sf~a=-^z3a7HLPKKcO6&C%Vpk3 zia8SUu3aOL3J8X!tsSIs`#Nt~ouR*4){`*N1Op9f<~#S0bh{ag{A3p8K~dDKZ0HAJ6r;_S#LE zU%YMn-ibb9!lYDGEgJILDE9WZKwrM6WLYd7Ntx@%Qq2EUTh}6iAsOf<+k=}O z4&&){To}0GqT7S4msdM}g(~;XE2Lb>5}YghogetKuk^xwmcSJL^EW!mqf((|oS%iS z2n46b!eW(fiOIXEbg+Gow9CqNpmPltH(i~Ix|TMif61@-sgrYyT7#tN*%9epxdhIY zWdwKwWd%u<%dba~lU2bJ8y}Y7zV0(o^$v^IRSbtRDQq&~j0f&g0Vf7`msqUn09+(@ zKrcXA8a`&+7CuORIU5pB>#%JYnM&yq5lUYTloby8dmRIlD*?W)hjHG&B!xW}-bf9v zVhOnw7@^g@ys$6Z@sC$1-|V}{k*z{6MZFJ0Z|O5>o9=naJ#zyhYh2B82JL#E(TK~`ek{37XukxalYg|_j!(ecz+>7UkNEsR09g$ zHpA$hM9vhmh1*o+{?iykB6cfK{rIvmiu_hyC6@$=8@A+}P1S6>XXPp?%ALe(r6A&g z>SIf>&5sj>Fw3sy=L3?HhM?lp@Yh#8OuK~D(N}qdDs!&5b6OGZ!$k+^ygm-1YULuA z-A#U{YPyH#l$VqqQ9|{?+SdJ&PMn2LMu+VxGt9#cA8i(@3t8ts@xi z^F^o4HNdjn_Gy@$1;pZp?!BT1tcp)5(fH`8UoA{RdTT4*BBO$^D3M z$~>zttqWy+b59h!KPM$!ariieHtutXL*!i?pMskspHtUqccuN3VWCcd@%kf>SF|hVjEM! zfcw*pQ4maO|K@|;6Wvo_ZZBTIkM$nf+0})WU&N$rJEhJbM-aTTWW_{mEretE{YGgh zdI4F54>^p;BDL_#Ip~+2n+5G@X|AnW3UU5LKg=pQK4dNE!F9jD78jG-U5G{m%-%&S z0^^p;y8s0zJ(^6<#Uw7+z30JefV#FmW>YmSl9OglYf+cCg`lJwO(v(^pacuGNw;gW{60v9sVd^~n(7hKyNsy7HTX za&N44Xgnh?&zPT3g@9wLj~7?*;1#U-p88(j8}B?=E`&Oh@Wnh^Uw47*tZC zQDRhLDS49e>*Fj?swG(Xvg)TUlf|+UgKywn5Bb1}@}IWV#psP?(C7s{^;icb^1lpX(( zZYQDKogj31i38e|)SX#34u3Dx#uA_!Owai>$|%OCaeekma7CSKWtOeyfQNmYNmbe6Oyy8hEti6_sVoh-CwD2IAS4_hYkD zgpPY7GBFDVvFvnwsfRjxu|FQ?q^hYxqoqN##9d>n;^olq_#vtD*$85^?@}5oI4*#C zpmB*``P|!7&GEA;#Q2ofBmKL%BMn)$r;KxBAnq;M?3Dj=kd~-=4TVRpp*{kW7QdcHeS2PPvWOLiq8w)WLpzE{8sQ(s gfjP3o?_3rkb37XPZF+}}HJFbog?dW^0!A1759c;}7XSbN literal 2218 zcmc&#c{J2}AO6j6##p8qN?BvFRb=_i*anSqLngbLJIDyp!Wet9#LS=*vc9B9Q?|I| zjz;!St|er-SrS<)%ZV1_5@oq_@9W-k-_!s1{o{Mi=Xw5m&gc6)pJ(&K<_7=(z`q~z zEbXEfJs1FB`t8?Z9RUGHOTAm9Q3 zM8kEZ~wIOjj1 z0i?SP|7A5SST)3gssFVO7$0CkCKK3*KFO2|*`?|*apJb-!n@g(oz)J9KbZn8@<$j| z_WLiIQI9Ixe{&)xIeu+wU9N*xx^l?0VWts7PREaT&aK+#mR(^1n>~k!5L0|=TVFD~ zS=}bPYDAfBD=yk2iue|@@}gC~1=Kwy?ktgpUkO<%Kjc2F_KBwmm(K0CfVG2rfGjt= zls!tQ?uY5Mti1VH)Mv-N%YR7wGIcj@D2F)S)lll9S3{`{%lNtetKoar8pR4l6l9&Z zccLWM4LI4(1PF>T8I>f70HryrgA(G^y~6nG9Ys@zCBhR=MJ64b%rBQe#4QR*qvuDX zzAET?EPnF{hbbF%7c4dw+o#4C$4pE)PqUHfiRWAy5o8rln3d6HFG&r~j0Hd@5;mJ$h{g66Z*NqujEBRjo*f@-Fyr9<2rDjfe#A`g`1K1g4(fmZ+DN>q+>0HlI zsoX+!=JnV#^<#wJVncVh$(E+VKUaogxxq|e;78XE zezahKfVWq0a2S8mK0ar;!4F&=KT1{nxQHA=!VoYua3>5V03cXkUh2t}OJ&I?8+&f3 z4&3v-TFJJ~)ppElvf9wXGmS6i%YZP zebu%dkL%(0)e?AFFxu*y7xR6q?no)cgD+)r3a`AK*A`sRY0@D>9xpe!H z9sB&u9z5Mt)oTq^JFK~3dNz&F5o7F$N?a9NxB`V_(=Mu}Pt<13C_7zzZ8Kt$Y>`DY zahNif5nmKJZD5j{;O>*?R9^VXd`A*xdRD&k;}4|3Vcq-}yWHri1)wi@NmP5I-!Snu zSvIRo05oGf?+wjM7%!oQh??8gGz7o9<)>l|HqsrR)5u+|iK=A_{-s{;O*u!{Tn(}F zqHhpK0zOB&ZTiT^)^c6YVx{4#_w(*1dQ2_0E!l;jc1HBXUzaEOIPbM#-zF7GN%ray zn%<^(5G)S*?qZH4BoiraN$Le#fc`L{B(8E$_~t&B9-HXR@wR@$_=aIY+krCd{IT_P z%pQ09SoG}7VCV(Om|)dmSM6U$L(C$gnhDXLx%5tV%kwGRn^E}x8hX1Iz>|C<+w#v{ z&2}%?lq~l{XrwT|7p{Im^IiRaX$Bk)!V@3}kJs651{_{*8~)49z-0e(8~l*NnMnY| zjQ`^VXI{Uiedd66-UA=wiWSMATgm8h>H%T&GO-a^6nrWTyZ>^SfBlugoC;F5Zvu5z zZb$TQo9CokAMl!so>=cCOU1&KAy3I6j(&s0LY#(jf{jW{kfn*VfbC?-h;VysGo?Yw ziAat}-oR*Rh}bF4dApX>y{-rqarCN_D^~I&w}%Hd0vn;cQ`saVoV~l z5>{Dz Date: Tue, 4 Aug 2020 20:40:35 -0700 Subject: [PATCH 041/107] Check for reflection warnings in CI build (#547) * Fix reflection warning * Migrate to log4j2 to address warnings * Check for reflection warnings in CI --- .github/workflows/clojure.yml | 3 +++ project.clj | 2 +- test/log4j.properties | 26 -------------------------- test/log4j2.properties | 19 +++++++++++++++++++ 4 files changed, 23 insertions(+), 27 deletions(-) delete mode 100755 test/log4j.properties create mode 100755 test/log4j2.properties diff --git a/.github/workflows/clojure.yml b/.github/workflows/clojure.yml index 865c5b04..f2a6e92d 100644 --- a/.github/workflows/clojure.yml +++ b/.github/workflows/clojure.yml @@ -34,3 +34,6 @@ jobs: - name: Run tests run: lein with-profile dev,${{matrix.clojure}} test :all + + - name: Check Reflection Warnings + run: '! lein with-profile dev,${{matrix.clojure}} check 2>&1 | egrep "Reflection warning|Performance warning"' diff --git a/project.clj b/project.clj index 4d549b8c..81daeed6 100644 --- a/project.clj +++ b/project.clj @@ -23,7 +23,7 @@ [org.jsoup/jsoup "1.11.3"] [org.clojure/tools.reader "1.3.2"] [com.cognitect/transit-clj "0.8.313"] - [ring/ring-codec "1.1.1"] + [ring/ring-codec "1.1.2"] ;; other (testing) deps [org.clojure/clojure "1.10.0"] [org.clojure/tools.logging "0.4.0"] diff --git a/test/log4j.properties b/test/log4j.properties deleted file mode 100755 index de3a2906..00000000 --- a/test/log4j.properties +++ /dev/null @@ -1,26 +0,0 @@ -############# -# Appenders # -############# - -# standard out appender -log4j.appender.C = org.apache.log4j.ConsoleAppender -log4j.appender.C.layout = org.apache.log4j.PatternLayout -log4j.appender.C.layout.ConversionPattern = %d | ES | %-5p | [%t] | %c | %m%n - -# daily rolling file appender -log4j.appender.F = org.apache.log4j.FileAppender -log4j.appender.F.File = http.log -log4j.appender.F.Append = true -log4j.appender.F.layout = org.apache.log4j.PatternLayout -log4j.appender.F.layout.ConversionPattern = %d | CLJ-HTTP | %-5p | [%t] | %c | %m%n - -########### -# Loggers # -########### - -# default -log4j.rootLogger = DEBUG, F - -# Things -log4j.logger.org.apache.http = DEBUG -log4j.logger.org.apache.http.wire = INFO diff --git a/test/log4j2.properties b/test/log4j2.properties new file mode 100755 index 00000000..56f6f014 --- /dev/null +++ b/test/log4j2.properties @@ -0,0 +1,19 @@ +status = error +dest = err +name = PropertiesConfig + +filter.threshold.type = ThresholdFilter +filter.threshold.level = debug + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d | %-5p | [%t] | %c | %m%n + +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT + +# Set this to debug to log all data to/from server +# See https://hc.apache.org/httpcomponents-client-4.5.x/logging.html +logger.wire.name = org.apache.http.wire +logger.wire.level = info \ No newline at end of file From cd61c48358cc9aa3723fcb0031cf4a12bd7416a9 Mon Sep 17 00:00:00 2001 From: Raymond Huang <1694040+rymndhng@users.noreply.github.com> Date: Tue, 4 Aug 2020 20:40:58 -0700 Subject: [PATCH 042/107] Prepare 3.10.2 release notes (#551) --- changelog.org | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/changelog.org b/changelog.org index 56b3bf16..eb0070ec 100644 --- a/changelog.org +++ b/changelog.org @@ -10,7 +10,18 @@ * Changelog List of user-visible changes that have gone into each release -** 3.10.2 (Unreleased) +** 3.10.3 (Unreleased) +** 3.10.2 +- Fix performance regressions from #528 + https://github.com/dakrone/clj-http/pull/546 +- Adds support for custom DNS Resolvers + https://github.com/dakrone/clj-http/pull/545 +- Buffer :debug output to improve readability + https://github.com/dakrone/clj-http/pull/544 +- Improve compatbility with GraalVM + https://github.com/dakrone/clj-http/pull/543 +- Bug fix: Check first byte before wrapping response stream with gunzip + https://github.com/dakrone/clj-http/pull/549 ** 3.10.1 - JSON parsing is always strict. See [[file:README.org::*Incrementally%20JSON%20Parsing][README#Incrementally JSON Parsing]]. Requires cheshire >= 5.9.0. ** 3.10.0 From 76964bccb628c9e43cbc1e31d5c4dfa27e0415bb Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Tue, 4 Aug 2020 20:47:25 -0700 Subject: [PATCH 043/107] Version 3.10.2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 81daeed6..122e6a37 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.10.2-SNAPSHOT" +(defproject clj-http "3.10.2" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From aee13b4f544e83e4eb6014d6d1f9178676899f13 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Tue, 4 Aug 2020 20:52:14 -0700 Subject: [PATCH 044/107] Version 3.10.3-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 122e6a37..8e3b0437 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.10.2" +(defproject clj-http "3.10.3-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 76130e65aa90df3eb9cf68a5ea5e8d8b2fb2d2e8 Mon Sep 17 00:00:00 2001 From: Oleksii Kachaiev Date: Mon, 17 Aug 2020 16:24:31 -0700 Subject: [PATCH 045/107] Properly handle "308 Permanent Redirect" status code (#554) Based on tools.ietf.org/html/rfc7538#section-3, "308 Permanent Redirect" status code is a permanent counterpart of status code 307 thus should be handled similarly. --- README.org | 2 +- src/clj_http/client.clj | 4 ++-- test/clj_http/test/client_test.clj | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.org b/README.org index d6ae74ee..ec1ad07b 100644 --- a/README.org +++ b/README.org @@ -704,7 +704,7 @@ APIs: clj-http conforms its behaviour regarding automatic redirects to the [[https://tools.ietf.org/html/rfc2616#section-10.3][RFC]]. -It means that redirects on status =301=, =302= and =307= are not redirected on +It means that redirects on status =301=, =302=, =307= and =308= are not redirected on methods other than =GET= and =HEAD=. If you want a behaviour closer to what most browser have, you can set =:redirect-strategy= to =:lax= in your request to have automatic redirection work on all methods by transforming the method of the diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index 4fdfb365..937b3914 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -196,7 +196,7 @@ ;; Statuses for which clj-http will not throw an exception (def unexceptional-status? - #{200 201 202 203 204 205 206 207 300 301 302 303 304 307}) + #{200 201 202 203 204 205 206 207 300 301 302 303 304 307 308}) (defn unexceptional-status-for-request? [req status] @@ -334,7 +334,7 @@ resp-r) :else (respond* resp-r req)) - (= 307 status) + (#{307 308} status) (if (or (#{:get :head} request-method) (opt req :force-redirects)) (follow-redirect client (assoc req :redirects-count diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index c15d4146..0b114fa3 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -457,7 +457,7 @@ (deftest pass-on-non-redirectable-methods (doseq [method [:put :post :delete] - status [301 302 307]] + status [301 302 307 308]] (let [client (fn [req] {:status status :body (:body req) :headers {"location" "http://example.com/bat"}}) r-client (client/wrap-redirects client) @@ -470,7 +470,7 @@ (deftest pass-on-non-redirectable-methods-async (doseq [method [:put :post :delete] - status [301 302 307]] + status [301 302 307 308]] (let [client (fn [req respond raise] (respond {:status status :body (:body req) :headers {"location" "http://example.com/bat"}})) From 9ddf5903e63ef6289d718448990f59da4126fbe8 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 9 Sep 2020 00:16:43 +0200 Subject: [PATCH 046/107] Update installation instructions to latest version (#557) --- README.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.org b/README.org index ec1ad07b..570a09c3 100644 --- a/README.org +++ b/README.org @@ -122,7 +122,7 @@ function. With Leiningen/Boot: #+BEGIN_SRC clojure -[clj-http "3.10.1"] +[clj-http "3.10.2"] #+END_SRC If you need an older version, a 2.x release is also available. From c48b93d70ea04a44e4860ffa63b2ee9597566b12 Mon Sep 17 00:00:00 2001 From: Raymond Huang <1694040+rymndhng@users.noreply.github.com> Date: Tue, 15 Sep 2020 10:21:56 -0700 Subject: [PATCH 047/107] Improve error message when using incompatible version of cheshire (< 5.9.0) (#558) The json-decode change in `v3.10.1` requires cheshire >= 5.9.0. When users do not upgrade cheshire, they will see a very unhelpful error similar to this. Syntax error (NullPointerException) compiling at (...) This change introduces a slightly more helpful error message to point users at how to resolve this issue. Fixes #555 --- changelog.org | 4 +++- src/clj_http/client.clj | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/changelog.org b/changelog.org index eb0070ec..50df5918 100644 --- a/changelog.org +++ b/changelog.org @@ -23,7 +23,9 @@ List of user-visible changes that have gone into each release - Bug fix: Check first byte before wrapping response stream with gunzip https://github.com/dakrone/clj-http/pull/549 ** 3.10.1 -- JSON parsing is always strict. See [[file:README.org::*Incrementally%20JSON%20Parsing][README#Incrementally JSON Parsing]]. Requires cheshire >= 5.9.0. +- JSON parsing is always strict. See [[file:README.org::*Incrementally%20JSON%20Parsing][README#Incrementally JSON Parsing]]. This is + a *breaking change* and users *must* upgrade to cheshire >= 5.9.0. + https://github.com/dakrone/clj-http/pull/507 ** 3.10.0 - Add trust-manager and key-managers support to the client https://github.com/dakrone/clj-http/pull/469 diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index 937b3914..9220688a 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -128,7 +128,10 @@ "Resolve and apply cheshire's json decoding dynamically." [& args] {:pre [json-enabled?]} - (apply (ns-resolve (symbol "cheshire.core") (symbol "parse-stream-strict")) args)) + (if-let [json-decode-fn (ns-resolve (symbol "cheshire.core") (symbol "parse-stream-strict"))] + (apply json-decode-fn args) + (throw + (IllegalStateException. "Missing #'cheshire.core/parse-stream-strict. Ensure the version of `cheshire` is >= 5.9.0")))) (defn ^:dynamic form-decode "Resolve and apply ring-codec's form decoding dynamically." From e1a84de92ce3a7132f26a90e8652693e07170173 Mon Sep 17 00:00:00 2001 From: Raymond Huang <1694040+rymndhng@users.noreply.github.com> Date: Fri, 18 Sep 2020 10:32:35 -0700 Subject: [PATCH 048/107] Prepare release notes for 3.10.3 (#559) --- README.org | 11 ++++------- changelog.org | 8 ++++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/README.org b/README.org index 570a09c3..b8b041a6 100644 --- a/README.org +++ b/README.org @@ -7,16 +7,12 @@ #+HTML_HEAD: #+LANGUAGE: en +[[https://clojars.org/clj-http][file:https://img.shields.io/clojars/v/clj-http.svg]] [[https://github.com/dakrone/clj-http/actions?query=workflow%3A%22Clojure+CI%22][file:https://github.com/dakrone/clj-http/workflows/Clojure%20CI/badge.svg]] [[https://gitter.im/clj-http/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge][file:https://badges.gitter.im/clj-http/Lobby.svg]] + * Table of Contents :TOC: :PROPERTIES: :CUSTOM_ID: h-aaf075ea-2f0e-4a45-871a-0f89c838fb4b :END: - -[[https://secure.travis-ci.org/dakrone/clj-http.png]] - -#+ATTR_HTML: title="Join the chat at https://gitter.im/clj-http/Lobby" -[[https://gitter.im/clj-http/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge][file:https://badges.gitter.im/clj-http/Lobby.svg]] - - [[#branches][Branches]] - [[#introduction][Introduction]] - [[#overview][Overview]] @@ -61,6 +57,7 @@ - [[#custom-middleware][Custom Middleware]] - [[#modifying-apache-specific-features-of-the-httpclientbuilder-and-httpasyncclientbuilder][Modifying Apache-specific features of the =HttpClientBuilder= and =HttpAsyncClientBuilder=]] - [[#incrementally-json-parsing][Incrementally JSON Parsing]] + - [[#dns-resolution][DNS Resolution]] - [[#development][Development]] - [[#faking-responses][Faking Responses]] - [[#optional-dependencies][Optional Dependencies]] @@ -122,7 +119,7 @@ function. With Leiningen/Boot: #+BEGIN_SRC clojure -[clj-http "3.10.2"] +[clj-http "3.10.3"] #+END_SRC If you need an older version, a 2.x release is also available. diff --git a/changelog.org b/changelog.org index 50df5918..0cfa7aa4 100644 --- a/changelog.org +++ b/changelog.org @@ -9,8 +9,12 @@ * Changelog List of user-visible changes that have gone into each release - -** 3.10.3 (Unreleased) +** 3.10.4 (Unreleased) +** 3.10.3 +- Improve error message when using incompatible version of cheshire + https://github.com/dakrone/clj-http/pull/558 +- Properly handle "308 Permanent Redirect" status code + https://github.com/dakrone/clj-http/pull/554 ** 3.10.2 - Fix performance regressions from #528 https://github.com/dakrone/clj-http/pull/546 From efaac4932d98369de43525724b1c7b7e9531c773 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Fri, 18 Sep 2020 10:38:09 -0700 Subject: [PATCH 049/107] Version 3.10.3 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 8e3b0437..af3931c4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.10.3-SNAPSHOT" +(defproject clj-http "3.10.3" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From ca5e9a9c8780fd6754b2429bc9d3603e2a33e95d Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Fri, 18 Sep 2020 10:38:24 -0700 Subject: [PATCH 050/107] Version 3.10.4-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index af3931c4..c8fe6660 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.10.3" +(defproject clj-http "3.10.4-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From d8ac232638d5e4014e3e6aa4dd013b7b098bf2cc Mon Sep 17 00:00:00 2001 From: Gleb Posobin Date: Tue, 10 Nov 2020 19:18:19 -0500 Subject: [PATCH 051/107] Handle quoted parameter values in content type (#573) --- src/clj_http/util.clj | 8 +++++--- test/clj_http/test/util_test.clj | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/clj_http/util.clj b/src/clj_http/util.clj index 0528703b..9d69f73c 100644 --- a/src/clj_http/util.clj +++ b/src/clj_http/util.clj @@ -145,15 +145,17 @@ false (or v1 v2))))) +(defn- trim-quotes [s] + (clojure.string/replace s #"^\s*(\"(.*)\"|(.*?))\s*$" "$2$3")) + (defn parse-content-type "Parse `s` as an RFC 2616 media type." [s] - (if-let [m (re-matches #"\s*(([^/]+)/([^ ;]+))\s*(\s*;.*)?" (str s))] + (when-let [m (re-matches #"\s*(([^/]+)/([^ ;]+))\s*(\s*;.*)?" (str s))] {:content-type (keyword (nth m 1)) :content-type-params (->> (split (str (nth m 4)) #"\s*;\s*") - (identity) (remove blank?) (map #(split % #"=")) - (mapcat (fn [[k v]] [(keyword (lower-case k)) (trim v)])) + (mapcat (fn [[k v]] [(keyword (lower-case k)) (trim-quotes v)])) (apply hash-map))})) diff --git a/test/clj_http/test/util_test.clj b/test/clj_http/test/util_test.clj index 249ddcef..4674b71c 100644 --- a/test/clj_http/test/util_test.clj +++ b/test/clj_http/test/util_test.clj @@ -41,6 +41,9 @@ " application/json; charset=UTF-8 " {:content-type :application/json :content-type-params {:charset "UTF-8"}} + " application/json; charset=\"utf-8\" " + {:content-type :application/json + :content-type-params {:charset "utf-8"}} "text/html; charset=ISO-8859-4" {:content-type :text/html :content-type-params {:charset "ISO-8859-4"}})) From 1d30bf453033f29b83421e0ee8ae33b42b51a7de Mon Sep 17 00:00:00 2001 From: ajchemist <1694505+ajchemist@users.noreply.github.com> Date: Mon, 16 Nov 2020 02:35:59 +0900 Subject: [PATCH 052/107] Fixed decode-json-body (#568) Consider body charset --- src/clj_http/client.clj | 2 +- test/clj_http/test/client_test.clj | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index 9220688a..54f4f155 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -450,7 +450,7 @@ (= coerce :exceptional)))) (defn- decode-json-body [body keyword? charset] - (let [^BufferedReader br (io/reader (util/force-stream body))] + (let [^BufferedReader br (io/reader (util/force-stream body) :encoding charset)] (try (.mark br 1) (let [first-char (int (try (.read br) (catch EOFException _ -1)))] diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index 0b114fa3..7d90611d 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -1511,8 +1511,13 @@ (is (= all-legal (client/url-encode-illegal-characters all-legal))))) +(defmethod client/coerce-response-body :json+ms949 + [req resp] + (client/coerce-json-body req resp true "MS949")) + (deftest t-coercion-methods (let [json-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"bar\"}")) + json-ms949-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"안뇽\"}" "MS949")) auto-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"bar\"}")) edn-body (ByteArrayInputStream. (.getBytes "{:foo \"bar\"}")) transit-json-body (ByteArrayInputStream. @@ -1526,6 +1531,8 @@ (ByteArrayInputStream. (.getBytes "foo=bar")) json-resp {:body json-body :status 200 :headers {"content-type" "application/json"}} + json-ms949-resp {:body json-ms949-body :status 200 + :headers {"content-type" "application/json; charset=ms949"}} auto-resp {:body auto-body :status 200 :headers {"content-type" "application/json"}} edn-resp {:body edn-body :status 200 @@ -1555,6 +1562,8 @@ auto-www-form-urlencoded-resp)) (:body (client/coerce-response-body {:as :x-www-form-urlencoded} www-form-urlencoded-resp)))) + (is (= {:foo "안뇽"} + (:body (client/coerce-response-body {:as :json+ms949} json-ms949-resp)))) (testing "throws AssertionError when optional libraries are not loaded" (with-redefs [client/json-enabled? false] @@ -1567,6 +1576,7 @@ (is (thrown? AssertionError (client/coerce-response-body {:as :x-www-form-urlencoded} www-form-urlencoded-resp))) (is (thrown? AssertionError (client/coerce-response-body {:as :auto} auto-www-form-urlencoded-resp))))))) + (deftest t-reader-coercion (let [read-lines (fn [reader] (vec (take-while not-empty (repeatedly #(.readLine reader))))) reader-body (ByteArrayInputStream. (.getBytes "foo\nbar\n")) From d6d725589b1fcb0c1072fc4a9f02b0a8823537f1 Mon Sep 17 00:00:00 2001 From: Alistair Dutton Date: Sun, 15 Nov 2020 17:36:33 +0000 Subject: [PATCH 053/107] Bump patch versions of apache httpcomponents to latest. (#569) Fixes https://bugzilla.redhat.com/show_bug.cgi\?id\=1886587. Technically, only the upgrade to apache-httpclient is necessary to fix this violation but it seems prudent to update the other components to the same version to ensure that they work together nicely.. --- project.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project.clj b/project.clj index c8fe6660..a77d5351 100644 --- a/project.clj +++ b/project.clj @@ -7,11 +7,11 @@ :global-vars {*warn-on-reflection* false} :min-lein-version "2.0.0" :exclusions [org.clojure/clojure] - :dependencies [[org.apache.httpcomponents/httpcore "4.4.12"] - [org.apache.httpcomponents/httpclient "4.5.10"] - [org.apache.httpcomponents/httpclient-cache "4.5.10"] + :dependencies [[org.apache.httpcomponents/httpcore "4.4.13"] + [org.apache.httpcomponents/httpclient "4.5.13"] + [org.apache.httpcomponents/httpclient-cache "4.5.13"] [org.apache.httpcomponents/httpasyncclient "4.1.4"] - [org.apache.httpcomponents/httpmime "4.5.10"] + [org.apache.httpcomponents/httpmime "4.5.13"] [commons-codec "1.12"] [commons-io "2.6"] [slingshot "0.12.2"] From 1a3bc03c627cd50aa53432a506cc7e70422aa9cd Mon Sep 17 00:00:00 2001 From: Raymond Huang <1694040+rymndhng@users.noreply.github.com> Date: Sun, 15 Nov 2020 09:37:00 -0800 Subject: [PATCH 054/107] Close transit input stream after reading response (#565) Connections are not closing immediately after coercing the body using transit. This is a regression introduced in https://github.com/dakrone/clj-http/pull/475. Previously, the response body always was also transferred into an intermediate byte-array which will read bytes until the stream is empty, and automatically close the connection. Since we no longer have this implicit behavior, the body coercion middleware must close the inputstream after reading a value. Fixes #564 --- src/clj_http/client.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index 54f4f155..c64f14ef 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -485,7 +485,8 @@ {:pre [transit-enabled?]} (let [charset (or charset (response-charset resp)) body (if (can-parse-body? request resp) - (parse-transit (util/force-stream body) type transit-opts) + (with-open [in (util/force-stream body)] + (parse-transit in type transit-opts)) (util/force-string body charset))] (assoc resp :body body))) From 5ae68705f843969b68bbecda5ef7e5b403bed626 Mon Sep 17 00:00:00 2001 From: sdaro Date: Sun, 15 Nov 2020 18:37:34 +0100 Subject: [PATCH 055/107] Enumerated multi-param-style added (#562) * Enumerated multi-param-style added * Refactor multi-param transformation step This cleans up the transformation for multi-param style. * PR Feedback: Rename :enumerated to :comma-separated * Fixing a mistyped keyword Co-authored-by: David Saro Co-authored-by: Raymond Huang --- README.org | 5 ++++- src/clj_http/client.clj | 27 +++++++++++++++++---------- test/clj_http/test/client_test.clj | 12 +++++++++++- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/README.org b/README.org index b8b041a6..b1788cd7 100644 --- a/README.org +++ b/README.org @@ -593,13 +593,14 @@ disabled by using with-middleware to specify different behavior. :CUSTOM_ID: h-dd49992c-a516-4af0-9735-4f4340773361 :END: -There are three different ways that query string parameters for array values can +There are four different ways that query string parameters for array values can be generated, depending on what the resulting query string should look like, they are: - A repeating parameter (default) - Array style - Indexed array style +- Comma separated style Here is an example of the input and output for the ~:query-params~ parameter, controlled by the ~:multi-param-style~ option: @@ -613,6 +614,8 @@ controlled by the ~:multi-param-style~ option: ;; with :multi-param-style :indexed, a repeating param with array suffix and ;; index (Rails-style): :a [1 2 3] => "a[0]=1&a[1]=2&a[2]=3" +;; with :multi-param-style :comma-separated, a param with comma-separated values +:a [1 2 3] => "a=1,2,3" #+END_SRC ** Meta Tag Headers diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index c64f14ef..41e02951 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -762,21 +762,28 @@ (second found)) "UTF-8")) -(defn- multi-param-suffix [index multi-param-style] - (case multi-param-style - :indexed (str "[" index "]") - :array "[]" - "")) +(defn- multi-param-entries [key values multi-param-style encoding] + (let [key (util/url-encode (name key) encoding) + values (map #(util/url-encode (str %) encoding) values)] + (case multi-param-style + :indexed + (map-indexed #(vector (str key \[ %1 \]) %2) values) + + :array + (map #(vector (str key "[]") %) values) + + :comma-separated + ;; See sub-delims in https://tools.ietf.org/html/rfc3986#section-2.2 + [[key (str/join "," values)]] + + ;; default: repeat the key multiple times + (map #(vector key %) values)))) (defn generate-query-string-with-encoding [params encoding multi-param-style] (str/join "&" (mapcat (fn [[k v]] (if (sequential? v) - (map-indexed - #(str (util/url-encode (name k) encoding) - (multi-param-suffix %1 multi-param-style) - "=" - (util/url-encode (str %2) encoding)) v) + (map #(str/join "=" %) (multi-param-entries k v multi-param-style encoding)) [(str (util/url-encode (name k) encoding) "=" (util/url-encode (str v) encoding))])) diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index 7d90611d..4a4ec3d0 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -1677,7 +1677,17 @@ query-string (-> resp :body form-decode-str)] (is (= 200 (:status resp))) (is (.contains query-string "a[]=1&a[]=2&a[]=3") query-string) - (is (.contains query-string "b[]=x&b[]=y&b[]=z") query-string)))) + (is (.contains query-string "b[]=x&b[]=y&b[]=z") query-string))) + (testing "multi-valued query params in comma-separated" + (let [resp (request {:uri "/query-string" + :method :get + :multi-param-style :comma-separated + :query-params {:a [1 2 3] + :b ["x" "y" "z"]}}) + query-string (-> resp :body form-decode-str)] + (is (= 200 (:status resp))) + (is (.contains query-string "a=1,2,3") query-string) + (is (.contains query-string "b=x,y,z") query-string)))) (deftest t-wrap-flatten-nested-params (is-applied client/wrap-flatten-nested-params From bfce49bf2b28305bf85cf3adaed5ef5db1f1ca33 Mon Sep 17 00:00:00 2001 From: Raymond Huang <1694040+rymndhng@users.noreply.github.com> Date: Sun, 15 Nov 2020 09:39:39 -0800 Subject: [PATCH 056/107] Adds workaround for Async Multipart uploads greater than 25 kb (#574) Fixes #560 --- src/clj_http/multipart.clj | 24 +++++++++++++++++++++++- test/clj_http/test/client_test.clj | 21 ++++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/clj_http/multipart.clj b/src/clj_http/multipart.clj index ce72b5fa..29adef84 100644 --- a/src/clj_http/multipart.clj +++ b/src/clj_http/multipart.clj @@ -125,6 +125,27 @@ [{:keys [^ContentBody content]}] content) +(defn- multipart-workaround + "Workaround for AsyncHttpClient to bypass 25kb restriction on getContent. + + See https://github.com/dakrone/clj-http/issues/560. + " + [^org.apache.http.entity.mime.MultipartFormEntity mp-entity] + (reify org.apache.http.HttpEntity + (isRepeatable [_] (.isRepeatable mp-entity)) + (isChunked [_] (.isChunked mp-entity)) + (isStreaming [_] (.isStreaming mp-entity)) + (getContentLength [_] (.getContentLength mp-entity)) + (getContentType [_] (.getContentType mp-entity)) + (getContentEncoding [_] (.getContentEncoding mp-entity)) + (consumeContent [_] (.consumeContent mp-entity)) + (getContent [_] + (let [os (java.io.ByteArrayOutputStream.)] + (.writeTo mp-entity os) + (.flush os) + (java.io.ByteArrayInputStream. (.toByteArray os)))) + (writeTo [_ output-stream] (.writeTo mp-entity output-stream)))) + (defn create-multipart-entity "Takes a multipart vector of maps and creates a MultipartEntity with each map added as a part, depending on the type of content." @@ -140,4 +161,5 @@ (let [name (or (:part-name m) (:name m)) part (make-multipart-body m)] (.addPart mp-entity name part))) - (.build mp-entity))) + (multipart-workaround + (.build mp-entity)))) diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index 4a4ec3d0..999ed2a7 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -160,7 +160,26 @@ )] (is (= 200 (:status @resp))) (is (not (realized? exception))) - #_(when (realized? exception) (prn @exception)))) + #_(when (realized? exception) (prn @exception))) + + ;; Regression Testing https://github.com/dakrone/clj-http/issues/560 + (testing "multipart uploads larger than 25kb" + (let [resp (promise) + exception (promise) + ;; assumption: file > 5kb + file (clojure.java.io/file "test-resources/big_array_json.json") + + _ (request {:uri "/post" :method :post + :async? true + :multipart [{:name "part-1" :content file} + {:name "part-2" :content file} + {:name "part-3" :content file} + {:name "part-4" :content file} + {:name "part-5" :content file}]} + resp + exception)] + (is (= 200 (:status (deref resp 500 :failed)))) + (is (not (realized? exception)))))) (deftest ^:integration nil-input (is (thrown-with-msg? Exception #"Host URL cannot be nil" From d2f523dec05c0a85ef4cb21c7821c7e37e437e07 Mon Sep 17 00:00:00 2001 From: Ankit Singh <18137713+gitankitsingh@users.noreply.github.com> Date: Sun, 15 Nov 2020 23:12:25 +0530 Subject: [PATCH 057/107] Remove unused imports (#570) Co-authored-by: Ankit Singh --- src/clj_http/client.clj | 16 +++---- src/clj_http/conn_mgr.clj | 40 ++++++----------- src/clj_http/cookies.clj | 17 +++---- src/clj_http/core.clj | 64 +++++++++------------------ src/clj_http/core_old.clj | 46 ++++++++----------- src/clj_http/headers.clj | 4 +- src/clj_http/multipart.clj | 15 +++---- src/clj_http/util.clj | 12 +++-- test/clj_http/test/client_test.clj | 12 ++--- test/clj_http/test/conn_mgr_test.clj | 20 +++------ test/clj_http/test/cookies_test.clj | 3 +- test/clj_http/test/core_test.clj | 36 ++++++--------- test/clj_http/test/headers_test.clj | 7 ++- test/clj_http/test/multipart_test.clj | 9 ++-- test/clj_http/test/util_test.clj | 8 ++-- 15 files changed, 117 insertions(+), 192 deletions(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index 41e02951..7f214c58 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -1,24 +1,22 @@ (ns clj-http.client "Batteries-included HTTP client." + (:refer-clojure :exclude [get update]) (:require [clj-http.conn-mgr :as conn] [clj-http.cookies :refer [wrap-cookies]] [clj-http.core :as core] [clj-http.headers :refer [wrap-header-map]] [clj-http.links :refer [wrap-links]] - [clj-http.util :refer [opt] :as util] + [clj-http.util :as util :refer [opt]] [clojure.java.io :as io] [clojure.stacktrace :refer [root-cause]] [clojure.string :as str] [clojure.walk :refer [keywordize-keys prewalk]] [slingshot.slingshot :refer [throw+]]) - (:import (java.io InputStream File ByteArrayOutputStream ByteArrayInputStream EOFException BufferedReader) - (java.net URL UnknownHostException) - (org.apache.http.entity BufferedHttpEntity ByteArrayEntity - InputStreamEntity FileEntity StringEntity) - (org.apache.http.impl.conn PoolingHttpClientConnectionManager) - (org.apache.http.impl.nio.conn PoolingNHttpClientConnectionManager) - (org.apache.http.impl.nio.client HttpAsyncClients)) - (:refer-clojure :exclude [get update])) + (:import [java.io BufferedReader ByteArrayInputStream ByteArrayOutputStream EOFException File InputStream] + [java.net UnknownHostException URL] + [org.apache.http.entity BufferedHttpEntity ByteArrayEntity FileEntity InputStreamEntity StringEntity] + org.apache.http.impl.conn.PoolingHttpClientConnectionManager + org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager)) ;; Cheshire is an optional dependency, so we check for it at compile time. (def json-enabled? diff --git a/src/clj_http/conn_mgr.clj b/src/clj_http/conn_mgr.clj index f9f646eb..c7d86529 100644 --- a/src/clj_http/conn_mgr.clj +++ b/src/clj_http/conn_mgr.clj @@ -2,32 +2,20 @@ "Utility methods for Scheme registries and HTTP connection managers" (:require [clj-http.util :refer [opt]] [clojure.java.io :as io]) - (:import (java.net Socket Proxy Proxy$Type InetSocketAddress) - (java.security KeyStore) - (javax.net.ssl KeyManager - TrustManager) - (org.apache.http.config RegistryBuilder Registry SocketConfig) - (org.apache.http.conn HttpClientConnectionManager) - (org.apache.http.conn.ssl DefaultHostnameVerifier - NoopHostnameVerifier - SSLConnectionSocketFactory - SSLContexts - TrustStrategy) - (org.apache.http.conn.socket PlainConnectionSocketFactory) - (org.apache.http.impl.conn BasicHttpClientConnectionManager - PoolingHttpClientConnectionManager) - (org.apache.http.impl.nio.conn PoolingNHttpClientConnectionManager) - (javax.net.ssl SSLContext HostnameVerifier) - (org.apache.http.nio.conn NHttpClientConnectionManager) - (org.apache.http.nio.conn.ssl SSLIOSessionStrategy) - (org.apache.http.impl.nio.reactor - IOReactorConfig - AbstractMultiworkerIOReactor$DefaultThreadFactory - DefaultConnectingIOReactor) - (org.apache.http.nio.conn NoopIOSessionStrategy) - (org.apache.http.nio.protocol HttpAsyncRequestExecutor) - (org.apache.http.impl.nio DefaultHttpClientIODispatch) - (org.apache.http.config ConnectionConfig))) + (:import [java.net InetSocketAddress Proxy Proxy$Type Socket] + java.security.KeyStore + [javax.net.ssl HostnameVerifier KeyManager SSLContext TrustManager] + [org.apache.http.config ConnectionConfig Registry RegistryBuilder SocketConfig] + org.apache.http.conn.HttpClientConnectionManager + org.apache.http.conn.socket.PlainConnectionSocketFactory + [org.apache.http.conn.ssl DefaultHostnameVerifier NoopHostnameVerifier SSLConnectionSocketFactory SSLContexts TrustStrategy] + [org.apache.http.impl.conn BasicHttpClientConnectionManager PoolingHttpClientConnectionManager] + org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager + org.apache.http.impl.nio.DefaultHttpClientIODispatch + [org.apache.http.impl.nio.reactor DefaultConnectingIOReactor IOReactorConfig] + [org.apache.http.nio.conn NHttpClientConnectionManager NoopIOSessionStrategy] + org.apache.http.nio.conn.ssl.SSLIOSessionStrategy + org.apache.http.nio.protocol.HttpAsyncRequestExecutor)) (def ^:private insecure-context-verifier (delay { diff --git a/src/clj_http/cookies.clj b/src/clj_http/cookies.clj index d42dc9a9..917551e0 100644 --- a/src/clj_http/cookies.clj +++ b/src/clj_http/cookies.clj @@ -2,16 +2,13 @@ "Namespace dealing with HTTP cookies" (:require [clj-http.util :refer [opt]] [clojure.string :refer [blank? join lower-case]]) - (:import (org.apache.http.client.params ClientPNames CookiePolicy) - (org.apache.http.cookie ClientCookie CookieOrigin CookieSpec) - (org.apache.http.params BasicHttpParams) - (org.apache.http.impl.cookie BasicClientCookie2) - (org.apache.http.impl.cookie BrowserCompatSpecFactory) - (org.apache.http.message BasicHeader) - org.apache.http.client.CookieStore - (org.apache.http.impl.client BasicCookieStore) - (org.apache.http Header) - (org.apache.http.protocol BasicHttpContext))) + (:import org.apache.http.client.CookieStore + [org.apache.http.cookie ClientCookie CookieOrigin CookieSpec] + org.apache.http.Header + org.apache.http.impl.client.BasicCookieStore + [org.apache.http.impl.cookie BasicClientCookie2 BrowserCompatSpecFactory] + org.apache.http.message.BasicHeader + org.apache.http.protocol.BasicHttpContext)) (defn cookie-spec ^org.apache.http.cookie.CookieSpec [] (.create diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index dacc77bf..1ea4212a 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -4,50 +4,26 @@ [clj-http.headers :as headers] [clj-http.multipart :as mp] [clj-http.util :refer [opt]] - [clojure.pprint]) - (:import (java.io ByteArrayOutputStream FilterInputStream InputStream) - (java.net URI URL ProxySelector InetAddress) - (java.util Locale) - (org.apache.http HttpEntity HeaderIterator HttpHost HttpRequest - HttpEntityEnclosingRequest HttpResponse - HttpRequestInterceptor HttpResponseInterceptor - ProtocolException) - (org.apache.http.auth UsernamePasswordCredentials AuthScope - NTCredentials) - (org.apache.http.client HttpRequestRetryHandler RedirectStrategy - CredentialsProvider) - (org.apache.http.client.config RequestConfig CookieSpecs) - (org.apache.http.client.methods HttpDelete HttpGet HttpPost HttpPut - HttpOptions HttpPatch - HttpHead - HttpEntityEnclosingRequestBase - CloseableHttpResponse - HttpUriRequest HttpRequestBase) - (org.apache.http.client.protocol HttpClientContext) - (org.apache.http.client.utils URIUtils) - (org.apache.http.config RegistryBuilder) - (org.apache.http.conn.routing HttpRoute HttpRoutePlanner) - (org.apache.http.conn.ssl BrowserCompatHostnameVerifier - SSLConnectionSocketFactory SSLContexts) - (org.apache.http.conn.socket PlainConnectionSocketFactory) - (org.apache.http.conn.util PublicSuffixMatcherLoader) - (org.apache.http.cookie CookieSpecProvider) - (org.apache.http.entity ByteArrayEntity StringEntity) - (org.apache.http.impl.client BasicCredentialsProvider - CloseableHttpClient HttpClients - DefaultRedirectStrategy - LaxRedirectStrategy HttpClientBuilder) - (org.apache.http.client.cache HttpCacheContext) - (org.apache.http.impl.client.cache CacheConfig - CachingHttpClientBuilder) - (org.apache.http.impl.cookie DefaultCookieSpecProvider) - (org.apache.http.impl.conn SystemDefaultRoutePlanner - DefaultProxyRoutePlanner) - (org.apache.http.impl.nio.client HttpAsyncClientBuilder - HttpAsyncClients - CloseableHttpAsyncClient) - (org.apache.http.message BasicHttpResponse) - (java.util.concurrent ExecutionException))) + clojure.pprint) + (:import [java.io ByteArrayOutputStream FilterInputStream InputStream] + [java.net InetAddress ProxySelector URI URL] + java.util.Locale + [org.apache.http HeaderIterator HttpEntity HttpEntityEnclosingRequest HttpHost HttpRequestInterceptor HttpResponse HttpResponseInterceptor ProtocolException] + [org.apache.http.auth AuthScope NTCredentials UsernamePasswordCredentials] + [org.apache.http.client CredentialsProvider HttpRequestRetryHandler RedirectStrategy] + org.apache.http.client.cache.HttpCacheContext + [org.apache.http.client.config CookieSpecs RequestConfig] + [org.apache.http.client.methods CloseableHttpResponse HttpDelete HttpEntityEnclosingRequestBase HttpGet HttpHead HttpOptions HttpPatch HttpPost HttpPut HttpRequestBase HttpUriRequest] + org.apache.http.client.protocol.HttpClientContext + org.apache.http.client.utils.URIUtils + org.apache.http.config.RegistryBuilder + org.apache.http.conn.routing.HttpRoutePlanner + org.apache.http.cookie.CookieSpecProvider + [org.apache.http.entity ByteArrayEntity StringEntity] + [org.apache.http.impl.client BasicCredentialsProvider CloseableHttpClient DefaultRedirectStrategy HttpClientBuilder HttpClients LaxRedirectStrategy] + [org.apache.http.impl.client.cache CacheConfig CachingHttpClientBuilder] + [org.apache.http.impl.conn DefaultProxyRoutePlanner SystemDefaultRoutePlanner] + [org.apache.http.impl.nio.client CloseableHttpAsyncClient HttpAsyncClientBuilder HttpAsyncClients])) (def CUSTOM_COOKIE_POLICY "_custom") diff --git a/src/clj_http/core_old.clj b/src/clj_http/core_old.clj index 8695d643..52e17077 100644 --- a/src/clj_http/core_old.clj +++ b/src/clj_http/core_old.clj @@ -4,34 +4,24 @@ [clj-http.headers :as headers] [clj-http.multipart :as mp] [clj-http.util :refer [opt]] - [clojure.pprint]) - (:import (java.io ByteArrayOutputStream FilterInputStream InputStream) - - (org.apache.http HeaderIterator HttpEntity - HttpEntityEnclosingRequest - HttpResponse Header HttpHost - HttpResponseInterceptor) - (org.apache.http.auth UsernamePasswordCredentials AuthScope - NTCredentials) - (org.apache.http.params CoreConnectionPNames) - (org.apache.http.client HttpClient HttpRequestRetryHandler) - (org.apache.http.client.methods HttpDelete - HttpEntityEnclosingRequestBase - HttpGet HttpHead HttpOptions - HttpPatch HttpPost HttpPut - HttpUriRequest) - (org.apache.http.client.params CookiePolicy ClientPNames) - (org.apache.http.conn ClientConnectionManager) - (org.apache.http.conn.routing HttpRoute) - (org.apache.http.conn.params ConnRoutePNames) - (org.apache.http.cookie CookieSpecFactory) - (org.apache.http.cookie.params CookieSpecPNames) - (org.apache.http.entity ByteArrayEntity StringEntity) - - (org.apache.http.impl.client DefaultHttpClient) - (org.apache.http.impl.conn ProxySelectorRoutePlanner) - (org.apache.http.impl.cookie BrowserCompatSpec) - (java.net URI))) + clojure.pprint) + (:import [java.io ByteArrayOutputStream FilterInputStream InputStream] + java.net.URI + [org.apache.http HeaderIterator HttpEntity HttpEntityEnclosingRequest HttpHost HttpResponseInterceptor] + [org.apache.http.auth AuthScope NTCredentials UsernamePasswordCredentials] + [org.apache.http.client HttpClient HttpRequestRetryHandler] + [org.apache.http.client.methods HttpDelete HttpEntityEnclosingRequestBase HttpGet HttpHead HttpOptions HttpPatch HttpPost HttpPut HttpUriRequest] + [org.apache.http.client.params ClientPNames CookiePolicy] + org.apache.http.conn.ClientConnectionManager + org.apache.http.conn.params.ConnRoutePNames + org.apache.http.conn.routing.HttpRoute + org.apache.http.cookie.CookieSpecFactory + org.apache.http.cookie.params.CookieSpecPNames + [org.apache.http.entity ByteArrayEntity StringEntity] + org.apache.http.impl.client.DefaultHttpClient + org.apache.http.impl.conn.ProxySelectorRoutePlanner + org.apache.http.impl.cookie.BrowserCompatSpec + org.apache.http.params.CoreConnectionPNames)) (defn parse-headers "Takes a HeaderIterator and returns a map of names to values. diff --git a/src/clj_http/headers.clj b/src/clj_http/headers.clj index 59743008..d229320c 100644 --- a/src/clj_http/headers.clj +++ b/src/clj_http/headers.clj @@ -8,8 +8,8 @@ \"Accept-Encoding\")." (:require [clojure.string :as s] [potemkin :as potemkin]) - (:import (java.util Locale) - (org.apache.http Header HeaderIterator))) + (:import java.util.Locale + [org.apache.http Header HeaderIterator])) (def special-cases "A collection of HTTP headers that do not follow the normal diff --git a/src/clj_http/multipart.clj b/src/clj_http/multipart.clj index 29adef84..e4381ace 100644 --- a/src/clj_http/multipart.clj +++ b/src/clj_http/multipart.clj @@ -1,15 +1,10 @@ (ns clj-http.multipart "Namespace used for clj-http to create multipart entities and bodies." - (:import (java.io File InputStream) - (org.apache.http.entity ContentType) - (org.apache.http.entity.mime MultipartEntityBuilder) - (org.apache.http.entity.mime HttpMultipartMode) - (org.apache.http.entity.mime.content ContentBody - ByteArrayBody - FileBody - InputStreamBody - StringBody) - (org.apache.http Consts))) + (:import [java.io File InputStream] + org.apache.http.Consts + org.apache.http.entity.ContentType + [org.apache.http.entity.mime HttpMultipartMode MultipartEntityBuilder] + [org.apache.http.entity.mime.content ByteArrayBody ContentBody FileBody InputStreamBody StringBody])) ;; we don't need to make a fake byte-array every time, only once (def byte-array-type (type (byte-array 0))) diff --git a/src/clj_http/util.clj b/src/clj_http/util.clj index 9d69f73c..f45ded64 100644 --- a/src/clj_http/util.clj +++ b/src/clj_http/util.clj @@ -2,13 +2,11 @@ "Helper functions for the HTTP client." (:require [clojure.string :refer [blank? lower-case split trim]] [clojure.walk :refer [postwalk]]) - (:import (org.apache.commons.codec.binary Base64) - (org.apache.commons.io IOUtils) - (java.io InputStream BufferedInputStream ByteArrayInputStream - ByteArrayOutputStream EOFException PushbackInputStream) - (java.net URLEncoder URLDecoder) - (java.util.zip InflaterInputStream DeflaterInputStream - GZIPInputStream GZIPOutputStream))) + (:import [java.io BufferedInputStream ByteArrayInputStream ByteArrayOutputStream EOFException InputStream PushbackInputStream] + [java.net URLDecoder URLEncoder] + [java.util.zip DeflaterInputStream GZIPInputStream GZIPOutputStream InflaterInputStream] + org.apache.commons.codec.binary.Base64 + org.apache.commons.io.IOUtils)) (defn utf8-bytes "Returns the encoding's bytes corresponding to the given string. If no diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index 999ed2a7..3f651c45 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -4,17 +4,17 @@ [clj-http.conn-mgr :as conn] [clj-http.test.core-test :refer [run-server]] [clj-http.util :as util] - [clojure.string :as str] [clojure.java.io :refer [resource]] + [clojure.string :as str] [clojure.test :refer :all] [cognitect.transit :as transit] - [ring.util.codec :refer [form-decode-str]] [ring.middleware.nested-params :refer [parse-nested-keys]] + [ring.util.codec :refer [form-decode-str]] [slingshot.slingshot :refer [try+]]) - (:import (java.net UnknownHostException) - (java.io ByteArrayInputStream) - (org.apache.http HttpEntity) - (org.apache.logging.log4j LogManager))) + (:import java.io.ByteArrayInputStream + java.net.UnknownHostException + org.apache.http.HttpEntity + org.apache.logging.log4j.LogManager)) (defonce logger (LogManager/getLogger "clj-http.test.client-test")) diff --git a/test/clj_http/test/conn_mgr_test.clj b/test/clj_http/test/conn_mgr_test.clj index e8b446aa..aca7463d 100644 --- a/test/clj_http/test/conn_mgr_test.clj +++ b/test/clj_http/test/conn_mgr_test.clj @@ -4,19 +4,13 @@ [clj-http.test.core-test :refer [run-server]] [clojure.test :refer :all] [ring.adapter.jetty :as ring]) - (:import (java.security KeyStore) - (javax.net.ssl KeyManager - KeyManagerFactory - TrustManager - TrustManagerFactory) - (org.apache.http.impl.conn BasicHttpClientConnectionManager) - (org.apache.http.conn.ssl SSLConnectionSocketFactory - DefaultHostnameVerifier - NoopHostnameVerifier - TrustStrategy) - (org.apache.http.conn.socket PlainConnectionSocketFactory) - (org.apache.http.nio.conn NoopIOSessionStrategy) - (org.apache.http.nio.conn.ssl SSLIOSessionStrategy))) + (:import java.security.KeyStore + [javax.net.ssl KeyManagerFactory TrustManagerFactory] + org.apache.http.conn.socket.PlainConnectionSocketFactory + org.apache.http.conn.ssl.SSLConnectionSocketFactory + org.apache.http.impl.conn.BasicHttpClientConnectionManager + org.apache.http.nio.conn.NoopIOSessionStrategy + org.apache.http.nio.conn.ssl.SSLIOSessionStrategy)) (def client-ks "test-resources/client-keystore") (def client-ks-pass "keykey") diff --git a/test/clj_http/test/cookies_test.clj b/test/clj_http/test/cookies_test.clj index e5a5e43d..8dd2b351 100644 --- a/test/clj_http/test/cookies_test.clj +++ b/test/clj_http/test/cookies_test.clj @@ -1,8 +1,7 @@ (ns clj-http.test.cookies-test (:require [clj-http.cookies :refer :all] - [clj-http.util :refer :all] [clojure.test :refer :all]) - (:import (org.apache.http.impl.cookie BasicClientCookie BasicClientCookie2))) + (:import [org.apache.http.impl.cookie BasicClientCookie BasicClientCookie2])) (defn refer-private [ns] (doseq [[symbol var] (ns-interns ns)] diff --git a/test/clj_http/test/core_test.clj b/test/clj_http/test/core_test.clj index f551c804..da9f3681 100644 --- a/test/clj_http/test/core_test.clj +++ b/test/clj_http/test/core_test.clj @@ -5,30 +5,22 @@ [clj-http.core :as core] [clj-http.util :as util] [clojure.java.io :refer [file]] - [clojure.pprint :as pp] [clojure.test :refer :all] [ring.adapter.jetty :as ring]) - (:import (java.io ByteArrayInputStream) - (java.net InetAddress SocketTimeoutException) - (java.util.concurrent TimeoutException TimeUnit) - (org.apache.http.params CoreConnectionPNames CoreProtocolPNames) - (org.apache.http.message BasicHeader BasicHeaderIterator) - (org.apache.http.client.methods HttpPost) - (org.apache.http.client.protocol HttpClientContext) - (org.apache.http.client.config RequestConfig) - (org.apache.http.client.params CookiePolicy ClientPNames) - (org.apache.http.conn.util PublicSuffixMatcherLoader) - (org.apache.http.cookie CommonCookieAttributeHandler) - (org.apache.http HttpRequest HttpResponse HttpConnection - HttpInetConnection HttpVersion ProtocolException) - (org.apache.http.protocol HttpContext ExecutionContext) - (org.apache.http.impl.client DefaultHttpClient) - (org.apache.http.impl.cookie RFC6265CookieSpec RFC6265CookieSpecProvider - RFC6265CookieSpecProvider$CompatibilityLevel) - (org.apache.http.impl.conn InMemoryDnsResolver) - (org.apache.http.client.params ClientPNames) - (org.apache.logging.log4j LogManager) - (sun.security.provider.certpath SunCertPathBuilderException))) + (:import java.io.ByteArrayInputStream + [java.net InetAddress SocketTimeoutException] + [java.util.concurrent TimeoutException TimeUnit] + [org.apache.http HttpConnection HttpInetConnection HttpRequest HttpResponse ProtocolException] + org.apache.http.client.config.RequestConfig + org.apache.http.client.params.ClientPNames + org.apache.http.client.protocol.HttpClientContext + org.apache.http.impl.conn.InMemoryDnsResolver + org.apache.http.impl.cookie.RFC6265CookieSpecProvider + [org.apache.http.message BasicHeader BasicHeaderIterator] + [org.apache.http.params CoreConnectionPNames CoreProtocolPNames] + [org.apache.http.protocol ExecutionContext HttpContext] + org.apache.logging.log4j.LogManager + sun.security.provider.certpath.SunCertPathBuilderException)) (defonce logger (LogManager/getLogger "clj-http.test.core-test")) diff --git a/test/clj_http/test/headers_test.clj b/test/clj_http/test/headers_test.clj index 1470017d..a2c5e835 100644 --- a/test/clj_http/test/headers_test.clj +++ b/test/clj_http/test/headers_test.clj @@ -3,10 +3,9 @@ [clj-http.headers :refer :all] [clj-http.util :refer [lower-case-keys]] [clojure.test :refer :all]) - (:import (javax.servlet.http HttpServletRequest - HttpServletResponse) - (org.eclipse.jetty.server Request Server) - (org.eclipse.jetty.server.handler AbstractHandler))) + (:import [javax.servlet.http HttpServletRequest HttpServletResponse] + [org.eclipse.jetty.server Request Server] + org.eclipse.jetty.server.handler.AbstractHandler)) (deftest test-special-case (are [expected given] diff --git a/test/clj_http/test/multipart_test.clj b/test/clj_http/test/multipart_test.clj index eebb69a3..7c309cf5 100644 --- a/test/clj_http/test/multipart_test.clj +++ b/test/clj_http/test/multipart_test.clj @@ -1,11 +1,10 @@ (ns clj-http.test.multipart-test (:require [clj-http.multipart :refer :all] [clojure.test :refer :all]) - (:import (java.io File ByteArrayOutputStream ByteArrayInputStream) - (java.nio.charset Charset) - (org.apache.http.entity.mime.content FileBody StringBody ContentBody - ByteArrayBody InputStreamBody) - (org.apache.http.util EntityUtils))) + (:import [java.io ByteArrayInputStream ByteArrayOutputStream File] + java.nio.charset.Charset + [org.apache.http.entity.mime.content ByteArrayBody ContentBody FileBody InputStreamBody StringBody] + org.apache.http.util.EntityUtils)) (defn body-str [^StringBody body] (-> body .getReader slurp)) diff --git a/test/clj_http/test/util_test.clj b/test/clj_http/test/util_test.clj index 4674b71c..e886c4f5 100644 --- a/test/clj_http/test/util_test.clj +++ b/test/clj_http/test/util_test.clj @@ -1,9 +1,9 @@ (ns clj-http.test.util-test (:require [clj-http.util :refer :all] - [clojure.test :refer :all] - [clojure.java.io :as io]) - (:import (org.apache.commons.io IOUtils) - (org.apache.commons.io.input NullInputStream))) + [clojure.java.io :as io] + [clojure.test :refer :all]) + (:import org.apache.commons.io.input.NullInputStream + org.apache.commons.io.IOUtils)) (deftest test-lower-case-keys (are [map expected] From 2860fc73e6af14fefbd10d019dd1249c08912a01 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Sun, 15 Nov 2020 09:55:03 -0800 Subject: [PATCH 058/107] Update changelog for 3.11.0 --- changelog.org | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/changelog.org b/changelog.org index 0cfa7aa4..c05437c1 100644 --- a/changelog.org +++ b/changelog.org @@ -9,7 +9,20 @@ * Changelog List of user-visible changes that have gone into each release -** 3.10.4 (Unreleased) +** 3.11.1 (Unreleased) +** 3.11.0 +- Adds workaround for Async Multipart uploads greater than 25 kb (#574) + https://github.com/dakrone/clj-http/pull/571 +- Adds an additional style for multi-param-style added (#562) + https://github.com/dakrone/clj-http/pull/562 +- Close transit input stream after reading response (#565) + https://github.com/dakrone/clj-http/pull/565 +- Bump patch versions of apache httpcomponents to latest. (#569) + https://github.com/dakrone/clj-http/pull/569 +- Fixed decode-json-body (#568) + https://github.com/dakrone/clj-http/pull/568 +- Handle quoted parameter values in content type (#573) + https://github.com/dakrone/clj-http/pull/573 ** 3.10.3 - Improve error message when using incompatible version of cheshire https://github.com/dakrone/clj-http/pull/558 From 7de01ffb8d3ed0c89c736bcd224c4ef4c13e0f30 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Sun, 15 Nov 2020 09:58:48 -0800 Subject: [PATCH 059/107] Bump version to 3.11.0-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index a77d5351..0dea9fe5 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.10.4-SNAPSHOT" +(defproject clj-http "3.11.0-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 24d195596b6f1ace411f1a3690526338fca9e331 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Sun, 15 Nov 2020 09:59:38 -0800 Subject: [PATCH 060/107] Version 3.11.0 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 0dea9fe5..5c291937 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.11.0-SNAPSHOT" +(defproject clj-http "3.11.0" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 65aaa9b9cc2bd2cfd15634d4f4b2cea802852cf2 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Sun, 15 Nov 2020 09:59:51 -0800 Subject: [PATCH 061/107] Version 3.11.1-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 5c291937..1e880e1d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.11.0" +(defproject clj-http "3.11.1-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 8006afcc1b57e2a2b38f057394b9fe635e9a2d76 Mon Sep 17 00:00:00 2001 From: Reynald Borer Date: Wed, 25 Nov 2020 06:44:47 +0100 Subject: [PATCH 062/107] Fix typo in changelog (#576) Small typo in changelog, the link to PR 574 points to PR 571. --- changelog.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.org b/changelog.org index c05437c1..545b144c 100644 --- a/changelog.org +++ b/changelog.org @@ -12,7 +12,7 @@ List of user-visible changes that have gone into each release ** 3.11.1 (Unreleased) ** 3.11.0 - Adds workaround for Async Multipart uploads greater than 25 kb (#574) - https://github.com/dakrone/clj-http/pull/571 + https://github.com/dakrone/clj-http/pull/574 - Adds an additional style for multi-param-style added (#562) https://github.com/dakrone/clj-http/pull/562 - Close transit input stream after reading response (#565) From 1c751431a3a8d38a795a70609a60cee24ad62757 Mon Sep 17 00:00:00 2001 From: Raymond Huang <1694040+rymndhng@users.noreply.github.com> Date: Wed, 23 Dec 2020 11:06:15 -0800 Subject: [PATCH 063/107] Create SSLContext consistently for all connection managers (#575) Refactors the internals of clj-http.conn-mgr to create SSLContexts identically for both Regular and Async connection managers. Fixes #572 --- src/clj_http/conn_mgr.clj | 277 +++++++++------------------ test/clj_http/test/conn_mgr_test.clj | 68 ++----- 2 files changed, 111 insertions(+), 234 deletions(-) diff --git a/src/clj_http/conn_mgr.clj b/src/clj_http/conn_mgr.clj index c7d86529..111c64ff 100644 --- a/src/clj_http/conn_mgr.clj +++ b/src/clj_http/conn_mgr.clj @@ -17,31 +17,22 @@ org.apache.http.nio.conn.ssl.SSLIOSessionStrategy org.apache.http.nio.protocol.HttpAsyncRequestExecutor)) -(def ^:private insecure-context-verifier - (delay { - :context (-> (SSLContexts/custom) - (.loadTrustMaterial nil (reify TrustStrategy - (isTrusted [_ _ _] true))) - (.build)) - :verifier NoopHostnameVerifier/INSTANCE})) - -(def ^:private insecure-socket-factory - (delay - (let [{:keys [context verifier]} @insecure-context-verifier] - (SSLConnectionSocketFactory. ^SSLContext context - ^HostnameVerifier verifier)))) - -(def ^:private insecure-strategy - (delay - (let [{:keys [context verifier]} @insecure-context-verifier] - (SSLIOSessionStrategy. ^SSLContext context ^HostnameVerifier verifier)))) - -(def ^:private ^SSLConnectionSocketFactory secure-ssl-socket-factory - (delay (SSLConnectionSocketFactory/getSocketFactory))) - -(def ^:private ^SSLIOSessionStrategy secure-strategy - (delay (SSLIOSessionStrategy/getDefaultStrategy))) - +;; -- Interop Helpers --------------------------------------------------------- +(defn ^Registry into-registry [registry] + (cond + (instance? Registry registry) + registry + + (map? registry) + (let [registry-builder (RegistryBuilder/create)] + (doseq [[k v] registry] + (.register registry-builder k v)) + (.build registry-builder)) + + :else + (throw (IllegalArgumentException. "Cannot coerce into a Registry")))) + +;; -- SocketFactory ----------------------------------------------------------- (defn ^SSLConnectionSocketFactory SSLGenericSocketFactory "Given a function that returns a new socket, create an SSLConnectionSocketFactory that will use that socket." @@ -68,6 +59,7 @@ [^String hostname ^Integer port] (Socket. (Proxy. Proxy$Type/SOCKS (InetSocketAddress. hostname port)))) +;; -- SSL Contexts ------------------------------------------------------------ (defn ^KeyStore get-keystore* [keystore-file keystore-type ^String keystore-pass] (when keystore-file @@ -82,31 +74,26 @@ keystore (apply get-keystore* keystore args))) -(defn get-keystore-context-verifier +(defn- ssl-context-for-keystore ;; TODO: use something else for passwords ;; Note: JVM strings aren't ideal for passwords - see ;; https://tinyurl.com/azm3ab9 - [{:keys [keystore keystore-type ^String keystore-pass keystore-instance - trust-store trust-store-type trust-store-pass] - :as req}] + [{:keys [keystore keystore-type ^String keystore-pass + trust-store trust-store-type trust-store-pass]}] (let [ks (get-keystore keystore keystore-type keystore-pass) ts (get-keystore trust-store trust-store-type trust-store-pass)] - {:context (-> (SSLContexts/custom) - (.loadKeyMaterial - ks (when keystore-pass - (.toCharArray keystore-pass))) - (.loadTrustMaterial - ts nil) - (.build)) - :verifier (if (opt req :insecure) - NoopHostnameVerifier/INSTANCE - (DefaultHostnameVerifier.))})) - -(defn get-managers-context-verifier + (-> (SSLContexts/custom) + (.loadKeyMaterial + ks (when keystore-pass + (.toCharArray keystore-pass))) + (.loadTrustMaterial + ts nil) + (.build)))) + +(defn- ssl-context-for-trust-or-key-manager "Given an instance or seqable data structure of TrustManager or KeyManager will create and return an SSLContexts object including the resulting managers" - [{:keys [trust-managers key-managers] - :as req}] + [{:keys [trust-managers key-managers]}] (let [x-or-xs->x-array (fn [type x-or-xs] (cond (or (-> x-or-xs class .isArray) @@ -119,12 +106,38 @@ (x-or-xs->x-array TrustManager trust-managers)) key-managers (when key-managers (x-or-xs->x-array KeyManager key-managers))] - {:context (doto (.build (SSLContexts/custom)) - (.init key-managers trust-managers nil)) - :verifier (if (opt req :insecure) - NoopHostnameVerifier/INSTANCE - (DefaultHostnameVerifier.))})) - + (doto (.build (SSLContexts/custom)) + (.init key-managers trust-managers nil)))) + +(defn- ssl-context-insecure + "Creates a SSL Context that trusts all material." + [] + (-> (SSLContexts/custom) + (.loadTrustMaterial nil (reify TrustStrategy + (isTrusted [_ chain auth-type] true))) + (.build))) + +(defn ^SSLContext get-ssl-context + "Gets the SSL Context from a request or connection pool settings" + [{:keys [keystore trust-store key-managers trust-managers] :as config}] + (cond (or keystore trust-store) + (ssl-context-for-keystore config) + + (or key-managers trust-managers) + (ssl-context-for-trust-or-key-manager config) + + (opt config :insecure) + (ssl-context-insecure) + + :else + (SSLContexts/createDefault))) + +(defn ^HostnameVerifier get-hostname-verifier [config] + (if (opt config :insecure) + NoopHostnameVerifier/INSTANCE + (DefaultHostnameVerifier.))) + +;; -- Connection Managers ----------------------------------------------------- (defn make-socks-proxied-conn-manager "Given an optional hostname and a port, create a connection manager that's proxied using a SOCKS proxy." @@ -133,115 +146,27 @@ ([^String hostname ^Integer port {:keys [keystore keystore-type keystore-pass trust-store trust-store-type trust-store-pass - trust-managers key-managers] :as opts}] + trust-managers key-managers] :as config}] (let [socket-factory #(socks-proxied-socket hostname port) - ssl-context (cond - (or trust-managers key-managers) - (-> opts get-managers-context-verifier :context) - - (some (complement nil?) - [keystore keystore-type keystore-pass trust-store - trust-store-type trust-store-pass]) - (-> opts get-keystore-context-verifier :context)) - reg (-> (RegistryBuilder/create) - (.register "http" (PlainGenericSocketFactory socket-factory)) - (.register "https" - (SSLGenericSocketFactory - socket-factory ssl-context)) - (.build))] - (PoolingHttpClientConnectionManager. reg)))) - -(def ^:private insecure-scheme-registry - (delay - (-> (RegistryBuilder/create) - (.register "http" PlainConnectionSocketFactory/INSTANCE) - (.register "https" ^SSLConnectionSocketFactory @insecure-socket-factory) - (.build)))) - -(def ^:private insecure-strategy-registry - (delay - (-> (RegistryBuilder/create) - (.register "http" NoopIOSessionStrategy/INSTANCE) - (.register "https" ^SSLIOSessionStrategy @insecure-strategy) - (.build)))) - -(def ^:private regular-scheme-registry - (delay (-> (RegistryBuilder/create) - (.register "http" (PlainConnectionSocketFactory/getSocketFactory)) - (.register "https" @secure-ssl-socket-factory) - (.build)))) - -(def ^:private regular-strategy-registry - (delay (-> (RegistryBuilder/create) - (.register "http" NoopIOSessionStrategy/INSTANCE) - (.register "https" @secure-strategy) - (.build)))) - -(defn ^Registry get-custom-scheme-registry - [{:keys [context verifier]}] - (let [factory (SSLConnectionSocketFactory. ^SSLContext context - ^HostnameVerifier verifier)] - (-> (RegistryBuilder/create) - (.register "http" (PlainConnectionSocketFactory/getSocketFactory)) - (.register "https" factory) - (.build)))) - -(defn ^Registry get-custom-strategy-registry - [{:keys [context verifier]}] - (let [strategy (SSLIOSessionStrategy. ^SSLContext context - ^HostnameVerifier verifier)] - (-> (RegistryBuilder/create) - (.register "http" NoopIOSessionStrategy/INSTANCE) - (.register "https" strategy) - (.build)))) - -(defn ^Registry get-keystore-scheme-registry - [req] - (-> req - get-keystore-context-verifier - get-custom-scheme-registry)) - -(defn ^Registry get-keystore-strategy-registry - [req] - (-> req - get-keystore-context-verifier - get-custom-strategy-registry)) - -(defn ^Registry get-managers-scheme-registry - [req] - (-> req - get-managers-context-verifier - get-custom-scheme-registry)) - -(defn ^Registry get-managers-strategy-registry - [req] - (-> req - get-managers-context-verifier - get-custom-strategy-registry)) + registry (into-registry + {"http" (PlainGenericSocketFactory socket-factory) + "https" (SSLGenericSocketFactory socket-factory (get-ssl-context config))})] + (PoolingHttpClientConnectionManager. registry)))) (defn ^BasicHttpClientConnectionManager make-regular-conn-manager [{:keys [dns-resolver keystore trust-store key-managers trust-managers - socket-timeout] :as req}] - - (let [conn-manager (cond - (or key-managers trust-managers) - (BasicHttpClientConnectionManager. (get-managers-scheme-registry req) - nil nil - dns-resolver) - - (or keystore trust-store) - (BasicHttpClientConnectionManager. (get-keystore-scheme-registry req) - nil nil - dns-resolver) - - (opt req :insecure) (BasicHttpClientConnectionManager. - @insecure-scheme-registry nil nil - dns-resolver) - - :else (BasicHttpClientConnectionManager. @regular-scheme-registry nil nil - dns-resolver))] + socket-timeout] :as config}] + + (let [registry (into-registry + {"http" (PlainConnectionSocketFactory/getSocketFactory) + "https" (SSLConnectionSocketFactory. + (get-ssl-context config) + (get-hostname-verifier config))}) + conn-manager (BasicHttpClientConnectionManager. registry + nil nil + dns-resolver)] (when socket-timeout (.setSocketConfig conn-manager (-> (.getSocketConfig conn-manager) @@ -271,18 +196,12 @@ (defn ^PoolingNHttpClientConnectionManager make-regular-async-conn-manager [{:keys [keystore trust-store - key-managers trust-managers] :as req}] - (let [^Registry registry (cond - (or key-managers trust-managers) - (get-managers-strategy-registry req) - - (or keystore trust-store) - (get-keystore-strategy-registry req) - - (opt req :insecure) - @insecure-strategy-registry - - :else @regular-strategy-registry) + key-managers trust-managers] :as config}] + (let [^Registry registry (into-registry + {"http" (NoopIOSessionStrategy/INSTANCE) + "https" (SSLIOSessionStrategy. + (get-ssl-context config) + (get-hostname-verifier config))}) io-reactor (make-ioreactor {:shutdown-grace-period 1})] (doto (PoolingNHttpClientConnectionManager. io-reactor registry) (.setMaxTotal 1)))) @@ -300,16 +219,11 @@ timeout keystore trust-store key-managers trust-managers] :as config}] - (let [registry (cond - (opt config :insecure) @insecure-scheme-registry - - (or key-managers trust-managers) - (get-managers-scheme-registry config) - - (or keystore trust-store) - (get-keystore-scheme-registry config) - - :else @regular-scheme-registry)] + (let [registry (into-registry + {"http" (PlainConnectionSocketFactory/getSocketFactory) + "https" (SSLConnectionSocketFactory. + (get-ssl-context config) + (get-hostname-verifier config))})] (PoolingHttpClientConnectionManager. registry nil nil dns-resolver timeout java.util.concurrent.TimeUnit/SECONDS))) @@ -366,16 +280,11 @@ [{:keys [dns-resolver timeout keystore trust-store io-config key-managers trust-managers] :as config}] - (let [registry (cond - (opt config :insecure) @insecure-strategy-registry - - (or key-managers trust-managers) - (get-managers-scheme-registry config) - - (or keystore trust-store) - (get-keystore-scheme-registry config) - - :else @regular-strategy-registry) + (let [registry (into-registry + {"http" (NoopIOSessionStrategy/INSTANCE) + "https" (SSLIOSessionStrategy. + (get-ssl-context config) + (get-hostname-verifier config))}) io-reactor (make-ioreactor io-config) protocol-handler (HttpAsyncRequestExecutor.) io-event-dispatch (DefaultHttpClientIODispatch. protocol-handler diff --git a/test/clj_http/test/conn_mgr_test.clj b/test/clj_http/test/conn_mgr_test.clj index aca7463d..1cc7dbad 100644 --- a/test/clj_http/test/conn_mgr_test.clj +++ b/test/clj_http/test/conn_mgr_test.clj @@ -6,11 +6,7 @@ [ring.adapter.jetty :as ring]) (:import java.security.KeyStore [javax.net.ssl KeyManagerFactory TrustManagerFactory] - org.apache.http.conn.socket.PlainConnectionSocketFactory - org.apache.http.conn.ssl.SSLConnectionSocketFactory - org.apache.http.impl.conn.BasicHttpClientConnectionManager - org.apache.http.nio.conn.NoopIOSessionStrategy - org.apache.http.nio.conn.ssl.SSLIOSessionStrategy)) + org.apache.http.impl.conn.BasicHttpClientConnectionManager)) (def client-ks "test-resources/client-keystore") (def client-ks-pass "keykey") @@ -40,26 +36,6 @@ (let [ks (conn-mgr/get-keystore "test-resources/keystore" nil nil)] (is (instance? KeyStore ks)))) -(deftest keystore-scheme-factory - (let [sr (conn-mgr/get-keystore-scheme-registry - {:keystore client-ks :keystore-pass client-ks-pass - :trust-store client-ks :trust-store-pass client-ks-pass}) - plain-socket-factory (.lookup sr "http") - ssl-socket-factory (.lookup sr "https")] - (is (instance? PlainConnectionSocketFactory plain-socket-factory)) - (is (instance? SSLConnectionSocketFactory ssl-socket-factory)))) - -(deftest keystore-session-strategy - (let [strategy-registry (conn-mgr/get-keystore-strategy-registry - {:keystore client-ks - :keystore-pass client-ks-pass - :trust-store client-ks - :trust-store-pass client-ks-pass}) - noop-session-strategy (.lookup strategy-registry "http") - ssl-session-strategy (.lookup strategy-registry "https")] - (is (instance? NoopIOSessionStrategy noop-session-strategy)) - (is (instance? SSLIOSessionStrategy ssl-session-strategy)))) - (def array-of-trust-manager (let [ks (conn-mgr/get-keystore "test-resources/keystore" nil "keykey") tmf (doto (TrustManagerFactory/getInstance (TrustManagerFactory/getDefaultAlgorithm)) @@ -72,31 +48,6 @@ (.init ks (.toCharArray "keykey")))] (.getKeyManagers tmf))) -(deftest managers-scheme-factory - (doseq [[trust-managers key-managers] [[array-of-trust-manager array-of-key-manager] - [(seq array-of-trust-manager) (seq array-of-key-manager)] - [(first (seq array-of-trust-manager)) (first (seq array-of-key-manager))]]] - (let [scheme-registry (conn-mgr/get-managers-scheme-registry - {:trust-managers trust-managers - :key-managers key-managers}) - plain-socket-factory (.lookup scheme-registry "http") - ssl-socket-factory (.lookup scheme-registry "https")] - (is (instance? PlainConnectionSocketFactory plain-socket-factory)) - (is (instance? SSLConnectionSocketFactory ssl-socket-factory))))) - -(deftest managers-session-strategy - (doseq [[trust-managers key-managers] [[array-of-trust-manager array-of-key-manager] - [(seq array-of-trust-manager) (seq array-of-key-manager)] - [(first (seq array-of-trust-manager)) (first (seq array-of-key-manager))]]] - (let [strategy-registry (conn-mgr/get-managers-strategy-registry - {:trust-managers trust-managers - :key-managers key-managers}) - noop-session-strategy (.lookup strategy-registry "http") - ssl-session-strategy (.lookup strategy-registry "https")] - (is (instance? NoopIOSessionStrategy noop-session-strategy)) - (is (instance? SSLIOSessionStrategy ssl-session-strategy))))) - - (deftest ^:integration ssl-client-cert-get (let [server (ring/run-jetty secure-handler {:port 18083 :ssl-port 18084 @@ -135,6 +86,23 @@ exception (promise) _ (core/request (assoc secure-request :async? true) resp exception)] (is (= 200 (:status (deref resp 1000 {:status :timeout}))))) + + (testing "with reusable connection pool" + (let [pool (conn-mgr/make-reusable-async-conn-manager {:timeout 10000 + :keystore client-ks :keystore-pass client-ks-pass + :trust-store client-ks :trust-store-pass client-ks-pass + :insecure? true})] + (try + (let [resp (promise) exception (promise) + _ (core/request {:request-method :get :uri "/get" + :server-port 18084 :scheme :https + :server-name "localhost" + :connection-manager pool :async? true} resp exception)] + (is (= 200 (:status (deref resp 1000 {:status :timeout})))) + (is (:body @resp)) + (is (not (realized? exception)))) + (finally + (conn-mgr/shutdown-manager pool))))) (finally (.stop server))))) From 8f48899835e1ea3f1d14a3408dafbfc58b1f29b2 Mon Sep 17 00:00:00 2001 From: torkus <48790775+torkus@users.noreply.github.com> Date: Sat, 30 Jan 2021 11:21:50 +1030 Subject: [PATCH 064/107] core/request-config, adds 'normalize-uri' parameter. default true. (#583) Co-authored-by: Torkus <48141663+ogri-la@users.noreply.github.com> --- src/clj_http/core.clj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index 1ea4212a..92a8c8be 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -162,7 +162,8 @@ cookie-spec ; deprecated conn-request-timeout - conn-timeout] + conn-timeout + normalize-uri] :as req}] (let [config (-> (RequestConfig/custom) (.setConnectTimeout (or connection-timeout conn-timeout -1)) @@ -174,7 +175,8 @@ (boolean (opt req :allow-circular-redirects))) (.setRelativeRedirectsAllowed ((complement false?) - (opt req :allow-relative-redirects))))] + (opt req :allow-relative-redirects))) + (.setNormalizeUri (or normalize-uri true)))] (if cookie-spec (.setCookieSpec config CUSTOM_COOKIE_POLICY) (.setCookieSpec config (get-cookie-policy req))) From 9b608657cef3b0267251685af790316674b9a10e Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Fri, 29 Jan 2021 16:55:56 -0800 Subject: [PATCH 065/107] Update changelog --- README.org | 2 +- changelog.org | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index b1788cd7..4de5f2b3 100644 --- a/README.org +++ b/README.org @@ -119,7 +119,7 @@ function. With Leiningen/Boot: #+BEGIN_SRC clojure -[clj-http "3.10.3"] +[clj-http "3.12.0"] #+END_SRC If you need an older version, a 2.x release is also available. diff --git a/changelog.org b/changelog.org index 545b144c..08039701 100644 --- a/changelog.org +++ b/changelog.org @@ -9,7 +9,12 @@ * Changelog List of user-visible changes that have gone into each release -** 3.11.1 (Unreleased) +** 3.12.1 (unreleased) +** 3.12.0 +- Create SSLContext consistently for all connection managers (#575) + https://github.com/dakrone/clj-http/pull/575 +- Adds RequestConfig Option :normalize-uri (#583) + https://github.com/dakrone/clj-http/pull/583 ** 3.11.0 - Adds workaround for Async Multipart uploads greater than 25 kb (#574) https://github.com/dakrone/clj-http/pull/574 From ac15783bba4c6a0a9d04af34b3abb5d1d583b577 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Fri, 29 Jan 2021 17:01:40 -0800 Subject: [PATCH 066/107] Version 3.12.0 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 1e880e1d..076c655b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.11.1-SNAPSHOT" +(defproject clj-http "3.12.0" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From be95cb8f57ad9a0bdf80a97629ec42d4e609000d Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Fri, 29 Jan 2021 17:11:50 -0800 Subject: [PATCH 067/107] Version 3.12.1-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 076c655b..f0062c05 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.12.0" +(defproject clj-http "3.12.1-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 4ea37ac6570321ca7d9b632da51feedc9e751524 Mon Sep 17 00:00:00 2001 From: torkus <48790775+torkus@users.noreply.github.com> Date: Sun, 31 Jan 2021 04:47:33 +1030 Subject: [PATCH 068/107] core/request-config, fixes incorrect setNormalizeUri. (#584) adds test for core/request-config Co-authored-by: Torkus <48141663+ogri-la@users.noreply.github.com> --- src/clj_http/core.clj | 8 ++++---- test/clj_http/test/core_test.clj | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index 92a8c8be..dfa8544a 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -160,10 +160,10 @@ socket-timeout max-redirects cookie-spec + normalize-uri ; deprecated conn-request-timeout - conn-timeout - normalize-uri] + conn-timeout] :as req}] (let [config (-> (RequestConfig/custom) (.setConnectTimeout (or connection-timeout conn-timeout -1)) @@ -175,12 +175,12 @@ (boolean (opt req :allow-circular-redirects))) (.setRelativeRedirectsAllowed ((complement false?) - (opt req :allow-relative-redirects))) - (.setNormalizeUri (or normalize-uri true)))] + (opt req :allow-relative-redirects))))] (if cookie-spec (.setCookieSpec config CUSTOM_COOKIE_POLICY) (.setCookieSpec config (get-cookie-policy req))) (when max-redirects (.setMaxRedirects config max-redirects)) + (when-not (nil? normalize-uri) (.setNormalizeUri config normalize-uri)) (.build config))) (defmulti ^:private construct-http-host (fn [proxy-host proxy-port] diff --git a/test/clj_http/test/core_test.clj b/test/clj_http/test/core_test.clj index da9f3681..9fb91c10 100644 --- a/test/clj_http/test/core_test.clj +++ b/test/clj_http/test/core_test.clj @@ -744,6 +744,23 @@ (is (= (:trace-redirects resp-without-redirects) [])))) +(deftest t-request-config + (let [params {:conn-timeout 100 ;; deprecated + :connection-timeout 200 ;; takes precedence over `:conn-timeout` + :conn-request-timeout 300 ;; deprecated + :connection-request-timeout 400 ;; takes precedence over `:conn-request-timeout` + :socket-timeout 500 + :max-redirects 600 + :cookie-spec "foo" + :normalize-uri false} + request-config (core/request-config params)] + (is (= 200 (.getConnectTimeout request-config))) + (is (= 400 (.getConnectionRequestTimeout request-config))) + (is (= 500 (.getSocketTimeout request-config))) + (is (= 600 (.getMaxRedirects request-config))) + (is (= core/CUSTOM_COOKIE_POLICY (.getCookieSpec request-config))) + (is (false? (.isNormalizeUri request-config))))) + (deftest ^:integration t-override-request-config (run-server) (let [called-args (atom []) From 44162143ac2b7d363f76927d6cd06a83b2deb6e3 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Sat, 30 Jan 2021 10:31:58 -0800 Subject: [PATCH 069/107] Update changelog --- changelog.org | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changelog.org b/changelog.org index 08039701..e03bbe58 100644 --- a/changelog.org +++ b/changelog.org @@ -9,7 +9,10 @@ * Changelog List of user-visible changes that have gone into each release -** 3.12.1 (unreleased) +** 3.12.2 (unreleased) +** 3.12.1 +- Bugfix for :normalize-uri (#584) + https://github.com/dakrone/clj-http/pull/584 ** 3.12.0 - Create SSLContext consistently for all connection managers (#575) https://github.com/dakrone/clj-http/pull/575 From 25b3ce076f38574667fe73c8598aa3c600be8347 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Sat, 30 Jan 2021 10:35:42 -0800 Subject: [PATCH 070/107] Version 3.12.1 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index f0062c05..8223101d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.12.1-SNAPSHOT" +(defproject clj-http "3.12.1" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From dd15359451645f677b3e294164cf70330b92241d Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Sat, 30 Jan 2021 10:35:55 -0800 Subject: [PATCH 071/107] Version 3.12.2-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 8223101d..fb360e2a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.12.1" +(defproject clj-http "3.12.2-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 9fc17aa5ebf61f2264e0027fb4c2849ee62947af Mon Sep 17 00:00:00 2001 From: Raymond Huang <1694040+rymndhng@users.noreply.github.com> Date: Mon, 24 May 2021 11:07:57 -0700 Subject: [PATCH 072/107] Upgrade dependencies (#598) * Bump httpcore to version 4.4.14 * update commons-io to address CVE-2021-29425 https://nvd.nist.gov/vuln/detail/CVE-2021-29425 * Upgrade dependencies This also fixes #597 #596. Co-authored-by: Reynald Borer Co-authored-by: Anton Mostovoy --- project.clj | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/project.clj b/project.clj index fb360e2a..1aaf8fc9 100644 --- a/project.clj +++ b/project.clj @@ -7,35 +7,35 @@ :global-vars {*warn-on-reflection* false} :min-lein-version "2.0.0" :exclusions [org.clojure/clojure] - :dependencies [[org.apache.httpcomponents/httpcore "4.4.13"] + :dependencies [[org.apache.httpcomponents/httpcore "4.4.14"] [org.apache.httpcomponents/httpclient "4.5.13"] [org.apache.httpcomponents/httpclient-cache "4.5.13"] [org.apache.httpcomponents/httpasyncclient "4.1.4"] [org.apache.httpcomponents/httpmime "4.5.13"] - [commons-codec "1.12"] - [commons-io "2.6"] + [commons-codec "1.15"] + [commons-io "2.8.0"] [slingshot "0.12.2"] [potemkin "0.4.5"]] :resource-paths ["resources"] :profiles {:dev {:dependencies [;; optional deps - [cheshire "5.9.0"] + [cheshire "5.10.0"] [crouton "0.1.2" :exclusions [[org.jsoup/jsoup]]] - [org.jsoup/jsoup "1.11.3"] - [org.clojure/tools.reader "1.3.2"] - [com.cognitect/transit-clj "0.8.313"] - [ring/ring-codec "1.1.2"] + [org.jsoup/jsoup "1.13.1"] + [org.clojure/tools.reader "1.3.5"] + [com.cognitect/transit-clj "1.0.324"] + [ring/ring-codec "1.1.3"] ;; other (testing) deps - [org.clojure/clojure "1.10.0"] - [org.clojure/tools.logging "0.4.0"] - [ring/ring-jetty-adapter "1.7.1"] - [ring/ring-devel "1.7.1"] + [org.clojure/clojure "1.10.3"] + [org.clojure/tools.logging "1.1.0"] + [ring/ring-jetty-adapter "1.9.3"] + [ring/ring-devel "1.9.3"] ;; caching example deps - [org.clojure/core.cache "0.7.2"] + [org.clojure/core.cache "1.0.207"] ;; logging - [org.apache.logging.log4j/log4j-api "2.11.2"] - [org.apache.logging.log4j/log4j-core "2.11.2"] - [org.apache.logging.log4j/log4j-1.2-api "2.11.2"]] - :plugins [[lein-ancient "0.6.15"] + [org.apache.logging.log4j/log4j-api "2.14.1"] + [org.apache.logging.log4j/log4j-core "2.14.1"] + [org.apache.logging.log4j/log4j-1.2-api "2.14.1"]] + :plugins [[lein-ancient "0.7.0"] [jonase/eastwood "0.2.5"] [lein-kibit "0.1.5"] [lein-nvd "0.5.2"]]} From ee95dc7f8a3d2b6c77da67a1e3b0b6ec3b0a3ca4 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 24 May 2021 11:15:38 -0700 Subject: [PATCH 073/107] Update changelog --- changelog.org | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changelog.org b/changelog.org index e03bbe58..b7372945 100644 --- a/changelog.org +++ b/changelog.org @@ -9,7 +9,10 @@ * Changelog List of user-visible changes that have gone into each release -** 3.12.2 (unreleased) +** 3.12.3 (unreleased) +** 3.12.2 +- Upgrade Dependencies (#598) + https://github.com/dakrone/clj-http/pull/598 ** 3.12.1 - Bugfix for :normalize-uri (#584) https://github.com/dakrone/clj-http/pull/584 From ae9d40d1ce1cdf52170fd6701613e76f8433884a Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 24 May 2021 11:15:53 -0700 Subject: [PATCH 074/107] Version 3.12.2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 1aaf8fc9..93fc2350 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.12.2-SNAPSHOT" +(defproject clj-http "3.12.2" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 032a9bb6509e31877e0354c4d73f1a5e2aceb961 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 24 May 2021 11:19:19 -0700 Subject: [PATCH 075/107] Version 3.12.3-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 93fc2350..d0f75ff8 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.12.2" +(defproject clj-http "3.12.3-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 5f0f842f330201cdfbeedcde781bb62cbdd018b8 Mon Sep 17 00:00:00 2001 From: Reynald Borer Date: Tue, 6 Jul 2021 05:47:11 +0200 Subject: [PATCH 076/107] Allow http-client re-use in async situation (#599) * allow http-client re-use in async situation * Complete test-reusable-http-client to verify that http-client is reused across requests Co-authored-by: Reynald Borer --- src/clj_http/core.clj | 3 ++- test/clj_http/test/core_test.clj | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index dfa8544a..1a6a6b8b 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -625,7 +625,8 @@ (conn/shutdown-manager conn-mgr)) (throw t)))) (let [^CloseableHttpAsyncClient client - (build-async-http-client req conn-mgr http-url proxy-ignore-hosts) + (or http-client + (build-async-http-client req conn-mgr http-url proxy-ignore-hosts)) original-thread-bindings (clojure.lang.Var/getThreadBindingFrame)] (when cache? (throw (IllegalArgumentException. diff --git a/test/clj_http/test/core_test.clj b/test/clj_http/test/core_test.clj index 9fb91c10..fe039c37 100644 --- a/test/clj_http/test/core_test.clj +++ b/test/clj_http/test/core_test.clj @@ -883,7 +883,9 @@ :async true} (fn [resp] (is (= 200 (:status resp))) - (is (= {:foo "bar"} (:body resp)))) + (is (= {:foo "bar"} (:body resp))) + (is (= hc (:http-client resp)) + "http-client is correctly reused")) (fn [e] (is false (str "failed with " e))))) (let [cm (conn/make-reusable-conn-manager {}) hc (:http-client (client/get (localhost "/get") @@ -893,7 +895,9 @@ :http-client hc :as :json})] (is (= 200 (:status resp))) - (is (= {:foo "bar"} (:body resp))))) + (is (= {:foo "bar"} (:body resp))) + (is (= hc (:http-client resp)) + "http-client is correctly reused"))) (deftest ^:integration t-cookies-spec (run-server) From 0d6fa9b83b74f03f1be72e52b2c779a284d860b5 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 5 Jul 2021 20:56:30 -0700 Subject: [PATCH 077/107] Update README & Changelog --- README.org | 4 ++-- changelog.org | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.org b/README.org index 4de5f2b3..6a911f97 100644 --- a/README.org +++ b/README.org @@ -9,7 +9,7 @@ [[https://clojars.org/clj-http][file:https://img.shields.io/clojars/v/clj-http.svg]] [[https://github.com/dakrone/clj-http/actions?query=workflow%3A%22Clojure+CI%22][file:https://github.com/dakrone/clj-http/workflows/Clojure%20CI/badge.svg]] [[https://gitter.im/clj-http/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge][file:https://badges.gitter.im/clj-http/Lobby.svg]] -* Table of Contents :TOC: +* Table of Contents :TOC_3: :PROPERTIES: :CUSTOM_ID: h-aaf075ea-2f0e-4a45-871a-0f89c838fb4b :END: @@ -119,7 +119,7 @@ function. With Leiningen/Boot: #+BEGIN_SRC clojure -[clj-http "3.12.0"] +[clj-http "3.12.3"] #+END_SRC If you need an older version, a 2.x release is also available. diff --git a/changelog.org b/changelog.org index b7372945..04e5cfda 100644 --- a/changelog.org +++ b/changelog.org @@ -9,7 +9,10 @@ * Changelog List of user-visible changes that have gone into each release -** 3.12.3 (unreleased) +** 3.12.4 (unreleased) +** 3.12.3 +- Allow http-client re-use in async situation (#599) + https://github.com/dakrone/clj-http/pull/599 ** 3.12.2 - Upgrade Dependencies (#598) https://github.com/dakrone/clj-http/pull/598 From 44ce155754dbb375837dd1729b97e55c96046f9d Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 5 Jul 2021 20:58:00 -0700 Subject: [PATCH 078/107] Version 3.12.3 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index d0f75ff8..df5a1fa8 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.12.3-SNAPSHOT" +(defproject clj-http "3.12.3" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 7aa6d02ad83dff9af6217f39e517cde2ded73a25 Mon Sep 17 00:00:00 2001 From: Raymond Huang Date: Mon, 5 Jul 2021 20:58:16 -0700 Subject: [PATCH 079/107] Version 3.12.4-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index df5a1fa8..3420175f 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.12.3" +(defproject clj-http "3.12.4-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From c11e191f0a8c170117bce37317f1b957e62e9985 Mon Sep 17 00:00:00 2001 From: rgkirch <6143833+rgkirch@users.noreply.github.com> Date: Mon, 3 Jan 2022 18:19:32 -0500 Subject: [PATCH 080/107] Update README.org (#606) --- README.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.org b/README.org index 6a911f97..b4e47e6d 100644 --- a/README.org +++ b/README.org @@ -668,7 +668,7 @@ using: #+END_SRC Note that this feature is currently beta and uses [[https://github.com/weavejester/crouton][Crouton]] to parse the body of -the request. If you do not want to use this feature, you can include Crouton in +the request. If you want to use this feature, you can include Crouton in addition to clj-http as a dependency like so: #+BEGIN_SRC clojure From c37f2654f35c1b25ea7f7f7ff403d19e9526f9ce Mon Sep 17 00:00:00 2001 From: Elton Law Date: Thu, 20 Jan 2022 19:27:22 -0500 Subject: [PATCH 081/107] Bump log4j2 version for cve-2021-44228 (#604) --- project.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index 3420175f..52f6afe0 100644 --- a/project.clj +++ b/project.clj @@ -32,9 +32,9 @@ ;; caching example deps [org.clojure/core.cache "1.0.207"] ;; logging - [org.apache.logging.log4j/log4j-api "2.14.1"] - [org.apache.logging.log4j/log4j-core "2.14.1"] - [org.apache.logging.log4j/log4j-1.2-api "2.14.1"]] + [org.apache.logging.log4j/log4j-api "2.17.1"] + [org.apache.logging.log4j/log4j-core "2.17.1"] + [org.apache.logging.log4j/log4j-1.2-api "2.17.1"]] :plugins [[lein-ancient "0.7.0"] [jonase/eastwood "0.2.5"] [lein-kibit "0.1.5"] From 3ff32c3663c1f540076ec6ec16594e282a756c15 Mon Sep 17 00:00:00 2001 From: Andrew Van Dyke <34038526+vandyand@users.noreply.github.com> Date: Mon, 31 Jan 2022 16:16:59 -0500 Subject: [PATCH 082/107] Fix: Typo in README (#608) --- README.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index b4e47e6d..8ee8c78a 100644 --- a/README.org +++ b/README.org @@ -420,8 +420,8 @@ different ways to handle flattening them: :CUSTOM_ID: h-0e3eb987-5b2b-4874-97ef-b834394d083d :END: The new async HTTP request API is a Ring-style async API. -All options for synchronous request can use in asynchronous requests. -start an async request is easy, for example: +All options for synchronous requests can be used in asynchronous requests. +Starting an async request is easy, for example: #+BEGIN_SRC clojure ;; :async? in options map need to be true From c960a6a8075b1fb21a5c2cb3c64214015168ead1 Mon Sep 17 00:00:00 2001 From: Juho Teperi Date: Thu, 17 Mar 2022 04:22:08 +0200 Subject: [PATCH 083/107] =?UTF-8?q?Fix=20#609:=20Fix=20reading=20Transit?= =?UTF-8?q?=20response=20stream=20if=20data=20isn't=20available=E2=80=A6?= =?UTF-8?q?=20(#611)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … right-away InputStream .available only checks if data is available to read right now. It is possible that data becomes available leter. Read method (used by Transit internally) will block untill data is available or the stream is closed. It is best to let Transit read the stream. Empty streams throws an exception with Transit, but we can catch that specific case and return nil like previously. Two test cases test the existing functionality, and third new test case checks that Transit data is read correctly when the data becomes available a bit later. --- src/clj_http/client.clj | 9 ++++++-- test/clj_http/test/client_test.clj | 35 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index 7f214c58..8c38cf0c 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -101,10 +101,15 @@ "Resolve and apply Transit's JSON/MessagePack decoding." [^InputStream in type & [opts]] {:pre [transit-enabled?]} - (when (pos? (.available in)) + (try (let [reader (ns-resolve 'cognitect.transit 'reader) read (ns-resolve 'cognitect.transit 'read)] - (read (reader in type (transit-read-opts opts)))))) + (read (reader in type (transit-read-opts opts)))) + (catch RuntimeException e + ;; Ignore exceptions from trying to read an empty stream. + (if (instance? EOFException (.getCause e)) + nil + (throw e))))) (defn ^:dynamic transit-encode "Resolve and apply Transit's JSON/MessagePack encoding." diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index 3f651c45..d2c2c59b 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -12,6 +12,8 @@ [ring.util.codec :refer [form-decode-str]] [slingshot.slingshot :refer [try+]]) (:import java.io.ByteArrayInputStream + java.io.PipedInputStream + java.io.PipedOutputStream java.net.UnknownHostException org.apache.http.HttpEntity org.apache.logging.log4j.LogManager)) @@ -1754,3 +1756,36 @@ (is (= (.getMessage e) (str "only :flatten-nested-keys or :ignore-nested-query-string/" ":flatten-nested-keys may be specified, not both")))))) + +(defn transit-resp [body] + {:body body + :status 200 + :headers {"content-type" "application/transit-json"}}) + +(deftest issue-609-empty-transit-response + (testing "Body is available right away" + (is (= {:foo "bar"} + (:body (client/coerce-response-body + {:as :transit+json} + (transit-resp (ByteArrayInputStream. + (.getBytes "[\"^ \",\"~:foo\",\"bar\"]")))))))) + + (testing "Empty body is read as nil" + (is (nil? (:body (client/coerce-response-body + {:as :transit+json} + (transit-resp (ByteArrayInputStream. (.getBytes "")))))))) + + (testing "Body is read correctly even if the data becomes available later" + ;; Ensure both streams are closed (normally done inside future). + (with-open [o (PipedOutputStream.) + i (PipedInputStream.)] + (.connect i o) + (future + (Thread/sleep 10) + (.write o (.getBytes "[\"^ \",\"~:foo\",\"bar\"]")) + ;; Close right now, with-open will wait until test is done. + (.close o)) + (is (= {:foo "bar"} + (:body (client/coerce-response-body + {:as :transit+json} + (transit-resp i)))))))) From 5a5288319abab50141b8785625feca8de5a11dfc Mon Sep 17 00:00:00 2001 From: Austin Haas <91683199+AustinHaas-cisco@users.noreply.github.com> Date: Fri, 6 May 2022 14:57:06 -0700 Subject: [PATCH 084/107] Don't discard body for HEAD requests. (#615) Fixes #614 --- src/clj_http/core.clj | 5 ++++- test/clj_http/test/core_test.clj | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index 1a6a6b8b..2ed4ea11 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -391,6 +391,7 @@ (getMethod [] (.toUpperCase (name method) Locale/ROOT))) (.setURI (URI. url))))) +(def proxy-head-with-body (make-proxy-method-with-body :head)) (def proxy-delete-with-body (make-proxy-method-with-body :delete)) (def proxy-get-with-body (make-proxy-method-with-body :get)) (def proxy-copy-with-body (make-proxy-method-with-body :copy)) @@ -413,7 +414,9 @@ :get (if body (proxy-get-with-body http-url) (HttpGet. http-url)) - :head (HttpHead. http-url) + :head (if body + (proxy-head-with-body http-url) + (HttpHead. http-url)) :put (HttpPut. http-url) :post (HttpPost. http-url) :options (HttpOptions. http-url) diff --git a/test/clj_http/test/core_test.clj b/test/clj_http/test/core_test.clj index fe039c37..2567d799 100644 --- a/test/clj_http/test/core_test.clj +++ b/test/clj_http/test/core_test.clj @@ -104,6 +104,8 @@ {:status 200 :body "delete-with-body"} [:post "/multipart"] {:status 200 :body (:body req)} + [:head "/head-with-body"] + {:status 200 :headers {"body" (slurp (:body req))}} [:get "/get-with-body"] {:status 200 :body (:body req)} [:options "/options"] @@ -412,8 +414,10 @@ (deftest ^:integration head-with-body (run-server) - (let [resp (request {:request-method :head :uri "/head" :body "foo"})] - (is (= 200 (:status resp))))) + (let [resp (request {:request-method :head :uri "/head-with-body" + :body (.getBytes "foo")})] + (is (= 200 (:status resp))) + (is (= "foo" (get-in resp [:headers "body"]))))) (deftest ^:integration t-clojure-output-coercion (run-server) From 6fbec42eff3e6cb66234a8e28f570da43117dc71 Mon Sep 17 00:00:00 2001 From: Lee Read Date: Tue, 16 Aug 2022 20:14:23 -0400 Subject: [PATCH 085/107] readme: update link to clj-http-lite (#619) --- README.org | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.org b/README.org index 8ee8c78a..01d7ff23 100644 --- a/README.org +++ b/README.org @@ -1510,8 +1510,8 @@ without them. :CUSTOM_ID: h-ba6b263b-74a5-40f3-afc1-b0d785554c2b :END: -Like clj-http but need something more lightweight without as many external -dependencies? Check out [[https://github.com/hiredman/clj-http-lite][clj-http-lite]] for a project that can be used as a +Like clj-http but need something more lightweight without as any external +dependencies? Check out [[https://github.com/clj-commons/clj-http-lite][clj-http-lite]] for a project that can be used as a drop-in replacement for clj-http. ** Troubleshooting @@ -1590,7 +1590,7 @@ Libraries using clj-http: Libraries inspired by clj-http: - [[https://github.com/mpenet/jet][jet]] -- [[https://github.com/hiredman/clj-http-lite][clj-http-lite]] +- [[https://github.com/clj-commons/clj-http-lite][clj-http-lite]] * Other Libraries Providing Middleware :PROPERTIES: From 63a5341bdf14f99aaa200cc595bcf6b9da048300 Mon Sep 17 00:00:00 2001 From: Sam Umbach Date: Tue, 27 Sep 2022 10:36:03 -0400 Subject: [PATCH 086/107] Use standard Clojure map for cookies (#621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clojure `sorted-map` and `sorted-set` are useful in some scenarios but can present problems in others: * good: predictable iteration order, used to great effect in `apply-on-form-params`, `test-encode-cookies`, and `test-wrap-cookies` * bad: `get`, `contains?`, `find`, `dissoc`, `select-keys`, etc throw ClassCastException when given incompatible keys https://clojure.atlassian.net/browse/CLJ-2693 * bad: `=` and `clojure.data/diff` can yield exceptions https://clojure.atlassian.net/browse/CLJ-2325 https://clojure.atlassian.net/browse/CLJ-1983 * bad: `pr` and `print` generate strings indistinguishable from unsorted map/set * bad: `print-dup` generates unreadable strings https://clojure.atlassian.net/browse/CLJ-1733 I propose to avoid `sorted-map` for values in the clj-http `:cookies` map. Looking at the commit history, I'm unable to determine why a sorted-map was used initially. The change away from sorted-map doesn't break any tests and doesn't appear to have any substantive effect on clj-http behavior, but perhaps there are some use cases I'm not familiar with. Of course, I probably should provide a justification or example of obvious downside to this use of sorted-map 😉 so I'll attempt to explain: ```clj user> (-> resp pr-str clojure.edn/read-string (get-in [:cookies "someCookie" "value"]) nil user> (-> resp pr-str clojure.edn/read-string (get-in [:cookies "someCookie" :value]) "someValue" user> (-> resp (get-in [:cookies "someCookie" "value"]) ClassCastException clojure.lang.Keyword cannot be cast to java.lang.String java.lang.String.compareTo (String.java:111) user> (-> resp (get-in [:cookies "someCookie" :value]) "someValue" ``` We expect `get` or `find` to return `nil` when a key isn't present in a map, and that's what we observe when serializing the response map from service1 to EDN and reading this EDN in service2. We were surprised when the same code threw ClassCastException when executed within service1. While I wouldn't expect another clj-http user to run into exactly this set of circumstances, the potential for inconsistent behavior, this inconvenient behavior of sorted-map, the lack of obvious upside to using sorted-map for cookie values, and the minimal clj-http code change lead me to believe this is a worthwhile change. In the meantime, we've adjusted our code to coerce these sorted-maps to standard Clojure maps, so I won't claim that we're blocked on this clj-http change. My goal here is to document the concern and attempt to make a case for this change, but remain open-minded about the approach and probably learn something in the process 😄 Happy to discuss alternatives, concerns, etc. Thanks your ongoing maintenance and stewardship of clj-http @dakrone 😃 --- src/clj_http/cookies.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj_http/cookies.clj b/src/clj_http/cookies.clj index 917551e0..c307235c 100644 --- a/src/clj_http/cookies.clj +++ b/src/clj_http/cookies.clj @@ -22,7 +22,7 @@ (if (not (nil? (get m k))) (assoc newm k (get m k)) newm)) - (sorted-map) (sort (keys m)))) + {} (keys m))) (defn to-cookie "Converts a ClientCookie object into a tuple where the first item is From e3acd44c54d09eacc8b5ae127ec39e4ec5ad7db8 Mon Sep 17 00:00:00 2001 From: cavandavid Date: Mon, 3 Oct 2022 22:26:28 +0530 Subject: [PATCH 087/107] add documentation on proxy credentials (#622) --- README.org | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.org b/README.org index 01d7ff23..80fb242b 100644 --- a/README.org +++ b/README.org @@ -1304,6 +1304,12 @@ Additionally, per-request proxies can be specified with the =proxy-host= and (client/get "http://example.com" {:proxy-host "127.0.0.1" :proxy-port 8118}) #+END_SRC +Proxy credentials can also be explicitly set as + +#+BEGIN_SRC clojure +(client/get "http://example.com" {:proxy-host "127.0.0.1" :proxy-port 8118 :proxy-user "proxy-user" :proxy-pass "superSecurePassword"}) +#+END_SRC + You can also specify the =proxy-ignore-hosts= parameter with a list of hosts where the proxy should be ignored. By default this list is =#{"localhost" "127.0.0.1"}=. From 1cc8be62d258e79ce68d92f06f815458443240c2 Mon Sep 17 00:00:00 2001 From: The Alchemist Date: Tue, 8 Nov 2022 13:17:03 -0500 Subject: [PATCH 088/107] tiny capitalization fix (#624) --- README.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.org b/README.org index 80fb242b..50b98b01 100644 --- a/README.org +++ b/README.org @@ -79,7 +79,7 @@ There are branches for the major version numbers: - 2.x (no longer maintained except for security issues) - 3.x (current stable releases and the main Github branch) -- master (which is 4.x, unreleased, based on version 5 of the apache http client) +- master (which is 4.x, unreleased, based on version 5 of the Apache HTTP Client) * Introduction :PROPERTIES: From f11a8dad027714c413ab7c34b3553eea3d1dcf12 Mon Sep 17 00:00:00 2001 From: Sam Waggoner Date: Thu, 6 Apr 2023 10:01:20 -0500 Subject: [PATCH 089/107] Add transit to README optional deps. (#632) --- README.org | 1 + 1 file changed, 1 insertion(+) diff --git a/README.org b/README.org index 50b98b01..3235b2eb 100644 --- a/README.org +++ b/README.org @@ -1506,6 +1506,7 @@ adding them with the clj-http dependency in your project.clj: [crouton] ;; for :decode-body-headers [org.clojure/tools.reader] ;; for :as :clojure [ring/ring-codec] ;; for :as :x-www-form-urlencoded +[com.cognitect/transit-clj] ;; for transit support #+END_SRC Prior to 2.0.0, you can /exclude/ the dependencies and clj-http will work From 69be3c465cd85667ad6a4d84ca7225085ca60604 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 8 Jun 2023 15:30:53 -0600 Subject: [PATCH 090/107] Fix self-signed SSL test The exception we wanted to test against isn't visible any more, so this has been changed to a generic `Exception`. --- test/clj_http/test/core_test.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/clj_http/test/core_test.clj b/test/clj_http/test/core_test.clj index 2567d799..fc65247d 100644 --- a/test/clj_http/test/core_test.clj +++ b/test/clj_http/test/core_test.clj @@ -294,7 +294,7 @@ :keystore "test-resources/keystore" :key-password "keykey"})] (try - (is (thrown? SunCertPathBuilderException + (is (thrown? Exception (client/request {:scheme :https :server-name "localhost" :server-port 18082 From e31fba678f8b0f668461ce9f00be458048e0e292 Mon Sep 17 00:00:00 2001 From: Elliot Courant Date: Thu, 8 Jun 2023 16:32:43 -0500 Subject: [PATCH 091/107] feat(xml): Adding support for XML response body parsing. (#630) This adds support for parsing `text/xml` response bodies using the clojure.xml library. It also implements the auto multimethod for it. --- src/clj_http/client.clj | 29 +++++++++++++++++++++++++++++ test/clj_http/test/client_test.clj | 9 +++++++++ 2 files changed, 38 insertions(+) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index 8c38cf0c..f4c25ec7 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -11,10 +11,12 @@ [clojure.stacktrace :refer [root-cause]] [clojure.string :as str] [clojure.walk :refer [keywordize-keys prewalk]] + [clojure.xml :as xml] [slingshot.slingshot :refer [throw+]]) (:import [java.io BufferedReader ByteArrayInputStream ByteArrayOutputStream EOFException File InputStream] [java.net UnknownHostException URL] [org.apache.http.entity BufferedHttpEntity ByteArrayEntity FileEntity InputStreamEntity StringEntity] + [javax.xml.parsers SAXParserFactory] org.apache.http.impl.conn.PoolingHttpClientConnectionManager org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager)) @@ -472,6 +474,27 @@ (util/force-string body charset))] (assoc resp :body body))) +(defn- decode-xml-body [body] + (let [non-validating (fn [s ch] + (.. + (doto + (SAXParserFactory/newInstance) + (.setFeature + "http://apache.org/xml/features/nonvalidating/load-external-dtd" false)) + (newSAXParser) + (parse s ch)))] + (-> body + (util/force-stream) + (xml/parse non-validating)))) + +(defn coerce-xml-body + [request {:keys [body] :as resp} & [charset]] + (let [charset (or charset (response-charset resp)) + body (if (can-parse-body? request resp) + (decode-xml-body body) + (util/force-string body charset))] + (assoc resp :body body))) + (defn coerce-clojure-body [_request {:keys [body] :as resp}] (let [charset (response-charset resp) @@ -511,6 +534,9 @@ (defmethod coerce-content-type :application/json [req resp] (coerce-json-body req resp true false)) +(defmethod coerce-content-type :text/xml [req resp] + (coerce-xml-body req resp false)) + (defmethod coerce-content-type :application/transit+json [req resp] (coerce-transit-body req resp :json)) @@ -546,6 +572,9 @@ (defmethod coerce-response-body :json-strict-string-keys [req resp] (coerce-json-body req resp false)) +(defmethod coerce-response-body :xml [req resp] + (coerce-xml-body req resp false)) + (defmethod coerce-response-body :clojure [req resp] (coerce-clojure-body req resp)) diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index d2c2c59b..dd7290a1 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -1539,6 +1539,8 @@ (deftest t-coercion-methods (let [json-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"bar\"}")) json-ms949-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"안뇽\"}" "MS949")) + xml-body (ByteArrayInputStream. (.getBytes "bar")) + xml-auto-body (ByteArrayInputStream. (.getBytes "bar")) auto-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"bar\"}")) edn-body (ByteArrayInputStream. (.getBytes "{:foo \"bar\"}")) transit-json-body (ByteArrayInputStream. @@ -1554,6 +1556,10 @@ :headers {"content-type" "application/json"}} json-ms949-resp {:body json-ms949-body :status 200 :headers {"content-type" "application/json; charset=ms949"}} + xml-resp {:body xml-body :status 200 + :headers {"content-type" "text/xml;charset=utf-8"}} + xml-auto-resp {:body xml-auto-body :status 200 + :headers {"content-type" "text/xml;charset=utf-8"}} auto-resp {:body auto-body :status 200 :headers {"content-type" "application/json"}} edn-resp {:body edn-body :status 200 @@ -1583,6 +1589,9 @@ auto-www-form-urlencoded-resp)) (:body (client/coerce-response-body {:as :x-www-form-urlencoded} www-form-urlencoded-resp)))) + (is (= {:tag :foo, :attrs nil, :content ["bar"]} + (:body (client/coerce-response-body {:as :xml} xml-resp)) + (:body (client/coerce-response-body {:as :auto} xml-auto-resp)))) (is (= {:foo "안뇽"} (:body (client/coerce-response-body {:as :json+ms949} json-ms949-resp)))) From d92be158230e8094436f415324d96f2bd7cf95f7 Mon Sep 17 00:00:00 2001 From: Sam Waggoner Date: Sun, 1 Oct 2023 12:08:36 -0500 Subject: [PATCH 092/107] Specify unit is seconds in with-connection-pool docstring. (#637) --- src/clj_http/client.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index f4c25ec7..c6dcfbb0 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -1294,7 +1294,7 @@ The following options are supported: - :timeout - Time that connections are left open before automatically closing + :timeout - Time in seconds that connections are left open before automatically closing default: 5 :threads - Maximum number of threads that will be used for connecting default: 4 From 42d3a3887bd13d6ec5bf943428f7f29991895512 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 11 Apr 2024 11:05:37 -0600 Subject: [PATCH 093/107] Release 3.12.4 --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 52f6afe0..6589b87f 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.12.4-SNAPSHOT" +(defproject clj-http "3.12.4" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" @@ -25,7 +25,7 @@ [com.cognitect/transit-clj "1.0.324"] [ring/ring-codec "1.1.3"] ;; other (testing) deps - [org.clojure/clojure "1.10.3"] + [org.clojure/clojure "1.11.1"] [org.clojure/tools.logging "1.1.0"] [ring/ring-jetty-adapter "1.9.3"] [ring/ring-devel "1.9.3"] From ccc316b6c31548e0eb1d8b0d51e7e83c74fc8735 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 11 Apr 2024 11:07:18 -0600 Subject: [PATCH 094/107] Bump to 3.12.5-SNAPSHOT and update readme --- README.org | 5 ++++- project.clj | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index 3235b2eb..f14cf347 100644 --- a/README.org +++ b/README.org @@ -119,7 +119,7 @@ function. With Leiningen/Boot: #+BEGIN_SRC clojure -[clj-http "3.12.3"] +[clj-http "3.12.4"] #+END_SRC If you need an older version, a 2.x release is also available. @@ -1444,6 +1444,9 @@ In previous versions of =clj-http= (<= 3.10.0), =clj-http= defaulted to lazily p was slow and also confused users who didn't expect laziness. ** DNS Resolution +:PROPERTIES: +:CUSTOM_ID: h:52CC15DF-57A5-425E-9AFC-10C9B4C4FA83 +:END: Users may add their own DNS resolver function to override the default DNS Resolver. This is useful in situations where you are unable to change the name to IP Address mapping. It is analogous to the =--resolve= flag present in =curl=. This example uses =org.apache.http.impl.conn.InMemoryDnsResolver= to resolve =example.com= to IP Address =127.0.0.1=. diff --git a/project.clj b/project.clj index 6589b87f..d35c7f0b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.12.4" +(defproject clj-http "3.12.5-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 0a201c8c1284c6f943816948795de7d6c5622782 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 11 Apr 2024 11:14:19 -0600 Subject: [PATCH 095/107] Bump all dependencies to the latest --- project.clj | 43 +++++++++++++++-------------- test/clj_http/test/headers_test.clj | 7 ++--- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/project.clj b/project.clj index d35c7f0b..6389a982 100644 --- a/project.clj +++ b/project.clj @@ -7,34 +7,35 @@ :global-vars {*warn-on-reflection* false} :min-lein-version "2.0.0" :exclusions [org.clojure/clojure] - :dependencies [[org.apache.httpcomponents/httpcore "4.4.14"] - [org.apache.httpcomponents/httpclient "4.5.13"] - [org.apache.httpcomponents/httpclient-cache "4.5.13"] - [org.apache.httpcomponents/httpasyncclient "4.1.4"] - [org.apache.httpcomponents/httpmime "4.5.13"] - [commons-codec "1.15"] - [commons-io "2.8.0"] + :dependencies [[org.apache.httpcomponents/httpcore "4.4.16"] + [org.apache.httpcomponents/httpclient "4.5.14"] + [org.apache.httpcomponents/httpclient-cache "4.5.14"] + [org.apache.httpcomponents/httpasyncclient "4.1.5"] + [org.apache.httpcomponents/httpmime "4.5.14"] + [commons-codec "1.16.1"] + [commons-io "2.16.1"] [slingshot "0.12.2"] - [potemkin "0.4.5"]] + [potemkin "0.4.7"]] :resource-paths ["resources"] :profiles {:dev {:dependencies [;; optional deps - [cheshire "5.10.0"] + [cheshire "5.13.0"] [crouton "0.1.2" :exclusions [[org.jsoup/jsoup]]] - [org.jsoup/jsoup "1.13.1"] - [org.clojure/tools.reader "1.3.5"] - [com.cognitect/transit-clj "1.0.324"] - [ring/ring-codec "1.1.3"] + [org.jsoup/jsoup "1.17.2"] + [org.clojure/tools.reader "1.4.1"] + [com.cognitect/transit-clj "1.0.333"] + [ring/ring-codec "1.2.0"] ;; other (testing) deps - [org.clojure/clojure "1.11.1"] - [org.clojure/tools.logging "1.1.0"] - [ring/ring-jetty-adapter "1.9.3"] - [ring/ring-devel "1.9.3"] + [org.clojure/clojure "1.11.2"] + [org.clojure/tools.logging "1.3.0"] + [ring/ring-jetty-adapter "1.12.1"] + [ring/ring-devel "1.12.1"] + [javax.servlet/javax.servlet-api "4.0.1"] ;; caching example deps - [org.clojure/core.cache "1.0.207"] + [org.clojure/core.cache "1.1.234"] ;; logging - [org.apache.logging.log4j/log4j-api "2.17.1"] - [org.apache.logging.log4j/log4j-core "2.17.1"] - [org.apache.logging.log4j/log4j-1.2-api "2.17.1"]] + [org.apache.logging.log4j/log4j-api "2.23.1"] + [org.apache.logging.log4j/log4j-core "2.23.1"] + [org.apache.logging.log4j/log4j-1.2-api "2.23.1"]] :plugins [[lein-ancient "0.7.0"] [jonase/eastwood "0.2.5"] [lein-kibit "0.1.5"] diff --git a/test/clj_http/test/headers_test.clj b/test/clj_http/test/headers_test.clj index a2c5e835..0732f18c 100644 --- a/test/clj_http/test/headers_test.clj +++ b/test/clj_http/test/headers_test.clj @@ -3,8 +3,7 @@ [clj-http.headers :refer :all] [clj-http.util :refer [lower-case-keys]] [clojure.test :refer :all]) - (:import [javax.servlet.http HttpServletRequest HttpServletResponse] - [org.eclipse.jetty.server Request Server] + (:import [org.eclipse.jetty.server Request Server] org.eclipse.jetty.server.handler.AbstractHandler)) (deftest test-special-case @@ -90,8 +89,8 @@ (.setHandler (proxy [AbstractHandler] [] (handle [target ^Request base-request - ^HttpServletRequest request - ^HttpServletResponse response] + request + response] (.setHandled base-request true) (.setStatus response 200) ;; copy over request headers verbatim From 9e5e07baad66b7cbb59f18f5c2b93e14291482f2 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 11 Apr 2024 11:16:25 -0600 Subject: [PATCH 096/107] Drop support for Clojure 1.6.x and 1.7.x in 3.13.x --- README.org | 3 ++- project.clj | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.org b/README.org index f14cf347..7fb42c51 100644 --- a/README.org +++ b/README.org @@ -128,7 +128,8 @@ If you need an older version, a 2.x release is also available. [clj-http "2.3.0"] #+END_SRC -clj-http 3.x supports clojure 1.6.0 and higher. +clj-http 3.12.x supports clojure 1.6.0 and higher. +clj-http 3.13.x supports clojure 1.8.0 and higher. clj-http 4.x will support clojure 1.7.0 and higher. * Quickstart diff --git a/project.clj b/project.clj index 6389a982..7ec4bf34 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.12.5-SNAPSHOT" +(defproject clj-http "3.13.0-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" @@ -40,12 +40,10 @@ [jonase/eastwood "0.2.5"] [lein-kibit "0.1.5"] [lein-nvd "0.5.2"]]} - :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} - :1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} :1.8 {:dependencies [[org.clojure/clojure "1.8.0"]]} :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]} :1.10 {:dependencies [[org.clojure/clojure "1.10.1"]]}} - :aliases {"all" ["with-profile" "dev,1.6:dev,1.7:dev,1.8:dev,1.9:dev,1.10:dev"]} + :aliases {"all" ["with-profile" "dev,1.8:dev,1.9:dev,1.10:dev"]} :plugins [[codox "0.6.4"]] :test-selectors {:default #(not (:integration %)) :integration :integration From 6d5d7b6bc2809773e47e2a493e8cd5bb96048719 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 11 Apr 2024 11:17:13 -0600 Subject: [PATCH 097/107] Release 3.13.0 --- README.org | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index 7fb42c51..a8cc7ed7 100644 --- a/README.org +++ b/README.org @@ -119,7 +119,7 @@ function. With Leiningen/Boot: #+BEGIN_SRC clojure -[clj-http "3.12.4"] +[clj-http "3.13.0"] #+END_SRC If you need an older version, a 2.x release is also available. diff --git a/project.clj b/project.clj index 7ec4bf34..3030dd43 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.13.0-SNAPSHOT" +(defproject clj-http "3.13.0" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From 5158b4e3349ac6a26db5cace5cd5a4e4226f18c0 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 11 Apr 2024 11:18:01 -0600 Subject: [PATCH 098/107] Bump to 3.13.1-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 3030dd43..cda9996b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.13.0" +(defproject clj-http "3.13.1-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From b8e494725258940b1707e0e1413b056e613cfc2d Mon Sep 17 00:00:00 2001 From: Matthieu Sprunck Date: Mon, 15 Apr 2024 21:05:44 +0200 Subject: [PATCH 099/107] Avoid a NPE when the charset in a content type is empty (#640) * NPE with an empty charset in the content type * Avoid the NPE when the charset is empty --- src/clj_http/util.clj | 3 ++- test/clj_http/test/util_test.clj | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/clj_http/util.clj b/src/clj_http/util.clj index f45ded64..2b14a81a 100644 --- a/src/clj_http/util.clj +++ b/src/clj_http/util.clj @@ -144,7 +144,8 @@ (or v1 v2))))) (defn- trim-quotes [s] - (clojure.string/replace s #"^\s*(\"(.*)\"|(.*?))\s*$" "$2$3")) + (when s + (clojure.string/replace s #"^\s*(\"(.*)\"|(.*?))\s*$" "$2$3"))) (defn parse-content-type "Parse `s` as an RFC 2616 media type." diff --git a/test/clj_http/test/util_test.clj b/test/clj_http/test/util_test.clj index e886c4f5..6ee75bd2 100644 --- a/test/clj_http/test/util_test.clj +++ b/test/clj_http/test/util_test.clj @@ -46,7 +46,10 @@ :content-type-params {:charset "utf-8"}} "text/html; charset=ISO-8859-4" {:content-type :text/html - :content-type-params {:charset "ISO-8859-4"}})) + :content-type-params {:charset "ISO-8859-4"}} + "text/html; charset=" + {:content-type :text/html + :content-type-params {:charset nil}})) (deftest test-force-byte-array (testing "empty InputStream returns empty byte-array" From e7060e4df82b5159e290fffae1d24caa90f8096b Mon Sep 17 00:00:00 2001 From: Andrew David Nimmo <100512+andrewnimmo@users.noreply.github.com> Date: Thu, 8 Aug 2024 20:30:42 +0200 Subject: [PATCH 100/107] Correct documentation link (#648) Correct the link to the ring-clojure SPEC document. --- README.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.org b/README.org index a8cc7ed7..a0a66677 100644 --- a/README.org +++ b/README.org @@ -103,7 +103,7 @@ The design of =clj-http= is inspired by the [[https://github.com/ring-clojure/ri server applications. The client in =clj-http.core= makes HTTP requests according to a given Ring -request map and returns [[https://github.com/ring-clojure/ring/blob/master/SPEC][Ring response maps]] corresponding to the resulting HTTP +request map and returns [[https://github.com/ring-clojure/ring/blob/master/SPEC.md][Ring response maps]] corresponding to the resulting HTTP response. The function =clj-http.client/request= uses Ring-style middleware to layer functionality over the core HTTP request/response implementation. Methods like =clj-http.client/get= are sugar over this =clj-http.client/request= From ab3413452551a768433de8f65e23bdfb50c52144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?KARASZI=20Istv=C3=A1n?= Date: Sat, 17 Aug 2024 11:34:48 +0200 Subject: [PATCH 101/107] Fix reflection warning on javax.xml.parsers.SAXParser --- src/clj_http/client.clj | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj index c6dcfbb0..7d01d886 100644 --- a/src/clj_http/client.clj +++ b/src/clj_http/client.clj @@ -16,7 +16,8 @@ (:import [java.io BufferedReader ByteArrayInputStream ByteArrayOutputStream EOFException File InputStream] [java.net UnknownHostException URL] [org.apache.http.entity BufferedHttpEntity ByteArrayEntity FileEntity InputStreamEntity StringEntity] - [javax.xml.parsers SAXParserFactory] + [javax.xml.parsers SAXParser SAXParserFactory] + org.xml.sax.helpers.DefaultHandler org.apache.http.impl.conn.PoolingHttpClientConnectionManager org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager)) @@ -474,18 +475,25 @@ (util/force-string body charset))] (assoc resp :body body))) +(defn- sax-parser ^SAXParser [] + (.. + (doto + (SAXParserFactory/newInstance) + (.setFeature + "http://apache.org/xml/features/nonvalidating/load-external-dtd" false)) + (newSAXParser))) + +(defn- non-validating [s ^DefaultHandler ch] + (let [parser (sax-parser)] + (cond + (instance? String s) (.parse parser ^String s ch) + (instance? InputStream s) (.parse parser ^InputStream s ch) + :else (throw (ex-info "Unsupported input" {:s s}))))) + (defn- decode-xml-body [body] - (let [non-validating (fn [s ch] - (.. - (doto - (SAXParserFactory/newInstance) - (.setFeature - "http://apache.org/xml/features/nonvalidating/load-external-dtd" false)) - (newSAXParser) - (parse s ch)))] - (-> body - (util/force-stream) - (xml/parse non-validating)))) + (-> body + (util/force-stream) + (xml/parse non-validating))) (defn coerce-xml-body [request {:keys [body] :as resp} & [charset]] @@ -892,7 +900,6 @@ ([req respond raise] (client (oauth-request req) respond raise)))) - (defn parse-user-info [user-info] (when user-info (str/split user-info #":"))) From 3ec25c5edf1e72c848ec2f27c06e30c1e9f7fa7f Mon Sep 17 00:00:00 2001 From: Toby Crawley Date: Mon, 19 Aug 2024 15:43:34 -0400 Subject: [PATCH 102/107] Update redir strategy javadoc links (#649) The existing links are no longer resolvable, and 404. --- README.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index a0a66677..4e8e4a4e 100644 --- a/README.org +++ b/README.org @@ -717,8 +717,8 @@ Redirect Options: list of redirected URLs with key: =:trace-redirects=. - =:redirect-strategy= :: Sets the redirect strategy for clj-http. Accepts the following: - =:none= - Perform no redirects - - =:default= - See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/client/DefaultRedirectStrategy.html - - =:lax= - See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/client/LaxRedirectStrategy.html + - =:default= - See https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/client/DefaultRedirectStrategy.html + - =:lax= - See https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/client/LaxRedirectStrategy.html - =:graceful= - Similar to =:default=, but does not throw exceptions when max redirects is reached. This is the redirects behaviour in 2.x - =nil= - When nil, assumes =:default= From 37d906287d57d54269bf76f6496aa93f7b6ad6de Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Mon, 19 Aug 2024 13:47:49 -0600 Subject: [PATCH 103/107] Update Clojure workflows --- .github/workflows/clojure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clojure.yml b/.github/workflows/clojure.yml index f2a6e92d..46aafe4b 100644 --- a/.github/workflows/clojure.yml +++ b/.github/workflows/clojure.yml @@ -11,8 +11,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: ["8", "11", "14"] - clojure: ["1.6", "1.7", "1.8", "1.9", "1.10"] + java: ["14", "17", "21"] + clojure: ["1.8", "1.9", "1.10", "1.11"] name: Java ${{ matrix.java }} Clojure ${{ matrix.clojure }} steps: From ef90f4e8d0ef44448d350730ced9683f16a8543a Mon Sep 17 00:00:00 2001 From: Mark Sto Date: Wed, 18 Jun 2025 20:13:37 +0400 Subject: [PATCH 104/107] Update the `project.clj` file (#655) * Fix an issue with Clojure dependency leading to inability to run REPL or tests with accordance to CONTRIBUTING.md * Cover all latest Clojure versions in the deps matrix * Add a missing SLF4J bridge dependency (eternal classics) SLF4J: No SLF4J providers were found. SLF4J: Defaulting to no-operation (NOP) logger implementation SLF4J: See https://www.slf4j.org/codes.html#noProviders for further details. * Do not mention supported Clojure versions explicitly --- project.clj | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/project.clj b/project.clj index cda9996b..8251a5c8 100644 --- a/project.clj +++ b/project.clj @@ -6,7 +6,6 @@ :distribution :repo} :global-vars {*warn-on-reflection* false} :min-lein-version "2.0.0" - :exclusions [org.clojure/clojure] :dependencies [[org.apache.httpcomponents/httpcore "4.4.16"] [org.apache.httpcomponents/httpclient "4.5.14"] [org.apache.httpcomponents/httpclient-cache "4.5.14"] @@ -25,7 +24,7 @@ [com.cognitect/transit-clj "1.0.333"] [ring/ring-codec "1.2.0"] ;; other (testing) deps - [org.clojure/clojure "1.11.2"] + [org.clojure/clojure "1.12.1"] [org.clojure/tools.logging "1.3.0"] [ring/ring-jetty-adapter "1.12.1"] [ring/ring-devel "1.12.1"] @@ -35,16 +34,18 @@ ;; logging [org.apache.logging.log4j/log4j-api "2.23.1"] [org.apache.logging.log4j/log4j-core "2.23.1"] - [org.apache.logging.log4j/log4j-1.2-api "2.23.1"]] + [org.apache.logging.log4j/log4j-1.2-api "2.23.1"] + [org.apache.logging.log4j/log4j-slf4j2-impl "2.23.1"]] :plugins [[lein-ancient "0.7.0"] [jonase/eastwood "0.2.5"] [lein-kibit "0.1.5"] [lein-nvd "0.5.2"]]} :1.8 {:dependencies [[org.clojure/clojure "1.8.0"]]} :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]} - :1.10 {:dependencies [[org.clojure/clojure "1.10.1"]]}} - :aliases {"all" ["with-profile" "dev,1.8:dev,1.9:dev,1.10:dev"]} + :1.10 {:dependencies [[org.clojure/clojure "1.10.3"]]} + :1.11 {:dependencies [[org.clojure/clojure "1.11.4"]]}} + :aliases {"all" ["with-profile" "dev,1.8:dev,1.9:dev,1.10:dev,1.11:dev"]} :plugins [[codox "0.6.4"]] - :test-selectors {:default #(not (:integration %)) + :test-selectors {:default #(not (:integration %)) :integration :integration :all (constantly true)}) From b07de01d4c79c33018fb14f7a88de0fa335f2d74 Mon Sep 17 00:00:00 2001 From: Mark Sto Date: Wed, 18 Jun 2025 20:15:06 +0400 Subject: [PATCH 105/107] Drop a "Content-Type" header, if any, for multipart requests (#654) * Drop a "Content-Type" header, if any, for multipart requests * Add test coverage to avoid regression (this one is subtle) --------- Co-authored-by: Lee Hinman --- src/clj_http/core.clj | 5 ++- test/clj_http/test/client_test.clj | 63 +++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj index 2ed4ea11..7cc54785 100644 --- a/src/clj_http/core.clj +++ b/src/clj_http/core.clj @@ -609,7 +609,10 @@ (if (string? body) (StringEntity. ^String body "UTF-8") (ByteArrayEntity. body)))))) - (doseq [[header-n header-v] headers] + (doseq [[header-n header-v] headers + :when (or (not multipart) + (and (not= "content-type" header-n) + (not= "Content-Type" header-n)))] (if (coll? header-v) (doseq [header-vth header-v] (.addHeader http-req header-n header-vth)) diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj index dd7290a1..1b111e99 100644 --- a/test/clj_http/test/client_test.clj +++ b/test/clj_http/test/client_test.clj @@ -4,7 +4,7 @@ [clj-http.conn-mgr :as conn] [clj-http.test.core-test :refer [run-server]] [clj-http.util :as util] - [clojure.java.io :refer [resource]] + [clojure.java.io :as io :refer [resource]] [clojure.string :as str] [clojure.test :refer :all] [cognitect.transit :as transit] @@ -16,6 +16,7 @@ java.io.PipedOutputStream java.net.UnknownHostException org.apache.http.HttpEntity + org.apache.http.HttpMessage org.apache.logging.log4j.LogManager)) (defonce logger (LogManager/getLogger "clj-http.test.client-test")) @@ -146,30 +147,37 @@ "Verify that we went through the failure path, not the success") (is (= @fail-p expected-var *test-dynamic-var*)))))))) +(defn retrieve-http-request-content-type-header + [response] + (let [http-req (get-in response [:request :http-req])] + (->> (.getAllHeaders ^HttpMessage http-req) + (map str) + (some #(when (str/starts-with? (str/lower-case %) "content-type") %))))) + (deftest ^:integration multipart-async (run-server) - (let [resp (promise) - exception (promise) - _ (request {:uri "/post" :method :post - :async? true - :multipart [{:name "title" :content "some-file"} - {:name "Content/Type" :content "text/plain"} - {:name "file" - :content (clojure.java.io/file - "test-resources/m.txt")}]} - resp - exception - )] - (is (= 200 (:status @resp))) - (is (not (realized? exception))) - #_(when (realized? exception) (prn @exception))) + + (testing "basics" + (let [resp (promise) + exception (promise) + _ (request {:uri "/post" :method :post + :async? true + :multipart [{:name "title" :content "some-file"} + {:name "Content/Type" :content "text/plain"} + {:name "file" + :content (io/file "test-resources/m.txt")}]} + resp + exception)] + (is (= 200 (:status (deref resp 500 :failed)))) + (is (not (realized? exception))) + #_(when (realized? exception) (prn @exception)))) ;; Regression Testing https://github.com/dakrone/clj-http/issues/560 (testing "multipart uploads larger than 25kb" (let [resp (promise) exception (promise) ;; assumption: file > 5kb - file (clojure.java.io/file "test-resources/big_array_json.json") + file (io/file "test-resources/big_array_json.json") _ (request {:uri "/post" :method :post :async? true @@ -181,7 +189,26 @@ resp exception)] (is (= 200 (:status (deref resp 500 :failed)))) - (is (not (realized? exception)))))) + (is (not (realized? exception))))) + + ;; Find the details in https://github.com/dakrone/clj-http/pull/654 + (testing "existing \"Content-Type\" request header is discarded" + (let [resp (request {:uri "/post" :method :post + :headers {"content-type" "multipart/form-data"} + :multipart [{:name "some" :content "thing"}] + :save-request? true}) + content-type (retrieve-http-request-content-type-header resp)] + (is (= 200 (:status resp))) + (is (not= "multipart/form-data" content-type)) + (is (nil? content-type))) + (let [resp (request {:uri "/post" :method :post + :headers {"Content-Type" "multipart/form-data"} + :multipart [{:name "some" :content "thing"}] + :save-request? true}) + content-type (retrieve-http-request-content-type-header resp)] + (is (= 200 (:status resp))) + (is (not= "multipart/form-data" content-type)) + (is (nil? content-type))))) (deftest ^:integration nil-input (is (thrown-with-msg? Exception #"Host URL cannot be nil" From 30950cd83fb8a7f19fb53c93a92d5dce9882526f Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 26 Jun 2025 12:34:47 -0600 Subject: [PATCH 106/107] Release 3.13.1 --- README.org | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.org b/README.org index 4e8e4a4e..3681fbd5 100644 --- a/README.org +++ b/README.org @@ -119,7 +119,7 @@ function. With Leiningen/Boot: #+BEGIN_SRC clojure -[clj-http "3.13.0"] +[clj-http "3.13.1"] #+END_SRC If you need an older version, a 2.x release is also available. diff --git a/project.clj b/project.clj index 8251a5c8..85fa18af 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.13.1-SNAPSHOT" +(defproject clj-http "3.13.1" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License" From cfcc70e435bc753b56f258348932a850dc4c6d09 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 26 Jun 2025 12:38:03 -0600 Subject: [PATCH 107/107] Bump to 3.13.2-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 85fa18af..a0c875f4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-http "3.13.1" +(defproject clj-http "3.13.2-SNAPSHOT" :description "A Clojure HTTP library wrapping the Apache HttpComponents client." :url "https://github.com/dakrone/clj-http/" :license {:name "The MIT License"