diff --git a/.github/workflows/clojure.yml b/.github/workflows/clojure.yml
new file mode 100644
index 00000000..46aafe4b
--- /dev/null
+++ b/.github/workflows/clojure.yml
@@ -0,0 +1,39 @@
+name: Clojure CI
+
+on:
+ push:
+ branches: [ 3.x ]
+ pull_request:
+ branches: [ 3.x ]
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ java: ["14", "17", "21"]
+ clojure: ["1.8", "1.9", "1.10", "1.11"]
+
+ name: Java ${{ matrix.java }} Clojure ${{ matrix.clojure }}
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/cache@v2
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-lein-${{ hashFiles('**/project.clj') }}
+ restore-keys: |
+ ${{ runner.os }}-lein-
+
+ - name: Setup java
+ uses: actions/setup-java@v1
+ with:
+ java-version: ${{ matrix.java }}
+
+ - name: Install dependencies
+ run: lein deps
+
+ - name: Run tests
+ run: lein with-profile dev,${{matrix.clojure}} test :all
+
+ - name: Check Reflection Warnings
+ run: '! lein with-profile dev,${{matrix.clojure}} check 2>&1 | egrep "Reflection warning|Performance warning"'
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index cba283ec..00000000
--- a/.travis.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-language: clojure
-lein: lein
-script: lein all do clean, test
-branches:
- only:
- - 3.x
- # 'master' is the new apache 5 client, and not passing yet
- # - master
-jdk:
- - openjdk7
- - oraclejdk7
- - oraclejdk8
diff --git a/README.org b/README.org
index 67a344bb..3681fbd5 100644
--- a/README.org
+++ b/README.org
@@ -7,16 +7,13 @@
#+HTML_HEAD:
#+LANGUAGE: en
-* Table of Contents :TOC:
+[[https://clojars.org/clj-http][file:https://img.shields.io/clojars/v/clj-http.svg]] [[https://github.com/dakrone/clj-http/actions?query=workflow%3A%22Clojure+CI%22][file:https://github.com/dakrone/clj-http/workflows/Clojure%20CI/badge.svg]] [[https://gitter.im/clj-http/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge][file:https://badges.gitter.im/clj-http/Lobby.svg]]
+
+* Table of Contents :TOC_3:
:PROPERTIES:
:CUSTOM_ID: h-aaf075ea-2f0e-4a45-871a-0f89c838fb4b
:END:
-
-[[https://secure.travis-ci.org/dakrone/clj-http.png]]
-
-#+ATTR_HTML: title="Join the chat at https://gitter.im/clj-http/Lobby"
-[[https://gitter.im/clj-http/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge][file:https://badges.gitter.im/clj-http/Lobby.svg]]
-
+- [[#branches][Branches]]
- [[#introduction][Introduction]]
- [[#overview][Overview]]
- [[#philosophy][Philosophy]]
@@ -37,12 +34,14 @@
- [[#meta-tag-headers][Meta Tag Headers]]
- [[#link-headers][Link Headers]]
- [[#redirects][Redirects]]
+ - [[#how-to-create-a-custom-redirectstrategy][How to create a custom RedirectStrategy]]
- [[#cookies][Cookies]]
- [[#cookiestores][Cookiestores]]
- [[#keystores-trust-stores][Keystores, Trust-stores]]
- [[#exceptions][Exceptions]]
- [[#decompression][Decompression]]
- [[#debugging][Debugging]]
+ - [[#logging][Logging]]
- [[#caching][Caching]]
- [[#authentication][Authentication]]
- [[#basic-auth][Basic Auth]]
@@ -53,8 +52,12 @@
- [[#raw-request][Raw Request]]
- [[#boolean-options][Boolean options]]
- [[#persistent-connections][Persistent Connections]]
+ - [[#re-using-httpclient-between-requests][Re-using =HttpClient= between requests]]
- [[#proxies][Proxies]]
- [[#custom-middleware][Custom Middleware]]
+ - [[#modifying-apache-specific-features-of-the-httpclientbuilder-and-httpasyncclientbuilder][Modifying Apache-specific features of the =HttpClientBuilder= and =HttpAsyncClientBuilder=]]
+ - [[#incrementally-json-parsing][Incrementally JSON Parsing]]
+ - [[#dns-resolution][DNS Resolution]]
- [[#development][Development]]
- [[#faking-responses][Faking Responses]]
- [[#optional-dependencies][Optional Dependencies]]
@@ -64,10 +67,9 @@
- [[#nohttpresponseexception--due-to-stale-connections][NoHttpResponseException ... due to stale connections**]]
- [[#tests][Tests]]
- [[#testimonials][Testimonials]]
-- [[#other-middleware][Other Middleware]]
+- [[#other-libraries-providing-middleware][Other Libraries Providing Middleware]]
- [[#license][License]]
-
* Branches
:PROPERTIES:
:CUSTOM_ID: h-e390585c-cbd8-4e94-b36b-4e9c27c16720
@@ -77,7 +79,7 @@ There are branches for the major version numbers:
- 2.x (no longer maintained except for security issues)
- 3.x (current stable releases and the main Github branch)
-- master (which is 4.x, unreleased, based on version 5 of the apache http client)
+- master (which is 4.x, unreleased, based on version 5 of the Apache HTTP Client)
* Introduction
:PROPERTIES:
@@ -101,7 +103,7 @@ The design of =clj-http= is inspired by the [[https://github.com/ring-clojure/ri
server applications.
The client in =clj-http.core= makes HTTP requests according to a given Ring
-request map and returns [[https://github.com/ring-clojure/ring/blob/master/SPEC][Ring response maps]] corresponding to the resulting HTTP
+request map and returns [[https://github.com/ring-clojure/ring/blob/master/SPEC.md][Ring response maps]] corresponding to the resulting HTTP
response. The function =clj-http.client/request= uses Ring-style middleware to
layer functionality over the core HTTP request/response implementation. Methods
like =clj-http.client/get= are sugar over this =clj-http.client/request=
@@ -117,7 +119,7 @@ function.
With Leiningen/Boot:
#+BEGIN_SRC clojure
-[clj-http "3.9.1"]
+[clj-http "3.13.1"]
#+END_SRC
If you need an older version, a 2.x release is also available.
@@ -126,7 +128,8 @@ If you need an older version, a 2.x release is also available.
[clj-http "2.3.0"]
#+END_SRC
-clj-http 3.x supports clojure 1.6.0 and higher.
+clj-http 3.12.x supports clojure 1.6.0 and higher.
+clj-http 3.13.x supports clojure 1.8.0 and higher.
clj-http 4.x will support clojure 1.7.0 and higher.
* Quickstart
@@ -177,7 +180,9 @@ Example requests:
(client/get "http://example.com/resources/id")
+;; Setting options
(client/get "http://example.com/resources/3" {:accept :json})
+(client/get "http://example.com/resources/3" {:accept :json :query-params {"q" "foo, bar"}})
;; Specifying headers as either a string or collection:
(client/get "http://example.com"
@@ -187,12 +192,6 @@ Example requests:
(client/get "http://example.com"
{:headers {:foo ["bar" "baz"], :eggplant "quux"}})
-;; Set any specific client parameters manually:
-(client/post "http://example.com"
- {:client-params {"http.protocol.allow-circular-redirects" false
- "http.protocol.version" HttpVersion/HTTP_1_0
- "http.useragent" "clj-http"}})
-
;; Completely ignore cookies:
(client/post "http://example.com" {:cookie-policy :none})
;; There are also multiple ways to handle cookies
@@ -268,7 +267,7 @@ Example requests:
(client/get "http://example.com/redirects-somewhere" {:max-redirects 5 :redirect-strategy :graceful})
;; Throw an exception if the get takes too long. Timeouts in milliseconds.
-(client/get "http://example.com/redirects-somewhere" {:socket-timeout 1000 :conn-timeout 1000})
+(client/get "http://example.com/redirects-somewhere" {:socket-timeout 1000 :connection-timeout 1000})
;; Query parameters
(client/get "http://example.com/search" {:query-params {"q" "foo, bar"}})
@@ -325,8 +324,8 @@ content encodings.
:body "{\"json\": \"input\"}"
:headers {"X-Api-Version" "2"}
:content-type :json
- :socket-timeout 1000 ;; in milliseconds
- :conn-timeout 1000 ;; in milliseconds
+ :socket-timeout 1000 ;; in milliseconds
+ :connection-timeout 1000 ;; in milliseconds
:accept :json})
;; Send form params as a urlencoded body (POST or PUT)
@@ -382,6 +381,15 @@ content encodings.
:retry-handler (fn [ex try-count http-context]
(println "Got:" ex)
(if (> try-count 4) false true))})
+
+;; to handle a file with non-ascii filename, try :multipart-charset "UTF-8" and :multipart-mode BROWSER_COMPATIBLE
+;; see also: https://stackoverflow.com/questions/3393445/international-characters-in-filename-in-mutipart-formdata
+(import (org.apache.http.entity.mime HttpMultipartMode))
+
+(client/post "http://example.org" {:multipart [{:content (clojure.java.io/file "日本語.txt")}]
+ :multipart-mode HttpMultipartMode/BROWSER_COMPATIBLE
+ :multipart-charset "UTF-8"} )
+
#+END_SRC
A word about flattening nested =:query-params= and =:form-params= maps. There are essentially three
@@ -413,8 +421,8 @@ different ways to handle flattening them:
:CUSTOM_ID: h-0e3eb987-5b2b-4874-97ef-b834394d083d
:END:
The new async HTTP request API is a Ring-style async API.
-All options for synchronous request can use in asynchronous requests.
-start an async request is easy, for example:
+All options for synchronous requests can be used in asynchronous requests.
+Starting an async request is easy, for example:
#+BEGIN_SRC clojure
;; :async? in options map need to be true
@@ -428,7 +436,7 @@ start an async request is easy, for example:
All exceptions thrown during the request will be passed to the raise callback.
-*** Cancelling requests
+*** Cancelling Requests
:PROPERTIES:
:CUSTOM_ID: cancelling-requests
:END:
@@ -504,9 +512,7 @@ it's received (output coercion) from the server.
;; Coerce as json
(client/get "http://example.com/foo.json" {:as :json})
-(client/get "http://example.com/foo.json" {:as :json-strict})
(client/get "http://example.com/foo.json" {:as :json-string-keys})
-(client/get "http://example.com/foo.json" {:as :json-strict-string-keys})
;; Coerce as Transit encoded JSON or MessagePack
(client/get "http://example.com/foo" {:as :transit+json})
@@ -529,19 +535,29 @@ it's received (output coercion) from the server.
(client/get "http://example.com/bigrequest.html" {:as :stream})
;; Note that the connection to the server will NOT be closed until the
;; stream has been read
+
+;; Return the body as a java.io.BufferedReader
+(client/get "http://example.com/bigrequest.html" {:as :reader})
+;; As above, the connection will remain open until the stream has been
+;; read. The reader will attempt to respect the server-specified charset,
+;; if any, defaulting to UTF-8.
#+END_SRC
-Output coercion with =:as :json=, =:as :json-strict=, =:as :json-strict-string-keys=, =:as :json-string-keys= or =:as :x-www-form-urlencoded= will only work with an optional dependency, see [[#optional-dependencies][Optional Dependencies]].
+Output coercion with =:as :json=, =:as :json-string-keys= or =:as :x-www-form-urlencoded=, will only work with an optional dependency, see [[#optional-dependencies][Optional Dependencies]].
-JSON coercion defaults to only an "unexceptional" statuses, meaning status codes
-in the #{200 201 202 203 204 205 206 207 300 301 302 303 304 307} range. If you
-would like to change this, you can send the =:coerce= option, which can be set
-to:
+By default, JSON coercion is only applied when the response's status
+is considered "unexceptional". If the =:unexceptional-status= option
+is provided, then its value is a function which specifies what status
+codes are unexceptional. =:unexceptional-status= defaults to
+=clj-http.client/unexceptional-status?=.
+
+If you would like to change under what conditions coercion is applied,
+you can send the =:coerce= option, which can be set to:
#+BEGIN_SRC clojure
:always ;; always json decode the body
-:unexceptional ;; only json decode when not an HTTP error response
-:exceptional ;; only json decode when it IS an HTTP error response
+:unexceptional ;; json decode when an HTTP response is considered unexceptional
+:exceptional ;; json decode when an HTTP response is considered exceptional
#+END_SRC
The =:coerce= setting defaults to =:unexceptional=.
@@ -578,13 +594,14 @@ disabled by using with-middleware to specify different behavior.
:CUSTOM_ID: h-dd49992c-a516-4af0-9735-4f4340773361
:END:
-There are three different ways that query string parameters for array values can
+There are four different ways that query string parameters for array values can
be generated, depending on what the resulting query string should look like,
they are:
- A repeating parameter (default)
- Array style
- Indexed array style
+- Comma separated style
Here is an example of the input and output for the ~:query-params~ parameter,
controlled by the ~:multi-param-style~ option:
@@ -598,6 +615,8 @@ controlled by the ~:multi-param-style~ option:
;; with :multi-param-style :indexed, a repeating param with array suffix and
;; index (Rails-style):
:a [1 2 3] => "a[0]=1&a[1]=2&a[2]=3"
+;; with :multi-param-style :comma-separated, a param with comma-separated values
+:a [1 2 3] => "a=1,2,3"
#+END_SRC
** Meta Tag Headers
@@ -650,7 +669,7 @@ using:
#+END_SRC
Note that this feature is currently beta and uses [[https://github.com/weavejester/crouton][Crouton]] to parse the body of
-the request. If you do not want to use this feature, you can include Crouton in
+the request. If you want to use this feature, you can include Crouton in
addition to clj-http as a dependency like so:
#+BEGIN_SRC clojure
@@ -686,7 +705,7 @@ APIs:
clj-http conforms its behaviour regarding automatic redirects to the [[https://tools.ietf.org/html/rfc2616#section-10.3][RFC]].
-It means that redirects on status =301=, =302= and =307= are not redirected on
+It means that redirects on status =301=, =302=, =307= and =308= are not redirected on
methods other than =GET= and =HEAD=. If you want a behaviour closer to what most
browser have, you can set =:redirect-strategy= to =:lax= in your request to have
automatic redirection work on all methods by transforming the method of the
@@ -698,8 +717,8 @@ Redirect Options:
list of redirected URLs with key: =:trace-redirects=.
- =:redirect-strategy= :: Sets the redirect strategy for clj-http. Accepts the following:
- =:none= - Perform no redirects
- - =:default= - See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/client/DefaultRedirectStrategy.html
- - =:lax= - See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/client/LaxRedirectStrategy.html
+ - =:default= - See https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/client/DefaultRedirectStrategy.html
+ - =:lax= - See https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/client/LaxRedirectStrategy.html
- =:graceful= - Similar to =:default=, but does not throw exceptions when max redirects is reached. This is the redirects behaviour in 2.x
- =nil= - When nil, assumes =:default=
@@ -712,6 +731,37 @@ this by setting =:validate-redirects false= in the request (the default is true)
NOTE: The options =:force-redirects= and =:follow-redirects= (present in clj-http 2.x are no longer
used). You can use =:graceful= to mostly emulate the old redirect behaviour.
+*** How to create a custom RedirectStrategy
+:PROPERTIES:
+:CUSTOM_ID: h:a3b8b124-411f-4c4c-ac4b-777624e76bf1
+:END:
+As mentioned earlier, it's possible to pass a custom instance of RedirectStrategy. The snippet below shows how to create a custom =RedirectStrategy= by wrapping the default strategy.
+
+#+begin_src clojure
+ (def default-strategy org.apache.http.impl.client.DefaultRedirectStrategy/INSTANCE)
+
+ (def logging-redirect-strategy
+ (reify org.apache.http.client.RedirectStrategy
+ (getRedirect [this request response context]
+ (println "attempting redirect...")
+ (.getRedirect default-strategy request response context))
+ (isRedirected [this request response context]
+ (println "checking isRedirected")
+ (.isRedirected default-strategy request response context))))
+
+ (client/get "https://httpbin.org/absolute-redirect/3" {:redirect-strategy logging-redirect-strategy})
+ ;; this will output the following:
+ ;;
+ ;; checking isRedirected
+ ;; attempting redirect...
+ ;; checking isRedirected
+ ;; attempting redirect...
+ ;; checking isRedirected
+ ;; attempting redirect...
+ ;; checking isRedirected
+#+end_src
+
+
** Cookies
:PROPERTIES:
:CUSTOM_ID: h-3bb89b16-4be3-455e-98ec-c5ca5830ddb9
@@ -828,7 +878,7 @@ HTTP responses other than =#{200 201 202 203 204 205 206 207 300 301 302 303 304
;; Or ignore an unknown host (methods return 'nil' if this is set to
;; true and the host does not exist:
(client/get "http://example.invalid" {:ignore-unknown-host? true})
-;; Or customize the http statuses that will throw:
+;; Or customize the http statuses that will not throw:
(client/get "http://example.com/broken" {:unexceptional-status #(<= 200 % 299)})
#+END_SRC
@@ -1217,7 +1267,7 @@ manager must be used.
;; You can also build your own, using clj-http's helper or manually building it:
(let [cm (conn/make-reusable-conn-manager {})
- hclient (core/build-http-client {} cm "https://example.com" false)]
+ hclient (core/build-http-client {} false cm)]
(client/get "http://example.com/1"
{:connection-manager cm :http-client hclient})
(client/get "http://example.com/2"
@@ -1227,7 +1277,7 @@ manager must be used.
;; Async http clients may also be created and re-used:
(let [acm (conn/make-reuseable-async-conn-manager {})
- ahclient (core/build-async-http-client {} acm "https://example.com" false)]
+ ahclient (core/build-async-http-client {} acm)]
(client/get "http://example.com/1"
{:connection-manager cm :http-client ahclient}
handle-response handle-failure)
@@ -1255,6 +1305,12 @@ Additionally, per-request proxies can be specified with the =proxy-host= and
(client/get "http://example.com" {:proxy-host "127.0.0.1" :proxy-port 8118})
#+END_SRC
+Proxy credentials can also be explicitly set as
+
+#+BEGIN_SRC clojure
+(client/get "http://example.com" {:proxy-host "127.0.0.1" :proxy-port 8118 :proxy-user "proxy-user" :proxy-pass "superSecurePassword"})
+#+END_SRC
+
You can also specify the =proxy-ignore-hosts= parameter with a list of
hosts where the proxy should be ignored. By default this list is
=#{"localhost" "127.0.0.1"}=.
@@ -1345,6 +1401,81 @@ Each of these variables is a sequence of functions of two arguments, the http bu
The functions are run in the order they are passed in (inside a =doseq=).
+By specifying =:http-client-builder=, your own instance of
+=HttpClientBuilder= will be used. A supplied =HttpClientBuilder= which
+sets the connection manager, redirect strategy, retry handler, route
+planner, cache, or cookie spec registry may find these overridden by
+clj-http's =:connection-manager=, =:redirect-strategy=,
+=:retry-handler=, =:cache=, or =:cookie-policy-registry= or
+=:cookie-spec=, respectively.
+
+** Incrementally JSON Parsing
+:PROPERTIES:
+:CUSTOM_ID: h:b01c16e8-7179-468e-8890-316939ec0e38
+:END:
+[[https://github.com/dakrone/cheshire][cheshire]] supports incrementally parsing JSON using lazy sequences. This approach can useful for
+processing large top-level JSON arrays because it doesn't require upfront work consuming the entire stream.
+
+#+begin_src clojure
+ (require '[cheshire.core :as json])
+
+ (defn print-all-pokemon-names [pokemons]
+ (for [pokemon pokemons]
+ (println (get-in pokemon [:name :english]))))
+
+ (let [url "https://raw.githubusercontent.com/fanzeyi/pokemon.json/master/pokedex.json"
+ response (get url {:as :reader})]
+ (with-open [reader (:body response)] ; closes the underlying connection when we're done
+ (let [pokemons (json/parse-stream reader true)]
+ ; You must perform all reads from the stream inside `with-open`,
+ ; any , any lazy
+ (doall (print-all-pokemon-names pokemons)))))
+#+end_src
+
+Keep in mind that the =reader= object wraps a HTTP connection. The user needs to be aware of two
+things:
+
+1. The user should close the reader after processing the stream, otherwise the underlying HTTP
+ Connection may leak and create subtle bugs. Clojure's [[https://clojuredocs.org/clojure.core/with-open][with-open]] is useful here.
+
+2. You should realize any lazy sequences before closing the connection. Use [[https://clojuredocs.org/clojure.core/doall][doall]] or [[https://clojure.org/reference/transducers][transducers]] to
+ prevent bugs from lazy IO. See [[https://stuartsierra.com/2015/08/25/clojure-donts-lazy-effects][Clojure Don'ts: Lazy Effects]].
+
+In previous versions of =clj-http= (<= 3.10.0), =clj-http= defaulted to lazily parsing JSON, but this
+was slow and also confused users who didn't expect laziness.
+
+** DNS Resolution
+:PROPERTIES:
+:CUSTOM_ID: h:52CC15DF-57A5-425E-9AFC-10C9B4C4FA83
+:END:
+
+Users may add their own DNS resolver function to override the default DNS Resolver. This is useful in situations where you are unable to change the name to IP Address mapping. It is analogous to the =--resolve= flag present in =curl=. This example uses =org.apache.http.impl.conn.InMemoryDnsResolver= to resolve =example.com= to IP Address =127.0.0.1=.
+
+#+BEGIN_SRC clojure
+(client/get "https://example.com" {:dns-resolver (doto (InMemoryDnsResolver.)
+ (.add "example.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))])))})
+#+END_SRC
+
+This option is supported for all of the connection managers.
+
+The =dns-resolver= can be any instance of =DnsResolver=. Here is an example of a custom implementation that attempts to look up the hostname in the supplied map and falls back to the default SystemDnsResolver if not found. Note how IPV6 addresses are specified.
+
+#+BEGIN_SRC clojure
+(defn custom-dns-resolver
+ [host-map]
+ (let [system-dns-resolver (org.apache.http.impl.conn.SystemDefaultDnsResolver.)]
+ (reify
+ org.apache.http.conn.DnsResolver
+ (^"[Ljava.net.InetAddress;" resolve [this ^String host]
+ (if-let [address (get host-map host)]
+ (into-array [(java.net.InetAddress/getByAddress host (byte-array address))])
+ (.resolve system-dns-resolver host))))))
+
+(client/get "https://example.com" {:dns-resolver (custom-dns-resolver {"example.com" [127 0 0 1]
+ "www.google.com" [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1]})})
+#+END_SRC
+
+
* Development
:PROPERTIES:
:CUSTOM_ID: h-65bbf017-2e8b-4c43-824b-24b89cc27a70
@@ -1379,6 +1510,7 @@ adding them with the clj-http dependency in your project.clj:
[crouton] ;; for :decode-body-headers
[org.clojure/tools.reader] ;; for :as :clojure
[ring/ring-codec] ;; for :as :x-www-form-urlencoded
+[com.cognitect/transit-clj] ;; for transit support
#+END_SRC
Prior to 2.0.0, you can /exclude/ the dependencies and clj-http will work
@@ -1389,8 +1521,8 @@ without them.
:CUSTOM_ID: h-ba6b263b-74a5-40f3-afc1-b0d785554c2b
:END:
-Like clj-http but need something more lightweight without as many external
-dependencies? Check out [[https://github.com/hiredman/clj-http-lite][clj-http-lite]] for a project that can be used as a
+Like clj-http but need something more lightweight without as any external
+dependencies? Check out [[https://github.com/clj-commons/clj-http-lite][clj-http-lite]] for a project that can be used as a
drop-in replacement for clj-http.
** Troubleshooting
@@ -1469,9 +1601,9 @@ Libraries using clj-http:
Libraries inspired by clj-http:
- [[https://github.com/mpenet/jet][jet]]
-- [[https://github.com/hiredman/clj-http-lite][clj-http-lite]]
+- [[https://github.com/clj-commons/clj-http-lite][clj-http-lite]]
-* Other libraries providing middleware
+* Other Libraries Providing Middleware
:PROPERTIES:
:CUSTOM_ID: other-middleware
:END:
@@ -1487,3 +1619,7 @@ Libraries inspired by clj-http:
Released under the MIT License:
+
+# Local Variables:
+# fill-column: 100
+# End:
diff --git a/changelog.org b/changelog.org
index 8f487252..04e5cfda 100644
--- a/changelog.org
+++ b/changelog.org
@@ -9,9 +9,69 @@
* Changelog
List of user-visible changes that have gone into each release
-
-** 3.10.0 (Unreleased)
-
+** 3.12.4 (unreleased)
+** 3.12.3
+- Allow http-client re-use in async situation (#599)
+ https://github.com/dakrone/clj-http/pull/599
+** 3.12.2
+- Upgrade Dependencies (#598)
+ https://github.com/dakrone/clj-http/pull/598
+** 3.12.1
+- Bugfix for :normalize-uri (#584)
+ https://github.com/dakrone/clj-http/pull/584
+** 3.12.0
+- Create SSLContext consistently for all connection managers (#575)
+ https://github.com/dakrone/clj-http/pull/575
+- Adds RequestConfig Option :normalize-uri (#583)
+ https://github.com/dakrone/clj-http/pull/583
+** 3.11.0
+- Adds workaround for Async Multipart uploads greater than 25 kb (#574)
+ https://github.com/dakrone/clj-http/pull/574
+- Adds an additional style for multi-param-style added (#562)
+ https://github.com/dakrone/clj-http/pull/562
+- Close transit input stream after reading response (#565)
+ https://github.com/dakrone/clj-http/pull/565
+- Bump patch versions of apache httpcomponents to latest. (#569)
+ https://github.com/dakrone/clj-http/pull/569
+- Fixed decode-json-body (#568)
+ https://github.com/dakrone/clj-http/pull/568
+- Handle quoted parameter values in content type (#573)
+ https://github.com/dakrone/clj-http/pull/573
+** 3.10.3
+- Improve error message when using incompatible version of cheshire
+ https://github.com/dakrone/clj-http/pull/558
+- Properly handle "308 Permanent Redirect" status code
+ https://github.com/dakrone/clj-http/pull/554
+** 3.10.2
+- Fix performance regressions from #528
+ https://github.com/dakrone/clj-http/pull/546
+- Adds support for custom DNS Resolvers
+ https://github.com/dakrone/clj-http/pull/545
+- Buffer :debug output to improve readability
+ https://github.com/dakrone/clj-http/pull/544
+- Improve compatbility with GraalVM
+ https://github.com/dakrone/clj-http/pull/543
+- Bug fix: Check first byte before wrapping response stream with gunzip
+ https://github.com/dakrone/clj-http/pull/549
+** 3.10.1
+- JSON parsing is always strict. See [[file:README.org::*Incrementally%20JSON%20Parsing][README#Incrementally JSON Parsing]]. This is
+ a *breaking change* and users *must* upgrade to cheshire >= 5.9.0.
+ https://github.com/dakrone/clj-http/pull/507
+** 3.10.0
+- Add trust-manager and key-managers support to the client
+ https://github.com/dakrone/clj-http/pull/469
+- Improving consistency of connection option names
+ https://github.com/dakrone/clj-http/pull/483
+ https://github.com/dakrone/clj-http/issues/477
+- Ensure Socket Timeout is set for BasicHttpClientConnectionManager
+ https://github.com/dakrone/clj-http/pull/463
+- Reduce body allocation and copying
+ https://github.com/dakrone/clj-http/pull/475
+** 3.9.1
+- Fix body parsing when first byte value is 255
+ https://github.com/dakrone/clj-http/pull/449
+- Add custom =:unexceptional-status= option
+ https://github.com/dakrone/clj-http/pull/451
** 3.9.0
- Add support for reusable http clients, returning the client in =:http-client= and allowing one to
be specified (with the same setting) - https://github.com/dakrone/clj-http/issues/441
diff --git a/examples/kubernetes_pod.clj b/examples/kubernetes_pod.clj
new file mode 100644
index 00000000..4add03bf
--- /dev/null
+++ b/examples/kubernetes_pod.clj
@@ -0,0 +1,20 @@
+(:ns clj-http.examples.kubernetes-pod
+ "This is an example of calling the Kubernetes API from inside a pod. K8s uses a
+ custom CA so that you can authenticate the API server, and provides a token per pod
+ so that each pod can authenticate itself with the APi server.
+
+ If you are still having 401/403 errors, look carefully at the message, if it includes
+ a ServiceAccount name, this part worked, and your problem is likely at the Role/RoleBinding level."
+ (:require [clj-http.client :as http]
+ [less.awful.ssl :refer [trust-store]]))
+
+;; Note that this is not a working example, you'll need to figure out your K8s API path.
+(let [k8s-trust-store (trust-store (clojure.java.io/file "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"))
+ bearer-token (format "Bearer %s" (slurp "/var/run/secrets/kubernetes.io/serviceaccount/token"))
+ kube-api-host (System/getenv "KUBERNETES_SERVICE_HOST")
+ kube-api-port (System/getenv "KUBERNETES_SERVICE_PORT")]
+ (http/get
+ (format "https://%s:%s/apis/" kube-api-host kube-api-port)
+ {:trust-store k8s-trust-store
+ :headers {:authorization bearer-token}}))
+
diff --git a/project.clj b/project.clj
index f80302df..a0c875f4 100644
--- a/project.clj
+++ b/project.clj
@@ -1,4 +1,4 @@
-(defproject clj-http "3.9.2-SNAPSHOT"
+(defproject clj-http "3.13.2-SNAPSHOT"
:description "A Clojure HTTP library wrapping the Apache HttpComponents client."
:url "https://github.com/dakrone/clj-http/"
:license {:name "The MIT License"
@@ -6,44 +6,46 @@
:distribution :repo}
:global-vars {*warn-on-reflection* false}
:min-lein-version "2.0.0"
- :exclusions [org.clojure/clojure]
- :dependencies [[org.apache.httpcomponents/httpcore "4.4.9"]
- [org.apache.httpcomponents/httpclient "4.5.5"]
- [org.apache.httpcomponents/httpclient-cache "4.5.5"]
- [org.apache.httpcomponents/httpasyncclient "4.1.3"]
- [org.apache.httpcomponents/httpmime "4.5.5"]
- [commons-codec "1.11"]
- [commons-io "2.6"]
+ :dependencies [[org.apache.httpcomponents/httpcore "4.4.16"]
+ [org.apache.httpcomponents/httpclient "4.5.14"]
+ [org.apache.httpcomponents/httpclient-cache "4.5.14"]
+ [org.apache.httpcomponents/httpasyncclient "4.1.5"]
+ [org.apache.httpcomponents/httpmime "4.5.14"]
+ [commons-codec "1.16.1"]
+ [commons-io "2.16.1"]
[slingshot "0.12.2"]
- [potemkin "0.4.5"]]
+ [potemkin "0.4.7"]]
:resource-paths ["resources"]
:profiles {:dev {:dependencies [;; optional deps
- [cheshire "5.8.0"]
+ [cheshire "5.13.0"]
[crouton "0.1.2" :exclusions [[org.jsoup/jsoup]]]
- [org.jsoup/jsoup "1.11.3"]
- [org.clojure/tools.reader "1.2.2"]
- [com.cognitect/transit-clj "0.8.300"]
- [ring/ring-codec "1.1.1"]
+ [org.jsoup/jsoup "1.17.2"]
+ [org.clojure/tools.reader "1.4.1"]
+ [com.cognitect/transit-clj "1.0.333"]
+ [ring/ring-codec "1.2.0"]
;; other (testing) deps
- [org.clojure/clojure "1.9.0"]
- [org.clojure/tools.logging "0.4.0"]
- [ring/ring-jetty-adapter "1.6.3"]
- [ring/ring-devel "1.6.3"]
+ [org.clojure/clojure "1.12.1"]
+ [org.clojure/tools.logging "1.3.0"]
+ [ring/ring-jetty-adapter "1.12.1"]
+ [ring/ring-devel "1.12.1"]
+ [javax.servlet/javax.servlet-api "4.0.1"]
;; caching example deps
- [org.clojure/core.cache "0.7.1"]
+ [org.clojure/core.cache "1.1.234"]
;; logging
- [org.apache.logging.log4j/log4j-api "2.11.0"]
- [org.apache.logging.log4j/log4j-core "2.11.0"]
- [org.apache.logging.log4j/log4j-1.2-api "2.11.0"]]
- :plugins [[lein-ancient "0.6.15"]
+ [org.apache.logging.log4j/log4j-api "2.23.1"]
+ [org.apache.logging.log4j/log4j-core "2.23.1"]
+ [org.apache.logging.log4j/log4j-1.2-api "2.23.1"]
+ [org.apache.logging.log4j/log4j-slf4j2-impl "2.23.1"]]
+ :plugins [[lein-ancient "0.7.0"]
[jonase/eastwood "0.2.5"]
[lein-kibit "0.1.5"]
[lein-nvd "0.5.2"]]}
- :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]}
- :1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]}
- :1.8 {:dependencies [[org.clojure/clojure "1.8.0"]]}}
- :aliases {"all" ["with-profile" "dev,1.6:dev,1.7:dev,1.8:dev"]}
+ :1.8 {:dependencies [[org.clojure/clojure "1.8.0"]]}
+ :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]}
+ :1.10 {:dependencies [[org.clojure/clojure "1.10.3"]]}
+ :1.11 {:dependencies [[org.clojure/clojure "1.11.4"]]}}
+ :aliases {"all" ["with-profile" "dev,1.8:dev,1.9:dev,1.10:dev,1.11:dev"]}
:plugins [[codox "0.6.4"]]
- :test-selectors {:default #(not (:integration %))
+ :test-selectors {:default #(not (:integration %))
:integration :integration
:all (constantly true)})
diff --git a/src/clj_http/client.clj b/src/clj_http/client.clj
index 9d7f6747..7d01d886 100644
--- a/src/clj_http/client.clj
+++ b/src/clj_http/client.clj
@@ -1,23 +1,25 @@
(ns clj-http.client
"Batteries-included HTTP client."
+ (:refer-clojure :exclude [get update])
(:require [clj-http.conn-mgr :as conn]
[clj-http.cookies :refer [wrap-cookies]]
[clj-http.core :as core]
[clj-http.headers :refer [wrap-header-map]]
[clj-http.links :refer [wrap-links]]
- [clj-http.util :refer [opt] :as util]
+ [clj-http.util :as util :refer [opt]]
+ [clojure.java.io :as io]
[clojure.stacktrace :refer [root-cause]]
[clojure.string :as str]
[clojure.walk :refer [keywordize-keys prewalk]]
+ [clojure.xml :as xml]
[slingshot.slingshot :refer [throw+]])
- (:import (java.io InputStream File ByteArrayOutputStream ByteArrayInputStream)
- (java.net URL UnknownHostException)
- (org.apache.http.entity BufferedHttpEntity ByteArrayEntity
- InputStreamEntity FileEntity StringEntity)
- (org.apache.http.impl.conn PoolingHttpClientConnectionManager)
- (org.apache.http.impl.nio.conn PoolingNHttpClientConnectionManager)
- (org.apache.http.impl.nio.client HttpAsyncClients))
- (:refer-clojure :exclude [get update]))
+ (:import [java.io BufferedReader ByteArrayInputStream ByteArrayOutputStream EOFException File InputStream]
+ [java.net UnknownHostException URL]
+ [org.apache.http.entity BufferedHttpEntity ByteArrayEntity FileEntity InputStreamEntity StringEntity]
+ [javax.xml.parsers SAXParser SAXParserFactory]
+ org.xml.sax.helpers.DefaultHandler
+ org.apache.http.impl.conn.PoolingHttpClientConnectionManager
+ org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager))
;; Cheshire is an optional dependency, so we check for it at compile time.
(def json-enabled?
@@ -100,12 +102,17 @@
(defn ^:dynamic parse-transit
"Resolve and apply Transit's JSON/MessagePack decoding."
- [in type & [opts]]
+ [^InputStream in type & [opts]]
{:pre [transit-enabled?]}
- (when (pos? (.available in))
+ (try
(let [reader (ns-resolve 'cognitect.transit 'reader)
read (ns-resolve 'cognitect.transit 'read)]
- (read (reader in type (transit-read-opts opts))))))
+ (read (reader in type (transit-read-opts opts))))
+ (catch RuntimeException e
+ ;; Ignore exceptions from trying to read an empty stream.
+ (if (instance? EOFException (.getCause e))
+ nil
+ (throw e)))))
(defn ^:dynamic transit-encode
"Resolve and apply Transit's JSON/MessagePack encoding."
@@ -127,14 +134,10 @@
"Resolve and apply cheshire's json decoding dynamically."
[& args]
{:pre [json-enabled?]}
- (apply (ns-resolve (symbol "cheshire.core") (symbol "decode")) args))
-
-(defn ^:dynamic json-decode-strict
- "Resolve and apply cheshire's json decoding dynamically (with lazy parsing
- disabled)."
- [& args]
- {:pre [json-enabled?]}
- (apply (ns-resolve (symbol "cheshire.core") (symbol "decode-strict")) args))
+ (if-let [json-decode-fn (ns-resolve (symbol "cheshire.core") (symbol "parse-stream-strict"))]
+ (apply json-decode-fn args)
+ (throw
+ (IllegalStateException. "Missing #'cheshire.core/parse-stream-strict. Ensure the version of `cheshire` is >= 5.9.0"))))
(defn ^:dynamic form-decode
"Resolve and apply ring-codec's form decoding dynamically."
@@ -202,11 +205,12 @@
;; Statuses for which clj-http will not throw an exception
(def unexceptional-status?
- #{200 201 202 203 204 205 206 207 300 301 302 303 304 307})
+ #{200 201 202 203 204 205 206 207 300 301 302 303 304 307 308})
(defn unexceptional-status-for-request?
[req status]
- ((:unexceptional-status req unexceptional-status?) status))
+ ((or (:unexceptional-status req) unexceptional-status?)
+ status))
;; helper methods to determine realm of a response
(defn success?
@@ -339,7 +343,7 @@
resp-r)
:else
(respond* resp-r req))
- (= 307 status)
+ (#{307 308} status)
(if (or (#{:get :head} request-method)
(opt req :force-redirects))
(follow-redirect client (assoc req :redirects-count
@@ -427,83 +431,105 @@
(defmulti coerce-response-body (fn [req _] (:as req)))
(defmethod coerce-response-body :byte-array [_ resp]
- (assoc resp :body (util/force-byte-array (:body resp))))
+ (update resp :body util/force-byte-array))
(defmethod coerce-response-body :stream [_ resp]
- (let [body (:body resp)]
- (cond (instance? InputStream body) resp
- ;; This shouldn't happen, but we plan for it anyway
- (instance? (Class/forName "[B") body)
- (assoc resp :body (ByteArrayInputStream. body)))))
+ (update resp :body util/force-stream))
+
+(defn- response-charset [response]
+ (or (-> response :content-type-params :charset)
+ "UTF-8"))
+
+(defmethod coerce-response-body :reader
+ [_ {:keys [body] :as resp}]
+ (let [header (get-in resp [:headers "content-type"])
+ parsed-values (util/parse-content-type header)
+ charset (response-charset parsed-values)]
+ (assoc resp :body (io/reader body :encoding charset))))
+
+(defn- can-parse-body? [{:keys [coerce] :as request} {:keys [status] :as _response}]
+ (or (= coerce :always)
+ (and (unexceptional-status-for-request? request status)
+ (or (nil? coerce)
+ (= coerce :unexceptional)))
+ (and (not (unexceptional-status-for-request? request status))
+ (= coerce :exceptional))))
+
+(defn- decode-json-body [body keyword? charset]
+ (let [^BufferedReader br (io/reader (util/force-stream body) :encoding charset)]
+ (try
+ (.mark br 1)
+ (let [first-char (int (try (.read br) (catch EOFException _ -1)))]
+ (case first-char
+ -1 nil
+ (do (.reset br)
+ (json-decode br keyword?))))
+ (finally (.close br)))))
(defn coerce-json-body
- [{:keys [coerce] :as request}
- {:keys [body status] :as resp} keyword? strict? & [charset]]
- (let [^String charset (or charset (-> resp :content-type-params :charset)
- "UTF-8")
- body (util/force-byte-array body)
- decode-func (if strict? json-decode-strict json-decode)]
- (if json-enabled?
- (cond
- (= coerce :always)
- (assoc resp :body (decode-func (String. ^"[B" body charset) keyword?))
-
- (and (unexceptional-status-for-request? request status)
- (or (nil? coerce) (= coerce :unexceptional)))
- (assoc resp :body (decode-func (String. ^"[B" body charset) keyword?))
-
- (and (not (unexceptional-status-for-request? request status))
- (= coerce :exceptional))
- (assoc resp :body (decode-func (String. ^"[B" body charset) keyword?))
-
- :else (assoc resp :body (String. ^"[B" body charset)))
- (assoc resp :body (String. ^"[B" body charset)))))
+ [request {:keys [body] :as resp} keyword? & [charset]]
+ {:pre [json-enabled?]}
+ (let [charset (or charset (response-charset resp))
+ body (if (can-parse-body? request resp)
+ (decode-json-body body keyword? charset)
+ (util/force-string body charset))]
+ (assoc resp :body body)))
+
+(defn- sax-parser ^SAXParser []
+ (..
+ (doto
+ (SAXParserFactory/newInstance)
+ (.setFeature
+ "http://apache.org/xml/features/nonvalidating/load-external-dtd" false))
+ (newSAXParser)))
+
+(defn- non-validating [s ^DefaultHandler ch]
+ (let [parser (sax-parser)]
+ (cond
+ (instance? String s) (.parse parser ^String s ch)
+ (instance? InputStream s) (.parse parser ^InputStream s ch)
+ :else (throw (ex-info "Unsupported input" {:s s})))))
+
+(defn- decode-xml-body [body]
+ (-> body
+ (util/force-stream)
+ (xml/parse non-validating)))
+
+(defn coerce-xml-body
+ [request {:keys [body] :as resp} & [charset]]
+ (let [charset (or charset (response-charset resp))
+ body (if (can-parse-body? request resp)
+ (decode-xml-body body)
+ (util/force-string body charset))]
+ (assoc resp :body body)))
(defn coerce-clojure-body
- [request {:keys [body] :as resp}]
- (let [^String charset (or (-> resp :content-type-params :charset) "UTF-8")
- body (util/force-byte-array body)]
+ [_request {:keys [body] :as resp}]
+ (let [charset (response-charset resp)
+ body (util/force-string body charset)]
(assoc resp :body (cond
(empty? body) nil
- edn-enabled? (parse-edn (String. ^"[B" body charset))
+ edn-enabled? (parse-edn body)
:else (binding [*read-eval* false]
- (read-string (String. ^"[B" body charset)))))))
+ (read-string body))))))
(defn coerce-transit-body
- [{:keys [transit-opts coerce] :as request}
- {:keys [body status] :as resp} type & [charset]]
- (let [^String charset (or charset (-> resp :content-type-params :charset)
- "UTF-8")
- body (util/force-byte-array body)]
- (if-not (empty? body)
- (if transit-enabled?
- (cond
- (= coerce :always)
- (assoc resp :body (parse-transit
- (ByteArrayInputStream. body) type transit-opts))
-
- (and (unexceptional-status-for-request? request status)
- (or (nil? coerce) (= coerce :unexceptional)))
- (assoc resp :body (parse-transit
- (ByteArrayInputStream. body) type transit-opts))
-
- (and (not (unexceptional-status-for-request? request status))
- (= coerce :exceptional))
- (assoc resp :body (parse-transit
- (ByteArrayInputStream. body) type transit-opts))
-
- :else (assoc resp :body (String. ^"[B" body charset)))
- (assoc resp :body (String. ^"[B" body charset)))
- (assoc resp :body nil))))
+ [{:keys [transit-opts] :as request}
+ {:keys [body] :as resp} type & [charset]]
+ {:pre [transit-enabled?]}
+ (let [charset (or charset (response-charset resp))
+ body (if (can-parse-body? request resp)
+ (with-open [in (util/force-stream body)]
+ (parse-transit in type transit-opts))
+ (util/force-string body charset))]
+ (assoc resp :body body)))
(defn coerce-form-urlencoded-body
- [request {:keys [body] :as resp}]
- (let [^String charset (or (-> resp :content-type-params :charset) "UTF-8")
- body-bytes (util/force-byte-array body)]
- (if ring-codec-enabled?
- (assoc resp :body (-> (String. ^"[B" body-bytes charset)
- form-decode keywordize-keys))
- (assoc resp :body (String. ^"[B" body-bytes charset)))))
+ [_request {:keys [body] :as resp}]
+ {:pre [ring-codec-enabled?]}
+ (let [charset (response-charset resp)
+ body (util/force-string body charset)]
+ (assoc resp :body (-> body form-decode keywordize-keys))))
(defmulti coerce-content-type (fn [req resp] (:content-type resp)))
@@ -516,6 +542,9 @@
(defmethod coerce-content-type :application/json [req resp]
(coerce-json-body req resp true false))
+(defmethod coerce-content-type :text/xml [req resp]
+ (coerce-xml-body req resp false))
+
(defmethod coerce-content-type :application/transit+json [req resp]
(coerce-transit-body req resp :json))
@@ -536,16 +565,23 @@
(coerce-content-type request))))
(defmethod coerce-response-body :json [req resp]
- (coerce-json-body req resp true false))
+ (coerce-json-body req resp true))
+(defmethod coerce-response-body :json-string-keys [req resp]
+ (coerce-json-body req resp false))
+
+;; There is no longer any distinction between strict and non-strict JSON parsing
+;; options.
+;;
+;; `:json-strict` and `:json-strict-string-keys` will be removed in a future version
(defmethod coerce-response-body :json-strict [req resp]
- (coerce-json-body req resp true true))
+ (coerce-json-body req resp true))
(defmethod coerce-response-body :json-strict-string-keys [req resp]
- (coerce-json-body req resp false true))
+ (coerce-json-body req resp false))
-(defmethod coerce-response-body :json-string-keys [req resp]
- (coerce-json-body req resp false false))
+(defmethod coerce-response-body :xml [req resp]
+ (coerce-xml-body req resp false))
(defmethod coerce-response-body :clojure [req resp]
(coerce-clojure-body req resp))
@@ -561,10 +597,7 @@
(defmethod coerce-response-body :default
[{:keys [as]} {:keys [body] :as resp}]
- (let [body-bytes (util/force-byte-array body)]
- (cond
- (string? as) (assoc resp :body (String. ^"[B" body-bytes ^String as))
- :else (assoc resp :body (String. ^"[B" body-bytes "UTF-8")))))
+ (assoc resp :body (util/force-string body (if (string? as) as "UTF-8"))))
(defn- output-coercion-response
[req {:keys [body] :as resp}]
@@ -769,21 +802,28 @@
(second found))
"UTF-8"))
-(defn- multi-param-suffix [index multi-param-style]
- (case multi-param-style
- :indexed (str "[" index "]")
- :array "[]"
- ""))
+(defn- multi-param-entries [key values multi-param-style encoding]
+ (let [key (util/url-encode (name key) encoding)
+ values (map #(util/url-encode (str %) encoding) values)]
+ (case multi-param-style
+ :indexed
+ (map-indexed #(vector (str key \[ %1 \]) %2) values)
+
+ :array
+ (map #(vector (str key "[]") %) values)
+
+ :comma-separated
+ ;; See sub-delims in https://tools.ietf.org/html/rfc3986#section-2.2
+ [[key (str/join "," values)]]
+
+ ;; default: repeat the key multiple times
+ (map #(vector key %) values))))
(defn generate-query-string-with-encoding [params encoding multi-param-style]
(str/join "&"
(mapcat (fn [[k v]]
(if (sequential? v)
- (map-indexed
- #(str (util/url-encode (name k) encoding)
- (multi-param-suffix %1 multi-param-style)
- "="
- (util/url-encode (str %2) encoding)) v)
+ (map #(str/join "=" %) (multi-param-entries k v multi-param-style encoding))
[(str (util/url-encode (name k) encoding)
"="
(util/url-encode (str v) encoding))]))
@@ -860,7 +900,6 @@
([req respond raise]
(client (oauth-request req) respond raise))))
-
(defn parse-user-info [user-info]
(when user-info
(str/split user-info #":")))
@@ -1262,7 +1301,7 @@
The following options are supported:
- :timeout - Time that connections are left open before automatically closing
+ :timeout - Time in seconds that connections are left open before automatically closing
default: 5
:threads - Maximum number of threads that will be used for connecting
default: 4
diff --git a/src/clj_http/conn_mgr.clj b/src/clj_http/conn_mgr.clj
index 63cc5098..111c64ff 100644
--- a/src/clj_http/conn_mgr.clj
+++ b/src/clj_http/conn_mgr.clj
@@ -2,56 +2,37 @@
"Utility methods for Scheme registries and HTTP connection managers"
(:require [clj-http.util :refer [opt]]
[clojure.java.io :as io])
- (:import (java.net Socket Proxy Proxy$Type InetSocketAddress)
- (java.security KeyStore)
- (org.apache.http.config RegistryBuilder Registry SocketConfig)
- (org.apache.http.conn HttpClientConnectionManager)
- (org.apache.http.conn.ssl DefaultHostnameVerifier
- NoopHostnameVerifier
- SSLConnectionSocketFactory
- SSLContexts
- TrustStrategy)
- (org.apache.http.conn.socket PlainConnectionSocketFactory)
- (org.apache.http.impl.conn BasicHttpClientConnectionManager
- PoolingHttpClientConnectionManager)
- (org.apache.http.impl.nio.conn PoolingNHttpClientConnectionManager)
- (javax.net.ssl SSLContext HostnameVerifier)
- (org.apache.http.nio.conn NHttpClientConnectionManager)
- (org.apache.http.nio.conn.ssl SSLIOSessionStrategy)
- (org.apache.http.impl.nio.reactor
- IOReactorConfig
- AbstractMultiworkerIOReactor$DefaultThreadFactory
- DefaultConnectingIOReactor)
- (org.apache.http.nio.conn NoopIOSessionStrategy)
- (org.apache.http.nio.protocol HttpAsyncRequestExecutor)
- (org.apache.http.impl.nio DefaultHttpClientIODispatch)
- (org.apache.http.config ConnectionConfig)))
-
-(def ^:private insecure-context-verifier
- (delay {
- :context (-> (SSLContexts/custom)
- (.loadTrustMaterial nil (reify TrustStrategy
- (isTrusted [_ _ _] true)))
- (.build))
- :verifier NoopHostnameVerifier/INSTANCE}))
-
-(def ^:private insecure-socket-factory
- (delay
- (let [{:keys [context verifier]} @insecure-context-verifier]
- (SSLConnectionSocketFactory. ^SSLContext context
- ^HostnameVerifier verifier))))
-
-(def ^:private insecure-strategy
- (delay
- (let [{:keys [context verifier]} @insecure-context-verifier]
- (SSLIOSessionStrategy. ^SSLContext context ^HostnameVerifier verifier))))
-
-(def ^:private ^SSLConnectionSocketFactory secure-ssl-socket-factory
- (SSLConnectionSocketFactory/getSocketFactory))
-
-(def ^:private ^SSLIOSessionStrategy secure-strategy
- (SSLIOSessionStrategy/getDefaultStrategy))
-
+ (:import [java.net InetSocketAddress Proxy Proxy$Type Socket]
+ java.security.KeyStore
+ [javax.net.ssl HostnameVerifier KeyManager SSLContext TrustManager]
+ [org.apache.http.config ConnectionConfig Registry RegistryBuilder SocketConfig]
+ org.apache.http.conn.HttpClientConnectionManager
+ org.apache.http.conn.socket.PlainConnectionSocketFactory
+ [org.apache.http.conn.ssl DefaultHostnameVerifier NoopHostnameVerifier SSLConnectionSocketFactory SSLContexts TrustStrategy]
+ [org.apache.http.impl.conn BasicHttpClientConnectionManager PoolingHttpClientConnectionManager]
+ org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager
+ org.apache.http.impl.nio.DefaultHttpClientIODispatch
+ [org.apache.http.impl.nio.reactor DefaultConnectingIOReactor IOReactorConfig]
+ [org.apache.http.nio.conn NHttpClientConnectionManager NoopIOSessionStrategy]
+ org.apache.http.nio.conn.ssl.SSLIOSessionStrategy
+ org.apache.http.nio.protocol.HttpAsyncRequestExecutor))
+
+;; -- Interop Helpers ---------------------------------------------------------
+(defn ^Registry into-registry [registry]
+ (cond
+ (instance? Registry registry)
+ registry
+
+ (map? registry)
+ (let [registry-builder (RegistryBuilder/create)]
+ (doseq [[k v] registry]
+ (.register registry-builder k v))
+ (.build registry-builder))
+
+ :else
+ (throw (IllegalArgumentException. "Cannot coerce into a Registry"))))
+
+;; -- SocketFactory -----------------------------------------------------------
(defn ^SSLConnectionSocketFactory SSLGenericSocketFactory
"Given a function that returns a new socket, create an
SSLConnectionSocketFactory that will use that socket."
@@ -78,6 +59,7 @@
[^String hostname ^Integer port]
(Socket. (Proxy. Proxy$Type/SOCKS (InetSocketAddress. hostname port))))
+;; -- SSL Contexts ------------------------------------------------------------
(defn ^KeyStore get-keystore*
[keystore-file keystore-type ^String keystore-pass]
(when keystore-file
@@ -92,26 +74,70 @@
keystore
(apply get-keystore* keystore args)))
-(defn get-keystore-context-verifier
+(defn- ssl-context-for-keystore
;; TODO: use something else for passwords
;; Note: JVM strings aren't ideal for passwords - see
;; https://tinyurl.com/azm3ab9
- [{:keys [keystore keystore-type ^String keystore-pass keystore-instance
- trust-store trust-store-type trust-store-pass]
- :as req}]
+ [{:keys [keystore keystore-type ^String keystore-pass
+ trust-store trust-store-type trust-store-pass]}]
(let [ks (get-keystore keystore keystore-type keystore-pass)
ts (get-keystore trust-store trust-store-type trust-store-pass)]
- {:context (-> (SSLContexts/custom)
- (.loadKeyMaterial
- ks (when keystore-pass
- (.toCharArray keystore-pass)))
- (.loadTrustMaterial
- ts nil)
- (.build))
- :verifier (if (opt req :insecure)
- NoopHostnameVerifier/INSTANCE
- (DefaultHostnameVerifier.))}))
+ (-> (SSLContexts/custom)
+ (.loadKeyMaterial
+ ks (when keystore-pass
+ (.toCharArray keystore-pass)))
+ (.loadTrustMaterial
+ ts nil)
+ (.build))))
+
+(defn- ssl-context-for-trust-or-key-manager
+ "Given an instance or seqable data structure of TrustManager or KeyManager
+ will create and return an SSLContexts object including the resulting managers"
+ [{:keys [trust-managers key-managers]}]
+ (let [x-or-xs->x-array (fn [type x-or-xs]
+ (cond
+ (or (-> x-or-xs class .isArray)
+ (sequential? x-or-xs))
+ (into-array type (seq x-or-xs))
+
+ :else
+ (into-array type [x-or-xs])))
+ trust-managers (when trust-managers
+ (x-or-xs->x-array TrustManager trust-managers))
+ key-managers (when key-managers
+ (x-or-xs->x-array KeyManager key-managers))]
+ (doto (.build (SSLContexts/custom))
+ (.init key-managers trust-managers nil))))
+
+(defn- ssl-context-insecure
+ "Creates a SSL Context that trusts all material."
+ []
+ (-> (SSLContexts/custom)
+ (.loadTrustMaterial nil (reify TrustStrategy
+ (isTrusted [_ chain auth-type] true)))
+ (.build)))
+
+(defn ^SSLContext get-ssl-context
+ "Gets the SSL Context from a request or connection pool settings"
+ [{:keys [keystore trust-store key-managers trust-managers] :as config}]
+ (cond (or keystore trust-store)
+ (ssl-context-for-keystore config)
+
+ (or key-managers trust-managers)
+ (ssl-context-for-trust-or-key-manager config)
+
+ (opt config :insecure)
+ (ssl-context-insecure)
+
+ :else
+ (SSLContexts/createDefault)))
+(defn ^HostnameVerifier get-hostname-verifier [config]
+ (if (opt config :insecure)
+ NoopHostnameVerifier/INSTANCE
+ (DefaultHostnameVerifier.)))
+
+;; -- Connection Managers -----------------------------------------------------
(defn make-socks-proxied-conn-manager
"Given an optional hostname and a port, create a connection manager that's
proxied using a SOCKS proxy."
@@ -119,82 +145,33 @@
(make-socks-proxied-conn-manager hostname port {}))
([^String hostname ^Integer port
{:keys [keystore keystore-type keystore-pass
- trust-store trust-store-type trust-store-pass] :as opts}]
+ trust-store trust-store-type trust-store-pass
+ trust-managers key-managers] :as config}]
(let [socket-factory #(socks-proxied-socket hostname port)
- ssl-context (when
- (some (complement nil?)
- [keystore keystore-type keystore-pass trust-store
- trust-store-type trust-store-pass])
- (-> opts get-keystore-context-verifier :context))
- reg (-> (RegistryBuilder/create)
- (.register "http" (PlainGenericSocketFactory socket-factory))
- (.register "https"
- (SSLGenericSocketFactory
- socket-factory ssl-context))
- (.build))]
- (PoolingHttpClientConnectionManager. reg))))
-
-(def ^:private insecure-scheme-registry
- (delay
- (-> (RegistryBuilder/create)
- (.register "http" PlainConnectionSocketFactory/INSTANCE)
- (.register "https" ^SSLConnectionSocketFactory @insecure-socket-factory)
- (.build))))
-
-(def ^:private insecure-strategy-registry
- (delay
- (-> (RegistryBuilder/create)
- (.register "http" NoopIOSessionStrategy/INSTANCE)
- (.register "https" ^SSLIOSessionStrategy @insecure-strategy)
- (.build))))
-
-(def ^:private regular-scheme-registry
- (-> (RegistryBuilder/create)
- (.register "http" (PlainConnectionSocketFactory/getSocketFactory))
- (.register "https" secure-ssl-socket-factory)
- (.build)))
-
-(def ^:private regular-strategy-registry
- (-> (RegistryBuilder/create)
- (.register "http" NoopIOSessionStrategy/INSTANCE)
- (.register "https" secure-strategy)
- (.build)))
-
-(defn ^Registry get-keystore-scheme-registry
- [req]
- (let [{:keys [context verifier]} (get-keystore-context-verifier req)
- factory (SSLConnectionSocketFactory. ^SSLContext context
- ^HostnameVerifier verifier)]
- (-> (RegistryBuilder/create)
- (.register "http" (PlainConnectionSocketFactory/getSocketFactory))
- (.register "https" factory)
- (.build))))
-
-(defn ^Registry get-keystore-strategy-registry
- [req]
- (let [{:keys [context verifier]} (get-keystore-context-verifier req)
- strategy (SSLIOSessionStrategy. ^SSLContext context
- ^HostnameVerifier verifier)]
- (-> (RegistryBuilder/create)
- (.register "http" NoopIOSessionStrategy/INSTANCE)
- (.register "https" strategy)
- (.build))))
+ registry (into-registry
+ {"http" (PlainGenericSocketFactory socket-factory)
+ "https" (SSLGenericSocketFactory socket-factory (get-ssl-context config))})]
+ (PoolingHttpClientConnectionManager. registry))))
(defn ^BasicHttpClientConnectionManager make-regular-conn-manager
- [{:keys [keystore trust-store socket-timeout] :as req}]
- (let [conn-manager (cond
- (or keystore trust-store)
- (BasicHttpClientConnectionManager. (get-keystore-scheme-registry req))
-
- (opt req :insecure) (BasicHttpClientConnectionManager.
- @insecure-scheme-registry)
-
- :else (BasicHttpClientConnectionManager. regular-scheme-registry))]
+ [{:keys [dns-resolver
+ keystore trust-store
+ key-managers trust-managers
+ socket-timeout] :as config}]
+
+ (let [registry (into-registry
+ {"http" (PlainConnectionSocketFactory/getSocketFactory)
+ "https" (SSLConnectionSocketFactory.
+ (get-ssl-context config)
+ (get-hostname-verifier config))})
+ conn-manager (BasicHttpClientConnectionManager. registry
+ nil nil
+ dns-resolver)]
(when socket-timeout
(.setSocketConfig conn-manager
(-> (.getSocketConfig conn-manager)
(SocketConfig/copy)
- (.setSocketTimeout socket-timeout) ;modify only the socket-timeout
+ (.setSoTimeout socket-timeout) ;modify only the socket-timeout
(.build))))
conn-manager))
@@ -218,15 +195,13 @@
(defn ^PoolingNHttpClientConnectionManager
make-regular-async-conn-manager
- [{:keys [keystore trust-store] :as req}]
- (let [^Registry registry (cond
- (or keystore trust-store)
- (get-keystore-strategy-registry req)
-
- (opt req :insecure)
- @insecure-strategy-registry
-
- :else regular-strategy-registry)
+ [{:keys [keystore trust-store
+ key-managers trust-managers] :as config}]
+ (let [^Registry registry (into-registry
+ {"http" (NoopIOSessionStrategy/INSTANCE)
+ "https" (SSLIOSessionStrategy.
+ (get-ssl-context config)
+ (get-hostname-verifier config))})
io-reactor (make-ioreactor {:shutdown-grace-period 1})]
(doto (PoolingNHttpClientConnectionManager. io-reactor registry)
(.setMaxTotal 1))))
@@ -240,16 +215,17 @@
"Given an timeout and optional insecure? flag, create a
PoolingHttpClientConnectionManager with seconds set as the
timeout value."
- [{:keys [timeout keystore trust-store] :as config}]
- (let [registry (cond
- (opt config :insecure) @insecure-scheme-registry
-
- (or keystore trust-store)
- (get-keystore-scheme-registry config)
-
- :else regular-scheme-registry)]
+ [{:keys [dns-resolver
+ timeout
+ keystore trust-store
+ key-managers trust-managers] :as config}]
+ (let [registry (into-registry
+ {"http" (PlainConnectionSocketFactory/getSocketFactory)
+ "https" (SSLConnectionSocketFactory.
+ (get-ssl-context config)
+ (get-hostname-verifier config))})]
(PoolingHttpClientConnectionManager.
- registry nil nil nil timeout java.util.concurrent.TimeUnit/SECONDS)))
+ registry nil nil dns-resolver timeout java.util.concurrent.TimeUnit/SECONDS)))
(defn reusable? [conn-mgr]
(or (instance? PoolingHttpClientConnectionManager conn-mgr)
@@ -274,7 +250,15 @@
:trust-store - trust store file to be used for connection manager
:trust-store-pass - trust store password
- Note that :insecure? and :keystore/:trust-store options are mutually exclusive
+ :key-managers - KeyManager objects to be used for connection manager
+ :trust-managers - TrustManager objects to be used for connection manager
+
+ :dns-resolver - Use a custom DNS resolver instead of the default DNS resolver.
+
+ Note that :insecure? and :keystore/:trust-store/:key-managers/:trust-managers options are mutually exclusive
+
+ Note that :key-managers/:trust-managers have precedence over :keystore/:trust-store options
+
If the value 'nil' is specified or the value is not set, the default value
will be used."
@@ -293,24 +277,24 @@
conn-man))
(defn- ^PoolingNHttpClientConnectionManager make-reusable-async-conn-manager*
- [{:keys [timeout keystore trust-store io-config] :as config}]
- (let [registry (cond
- (opt config :insecure) @insecure-strategy-registry
-
- (or keystore trust-store)
- (get-keystore-scheme-registry config)
-
- :else regular-strategy-registry)
+ [{:keys [dns-resolver
+ timeout keystore trust-store io-config
+ key-managers trust-managers] :as config}]
+ (let [registry (into-registry
+ {"http" (NoopIOSessionStrategy/INSTANCE)
+ "https" (SSLIOSessionStrategy.
+ (get-ssl-context config)
+ (get-hostname-verifier config))})
io-reactor (make-ioreactor io-config)
protocol-handler (HttpAsyncRequestExecutor.)
io-event-dispatch (DefaultHttpClientIODispatch. protocol-handler
ConnectionConfig/DEFAULT)]
(future (.execute io-reactor io-event-dispatch))
(proxy [PoolingNHttpClientConnectionManager ReuseableAsyncConnectionManager]
- [io-reactor nil registry nil nil timeout
+ [io-reactor nil registry nil dns-resolver timeout
java.util.concurrent.TimeUnit/SECONDS])))
-(defn ^PoolingNHttpClientConnectionManager make-reuseable-async-conn-manager
+(defn ^PoolingNHttpClientConnectionManager make-reusable-async-conn-manager
"Creates a default pooling async connection manager with the specified
options. Handles the same options as make-reusable-conn-manager plus
:io-config which should be a map containing some of the following keys:
@@ -354,6 +338,11 @@
(.setDefaultMaxPerRoute conn-man default-per-route))
conn-man))
+(defn ^PoolingNHttpClientConnectionManager make-reuseable-async-conn-manager
+ "Wraps correctly-spelled version - keeping for backwards compatibility."
+ [opts]
+ (make-reusable-async-conn-manager opts))
+
(defmulti shutdown-manager
"Shut down the given connection manager, if it is not nil"
class)
diff --git a/src/clj_http/cookies.clj b/src/clj_http/cookies.clj
index d42dc9a9..c307235c 100644
--- a/src/clj_http/cookies.clj
+++ b/src/clj_http/cookies.clj
@@ -2,16 +2,13 @@
"Namespace dealing with HTTP cookies"
(:require [clj-http.util :refer [opt]]
[clojure.string :refer [blank? join lower-case]])
- (:import (org.apache.http.client.params ClientPNames CookiePolicy)
- (org.apache.http.cookie ClientCookie CookieOrigin CookieSpec)
- (org.apache.http.params BasicHttpParams)
- (org.apache.http.impl.cookie BasicClientCookie2)
- (org.apache.http.impl.cookie BrowserCompatSpecFactory)
- (org.apache.http.message BasicHeader)
- org.apache.http.client.CookieStore
- (org.apache.http.impl.client BasicCookieStore)
- (org.apache.http Header)
- (org.apache.http.protocol BasicHttpContext)))
+ (:import org.apache.http.client.CookieStore
+ [org.apache.http.cookie ClientCookie CookieOrigin CookieSpec]
+ org.apache.http.Header
+ org.apache.http.impl.client.BasicCookieStore
+ [org.apache.http.impl.cookie BasicClientCookie2 BrowserCompatSpecFactory]
+ org.apache.http.message.BasicHeader
+ org.apache.http.protocol.BasicHttpContext))
(defn cookie-spec ^org.apache.http.cookie.CookieSpec []
(.create
@@ -25,7 +22,7 @@
(if (not (nil? (get m k)))
(assoc newm k (get m k))
newm))
- (sorted-map) (sort (keys m))))
+ {} (keys m)))
(defn to-cookie
"Converts a ClientCookie object into a tuple where the first item is
diff --git a/src/clj_http/core.clj b/src/clj_http/core.clj
index 214b840e..7cc54785 100644
--- a/src/clj_http/core.clj
+++ b/src/clj_http/core.clj
@@ -4,50 +4,26 @@
[clj-http.headers :as headers]
[clj-http.multipart :as mp]
[clj-http.util :refer [opt]]
- [clojure.pprint])
- (:import (java.io ByteArrayOutputStream FilterInputStream InputStream)
- (java.net URI URL ProxySelector InetAddress)
- (java.util Locale)
- (org.apache.http HttpEntity HeaderIterator HttpHost HttpRequest
- HttpEntityEnclosingRequest HttpResponse
- HttpRequestInterceptor HttpResponseInterceptor
- ProtocolException)
- (org.apache.http.auth UsernamePasswordCredentials AuthScope
- NTCredentials)
- (org.apache.http.client HttpRequestRetryHandler RedirectStrategy
- CredentialsProvider)
- (org.apache.http.client.config RequestConfig CookieSpecs)
- (org.apache.http.client.methods HttpDelete HttpGet HttpPost HttpPut
- HttpOptions HttpPatch
- HttpHead
- HttpEntityEnclosingRequestBase
- CloseableHttpResponse
- HttpUriRequest HttpRequestBase)
- (org.apache.http.client.protocol HttpClientContext)
- (org.apache.http.client.utils URIUtils)
- (org.apache.http.config RegistryBuilder)
- (org.apache.http.conn.routing HttpRoute HttpRoutePlanner)
- (org.apache.http.conn.ssl BrowserCompatHostnameVerifier
- SSLConnectionSocketFactory SSLContexts)
- (org.apache.http.conn.socket PlainConnectionSocketFactory)
- (org.apache.http.conn.util PublicSuffixMatcherLoader)
- (org.apache.http.cookie CookieSpecProvider)
- (org.apache.http.entity ByteArrayEntity StringEntity)
- (org.apache.http.impl.client BasicCredentialsProvider
- CloseableHttpClient HttpClients
- DefaultRedirectStrategy
- LaxRedirectStrategy HttpClientBuilder)
- (org.apache.http.client.cache HttpCacheContext)
- (org.apache.http.impl.client.cache CacheConfig
- CachingHttpClientBuilder)
- (org.apache.http.impl.cookie DefaultCookieSpecProvider)
- (org.apache.http.impl.conn SystemDefaultRoutePlanner
- DefaultProxyRoutePlanner)
- (org.apache.http.impl.nio.client HttpAsyncClientBuilder
- HttpAsyncClients
- CloseableHttpAsyncClient)
- (org.apache.http.message BasicHttpResponse)
- (java.util.concurrent ExecutionException)))
+ clojure.pprint)
+ (:import [java.io ByteArrayOutputStream FilterInputStream InputStream]
+ [java.net InetAddress ProxySelector URI URL]
+ java.util.Locale
+ [org.apache.http HeaderIterator HttpEntity HttpEntityEnclosingRequest HttpHost HttpRequestInterceptor HttpResponse HttpResponseInterceptor ProtocolException]
+ [org.apache.http.auth AuthScope NTCredentials UsernamePasswordCredentials]
+ [org.apache.http.client CredentialsProvider HttpRequestRetryHandler RedirectStrategy]
+ org.apache.http.client.cache.HttpCacheContext
+ [org.apache.http.client.config CookieSpecs RequestConfig]
+ [org.apache.http.client.methods CloseableHttpResponse HttpDelete HttpEntityEnclosingRequestBase HttpGet HttpHead HttpOptions HttpPatch HttpPost HttpPut HttpRequestBase HttpUriRequest]
+ org.apache.http.client.protocol.HttpClientContext
+ org.apache.http.client.utils.URIUtils
+ org.apache.http.config.RegistryBuilder
+ org.apache.http.conn.routing.HttpRoutePlanner
+ org.apache.http.cookie.CookieSpecProvider
+ [org.apache.http.entity ByteArrayEntity StringEntity]
+ [org.apache.http.impl.client BasicCredentialsProvider CloseableHttpClient DefaultRedirectStrategy HttpClientBuilder HttpClients LaxRedirectStrategy]
+ [org.apache.http.impl.client.cache CacheConfig CachingHttpClientBuilder]
+ [org.apache.http.impl.conn DefaultProxyRoutePlanner SystemDefaultRoutePlanner]
+ [org.apache.http.impl.nio.client CloseableHttpAsyncClient HttpAsyncClientBuilder HttpAsyncClients]))
(def CUSTOM_COOKIE_POLICY "_custom")
@@ -179,17 +155,21 @@
(defmethod get-cookie-policy :stardard-strict standard-strict-cookie-policy
[_] CookieSpecs/STANDARD_STRICT)
-(defn request-config [{:keys [conn-timeout
+(defn request-config [{:keys [connection-timeout
+ connection-request-timeout
socket-timeout
- conn-request-timeout
max-redirects
- cookie-spec]
+ cookie-spec
+ normalize-uri
+ ; deprecated
+ conn-request-timeout
+ conn-timeout]
:as req}]
(let [config (-> (RequestConfig/custom)
- (.setConnectTimeout (or conn-timeout -1))
+ (.setConnectTimeout (or connection-timeout conn-timeout -1))
(.setSocketTimeout (or socket-timeout -1))
(.setConnectionRequestTimeout
- (or conn-request-timeout -1))
+ (or connection-request-timeout conn-request-timeout -1))
(.setRedirectsEnabled true)
(.setCircularRedirectsAllowed
(boolean (opt req :allow-circular-redirects)))
@@ -200,6 +180,7 @@
(.setCookieSpec config CUSTOM_COOKIE_POLICY)
(.setCookieSpec config (get-cookie-policy req)))
(when max-redirects (.setMaxRedirects config max-redirects))
+ (when-not (nil? normalize-uri) (.setNormalizeUri config normalize-uri))
(.build config)))
(defmulti ^:private construct-http-host (fn [proxy-host proxy-port]
@@ -306,28 +287,37 @@
[{:keys [retry-handler request-interceptor
response-interceptor proxy-host proxy-port
http-builder-fns cookie-spec
- cookie-policy-registry]
+ cookie-policy-registry
+ ^HttpClientBuilder http-client-builder]
:as req}
caching?
conn-mgr
& [http-url proxy-ignore-hosts]]
;; have to let first, otherwise we get a reflection warning on (.build)
(let [cache? (opt req :cache)
- ^HttpClientBuilder builder (-> (if caching?
- (CachingHttpClientBuilder/create)
- (HttpClients/custom))
- (.setConnectionManager conn-mgr)
- (.setRedirectStrategy
- (get-redirect-strategy req))
- (add-retry-handler retry-handler)
- ;; By default, get the proxy settings
- ;; from the jvm or system properties
- (.setRoutePlanner
- (get-route-planner
- proxy-host proxy-port
- proxy-ignore-hosts http-url)))]
+ builder (-> (cond
+ http-client-builder http-client-builder
+ caching?
+ ^HttpClientBuilder (CachingHttpClientBuilder/create)
+ :else
+ ^HttpClientBuilder (HttpClients/custom))
+ (.setConnectionManager conn-mgr)
+ (.setRedirectStrategy
+ (get-redirect-strategy req))
+ (add-retry-handler retry-handler)
+
+ ;; prefer using clj-http.client/wrap-decompression
+ ;; for consistency between sync/async client options
+ (.disableContentCompression)
+
+ ;; By default, get the proxy settings
+ ;; from the jvm or system properties
+ (.setRoutePlanner
+ (get-route-planner
+ proxy-host proxy-port
+ proxy-ignore-hosts http-url)))]
(when cache?
- (.setCacheConfig builder (build-cache-config req)))
+ (.setCacheConfig ^CachingHttpClientBuilder builder (build-cache-config req)))
(when (or cookie-policy-registry cookie-spec)
(if cookie-policy-registry
;; They have a custom registry they'd like to re-use, so use that
@@ -401,6 +391,7 @@
(getMethod [] (.toUpperCase (name method) Locale/ROOT)))
(.setURI (URI. url)))))
+(def proxy-head-with-body (make-proxy-method-with-body :head))
(def proxy-delete-with-body (make-proxy-method-with-body :delete))
(def proxy-get-with-body (make-proxy-method-with-body :get))
(def proxy-copy-with-body (make-proxy-method-with-body :copy))
@@ -423,7 +414,9 @@
:get (if body
(proxy-get-with-body http-url)
(HttpGet. http-url))
- :head (HttpHead. http-url)
+ :head (if body
+ (proxy-head-with-body http-url)
+ (HttpHead. http-url))
:put (HttpPut. http-url)
:post (HttpPost. http-url)
:options (HttpOptions. http-url)
@@ -473,27 +466,29 @@
(defn- print-debug!
"Print out debugging information to *out* for a given request."
[{:keys [debug-body body] :as req} http-req]
- (println "Request:" (type body))
- (clojure.pprint/pprint
- (assoc req
- :body (if (opt req :debug-body)
- (cond
- (isa? (type body) String)
- body
-
- (isa? (type body) HttpEntity)
- (let [baos (ByteArrayOutputStream.)]
- (.writeTo ^HttpEntity body baos)
- (.toString baos "UTF-8"))
-
- :else nil)
- (if (isa? (type body) String)
- (format "... %s bytes ..."
- (count body))
- (and body (bean body))))
- :body-type (type body)))
- (println "HttpRequest:")
- (clojure.pprint/pprint (bean http-req)))
+ (println
+ (with-out-str
+ (println "Request:" (type body))
+ (clojure.pprint/pprint
+ (assoc req
+ :body (if (opt req :debug-body)
+ (cond
+ (isa? (type body) String)
+ body
+
+ (isa? (type body) HttpEntity)
+ (let [baos (ByteArrayOutputStream.)]
+ (.writeTo ^HttpEntity body baos)
+ (.toString baos "UTF-8"))
+
+ :else nil)
+ (if (isa? (type body) String)
+ (format "... %s bytes ..."
+ (count body))
+ (and body (bean body))))
+ :body-type (type body)))
+ (println "HttpRequest:")
+ (clojure.pprint/pprint (bean http-req)))))
(defn- build-response-map
[^HttpResponse response req ^HttpUriRequest http-req http-url
@@ -519,7 +514,7 @@
:reason-phrase (.getReasonPhrase status)
:trace-redirects (mapv str (.getRedirectLocations context))
:cached (when (instance? HttpCacheContext context)
- (when-let [cache-resp (.getCacheResponseStatus context)]
+ (when-let [cache-resp (.getCacheResponseStatus ^HttpCacheContext context)]
(-> cache-resp str keyword)))}]
(if (opt req :save-request)
(-> response
@@ -553,13 +548,16 @@
(defn request
([req] (request req nil nil))
- ([{:keys [body conn-timeout conn-request-timeout connection-manager
+ ([{:keys [body connection-timeout connection-request-timeout connection-manager
cookie-store cookie-policy headers multipart query-string
redirect-strategy max-redirects retry-handler
request-method scheme server-name server-port socket-timeout
uri response-interceptor proxy-host proxy-port
http-client-context http-request-config http-client
- proxy-ignore-hosts proxy-user proxy-pass digest-auth ntlm-auth]
+ proxy-ignore-hosts proxy-user proxy-pass digest-auth ntlm-auth
+ multipart-mode multipart-charset
+ ; deprecated
+ conn-timeout conn-request-timeout]
:as req} respond raise]
(let [async? (opt req :async)
cache? (opt req :cache)
@@ -611,7 +609,10 @@
(if (string? body)
(StringEntity. ^String body "UTF-8")
(ByteArrayEntity. body))))))
- (doseq [[header-n header-v] headers]
+ (doseq [[header-n header-v] headers
+ :when (or (not multipart)
+ (and (not= "content-type" header-n)
+ (not= "Content-Type" header-n)))]
(if (coll? header-v)
(doseq [header-vth header-v]
(.addHeader http-req header-n header-vth))
@@ -630,7 +631,9 @@
(conn/shutdown-manager conn-mgr))
(throw t))))
(let [^CloseableHttpAsyncClient client
- (build-async-http-client req conn-mgr http-url proxy-ignore-hosts)]
+ (or http-client
+ (build-async-http-client req conn-mgr http-url proxy-ignore-hosts))
+ original-thread-bindings (clojure.lang.Var/getThreadBindingFrame)]
(when cache?
(throw (IllegalArgumentException.
"caching is not yet supported for async clients")))
@@ -638,12 +641,14 @@
(.execute client http-req context
(reify org.apache.http.concurrent.FutureCallback
(failed [this ex]
+ (clojure.lang.Var/resetThreadBindingFrame original-thread-bindings)
(when-not (conn/reusable? conn-mgr)
(conn/shutdown-manager conn-mgr))
(if (opt req :ignore-unknown-host)
((:unknown-host-respond req) nil)
(raise ex)))
(completed [this resp]
+ (clojure.lang.Var/resetThreadBindingFrame original-thread-bindings)
(try
(respond (build-response-map
resp req http-req http-url
@@ -653,6 +658,7 @@
(conn/shutdown-manager conn-mgr))
(raise t))))
(cancelled [this]
+ (clojure.lang.Var/resetThreadBindingFrame original-thread-bindings)
;; Run the :oncancel function if available
(when-let [oncancel (:oncancel req)]
(oncancel))
diff --git a/src/clj_http/core_old.clj b/src/clj_http/core_old.clj
index d6a7e8c1..52e17077 100644
--- a/src/clj_http/core_old.clj
+++ b/src/clj_http/core_old.clj
@@ -4,34 +4,24 @@
[clj-http.headers :as headers]
[clj-http.multipart :as mp]
[clj-http.util :refer [opt]]
- [clojure.pprint])
- (:import (java.io ByteArrayOutputStream FilterInputStream InputStream)
-
- (org.apache.http HeaderIterator HttpEntity
- HttpEntityEnclosingRequest
- HttpResponse Header HttpHost
- HttpResponseInterceptor)
- (org.apache.http.auth UsernamePasswordCredentials AuthScope
- NTCredentials)
- (org.apache.http.params CoreConnectionPNames)
- (org.apache.http.client HttpClient HttpRequestRetryHandler)
- (org.apache.http.client.methods HttpDelete
- HttpEntityEnclosingRequestBase
- HttpGet HttpHead HttpOptions
- HttpPatch HttpPost HttpPut
- HttpUriRequest)
- (org.apache.http.client.params CookiePolicy ClientPNames)
- (org.apache.http.conn ClientConnectionManager)
- (org.apache.http.conn.routing HttpRoute)
- (org.apache.http.conn.params ConnRoutePNames)
- (org.apache.http.cookie CookieSpecFactory)
- (org.apache.http.cookie.params CookieSpecPNames)
- (org.apache.http.entity ByteArrayEntity StringEntity)
-
- (org.apache.http.impl.client DefaultHttpClient)
- (org.apache.http.impl.conn ProxySelectorRoutePlanner)
- (org.apache.http.impl.cookie BrowserCompatSpec)
- (java.net URI)))
+ clojure.pprint)
+ (:import [java.io ByteArrayOutputStream FilterInputStream InputStream]
+ java.net.URI
+ [org.apache.http HeaderIterator HttpEntity HttpEntityEnclosingRequest HttpHost HttpResponseInterceptor]
+ [org.apache.http.auth AuthScope NTCredentials UsernamePasswordCredentials]
+ [org.apache.http.client HttpClient HttpRequestRetryHandler]
+ [org.apache.http.client.methods HttpDelete HttpEntityEnclosingRequestBase HttpGet HttpHead HttpOptions HttpPatch HttpPost HttpPut HttpUriRequest]
+ [org.apache.http.client.params ClientPNames CookiePolicy]
+ org.apache.http.conn.ClientConnectionManager
+ org.apache.http.conn.params.ConnRoutePNames
+ org.apache.http.conn.routing.HttpRoute
+ org.apache.http.cookie.CookieSpecFactory
+ org.apache.http.cookie.params.CookieSpecPNames
+ [org.apache.http.entity ByteArrayEntity StringEntity]
+ org.apache.http.impl.client.DefaultHttpClient
+ org.apache.http.impl.conn.ProxySelectorRoutePlanner
+ org.apache.http.impl.cookie.BrowserCompatSpec
+ org.apache.http.params.CoreConnectionPNames))
(defn parse-headers
"Takes a HeaderIterator and returns a map of names to values.
@@ -212,10 +202,13 @@
Note that where Ring uses InputStreams for the request and response bodies,
the clj-http uses ByteArrays for the bodies."
[{:keys [request-method scheme server-name server-port uri query-string
- headers body multipart socket-timeout conn-timeout proxy-host
+ headers body multipart socket-timeout connection-timeout proxy-host
proxy-ignore-hosts proxy-port proxy-user proxy-pass as cookie-store
retry-handler response-interceptor digest-auth ntlm-auth
- connection-manager client-params]
+ connection-manager client-params
+ ; deprecated
+ conn-timeout
+ ]
:as req}]
(let [^ClientConnectionManager conn-mgr
(or connection-manager
@@ -237,7 +230,8 @@
;; merge in map of specified timeouts, to
;; support backward compatibility.
(merge {CoreConnectionPNames/SO_TIMEOUT socket-timeout
- CoreConnectionPNames/CONNECTION_TIMEOUT conn-timeout}
+ CoreConnectionPNames/CONNECTION_TIMEOUT (or connection-timeout
+ conn-timeout)}
client-params))
(when-let [[user pass] digest-auth]
diff --git a/src/clj_http/headers.clj b/src/clj_http/headers.clj
index 9b1e068e..d229320c 100644
--- a/src/clj_http/headers.clj
+++ b/src/clj_http/headers.clj
@@ -8,8 +8,8 @@
\"Accept-Encoding\")."
(:require [clojure.string :as s]
[potemkin :as potemkin])
- (:import (java.util Locale)
- (org.apache.http Header HeaderIterator)))
+ (:import java.util.Locale
+ [org.apache.http Header HeaderIterator]))
(def special-cases
"A collection of HTTP headers that do not follow the normal
@@ -119,9 +119,14 @@
mta)
(with-meta [_ mta]
(HeaderMap. m mta))
+
clojure.lang.Associative
(containsKey [_ k]
(contains? m (normalize k)))
+ (entryAt [_ k]
+ (if (contains? m (normalize k))
+ (clojure.lang.MapEntry. k (get _ k))))
+
(empty [_]
(HeaderMap. {} nil)))
diff --git a/src/clj_http/multipart.clj b/src/clj_http/multipart.clj
index e2691556..e4381ace 100644
--- a/src/clj_http/multipart.clj
+++ b/src/clj_http/multipart.clj
@@ -1,15 +1,10 @@
(ns clj-http.multipart
"Namespace used for clj-http to create multipart entities and bodies."
- (:import (java.io File InputStream)
- (org.apache.http.entity ContentType)
- (org.apache.http.entity.mime MultipartEntityBuilder)
- (org.apache.http.entity.mime HttpMultipartMode)
- (org.apache.http.entity.mime.content ContentBody
- ByteArrayBody
- FileBody
- InputStreamBody
- StringBody)
- (org.apache.http Consts)))
+ (:import [java.io File InputStream]
+ org.apache.http.Consts
+ org.apache.http.entity.ContentType
+ [org.apache.http.entity.mime HttpMultipartMode MultipartEntityBuilder]
+ [org.apache.http.entity.mime.content ByteArrayBody ContentBody FileBody InputStreamBody StringBody]))
;; we don't need to make a fake byte-array every time, only once
(def byte-array-type (type (byte-array 0)))
@@ -125,16 +120,41 @@
[{:keys [^ContentBody content]}]
content)
+(defn- multipart-workaround
+ "Workaround for AsyncHttpClient to bypass 25kb restriction on getContent.
+
+ See https://github.com/dakrone/clj-http/issues/560.
+ "
+ [^org.apache.http.entity.mime.MultipartFormEntity mp-entity]
+ (reify org.apache.http.HttpEntity
+ (isRepeatable [_] (.isRepeatable mp-entity))
+ (isChunked [_] (.isChunked mp-entity))
+ (isStreaming [_] (.isStreaming mp-entity))
+ (getContentLength [_] (.getContentLength mp-entity))
+ (getContentType [_] (.getContentType mp-entity))
+ (getContentEncoding [_] (.getContentEncoding mp-entity))
+ (consumeContent [_] (.consumeContent mp-entity))
+ (getContent [_]
+ (let [os (java.io.ByteArrayOutputStream.)]
+ (.writeTo mp-entity os)
+ (.flush os)
+ (java.io.ByteArrayInputStream. (.toByteArray os))))
+ (writeTo [_ output-stream] (.writeTo mp-entity output-stream))))
+
(defn create-multipart-entity
"Takes a multipart vector of maps and creates a MultipartEntity with each
map added as a part, depending on the type of content."
- [multipart {:keys [mime-subtype]}]
+ [multipart {:keys [mime-subtype multipart-mode multipart-charset]
+ :or {mime-subtype "form-data"
+ multipart-mode HttpMultipartMode/STRICT}}]
(let [mp-entity (doto (MultipartEntityBuilder/create)
- (.setStrictMode)
- (.setCharset (encoding-to-charset "UTF-8"))
- (.setMimeSubtype (or mime-subtype "form-data")))]
+ (.setMode multipart-mode)
+ (.setMimeSubtype mime-subtype))]
+ (when multipart-charset
+ (.setCharset mp-entity (encoding-to-charset multipart-charset)))
(doseq [m multipart]
(let [name (or (:part-name m) (:name m))
part (make-multipart-body m)]
(.addPart mp-entity name part)))
- (.build mp-entity)))
+ (multipart-workaround
+ (.build mp-entity))))
diff --git a/src/clj_http/util.clj b/src/clj_http/util.clj
index 5a5a4a59..2b14a81a 100644
--- a/src/clj_http/util.clj
+++ b/src/clj_http/util.clj
@@ -2,13 +2,11 @@
"Helper functions for the HTTP client."
(:require [clojure.string :refer [blank? lower-case split trim]]
[clojure.walk :refer [postwalk]])
- (:import (org.apache.commons.codec.binary Base64)
- (org.apache.commons.io IOUtils)
- (java.io BufferedInputStream ByteArrayInputStream
- ByteArrayOutputStream EOFException)
- (java.net URLEncoder URLDecoder)
- (java.util.zip InflaterInputStream DeflaterInputStream
- GZIPInputStream GZIPOutputStream)))
+ (:import [java.io BufferedInputStream ByteArrayInputStream ByteArrayOutputStream EOFException InputStream PushbackInputStream]
+ [java.net URLDecoder URLEncoder]
+ [java.util.zip DeflaterInputStream GZIPInputStream GZIPOutputStream InflaterInputStream]
+ org.apache.commons.codec.binary.Base64
+ org.apache.commons.io.IOUtils))
(defn utf8-bytes
"Returns the encoding's bytes corresponding to the given string. If no
@@ -25,12 +23,12 @@
(defn url-decode
"Returns the form-url-decoded version of the given string, using either a
specified encoding or UTF-8 by default."
- [encoded & [encoding]]
+ [^String encoded & [^String encoding]]
(URLDecoder/decode encoded (or encoding "UTF-8")))
(defn url-encode
"Returns an UTF-8 URL encoded version of the given string."
- [unencoded & [encoding]]
+ [^String unencoded & [^String encoding]]
(URLEncoder/encode unencoded (or encoding "UTF-8")))
(defn base64-encode
@@ -43,8 +41,13 @@
[b]
(when b
(cond
- (instance? java.io.InputStream b)
- (GZIPInputStream. b)
+ (instance? InputStream b)
+ (let [^PushbackInputStream b (PushbackInputStream. b)
+ first-byte (int (try (.read b) (catch EOFException _ -1)))]
+ (case first-byte
+ -1 b
+ (do (.unread b first-byte)
+ (GZIPInputStream. b))))
:else
(IOUtils/toByteArray (GZIPInputStream. (ByteArrayInputStream. b))))))
@@ -58,24 +61,41 @@
(.close gos)
(.toByteArray baos))))
+(defn force-stream
+ "Force b as InputStream if it is a ByteArray."
+ ^InputStream [b]
+ (if (instance? InputStream b)
+ b
+ (ByteArrayInputStream. b)))
+
(defn force-byte-array
"force b as byte array if it is an InputStream, also close the stream"
^bytes [b]
- (if (instance? java.io.InputStream b)
- (try
- (let [^int first-byte (try
- (.read b)
- (catch EOFException e -1))]
- (if (= -1 first-byte)
- (byte-array 0)
- (let [rest-bytes (IOUtils/toByteArray ^java.io.InputStream b)
- barray (byte-array (inc (count rest-bytes)))]
- (aset-byte barray 0 (unchecked-byte first-byte))
- (System/arraycopy rest-bytes 0 barray 1 (count rest-bytes))
- barray)))
- (finally (.close ^java.io.InputStream b)))
+ (if (instance? InputStream b)
+ (let [^PushbackInputStream bs (PushbackInputStream. b)]
+ (try
+ (let [first-byte (int (try (.read bs) (catch EOFException _ -1)))]
+ (case first-byte
+ -1 (byte-array 0)
+ (do (.unread bs first-byte)
+ (IOUtils/toByteArray bs))))
+ (finally (.close bs))))
b))
+(defn force-string
+ "Convert s (a ByteArray or InputStream) to String."
+ ^String [s ^String charset]
+ (if (instance? InputStream s)
+ (let [^PushbackInputStream bs (PushbackInputStream. s)]
+ (try
+ (let [first-byte (int (try (.read bs) (catch EOFException _ -1)))]
+ (case first-byte
+ -1 ""
+ (do (.unread bs first-byte)
+ (IOUtils/toString bs charset))))
+ (finally (.close bs))))
+ (IOUtils/toString ^"[B" s charset)))
+
(defn inflate
"Returns a zlib inflate'd version of the given byte array or InputStream."
[b]
@@ -83,7 +103,7 @@
;; This weirdness is because HTTP servers lie about what kind of deflation
;; they're using, so we try one way, then if that doesn't work, reset and
;; try the other way
- (let [stream (BufferedInputStream. (if (instance? java.io.InputStream b)
+ (let [stream (BufferedInputStream. (if (instance? InputStream b)
b
(ByteArrayInputStream. b)))
_ (.mark stream 512)
@@ -123,15 +143,18 @@
false
(or v1 v2)))))
+(defn- trim-quotes [s]
+ (when s
+ (clojure.string/replace s #"^\s*(\"(.*)\"|(.*?))\s*$" "$2$3")))
+
(defn parse-content-type
"Parse `s` as an RFC 2616 media type."
[s]
- (if-let [m (re-matches #"\s*(([^/]+)/([^ ;]+))\s*(\s*;.*)?" (str s))]
+ (when-let [m (re-matches #"\s*(([^/]+)/([^ ;]+))\s*(\s*;.*)?" (str s))]
{:content-type (keyword (nth m 1))
:content-type-params
(->> (split (str (nth m 4)) #"\s*;\s*")
- (identity)
(remove blank?)
(map #(split % #"="))
- (mapcat (fn [[k v]] [(keyword (lower-case k)) (trim v)]))
+ (mapcat (fn [[k v]] [(keyword (lower-case k)) (trim-quotes v)]))
(apply hash-map))}))
diff --git a/test-resources/big_array_json.json b/test-resources/big_array_json.json
new file mode 100644
index 00000000..51ccef7d
--- /dev/null
+++ b/test-resources/big_array_json.json
@@ -0,0 +1,102 @@
+[
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]},
+ {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}
+]
diff --git a/test-resources/client-keystore b/test-resources/client-keystore
index d30af3db..55bdc0c4 100644
Binary files a/test-resources/client-keystore and b/test-resources/client-keystore differ
diff --git a/test-resources/keystore b/test-resources/keystore
index 13d78bf9..2944ca21 100644
Binary files a/test-resources/keystore and b/test-resources/keystore differ
diff --git a/test/clj_http/test/client_test.clj b/test/clj_http/test/client_test.clj
index 58b56f0c..1b111e99 100644
--- a/test/clj_http/test/client_test.clj
+++ b/test/clj_http/test/client_test.clj
@@ -4,17 +4,20 @@
[clj-http.conn-mgr :as conn]
[clj-http.test.core-test :refer [run-server]]
[clj-http.util :as util]
+ [clojure.java.io :as io :refer [resource]]
[clojure.string :as str]
- [clojure.java.io :refer [resource]]
[clojure.test :refer :all]
[cognitect.transit :as transit]
- [ring.util.codec :refer [form-decode-str]]
[ring.middleware.nested-params :refer [parse-nested-keys]]
+ [ring.util.codec :refer [form-decode-str]]
[slingshot.slingshot :refer [try+]])
- (:import (java.net UnknownHostException)
- (java.io ByteArrayInputStream)
- (org.apache.http HttpEntity)
- (org.apache.logging.log4j LogManager)))
+ (:import java.io.ByteArrayInputStream
+ java.io.PipedInputStream
+ java.io.PipedOutputStream
+ java.net.UnknownHostException
+ org.apache.http.HttpEntity
+ org.apache.http.HttpMessage
+ org.apache.logging.log4j.LogManager))
(defonce logger (LogManager/getLogger "clj-http.test.client-test"))
@@ -111,23 +114,101 @@
(is (= params (read-fn (:body @resp))))
(is (not (realized? exception)))))))
+(def ^:dynamic *test-dynamic-var* nil)
+
+(deftest ^:integration async-preserves-dynamic-variable-bindings
+ (run-server)
+ (let [expected-var "cat"]
+ (binding [*test-dynamic-var* expected-var]
+ (let [test-fn (fn [uri success-p fail-p]
+ (request {:uri uri
+ :method :get
+ :scheme "http"
+ :async? true}
+ (fn [_]
+ (deliver success-p *test-dynamic-var*)
+ (deliver fail-p :success))
+ (fn [_]
+ (deliver success-p :fail)
+ (deliver fail-p *test-dynamic-var*))))]
+ (testing "dynamic variables on success responses"
+ (let [success-p (promise)
+ fail-p (promise)]
+ (test-fn "/get" success-p fail-p)
+ (is (= @success-p expected-var *test-dynamic-var*))
+ (is (= @fail-p :success)
+ "Verify that we went through the success path, not the failure")))
+
+ (testing "dynamic variables on fail responses"
+ (let [success-p (promise)
+ fail-p (promise)]
+ (test-fn "/json-bad" success-p fail-p)
+ (is (= @success-p :fail)
+ "Verify that we went through the failure path, not the success")
+ (is (= @fail-p expected-var *test-dynamic-var*))))))))
+
+(defn retrieve-http-request-content-type-header
+ [response]
+ (let [http-req (get-in response [:request :http-req])]
+ (->> (.getAllHeaders ^HttpMessage http-req)
+ (map str)
+ (some #(when (str/starts-with? (str/lower-case %) "content-type") %)))))
+
(deftest ^:integration multipart-async
(run-server)
- (let [resp (promise)
- exception (promise)
- _ (request {:uri "/post" :method :post
- :async? true
- :multipart [{:name "title" :content "some-file"}
- {:name "Content/Type" :content "text/plain"}
- {:name "file"
- :content (clojure.java.io/file
- "test-resources/m.txt")}]}
- resp
- exception
- )]
- (is (= 200 (:status @resp)))
- (is (not (realized? exception)))
- #_(when (realized? exception) (prn @exception))))
+
+ (testing "basics"
+ (let [resp (promise)
+ exception (promise)
+ _ (request {:uri "/post" :method :post
+ :async? true
+ :multipart [{:name "title" :content "some-file"}
+ {:name "Content/Type" :content "text/plain"}
+ {:name "file"
+ :content (io/file "test-resources/m.txt")}]}
+ resp
+ exception)]
+ (is (= 200 (:status (deref resp 500 :failed))))
+ (is (not (realized? exception)))
+ #_(when (realized? exception) (prn @exception))))
+
+ ;; Regression Testing https://github.com/dakrone/clj-http/issues/560
+ (testing "multipart uploads larger than 25kb"
+ (let [resp (promise)
+ exception (promise)
+ ;; assumption: file > 5kb
+ file (io/file "test-resources/big_array_json.json")
+
+ _ (request {:uri "/post" :method :post
+ :async? true
+ :multipart [{:name "part-1" :content file}
+ {:name "part-2" :content file}
+ {:name "part-3" :content file}
+ {:name "part-4" :content file}
+ {:name "part-5" :content file}]}
+ resp
+ exception)]
+ (is (= 200 (:status (deref resp 500 :failed))))
+ (is (not (realized? exception)))))
+
+ ;; Find the details in https://github.com/dakrone/clj-http/pull/654
+ (testing "existing \"Content-Type\" request header is discarded"
+ (let [resp (request {:uri "/post" :method :post
+ :headers {"content-type" "multipart/form-data"}
+ :multipart [{:name "some" :content "thing"}]
+ :save-request? true})
+ content-type (retrieve-http-request-content-type-header resp)]
+ (is (= 200 (:status resp)))
+ (is (not= "multipart/form-data" content-type))
+ (is (nil? content-type)))
+ (let [resp (request {:uri "/post" :method :post
+ :headers {"Content-Type" "multipart/form-data"}
+ :multipart [{:name "some" :content "thing"}]
+ :save-request? true})
+ content-type (retrieve-http-request-content-type-header resp)]
+ (is (= 200 (:status resp)))
+ (is (not= "multipart/form-data" content-type))
+ (is (nil? content-type)))))
(deftest ^:integration nil-input
(is (thrown-with-msg? Exception #"Host URL cannot be nil"
@@ -424,7 +505,7 @@
(deftest pass-on-non-redirectable-methods
(doseq [method [:put :post :delete]
- status [301 302 307]]
+ status [301 302 307 308]]
(let [client (fn [req] {:status status :body (:body req)
:headers {"location" "http://example.com/bat"}})
r-client (client/wrap-redirects client)
@@ -437,7 +518,7 @@
(deftest pass-on-non-redirectable-methods-async
(doseq [method [:put :post :delete]
- status [301 302 307]]
+ status [301 302 307 308]]
(let [client (fn [req respond raise]
(respond {:status status :body (:body req)
:headers {"location" "http://example.com/bat"}}))
@@ -1478,8 +1559,15 @@
(is (= all-legal
(client/url-encode-illegal-characters all-legal)))))
+(defmethod client/coerce-response-body :json+ms949
+ [req resp]
+ (client/coerce-json-body req resp true "MS949"))
+
(deftest t-coercion-methods
(let [json-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"bar\"}"))
+ json-ms949-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"안뇽\"}" "MS949"))
+ xml-body (ByteArrayInputStream. (.getBytes "bar"))
+ xml-auto-body (ByteArrayInputStream. (.getBytes "bar"))
auto-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"bar\"}"))
edn-body (ByteArrayInputStream. (.getBytes "{:foo \"bar\"}"))
transit-json-body (ByteArrayInputStream.
@@ -1493,6 +1581,12 @@
(ByteArrayInputStream. (.getBytes "foo=bar"))
json-resp {:body json-body :status 200
:headers {"content-type" "application/json"}}
+ json-ms949-resp {:body json-ms949-body :status 200
+ :headers {"content-type" "application/json; charset=ms949"}}
+ xml-resp {:body xml-body :status 200
+ :headers {"content-type" "text/xml;charset=utf-8"}}
+ xml-auto-resp {:body xml-auto-body :status 200
+ :headers {"content-type" "text/xml;charset=utf-8"}}
auto-resp {:body auto-body :status 200
:headers {"content-type" "application/json"}}
edn-resp {:body edn-body :status 200
@@ -1521,7 +1615,39 @@
(:body (client/coerce-response-body {:as :auto}
auto-www-form-urlencoded-resp))
(:body (client/coerce-response-body {:as :x-www-form-urlencoded}
- www-form-urlencoded-resp))))))
+ www-form-urlencoded-resp))))
+ (is (= {:tag :foo, :attrs nil, :content ["bar"]}
+ (:body (client/coerce-response-body {:as :xml} xml-resp))
+ (:body (client/coerce-response-body {:as :auto} xml-auto-resp))))
+ (is (= {:foo "안뇽"}
+ (:body (client/coerce-response-body {:as :json+ms949} json-ms949-resp))))
+
+ (testing "throws AssertionError when optional libraries are not loaded"
+ (with-redefs [client/json-enabled? false]
+ (is (thrown? AssertionError (client/coerce-response-body {:as :json} json-resp)))
+ (is (thrown? AssertionError (client/coerce-response-body {:as :auto} json-resp))))
+ (with-redefs [client/transit-enabled? false]
+ (is (thrown? AssertionError (client/coerce-response-body {:as :transit+json} transit-json-resp)))
+ (is (thrown? AssertionError (client/coerce-response-body {:as :transit+msgpack} transit-msgpack-resp))))
+ (with-redefs [client/ring-codec-enabled? false]
+ (is (thrown? AssertionError (client/coerce-response-body {:as :x-www-form-urlencoded} www-form-urlencoded-resp)))
+ (is (thrown? AssertionError (client/coerce-response-body {:as :auto} auto-www-form-urlencoded-resp)))))))
+
+
+(deftest t-reader-coercion
+ (let [read-lines (fn [reader] (vec (take-while not-empty (repeatedly #(.readLine reader)))))
+ reader-body (ByteArrayInputStream. (.getBytes "foo\nbar\n"))
+ reader-resp {:body reader-body :status 200 :headers {"content-type" "text/plain; charset=utf-8"}}
+ encoded-body (ByteArrayInputStream. (byte-array [0xA9]))
+ encoded-resp {:body encoded-body :status 200 :headers {"content-type" "text/plain; charset=iso-8859-1"}}
+ utf8-body (ByteArrayInputStream. (byte-array [0xC2 0xA9]))
+ utf8-resp {:body utf8-body :status 200 :headers {"content-type" "text/plain; charset=utf-8"}}]
+ (is (= ["foo" "bar"]
+ (read-lines (:body (client/coerce-response-body {:as :reader} reader-resp)))))
+
+ (is (= "©"
+ (.readLine (:body (client/coerce-response-body {:as :reader} encoded-resp)))
+ (.readLine (:body (client/coerce-response-body {:as :reader} utf8-resp)))))))
(deftest ^:integration t-with-middleware
(run-server)
@@ -1608,7 +1734,17 @@
query-string (-> resp :body form-decode-str)]
(is (= 200 (:status resp)))
(is (.contains query-string "a[]=1&a[]=2&a[]=3") query-string)
- (is (.contains query-string "b[]=x&b[]=y&b[]=z") query-string))))
+ (is (.contains query-string "b[]=x&b[]=y&b[]=z") query-string)))
+ (testing "multi-valued query params in comma-separated"
+ (let [resp (request {:uri "/query-string"
+ :method :get
+ :multi-param-style :comma-separated
+ :query-params {:a [1 2 3]
+ :b ["x" "y" "z"]}})
+ query-string (-> resp :body form-decode-str)]
+ (is (= 200 (:status resp)))
+ (is (.contains query-string "a=1,2,3") query-string)
+ (is (.contains query-string "b=x,y,z") query-string))))
(deftest t-wrap-flatten-nested-params
(is-applied client/wrap-flatten-nested-params
@@ -1656,3 +1792,36 @@
(is (= (.getMessage e)
(str "only :flatten-nested-keys or :ignore-nested-query-string/"
":flatten-nested-keys may be specified, not both"))))))
+
+(defn transit-resp [body]
+ {:body body
+ :status 200
+ :headers {"content-type" "application/transit-json"}})
+
+(deftest issue-609-empty-transit-response
+ (testing "Body is available right away"
+ (is (= {:foo "bar"}
+ (:body (client/coerce-response-body
+ {:as :transit+json}
+ (transit-resp (ByteArrayInputStream.
+ (.getBytes "[\"^ \",\"~:foo\",\"bar\"]"))))))))
+
+ (testing "Empty body is read as nil"
+ (is (nil? (:body (client/coerce-response-body
+ {:as :transit+json}
+ (transit-resp (ByteArrayInputStream. (.getBytes ""))))))))
+
+ (testing "Body is read correctly even if the data becomes available later"
+ ;; Ensure both streams are closed (normally done inside future).
+ (with-open [o (PipedOutputStream.)
+ i (PipedInputStream.)]
+ (.connect i o)
+ (future
+ (Thread/sleep 10)
+ (.write o (.getBytes "[\"^ \",\"~:foo\",\"bar\"]"))
+ ;; Close right now, with-open will wait until test is done.
+ (.close o))
+ (is (= {:foo "bar"}
+ (:body (client/coerce-response-body
+ {:as :transit+json}
+ (transit-resp i))))))))
diff --git a/test/clj_http/test/conn_mgr_test.clj b/test/clj_http/test/conn_mgr_test.clj
index 9fd678ba..1cc7dbad 100644
--- a/test/clj_http/test/conn_mgr_test.clj
+++ b/test/clj_http/test/conn_mgr_test.clj
@@ -4,15 +4,9 @@
[clj-http.test.core-test :refer [run-server]]
[clojure.test :refer :all]
[ring.adapter.jetty :as ring])
- (:import (java.security KeyStore)
- (org.apache.http.impl.conn BasicHttpClientConnectionManager)
- (org.apache.http.conn.ssl SSLConnectionSocketFactory
- DefaultHostnameVerifier
- NoopHostnameVerifier
- TrustStrategy)
- (org.apache.http.conn.socket PlainConnectionSocketFactory)
- (org.apache.http.nio.conn NoopIOSessionStrategy)
- (org.apache.http.nio.conn.ssl SSLIOSessionStrategy)))
+ (:import java.security.KeyStore
+ [javax.net.ssl KeyManagerFactory TrustManagerFactory]
+ org.apache.http.impl.conn.BasicHttpClientConnectionManager))
(def client-ks "test-resources/client-keystore")
(def client-ks-pass "keykey")
@@ -42,25 +36,17 @@
(let [ks (conn-mgr/get-keystore "test-resources/keystore" nil nil)]
(is (instance? KeyStore ks))))
-(deftest keystore-scheme-factory
- (let [sr (conn-mgr/get-keystore-scheme-registry
- {:keystore client-ks :keystore-pass client-ks-pass
- :trust-store client-ks :trust-store-pass client-ks-pass})
- plain-socket-factory (.lookup sr "http")
- ssl-socket-factory (.lookup sr "https")]
- (is (instance? PlainConnectionSocketFactory plain-socket-factory))
- (is (instance? SSLConnectionSocketFactory ssl-socket-factory))))
+(def array-of-trust-manager
+ (let [ks (conn-mgr/get-keystore "test-resources/keystore" nil "keykey")
+ tmf (doto (TrustManagerFactory/getInstance (TrustManagerFactory/getDefaultAlgorithm))
+ (.init ks))]
+ (.getTrustManagers tmf)))
-(deftest keystore-session-strategy
- (let [strategy-registry (conn-mgr/get-keystore-strategy-registry
- {:keystore client-ks
- :keystore-pass client-ks-pass
- :trust-store client-ks
- :trust-store-pass client-ks-pass})
- noop-session-strategy (.lookup strategy-registry "http")
- ssl-session-strategy (.lookup strategy-registry "https")]
- (is (instance? NoopIOSessionStrategy noop-session-strategy))
- (is (instance? SSLIOSessionStrategy ssl-session-strategy))))
+(def array-of-key-manager
+ (let [ks (conn-mgr/get-keystore "test-resources/keystore" nil "keykey")
+ tmf (doto (KeyManagerFactory/getInstance (KeyManagerFactory/getDefaultAlgorithm))
+ (.init ks (.toCharArray "keykey")))]
+ (.getKeyManagers tmf)))
(deftest ^:integration ssl-client-cert-get
(let [server (ring/run-jetty secure-handler
@@ -100,6 +86,23 @@
exception (promise)
_ (core/request (assoc secure-request :async? true) resp exception)]
(is (= 200 (:status (deref resp 1000 {:status :timeout})))))
+
+ (testing "with reusable connection pool"
+ (let [pool (conn-mgr/make-reusable-async-conn-manager {:timeout 10000
+ :keystore client-ks :keystore-pass client-ks-pass
+ :trust-store client-ks :trust-store-pass client-ks-pass
+ :insecure? true})]
+ (try
+ (let [resp (promise) exception (promise)
+ _ (core/request {:request-method :get :uri "/get"
+ :server-port 18084 :scheme :https
+ :server-name "localhost"
+ :connection-manager pool :async? true} resp exception)]
+ (is (= 200 (:status (deref resp 1000 {:status :timeout}))))
+ (is (:body @resp))
+ (is (not (realized? exception))))
+ (finally
+ (conn-mgr/shutdown-manager pool)))))
(finally
(.stop server)))))
@@ -115,7 +118,7 @@
:server-name "localhost"
;; timeouts forces an exception being thrown
:socket-timeout 1
- :conn-timeout 1
+ :connection-timeout 1
:connection-manager cm
:as :stream})
(is false "request should have thrown an exception")
@@ -140,8 +143,10 @@
(let [regular (conn-mgr/make-regular-conn-manager {})
regular-reusable (conn-mgr/make-reusable-conn-manager {})
async (conn-mgr/make-regular-async-conn-manager {})
- async-reusable (conn-mgr/make-reuseable-async-conn-manager {})]
+ async-reusable (conn-mgr/make-reusable-async-conn-manager {})
+ async-reuseable (conn-mgr/make-reuseable-async-conn-manager {})]
(is (false? (conn-mgr/reusable? regular)))
(is (true? (conn-mgr/reusable? regular-reusable)))
(is (false? (conn-mgr/reusable? async)))
- (is (true? (conn-mgr/reusable? async-reusable)))))
+ (is (true? (conn-mgr/reusable? async-reusable)))
+ (is (true? (conn-mgr/reusable? async-reuseable)))))
diff --git a/test/clj_http/test/cookies_test.clj b/test/clj_http/test/cookies_test.clj
index e5a5e43d..8dd2b351 100644
--- a/test/clj_http/test/cookies_test.clj
+++ b/test/clj_http/test/cookies_test.clj
@@ -1,8 +1,7 @@
(ns clj-http.test.cookies-test
(:require [clj-http.cookies :refer :all]
- [clj-http.util :refer :all]
[clojure.test :refer :all])
- (:import (org.apache.http.impl.cookie BasicClientCookie BasicClientCookie2)))
+ (:import [org.apache.http.impl.cookie BasicClientCookie BasicClientCookie2]))
(defn refer-private [ns]
(doseq [[symbol var] (ns-interns ns)]
diff --git a/test/clj_http/test/core_test.clj b/test/clj_http/test/core_test.clj
index 582aec15..fc65247d 100644
--- a/test/clj_http/test/core_test.clj
+++ b/test/clj_http/test/core_test.clj
@@ -5,29 +5,22 @@
[clj-http.core :as core]
[clj-http.util :as util]
[clojure.java.io :refer [file]]
- [clojure.pprint :as pp]
[clojure.test :refer :all]
[ring.adapter.jetty :as ring])
- (:import (java.io ByteArrayInputStream)
- (java.net SocketTimeoutException)
- (java.util.concurrent TimeoutException TimeUnit)
- (org.apache.http.params CoreConnectionPNames CoreProtocolPNames)
- (org.apache.http.message BasicHeader BasicHeaderIterator)
- (org.apache.http.client.methods HttpPost)
- (org.apache.http.client.protocol HttpClientContext)
- (org.apache.http.client.config RequestConfig)
- (org.apache.http.client.params CookiePolicy ClientPNames)
- (org.apache.http.conn.util PublicSuffixMatcherLoader)
- (org.apache.http.cookie CommonCookieAttributeHandler)
- (org.apache.http HttpRequest HttpResponse HttpConnection
- HttpInetConnection HttpVersion ProtocolException)
- (org.apache.http.protocol HttpContext ExecutionContext)
- (org.apache.http.impl.client DefaultHttpClient)
- (org.apache.http.impl.cookie RFC6265CookieSpec RFC6265CookieSpecProvider
- RFC6265CookieSpecProvider$CompatibilityLevel)
- (org.apache.http.client.params ClientPNames)
- (org.apache.logging.log4j LogManager)
- (sun.security.provider.certpath SunCertPathBuilderException)))
+ (:import java.io.ByteArrayInputStream
+ [java.net InetAddress SocketTimeoutException]
+ [java.util.concurrent TimeoutException TimeUnit]
+ [org.apache.http HttpConnection HttpInetConnection HttpRequest HttpResponse ProtocolException]
+ org.apache.http.client.config.RequestConfig
+ org.apache.http.client.params.ClientPNames
+ org.apache.http.client.protocol.HttpClientContext
+ org.apache.http.impl.conn.InMemoryDnsResolver
+ org.apache.http.impl.cookie.RFC6265CookieSpecProvider
+ [org.apache.http.message BasicHeader BasicHeaderIterator]
+ [org.apache.http.params CoreConnectionPNames CoreProtocolPNames]
+ [org.apache.http.protocol ExecutionContext HttpContext]
+ org.apache.logging.log4j.LogManager
+ sun.security.provider.certpath.SunCertPathBuilderException))
(defonce logger (LogManager/getLogger "clj-http.test.core-test"))
@@ -58,6 +51,9 @@
[:get "/json-array"]
{:status 200 :body "[\"foo\", \"bar\"]"
:headers {"content-type" "application/json"}}
+ [:get "/json-large-array"]
+ {:status 200 :body (file "test-resources/big_array_json.json")
+ :headers {"content-type" "application/json"}}
[:get "/json-bad"]
{:status 400 :body "{\"foo\":\"bar\"}"}
[:get "/redirect"]
@@ -108,6 +104,8 @@
{:status 200 :body "delete-with-body"}
[:post "/multipart"]
{:status 200 :body (:body req)}
+ [:head "/head-with-body"]
+ {:status 200 :headers {"body" (slurp (:body req))}}
[:get "/get-with-body"]
{:status 200 :body (:body req)}
[:options "/options"]
@@ -147,7 +145,7 @@
(defn run-server
[]
(defonce server
- (ring/run-jetty (add-headers-if-requested handler) {:port 18080 :join? false})))
+ (ring/run-jetty (add-headers-if-requested #'handler) {:port 18080 :join? false})))
(defn localhost [path]
(str "http://localhost:18080" path))
@@ -169,6 +167,52 @@
(is (= 200 (:status resp)))
(is (= "get" (slurp-body resp)))))
+(deftest ^:integration dns-resolver
+ (run-server)
+ (let [custom-dns-resolver (doto (InMemoryDnsResolver.)
+ (.add "foo.bar.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))])))
+ resp (request {:request-method :get :uri "/get"
+ :server-name "foo.bar.com"
+ :dns-resolver custom-dns-resolver})]
+ (is (= 200 (:status resp)))
+ (is (= "get" (slurp-body resp)))))
+
+(deftest ^:integration dns-resolver-unknown-host
+ (run-server)
+ (let [custom-dns-resolver (doto (InMemoryDnsResolver.)
+ (.add "foo.bar.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))])))]
+ (is (thrown? java.net.UnknownHostException (request {:request-method :get :uri "/get"
+ :server-name "www.google.com"
+ :dns-resolver custom-dns-resolver})))))
+
+(deftest ^:integration dns-resolver-reusable-connection-manager
+ (run-server)
+ (let [custom-dns-resolver (doto (InMemoryDnsResolver.)
+ (.add "totallynonexistant.google.com"
+ (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))])))
+ cm (conn/make-reuseable-async-conn-manager {:dns-resolver custom-dns-resolver})
+ hc (core/build-async-http-client {} cm)]
+ (client/get "http://totallynonexistant.google.com:18080/json"
+ {:connection-manager cm
+ :http-client hc
+ :as :json
+ :async true}
+ (fn [resp]
+ (is (= 200 (:status resp)))
+ (is (= {:foo "bar"} (:body resp))))
+ (fn [e] (is false (str "failed with " e)))))
+ (let [custom-dns-resolver (doto (InMemoryDnsResolver.)
+ (.add "nonexistant.google.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))])))
+ cm (conn/make-reusable-conn-manager {:dns-resolver custom-dns-resolver})
+ hc (:http-client (client/get "http://nonexistant.google.com:18080/get"
+ {:connection-manager cm}))
+ resp (client/get "http://nonexistant.google.com:18080/json"
+ {:connection-manager cm
+ :http-client hc
+ :as :json})]
+ (is (= 200 (:status resp)))
+ (is (= {:foo "bar"} (:body resp)))))
+
(deftest ^:integration save-request-option
(run-server)
(let [resp (request {:request-method :post
@@ -250,7 +294,7 @@
:keystore "test-resources/keystore"
:key-password "keykey"})]
(try
- (is (thrown? SunCertPathBuilderException
+ (is (thrown? Exception
(client/request {:scheme :https
:server-name "localhost"
:server-port 18082
@@ -370,8 +414,10 @@
(deftest ^:integration head-with-body
(run-server)
- (let [resp (request {:request-method :head :uri "/head" :body "foo"})]
- (is (= 200 (:status resp)))))
+ (let [resp (request {:request-method :head :uri "/head-with-body"
+ :body (.getBytes "foo")})]
+ (is (= 200 (:status resp)))
+ (is (= "foo" (get-in resp [:headers "body"])))))
(deftest ^:integration t-clojure-output-coercion
(run-server)
@@ -427,7 +473,10 @@
(deftest ^:integration t-json-output-coercion
(run-server)
(let [resp (client/get (localhost "/json") {:as :json})
- resp-array (client/get (localhost "/json-array") {:as :json-strict})
+ resp-array (client/get (localhost "/json-array") {:as :json})
+ resp-array-strict (client/get (localhost "/json-array") {:as :json-strict})
+ resp-large-array (client/get (localhost "/json-large-array") {:as :json})
+ resp-large-array-strict (client/get (localhost "/json-large-array") {:as :json-strict})
resp-str (client/get (localhost "/json")
{:as :json :coerce :exceptional})
resp-str-keys (client/get (localhost "/json") {:as :json-string-keys})
@@ -445,6 +494,9 @@
(is (= 200
(:status resp)
(:status resp-array)
+ (:status resp-array-strict)
+ (:status resp-large-array)
+ (:status resp-large-array-strict)
(:status resp-str)
(:status resp-str-keys)
(:status resp-strict-str-keys)
@@ -459,6 +511,7 @@
(:body resp-str-keys)))
;; '("foo" "bar") and ["foo" "bar"] compare as equal with =.
(is (vector? (:body resp-array)))
+ (is (vector? (:body resp-array-strict)))
(is (= "{\"foo\":\"bar\"}" (:body resp-str)))
(is (= 400
(:status bad-resp)
@@ -467,7 +520,11 @@
(is (= "{\"foo\":\"bar\"}" (:body bad-resp))
"don't coerce on bad response status by default")
(is (= {:foo "bar"} (:body bad-resp-json)))
- (is (= "{\"foo\":\"bar\"}" (:body bad-resp-json2)))))
+ (is (= "{\"foo\":\"bar\"}" (:body bad-resp-json2)))
+
+ (testing "lazily parsed stream completes parsing."
+ (is (= 100 (count (:body resp-large-array)))))
+ (is (= 100 (count (:body resp-large-array-strict))))))
(deftest ^:integration t-ipv6
(run-server)
@@ -605,7 +662,7 @@
;; This relies on connections to writequit.org being slower than 10ms, if this
;; fails, you must have very nice internet.
-(deftest ^:integration sets-conn-timeout
+(deftest ^:integration sets-connection-timeout
(run-server)
(try
(is (thrown? SocketTimeoutException
@@ -613,7 +670,7 @@
:server-name "writequit.org"
:server-port 80
:request-method :get :uri "/"
- :conn-timeout 10})))))
+ :connection-timeout 10})))))
(deftest ^:integration connection-pool-timeout
(run-server)
@@ -622,8 +679,8 @@
:server-name "localhost"
:server-port 18080
:request-method :get
- :conn-timeout 1
- :conn-request-timeout 1
+ :connection-timeout 1
+ :connection-request-timeout 1
:uri "/timeout"}))
is-pool-timeout-error?
(fn [req-fut]
@@ -691,6 +748,23 @@
(is (= (:trace-redirects resp-without-redirects) []))))
+(deftest t-request-config
+ (let [params {:conn-timeout 100 ;; deprecated
+ :connection-timeout 200 ;; takes precedence over `:conn-timeout`
+ :conn-request-timeout 300 ;; deprecated
+ :connection-request-timeout 400 ;; takes precedence over `:conn-request-timeout`
+ :socket-timeout 500
+ :max-redirects 600
+ :cookie-spec "foo"
+ :normalize-uri false}
+ request-config (core/request-config params)]
+ (is (= 200 (.getConnectTimeout request-config)))
+ (is (= 400 (.getConnectionRequestTimeout request-config)))
+ (is (= 500 (.getSocketTimeout request-config)))
+ (is (= 600 (.getMaxRedirects request-config)))
+ (is (= core/CUSTOM_COOKIE_POLICY (.getCookieSpec request-config)))
+ (is (false? (.isNormalizeUri request-config)))))
+
(deftest ^:integration t-override-request-config
(run-server)
(let [called-args (atom [])
@@ -740,6 +814,23 @@
"Headers should have included the new default headers")
(is (not (realized? error)))))
+(deftest ^:integration test-custom-http-client-builder
+ (run-server)
+ (let [methods (atom nil)
+ resp (client/get
+ (localhost "/get")
+ {:http-client-builder
+ (-> (org.apache.http.impl.client.HttpClientBuilder/create)
+ (.setRequestExecutor
+ (proxy [org.apache.http.protocol.HttpRequestExecutor] []
+ (execute [request connection context]
+ (->> request
+ .getRequestLine
+ .getMethod
+ (swap! methods conj))
+ (proxy-super execute request connection context)))))})]
+ (is (= ["GET"] @methods))))
+
(deftest ^:integration test-bad-redirects
(run-server)
(try
@@ -796,7 +887,9 @@
:async true}
(fn [resp]
(is (= 200 (:status resp)))
- (is (= {:foo "bar"} (:body resp))))
+ (is (= {:foo "bar"} (:body resp)))
+ (is (= hc (:http-client resp))
+ "http-client is correctly reused"))
(fn [e] (is false (str "failed with " e)))))
(let [cm (conn/make-reusable-conn-manager {})
hc (:http-client (client/get (localhost "/get")
@@ -806,7 +899,9 @@
:http-client hc
:as :json})]
(is (= 200 (:status resp)))
- (is (= {:foo "bar"} (:body resp)))))
+ (is (= {:foo "bar"} (:body resp)))
+ (is (= hc (:http-client resp))
+ "http-client is correctly reused")))
(deftest ^:integration t-cookies-spec
(run-server)
diff --git a/test/clj_http/test/headers_test.clj b/test/clj_http/test/headers_test.clj
index 74dc0e69..0732f18c 100644
--- a/test/clj_http/test/headers_test.clj
+++ b/test/clj_http/test/headers_test.clj
@@ -3,10 +3,8 @@
[clj-http.headers :refer :all]
[clj-http.util :refer [lower-case-keys]]
[clojure.test :refer :all])
- (:import (javax.servlet.http HttpServletRequest
- HttpServletResponse)
- (org.eclipse.jetty.server Request Server)
- (org.eclipse.jetty.server.handler AbstractHandler)))
+ (:import [org.eclipse.jetty.server Request Server]
+ org.eclipse.jetty.server.handler.AbstractHandler))
(deftest test-special-case
(are [expected given]
@@ -66,7 +64,13 @@
(is (= "baz" (:foo (merge (header-map :foo "bar")
{"Foo" "baz"}))))
(let [m-with-meta (with-meta m {:withmeta-test true})]
- (is (= (:withmeta-test (meta m-with-meta)) true)))))
+ (is (= (:withmeta-test (meta m-with-meta)) true)))
+
+ (testing "select-keys"
+ (are [expected keyset] (= expected (select-keys m keyset))
+ {"foo" "bar"} ["foo"]
+ {"foo" "bar"} ["foo" "non-existent-key"]
+ {"foo" "bar" "Foo" "bar" :foo "bar"} ["foo" "Foo" :foo]))))
(deftest test-empty
(testing "an empty header-map is a header-map"
@@ -85,8 +89,8 @@
(.setHandler (proxy [AbstractHandler] []
(handle [target
^Request base-request
- ^HttpServletRequest request
- ^HttpServletResponse response]
+ request
+ response]
(.setHandled base-request true)
(.setStatus response 200)
;; copy over request headers verbatim
diff --git a/test/clj_http/test/multipart_test.clj b/test/clj_http/test/multipart_test.clj
index 7151c12e..7c309cf5 100644
--- a/test/clj_http/test/multipart_test.clj
+++ b/test/clj_http/test/multipart_test.clj
@@ -1,10 +1,10 @@
(ns clj-http.test.multipart-test
(:require [clj-http.multipart :refer :all]
[clojure.test :refer :all])
- (:import (java.io File ByteArrayOutputStream ByteArrayInputStream)
- (org.apache.http.entity.mime.content FileBody StringBody ContentBody
- ByteArrayBody InputStreamBody)
- (java.nio.charset Charset)))
+ (:import [java.io ByteArrayInputStream ByteArrayOutputStream File]
+ java.nio.charset.Charset
+ [org.apache.http.entity.mime.content ByteArrayBody ContentBody FileBody InputStreamBody StringBody]
+ org.apache.http.util.EntityUtils))
(defn body-str [^StringBody body]
(-> body .getReader slurp))
@@ -173,3 +173,11 @@
(is (= (Charset/forName "ascii") (body-charset body)))
(is (= test-file (.getFile body) ))
(is (= "testname" (.getFilename body)))))))
+
+(deftest test-multipart-content-charset
+ (testing "charset is nil if no multipart-charset is supplied"
+ (let [mp-entity (create-multipart-entity [] nil)]
+ (is (nil? (EntityUtils/getContentCharSet mp-entity)))))
+ (testing "charset is set if a multipart-charset is supplied"
+ (let [mp-entity (create-multipart-entity [] {:multipart-charset "UTF-8"})]
+ (is (= "UTF-8" (EntityUtils/getContentCharSet mp-entity))))))
diff --git a/test/clj_http/test/util_test.clj b/test/clj_http/test/util_test.clj
index 781a9541..6ee75bd2 100644
--- a/test/clj_http/test/util_test.clj
+++ b/test/clj_http/test/util_test.clj
@@ -1,9 +1,9 @@
(ns clj-http.test.util-test
(:require [clj-http.util :refer :all]
- [clojure.test :refer :all]
- [clojure.java.io :as io])
- (:import (org.apache.commons.io IOUtils)
- (org.apache.commons.io.input NullInputStream)))
+ [clojure.java.io :as io]
+ [clojure.test :refer :all])
+ (:import org.apache.commons.io.input.NullInputStream
+ org.apache.commons.io.IOUtils))
(deftest test-lower-case-keys
(are [map expected]
@@ -41,9 +41,15 @@
" application/json; charset=UTF-8 "
{:content-type :application/json
:content-type-params {:charset "UTF-8"}}
+ " application/json; charset=\"utf-8\" "
+ {:content-type :application/json
+ :content-type-params {:charset "utf-8"}}
"text/html; charset=ISO-8859-4"
{:content-type :text/html
- :content-type-params {:charset "ISO-8859-4"}}))
+ :content-type-params {:charset "ISO-8859-4"}}
+ "text/html; charset="
+ {:content-type :text/html
+ :content-type-params {:charset nil}}))
(deftest test-force-byte-array
(testing "empty InputStream returns empty byte-array"
@@ -53,3 +59,12 @@
;; coerce to seq to force byte-by-byte comparison
(is (= (seq (IOUtils/toByteArray (io/input-stream jpg-path)))
(seq (force-byte-array (io/input-stream jpg-path))))))))
+
+(deftest test-gunzip
+ (testing "with input streams"
+ (testing "with empty stream, does not apply gunzip stream"
+ (is (= "" (slurp (gunzip (force-stream (byte-array 0)))))))
+ (testing "with non-empty stream, gunzip decompresses data"
+ (let [data "hello world"]
+ (is (= data
+ (slurp (gunzip (force-stream (gzip (.getBytes data)))))))))))
diff --git a/test/log4j.properties b/test/log4j.properties
deleted file mode 100755
index de3a2906..00000000
--- a/test/log4j.properties
+++ /dev/null
@@ -1,26 +0,0 @@
-#############
-# Appenders #
-#############
-
-# standard out appender
-log4j.appender.C = org.apache.log4j.ConsoleAppender
-log4j.appender.C.layout = org.apache.log4j.PatternLayout
-log4j.appender.C.layout.ConversionPattern = %d | ES | %-5p | [%t] | %c | %m%n
-
-# daily rolling file appender
-log4j.appender.F = org.apache.log4j.FileAppender
-log4j.appender.F.File = http.log
-log4j.appender.F.Append = true
-log4j.appender.F.layout = org.apache.log4j.PatternLayout
-log4j.appender.F.layout.ConversionPattern = %d | CLJ-HTTP | %-5p | [%t] | %c | %m%n
-
-###########
-# Loggers #
-###########
-
-# default
-log4j.rootLogger = DEBUG, F
-
-# Things
-log4j.logger.org.apache.http = DEBUG
-log4j.logger.org.apache.http.wire = INFO
diff --git a/test/log4j2.properties b/test/log4j2.properties
new file mode 100755
index 00000000..56f6f014
--- /dev/null
+++ b/test/log4j2.properties
@@ -0,0 +1,19 @@
+status = error
+dest = err
+name = PropertiesConfig
+
+filter.threshold.type = ThresholdFilter
+filter.threshold.level = debug
+
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d | %-5p | [%t] | %c | %m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.stdout.ref = STDOUT
+
+# Set this to debug to log all data to/from server
+# See https://hc.apache.org/httpcomponents-client-4.5.x/logging.html
+logger.wire.name = org.apache.http.wire
+logger.wire.level = info
\ No newline at end of file