Merge branch 'main' into feature-set-role

Author: chernser

CHANGELOG.md

@@ -1,10 +1,15 @@
-## Latest
+## Latest 
 
 ### New Features
-- [HTTP] Persistence of a role after it is set by `SET ROLE <role>` 
+- Added possibility to set client ID in `Referer` HTTP Header (https://github.com/ClickHouse/clickhouse-java/issues/1572)
+- [HTTP] Persistence of a role after it is set by `SET ROLE <role>`
 
 ### Bug Fixes
-- Fix param error in ByteUtils#equals in java9 [#1551](https://github.com/ClickHouse/clickhouse-java/issues/1551)
+- Change RowBinaryWithDefaults settings. Output is changed from true to false
+- Fix fail over for Apache HTTP client. Connect timeout error is not detected correctly
+- Fix password logging in DEBUG (https://github.com/ClickHouse/clickhouse-java/issues/1571)
+- Fix handling "NoHttpResponseException" in Apache HTTP client
+- Fix param error in ByteUtils#equals in java9 
 
 ## 0.6.0-patch3
 

CONTRIBUTING.md

@@ -103,7 +103,7 @@ In the case you don't want to use docker and/or prefer to test against an existi
   ```
 
 ### Tooling
-We use [TestNG](http://org.testng/doc) as testing framework and for running ClickHouse Local instance [testcontainers](https://www.testcontainers.org/modules/databases/clickhouse/).
+We use [TestNG](https://testng.org/) as testing framework and for running ClickHouse Local instance [testcontainers](https://www.testcontainers.org/modules/databases/clickhouse/).
 
 ### Running unit tests
 

clickhouse-cli-client/src/test/java/com/clickhouse/client/cli/ClickHouseCommandLineClientTest.java

@@ -3,22 +3,21 @@
 import com.clickhouse.client.ClickHouseClient;
 import com.clickhouse.client.ClickHouseClientBuilder;
 import com.clickhouse.client.ClickHouseException;
+import com.clickhouse.client.ClickHouseLoadBalancingPolicy;
 import com.clickhouse.client.ClickHouseNode;
 import com.clickhouse.client.ClickHouseProtocol;
 import com.clickhouse.client.ClickHouseServerForTest;
 import com.clickhouse.client.ClientIntegrationTest;
 import com.clickhouse.client.cli.config.ClickHouseCommandLineOption;
 import com.clickhouse.data.ClickHouseCompression;
-
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-
 import org.testcontainers.containers.GenericContainer;
-import org.testng.Assert;
 import org.testng.SkipException;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
 // deprecate from version 0.6.0
 @Deprecated
 public class ClickHouseCommandLineClientTest extends ClientIntegrationTest {
@@ -190,4 +189,14 @@ public void testTransactionTimeout() throws ClickHouseException {
     public void testRowBinaryWithDefaults() throws ClickHouseException, IOException, ExecutionException, InterruptedException {
         throw new SkipException("Skip due to supported");
     }
+
+    @Test(dataProvider = "loadBalancingPolicies", groups = {"unit"})
+    public void testLoadBalancingPolicyFailover(ClickHouseLoadBalancingPolicy loadBalancingPolicy) {
+        throw new SkipException("Skip due to failover is not supported");
+    }
+
+    @Test(groups = {"integration"})
+    public void testFailover() {
+        throw new SkipException("Skip due to failover is not supported");
+    }
 }

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseException.java

@@ -1,9 +1,9 @@
 package com.clickhouse.client;
 
+import java.io.IOException;
 import java.net.ConnectException;
 import java.net.SocketTimeoutException;
 import java.net.UnknownHostException;
-import java.util.Locale;
 import java.util.concurrent.TimeoutException;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -120,10 +120,7 @@ public static ClickHouseException forCancellation(Exception e, ClickHouseNode se
     public static boolean isConnectTimedOut(Throwable t) {
         if (t instanceof SocketTimeoutException || t instanceof TimeoutException) {
             String msg = t.getMessage();
-            if (msg != null && msg.length() >= MSG_CONNECT_TIMED_OUT.length()) {
-                msg = msg.substring(0, MSG_CONNECT_TIMED_OUT.length()).toLowerCase(Locale.ROOT);
-            }
-            return MSG_CONNECT_TIMED_OUT.equals(msg);
+            return msg != null && msg.toLowerCase().contains(MSG_CONNECT_TIMED_OUT);
         }
 
         return false;

clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java

@@ -41,13 +41,11 @@
 import com.clickhouse.data.value.UnsignedInteger;
 import com.clickhouse.data.value.UnsignedLong;
 import com.clickhouse.data.value.UnsignedShort;
-
 import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
 import org.testng.Assert;
 import org.testng.SkipException;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
-import org.testng.asserts.Assertion;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
@@ -75,9 +73,10 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
 import java.util.TimeZone;
 import java.util.UUID;
-import java.util.Map.Entry;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -150,13 +149,23 @@ protected ClickHouseResponseSummary execute(ClickHouseRequest<?> request, String
 
     protected ClickHouseRequest<?> newRequest(ClickHouseClient client, ClickHouseNode server) {
         ClickHouseRequest<?> request = client.read(server);
+        setClientOptions(request);
+        return request;
+    }
+
+    protected ClickHouseRequest<?> newRequest(ClickHouseClient client, ClickHouseNodes servers) {
+        ClickHouseRequest<?> request = client.read(servers);
+        setClientOptions(request);
+        return request;
+    }
+
+    private void setClientOptions(ClickHouseRequest<?> request) {
         Map<ClickHouseOption, Serializable> options = getClientOptions();
         if (options != null) {
             for (Entry<ClickHouseOption, Serializable> e : options.entrySet()) {
                 request.option(e.getKey(), e.getValue());
             }
         }
-        return request;
     }
 
     protected abstract ClickHouseProtocol getProtocol();
@@ -398,6 +407,16 @@ protected Object[][] getSimpleTypes() {
                         "00000000-0000-0000-ffff-ffffffffffff", "00000000-0000-0000-0000-000000000001" } };
     }
 
+    @DataProvider(name = "loadBalancingPolicies")
+    protected Object[][] getLoadBalancingPolicies() {
+        return new Object[][]{
+                {ClickHouseLoadBalancingPolicy.of(null)},
+                {ClickHouseLoadBalancingPolicy.of(ClickHouseLoadBalancingPolicy.FIRST_ALIVE)},
+                {ClickHouseLoadBalancingPolicy.of(ClickHouseLoadBalancingPolicy.ROUND_ROBIN)},
+                {ClickHouseLoadBalancingPolicy.of(ClickHouseLoadBalancingPolicy.RANDOM)},
+        };
+    }
+
     @Test(groups = { "unit" })
     public void testInitialization() {
         Assert.assertNotNull(getProtocol(), "The client should support non-null protocol");
@@ -886,7 +905,7 @@ public void testQuery() {
                  * while (response.hasError()) { int index = 0; for (ClickHouseColumn c :
                  * columns) { // RawValue v = response.getRawValue(index++); // String v =
                  * response.getValue(index++, String.class) }
-                 * 
+                 *
                  * } byte[] bytes = in.readAllBytes(); String str = new String(bytes);
                  */
             } catch (Exception e) {
@@ -2555,4 +2574,50 @@ public void testRowBinaryWithDefaults() throws ClickHouseException, IOException,
             }
         }
     }
+
+    @Test(dataProvider = "loadBalancingPolicies", groups = {"unit"})
+    public void testLoadBalancingPolicyFailover(ClickHouseLoadBalancingPolicy loadBalancingPolicy) {
+        String firstEndpoint = "111.1.1.1";
+        String secondEndpoint = "222.2.2.2";
+
+        Properties props = new Properties();
+        props.setProperty("failover", "1");
+
+        // nodes where the first node is failed
+        ClickHouseNodes nodes = ClickHouseNodes.of(
+                getProtocol() + "://" + firstEndpoint + "," + secondEndpoint,
+                props
+        );
+
+        try (ClickHouseClient client = getClient();
+             ClickHouseResponse response = newRequest(client, nodes.nodes.getFirst())
+                     .query("select 1")
+                     .executeAndWait()) {
+            Assert.fail("Exception expected for query on failed node");
+        } catch (Exception failoverException) {
+            ClickHouseNode failoverNode = loadBalancingPolicy.suggest(nodes, nodes.nodes.getFirst(), failoverException);
+            Assert.assertEquals(failoverNode.getHost(), secondEndpoint, "The next node is expected to be suggested by the load balancing policy");
+        }
+    }
+
+    @Test(groups = {"integration"})
+    public void testFailover() throws ClickHouseException {
+        ClickHouseNode availableNode = getServer();
+        Properties props = new Properties();
+        props.setProperty("failover", "1");
+
+        // nodes with the first node is unavailable
+        ClickHouseNodes nodes = ClickHouseNodes.of(
+                getProtocol() + "://111.1.1.1," + availableNode.getBaseUri(),
+                props
+        );
+
+        // should fail over to next node and successfully perform request if the first node is failed
+        try (ClickHouseClient client = getClient();
+             ClickHouseResponse response = newRequest(client, nodes)
+                     .query("select 1")
+                     .executeAndWait()) {
+            Assert.assertEquals(response.firstRecord().getValue(0).asInteger(), 1);
+        }
+    }
 }

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java

@@ -1,16 +1,5 @@
 package com.clickhouse.data;
 
-import java.io.Serializable;
-import java.lang.reflect.Array;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Objects;
-import java.util.TimeZone;
-
 import com.clickhouse.data.value.ClickHouseArrayValue;
 import com.clickhouse.data.value.ClickHouseBigDecimalValue;
 import com.clickhouse.data.value.ClickHouseBigIntegerValue;
@@ -46,6 +35,17 @@
 import com.clickhouse.data.value.array.ClickHouseLongArrayValue;
 import com.clickhouse.data.value.array.ClickHouseShortArrayValue;
 
+import java.io.Serializable;
+import java.lang.reflect.Array;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.TimeZone;
+
 /**
  * This class represents a column defined in database.
  */

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFormat.java

@@ -12,7 +12,7 @@ public enum ClickHouseFormat {
     RowBinary(true, true, true, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#rowbinary
     RowBinaryWithNames(true, true, true, true, true, RowBinary), // https://clickhouse.com/docs/en/interfaces/formats/#rowbinarywithnames
     RowBinaryWithNamesAndTypes(true, true, true, true, true, RowBinary), // https://clickhouse.com/docs/en/interfaces/formats/#rowbinarywithnamesandtypes
-    RowBinaryWithDefaults(true, true, true, true, true, RowBinary), // https://clickhouse.com/docs/en/interfaces/formats#rowbinarywithdefaults
+    RowBinaryWithDefaults(true, false, true, true, true, RowBinary), // https://clickhouse.com/docs/en/interfaces/formats#rowbinarywithdefaults
     TabSeparated(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#tabseparated
     TabSeparatedRaw(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#tabseparatedraw
     TabSeparatedRawWithNames(true, true, false, true, true, TabSeparatedRaw), // https://clickhouse.com/docs/en/interfaces/formats/#tabseparatedrawwithnames

clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java

@@ -1,17 +1,8 @@
 package com.clickhouse.client.grpc;
 
-import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
-import org.testng.Assert;
-import org.testng.SkipException;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-
 import com.clickhouse.client.ClickHouseClient;
 import com.clickhouse.client.ClickHouseException;
+import com.clickhouse.client.ClickHouseLoadBalancingPolicy;
 import com.clickhouse.client.ClickHouseNode;
 import com.clickhouse.client.ClickHouseProtocol;
 import com.clickhouse.client.ClickHouseResponse;
@@ -23,6 +14,15 @@
 import com.clickhouse.data.ClickHouseInputStream;
 import com.clickhouse.data.ClickHouseRecord;
 import com.clickhouse.data.ClickHouseVersion;
+import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
+import org.testng.Assert;
+import org.testng.SkipException;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
 @Deprecated
 public class ClickHouseGrpcClientTest extends ClientIntegrationTest {
     @DataProvider(name = "requestCompressionMatrix")
@@ -144,7 +144,7 @@ public void testErrorDuringInsert() throws ClickHouseException {
     public void testErrorDuringQuery() throws ClickHouseException {
         throw new SkipException("Skip due to grpc is too slow");
     }
-    
+
     @Test(groups = { "integration" })
     @Override
     public void testSessionLock() {
@@ -205,4 +205,13 @@ public void testRowBinaryWithDefaults() throws ClickHouseException, IOException,
         throw new SkipException("Skip due to supported");
     }
 
+    @Test(dataProvider = "loadBalancingPolicies", groups = {"unit"})
+    public void testLoadBalancingPolicyFailover(ClickHouseLoadBalancingPolicy loadBalancingPolicy) {
+        throw new SkipException("Skip due to failover is not supported");
+    }
+
+    @Test(groups = {"integration"})
+    public void testFailover() {
+        throw new SkipException("Skip due to failover is not supported");
+    }
 }

clickhouse-http-client/pom.xml

@@ -103,6 +103,14 @@
             <artifactId>testng</artifactId>
             <scope>test</scope>
         </dependency>
+        <!-- Replace with newer version as soon java 11 is minimal version -->
+        <dependency>
+            <groupId>com.github.tomakehurst</groupId>
+            <artifactId>wiremock-jre8</artifactId>
+            <version>2.35.2</version>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>

clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java

@@ -3,6 +3,7 @@
 import com.clickhouse.client.AbstractSocketClient;
 import com.clickhouse.client.ClickHouseClient;
 import com.clickhouse.client.ClickHouseConfig;
+import com.clickhouse.client.ClickHouseException;
 import com.clickhouse.client.ClickHouseNode;
 import com.clickhouse.client.ClickHouseRequest;
 import com.clickhouse.client.ClickHouseSocketFactory;
@@ -249,7 +250,12 @@ protected ClickHouseHttpResponse post(ClickHouseConfig config, String sql, Click
         ClickHouseHttpEntity postBody = new ClickHouseHttpEntity(config, contentType, contentEncoding, boundary,
                 sql, data, tables);
         post.setEntity(postBody);
-        CloseableHttpResponse response = client.execute(post);
+        CloseableHttpResponse response;
+        try {
+            response = client.execute(post);
+        } catch (IOException e) {
+            throw new ConnectException(ClickHouseUtils.format("HTTP request failed: %s", e.getMessage()));
+        }
 
         checkResponse(config, response);
         // buildResponse should use the config of current request in case of reusable