From 344d751ff29690f3876ad98f603175fb58b9f66e Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 16 Apr 2026 13:55:39 -0700 Subject: [PATCH 1/8] Fixed overriding server settings in jdbc connection --- .../client/api/internal/ServerSettings.java | 5 ++ .../com/clickhouse/jdbc/ConnectionImpl.java | 4 +- .../jdbc/internal/JdbcConfiguration.java | 28 +++++++++-- .../com/clickhouse/jdbc/StatementTest.java | 48 +++++++++++++++++++ .../jdbc/internal/JdbcConfigurationTest.java | 5 +- 5 files changed, 83 insertions(+), 7 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/ServerSettings.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/ServerSettings.java index 24210866f..84c1b4400 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/ServerSettings.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/ServerSettings.java @@ -46,4 +46,9 @@ public final class ServerSettings { public static final String ASYNC_INSERT = "async_insert"; public static final String WAIT_ASYNC_INSERT = "wait_for_async_insert"; + + // Misc + public static final String ON = "1"; + + public static final String OFF = "0"; } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java index 7328048ce..73cd00d7e 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java @@ -110,9 +110,7 @@ public ConnectionImpl(String url, Properties info) throws SQLException { this.client.loadServerInfo(); } this.schema = client.getDefaultDatabase(); - this.defaultQuerySettings = new QuerySettings() - .serverSetting(ServerSettings.ASYNC_INSERT, "0") - .serverSetting(ServerSettings.WAIT_END_OF_QUERY, "0"); + this.defaultQuerySettings = new QuerySettings(); this.metadata = new DatabaseMetaDataImpl(this, false, url); this.defaultCalendar = Calendar.getInstance(); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java index 3680cc32b..f1c5be2ad 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java @@ -3,6 +3,7 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.http.ClickHouseHttpProto; +import com.clickhouse.client.api.internal.ServerSettings; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.jdbc.Driver; import com.clickhouse.jdbc.DriverProperties; @@ -89,7 +90,7 @@ public JdbcConfiguration(String url, Properties info) throws SQLException { Map urlProperties = parseUrl(url); String tmpConnectionUrl = urlProperties.remove(PARSE_URL_CONN_URL_PROP); - initProperties(urlProperties, props); + buildFinalProperties(urlProperties, props); // after initializing all properties - set final connection URL boolean useSSLInfo = Boolean.parseBoolean(props.getProperty(DriverProperties.SECURE_CONNECTION.getKey(), "false")); @@ -266,10 +267,31 @@ private Map parseUrl(String url) throws SQLException { return properties; } - private void initProperties(Map urlProperties, Properties providedProperties) { + /** + * Creates initial properties with defaults. + * @return mutable HashMap with default values. + */ + Map createProperties() { + Map props = new HashMap<>(); + + // Requires to wait result on insert + props.put(DriverProperties.serverSetting(ServerSettings.ASYNC_INSERT), ServerSettings.OFF); + + // Requires to wait result of query (manly insert) + props.put(DriverProperties.serverSetting(ServerSettings.WAIT_END_OF_QUERY), ServerSettings.ON); + + return props; + } + + /** + * Combines url properties and provided ones via {@link java.sql.Driver#connect(String, Properties)} + * @param urlProperties - properties parsed from URL + * @param providedProperties - properties object provided by application + */ + private void buildFinalProperties(Map urlProperties, Properties providedProperties) { // Copy provided properties - Map props = new HashMap<>(); + Map props = createProperties(); // Set driver properties defaults (client will do the same) for (DriverProperties prop : DriverProperties.values()) { if (prop.getDefaultValue() != null) { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index 575dde388..dce8daca6 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -264,6 +264,54 @@ public void testExecuteUpdateDates() throws Exception { } } + @DataProvider(name = "asyncInsertSettingsDP") + public static Object[][] asyncInsertSettingsDP() { + return new Object[][]{ + // asyncInsert, waitEndOfQuery, expectedUpdateCount, expectedSelectCount + {ServerSettings.OFF, ServerSettings.OFF, 10000}, + {ServerSettings.OFF, ServerSettings.ON, 10000}, + {ServerSettings.ON, ServerSettings.OFF, 0, -1}, + {ServerSettings.ON, ServerSettings.ON, 0, 10000} + }; + } + + @Test(groups = {"integration"}, dataProvider = "asyncInsertSettingsDP") + public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert, int expectedUpdateCount, int expectedSelectCount) throws Exception { + String tableName = "test_async_insert_param_" + asyncInsert + "_" + waitAsyncInsert; + + Properties props = new Properties(); + props.setProperty(ClientConfigProperties.serverSetting(ServerSettings.ASYNC_INSERT), asyncInsert); + props.setProperty(ClientConfigProperties.serverSetting(ServerSettings.WAIT_ASYNC_INSERT), waitAsyncInsert); + // Wait end of query off for isolation of this logic + props.setProperty(ClientConfigProperties.serverSetting(ServerSettings.WAIT_END_OF_QUERY), ServerSettings.OFF); + + try (Connection conn = getJdbcConnection(props)) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE TABLE IF NOT EXISTS " + getDatabase() + "." + tableName + " (id UInt32) ENGINE = MergeTree ORDER BY id"); + stmt.execute("TRUNCATE TABLE " + getDatabase() + "." + tableName); + + StringBuilder sb = new StringBuilder("INSERT INTO " + getDatabase() + "." + tableName + " VALUES "); + for (int i = 0; i < 10000; i++) { + if (i > 0) sb.append(", "); + sb.append("(").append(i).append(")"); + } + + int updateCount = stmt.executeUpdate(sb.toString()); + assertEquals(updateCount, expectedUpdateCount); + + try (ResultSet rs = stmt.executeQuery("SELECT count() FROM " + getDatabase() + "." + tableName)) { + assertTrue(rs.next()); + int count = rs.getInt(1); + if (expectedSelectCount == -1) { + assertTrue(count < 10000, "Expected count to be < 10000, but was: " + count); + } else { + assertEquals(count, expectedSelectCount); + } + } + } + } + } + @Test(groups = {"integration"}) public void testExecuteUpdateBatch() throws Exception { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java index a5a3e972f..db756d29c 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java @@ -121,8 +121,11 @@ public void testParseURLValid(String jdbcURL, Properties properties, throws Exception { JdbcConfiguration configuration = new JdbcConfiguration(jdbcURL, properties); + // We need to copy defaults to expected. + Map expectedWithDefaults = configuration.createProperties(); + expectedWithDefaults.putAll(expectedClientProps); assertEquals(configuration.getConnectionUrl(), connectionURL, "URL: " + jdbcURL); - assertEquals(configuration.clientProperties, expectedClientProps, "URL: " + jdbcURL); + assertEquals(configuration.clientProperties, expectedWithDefaults, "URL: " + jdbcURL); Client.Builder bob = new Client.Builder(); configuration.applyClientProperties(bob); Client client = bob.build(); From ab67235568b63fc7c2ab95aba462928f735160ea Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 16 Apr 2026 14:44:39 -0700 Subject: [PATCH 2/8] fixed test --- jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index dce8daca6..9648ef806 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -268,8 +268,8 @@ public void testExecuteUpdateDates() throws Exception { public static Object[][] asyncInsertSettingsDP() { return new Object[][]{ // asyncInsert, waitEndOfQuery, expectedUpdateCount, expectedSelectCount - {ServerSettings.OFF, ServerSettings.OFF, 10000}, - {ServerSettings.OFF, ServerSettings.ON, 10000}, + {ServerSettings.OFF, ServerSettings.OFF, 10000, 10000}, + {ServerSettings.OFF, ServerSettings.ON, 10000, 10000}, {ServerSettings.ON, ServerSettings.OFF, 0, -1}, {ServerSettings.ON, ServerSettings.ON, 0, 10000} }; From 0f0d82e940ced973a851bf225b334bb4f2073982 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 16 Apr 2026 16:44:14 -0700 Subject: [PATCH 3/8] attempt first to fix flaky test --- .../test/java/com/clickhouse/jdbc/StatementTest.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index 9648ef806..f692be809 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -264,14 +264,16 @@ public void testExecuteUpdateDates() throws Exception { } } + private static final int ASYNC_INSERT_SETTINGS_DP_ROWS = 1000_000; + @DataProvider(name = "asyncInsertSettingsDP") public static Object[][] asyncInsertSettingsDP() { return new Object[][]{ // asyncInsert, waitEndOfQuery, expectedUpdateCount, expectedSelectCount - {ServerSettings.OFF, ServerSettings.OFF, 10000, 10000}, - {ServerSettings.OFF, ServerSettings.ON, 10000, 10000}, + {ServerSettings.OFF, ServerSettings.OFF, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS}, + {ServerSettings.OFF, ServerSettings.ON, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS}, {ServerSettings.ON, ServerSettings.OFF, 0, -1}, - {ServerSettings.ON, ServerSettings.ON, 0, 10000} + {ServerSettings.ON, ServerSettings.ON, 0, ASYNC_INSERT_SETTINGS_DP_ROWS} }; } @@ -291,7 +293,7 @@ public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert stmt.execute("TRUNCATE TABLE " + getDatabase() + "." + tableName); StringBuilder sb = new StringBuilder("INSERT INTO " + getDatabase() + "." + tableName + " VALUES "); - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < ASYNC_INSERT_SETTINGS_DP_ROWS; i++) { if (i > 0) sb.append(", "); sb.append("(").append(i).append(")"); } @@ -303,7 +305,7 @@ public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert assertTrue(rs.next()); int count = rs.getInt(1); if (expectedSelectCount == -1) { - assertTrue(count < 10000, "Expected count to be < 10000, but was: " + count); + assertTrue(count < ASYNC_INSERT_SETTINGS_DP_ROWS, "Expected count to be < " + ASYNC_INSERT_SETTINGS_DP_ROWS + ", but was: " + count); } else { assertEquals(count, expectedSelectCount); } From 0709cb6628cca72ad468d1bf4aeafa4b7051715c Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 16 Apr 2026 17:26:33 -0700 Subject: [PATCH 4/8] switched to another format --- .../java/com/clickhouse/jdbc/StatementTest.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index f692be809..47a430e89 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -264,7 +264,7 @@ public void testExecuteUpdateDates() throws Exception { } } - private static final int ASYNC_INSERT_SETTINGS_DP_ROWS = 1000_000; + private static final int ASYNC_INSERT_SETTINGS_DP_ROWS = 100_000; @DataProvider(name = "asyncInsertSettingsDP") public static Object[][] asyncInsertSettingsDP() { @@ -289,13 +289,16 @@ public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert try (Connection conn = getJdbcConnection(props)) { try (Statement stmt = conn.createStatement()) { - stmt.execute("CREATE TABLE IF NOT EXISTS " + getDatabase() + "." + tableName + " (id UInt32) ENGINE = MergeTree ORDER BY id"); + stmt.execute("CREATE TABLE IF NOT EXISTS " + getDatabase() + "." + tableName + " (id UInt32, name String, value Float64, status Int8, timestamp DateTime) ENGINE = MergeTree ORDER BY id"); stmt.execute("TRUNCATE TABLE " + getDatabase() + "." + tableName); - StringBuilder sb = new StringBuilder("INSERT INTO " + getDatabase() + "." + tableName + " VALUES "); + StringBuilder sb = new StringBuilder("INSERT INTO " + getDatabase() + "." + tableName + " FORMAT TSV\n"); for (int i = 0; i < ASYNC_INSERT_SETTINGS_DP_ROWS; i++) { - if (i > 0) sb.append(", "); - sb.append("(").append(i).append(")"); + sb.append(i).append("\t") + .append("name_").append(i).append("\t") + .append(i * 1.1).append("\t") + .append(i % 2).append("\t") + .append("2023-01-01 10:11:12").append("\n"); } int updateCount = stmt.executeUpdate(sb.toString()); From 25597501cd0cd018b91d46864ae2e289305cf47c Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 16 Apr 2026 18:09:14 -0700 Subject: [PATCH 5/8] added insert_deduplicate=0 to make tests stable --- jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index 47a430e89..d61a534fd 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -286,6 +286,7 @@ public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert props.setProperty(ClientConfigProperties.serverSetting(ServerSettings.WAIT_ASYNC_INSERT), waitAsyncInsert); // Wait end of query off for isolation of this logic props.setProperty(ClientConfigProperties.serverSetting(ServerSettings.WAIT_END_OF_QUERY), ServerSettings.OFF); + props.setProperty(ClientConfigProperties.serverSetting("insert_deduplicate"), ServerSettings.OFF); try (Connection conn = getJdbcConnection(props)) { try (Statement stmt = conn.createStatement()) { From 9a94ae625167b8f5f0ef1e00e2f702af5e230b4b Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 16 Apr 2026 19:28:28 -0700 Subject: [PATCH 6/8] Forced flashing to disk when wait_async_insert=1 --- .../test/java/com/clickhouse/jdbc/StatementTest.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index d61a534fd..80cb6f7be 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import static org.testng.Assert.assertEquals; @@ -273,20 +274,24 @@ public static Object[][] asyncInsertSettingsDP() { {ServerSettings.OFF, ServerSettings.OFF, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS}, {ServerSettings.OFF, ServerSettings.ON, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS}, {ServerSettings.ON, ServerSettings.OFF, 0, -1}, - {ServerSettings.ON, ServerSettings.ON, 0, ASYNC_INSERT_SETTINGS_DP_ROWS} + {ServerSettings.ON, ServerSettings.ON, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS} }; } @Test(groups = {"integration"}, dataProvider = "asyncInsertSettingsDP") public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert, int expectedUpdateCount, int expectedSelectCount) throws Exception { - String tableName = "test_async_insert_param_" + asyncInsert + "_" + waitAsyncInsert; + String tableName = "test_async_insert_param_" + asyncInsert + "_" + waitAsyncInsert + "_" + UUID.randomUUID().toString().replace("-", "_"); Properties props = new Properties(); props.setProperty(ClientConfigProperties.serverSetting(ServerSettings.ASYNC_INSERT), asyncInsert); props.setProperty(ClientConfigProperties.serverSetting(ServerSettings.WAIT_ASYNC_INSERT), waitAsyncInsert); // Wait end of query off for isolation of this logic props.setProperty(ClientConfigProperties.serverSetting(ServerSettings.WAIT_END_OF_QUERY), ServerSettings.OFF); - props.setProperty(ClientConfigProperties.serverSetting("insert_deduplicate"), ServerSettings.OFF); + + if (waitAsyncInsert.equals(ServerSettings.ON)) { + // make it flash to disk to check that we get result. If buffer is bigger server may wait flashing to disk. + props.setProperty(ClientConfigProperties.serverSetting("async_insert_max_data_size"), "3488890"); + } try (Connection conn = getJdbcConnection(props)) { try (Statement stmt = conn.createStatement()) { From 28cf959980da73dc2c0136a55ead3b7e57e8102a Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 16 Apr 2026 19:32:28 -0700 Subject: [PATCH 7/8] updated change.log --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f836a5c89..3afd247d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.9.9 + + +### Bug Fixes +- **[jdbc-v2]** Fixed issues with hardcoded `async_insert` settings in `ConnectionImpl`. Now it let override them. (https://github.com/ClickHouse/clickhouse-java/issues/2652, https://github.com/ClickHouse/clickhouse-java/issues/2825) + ## 0.9.8 ### Improvements From 5d4d7e41e0a3a7a559ab66b00a9dc0f62cf48595 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 17 Apr 2026 14:01:31 -0700 Subject: [PATCH 8/8] removed hardcoded settings. Updated changelog --- CHANGELOG.md | 10 ++- .../jdbc/internal/JdbcConfiguration.java | 21 +----- .../clickhouse/jdbc/JdbcIntegrationTest.java | 14 +++- .../jdbc/PreparedStatementTest.java | 11 +-- .../com/clickhouse/jdbc/StatementTest.java | 72 +++++++++++-------- .../jdbc/internal/JdbcConfigurationTest.java | 5 +- 6 files changed, 73 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3afd247d9..194994c72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,14 @@ ## 0.9.9 +### Breaking Changes -### Bug Fixes -- **[jdbc-v2]** Fixed issues with hardcoded `async_insert` settings in `ConnectionImpl`. Now it let override them. (https://github.com/ClickHouse/clickhouse-java/issues/2652, https://github.com/ClickHouse/clickhouse-java/issues/2825) +- **[jdbc-v2]** Hardcoded server setting `async_insert=0` is removed as well as others. This is done to +fix issue with overriding these settings and using client with read-only profiles. The change, first of all, makes +driver behavior to follow default what is set on server side (note: starting ClickHouse 26.3 `async_insert` is on by default). +In second, this fix changes what number of affected rows returned by method like `java.sql.Statement.executeUpdate(java.lang.String)`. +Previously they return more accurate values because insert was synchronous, but in case of asynchronous insert it is not +guaranteed anymore (see also https://github.com/ClickHouse/ClickHouse/issues/57768). Read more about asynchronous insert https://clickhouse.com/docs/optimize/asynchronous-inserts. +(https://github.com/ClickHouse/clickhouse-java/issues/2652, https://github.com/ClickHouse/clickhouse-java/issues/2825) ## 0.9.8 diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java index f1c5be2ad..d4d1c4987 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java @@ -3,7 +3,6 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.http.ClickHouseHttpProto; -import com.clickhouse.client.api.internal.ServerSettings; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.jdbc.Driver; import com.clickhouse.jdbc.DriverProperties; @@ -75,7 +74,7 @@ public boolean isIgnoreUnsupportedRequests() { /** * Parses URL to get property and target host. - * Properties that are passed in the {@code info} parameter will override that are set in the {@code url}. + * Properties that are passed in the {@code url} will override the {@code info} ones. * @param url - JDBC url * @param info - Driver and Client properties. */ @@ -267,22 +266,6 @@ private Map parseUrl(String url) throws SQLException { return properties; } - /** - * Creates initial properties with defaults. - * @return mutable HashMap with default values. - */ - Map createProperties() { - Map props = new HashMap<>(); - - // Requires to wait result on insert - props.put(DriverProperties.serverSetting(ServerSettings.ASYNC_INSERT), ServerSettings.OFF); - - // Requires to wait result of query (manly insert) - props.put(DriverProperties.serverSetting(ServerSettings.WAIT_END_OF_QUERY), ServerSettings.ON); - - return props; - } - /** * Combines url properties and provided ones via {@link java.sql.Driver#connect(String, Properties)} * @param urlProperties - properties parsed from URL @@ -291,7 +274,7 @@ Map createProperties() { private void buildFinalProperties(Map urlProperties, Properties providedProperties) { // Copy provided properties - Map props = createProperties(); + Map props = new HashMap<>(); // Set driver properties defaults (client will do the same) for (DriverProperties prop : DriverProperties.values()) { if (prop.getDefaultValue() != null) { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java index 69bc2a11e..fcf3f98f8 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java @@ -4,6 +4,7 @@ import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseServerForTest; import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.internal.ServerSettings; import com.clickhouse.client.api.query.GenericRecord; import com.clickhouse.data.ClickHouseVersion; import com.clickhouse.logging.Logger; @@ -13,11 +14,16 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.List; +import java.util.Map; import java.util.Properties; public abstract class JdbcIntegrationTest extends BaseIntegrationTest { private static final Logger LOGGER = LoggerFactory.getLogger(JdbcIntegrationTest.class); + public static final String WAIT_ASYNC_SETTING_KEY = DriverProperties.serverSetting(ServerSettings.WAIT_ASYNC_INSERT); + public static final String WAIT_QUERY_SETTING_KEY = DriverProperties.serverSetting(ServerSettings.WAIT_END_OF_QUERY); + public static final String ASYNC_INSERT_SETTING_KEY = DriverProperties.serverSetting(ServerSettings.ASYNC_INSERT); + public String getEndpointString() { return getEndpointString(isCloud()); } @@ -28,7 +34,13 @@ public String getEndpointString(boolean includeDbName) { } public Connection getJdbcConnection() throws SQLException { - return getJdbcConnection(null); + return getJdbcConnection((Properties) null); + } + + public Connection getJdbcConnection(Map propertiesMap) throws SQLException { + Properties config = new Properties(); + config.putAll(propertiesMap); + return getJdbcConnection(config); } public Connection getJdbcConnection(Properties properties) throws SQLException { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java index d5bfb7978..6c76fe6f3 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java @@ -1,6 +1,7 @@ package com.clickhouse.jdbc; import com.clickhouse.client.api.DataTypeUtils; +import com.clickhouse.client.api.internal.ServerSettings; import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseVersion; @@ -956,7 +957,7 @@ void testBatchInsertTextStatement(String sql) throws Exception { String table = "test_batch_text"; long seed = System.currentTimeMillis(); Random rnd = new Random(seed); - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { stmt.execute("CREATE TABLE IF NOT EXISTS " + table + @@ -1007,7 +1008,7 @@ void testBatchInsertNoValuesReuse() throws Exception { String sql = "INSERT INTO %s (v1, v2) VALUES (?, ?)"; long seed = System.currentTimeMillis(); Random rnd = new Random(seed); - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { stmt.execute("CREATE TABLE IF NOT EXISTS " + table + @@ -1062,7 +1063,7 @@ void testBatchInsertValuesReuse() throws Exception { String sql = "INSERT INTO %s (v1, v2) VALUES (1, ?)"; long seed = System.currentTimeMillis(); Random rnd = new Random(seed); - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { stmt.execute("CREATE TABLE IF NOT EXISTS " + table + @@ -1266,7 +1267,7 @@ public void testJdbcEscapeSyntax() throws Exception { @Test(groups = {"integration "}) public void testStatementsWithDatabaseInTableIdentifier() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { final String db1Name = conn.getSchema() + "_db1"; final String table1Name = "table1"; try (Statement stmt = conn.createStatement()) { @@ -1296,7 +1297,7 @@ public void testStatementsWithDatabaseInTableIdentifier() throws Exception { @Test(groups = {"integration "}) public void testNullValues() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { final String table = "test_null_values"; try (Statement stmt = conn.createStatement()) { stmt.execute("DROP TABLE IF EXISTS " + table); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index 80cb6f7be..5fe6abeb4 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -149,7 +149,7 @@ public void testExecuteQueryDates() throws Exception { @Test(groups = {"integration"}) public void testExecuteUpdateSimpleNumbers() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { assertEquals(stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + getDatabase() + ".simpleNumbers (num UInt8) ENGINE = MergeTree ORDER BY ()"), 0); assertEquals(stmt.executeUpdate("INSERT INTO " + getDatabase() + ".simpleNumbers VALUES (1), (2), (3)"), 3); @@ -168,7 +168,7 @@ public void testExecuteUpdateSimpleNumbers() throws Exception { @Test(groups = {"integration"}) public void testExecuteUpdateSimpleFloats() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { assertEquals(stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + getDatabase() + ".simpleFloats (num Float32) ENGINE = MergeTree ORDER BY ()"), 0); assertEquals(stmt.executeUpdate("INSERT INTO " + getDatabase() + ".simpleFloats VALUES (1.1), (2.2), (3.3)"), 3); @@ -188,7 +188,7 @@ public void testExecuteUpdateSimpleFloats() throws Exception { @Test(groups = {"integration"}) public void testExecuteUpdateBooleans() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { assertEquals(stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + getDatabase() + ".booleans (id UInt8, flag Boolean) ENGINE = MergeTree ORDER BY ()"), 0); assertEquals(stmt.executeUpdate("INSERT INTO " + getDatabase() + ".booleans VALUES (0, true), (1, false), (2, true)"), 3); @@ -207,7 +207,7 @@ public void testExecuteUpdateBooleans() throws Exception { @Test(groups = {"integration"}) public void testExecuteUpdateStrings() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { assertEquals(stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + getDatabase() + ".strings (id UInt8, words String) ENGINE = MergeTree ORDER BY ()"), 0); assertEquals(stmt.executeUpdate("INSERT INTO " + getDatabase() + ".strings VALUES (0, 'Hello'), (1, 'World'), (2, 'ClickHouse')"), 3); @@ -226,7 +226,7 @@ public void testExecuteUpdateStrings() throws Exception { @Test(groups = {"integration"}) public void testExecuteUpdateNulls() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { assertEquals(stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + getDatabase() + ".nulls (id UInt8, nothing Nullable(String)) ENGINE = MergeTree ORDER BY ()"), 0); assertEquals(stmt.executeUpdate("INSERT INTO " + getDatabase() + ".nulls VALUES (0, 'Hello'), (1, NULL), (2, 'ClickHouse')"), 3); @@ -245,7 +245,7 @@ public void testExecuteUpdateNulls() throws Exception { @Test(groups = {"integration"}) public void testExecuteUpdateDates() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { assertEquals(stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + getDatabase() + ".dates (id UInt8, date Nullable(Date), datetime Nullable(DateTime)) ENGINE = MergeTree ORDER BY ()"), 0); assertEquals(stmt.executeUpdate("INSERT INTO " + getDatabase() + ".dates VALUES (0, '2020-01-01', '2020-01-01 10:11:12'), (1, NULL, '2020-01-01 12:10:07'), (2, '2020-01-01', NULL)"), 3); @@ -270,16 +270,16 @@ public void testExecuteUpdateDates() throws Exception { @DataProvider(name = "asyncInsertSettingsDP") public static Object[][] asyncInsertSettingsDP() { return new Object[][]{ - // asyncInsert, waitEndOfQuery, expectedUpdateCount, expectedSelectCount - {ServerSettings.OFF, ServerSettings.OFF, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS}, - {ServerSettings.OFF, ServerSettings.ON, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS}, - {ServerSettings.ON, ServerSettings.OFF, 0, -1}, - {ServerSettings.ON, ServerSettings.ON, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS} + // asyncInsert, waitAsyncInsert, expectedUpdateCount, expectedSelectCount, should fail + {ServerSettings.OFF, ServerSettings.OFF, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS, true}, + {ServerSettings.OFF, ServerSettings.ON, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS, true}, + {ServerSettings.ON, ServerSettings.OFF, 0, -1, false}, // return immediately + {ServerSettings.ON, ServerSettings.ON, ASYNC_INSERT_SETTINGS_DP_ROWS, ASYNC_INSERT_SETTINGS_DP_ROWS, true} }; } @Test(groups = {"integration"}, dataProvider = "asyncInsertSettingsDP") - public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert, int expectedUpdateCount, int expectedSelectCount) throws Exception { + public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert, int expectedUpdateCount, int expectedSelectCount, boolean fails) throws Exception { String tableName = "test_async_insert_param_" + asyncInsert + "_" + waitAsyncInsert + "_" + UUID.randomUUID().toString().replace("-", "_"); Properties props = new Properties(); @@ -289,25 +289,27 @@ public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert props.setProperty(ClientConfigProperties.serverSetting(ServerSettings.WAIT_END_OF_QUERY), ServerSettings.OFF); if (waitAsyncInsert.equals(ServerSettings.ON)) { - // make it flash to disk to check that we get result. If buffer is bigger server may wait flashing to disk. + // make it flush to disk to check that we get result. If buffer is bigger server may wait flushing to disk. props.setProperty(ClientConfigProperties.serverSetting("async_insert_max_data_size"), "3488890"); } + StringBuilder sb = new StringBuilder("INSERT INTO " + getDatabase() + "." + tableName + " FORMAT TSV\n"); + for (int i = 0; i < ASYNC_INSERT_SETTINGS_DP_ROWS; i++) { + sb.append(i).append("\t") + .append("name_").append(i).append("\t") + .append(i * 1.1).append("\t") + .append(i % 2).append("\t") + .append("2023-01-01 10:11:12").append("\n"); + } + final String insertStatement = sb.toString(); + try (Connection conn = getJdbcConnection(props)) { try (Statement stmt = conn.createStatement()) { stmt.execute("CREATE TABLE IF NOT EXISTS " + getDatabase() + "." + tableName + " (id UInt32, name String, value Float64, status Int8, timestamp DateTime) ENGINE = MergeTree ORDER BY id"); stmt.execute("TRUNCATE TABLE " + getDatabase() + "." + tableName); - - StringBuilder sb = new StringBuilder("INSERT INTO " + getDatabase() + "." + tableName + " FORMAT TSV\n"); - for (int i = 0; i < ASYNC_INSERT_SETTINGS_DP_ROWS; i++) { - sb.append(i).append("\t") - .append("name_").append(i).append("\t") - .append(i * 1.1).append("\t") - .append(i % 2).append("\t") - .append("2023-01-01 10:11:12").append("\n"); - } - - int updateCount = stmt.executeUpdate(sb.toString()); + + int updateCount = stmt.executeUpdate(insertStatement); + assertEquals(updateCount, expectedUpdateCount); try (ResultSet rs = stmt.executeQuery("SELECT count() FROM " + getDatabase() + "." + tableName)) { @@ -319,6 +321,17 @@ public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert assertEquals(count, expectedSelectCount); } } + + // verify error scenario + final String failingInsertStatement = insertStatement + + "some\t1invalid\trow\t10\n"; + + try { + stmt.executeUpdate(failingInsertStatement); + assertFalse(fails, "should fail"); + } catch (Exception e) { + assertTrue(fails, "should not fail"); + } } } } @@ -326,7 +339,7 @@ public void testInsertWithAsyncInsert(String asyncInsert, String waitAsyncInsert @Test(groups = {"integration"}) public void testExecuteUpdateBatch() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { assertEquals(stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + getDatabase() + ".batch (id UInt8, num UInt8) ENGINE = MergeTree ORDER BY ()"), 0); stmt.addBatch("INSERT INTO " + getDatabase() + ".batch VALUES (0, 1)"); @@ -355,7 +368,7 @@ public void testExecuteUpdateBatch() throws Exception { @Test(groups = {"integration"}) public void testExecuteUpdateBatchReuse() throws Exception { String tableClause = getDatabase() + ".batch_reuse"; - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { try (Statement stmt = conn.createStatement()) { assertEquals(stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + tableClause + " (id UInt8, num UInt8) ENGINE = MergeTree ORDER BY ()"), 0); // add and execute first invalid batch @@ -723,7 +736,7 @@ public void testSwitchDatabase() throws Exception { @Test(groups = {"integration"}) public void testNewLineSQLParsing() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { String sqlCreate = "CREATE TABLE balance ( `id` UUID, `currency` String, `amount` Decimal(64, 18), `create_time` DateTime64(6), `_version` UInt64, `_sign` UInt8 ) ENGINE = ReplacingMergeTree PRIMARY KEY id ORDER BY id;"; try (Statement stmt = conn.createStatement()) { int r = stmt.executeUpdate(sqlCreate); @@ -788,7 +801,7 @@ public void testNewLineSQLParsing() throws Exception { @Test(groups = {"integration"}) public void testNullableFixedStringType() throws Exception { - try (Connection conn = getJdbcConnection()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF))) { String sqlCreate = "CREATE TABLE `data_types` (`f1` FixedString(4),`f2` LowCardinality(FixedString(4)), `f3` Nullable(FixedString(4)), `f4` LowCardinality(Nullable(FixedString(4))) ) ENGINE Memory;"; try (Statement stmt = conn.createStatement()) { int r = stmt.executeUpdate(sqlCreate); @@ -1192,7 +1205,7 @@ public void testExecute() throws Exception { Assert.assertNull(stmt.getResultSet()); } - try (Connection conn = getJdbcConnection(); Statement stmt = conn.createStatement()) { + try (Connection conn = getJdbcConnection(Map.of(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF)); Statement stmt = conn.createStatement()) { // no result set and update count Assert.assertFalse(stmt.execute("CREATE TABLE test_multi_result (id Int32) Engine MergeTree ORDER BY ()")); Assert.assertNull(stmt.getResultSet()); @@ -1385,6 +1398,7 @@ public void testDescribeStatement() throws Exception { public void testUnknownStatement(String parserName) throws Exception { Properties properties = new Properties(); properties.setProperty(DriverProperties.SQL_PARSER.getKey(), parserName); + properties.setProperty(ASYNC_INSERT_SETTING_KEY, ServerSettings.OFF); try (Connection conn = getJdbcConnection(properties)) { try (Statement stmt = conn.createStatement()) { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java index db756d29c..a5a3e972f 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java @@ -121,11 +121,8 @@ public void testParseURLValid(String jdbcURL, Properties properties, throws Exception { JdbcConfiguration configuration = new JdbcConfiguration(jdbcURL, properties); - // We need to copy defaults to expected. - Map expectedWithDefaults = configuration.createProperties(); - expectedWithDefaults.putAll(expectedClientProps); assertEquals(configuration.getConnectionUrl(), connectionURL, "URL: " + jdbcURL); - assertEquals(configuration.clientProperties, expectedWithDefaults, "URL: " + jdbcURL); + assertEquals(configuration.clientProperties, expectedClientProps, "URL: " + jdbcURL); Client.Builder bob = new Client.Builder(); configuration.applyClientProperties(bob); Client client = bob.build();