Merge pull request #86 from zhicwu/develop

Fix mutation error due to closed response

Author: zhicwu

Dockerfile

@@ -38,16 +38,16 @@ RUN apt-get update \
 		apt-transport-https curl htop iftop iptraf iputils-ping jq lsof net-tools tzdata wget \
 	&& apt-get clean \
 	&& wget -q -P $JDBC_BRIDGE_HOME $JDBC_BRIDGE_REL_URL/LICENSE $JDBC_BRIDGE_REL_URL/NOTICE \
-		$JDBC_BRIDGE_REL_URL/clickhouse-jdbc-bridge-${revision}.jar \
+		$JDBC_BRIDGE_REL_URL/clickhouse-jdbc-bridge-${revision}-shaded.jar \
 	&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 COPY --chown=root:root docker/ $JDBC_BRIDGE_HOME
 
 RUN chmod +x $JDBC_BRIDGE_HOME/*.sh \
 	&& mkdir -p $JDBC_BRIDGE_HOME/logs /usr/local/lib/java \
 	&& ln -s $JDBC_BRIDGE_HOME/logs /var/log/clickhouse-jdbc-bridge \
-	&& ln -s $JDBC_BRIDGE_HOME/clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION.jar \
-		/usr/local/lib/java/clickhouse-jdbc-bridge.jar \
+	&& ln -s $JDBC_BRIDGE_HOME/clickhouse-jdbc-bridge-${revision}-shaded.jar \
+		/usr/local/lib/java/clickhouse-jdbc-bridge-shaded.jar \
 	&& ln -s $JDBC_BRIDGE_HOME /etc/clickhouse-jdbc-bridge
 
 WORKDIR $JDBC_BRIDGE_HOME

README.md

@@ -89,16 +89,38 @@ JDBC bridge for ClickHouse®. It acts as a stateless proxy passing queries from
         --query="select * from jdbc('self?datasource_column', 'select 1')"
     ```
 
+* Debian/RPM Package
+
+    Besides docker, you can download and install released Debian/RPM package on existing Linux system.
+
+    Debian/Ubuntu
+    ```bash
+    apt update && apt install -y procps wget
+    export JDBC_BRIDGE_VERSION=2.0.0
+    wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$JDBC_BRIDGE_VERSION/clickhouse-jdbc-bridge_$JDBC_BRIDGE_VERSION-1_all.deb
+    apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$JDBC_BRIDGE_VERSION-1_all.deb
+    clickhouse-jdbc-bridge
+    ```
+
+    CentOS/RHEL
+    ```bash
+    yum install -y wget
+    export JDBC_BRIDGE_VERSION=2.0.0
+    wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$JDBC_BRIDGE_VERSION/clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION-1.noarch.rpm
+    yum localinstall -y clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION-1.noarch.rpm
+    clickhouse-jdbc-bridge
+    ```
+
 * Java CLI
 
     ```bash
     export JDBC_BRIDGE_VERSION=2.0.0
-    wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$JDBC_BRIDGE_VERSION/clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION.jar
+    wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$JDBC_BRIDGE_VERSION/clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION-shaded.jar
     # add named datasource
     wget -P config/datasources https://raw.githubusercontent.com/ClickHouse/clickhouse-jdbc-bridge/master/misc/quick-start/jdbc-bridge/config/datasources/ch-server.json
     # start jdbc bridge, and then issue below query in ClickHouse for testing
     # select * from jdbc('ch-server', 'select 1')
-    java -jar clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION.jar
+    java -jar clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION-shaded.jar
     ```
 
 
@@ -195,6 +217,13 @@ Assuming you started a test environment using docker-compose, please refer to ex
 
 * Mutation
     ```sql
+    -- use query parameter
+    select * from jdbc('ch-server?mutation', 'drop table if exists system.test_table');
+    select * from jdbc('ch-server?mutation', 'create table system.test_table(a String, b UInt8) engine=Memory()');
+    select * from jdbc('ch-server?mutation', 'insert into system.test_table values(''a'', 1)');
+	select * from jdbc('ch-server?mutation', 'truncate table system.test_table');
+
+    -- use JDBC table engine
     drop table if exists system.test_table;
     create table system.test_table (
         a String,
@@ -205,7 +234,7 @@ Assuming you started a test environment using docker-compose, please refer to ex
     create table system.jdbc_table (
         a String,
         b UInt8
-    ) engine=JDBC('ch-server', 'system', 'test_table');
+    ) engine=JDBC('ch-server?batch_size=1000', 'system', 'test_table');
 
     insert into system.jdbc_table(a, b) values('a', 1);
 
@@ -280,8 +309,9 @@ Assuming you started a test environment using docker-compose, please refer to ex
     Couple of timeout settings you should be aware of:
     1. datasource timeout, for example: `max_execution_time` in MariaDB
     2. JDBC driver timeout, for example: `connectTimeout` and `socketTimeout` in [MariaDB Connector/J](https://mariadb.com/kb/en/about-mariadb-connector-j/)
-    3. Vertx timeout - see `config/server.json` and `config/vertx.json`
-    4. Client(ClickHouse JDBC driver) timeout - see timeout settings in ClickHouse JDBC driver
+    3. JDBC bridge timeout, for examples: `queryTimeout` in `config/server.json`, and `maxWorkerExecuteTime` in `config/vertx.json`
+    4. ClickHouse timeout like `max_execution_time` and `keep_alive_timeout` etc.
+    5. Client timeout, for example: `socketTimeout` in ClickHouse JDBC driver
 
 
 ## Migration
@@ -375,7 +405,7 @@ Test Case | Time Spent(s) | Throughput(#/s) | Failed Requests | Min(ms) | Mean(m
 [clickhouse_constant-query](misc/perf-test/results/clickhouse_constant-query.txt) | 797.775 | 125.35 | 0 | 1 | 159 | 4 | 1,077
 [clickhouse_constant-query(mysql)](misc/perf-test/results/clickhouse_constant-query(mysql).txt) | 1,598.426 | 62.56 | 0 | 7 | 320 | 18 | 2,049
 [clickhouse_constant-query(remote)](misc/perf-test/results/clickhouse_constant-query(remote).txt) | 802.212 | 124.66 | 0 | 2 | 160 | 8 | 3,073
-[clickhouse_constant-query(url)](misc/perf-test/results/clickhouclickhouse_constant-query(url)se_ping.txt) | 801.686 | 124.74 | 0 | 3 | 160 | 11 | 1,123
+[clickhouse_constant-query(url)](misc/perf-test/results/clickhouse_constant-query(url).txt) | 801.686 | 124.74 | 0 | 3 | 160 | 11 | 1,123
 [clickhouse_constant-query(jdbc)](misc/perf-test/results/clickhouse_constant-query(jdbc).txt) | 925.087 | 108.10 | 5,813 | 14 | 185 | 75 | 4,091
 [clickhouse(patched)_constant-query(jdbc)](misc/perf-test/results/clickhouse(patched)_constant-query(jdbc).txt) | 833.892 | 119.92 | 1,577 | 10 | 167 | 51 | 3,109
 [clickhouse(patched)_constant-query(jdbc-dual)](misc/perf-test/results/clickhouse(patched)_constant-query(jdbc-dual).txt) | 846.403 | 118.15 | 3,021 | 8 | 169 | 50 | 3,054

all-in-one.Dockerfile

@@ -69,7 +69,7 @@ RUN apt-get update \
 		https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.21/mysql-connector-java-8.0.21.jar \
 	&& wget -q -P /etc/clickhouse-jdbc-bridge/drivers/postgres \
 		https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar \
-	&& sed -i -e 's|\(^[[:space:]]*\)\(exec.*clickhouse-server.*$\)|\1exec clickhouse-jdbc-bridge\&\n\1\2|' /entrypoint.sh \
+	&& sed -i -e 's|\(^[[:space:]]*\)\(exec.*clickhouse-server.*$\)|\1exec -c clickhouse-jdbc-bridge >/dev/null \&\n\1\2|' /entrypoint.sh \
 	&& echo '{\n\
   "$schema": "../datasource-schema.json",\n\
   "self": {\n\

docker/config/server.json

@@ -1,6 +1,6 @@
 {
   "requestTimeout": 5000,
-  "queryTimeout": 30000,
+  "queryTimeout": 60000,
   "configScanPeriod": 5000,
   "repositories": [
     {

docker/docker-entrypoint.sh

@@ -21,7 +21,7 @@ start_server() {
 	fi
 
 	if [ "$(echo ${CUSTOM_DRIVER_LOADER:="true"} | tr '[:upper:]' '[:lower:]')" != "true" ]; then
-		local classpath="./clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION.jar:$(echo $(ls ${DRIVER_DIR:="drivers"}/*.jar) | tr ' ' ':'):."
+		local classpath="./clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION-shaded.jar:$(echo $(ls ${DRIVER_DIR:="drivers"}/*.jar) | tr ' ' ':'):."
 		java -XX:+UseContainerSupport -XX:+IdleTuningCompactOnIdle -XX:+IdleTuningGcOnIdle \
 			-Xdump:none -Xdump:tool:events=systhrow+throw,filter=*OutOfMemoryError,exec="kill -9 %pid" \
 			-Dlog4j.configuration=file:///$JDBC_BRIDGE_HOME/log4j.properties -Dnashorn.args=--language=es6 \
@@ -30,7 +30,7 @@ start_server() {
 		java -XX:+UseContainerSupport -XX:+IdleTuningCompactOnIdle -XX:+IdleTuningGcOnIdle \
 			-Xdump:none -Xdump:tool:events=systhrow+throw,filter=*OutOfMemoryError,exec="kill -9 %pid" \
 			-Dlog4j.configuration=file:///$JDBC_BRIDGE_HOME/log4j.properties -Dnashorn.args=--language=es6 \
-			${JDBC_BRIDGE_JVM_OPTS:=""} -jar clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION.jar
+			${JDBC_BRIDGE_JVM_OPTS:=""} -jar clickhouse-jdbc-bridge-$JDBC_BRIDGE_VERSION-shaded.jar
 	fi
 }
 

misc/perf-test/jdbc-bridge/config/datasources/mariadb.json

@@ -5,7 +5,7 @@
       "https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.7.0/mariadb-java-client-2.7.0.jar"
     ],
     "driverClassName": "org.mariadb.jdbc.Driver",
-    "jdbcUrl": "jdbc:mariadb://mariadb:3306/test?useSSL=false&useCompression=false",
+    "jdbcUrl": "jdbc:mariadb://mariadb:3306/test?useSSL=false&useCompression=false&rewriteBatchedStatements=true",
     "dataSource": {
       "user": "root",
       "password": "root"

pom.xml

@@ -240,7 +240,7 @@
                             <packageDescription>JDBC bridge for ClickHouse. It acts as a stateless proxy passing queries from ClickHouse to external datasources.</packageDescription>
                             <createSymLinks>true</createSymLinks>
                             <createIncludeFiles>false</createIncludeFiles>
-                            <excludeAllArtifacts>false</excludeAllArtifacts>
+                            <excludeAllArtifacts>true</excludeAllArtifacts>
                             <excludeAllDependencies>true</excludeAllDependencies>
                             <libDirectory>/usr/local/lib/java</libDirectory>
                             <useDefaultCopyResources>false</useDefaultCopyResources>
@@ -327,11 +327,11 @@
                                     <groupname>root</groupname>
                                     <sources>
                                         <source>
-                                            <location>target/clickhouse-jdbc-bridge-${project.version}.jar</location>
+                                            <location>target/clickhouse-jdbc-bridge-${project.version}-shaded.jar</location>
                                         </source>
                                         <softlinkSource>
-                                            <location>/usr/local/lib/java/clickhouse-jdbc-bridge-${project.version}.jar</location>
-                                            <destination>clickhouse-jdbc-bridge.jar</destination>
+                                            <location>/usr/local/lib/java/clickhouse-jdbc-bridge-${project.version}-shaded.jar</location>
+                                            <destination>clickhouse-jdbc-bridge-shaded.jar</destination>
                                         </softlinkSource>
                                     </sources>
                                 </mapping>
@@ -513,7 +513,7 @@
                 <artifactId>maven-shade-plugin</artifactId>
                 <version>${shade-plugin.version}</version>
                 <configuration>
-                    <shadedArtifactAttached>false</shadedArtifactAttached>
+                    <shadedArtifactAttached>true</shadedArtifactAttached>
                     <createDependencyReducedPom>true</createDependencyReducedPom>
                     <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
                     <shadedClassifierName>shaded</shadedClassifierName>

src/main/bin/clickhouse-jdbc-bridge

@@ -43,7 +43,7 @@ init_env() {
 
 start_server() {
     # override below environment variables as needed in $INIT_SCRIPT
-    : ${APP_PACKAGE:="/usr/local/lib/java/clickhouse-jdbc-bridge.jar"}
+    : ${APP_PACKAGE:="/usr/local/lib/java/clickhouse-jdbc-bridge-shaded.jar"}
     : ${JVM_ARGS:="-Xmx512m"}
     : ${WORK_DIRECTORY:=$(dirname "$INIT_SCRIPT")}
 
@@ -61,7 +61,7 @@ start_server() {
     fi
 
     echo "Work directory is set to [$WORK_DIRECTORY]"
-
+    cd "$WORK_DIRECTORY"
     java $JVM_ARGS -Dlog4j.configuration=file:///$WORK_DIRECTORY/log4j.properties \
         -Duser.dir="$WORK_DIRECTORY" -jar "$APP_PACKAGE"
 }