diff --git a/.github/workflows/build-connector.yml b/.github/workflows/build-connector.yml index 2b2a242be..783b44596 100644 --- a/.github/workflows/build-connector.yml +++ b/.github/workflows/build-connector.yml @@ -41,41 +41,78 @@ jobs: - name: Build flink connector 1.15 run: | cd flink-doris-connector && mvn clean package \ + -pl flink-doris-connector-flink1 -am \ + -Pflink1 \ -Dflink.version=1.15.0 \ - -Dflink.minor.version=1.15 \ + -Dflink.major.version=1.15 \ -Dflink.python.id=flink-python_2.12 - name: Build flink connector 1.16 run: | cd flink-doris-connector && mvn clean package \ + -pl flink-doris-connector-flink1 -am \ + -Pflink1 \ -Dflink.version=1.16.0 \ - -Dflink.minor.version=1.16 \ - -Dflink.python.id=flink-python + -Dflink.major.version=1.16 - name: Build flink connector 1.17 run: | cd flink-doris-connector && mvn clean package \ + -pl flink-doris-connector-flink1 -am \ + -Pflink1 \ -Dflink.version=1.17.0 \ - -Dflink.minor.version=1.17 \ - -Dflink.python.id=flink-python + -Dflink.major.version=1.17 - name: Build flink connector 1.18 run: | cd flink-doris-connector && mvn clean package \ + -pl flink-doris-connector-flink1 -am \ + -Pflink1 \ -Dflink.version=1.18.0 \ - -Dflink.minor.version=1.18 \ - -Dflink.python.id=flink-python + -Dflink.major.version=1.18 - name: Build flink connector 1.19 run: | cd flink-doris-connector && mvn clean package \ + -pl flink-doris-connector-flink1 -am \ + -Pflink1 \ -Dflink.version=1.19.0 \ - -Dflink.minor.version=1.19 \ - -Dflink.python.id=flink-python + -Dflink.major.version=1.19 - name: Build flink connector 1.20 run: | cd flink-doris-connector && mvn clean package \ + -pl flink-doris-connector-flink1 -am \ + -Pflink1 \ -Dflink.version=1.20.0 \ - -Dflink.minor.version=1.20 \ - -Dflink.python.id=flink-python + -Dflink.major.version=1.20 + + - name: Setup java + uses: actions/setup-java@v2 + with: + distribution: adopt + java-version: '17' + + - name: Build flink connector 2.0 + run: | + cd flink-doris-connector && mvn clean package \ + -pl flink-doris-connector-flink2 -am \ + -Pflink2 \ + -Dflink.version=2.0.0 \ + -Dflink.major.version=2.0 + + - name: Build flink connector 2.1 + run: | + cd flink-doris-connector && mvn clean package \ + -pl flink-doris-connector-flink2 -am \ + -Pflink2 \ + -Dflink.version=2.1.0 \ + -Dflink.major.version=2.1 + + - name: Build flink connector 2.2 + run: | + cd flink-doris-connector && mvn clean package \ + -pl flink-doris-connector-flink2 -am \ + -Pflink2 \ + -Dflink.version=2.2.0 \ + -Dflink.major.version=2.2 \ No newline at end of file diff --git a/.github/workflows/checkstyle.yaml b/.github/workflows/checkstyle.yaml index 325ed71a6..083729fc2 100644 --- a/.github/workflows/checkstyle.yaml +++ b/.github/workflows/checkstyle.yaml @@ -32,7 +32,7 @@ jobs: persist-credentials: false submodules: recursive - - name: Setup java + - name: Setup java 8 uses: actions/setup-java@v2 with: distribution: adopt @@ -40,4 +40,14 @@ jobs: - name: Run java checkstyle run: - cd flink-doris-connector && mvn clean compile checkstyle:checkstyle \ No newline at end of file + cd flink-doris-connector && mvn clean checkstyle:check -Pflink1 -pl flink-doris-connector-flink1 -am + + - name: Setup java 17 + uses: actions/setup-java@v2 + with: + distribution: adopt + java-version: '17' + + - name: Run java checkstyle + run: + cd flink-doris-connector && mvn clean checkstyle:check -Pflink2 -pl flink-doris-connector-flink2 -am \ No newline at end of file diff --git a/.github/workflows/run-e2ecase.yml b/.github/workflows/run-e2ecase-flink1.yml similarity index 79% rename from .github/workflows/run-e2ecase.yml rename to .github/workflows/run-e2ecase-flink1.yml index e6318f8f9..c3e861562 100644 --- a/.github/workflows/run-e2ecase.yml +++ b/.github/workflows/run-e2ecase-flink1.yml @@ -16,7 +16,7 @@ # under the License. # --- -name: Run E2ECases +name: Run E2ECases Flink1.x on: pull_request: push: @@ -38,7 +38,10 @@ jobs: distribution: adopt java-version: '8' - - name: Run E2ECases + - name: Run E2ECases (Flink1 module) run: | - cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="apache/doris:doris-all-in-one-2.1.0" - + cd flink-doris-connector && mvn test \ + -Pflink1 -pl flink-doris-connector-it -am \ + -DfailIfNoTests=false \ + -Dtest="*E2ECase" \ + -Dimage="apache/doris:doris-all-in-one-2.1.0" \ No newline at end of file diff --git a/.github/workflows/run-e2ecase-flink2.yml b/.github/workflows/run-e2ecase-flink2.yml new file mode 100644 index 000000000..8883e3247 --- /dev/null +++ b/.github/workflows/run-e2ecase-flink2.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Run E2ECases Flink2.x +on: + pull_request: + push: + +jobs: + build-extension: + name: "Run E2ECases" + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Setup java + uses: actions/setup-java@v2 + with: + distribution: adopt + java-version: '17' + + - name: Run E2ECases (Flink2 module) + run: | + cd flink-doris-connector && mvn test \ + -Pflink2 -pl flink-doris-connector-it -am \ + -DfailIfNoTests=false \ + -Dtest="*E2ECase" \ + -Dimage="apache/doris:doris-all-in-one-2.1.0" \ No newline at end of file diff --git a/.github/workflows/run-itcase.yml b/.github/workflows/run-itcase-flink1.yml similarity index 80% rename from .github/workflows/run-itcase.yml rename to .github/workflows/run-itcase-flink1.yml index 0b6706560..f7d7a76ca 100644 --- a/.github/workflows/run-itcase.yml +++ b/.github/workflows/run-itcase-flink1.yml @@ -16,7 +16,7 @@ # under the License. # --- -name: Run ITCases +name: Run ITCases Flink1.x on: pull_request: push: @@ -38,7 +38,10 @@ jobs: distribution: adopt java-version: '8' - - name: Run ITCases + - name: Run ITCases (Flink1 module) run: | - cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="apache/doris:doris-all-in-one-2.1.0" - + cd flink-doris-connector && mvn test \ + -Pflink1 -pl flink-doris-connector-it -am \ + -DfailIfNoTests=false \ + -Dtest="*ITCase" \ + -Dimage="apache/doris:doris-all-in-one-2.1.0" diff --git a/.github/workflows/run-itcase-flink2.yml b/.github/workflows/run-itcase-flink2.yml new file mode 100644 index 000000000..4c55f9e72 --- /dev/null +++ b/.github/workflows/run-itcase-flink2.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Run ITCases Flink2.x +on: + pull_request: + push: + +jobs: + build-extension: + name: "Run ITCases" + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Setup java + uses: actions/setup-java@v2 + with: + distribution: adopt + java-version: '17' + + - name: Run ITCases (Flink2 module) + run: | + cd flink-doris-connector && mvn test \ + -Pflink2 -pl flink-doris-connector-it -am \ + -DfailIfNoTests=false \ + -Dtest="*ITCase" \ + -Dimage="apache/doris:doris-all-in-one-2.1.0" + diff --git a/.licenserc.yaml b/.licenserc.yaml index 27e1080ed..4ccca0d23 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -12,6 +12,6 @@ header: - '.github/PULL_REQUEST_TEMPLATE.md' - '.licenserc.yaml' - 'custom_env.sh.tpl' - - 'flink-doris-connector/src/test/resources/container/' + - 'flink-doris-connector/flink-doris-connector-base/src/test/resources/container/' comment: on-failure diff --git a/README.md b/README.md index 51481f6fd..b2892d52f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,19 @@ under the License. ## Flink Doris Connector -Flink Doris Connector now support flink version from 1.11 to 1.20. +Flink Doris Connector supports the following Flink versions: + +| Flink Version | JDK Requirement | +|:--------------|:----------------| +| 1.15.x | JDK 8 | +| 1.16.x | JDK 8 | +| 1.17.x | JDK 8 | +| 1.18.x | JDK 8 | +| 1.19.x | JDK 8 | +| 1.20.x | JDK 8 | +| 2.0.x | JDK 17 | +| 2.1.x | JDK 17 | +| 2.2.x | JDK 17 | If you wish to contribute or use a connector from flink 1.13 (and earlier), please use the [branch-for-flink-before-1.13](https://github.com/apache/doris-flink-connector/tree/branch-for-flink-before-1.13) @@ -37,6 +49,9 @@ More information about compilation and usage, please visit [Flink Doris Connecto ## How to Build You need to copy customer_env.sh.tpl to customer_env.sh before build and you need to configure it before build. + +> **JDK requirement:** Use JDK 8 for Flink 1.x, and JDK 17 for Flink 2.x. + ```shell git clone git@github.com:apache/doris-flink-connector.git cd doris-flink-connector/flink-doris-connector diff --git a/flink-doris-connector/build.sh b/flink-doris-connector/build.sh index 4852553b5..3b4be40da 100755 --- a/flink-doris-connector/build.sh +++ b/flink-doris-connector/build.sh @@ -116,7 +116,9 @@ fi selectFlink() { echo 'Flink-Doris-Connector supports multiple versions of flink. Which version do you need ?' - select flink in "1.15.x" "1.16.x" "1.17.x" "1.18.x" "1.19.x" "1.20.x" + echo ' [Flink 1.x requires JDK 8]' + echo ' [Flink 2.x requires JDK 17]' + select flink in "1.15.x" "1.16.x" "1.17.x" "1.18.x" "1.19.x" "1.20.x" "2.0.x" "2.1.x" "2.2.x" do case $flink in "1.15.x") @@ -137,6 +139,15 @@ selectFlink() { "1.20.x") return 6 ;; + "2.0.x") + return 7 + ;; + "2.1.x") + return 8 + ;; + "2.2.x") + return 9 + ;; *) echo "invalid selected, exit.." exit 1 @@ -149,6 +160,8 @@ FLINK_VERSION=0 selectFlink flinkVer=$? FLINK_PYTHON_ID="flink-python" +FLINK_MODULE="flink-doris-connector-flink1" +FLINK_PROFILE="flink1" if [ ${flinkVer} -eq 1 ]; then FLINK_VERSION="1.15.0" FLINK_PYTHON_ID="flink-python_2.12" @@ -162,25 +175,61 @@ elif [ ${flinkVer} -eq 5 ]; then FLINK_VERSION="1.19.0" elif [ ${flinkVer} -eq 6 ]; then FLINK_VERSION="1.20.0" +elif [ ${flinkVer} -eq 7 ]; then + FLINK_VERSION="2.0.0" + FLINK_MODULE="flink-doris-connector-flink2" + FLINK_PROFILE="flink2" +elif [ ${flinkVer} -eq 8 ]; then + FLINK_VERSION="2.1.0" + FLINK_MODULE="flink-doris-connector-flink2" + FLINK_PROFILE="flink2" +elif [ ${flinkVer} -eq 9 ]; then + FLINK_VERSION="2.2.0" + FLINK_MODULE="flink-doris-connector-flink2" + FLINK_PROFILE="flink2" fi # extract major version: -# eg: 3.1.2 -> 3 +# eg: 3.1.2 -> 3.1 FLINK_MAJOR_VERSION=0 [ ${FLINK_VERSION} != 0 ] && FLINK_MAJOR_VERSION=${FLINK_VERSION%.*} +# Sanity check: Flink 1.x requires JDK 1.8, Flink 2.x requires JDK 17 +JAVA_VERSION=$(java -version 2>&1 | awk -F'"' '/version/ {print $2}') +if [ "${FLINK_PROFILE}" = "flink1" ]; then + if [[ "${JAVA_VERSION}" != 1.8* ]]; then + echo_r "Error: Flink ${FLINK_VERSION} requires JDK 1.8, but found version '${JAVA_VERSION}'." + echo_r "Please switch to JDK 1.8 before building Flink 1.x." + exit 1 + fi +elif [ "${FLINK_PROFILE}" = "flink2" ]; then + if [[ "${JAVA_VERSION}" != 17* ]]; then + echo_r "Error: Flink ${FLINK_VERSION} requires JDK 17, but found version '${JAVA_VERSION}'." + echo_r "Please switch to JDK 17 before building Flink 2.x." + exit 1 + fi +fi +echo_g " JDK version check passed: ${JAVA_VERSION}" + echo_g " flink version: ${FLINK_VERSION}, major version: ${FLINK_MAJOR_VERSION}" echo_g " build starting..." -${MVN_BIN} clean package -Dflink.version=${FLINK_VERSION} -Dflink.major.version=${FLINK_MAJOR_VERSION} -Dflink.python.id=${FLINK_PYTHON_ID} "$@" +${MVN_BIN} clean package \ + -pl ${FLINK_MODULE} -am \ + -P${FLINK_PROFILE} \ + -DskipTests \ + -Dflink.version=${FLINK_VERSION} \ + -Dflink.major.version=${FLINK_MAJOR_VERSION} \ + -Dflink.python.id=${FLINK_PYTHON_ID} \ + "$@" EXIT_CODE=$? if [ $EXIT_CODE -eq 0 ]; then DIST_DIR=${DORIS_HOME}/dist [ ! -d "$DIST_DIR" ] && mkdir "$DIST_DIR" - dist_jar=$(ls "${ROOT}"/target | grep "flink-doris-" | grep -v "sources.jar" | grep -v "original-") + dist_jar=$(ls "${ROOT}/${FLINK_MODULE}/target" | grep "flink-doris-" | grep -v "sources.jar" | grep -v "original-" | grep -v "\-tests\.jar") rm -rf "${DIST_DIR}"/"${dist_jar}" - cp "${ROOT}"/target/"${dist_jar}" "$DIST_DIR" + cp "${ROOT}/${FLINK_MODULE}/target/${dist_jar}" "$DIST_DIR" echo_g "*****************************************************************" echo_g "Successfully build Flink-Doris-Connector" echo_g "dist: $DIST_DIR/$dist_jar " diff --git a/flink-doris-connector/flink-doris-connector-base/pom.xml b/flink-doris-connector/flink-doris-connector-base/pom.xml new file mode 100644 index 000000000..2b87e2339 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-base/pom.xml @@ -0,0 +1,239 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector-parent + ${revision} + ../pom.xml + + + flink-doris-connector-base + Flink Doris Connector Base + Base code for Flink Doris Connector; built per Flink version via profile (flink1 / flink2) + jar + + + + org.apache.doris + thrift-service + + + + org.apache.flink + flink-table-planner-loader + + + org.apache.flink + flink-connector-base + + + org.apache.flink + flink-table-api-java-bridge + + + org.apache.flink + flink-table-runtime + + + org.apache.flink + ${flink.python.id} + + + org.apache.flink + flink-runtime-web + + + + org.apache.thrift + libthrift + + + httpclient + org.apache.httpcomponents + + + httpcore + org.apache.httpcomponents + + + + + org.apache.httpcomponents + httpclient + + + commons-codec + commons-codec + + + + + commons-codec + commons-codec + + + org.apache.arrow.adbc + adbc-driver-flight-sql + + + org.apache.arrow + flight-sql-jdbc-core + + + org.apache.arrow + flight-sql-jdbc-driver + + + org.apache.arrow + arrow-vector + + + org.apache.arrow.adbc + adbc-driver-manager + + + + + org.apache.arrow + arrow-vector + + + org.apache.arrow + arrow-memory-netty + runtime + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.google.guava + guava + + + com.github.jsqlparser + jsqlparser + + + + + junit + junit + + + org.hamcrest + hamcrest-core + + + org.mockito + mockito-core + + + org.mockito + mockito-inline + + + + mysql + mysql-connector-java + + + com.google.protobuf + protobuf-java + + + + + + org.testcontainers + testcontainers + + + org.testcontainers + mysql + + + org.apache.flink + flink-test-utils + + + org.apache.flink + flink-connector-test-utils + + + org.apache.flink + flink-table-common + ${flink.version} + test-jar + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + + test-jar + + + + + + + diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/backend/BackendClient.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/backend/BackendClient.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/backend/BackendClient.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/backend/BackendClient.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java similarity index 98% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java index bfca184ec..adfb3d366 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java +++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java @@ -79,6 +79,7 @@ public class DorisTypeMapper { /** Max size of varchar type of Doris. */ public static final int MAX_VARCHAR_SIZE = 65533; + /* Max precision of datetime type of Doris. */ public static final int MAX_SUPPORTED_DATE_TIME_PRECISION = 6; @@ -118,8 +119,8 @@ public static DataType toFlinkType( case STRING: case JSONB: case JSON: - // Currently, the subtype of the generic cannot be obtained, - // so it is mapped to string + // Currently, the subtype of the generic cannot be obtained, + // so it is mapped to string case ARRAY: case MAP: case STRUCT: diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java similarity index 99% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java index 37338b28b..4ea075766 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java +++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java @@ -31,6 +31,7 @@ public class DorisConnectionOptions implements Serializable { protected final String password; protected String jdbcUrl; protected String benodes; + /** * Used to enable automatic redirection of fe, When it is not enabled, it will actively request * the be list, and the polling will initiate a streamload request to be. diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java similarity index 99% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java index 2b50f2535..2e8f2fae4 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java +++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java @@ -259,6 +259,7 @@ public static class Builder { private Long rowLimit; private Integer thriftMaxMessageSize = ConfigurationOptions.DORIS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT; + /** * Sets the readFields for doris table to push down projection, such as name,age. * diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CreateTableException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/CreateTableException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CreateTableException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/CreateTableException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Get.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Get.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Get.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Get.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/GetAction.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/GetAction.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/GetAction.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/GetAction.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Record.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Record.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/RecordKey.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/RecordKey.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Worker.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Worker.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/RestService.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/RestService.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/RestService.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/RestService.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Field.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Field.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Field.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Field.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/RespContent.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/RespContent.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/RespContent.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/RespContent.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Schema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Schema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Schema.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Schema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Tablet.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Tablet.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Tablet.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Tablet.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/Routing.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/serialization/Routing.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/Routing.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/serialization/Routing.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/RowBatch.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/serialization/RowBatch.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/RowBatch.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/serialization/RowBatch.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/BackendUtil.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/BackendUtil.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpUtil.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpUtil.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/LoadStatus.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/LoadStatus.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java similarity index 92% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java index 8eb98037f..10d482698 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java +++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java @@ -73,7 +73,9 @@ public void setLabelName(String labelName) { this.labelName = labelName; } - /** @return true if buffer is empty */ + /** + * @return true if buffer is empty + */ public boolean isEmpty() { return numOfRecords == 0; } @@ -90,22 +92,30 @@ public LinkedList getBuffer() { return buffer; } - /** @return Number of records in this buffer */ + /** + * @return Number of records in this buffer + */ public int getNumOfRecords() { return numOfRecords; } - /** @return Buffer size in bytes */ + /** + * @return Buffer size in bytes + */ public long getBufferSizeBytes() { return bufferSizeBytes; } - /** @param numOfRecords Updates number of records (Usually by 1) */ + /** + * @param numOfRecords Updates number of records (Usually by 1) + */ public void setNumOfRecords(int numOfRecords) { this.numOfRecords = numOfRecords; } - /** @param bufferSizeBytes Updates sum of size of records present in this buffer (Bytes) */ + /** + * @param bufferSizeBytes Updates sum of size of records present in this buffer (Bytes) + */ public void setBufferSizeBytes(long bufferSizeBytes) { this.bufferSizeBytes = bufferSizeBytes; } diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java similarity index 90% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java index b0b256eeb..afde60238 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java +++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java @@ -17,8 +17,7 @@ package org.apache.doris.flink.sink.batch; -import org.apache.flink.api.connector.sink2.Sink; -import org.apache.flink.runtime.checkpoint.CheckpointIDCounter; +import org.apache.flink.api.connector.sink2.SinkWriter.Context; import org.apache.flink.util.Preconditions; import org.apache.flink.util.StringUtils; import org.apache.flink.util.concurrent.ExecutorThreadFactory; @@ -27,7 +26,6 @@ import org.apache.doris.flink.cfg.DorisOptions; import org.apache.doris.flink.cfg.DorisReadOptions; import org.apache.doris.flink.sink.DorisCommittable; -import org.apache.doris.flink.sink.writer.DorisAbstractWriter; import org.apache.doris.flink.sink.writer.DorisWriterState; import org.apache.doris.flink.sink.writer.LabelGenerator; import org.apache.doris.flink.sink.writer.serializer.DorisRecord; @@ -44,9 +42,15 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -/** Doris Batch StreamLoad. */ -public class DorisBatchWriter - implements DorisAbstractWriter { +/** + * Doris Batch StreamLoad. + * + *

Core batch stream load writer shared across Flink versions. + * + *

Version-specific modules should wrap this class (for example via DorisBatchWriterAdapter and + * implement the appropriate Flink sink interfaces there. + */ +public class DorisBatchWriter { private static final Logger LOG = LoggerFactory.getLogger(DorisBatchWriter.class); private DorisBatchStreamLoad batchStreamLoad; private final DorisOptions dorisOptions; @@ -63,16 +67,13 @@ public class DorisBatchWriter private int subtaskId; public DorisBatchWriter( - Sink.InitContext initContext, + long lastCheckpointId, + int subtaskId, DorisRecordSerializer serializer, DorisOptions dorisOptions, DorisReadOptions dorisReadOptions, DorisExecutionOptions executionOptions) { - - long restoreCheckpointId = - initContext - .getRestoredCheckpointId() - .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1); + long restoreCheckpointId = lastCheckpointId; LOG.info("restore from checkpointId {}", restoreCheckpointId); if (!StringUtils.isNullOrWhitespaceOnly(dorisOptions.getTableIdentifier())) { String[] tableInfo = dorisOptions.getTableIdentifier().split("\\."); @@ -84,8 +85,8 @@ public DorisBatchWriter( } LOG.info("labelPrefix " + executionOptions.getLabelPrefix()); - this.subtaskId = initContext.getSubtaskId(); - this.labelPrefix = executionOptions.getLabelPrefix() + "_" + initContext.getSubtaskId(); + this.subtaskId = subtaskId; + this.labelPrefix = executionOptions.getLabelPrefix() + "_" + subtaskId; this.labelGenerator = new LabelGenerator(labelPrefix, false); this.scheduledExecutorService = new ScheduledThreadPoolExecutor( @@ -122,13 +123,11 @@ private void intervalFlush() { } } - @Override public void write(IN in, Context context) throws IOException, InterruptedException { checkFlushException(); writeOneDorisRecord(serializer.serialize(in)); } - @Override public void flush(boolean flush) throws IOException, InterruptedException { checkFlushException(); writeOneDorisRecord(serializer.flush()); @@ -136,13 +135,11 @@ public void flush(boolean flush) throws IOException, InterruptedException { batchStreamLoad.checkpointFlush(); } - @Override public Collection prepareCommit() throws IOException, InterruptedException { checkFlushException(); return Collections.emptyList(); } - @Override public List snapshotState(long checkpointId) throws IOException { checkFlushException(); return new ArrayList<>(); @@ -163,7 +160,6 @@ public void writeOneDorisRecord(DorisRecord record) throws InterruptedException batchStreamLoad.writeRecord(db, tbl, record.getRow()); } - @Override public void close() throws Exception { LOG.info("DorisBatchWriter Close"); if (scheduledExecutorService != null) { diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java similarity index 93% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java index e5f4c4ebb..2d902050e 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java +++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java @@ -101,7 +101,9 @@ public void setLabelName(String labelName) { this.labelName = labelName; } - /** @return true if buffer is empty */ + /** + * @return true if buffer is empty + */ public boolean isEmpty() { return numOfRecords == 0; } @@ -125,22 +127,30 @@ public ByteBuffer getBuffer() { return buffer; } - /** @return Number of records in this buffer */ + /** + * @return Number of records in this buffer + */ public int getNumOfRecords() { return numOfRecords; } - /** @return Buffer size in bytes */ + /** + * @return Buffer size in bytes + */ public int getBufferSizeBytes() { return bufferSizeBytes; } - /** @param numOfRecords Updates number of records (Usually by 1) */ + /** + * @param numOfRecords Updates number of records (Usually by 1) + */ public void setNumOfRecords(int numOfRecords) { this.numOfRecords = numOfRecords; } - /** @param bufferSizeBytes Updates sum of size of records present in this buffer (Bytes) */ + /** + * @param bufferSizeBytes Updates sum of size of records present in this buffer (Bytes) + */ public void setBufferSizeBytes(int bufferSizeBytes) { this.bufferSizeBytes = bufferSizeBytes; } diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java similarity index 92% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java index b47f2ebee..fc7d4b4ba 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java +++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java @@ -18,8 +18,7 @@ package org.apache.doris.flink.sink.copy; import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.api.connector.sink2.Sink; -import org.apache.flink.runtime.checkpoint.CheckpointIDCounter; +import org.apache.flink.api.connector.sink2.SinkWriter.Context; import org.apache.flink.util.Preconditions; import org.apache.flink.util.StringUtils; import org.apache.flink.util.concurrent.ExecutorThreadFactory; @@ -27,7 +26,6 @@ import org.apache.doris.flink.cfg.DorisExecutionOptions; import org.apache.doris.flink.cfg.DorisOptions; import org.apache.doris.flink.cfg.DorisReadOptions; -import org.apache.doris.flink.sink.writer.DorisAbstractWriter; import org.apache.doris.flink.sink.writer.DorisWriterState; import org.apache.doris.flink.sink.writer.LabelGenerator; import org.apache.doris.flink.sink.writer.serializer.DorisRecord; @@ -46,8 +44,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -public class DorisCopyWriter - implements DorisAbstractWriter { +public class DorisCopyWriter { private static final Logger LOG = LoggerFactory.getLogger(DorisCopyWriter.class); private final long lastCheckpointId; @@ -65,7 +62,8 @@ public class DorisCopyWriter private String table; public DorisCopyWriter( - Sink.InitContext initContext, + long lastCheckpointId, + int subtaskId, DorisRecordSerializer serializer, DorisOptions dorisOptions, DorisReadOptions dorisReadOptions, @@ -78,17 +76,14 @@ public DorisCopyWriter( this.database = tableInfo[0]; this.table = tableInfo[1]; } - this.lastCheckpointId = - initContext - .getRestoredCheckpointId() - .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1); + this.lastCheckpointId = lastCheckpointId; LOG.info("restore checkpointId {}", lastCheckpointId); LOG.info("labelPrefix {}", executionOptions.getLabelPrefix()); this.labelPrefix = executionOptions.getLabelPrefix() + "_" + UUID.randomUUID().toString().replaceAll("-", ""); - this.labelGenerator = new LabelGenerator(labelPrefix, false, initContext.getSubtaskId()); + this.labelGenerator = new LabelGenerator(labelPrefix, false, subtaskId); this.scheduledExecutorService = new ScheduledThreadPoolExecutor( 1, new ExecutorThreadFactory("copy-upload-interval")); @@ -121,7 +116,6 @@ private void intervalFlush() { } } - @Override public void write(IN in, Context context) throws IOException, InterruptedException { checkFlushException(); writeOneDorisRecord(serializer.serialize(in)); @@ -142,7 +136,6 @@ public void writeOneDorisRecord(DorisRecord record) throws InterruptedException batchStageLoad.writeRecord(db, tbl, record.getRow()); } - @Override public void flush(boolean b) throws IOException, InterruptedException { checkFlushException(); writeOneDorisRecord(serializer.flush()); @@ -150,7 +143,6 @@ public void flush(boolean b) throws IOException, InterruptedException { batchStageLoad.flush(null, true); } - @Override public Collection prepareCommit() throws IOException, InterruptedException { Preconditions.checkState(batchStageLoad != null); @@ -168,7 +160,6 @@ public Collection prepareCommit() return committables; } - @Override public List snapshotState(long checkpointId) throws IOException { Preconditions.checkState(batchStageLoad != null); if (!batchStageLoad.getFileListMap().isEmpty()) { @@ -181,7 +172,6 @@ public List snapshotState(long checkpointId) throws IOExceptio return Collections.emptyList(); } - @Override public void close() throws Exception { LOG.info("DorisBatchWriter Close"); if (scheduledExecutorService != null) { diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java similarity index 96% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java index da07249fd..4435b360a 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java +++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java @@ -18,9 +18,8 @@ package org.apache.doris.flink.sink.writer; import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter.Context; import org.apache.flink.metrics.groups.SinkWriterMetricGroup; -import org.apache.flink.runtime.checkpoint.CheckpointIDCounter; import org.apache.commons.lang3.StringUtils; import org.apache.doris.flink.cfg.DorisExecutionOptions; @@ -51,10 +50,13 @@ /** * Doris Writer will load data to doris. * - * @param + *

This is the core implementation that is shared across different Flink versions. + * Version-specific modules should wrap this class and implement the appropriate Flink sink + * interfaces there (for example via DorisWriterAdapter). + * + * @param record type */ -public class DorisWriter - implements DorisAbstractWriter { +public class DorisWriter { private static final Logger LOG = LoggerFactory.getLogger(DorisWriter.class); private final long lastCheckpointId; private long curCheckpointId; @@ -78,21 +80,20 @@ public class DorisWriter private volatile boolean multiTableLoad = false; public DorisWriter( - Sink.InitContext initContext, + long lastCheckpointId, + int subtaskId, + SinkWriterMetricGroup sinkMetricGroup, Collection state, DorisRecordSerializer serializer, DorisOptions dorisOptions, DorisReadOptions dorisReadOptions, DorisExecutionOptions executionOptions) { - this.lastCheckpointId = - initContext - .getRestoredCheckpointId() - .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1); + this.lastCheckpointId = lastCheckpointId; this.curCheckpointId = lastCheckpointId + 1; LOG.info("restore from checkpointId {}", lastCheckpointId); LOG.info("labelPrefix {}", executionOptions.getLabelPrefix()); this.labelPrefix = executionOptions.getLabelPrefix(); - this.subtaskId = initContext.getSubtaskId(); + this.subtaskId = subtaskId; this.scheduledExecutorService = new ScheduledThreadPoolExecutor( 1, @@ -111,7 +112,7 @@ public DorisWriter( this.executionOptions = executionOptions; this.intervalTime = executionOptions.checkInterval(); this.globalLoading = false; - sinkMetricGroup = initContext.metricGroup(); + this.sinkMetricGroup = sinkMetricGroup; initializeLoad(state); serializer.initial(); } @@ -167,13 +168,11 @@ public void abortLingeringTransactions(Collection recoveredSta } } - @Override public void write(IN in, Context context) throws IOException, InterruptedException { checkLoadException(); writeOneDorisRecord(serializer.serialize(in)); } - @Override public void flush(boolean endOfInput) throws IOException, InterruptedException { writeOneDorisRecord(serializer.flush()); } @@ -216,7 +215,6 @@ public void registerMetrics(String tableKey) { sinkMetricsMap.put(tableKey, metrics); } - @Override public Collection prepareCommit() throws IOException, InterruptedException { // Verify whether data is written during a checkpoint if (!globalLoading && loadingMap.values().stream().noneMatch(Boolean::booleanValue)) { @@ -272,7 +270,6 @@ private void abortPossibleSuccessfulTransaction() { } } - @Override public List snapshotState(long checkpointId) throws IOException { List writerStates = new ArrayList<>(); for (DorisStreamLoad dorisStreamLoad : dorisStreamLoadMap.values()) { @@ -423,7 +420,6 @@ public void setBackendUtil(BackendUtil backendUtil) { this.backendUtil = backendUtil; } - @Override public void close() throws Exception { LOG.info("Close DorisWriter."); if (scheduledExecutorService != null) { diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/EventType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/EventType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/EventType.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/EventType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/DorisSource.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/DorisSource.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/DorisSource.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/DorisSource.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java similarity index 98% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java index a4b07677e..ccda25083 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java +++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java @@ -89,13 +89,7 @@ public void open(FunctionContext context) throws Exception { this.lookupMetrics = new LookupMetrics(context.getMetricGroup()); } - /** - * This is a lookup method which is called by Flink framework in runtime. - * - * @param keys lookup keys - */ - public void eval(Object... keys) throws IOException { - RowData keyRow = GenericRowData.of(keys); + public void eval(RowData keyRow) throws IOException { if (cache != null) { List cachedRows = cache.getIfPresent(keyRow); if (cachedRows != null) { @@ -114,6 +108,16 @@ public void eval(Object... keys) throws IOException { queryRecord(keyRow); } + /** + * This is a lookup method which is called by Flink framework in runtime. + * + * @param keys lookup keys + */ + public void eval(Object... keys) throws IOException { + RowData keyRow = GenericRowData.of(keys); + eval(keyRow); + } + private void queryRecord(RowData keyRow) throws IOException { List rowData = lookupReader.get(keyRow); if (rowData == null) { diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/converter/TableNameConverter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/converter/TableNameConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/converter/TableNameConverter.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/converter/TableNameConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java similarity index 82% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java index ddffb6d9f..aff073821 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java +++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java @@ -131,27 +131,27 @@ public static String toDorisType(String postgresType, Integer precision, Integer case JSON: case JSONB: return DorisType.JSONB; - /* Compatible with doris1.2 array type can only be used in dup table, - and then converted to array in the next version - case _BOOL: - return String.format("%s<%s>", DorisType.ARRAY, DorisType.BOOLEAN); - case _INT2: - return String.format("%s<%s>", DorisType.ARRAY, DorisType.TINYINT); - case _INT4: - return String.format("%s<%s>", DorisType.ARRAY, DorisType.INT); - case _INT8: - return String.format("%s<%s>", DorisType.ARRAY, DorisType.BIGINT); - case _FLOAT4: - return String.format("%s<%s>", DorisType.ARRAY, DorisType.FLOAT); - case _FLOAT8: - return String.format("%s<%s>", DorisType.ARRAY, DorisType.DOUBLE); - case _TEXT: - return String.format("%s<%s>", DorisType.ARRAY, DorisType.STRING); - case _DATE: - return String.format("%s<%s>", DorisType.ARRAY, DorisType.DATE_V2); - case _TIMESTAMP: - return String.format("%s<%s>", DorisType.ARRAY, DorisType.DATETIME_V2); - **/ + /* Compatible with doris1.2 array type can only be used in dup table, + and then converted to array in the next version + case _BOOL: + return String.format("%s<%s>", DorisType.ARRAY, DorisType.BOOLEAN); + case _INT2: + return String.format("%s<%s>", DorisType.ARRAY, DorisType.TINYINT); + case _INT4: + return String.format("%s<%s>", DorisType.ARRAY, DorisType.INT); + case _INT8: + return String.format("%s<%s>", DorisType.ARRAY, DorisType.BIGINT); + case _FLOAT4: + return String.format("%s<%s>", DorisType.ARRAY, DorisType.FLOAT); + case _FLOAT8: + return String.format("%s<%s>", DorisType.ARRAY, DorisType.DOUBLE); + case _TEXT: + return String.format("%s<%s>", DorisType.ARRAY, DorisType.STRING); + case _DATE: + return String.format("%s<%s>", DorisType.ARRAY, DorisType.DATE_V2); + case _TIMESTAMP: + return String.format("%s<%s>", DorisType.ARRAY, DorisType.DATETIME_V2); + **/ default: throw new UnsupportedOperationException( "Unsupported Postgres Type: " + postgresType); diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/ErrorMessages.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/ErrorMessages.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/ErrorMessages.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/ErrorMessages.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/FastDateUtil.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/FastDateUtil.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/FastDateUtil.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/FastDateUtil.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/IPUtils.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/IPUtils.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/IPUtils.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/IPUtils.java diff --git a/flink-doris-connector/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java diff --git a/flink-doris-connector/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-doris-connector/flink-doris-connector-base/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory similarity index 100% rename from flink-doris-connector/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory rename to flink-doris-connector/flink-doris-connector-base/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory diff --git a/flink-doris-connector/src/main/resources/log4j2.properties b/flink-doris-connector/flink-doris-connector-base/src/main/resources/log4j2.properties similarity index 100% rename from flink-doris-connector/src/main/resources/log4j2.properties rename to flink-doris-connector/flink-doris-connector-base/src/main/resources/log4j2.properties diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java similarity index 98% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java index 5433bbc2e..13adef182 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java +++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java @@ -20,6 +20,7 @@ import org.apache.doris.flink.container.instance.ContainerService; import org.apache.doris.flink.container.instance.DorisContainer; import org.apache.doris.flink.container.instance.DorisCustomerContainer; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,6 +84,7 @@ protected String getDorisInstanceHost() { return dorisContainerService.getInstanceHost(); } + @AfterClass public static void closeContainers() { LOG.info("Starting to close containers."); closeDorisContainer(); diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/ContainerUtils.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/ContainerUtils.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/RecordTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/lookup/RecordTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/RecordTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/lookup/RecordTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestRestService.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/TestRestService.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestRestService.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/TestRestService.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRouting.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/serialization/TestRouting.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRouting.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/serialization/TestRouting.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/OptionUtils.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/OptionUtils.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/OptionUtils.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/OptionUtils.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestUtil.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestUtil.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestUtil.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestUtil.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java similarity index 88% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java index 9704e2504..411c2c7f5 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java +++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java @@ -17,8 +17,6 @@ package org.apache.doris.flink.sink.batch; -import org.apache.flink.api.connector.sink2.Sink; - import org.apache.doris.flink.cfg.DorisExecutionOptions; import org.apache.doris.flink.cfg.DorisOptions; import org.apache.doris.flink.cfg.DorisReadOptions; @@ -33,7 +31,6 @@ import org.mockito.MockedStatic; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; public class TestDorisBatchWriter { @@ -57,8 +54,7 @@ public void testInit() { .build(); thrown.expect(IllegalStateException.class); thrown.expectMessage("tableIdentifier input error"); - Sink.InitContext initContext = mock(Sink.InitContext.class); - DorisBatchWriter batchWriter = new DorisBatchWriter(initContext, null, options, null, null); + DorisBatchWriter batchWriter = new DorisBatchWriter(1, 1, null, options, null, null); } @Test @@ -70,11 +66,10 @@ public void testWriteOneDorisRecord() throws Exception { .build(); DorisReadOptions readOptions = DorisReadOptions.builder().build(); DorisExecutionOptions executionOptions = DorisExecutionOptions.builder().build(); - Sink.InitContext context = mock(Sink.InitContext.class); SimpleStringSerializer simpleStringSerializer = new SimpleStringSerializer(); DorisBatchWriter batchWriter = new DorisBatchWriter( - context, simpleStringSerializer, options, readOptions, executionOptions); + 1, 1, simpleStringSerializer, options, readOptions, executionOptions); batchWriter.writeOneDorisRecord(null); batchWriter.writeOneDorisRecord(DorisRecord.of(null)); batchWriter.writeOneDorisRecord(DorisRecord.of("db", "tbl", "zhangsan,1".getBytes())); diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java similarity index 92% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java index 9e1327be0..e5a2867f0 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java +++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java @@ -18,7 +18,6 @@ package org.apache.doris.flink.sink.copy; import org.apache.flink.api.common.time.Deadline; -import org.apache.flink.api.connector.sink2.Sink; import org.apache.doris.flink.cfg.DorisExecutionOptions; import org.apache.doris.flink.cfg.DorisOptions; @@ -68,11 +67,10 @@ public void testPrepareCommit() throws Exception { CloseableHttpResponse uploadResponse = HttpTestUtil.getResponse("", false, true); CloseableHttpResponse preCommitResponse = HttpTestUtil.getResponse("", true, false); when(httpClient.execute(any())).thenReturn(uploadResponse).thenReturn(preCommitResponse); - Sink.InitContext initContext = mock(Sink.InitContext.class); - // when(initContext.getRestoredCheckpointId()).thenReturn(OptionalLong.of(1)); DorisCopyWriter copyWriter = new DorisCopyWriter( - initContext, + 0, + 0, new SimpleStringSerializer(), dorisOptions, readOptions, @@ -109,11 +107,10 @@ public void testPrepareCommit() throws Exception { @Test public void testSnapshot() throws Exception { - Sink.InitContext initContext = mock(Sink.InitContext.class); - // when(initContext.getRestoredCheckpointId()).thenReturn(OptionalLong.of(1)); DorisCopyWriter copyWriter = new DorisCopyWriter( - initContext, + 1, + 1, new SimpleStringSerializer(), dorisOptions, readOptions, diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java similarity index 99% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java index b1fed97a6..cda8674ad 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java +++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java @@ -57,6 +57,8 @@ public void setUp() throws Exception { dorisOptions = OptionUtils.buildDorisOptions(); readOptions = OptionUtils.buildDorisReadOptions(); executionOptions = OptionUtils.buildExecutionOptional(); + // clear thread interrupted status before each test + Thread.interrupted(); } @Test diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java similarity index 92% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java index c1f59c747..8edb7a589 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java +++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java @@ -17,7 +17,6 @@ package org.apache.doris.flink.sink.writer; -import org.apache.flink.api.connector.sink2.Sink; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Histogram; import org.apache.flink.metrics.groups.SinkWriterMetricGroup; @@ -25,7 +24,6 @@ import org.apache.doris.flink.cfg.DorisExecutionOptions; import org.apache.doris.flink.cfg.DorisOptions; import org.apache.doris.flink.cfg.DorisReadOptions; -import org.apache.doris.flink.exception.DorisRuntimeException; import org.apache.doris.flink.sink.BackendUtil; import org.apache.doris.flink.sink.DorisCommittable; import org.apache.doris.flink.sink.HttpTestUtil; @@ -45,7 +43,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.OptionalLong; import java.util.concurrent.ConcurrentHashMap; import static org.mockito.ArgumentMatchers.any; @@ -68,6 +65,8 @@ public void setUp() { executionOptions = OptionUtils.buildExecutionOptional(); backendUtilMockedStatic = mockStatic(BackendUtil.class); backendUtilMockedStatic.when(() -> BackendUtil.tryHttpConnection(any())).thenReturn(true); + // clear thread interrupted status before each test + Thread.interrupted(); } @Test @@ -94,10 +93,10 @@ public void testPrepareCommitLabelExists() throws Exception { HttpTestUtil.getResponse(HttpTestUtil.LABEL_EXIST_PRE_COMMIT_TABLE_RESPONSE, true); when(httpClient.execute(any(HttpUriRequest.class))).thenReturn(preCommitResponse); DorisWriter dorisWriter = initWriter(httpClient); - dorisWriter.write("doris,1", null); try { + dorisWriter.write("doris,1", null); dorisWriter.prepareCommit(); - } catch (DorisRuntimeException e) { + } catch (RuntimeException e) { Assert.assertTrue(e.getMessage().contains("Exist label abort finished, retry")); } } @@ -110,10 +109,10 @@ public void testPrepareCommitLabelExistsNoPrecommit() throws Exception { HttpTestUtil.LABEL_EXIST_PRE_COMMIT_TABLE_FINISH_RESPONSE, true); when(httpClient.execute(any(HttpUriRequest.class))).thenReturn(preCommitResponse); DorisWriter dorisWriter = initWriter(httpClient); - dorisWriter.write("doris,1", null); try { + dorisWriter.write("doris,1", null); dorisWriter.prepareCommit(); - } catch (DorisRuntimeException e) { + } catch (RuntimeException e) { Assert.assertTrue(e.getMessage().contains("stream load error")); } } @@ -127,10 +126,7 @@ public void testPrepareCommitLabelExistsNo2PC() throws Exception { when(httpClient.execute(any(HttpUriRequest.class))).thenReturn(preCommitResponse); Map dorisStreamLoadMap = new ConcurrentHashMap<>(); Map dorisWriteMetricsMap = new ConcurrentHashMap<>(); - Sink.InitContext initContext = mock(Sink.InitContext.class); SinkWriterMetricGroup sinkWriterMetricGroup = mock(SinkWriterMetricGroup.class); - when(initContext.getRestoredCheckpointId()).thenReturn(OptionalLong.of(1)); - when(initContext.getSubtaskId()).thenReturn(1); DorisWriteMetrics mockWriteMetrics = getMockWriteMetrics(sinkWriterMetricGroup); dorisWriteMetricsMap.put(dorisOptions.getTableIdentifier(), mockWriteMetrics); // skip abort in init @@ -138,7 +134,9 @@ public void testPrepareCommitLabelExistsNo2PC() throws Exception { executionOptions.setEnable2PC(false); DorisWriter dorisWriter = new DorisWriter( - initContext, + 1, + 1, + sinkWriterMetricGroup, Collections.emptyList(), new SimpleStringSerializer(), dorisOptions, @@ -156,10 +154,10 @@ public void testPrepareCommitLabelExistsNo2PC() throws Exception { dorisStreamLoadMap.put(dorisOptions.getTableIdentifier(), dorisStreamLoad); dorisWriter.setDorisStreamLoadMap(dorisStreamLoadMap); dorisWriter.setDorisMetricsMap(dorisWriteMetricsMap); - dorisWriter.write("doris,1", null); try { + dorisWriter.write("doris,1", null); dorisWriter.prepareCommit(); - } catch (DorisRuntimeException e) { + } catch (RuntimeException e) { Assert.assertTrue(e.getMessage().contains("stream load error")); } } @@ -171,10 +169,10 @@ public void testPrepareCommitFail() throws Exception { HttpTestUtil.getResponse(HttpTestUtil.PRE_COMMIT_FAIL_RESPONSE, true); when(httpClient.execute(any())).thenReturn(preCommitResponse); DorisWriter dorisWriter = initWriter(httpClient); - dorisWriter.write("doris,1", null); try { + dorisWriter.write("doris,1", null); dorisWriter.prepareCommit(); - } catch (DorisRuntimeException e) { + } catch (RuntimeException e) { Assert.assertTrue(e.getMessage().contains("stream load error")); } } @@ -182,17 +180,16 @@ public void testPrepareCommitFail() throws Exception { private DorisWriter initWriter(CloseableHttpClient httpClient) throws IOException { Map dorisStreamLoadMap = new ConcurrentHashMap<>(); Map dorisWriteMetricsMap = new ConcurrentHashMap<>(); - Sink.InitContext initContext = mock(Sink.InitContext.class); SinkWriterMetricGroup sinkWriterMetricGroup = mock(SinkWriterMetricGroup.class); - when(initContext.getRestoredCheckpointId()).thenReturn(OptionalLong.of(1)); - when(initContext.getSubtaskId()).thenReturn(1); DorisWriteMetrics mockWriteMetrics = getMockWriteMetrics(sinkWriterMetricGroup); dorisWriteMetricsMap.put(dorisOptions.getTableIdentifier(), mockWriteMetrics); // skip abort in init dorisOptions.setTableIdentifier(""); DorisWriter dorisWriter = new DorisWriter( - initContext, + 1, + 1, + sinkWriterMetricGroup, Collections.emptyList(), new SimpleStringSerializer(), dorisOptions, diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java similarity index 99% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java index ac10053dd..3c68a6820 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java +++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java @@ -56,6 +56,7 @@ public static void setUp() { rowData.setRowKind(RowKind.INSERT); dataTypes = new DataType[] {DataTypes.INT(), DataTypes.STRING(), DataTypes.DOUBLE()}; fieldNames = new String[] {"id", "name", "weight"}; + Thread.interrupted(); } @Test diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java diff --git a/flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql diff --git a/flink-doris-connector/src/test/resources/docker/doris/be.conf b/flink-doris-connector/flink-doris-connector-base/src/test/resources/docker/doris/be.conf similarity index 100% rename from flink-doris-connector/src/test/resources/docker/doris/be.conf rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/docker/doris/be.conf diff --git a/flink-doris-connector/src/test/resources/docker/doris/fe.conf b/flink-doris-connector/flink-doris-connector-base/src/test/resources/docker/doris/fe.conf similarity index 100% rename from flink-doris-connector/src/test/resources/docker/doris/fe.conf rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/docker/doris/fe.conf diff --git a/flink-doris-connector/src/test/resources/log4j2-test.properties b/flink-doris-connector/flink-doris-connector-base/src/test/resources/log4j2-test.properties similarity index 100% rename from flink-doris-connector/src/test/resources/log4j2-test.properties rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/log4j2-test.properties diff --git a/flink-doris-connector/flink-doris-connector-flink1/pom.xml b/flink-doris-connector/flink-doris-connector-flink1/pom.xml new file mode 100644 index 000000000..25795607d --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink1/pom.xml @@ -0,0 +1,275 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector-parent + ${revision} + ../pom.xml + + + flink-doris-connector-${flink.major.version} + Flink Doris Connector (Flink ${flink.version}) + Flink Doris Connector for Flink 1.15 - Flink 1.20 + + + 1.20.0 + 1.20 + 8 + 8 + + + + + org.apache.doris + flink-doris-connector-base + + + + org.apache.doris + flink-doris-connector-base + ${revision} + tests + test + + + org.apache.flink + flink-table-runtime + + + org.apache.flink + flink-connector-base + + + + + org.apache.flink + flink-sql-connector-mysql-cdc + ${flink.sql.cdc.version} + provided + + + flink-shaded-guava + org.apache.flink + + + + + org.apache.flink + flink-sql-connector-oracle-cdc + ${flink.sql.cdc.version} + provided + + + flink-shaded-guava + org.apache.flink + + + + + org.apache.flink + flink-sql-connector-postgres-cdc + ${flink.sql.cdc.version} + provided + + + flink-shaded-guava + org.apache.flink + + + + + org.apache.flink + flink-sql-connector-sqlserver-cdc + ${flink.sql.cdc.version} + provided + + + flink-shaded-guava + org.apache.flink + + + + + org.apache.flink + flink-sql-connector-db2-cdc + ${flink.sql.cdc.version} + provided + + + flink-shaded-guava + org.apache.flink + + + + + org.apache.flink + flink-sql-connector-mongodb-cdc + ${flink.sql.cdc.version} + provided + + + flink-shaded-guava + org.apache.flink + + + + + com.oracle.ojdbc + ojdbc8 + ${ojdbc.version} + provided + + + com.github.jsqlparser + jsqlparser + + + + + org.hamcrest + hamcrest-core + + + org.mockito + mockito-core + + + org.mockito + mockito-inline + + + mysql + mysql-connector-java + + + com.google.protobuf + protobuf-java + + + + + junit + junit + + + org.testcontainers + testcontainers + + + org.testcontainers + mysql + + + org.apache.flink + flink-test-utils + + + org.apache.flink + flink-connector-test-utils + + + org.apache.flink + flink-table-common + ${flink.version} + test-jar + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + ${maven.compiler.source} + ${maven.compiler.target} + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.4.1 + + + + org.apache.arrow + org.apache.doris.shaded.org.apache.arrow + + + io.netty + org.apache.doris.shaded.io.netty + + + com.fasterxml.jackson + org.apache.doris.shaded.com.fasterxml.jackson + + + org.apache.commons.codec + org.apache.doris.shaded.org.apache.commons.codec + + + com.google + org.apache.doris.shaded.com.google + + + org.apache.thrift + org.apache.doris.shaded.org.apache.thrift + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + shade + + package + + + + + + diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java similarity index 98% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java index ca7dc1146..ed40695bc 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java +++ b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java @@ -404,8 +404,10 @@ public List listPartitions(ObjectPath tablePath) @Override public List listPartitions( ObjectPath tablePath, CatalogPartitionSpec partitionSpec) - throws TableNotExistException, TableNotPartitionedException, - PartitionSpecInvalidException, CatalogException { + throws TableNotExistException, + TableNotPartitionedException, + PartitionSpecInvalidException, + CatalogException { return Collections.emptyList(); } @@ -434,8 +436,10 @@ public void createPartition( CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists) - throws TableNotExistException, TableNotPartitionedException, - PartitionSpecInvalidException, PartitionAlreadyExistsException, + throws TableNotExistException, + TableNotPartitionedException, + PartitionSpecInvalidException, + PartitionAlreadyExistsException, CatalogException { throw new UnsupportedOperationException(); } diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/DorisSink.java similarity index 96% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/DorisSink.java index d8e0d8277..f95a00979 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java +++ b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/DorisSink.java @@ -29,13 +29,14 @@ import org.apache.doris.flink.cfg.DorisOptions; import org.apache.doris.flink.cfg.DorisReadOptions; import org.apache.doris.flink.rest.RestService; -import org.apache.doris.flink.sink.batch.DorisBatchWriter; +import org.apache.doris.flink.sink.batch.DorisBatchWriterAdapter; import org.apache.doris.flink.sink.committer.DorisCommitter; import org.apache.doris.flink.sink.copy.CopyCommittableSerializer; import org.apache.doris.flink.sink.copy.DorisCopyCommitter; -import org.apache.doris.flink.sink.copy.DorisCopyWriter; +import org.apache.doris.flink.sink.copy.DorisCopyWriterAdapter; import org.apache.doris.flink.sink.writer.DorisAbstractWriter; import org.apache.doris.flink.sink.writer.DorisWriter; +import org.apache.doris.flink.sink.writer.DorisWriterAdapter; import org.apache.doris.flink.sink.writer.DorisWriterState; import org.apache.doris.flink.sink.writer.DorisWriterStateSerializer; import org.apache.doris.flink.sink.writer.WriteMode; @@ -111,7 +112,7 @@ public DorisAbstractWriter restoreWriter( public DorisAbstractWriter getDorisAbstractWriter( InitContext initContext, Collection states) { if (WriteMode.STREAM_LOAD.equals(dorisExecutionOptions.getWriteMode())) { - return new DorisWriter<>( + return new DorisWriterAdapter<>( initContext, states, serializer, @@ -119,10 +120,10 @@ public DorisAbstractWriter getDorisAbstractWriter( dorisReadOptions, dorisExecutionOptions); } else if (WriteMode.STREAM_LOAD_BATCH.equals(dorisExecutionOptions.getWriteMode())) { - return new DorisBatchWriter<>( + return new DorisBatchWriterAdapter<>( initContext, serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions); } else if (WriteMode.COPY.equals(dorisExecutionOptions.getWriteMode())) { - return new DorisCopyWriter( + return new DorisCopyWriterAdapter<>( initContext, serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions); } throw new IllegalArgumentException( diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java similarity index 97% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java index 993283d89..970466fdf 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java +++ b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java @@ -51,8 +51,8 @@ public DorisBatchSink( @Override public SinkWriter createWriter(InitContext initContext) throws IOException { - DorisBatchWriter dorisBatchWriter = - new DorisBatchWriter( + DorisBatchWriterAdapter dorisBatchWriter = + new DorisBatchWriterAdapter( initContext, serializer, dorisOptions, diff --git a/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java new file mode 100644 index 000000000..2a9bd4baa --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink.batch; + +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.runtime.checkpoint.CheckpointIDCounter; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.sink.DorisCommittable; +import org.apache.doris.flink.sink.writer.DorisAbstractWriter; +import org.apache.doris.flink.sink.writer.DorisWriterState; +import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +/** Flink 1.x specific wrapper for the shared core {@link DorisBatchWriter} implementation. */ +public class DorisBatchWriterAdapter + implements DorisAbstractWriter { + + private final DorisBatchWriter delegate; + + public DorisBatchWriterAdapter( + Sink.InitContext initContext, + DorisRecordSerializer serializer, + DorisOptions dorisOptions, + DorisReadOptions dorisReadOptions, + DorisExecutionOptions executionOptions) { + + long restoreCheckpointId = + initContext + .getRestoredCheckpointId() + .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1); + int subtaskId = initContext.getSubtaskId(); + + this.delegate = + new DorisBatchWriter<>( + restoreCheckpointId, + subtaskId, + serializer, + dorisOptions, + dorisReadOptions, + executionOptions); + } + + @Override + public void write(IN in, Context context) throws IOException, InterruptedException { + delegate.write(in, context); + } + + @Override + public void flush(boolean endOfInput) throws IOException, InterruptedException { + delegate.flush(endOfInput); + } + + @Override + public Collection prepareCommit() throws IOException, InterruptedException { + return delegate.prepareCommit(); + } + + @Override + public List snapshotState(long checkpointId) throws IOException { + return delegate.snapshotState(checkpointId); + } + + @Override + public void close() throws Exception { + delegate.close(); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java new file mode 100644 index 000000000..ba4b7658c --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink.copy; + +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.runtime.checkpoint.CheckpointIDCounter; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.sink.writer.DorisAbstractWriter; +import org.apache.doris.flink.sink.writer.DorisWriterState; +import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +/** Flink 1.x specific wrapper for the shared core {@link DorisCopyWriter} implementation. */ +public class DorisCopyWriterAdapter + implements DorisAbstractWriter { + + private final DorisCopyWriter delegate; + + public DorisCopyWriterAdapter( + Sink.InitContext initContext, + DorisRecordSerializer serializer, + DorisOptions dorisOptions, + DorisReadOptions dorisReadOptions, + DorisExecutionOptions executionOptions) { + + long restoreCheckpointId = + initContext + .getRestoredCheckpointId() + .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1); + int subtaskId = initContext.getSubtaskId(); + + this.delegate = + new DorisCopyWriter<>( + restoreCheckpointId, + subtaskId, + serializer, + dorisOptions, + dorisReadOptions, + executionOptions); + } + + @Override + public void write(IN in, Context context) throws IOException, InterruptedException { + delegate.write(in, context); + } + + @Override + public void flush(boolean endOfInput) throws IOException, InterruptedException { + delegate.flush(endOfInput); + } + + @Override + public Collection prepareCommit() + throws IOException, InterruptedException { + return delegate.prepareCommit(); + } + + @Override + public List snapshotState(long checkpointId) throws IOException { + return delegate.snapshotState(checkpointId); + } + + @Override + public void close() throws Exception { + delegate.close(); + } +} diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java diff --git a/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java new file mode 100644 index 000000000..a79f9999f --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink.writer; + +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.metrics.groups.SinkWriterMetricGroup; +import org.apache.flink.runtime.checkpoint.CheckpointIDCounter; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.sink.DorisCommittable; +import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +/** + * Flink 1.x specific wrapper for the shared core {@link DorisWriter} implementation. + * + *

This class adapts Flink 1.x's {@link Sink.InitContext} to the version-neutral core writer in + * the base module while implementing {@link DorisAbstractWriter} for use by the 1.x connector + * module. + */ +public class DorisWriterAdapter + implements DorisAbstractWriter { + + private final DorisWriter delegate; + + public DorisWriterAdapter( + Sink.InitContext initContext, + Collection state, + DorisRecordSerializer serializer, + DorisOptions dorisOptions, + DorisReadOptions dorisReadOptions, + DorisExecutionOptions executionOptions) { + + long lastCheckpointId = + initContext + .getRestoredCheckpointId() + .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1); + int subtaskId = initContext.getSubtaskId(); + SinkWriterMetricGroup sinkMetricGroup = initContext.metricGroup(); + + this.delegate = + new DorisWriter<>( + lastCheckpointId, + subtaskId, + sinkMetricGroup, + state, + serializer, + dorisOptions, + dorisReadOptions, + executionOptions); + } + + @Override + public void write(IN in, Context context) throws IOException, InterruptedException { + delegate.write(in, context); + } + + @Override + public void flush(boolean endOfInput) throws IOException, InterruptedException { + delegate.flush(endOfInput); + } + + @Override + public Collection prepareCommit() throws IOException, InterruptedException { + return delegate.prepareCommit(); + } + + @Override + public List snapshotState(long checkpointId) throws IOException { + return delegate.snapshotState(checkpointId); + } + + @Override + public void close() throws Exception { + delegate.close(); + } +} diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/utils/DorisTableUtil.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/utils/DorisTableUtil.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/utils/DorisTableUtil.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/utils/DorisTableUtil.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/utils/JsonNodeExtractUtil.java b/flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/utils/JsonNodeExtractUtil.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/utils/JsonNodeExtractUtil.java rename to flink-doris-connector/flink-doris-connector-flink1/src/main/java/org/apache/doris/flink/tools/cdc/utils/JsonNodeExtractUtil.java diff --git a/flink-doris-connector/flink-doris-connector-flink1/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-doris-connector/flink-doris-connector-flink1/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 000000000..5863c8bf2 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink1/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +org.apache.doris.flink.table.DorisDynamicTableFactory +org.apache.doris.flink.catalog.DorisCatalogFactory \ No newline at end of file diff --git a/flink-doris-connector/flink-doris-connector-flink1/src/main/resources/log4j2.properties b/flink-doris-connector/flink-doris-connector-flink1/src/main/resources/log4j2.properties new file mode 100644 index 000000000..591598b95 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink1/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c [%t] %x - %m%n diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java similarity index 91% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java index d21ad7513..5fd4268e2 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java +++ b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java @@ -35,8 +35,10 @@ public class DorisCatalogTest { @Before public void setup() - throws DatabaseAlreadyExistException, TableAlreadyExistException, - TableNotExistException, DatabaseNotExistException { + throws DatabaseAlreadyExistException, + TableAlreadyExistException, + TableNotExistException, + DatabaseNotExistException { DorisConnectionOptions connectionOptions = new DorisConnectionOptions.DorisConnectionOptionsBuilder() .withFenodes("127.0.0.1:8030") diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java similarity index 98% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java index cd3965a30..f0c027a04 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java +++ b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java @@ -117,7 +117,7 @@ protected void cancelE2EJob(String jobName) { private StreamExecutionEnvironment configFlinkEnvironment() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.setParallelism(DEFAULT_PARALLELISM); + env.setParallelism(AbstractContainerTestBase.DEFAULT_PARALLELISM); Map flinkMap = new HashMap<>(); flinkMap.put("execution.checkpointing.interval", "10s"); flinkMap.put("pipeline.operator-chaining", "false"); diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java similarity index 94% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java index 3b38a79b0..fde5a6b1a 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java +++ b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java @@ -22,10 +22,10 @@ import org.apache.doris.flink.cfg.DorisExecutionOptions; import org.apache.doris.flink.cfg.DorisOptions; import org.apache.doris.flink.cfg.DorisReadOptions; -import org.apache.doris.flink.sink.batch.DorisBatchWriter; -import org.apache.doris.flink.sink.copy.DorisCopyWriter; +import org.apache.doris.flink.sink.batch.DorisBatchWriterAdapter; +import org.apache.doris.flink.sink.copy.DorisCopyWriterAdapter; import org.apache.doris.flink.sink.writer.DorisAbstractWriter; -import org.apache.doris.flink.sink.writer.DorisWriter; +import org.apache.doris.flink.sink.writer.DorisWriterAdapter; import org.apache.doris.flink.sink.writer.WriteMode; import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer; import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer; @@ -65,7 +65,7 @@ public void testDorisSink() { dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer); DorisAbstractWriter dorisAbstractWriter = dorisSink.getDorisAbstractWriter(initContext, Collections.emptyList()); - Assert.assertTrue(dorisAbstractWriter instanceof DorisWriter); + Assert.assertTrue(dorisAbstractWriter instanceof DorisWriterAdapter); dorisExecutionOptions = DorisExecutionOptions.builder().setBatchMode(true).disable2PC().build(); @@ -74,7 +74,7 @@ public void testDorisSink() { dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer); dorisAbstractWriter = dorisSink.getDorisAbstractWriter(initContext, Collections.emptyList()); - Assert.assertTrue(dorisAbstractWriter instanceof DorisBatchWriter); + Assert.assertTrue(dorisAbstractWriter instanceof DorisBatchWriterAdapter); dorisExecutionOptions = DorisExecutionOptions.builder().disable2PC().setWriteMode(WriteMode.COPY).build(); @@ -83,7 +83,7 @@ public void testDorisSink() { dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer); dorisAbstractWriter = dorisSink.getDorisAbstractWriter(initContext, Collections.emptyList()); - Assert.assertTrue(dorisAbstractWriter instanceof DorisCopyWriter); + Assert.assertTrue(dorisAbstractWriter instanceof DorisCopyWriterAdapter); } @After diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java similarity index 96% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java index a3c210975..87c2f5205 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java +++ b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java @@ -20,6 +20,7 @@ import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.connector.sink.DynamicTableSink; import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.factories.utils.FactoryMocks; import org.apache.doris.flink.cfg.DorisExecutionOptions; import org.apache.doris.flink.cfg.DorisLookupOptions; @@ -44,8 +45,6 @@ import static org.apache.doris.flink.cfg.ConfigurationOptions.FLIGHT_SQL_PORT_DEFAULT; import static org.apache.doris.flink.cfg.ConfigurationOptions.USE_FLIGHT_SQL_DEFAULT; import static org.apache.doris.flink.utils.FactoryMocks.SCHEMA; -import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSink; -import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSource; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -71,7 +70,7 @@ public void testDorisSourceProperties() { properties.put("lookup.jdbc.read.thread-size", "1"); properties.put("source.use-flight-sql", "true"); properties.put("source.flight-sql-port", "-1"); - DynamicTableSource actual = createTableSource(SCHEMA, properties); + DynamicTableSource actual = FactoryMocks.createTableSource(SCHEMA, properties); DorisOptions options = DorisOptions.builder() .setTableIdentifier("db.tbl") @@ -147,7 +146,7 @@ public void testDorisSinkProperties() { properties.put("sink.ignore.commit-error", "false"); properties.put("sink.parallelism", "1"); - DynamicTableSink actual = createTableSink(SCHEMA, properties); + DynamicTableSink actual = FactoryMocks.createTableSink(SCHEMA, properties); DorisOptions options = DorisOptions.builder() .setTableIdentifier("db.tbl") @@ -238,7 +237,7 @@ public void testDorisSinkProperties() { assertNotEquals(actual, expected4); executionOptions.setEnable2PC(true); - DynamicTableSink actual2 = createTableSink(SCHEMA, new HashMap<>()); + DynamicTableSink actual2 = FactoryMocks.createTableSink(SCHEMA, new HashMap<>()); assertNotEquals(actual2, expected); } diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBCreateTableTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBCreateTableTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBCreateTableTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBCreateTableTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java diff --git a/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java new file mode 100644 index 000000000..431c3d646 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.utils; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.CatalogTableImpl; +import org.apache.flink.table.types.AtomicDataType; +import org.apache.flink.table.types.logical.VarCharType; + +import java.util.Map; + +public class CatalogUtil { + public static CatalogTable createTable(TableSchema tableSchema, Map options) { + return new CatalogTableImpl(tableSchema, options, "FlinkTable"); + } + + public static TableSchema getTableSchema() { + return TableSchema.builder() + .field("id", new AtomicDataType(new VarCharType(false, 128))) + .field("c_boolean", DataTypes.BOOLEAN()) + .field("c_char", DataTypes.CHAR(1)) + .field("c_date", DataTypes.DATE()) + .field("c_datetime", DataTypes.TIMESTAMP(0)) + .field("c_decimal", DataTypes.DECIMAL(10, 2)) + .field("c_double", DataTypes.DOUBLE()) + .field("c_float", DataTypes.FLOAT()) + .field("c_int", DataTypes.INT()) + .field("c_bigint", DataTypes.BIGINT()) + .field("c_largeint", DataTypes.STRING()) + .field("c_smallint", DataTypes.SMALLINT()) + .field("c_string", DataTypes.STRING()) + .field("c_tinyint", DataTypes.TINYINT()) + .field("c_array", DataTypes.ARRAY(DataTypes.INT())) + .field("c_map", DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING())) + .field("c_row", DataTypes.ROW()) + .field("c_varbinary", DataTypes.VARBINARY(16)) + .primaryKey("id") + .build(); + } + + public static String[] getColumns() { + return getTableSchema().getFieldNames(); + } +} diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockSource.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/MockSource.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockSource.java rename to flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/MockSource.java diff --git a/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java new file mode 100644 index 000000000..4b2eb9e30 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink1/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.utils; + +import org.apache.flink.streaming.api.functions.source.SourceFunction; + +public interface MockSourceFunction extends SourceFunction {} diff --git a/flink-doris-connector/flink-doris-connector-flink2/pom.xml b/flink-doris-connector/flink-doris-connector-flink2/pom.xml new file mode 100644 index 000000000..e2183c402 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/pom.xml @@ -0,0 +1,190 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector-parent + ${revision} + ../pom.xml + + + flink-doris-connector-${flink.major.version} + Flink Doris Connector (Flink ${flink.version}) + Flink Doris Connector for Flink 2.0+ + + + 2.2.0 + 2.2 + 17 + 17 + + + + + org.apache.doris + flink-doris-connector-base + + + + org.apache.doris + flink-doris-connector-base + ${revision} + tests + test + + + org.apache.flink + flink-table-runtime + + + org.apache.flink + flink-connector-base + + + + + org.hamcrest + hamcrest-core + + + org.mockito + mockito-core + + + org.mockito + mockito-inline + + + mysql + mysql-connector-java + + + com.google.protobuf + protobuf-java + + + + + junit + junit + + + org.testcontainers + testcontainers + + + org.testcontainers + mysql + + + org.apache.flink + flink-test-utils + + + org.apache.flink + flink-connector-test-utils + + + org.apache.flink + flink-table-common + ${flink.version} + test-jar + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.4.1 + + + + org.apache.arrow + org.apache.doris.shaded.org.apache.arrow + + + io.netty + org.apache.doris.shaded.io.netty + + + com.fasterxml.jackson + org.apache.doris.shaded.com.fasterxml.jackson + + + org.apache.commons.codec + org.apache.doris.shaded.org.apache.commons.codec + + + com.google + org.apache.doris.shaded.com.google + + + org.apache.thrift + org.apache.doris.shaded.org.apache.thrift + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + shade + + package + + + + + + diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java new file mode 100644 index 000000000..bcc619246 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java @@ -0,0 +1,578 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.catalog; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.AbstractCatalog; +import org.apache.flink.table.catalog.CatalogBaseTable; +import org.apache.flink.table.catalog.CatalogDatabase; +import org.apache.flink.table.catalog.CatalogDatabaseImpl; +import org.apache.flink.table.catalog.CatalogFunction; +import org.apache.flink.table.catalog.CatalogPartition; +import org.apache.flink.table.catalog.CatalogPartitionSpec; +import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException; +import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException; +import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException; +import org.apache.flink.table.catalog.exceptions.FunctionAlreadyExistException; +import org.apache.flink.table.catalog.exceptions.FunctionNotExistException; +import org.apache.flink.table.catalog.exceptions.PartitionAlreadyExistsException; +import org.apache.flink.table.catalog.exceptions.PartitionNotExistException; +import org.apache.flink.table.catalog.exceptions.PartitionSpecInvalidException; +import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException; +import org.apache.flink.table.catalog.exceptions.TableNotExistException; +import org.apache.flink.table.catalog.exceptions.TableNotPartitionedException; +import org.apache.flink.table.catalog.exceptions.TablePartitionedException; +import org.apache.flink.table.catalog.stats.CatalogColumnStatistics; +import org.apache.flink.table.catalog.stats.CatalogTableStatistics; +import org.apache.flink.table.expressions.Expression; +import org.apache.flink.table.factories.Factory; +import org.apache.flink.table.types.DataType; +import org.apache.flink.util.Preconditions; +import org.apache.flink.util.StringUtils; + +import org.apache.commons.compress.utils.Lists; +import org.apache.doris.flink.catalog.doris.DorisSchemaFactory; +import org.apache.doris.flink.catalog.doris.DorisSystem; +import org.apache.doris.flink.catalog.doris.FieldSchema; +import org.apache.doris.flink.catalog.doris.TableSchema; +import org.apache.doris.flink.cfg.DorisConnectionOptions; +import org.apache.doris.flink.table.DorisDynamicTableFactory; +import org.apache.doris.flink.tools.cdc.DorisTableConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.StringJoiner; + +import static org.apache.doris.flink.catalog.DorisCatalogOptions.DEFAULT_DATABASE; +import static org.apache.doris.flink.catalog.DorisCatalogOptions.getCreateTableProps; +import static org.apache.doris.flink.table.DorisConfigOptions.FENODES; +import static org.apache.doris.flink.table.DorisConfigOptions.IDENTIFIER; +import static org.apache.doris.flink.table.DorisConfigOptions.PASSWORD; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_LABEL_PREFIX; +import static org.apache.doris.flink.table.DorisConfigOptions.TABLE_IDENTIFIER; +import static org.apache.doris.flink.table.DorisConfigOptions.USERNAME; +import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR; +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** catalog for flink. */ +public class DorisCatalog extends AbstractCatalog { + + private static final Logger LOG = LoggerFactory.getLogger(DorisCatalog.class); + private DorisSystem dorisSystem; + private final DorisConnectionOptions connectionOptions; + private final Map properties; + + public DorisCatalog( + String catalogName, + DorisConnectionOptions connectionOptions, + String defaultDatabase, + Map properties) { + super(catalogName, defaultDatabase); + this.connectionOptions = connectionOptions; + this.properties = Collections.unmodifiableMap(properties); + } + + @Override + public void open() throws CatalogException { + dorisSystem = new DorisSystem(connectionOptions); + } + + @Override + public synchronized void close() throws CatalogException { + try { + LOG.info("Closed catalog {} ", getName()); + } catch (Exception e) { + throw new CatalogException(String.format("Closing catalog %s failed.", getName()), e); + } + } + + @Override + public Optional getFactory() { + return Optional.of(new DorisDynamicTableFactory()); + } + + // ------------- databases ------------- + + @Override + public List listDatabases() throws CatalogException { + return dorisSystem.listDatabases(); + } + + @Override + public CatalogDatabase getDatabase(String databaseName) + throws DatabaseNotExistException, CatalogException { + if (databaseExists(databaseName)) { + return new CatalogDatabaseImpl(Collections.emptyMap(), null); + } else { + throw new DatabaseNotExistException(getName(), databaseName); + } + } + + @Override + public boolean databaseExists(String databaseName) throws CatalogException { + checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName)); + return listDatabases().contains(databaseName); + } + + @Override + public void createDatabase(String name, CatalogDatabase database, boolean ignoreIfExists) + throws DatabaseAlreadyExistException, CatalogException { + if (databaseExists(name)) { + if (ignoreIfExists) { + return; + } + throw new DatabaseAlreadyExistException(getName(), name); + } else { + dorisSystem.createDatabase(name); + } + } + + @Override + public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) + throws DatabaseNotEmptyException, CatalogException, DatabaseNotExistException { + if (!databaseExists(name)) { + if (ignoreIfNotExists) { + return; + } + throw new DatabaseNotExistException(getName(), name); + } + + if (!cascade && !listTables(name).isEmpty()) { + throw new DatabaseNotEmptyException(getName(), name); + } + dorisSystem.dropDatabase(name); + } + + @Override + public void alterDatabase(String name, CatalogDatabase newDatabase, boolean ignoreIfNotExists) + throws DatabaseNotExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + // ------------- tables ------------- + + @Override + public List listTables(String databaseName) + throws DatabaseNotExistException, CatalogException { + Preconditions.checkState( + org.apache.commons.lang3.StringUtils.isNotBlank(databaseName), + "Database name must not be blank."); + if (!databaseExists(databaseName)) { + throw new DatabaseNotExistException(getName(), databaseName); + } + + return dorisSystem.extractColumnValuesBySQL( + "SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = ?", + 1, + null, + databaseName); + } + + @Override + public List listViews(String databaseName) + throws DatabaseNotExistException, CatalogException { + return Collections.emptyList(); + } + + @Override + public CatalogBaseTable getTable(ObjectPath tablePath) + throws TableNotExistException, CatalogException { + if (!tableExists(tablePath)) { + throw new TableNotExistException(getName(), tablePath); + } + String databaseName = tablePath.getDatabaseName(); + String tableName = tablePath.getObjectName(); + Map props = new HashMap<>(properties); + props.put(CONNECTOR.key(), IDENTIFIER); + if (!props.containsKey(FENODES.key())) { + props.put(FENODES.key(), queryFenodes()); + } + props.put(USERNAME.key(), connectionOptions.getUsername()); + props.put(PASSWORD.key(), connectionOptions.getPassword()); + props.put(TABLE_IDENTIFIER.key(), databaseName + "." + tableName); + + String labelPrefix = props.getOrDefault(SINK_LABEL_PREFIX.key(), ""); + props.put(SINK_LABEL_PREFIX.key(), String.join("_", labelPrefix, databaseName, tableName)); + // remove catalog option + props.remove(DEFAULT_DATABASE.key()); + return CatalogTable.newBuilder() + .schema(createTableSchema(databaseName, tableName)) + .comment(null) + .partitionKeys(Lists.newArrayList()) + .options(props) + .build(); + } + + @VisibleForTesting + protected String queryFenodes() { + try (Connection conn = + DriverManager.getConnection( + connectionOptions.getJdbcUrl(), + connectionOptions.getUsername(), + connectionOptions.getPassword())) { + StringJoiner fenodes = new StringJoiner(","); + PreparedStatement ps = conn.prepareStatement("SHOW FRONTENDS"); + ResultSet resultSet = ps.executeQuery(); + + // find target ip column name, Version 1.2 is IP, version 2.x is Host + String field = ""; + ResultSetMetaData metaData = resultSet.getMetaData(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnName = metaData.getColumnName(i); + if (columnName.equalsIgnoreCase("IP") || columnName.equalsIgnoreCase("Host")) { + field = columnName; + break; + } + } + + while (resultSet.next()) { + String ip = resultSet.getString(field); + String port = resultSet.getString("HttpPort"); + fenodes.add(ip + ":" + port); + } + return fenodes.toString(); + } catch (Exception e) { + throw new CatalogException("Failed getting fenodes", e); + } + } + + private Schema createTableSchema(String databaseName, String tableName) { + try (Connection conn = + DriverManager.getConnection( + connectionOptions.getJdbcUrl(), + connectionOptions.getUsername(), + connectionOptions.getPassword())) { + PreparedStatement ps = + conn.prepareStatement( + String.format( + "SELECT COLUMN_NAME,DATA_TYPE,COLUMN_SIZE,DECIMAL_DIGITS FROM `information_schema`.`COLUMNS` WHERE `TABLE_SCHEMA`= '%s' AND `TABLE_NAME`= '%s'", + databaseName, tableName)); + + List columnNames = new ArrayList<>(); + List columnTypes = new ArrayList<>(); + ResultSet resultSet = ps.executeQuery(); + while (resultSet.next()) { + String columnName = resultSet.getString("COLUMN_NAME"); + String columnType = resultSet.getString("DATA_TYPE"); + long columnSize = resultSet.getLong("COLUMN_SIZE"); + long columnDigit = resultSet.getLong("DECIMAL_DIGITS"); + DataType flinkType = + DorisTypeMapper.toFlinkType( + columnName, columnType, (int) columnSize, (int) columnDigit); + columnNames.add(columnName); + columnTypes.add(flinkType); + } + Schema.Builder schemaBuilder = Schema.newBuilder().fromFields(columnNames, columnTypes); + Schema tableSchema = schemaBuilder.build(); + return tableSchema; + } catch (Exception e) { + throw new CatalogException( + String.format( + "Failed getting catalog %s database %s table %s", + getName(), databaseName, tableName), + e); + } + } + + @Override + public boolean tableExists(ObjectPath tablePath) throws CatalogException { + try { + return databaseExists(tablePath.getDatabaseName()) + && listTables(tablePath.getDatabaseName()).contains(tablePath.getObjectName()); + } catch (DatabaseNotExistException e) { + return false; + } + } + + @Override + public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) + throws TableNotExistException, CatalogException { + if (!tableExists(tablePath)) { + if (ignoreIfNotExists) { + return; + } + throw new TableNotExistException(getName(), tablePath); + } + + dorisSystem.dropTable( + String.format("%s.%s", tablePath.getDatabaseName(), tablePath.getObjectName())); + } + + @Override + public void renameTable(ObjectPath tablePath, String newTableName, boolean ignoreIfNotExists) + throws TableNotExistException, TableAlreadyExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists) + throws TableAlreadyExistException, DatabaseNotExistException, CatalogException { + checkNotNull(tablePath, "tablePath cannot be null"); + checkNotNull(table, "table cannot be null"); + + if (!databaseExists(tablePath.getDatabaseName())) { + throw new DatabaseNotExistException(getName(), tablePath.getDatabaseName()); + } + if (tableExists(tablePath)) { + if (ignoreIfExists) { + return; + } + throw new TableAlreadyExistException(getName(), tablePath); + } + + Map options = table.getOptions(); + if (!IDENTIFIER.equals(options.get(CONNECTOR.key()))) { + return; + } + + List primaryKeys = getCreateDorisKeys(table.getUnresolvedSchema()); + TableSchema schema = + DorisSchemaFactory.createTableSchema( + tablePath.getDatabaseName(), + tablePath.getObjectName(), + getCreateDorisColumns(table.getUnresolvedSchema()), + primaryKeys, + new DorisTableConfig(getCreateTableProps(options)), + table.getComment()); + + dorisSystem.createTable(schema); + } + + public List getCreateDorisKeys(Schema schema) { + Preconditions.checkState(schema.getPrimaryKey().isPresent(), "primary key cannot be null"); + return schema.getPrimaryKey().get().getColumnNames(); + } + + public Map getCreateDorisColumns(Schema schema) { + ((Schema.UnresolvedPhysicalColumn) schema.getColumns().get(0)).getDataType(); + // Flink 2.x Schema cannot directly obtain the datatype. + Map fields = new LinkedHashMap<>(); + for (Schema.UnresolvedColumn col : schema.getColumns()) { + Schema.UnresolvedPhysicalColumn physCol = (Schema.UnresolvedPhysicalColumn) col; + String name = physCol.getName(); + DataType dataType = (DataType) physCol.getDataType(); + fields.put(name, new FieldSchema(name, DorisTypeMapper.toDorisType(dataType), null)); + } + return fields; + } + + @Override + public void alterTable( + ObjectPath tablePath, CatalogBaseTable newTable, boolean ignoreIfNotExists) + throws TableNotExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + // ------------- partitions ------------- + + @Override + public List listPartitions(ObjectPath tablePath) + throws TableNotExistException, TableNotPartitionedException, CatalogException { + return Collections.emptyList(); + } + + @Override + public List listPartitions( + ObjectPath tablePath, CatalogPartitionSpec partitionSpec) + throws TableNotExistException, + TableNotPartitionedException, + PartitionSpecInvalidException, + CatalogException { + return Collections.emptyList(); + } + + @Override + public List listPartitionsByFilter( + ObjectPath tablePath, List filters) + throws TableNotExistException, TableNotPartitionedException, CatalogException { + return Collections.emptyList(); + } + + @Override + public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) + throws PartitionNotExistException, CatalogException { + throw new PartitionNotExistException(getName(), tablePath, partitionSpec); + } + + @Override + public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) + throws CatalogException { + return false; + } + + @Override + public void createPartition( + ObjectPath tablePath, + CatalogPartitionSpec partitionSpec, + CatalogPartition partition, + boolean ignoreIfExists) + throws TableNotExistException, + TableNotPartitionedException, + PartitionSpecInvalidException, + PartitionAlreadyExistsException, + CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public void dropPartition( + ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists) + throws PartitionNotExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public void alterPartition( + ObjectPath tablePath, + CatalogPartitionSpec partitionSpec, + CatalogPartition newPartition, + boolean ignoreIfNotExists) + throws PartitionNotExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + // ------------- functions ------------- + + @Override + public List listFunctions(String dbName) + throws DatabaseNotExistException, CatalogException { + return Collections.emptyList(); + } + + @Override + public CatalogFunction getFunction(ObjectPath functionPath) + throws FunctionNotExistException, CatalogException { + throw new FunctionNotExistException(getName(), functionPath); + } + + @Override + public boolean functionExists(ObjectPath functionPath) throws CatalogException { + return false; + } + + @Override + public void createFunction( + ObjectPath functionPath, CatalogFunction function, boolean ignoreIfExists) + throws FunctionAlreadyExistException, DatabaseNotExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public void alterFunction( + ObjectPath functionPath, CatalogFunction newFunction, boolean ignoreIfNotExists) + throws FunctionNotExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists) + throws FunctionNotExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + // ------------- statistics ------------- + + @Override + public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) + throws TableNotExistException, CatalogException { + return CatalogTableStatistics.UNKNOWN; + } + + @Override + public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) + throws TableNotExistException, CatalogException { + return CatalogColumnStatistics.UNKNOWN; + } + + @Override + public CatalogTableStatistics getPartitionStatistics( + ObjectPath tablePath, CatalogPartitionSpec partitionSpec) + throws PartitionNotExistException, CatalogException { + return CatalogTableStatistics.UNKNOWN; + } + + @Override + public CatalogColumnStatistics getPartitionColumnStatistics( + ObjectPath tablePath, CatalogPartitionSpec partitionSpec) + throws PartitionNotExistException, CatalogException { + return CatalogColumnStatistics.UNKNOWN; + } + + @Override + public void alterTableStatistics( + ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists) + throws TableNotExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public void alterTableColumnStatistics( + ObjectPath tablePath, + CatalogColumnStatistics columnStatistics, + boolean ignoreIfNotExists) + throws TableNotExistException, CatalogException, TablePartitionedException { + throw new UnsupportedOperationException(); + } + + @Override + public void alterPartitionStatistics( + ObjectPath tablePath, + CatalogPartitionSpec partitionSpec, + CatalogTableStatistics partitionStatistics, + boolean ignoreIfNotExists) + throws PartitionNotExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public void alterPartitionColumnStatistics( + ObjectPath tablePath, + CatalogPartitionSpec partitionSpec, + CatalogColumnStatistics columnStatistics, + boolean ignoreIfNotExists) + throws PartitionNotExistException, CatalogException { + throw new UnsupportedOperationException(); + } + + @VisibleForTesting + public DorisConnectionOptions getConnectionOptions() { + return connectionOptions; + } + + @VisibleForTesting + public Map getProperties() { + return properties; + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java new file mode 100644 index 000000000..30fc8e5c3 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.catalog; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.table.factories.CatalogFactory; +import org.apache.flink.table.factories.FactoryUtil; + +import org.apache.doris.flink.cfg.DorisConnectionOptions; + +import java.util.HashSet; +import java.util.Set; + +import static org.apache.doris.flink.catalog.DorisCatalogOptions.DEFAULT_DATABASE; +import static org.apache.doris.flink.catalog.DorisCatalogOptions.TABLE_PROPERTIES_PREFIX; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_BATCH_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_DESERIALIZE_ARROW_ASYNC; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_DESERIALIZE_QUEUE_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_EXEC_MEM_LIMIT; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_CONNECT_TIMEOUT_MS; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_QUERY_TIMEOUT_S; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_READ_TIMEOUT_MS; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_RETRIES; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_TABLET_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.FENODES; +import static org.apache.doris.flink.table.DorisConfigOptions.IDENTIFIER; +import static org.apache.doris.flink.table.DorisConfigOptions.JDBC_URL; +import static org.apache.doris.flink.table.DorisConfigOptions.PASSWORD; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_COUNT; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_CHECK_INTERVAL; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_ENABLE_2PC; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_ENABLE_DELETE; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_LABEL_PREFIX; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_MAX_RETRIES; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_PARALLELISM; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_USE_CACHE; +import static org.apache.doris.flink.table.DorisConfigOptions.SOURCE_USE_OLD_API; +import static org.apache.doris.flink.table.DorisConfigOptions.STREAM_LOAD_PROP_PREFIX; +import static org.apache.doris.flink.table.DorisConfigOptions.TABLE_IDENTIFIER; +import static org.apache.doris.flink.table.DorisConfigOptions.USERNAME; + +/** Factory for {@link DorisCatalog}. */ +public class DorisCatalogFactory implements CatalogFactory { + + @Override + public String factoryIdentifier() { + return IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + final Set> options = new HashSet<>(); + options.add(JDBC_URL); + options.add(USERNAME); + options.add(PASSWORD); + return options; + } + + @Override + public Set> optionalOptions() { + final Set> options = new HashSet<>(); + options.add(JDBC_URL); + options.add(DEFAULT_DATABASE); + + options.add(FENODES); + options.add(TABLE_IDENTIFIER); + options.add(USERNAME); + options.add(PASSWORD); + + options.add(DORIS_TABLET_SIZE); + options.add(DORIS_REQUEST_CONNECT_TIMEOUT_MS); + options.add(DORIS_REQUEST_READ_TIMEOUT_MS); + options.add(DORIS_REQUEST_QUERY_TIMEOUT_S); + options.add(DORIS_REQUEST_RETRIES); + options.add(DORIS_DESERIALIZE_ARROW_ASYNC); + options.add(DORIS_DESERIALIZE_QUEUE_SIZE); + options.add(DORIS_BATCH_SIZE); + options.add(DORIS_EXEC_MEM_LIMIT); + + options.add(SINK_CHECK_INTERVAL); + options.add(SINK_ENABLE_2PC); + options.add(SINK_MAX_RETRIES); + options.add(SINK_ENABLE_DELETE); + options.add(SINK_LABEL_PREFIX); + options.add(SINK_BUFFER_SIZE); + options.add(SINK_BUFFER_COUNT); + options.add(SINK_PARALLELISM); + options.add(SINK_USE_CACHE); + + options.add(SOURCE_USE_OLD_API); + return options; + } + + @Override + public Catalog createCatalog(Context context) { + final FactoryUtil.CatalogFactoryHelper helper = + FactoryUtil.createCatalogFactoryHelper(this, context); + helper.validateExcept(STREAM_LOAD_PROP_PREFIX, TABLE_PROPERTIES_PREFIX); + + DorisConnectionOptions connectionOptions = + new DorisConnectionOptions.DorisConnectionOptionsBuilder() + .withFenodes(helper.getOptions().get(FENODES)) + .withJdbcUrl(helper.getOptions().get(JDBC_URL)) + .withUsername(helper.getOptions().get(USERNAME)) + .withPassword(helper.getOptions().get(PASSWORD)) + .build(); + return new DorisCatalog( + context.getName(), + connectionOptions, + helper.getOptions().get(DEFAULT_DATABASE), + ((Configuration) helper.getOptions()).toMap()); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java new file mode 100644 index 000000000..3229b1d8f --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.datastream; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.functions.OpenContext; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.streaming.api.functions.source.legacy.RichParallelSourceFunction; + +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.cfg.DorisStreamOptions; +import org.apache.doris.flink.deserialization.DorisDeserializationSchema; +import org.apache.doris.flink.exception.DorisException; +import org.apache.doris.flink.rest.PartitionDefinition; +import org.apache.doris.flink.rest.RestService; +import org.apache.doris.flink.source.reader.DorisValueReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** DorisSource. */ +@Deprecated +@PublicEvolving +public class DorisSourceFunction extends RichParallelSourceFunction> + implements ResultTypeQueryable> { + + private static final Logger logger = LoggerFactory.getLogger(DorisSourceFunction.class); + + private final DorisDeserializationSchema> deserializer; + private final DorisOptions options; + private final DorisReadOptions readOptions; + private transient volatile boolean isRunning; + private List dorisPartitions; + private List taskDorisPartitions = new ArrayList<>(); + + public DorisSourceFunction( + DorisStreamOptions streamOptions, DorisDeserializationSchema> deserializer) { + this.deserializer = deserializer; + this.options = streamOptions.getOptions(); + this.readOptions = streamOptions.getReadOptions(); + try { + this.dorisPartitions = RestService.findPartitions(options, readOptions, logger); + logger.info("Doris partitions size {}", dorisPartitions.size()); + } catch (DorisException e) { + throw new RuntimeException("Failed fetch doris partitions"); + } + } + + @Override + public void open(OpenContext parameters) throws Exception { + super.open(parameters); + this.isRunning = true; + assignTaskPartitions(); + } + + /** Assign partitions to each task. */ + private void assignTaskPartitions() { + int taskIndex = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); + int totalTasks = getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks(); + + for (int i = 0; i < dorisPartitions.size(); i++) { + if (i % totalTasks == taskIndex) { + taskDorisPartitions.add(dorisPartitions.get(i)); + } + } + logger.info("subtask {} process {} partitions ", taskIndex, taskDorisPartitions.size()); + } + + @Override + public void run(SourceContext> sourceContext) { + for (PartitionDefinition partitions : taskDorisPartitions) { + try (DorisValueReader valueReader = + new DorisValueReader(partitions, options, readOptions)) { + while (isRunning && valueReader.hasNext()) { + List next = valueReader.next(); + sourceContext.collect(next); + } + } catch (Exception e) { + logger.error("close reader resource failed,", e); + } + } + } + + @Override + public void cancel() { + isRunning = false; + } + + @Override + public void close() throws Exception { + super.close(); + isRunning = false; + } + + @Override + public TypeInformation> getProducedType() { + return this.deserializer.getProducedType(); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/DorisSink.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/DorisSink.java new file mode 100644 index 000000000..fe1e920ff --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/DorisSink.java @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.connector.sink2.Committer; +import org.apache.flink.api.connector.sink2.CommitterInitContext; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SupportsCommitter; +import org.apache.flink.api.connector.sink2.SupportsWriterState; +import org.apache.flink.api.connector.sink2.WriterInitContext; +import org.apache.flink.core.io.SimpleVersionedSerializer; +import org.apache.flink.util.Preconditions; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.rest.RestService; +import org.apache.doris.flink.sink.batch.DorisBatchWriterAdapter; +import org.apache.doris.flink.sink.committer.DorisCommitter; +import org.apache.doris.flink.sink.copy.CopyCommittableSerializer; +import org.apache.doris.flink.sink.copy.DorisCopyCommitter; +import org.apache.doris.flink.sink.copy.DorisCopyWriterAdapter; +import org.apache.doris.flink.sink.writer.DorisAbstractWriter; +import org.apache.doris.flink.sink.writer.DorisWriter; +import org.apache.doris.flink.sink.writer.DorisWriterAdapter; +import org.apache.doris.flink.sink.writer.DorisWriterState; +import org.apache.doris.flink.sink.writer.DorisWriterStateSerializer; +import org.apache.doris.flink.sink.writer.WriteMode; +import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +/** + * Load data into Doris based on 2PC. see {@link DorisWriter} and {@link DorisCommitter}. + * + * @param type of record. + */ +@PublicEvolving +public class DorisSink + implements Sink, + SupportsWriterState, + SupportsCommitter { + private static final Logger LOG = LoggerFactory.getLogger(DorisSink.class); + private final DorisOptions dorisOptions; + private final DorisReadOptions dorisReadOptions; + private final DorisExecutionOptions dorisExecutionOptions; + private final DorisRecordSerializer serializer; + + public DorisSink( + DorisOptions dorisOptions, + DorisReadOptions dorisReadOptions, + DorisExecutionOptions dorisExecutionOptions, + DorisRecordSerializer serializer) { + this.dorisOptions = dorisOptions; + this.dorisReadOptions = dorisReadOptions; + this.dorisExecutionOptions = dorisExecutionOptions; + this.serializer = serializer; + checkKeyType(); + } + + /** The uniq model has 2pc close by default unless 2pc is forced open. */ + private void checkKeyType() { + if (dorisExecutionOptions.enabled2PC() + && !dorisExecutionOptions.force2PC() + && RestService.isUniqueKeyType(dorisOptions, dorisReadOptions, LOG)) { + dorisExecutionOptions.setEnable2PC(false); + } + } + + @Override + public DorisAbstractWriter createWriter(WriterInitContext initContext) throws IOException { + return getDorisAbstractWriter(initContext, Collections.emptyList()); + } + + @Override + public Committer createCommitter(CommitterInitContext committerInitContext) throws IOException { + if (WriteMode.STREAM_LOAD.equals(dorisExecutionOptions.getWriteMode()) + || WriteMode.STREAM_LOAD_BATCH.equals(dorisExecutionOptions.getWriteMode())) { + return new DorisCommitter(dorisOptions, dorisReadOptions, dorisExecutionOptions); + } else if (WriteMode.COPY.equals(dorisExecutionOptions.getWriteMode())) { + return new DorisCopyCommitter(dorisOptions, dorisExecutionOptions.getMaxRetries()); + } + throw new IllegalArgumentException( + "Unsupported write mode " + dorisExecutionOptions.getWriteMode()); + } + + @Override + public DorisAbstractWriter restoreWriter( + WriterInitContext initContext, Collection recoveredState) + throws IOException { + return getDorisAbstractWriter(initContext, recoveredState); + } + + @VisibleForTesting + public DorisAbstractWriter getDorisAbstractWriter( + WriterInitContext initContext, Collection states) { + if (WriteMode.STREAM_LOAD.equals(dorisExecutionOptions.getWriteMode())) { + return new DorisWriterAdapter<>( + initContext, + states, + serializer, + dorisOptions, + dorisReadOptions, + dorisExecutionOptions); + } else if (WriteMode.STREAM_LOAD_BATCH.equals(dorisExecutionOptions.getWriteMode())) { + return new DorisBatchWriterAdapter<>( + initContext, serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions); + } else if (WriteMode.COPY.equals(dorisExecutionOptions.getWriteMode())) { + return new DorisCopyWriterAdapter<>( + initContext, serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions); + } + throw new IllegalArgumentException( + "Unsupported write mode " + dorisExecutionOptions.getWriteMode()); + } + + @Override + public SimpleVersionedSerializer getWriterStateSerializer() { + return new DorisWriterStateSerializer(); + } + + @Override + public SimpleVersionedSerializer getCommittableSerializer() { + if (WriteMode.STREAM_LOAD.equals(dorisExecutionOptions.getWriteMode()) + || WriteMode.STREAM_LOAD_BATCH.equals(dorisExecutionOptions.getWriteMode())) { + return new DorisCommittableSerializer(); + } else if (WriteMode.COPY.equals(dorisExecutionOptions.getWriteMode())) { + return new CopyCommittableSerializer(); + } + throw new IllegalArgumentException( + "Unsupported write mode " + dorisExecutionOptions.getWriteMode()); + } + + public static Builder builder() { + return new Builder<>(); + } + + /** + * build for DorisSink. + * + * @param record type. + */ + public static class Builder { + private DorisOptions dorisOptions; + private DorisReadOptions dorisReadOptions; + private DorisExecutionOptions dorisExecutionOptions; + private DorisRecordSerializer serializer; + + /** + * Sets the DorisOptions for the DorisSink. + * + * @param dorisOptions the common options of the doris cluster. + * @return this DorisSink.Builder. + */ + public Builder setDorisOptions(DorisOptions dorisOptions) { + this.dorisOptions = dorisOptions; + return this; + } + + /** + * Sets the DorisReadOptions for the DorisSink. + * + * @param dorisReadOptions the read options of the DorisSink. + * @return this DorisSink.Builder. + */ + public Builder setDorisReadOptions(DorisReadOptions dorisReadOptions) { + this.dorisReadOptions = dorisReadOptions; + return this; + } + + /** + * Sets the DorisExecutionOptions for the DorisSink. + * + * @param dorisExecutionOptions the execution options of the DorisSink. + * @return this DorisSink.Builder. + */ + public Builder setDorisExecutionOptions(DorisExecutionOptions dorisExecutionOptions) { + this.dorisExecutionOptions = dorisExecutionOptions; + return this; + } + + /** + * Sets the {@link DorisRecordSerializer serializer} that transforms incoming records to + * DorisRecord + * + * @param serializer + * @return this DorisSink.Builder. + */ + public Builder setSerializer(DorisRecordSerializer serializer) { + this.serializer = serializer; + return this; + } + + /** + * Build the {@link DorisSink}. + * + * @return a DorisSink with the settings made for this builder. + */ + public DorisSink build() { + Preconditions.checkNotNull(dorisOptions); + Preconditions.checkNotNull(dorisExecutionOptions); + Preconditions.checkNotNull(serializer); + if (dorisReadOptions == null) { + dorisReadOptions = DorisReadOptions.builder().build(); + } + return new DorisSink<>( + dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer); + } + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java new file mode 100644 index 000000000..de287a572 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink.batch; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; +import org.apache.flink.api.connector.sink2.WriterInitContext; +import org.apache.flink.util.Preconditions; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer; + +import java.io.IOException; + +@Deprecated +@PublicEvolving +public class DorisBatchSink implements Sink { + private final DorisOptions dorisOptions; + private final DorisReadOptions dorisReadOptions; + private final DorisExecutionOptions dorisExecutionOptions; + private final DorisRecordSerializer serializer; + + public DorisBatchSink( + DorisOptions dorisOptions, + DorisReadOptions dorisReadOptions, + DorisExecutionOptions dorisExecutionOptions, + DorisRecordSerializer serializer) { + this.dorisOptions = dorisOptions; + this.dorisReadOptions = dorisReadOptions; + this.dorisExecutionOptions = dorisExecutionOptions; + this.serializer = serializer; + } + + @Override + public SinkWriter createWriter(WriterInitContext initContext) throws IOException { + DorisBatchWriterAdapter dorisBatchWriter = + new DorisBatchWriterAdapter( + initContext, + serializer, + dorisOptions, + dorisReadOptions, + dorisExecutionOptions); + return dorisBatchWriter; + } + + public static Builder builder() { + return new Builder<>(); + } + + /** + * build for DorisBatchSink. + * + * @param record type. + */ + public static class Builder { + private DorisOptions dorisOptions; + private DorisReadOptions dorisReadOptions; + private DorisExecutionOptions dorisExecutionOptions; + private DorisRecordSerializer serializer; + + public Builder setDorisOptions(DorisOptions dorisOptions) { + this.dorisOptions = dorisOptions; + return this; + } + + public Builder setDorisReadOptions(DorisReadOptions dorisReadOptions) { + this.dorisReadOptions = dorisReadOptions; + return this; + } + + public Builder setDorisExecutionOptions(DorisExecutionOptions dorisExecutionOptions) { + this.dorisExecutionOptions = dorisExecutionOptions; + return this; + } + + public Builder setSerializer(DorisRecordSerializer serializer) { + this.serializer = serializer; + return this; + } + + public DorisBatchSink build() { + Preconditions.checkNotNull(dorisOptions); + Preconditions.checkNotNull(dorisExecutionOptions); + Preconditions.checkNotNull(serializer); + if (dorisReadOptions == null) { + dorisReadOptions = DorisReadOptions.builder().build(); + } + return new DorisBatchSink<>( + dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer); + } + } + + @VisibleForTesting + public DorisReadOptions getDorisReadOptions() { + return dorisReadOptions; + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java new file mode 100644 index 000000000..7ad85b19d --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink.batch; + +import org.apache.flink.api.connector.sink2.WriterInitContext; +import org.apache.flink.runtime.checkpoint.CheckpointIDCounter; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.sink.DorisCommittable; +import org.apache.doris.flink.sink.writer.DorisAbstractWriter; +import org.apache.doris.flink.sink.writer.DorisWriterState; +import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +/** Flink 2.x specific wrapper for the shared core {@link DorisBatchWriter} implementation. */ +public class DorisBatchWriterAdapter + implements DorisAbstractWriter { + + private final DorisBatchWriter delegate; + + public DorisBatchWriterAdapter( + WriterInitContext initContext, + DorisRecordSerializer serializer, + DorisOptions dorisOptions, + DorisReadOptions dorisReadOptions, + DorisExecutionOptions executionOptions) { + + long restoreCheckpointId = + initContext + .getRestoredCheckpointId() + .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1); + int subtaskId = initContext.getTaskInfo().getIndexOfThisSubtask(); + + this.delegate = + new DorisBatchWriter<>( + restoreCheckpointId, + subtaskId, + serializer, + dorisOptions, + dorisReadOptions, + executionOptions); + } + + @Override + public void write(IN in, Context context) throws IOException, InterruptedException { + delegate.write(in, context); + } + + @Override + public void flush(boolean endOfInput) throws IOException, InterruptedException { + delegate.flush(endOfInput); + } + + @Override + public Collection prepareCommit() throws IOException, InterruptedException { + return delegate.prepareCommit(); + } + + @Override + public List snapshotState(long checkpointId) throws IOException { + return delegate.snapshotState(checkpointId); + } + + @Override + public void close() throws Exception { + delegate.close(); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java new file mode 100644 index 000000000..3f6823322 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink.copy; + +import org.apache.flink.api.connector.sink2.WriterInitContext; +import org.apache.flink.runtime.checkpoint.CheckpointIDCounter; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.sink.writer.DorisAbstractWriter; +import org.apache.doris.flink.sink.writer.DorisWriterState; +import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +/** Flink 2.x specific wrapper for the shared core {@link DorisCopyWriter} implementation. */ +public class DorisCopyWriterAdapter + implements DorisAbstractWriter { + + private final DorisCopyWriter delegate; + + public DorisCopyWriterAdapter( + WriterInitContext initContext, + DorisRecordSerializer serializer, + DorisOptions dorisOptions, + DorisReadOptions dorisReadOptions, + DorisExecutionOptions executionOptions) { + + long restoreCheckpointId = + initContext + .getRestoredCheckpointId() + .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1); + int subtaskId = initContext.getTaskInfo().getIndexOfThisSubtask(); + + this.delegate = + new DorisCopyWriter<>( + restoreCheckpointId, + subtaskId, + serializer, + dorisOptions, + dorisReadOptions, + executionOptions); + } + + @Override + public void write(IN in, Context context) throws IOException, InterruptedException { + delegate.write(in, context); + } + + @Override + public void flush(boolean endOfInput) throws IOException, InterruptedException { + delegate.flush(endOfInput); + } + + @Override + public Collection prepareCommit() + throws IOException, InterruptedException { + return delegate.prepareCommit(); + } + + @Override + public List snapshotState(long checkpointId) throws IOException { + return delegate.snapshotState(checkpointId); + } + + @Override + public void close() throws Exception { + delegate.close(); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java new file mode 100644 index 000000000..8e8941b6e --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink.writer; + +import org.apache.flink.api.connector.sink2.CommittingSinkWriter; +import org.apache.flink.api.connector.sink2.StatefulSinkWriter; + +/** Abstract for different Doris Writer: Stream Load, Copy ... */ +public interface DorisAbstractWriter + extends StatefulSinkWriter, CommittingSinkWriter {} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java new file mode 100644 index 000000000..e9635992a --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink.writer; + +import org.apache.flink.api.connector.sink2.WriterInitContext; +import org.apache.flink.metrics.groups.SinkWriterMetricGroup; +import org.apache.flink.runtime.checkpoint.CheckpointIDCounter; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.sink.DorisCommittable; +import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +/** + * Flink 2.x specific wrapper for the shared core {@link DorisWriter} implementation. + * + *

This class adapts Flink 2.x's {@link WriterInitContext} to the version-neutral core writer in + * the base module while implementing {@link DorisAbstractWriter} for use by the 2.x connector + * module. + */ +public class DorisWriterAdapter + implements DorisAbstractWriter { + + private final DorisWriter delegate; + + public DorisWriterAdapter( + WriterInitContext initContext, + Collection state, + DorisRecordSerializer serializer, + DorisOptions dorisOptions, + DorisReadOptions dorisReadOptions, + DorisExecutionOptions executionOptions) { + + long lastCheckpointId = + initContext + .getRestoredCheckpointId() + .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1); + int subtaskId = initContext.getTaskInfo().getIndexOfThisSubtask(); + SinkWriterMetricGroup sinkMetricGroup = initContext.metricGroup(); + + this.delegate = + new DorisWriter<>( + lastCheckpointId, + subtaskId, + sinkMetricGroup, + state, + serializer, + dorisOptions, + dorisReadOptions, + executionOptions); + } + + @Override + public void write(IN in, Context context) throws IOException, InterruptedException { + delegate.write(in, context); + } + + @Override + public void flush(boolean endOfInput) throws IOException, InterruptedException { + delegate.flush(endOfInput); + } + + @Override + public Collection prepareCommit() throws IOException, InterruptedException { + return delegate.prepareCommit(); + } + + @Override + public List snapshotState(long checkpointId) throws IOException { + return delegate.snapshotState(checkpointId); + } + + @Override + public void close() throws Exception { + delegate.close(); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupAdapter.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupAdapter.java new file mode 100644 index 000000000..2c1354448 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupAdapter.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.doris.flink.table; + +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.functions.AsyncLookupFunction; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.types.DataType; + +import org.apache.doris.flink.cfg.DorisLookupOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.deserialization.converter.DorisRowConverter; + +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; + +/** + * Adapter that bridges Flink 2.x {@link AsyncLookupFunction} to the existing {@link + * DorisRowDataAsyncLookupFunction} which is implemented against the older {@link + * org.apache.flink.table.functions.AsyncTableFunction} API and shared with Flink 1.15–1.20. + */ +public class DorisAsyncLookupAdapter extends AsyncLookupFunction { + + private final DorisRowDataAsyncLookupFunction delegate; + private final DorisRowConverter keyConverter; + private final int[] keyIndex; + + public DorisAsyncLookupAdapter( + DorisOptions options, + DorisLookupOptions lookupOptions, + String[] selectFields, + DataType[] fieldTypes, + String[] conditionFields, + int[] keyIndex) { + this.delegate = + new DorisRowDataAsyncLookupFunction( + options, + lookupOptions, + selectFields, + fieldTypes, + conditionFields, + keyIndex); + this.keyIndex = keyIndex; + DataType[] keyTypes = new DataType[keyIndex.length]; + for (int i = 0; i < keyIndex.length; i++) { + keyTypes[i] = fieldTypes[keyIndex[i]]; + } + this.keyConverter = new DorisRowConverter(keyTypes); + } + + @Override + public void open(FunctionContext context) throws Exception { + super.open(context); + delegate.open(context); + } + + @Override + public CompletableFuture> asyncLookup(RowData keyRow) { + Object[] keys = new Object[keyIndex.length]; + for (int i = 0; i < keyIndex.length; i++) { + keys[i] = keyConverter.convertExternal(keyRow, i); + } + + CompletableFuture> future = new CompletableFuture<>(); + try { + delegate.eval(future, keys); + } catch (IOException e) { + future.completeExceptionally(e); + } + return future; + } + + @Override + public void close() throws Exception { + try { + delegate.close(); + } finally { + super.close(); + } + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java new file mode 100644 index 000000000..d931593fb --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java @@ -0,0 +1,306 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.table; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.factories.DynamicTableSinkFactory; +import org.apache.flink.table.factories.DynamicTableSourceFactory; +import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.table.legacy.api.TableSchema; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.utils.TableSchemaUtils; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisLookupOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.sink.writer.WriteMode; + +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; + +import static org.apache.doris.flink.table.DorisConfigOptions.AUTO_REDIRECT; +import static org.apache.doris.flink.table.DorisConfigOptions.BENODES; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_BATCH_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_DESERIALIZE_ARROW_ASYNC; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_DESERIALIZE_QUEUE_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_EXEC_MEM_LIMIT; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_FILTER_QUERY; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_READ_FIELD; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_CONNECT_TIMEOUT_MS; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_QUERY_TIMEOUT_S; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_READ_TIMEOUT_MS; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_RETRIES; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_TABLET_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_THRIFT_MAX_MESSAGE_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.FENODES; +import static org.apache.doris.flink.table.DorisConfigOptions.FLIGHT_SQL_PORT; +import static org.apache.doris.flink.table.DorisConfigOptions.IDENTIFIER; +import static org.apache.doris.flink.table.DorisConfigOptions.JDBC_URL; +import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_CACHE_MAX_ROWS; +import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_CACHE_TTL; +import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_JDBC_ASYNC; +import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_JDBC_READ_BATCH_QUEUE_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_JDBC_READ_BATCH_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_JDBC_READ_THREAD_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_MAX_RETRIES; +import static org.apache.doris.flink.table.DorisConfigOptions.PASSWORD; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_COUNT; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_FLUSH_INTERVAL; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_FLUSH_MAX_BYTES; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_FLUSH_MAX_ROWS; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_CHECK_INTERVAL; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_ENABLE_2PC; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_ENABLE_BATCH_MODE; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_ENABLE_DELETE; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_FLUSH_QUEUE_SIZE; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_HTTP_UTF8_CHARSET; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_IGNORE_COMMIT_ERROR; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_IGNORE_UPDATE_BEFORE; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_LABEL_PREFIX; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_MAX_RETRIES; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_PARALLELISM; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_USE_CACHE; +import static org.apache.doris.flink.table.DorisConfigOptions.SINK_WRITE_MODE; +import static org.apache.doris.flink.table.DorisConfigOptions.SOURCE_USE_OLD_API; +import static org.apache.doris.flink.table.DorisConfigOptions.STREAM_LOAD_PROP_PREFIX; +import static org.apache.doris.flink.table.DorisConfigOptions.TABLE_IDENTIFIER; +import static org.apache.doris.flink.table.DorisConfigOptions.USERNAME; +import static org.apache.doris.flink.table.DorisConfigOptions.USE_FLIGHT_SQL; + +/** + * The {@link DorisDynamicTableFactory} translates the catalog table to a table source. + * + *

Because the table source requires a decoding format, we are discovering the format using the + * provided {@link FactoryUtil} for convenience. + */ +public final class DorisDynamicTableFactory + implements DynamicTableSourceFactory, DynamicTableSinkFactory { + + @Override + public String factoryIdentifier() { + return IDENTIFIER; // used for matching to `connector = '...'` + } + + @Override + public Set> requiredOptions() { + final Set> options = new HashSet<>(); + options.add(FENODES); + options.add(TABLE_IDENTIFIER); + return options; + } + + @Override + public Set> optionalOptions() { + final Set> options = new HashSet<>(); + options.add(FENODES); + options.add(BENODES); + options.add(TABLE_IDENTIFIER); + options.add(USERNAME); + options.add(PASSWORD); + options.add(JDBC_URL); + options.add(AUTO_REDIRECT); + + options.add(DORIS_READ_FIELD); + options.add(DORIS_FILTER_QUERY); + options.add(DORIS_TABLET_SIZE); + options.add(DORIS_REQUEST_CONNECT_TIMEOUT_MS); + options.add(DORIS_REQUEST_READ_TIMEOUT_MS); + options.add(DORIS_REQUEST_QUERY_TIMEOUT_S); + options.add(DORIS_REQUEST_RETRIES); + options.add(DORIS_DESERIALIZE_ARROW_ASYNC); + options.add(DORIS_DESERIALIZE_QUEUE_SIZE); + options.add(DORIS_BATCH_SIZE); + options.add(DORIS_EXEC_MEM_LIMIT); + options.add(DORIS_THRIFT_MAX_MESSAGE_SIZE); + + options.add(LOOKUP_CACHE_MAX_ROWS); + options.add(LOOKUP_CACHE_TTL); + options.add(LOOKUP_MAX_RETRIES); + options.add(LOOKUP_JDBC_ASYNC); + options.add(LOOKUP_JDBC_READ_BATCH_SIZE); + options.add(LOOKUP_JDBC_READ_THREAD_SIZE); + options.add(LOOKUP_JDBC_READ_BATCH_QUEUE_SIZE); + + options.add(SINK_CHECK_INTERVAL); + options.add(SINK_ENABLE_2PC); + options.add(SINK_MAX_RETRIES); + options.add(SINK_ENABLE_DELETE); + options.add(SINK_LABEL_PREFIX); + options.add(SINK_BUFFER_SIZE); + options.add(SINK_BUFFER_COUNT); + options.add(SINK_PARALLELISM); + options.add(SINK_IGNORE_UPDATE_BEFORE); + + options.add(SINK_ENABLE_BATCH_MODE); + options.add(SINK_BUFFER_FLUSH_MAX_ROWS); + options.add(SINK_BUFFER_FLUSH_MAX_BYTES); + options.add(SINK_FLUSH_QUEUE_SIZE); + options.add(SINK_BUFFER_FLUSH_INTERVAL); + + options.add(SINK_USE_CACHE); + + options.add(SOURCE_USE_OLD_API); + options.add(SINK_WRITE_MODE); + options.add(SINK_IGNORE_COMMIT_ERROR); + + options.add(USE_FLIGHT_SQL); + options.add(FLIGHT_SQL_PORT); + options.add(SINK_HTTP_UTF8_CHARSET); + return options; + } + + @Override + public DynamicTableSource createDynamicTableSource(Context context) { + // either implement your custom validation logic here ... + // or use the provided helper utility + final FactoryUtil.TableFactoryHelper helper = + FactoryUtil.createTableFactoryHelper(this, context); + // validate all options + helper.validateExcept(STREAM_LOAD_PROP_PREFIX); + // get the validated options + final ReadableConfig options = helper.getOptions(); + // derive the produced data type (excluding computed columns) from the catalog table + final DataType producedDataType = + context.getCatalogTable().getSchema().toPhysicalRowDataType(); + TableSchema physicalSchema = + TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema()); + // create and return dynamic table source + return new DorisDynamicTableSource( + getDorisOptions(helper.getOptions()), + getDorisReadOptions(helper.getOptions()), + getDorisLookupOptions(helper.getOptions()), + physicalSchema, + context.getPhysicalRowDataType()); + } + + private DorisOptions getDorisOptions(ReadableConfig readableConfig) { + final String fenodes = readableConfig.get(FENODES); + final String benodes = readableConfig.get(BENODES); + final DorisOptions.Builder builder = + DorisOptions.builder() + .setFenodes(fenodes) + .setBenodes(benodes) + .setAutoRedirect(readableConfig.get(AUTO_REDIRECT)) + .setJdbcUrl(readableConfig.get(JDBC_URL)) + .setTableIdentifier(readableConfig.get(TABLE_IDENTIFIER)); + + readableConfig.getOptional(USERNAME).ifPresent(builder::setUsername); + readableConfig.getOptional(PASSWORD).ifPresent(builder::setPassword); + return builder.build(); + } + + private DorisReadOptions getDorisReadOptions(ReadableConfig readableConfig) { + final DorisReadOptions.Builder builder = DorisReadOptions.builder(); + builder.setDeserializeArrowAsync(readableConfig.get(DORIS_DESERIALIZE_ARROW_ASYNC)) + .setDeserializeQueueSize(readableConfig.get(DORIS_DESERIALIZE_QUEUE_SIZE)) + .setExecMemLimit(readableConfig.get(DORIS_EXEC_MEM_LIMIT).getBytes()) + .setThriftMaxMessageSize(readableConfig.get(DORIS_THRIFT_MAX_MESSAGE_SIZE)) + .setFilterQuery(readableConfig.get(DORIS_FILTER_QUERY)) + .setReadFields(readableConfig.get(DORIS_READ_FIELD)) + .setRequestQueryTimeoutS( + (int) readableConfig.get(DORIS_REQUEST_QUERY_TIMEOUT_S).getSeconds()) + .setRequestBatchSize(readableConfig.get(DORIS_BATCH_SIZE)) + .setRequestConnectTimeoutMs( + (int) readableConfig.get(DORIS_REQUEST_CONNECT_TIMEOUT_MS).toMillis()) + .setRequestReadTimeoutMs( + (int) readableConfig.get(DORIS_REQUEST_READ_TIMEOUT_MS).toMillis()) + .setRequestRetries(readableConfig.get(DORIS_REQUEST_RETRIES)) + .setRequestTabletSize(readableConfig.get(DORIS_TABLET_SIZE)) + .setUseOldApi(readableConfig.get(SOURCE_USE_OLD_API)) + .setUseFlightSql(readableConfig.get(USE_FLIGHT_SQL)) + .setFlightSqlPort(readableConfig.get(FLIGHT_SQL_PORT)); + return builder.build(); + } + + private DorisExecutionOptions getDorisExecutionOptions( + ReadableConfig readableConfig, Properties streamLoadProp) { + final DorisExecutionOptions.Builder builder = DorisExecutionOptions.builder(); + builder.setCheckInterval((int) readableConfig.get(SINK_CHECK_INTERVAL).toMillis()); + builder.setMaxRetries(readableConfig.get(SINK_MAX_RETRIES)); + builder.setBufferSize((int) readableConfig.get(SINK_BUFFER_SIZE).getBytes()); + builder.setBufferCount(readableConfig.get(SINK_BUFFER_COUNT)); + builder.setLabelPrefix(readableConfig.get(SINK_LABEL_PREFIX)); + builder.setStreamLoadProp(streamLoadProp); + builder.setDeletable(readableConfig.get(SINK_ENABLE_DELETE)); + builder.setIgnoreUpdateBefore(readableConfig.get(SINK_IGNORE_UPDATE_BEFORE)); + builder.setIgnoreCommitError(readableConfig.get(SINK_IGNORE_COMMIT_ERROR)); + + if (!readableConfig.get(SINK_ENABLE_2PC)) { + builder.disable2PC(); + } else if (readableConfig.getOptional(SINK_ENABLE_2PC).isPresent()) { + // force open 2pc + builder.enable2PC(); + } + + builder.setWriteMode(WriteMode.of(readableConfig.get(SINK_WRITE_MODE))); + builder.setBatchMode(readableConfig.get(SINK_ENABLE_BATCH_MODE)); + // Compatible with previous versions + if (readableConfig.get(SINK_ENABLE_BATCH_MODE)) { + builder.setWriteMode(WriteMode.STREAM_LOAD_BATCH); + } + builder.setFlushQueueSize(readableConfig.get(SINK_FLUSH_QUEUE_SIZE)); + builder.setBufferFlushMaxRows(readableConfig.get(SINK_BUFFER_FLUSH_MAX_ROWS)); + builder.setBufferFlushMaxBytes( + (int) readableConfig.get(SINK_BUFFER_FLUSH_MAX_BYTES).getBytes()); + builder.setBufferFlushIntervalMs(readableConfig.get(SINK_BUFFER_FLUSH_INTERVAL).toMillis()); + builder.setUseCache(readableConfig.get(SINK_USE_CACHE)); + builder.setHttpUtf8Charset(readableConfig.get(SINK_HTTP_UTF8_CHARSET)); + return builder.build(); + } + + private DorisLookupOptions getDorisLookupOptions(ReadableConfig readableConfig) { + final DorisLookupOptions.Builder builder = DorisLookupOptions.builder(); + builder.setCacheExpireMs(readableConfig.get(LOOKUP_CACHE_TTL).toMillis()); + builder.setCacheMaxSize(readableConfig.get(LOOKUP_CACHE_MAX_ROWS)); + builder.setMaxRetryTimes(readableConfig.get(LOOKUP_MAX_RETRIES)); + builder.setJdbcReadBatchSize(readableConfig.get(LOOKUP_JDBC_READ_BATCH_SIZE)); + builder.setJdbcReadBatchQueueSize(readableConfig.get(LOOKUP_JDBC_READ_BATCH_QUEUE_SIZE)); + builder.setJdbcReadThreadSize(readableConfig.get(LOOKUP_JDBC_READ_THREAD_SIZE)); + builder.setAsync(readableConfig.get(LOOKUP_JDBC_ASYNC)); + return builder.build(); + } + + @Override + public DynamicTableSink createDynamicTableSink(Context context) { + final FactoryUtil.TableFactoryHelper helper = + FactoryUtil.createTableFactoryHelper(this, context); + + // validate all options + helper.validateExcept(STREAM_LOAD_PROP_PREFIX); + // sink parallelism + final Integer parallelism = helper.getOptions().get(SINK_PARALLELISM); + + Properties streamLoadProp = + DorisConfigOptions.getStreamLoadProp(context.getCatalogTable().getOptions()); + TableSchema physicalSchema = + TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema()); + // create and return dynamic table sink + return new DorisDynamicTableSink( + getDorisOptions(helper.getOptions()), + getDorisReadOptions(helper.getOptions()), + getDorisExecutionOptions(helper.getOptions(), streamLoadProp), + physicalSchema, + parallelism); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java new file mode 100644 index 000000000..b3d75ded5 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.table; + +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.sink.SinkV2Provider; +import org.apache.flink.table.connector.sink.abilities.SupportsOverwrite; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.legacy.api.TableSchema; +import org.apache.flink.util.Preconditions; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.connection.SimpleJdbcConnectionProvider; +import org.apache.doris.flink.exception.DorisSystemException; +import org.apache.doris.flink.rest.RestService; +import org.apache.doris.flink.sink.DorisSink; +import org.apache.doris.flink.sink.writer.serializer.RowDataSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Objects; +import java.util.Properties; +import java.util.stream.Collectors; + +import static org.apache.doris.flink.sink.writer.LoadConstants.COLUMNS_KEY; +import static org.apache.doris.flink.sink.writer.LoadConstants.CSV; +import static org.apache.doris.flink.sink.writer.LoadConstants.DORIS_DELETE_SIGN; +import static org.apache.doris.flink.sink.writer.LoadConstants.FIELD_DELIMITER_DEFAULT; +import static org.apache.doris.flink.sink.writer.LoadConstants.FIELD_DELIMITER_KEY; +import static org.apache.doris.flink.sink.writer.LoadConstants.FORMAT_KEY; + +/** DorisDynamicTableSink. */ +public class DorisDynamicTableSink implements DynamicTableSink, SupportsOverwrite { + private static final Logger LOG = LoggerFactory.getLogger(DorisDynamicTableSink.class); + private final DorisOptions options; + private final DorisReadOptions readOptions; + private final DorisExecutionOptions executionOptions; + private final TableSchema tableSchema; + private final Integer sinkParallelism; + private boolean overwrite = false; + + public DorisDynamicTableSink( + DorisOptions options, + DorisReadOptions readOptions, + DorisExecutionOptions executionOptions, + TableSchema tableSchema, + Integer sinkParallelism) { + this.options = options; + this.readOptions = readOptions; + this.executionOptions = executionOptions; + this.tableSchema = tableSchema; + this.sinkParallelism = sinkParallelism; + } + + @Override + public ChangelogMode getChangelogMode(ChangelogMode changelogMode) { + if (executionOptions.getIgnoreUpdateBefore()) { + return ChangelogMode.upsert(); + } else { + return ChangelogMode.all(); + } + } + + @Override + public SinkRuntimeProvider getSinkRuntimeProvider(Context context) { + Properties loadProperties = executionOptions.getStreamLoadProp(); + boolean deletable = + executionOptions.getDeletable() + && RestService.isUniqueKeyType(options, readOptions, LOG); + if (!loadProperties.containsKey(COLUMNS_KEY)) { + String[] fieldNames = tableSchema.getFieldNames(); + Preconditions.checkState(fieldNames != null && fieldNames.length > 0); + String columns = + String.join( + ",", + Arrays.stream(fieldNames) + .map( + item -> + String.format( + "`%s`", item.trim().replace("`", ""))) + .collect(Collectors.toList())); + if (deletable) { + columns = String.format("%s,%s", columns, DORIS_DELETE_SIGN); + } + loadProperties.put(COLUMNS_KEY, columns); + } + + RowDataSerializer.Builder serializerBuilder = RowDataSerializer.builder(); + serializerBuilder + .setFieldNames(tableSchema.getFieldNames()) + .setFieldType(tableSchema.getFieldDataTypes()) + .setType(loadProperties.getProperty(FORMAT_KEY, CSV)) + .enableDelete(deletable) + .setFieldDelimiter( + loadProperties.getProperty(FIELD_DELIMITER_KEY, FIELD_DELIMITER_DEFAULT)); + + DorisSink.Builder dorisSinkBuilder = DorisSink.builder(); + dorisSinkBuilder + .setDorisOptions(options) + .setDorisReadOptions(readOptions) + .setDorisExecutionOptions(executionOptions) + .setSerializer(serializerBuilder.build()); + DorisSink dorisSink = dorisSinkBuilder.build(); + + // for insert overwrite + if (overwrite) { + if (context.isBounded()) { + // execute jdbc query to truncate table + Preconditions.checkArgument( + options.getJdbcUrl() != null, "jdbc-url is required for Overwrite mode."); + // todo: should be written to a temporary table first, + // and then use GlobalCommitter to perform the rename. + truncateTable(); + } else { + throw new IllegalStateException("Streaming mode not support overwrite."); + } + } + return SinkV2Provider.of(dorisSink, sinkParallelism); + } + + private void truncateTable() { + String truncateQuery = "TRUNCATE TABLE " + options.getTableIdentifier(); + SimpleJdbcConnectionProvider jdbcConnectionProvider = + new SimpleJdbcConnectionProvider(options); + try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection(); + Statement statement = connection.createStatement()) { + LOG.info("Executing truncate query: {}", truncateQuery); + statement.execute(truncateQuery); + } catch (Exception e) { + LOG.error("Failed to execute truncate query: {}", truncateQuery, e); + throw new DorisSystemException( + String.format("Failed to execute truncate query: %s", truncateQuery), e); + } + } + + @Override + public DynamicTableSink copy() { + DorisDynamicTableSink sink = + new DorisDynamicTableSink( + options, readOptions, executionOptions, tableSchema, sinkParallelism); + sink.overwrite = overwrite; + return sink; + } + + @Override + public String asSummaryString() { + return "Doris Table Sink"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DorisDynamicTableSink that = (DorisDynamicTableSink) o; + return Objects.equals(options, that.options) + && Objects.equals(readOptions, that.readOptions) + && Objects.equals(executionOptions, that.executionOptions) + && Objects.equals(tableSchema, that.tableSchema) + && Objects.equals(sinkParallelism, that.sinkParallelism) + && Objects.equals(overwrite, that.overwrite); + } + + @Override + public int hashCode() { + return Objects.hash( + options, readOptions, executionOptions, tableSchema, sinkParallelism, overwrite); + } + + @Override + public void applyOverwrite(boolean overwrite) { + this.overwrite = overwrite; + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java new file mode 100644 index 000000000..4cecadd98 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java @@ -0,0 +1,258 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.table; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.connector.Projection; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.connector.source.InputFormatProvider; +import org.apache.flink.table.connector.source.LookupTableSource; +import org.apache.flink.table.connector.source.ScanTableSource; +import org.apache.flink.table.connector.source.SourceProvider; +import org.apache.flink.table.connector.source.abilities.SupportsFilterPushDown; +import org.apache.flink.table.connector.source.abilities.SupportsLimitPushDown; +import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown; +import org.apache.flink.table.connector.source.lookup.AsyncLookupFunctionProvider; +import org.apache.flink.table.connector.source.lookup.LookupFunctionProvider; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.expressions.ResolvedExpression; +import org.apache.flink.table.legacy.api.TableSchema; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.StringUtils; + +import org.apache.doris.flink.cfg.DorisLookupOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.deserialization.RowDataDeserializationSchema; +import org.apache.doris.flink.exception.DorisException; +import org.apache.doris.flink.rest.PartitionDefinition; +import org.apache.doris.flink.rest.RestService; +import org.apache.doris.flink.source.DorisSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** The {@link DorisDynamicTableSource} is used during planning. */ +public final class DorisDynamicTableSource + implements ScanTableSource, + LookupTableSource, + SupportsFilterPushDown, + SupportsProjectionPushDown, + SupportsLimitPushDown { + + private static final Logger LOG = LoggerFactory.getLogger(DorisDynamicTableSource.class); + private final DorisOptions options; + private final DorisReadOptions readOptions; + private DorisLookupOptions lookupOptions; + private TableSchema physicalSchema; + private List resolvedFilterQuery = new ArrayList<>(); + private DataType physicalRowDataType; + + public DorisDynamicTableSource( + DorisOptions options, + DorisReadOptions readOptions, + DorisLookupOptions lookupOptions, + TableSchema physicalSchema, + DataType physicalRowDataType) { + this.options = options; + this.lookupOptions = lookupOptions; + this.readOptions = readOptions; + this.physicalSchema = physicalSchema; + this.physicalRowDataType = physicalRowDataType; + } + + @Override + public ChangelogMode getChangelogMode() { + // in our example the format decides about the changelog mode + // but it could also be the source itself + return ChangelogMode.insertOnly(); + } + + @Override + public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderContext) { + if (!resolvedFilterQuery.isEmpty()) { + String filterQuery = resolvedFilterQuery.stream().collect(Collectors.joining(" AND ")); + if (!StringUtils.isNullOrWhitespaceOnly(readOptions.getFilterQuery())) { + filterQuery = + String.format("(%s) AND (%s)", readOptions.getFilterQuery(), filterQuery); + } + readOptions.setFilterQuery(filterQuery); + } + + if (StringUtils.isNullOrWhitespaceOnly(readOptions.getReadFields())) { + String[] selectFields = + DataType.getFieldNames(physicalRowDataType).toArray(new String[0]); + readOptions.setReadFields( + Arrays.stream(selectFields) + .map(item -> String.format("`%s`", item.trim().replace("`", ""))) + .collect(Collectors.joining(", "))); + } + + if (readOptions.getUseOldApi()) { + List dorisPartitions; + try { + dorisPartitions = RestService.findPartitions(options, readOptions, LOG); + } catch (DorisException e) { + throw new RuntimeException("Failed fetch doris partitions"); + } + DorisRowDataInputFormat.Builder builder = + DorisRowDataInputFormat.builder() + .setFenodes(options.getFenodes()) + .setBenodes(options.getBenodes()) + .setUsername(options.getUsername()) + .setPassword(options.getPassword()) + .setTableIdentifier(options.getTableIdentifier()) + .setPartitions(dorisPartitions) + .setReadOptions(readOptions) + .setRowType((RowType) physicalRowDataType.getLogicalType()); + return InputFormatProvider.of(builder.build()); + } else { + // Read data using the interface of the FLIP-27 specification + DorisSource build = + DorisSource.builder() + .setDorisReadOptions(readOptions) + .setDorisOptions(options) + .setDeserializer( + new RowDataDeserializationSchema( + (RowType) physicalRowDataType.getLogicalType())) + .build(); + return SourceProvider.of(build); + } + } + + @Override + public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) { + String[] keyNames = new String[context.getKeys().length]; + int[] keyIndexs = new int[context.getKeys().length]; + for (int i = 0; i < keyNames.length; i++) { + int[] innerKeyArr = context.getKeys()[i]; + keyNames[i] = DataType.getFieldNames(physicalRowDataType).get(innerKeyArr[0]); + keyIndexs[i] = innerKeyArr[0]; + } + String[] selectFields = DataType.getFieldNames(physicalRowDataType).toArray(new String[0]); + DataType[] fieldTypes = + DataType.getFieldDataTypes(physicalRowDataType).toArray(new DataType[0]); + if (lookupOptions.isAsync()) { + return AsyncLookupFunctionProvider.of( + new DorisAsyncLookupAdapter( + options, lookupOptions, selectFields, fieldTypes, keyNames, keyIndexs)); + } else { + return LookupFunctionProvider.of( + new DorisJdbcLookupAdapter( + options, lookupOptions, selectFields, fieldTypes, keyNames, keyIndexs)); + } + } + + @Override + public DynamicTableSource copy() { + // filterQuery/readFields of readOption may be overwritten in union all sql + DorisDynamicTableSource newSource = + new DorisDynamicTableSource( + options, + readOptions.copy(), + lookupOptions, + physicalSchema, + physicalRowDataType); + newSource.resolvedFilterQuery = new ArrayList<>(this.resolvedFilterQuery); + return newSource; + } + + @Override + public String asSummaryString() { + return "Doris Table Source"; + } + + @Override + public Result applyFilters(List filters) { + List acceptedFilters = new ArrayList<>(); + List remainingFilters = new ArrayList<>(); + + DorisExpressionVisitor expressionVisitor = new DorisExpressionVisitor(); + for (ResolvedExpression filter : filters) { + String filterQuery = filter.accept(expressionVisitor); + if (!StringUtils.isNullOrWhitespaceOnly(filterQuery)) { + acceptedFilters.add(filter); + this.resolvedFilterQuery.add(filterQuery); + } else { + remainingFilters.add(filter); + } + } + return Result.of(acceptedFilters, remainingFilters); + } + + @Override + public boolean supportsNestedProjection() { + return false; + } + + @Override + public void applyProjection(int[][] projectedFields, DataType producedDataType) { + this.physicalRowDataType = Projection.of(projectedFields).project(physicalRowDataType); + String[] selectFields = DataType.getFieldNames(physicalRowDataType).toArray(new String[0]); + this.readOptions.setReadFields( + Arrays.stream(selectFields) + .map(item -> String.format("`%s`", item.trim().replace("`", ""))) + .collect(Collectors.joining(", "))); + } + + @VisibleForTesting + public List getResolvedFilterQuery() { + return resolvedFilterQuery; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DorisDynamicTableSource that = (DorisDynamicTableSource) o; + return Objects.equals(options, that.options) + && Objects.equals(readOptions, that.readOptions) + && Objects.equals(lookupOptions, that.lookupOptions) + && Objects.equals(physicalSchema, that.physicalSchema) + && Objects.equals(resolvedFilterQuery, that.resolvedFilterQuery) + && Objects.equals(physicalRowDataType, that.physicalRowDataType); + } + + @Override + public int hashCode() { + return Objects.hash( + options, + readOptions, + lookupOptions, + physicalSchema, + resolvedFilterQuery, + physicalRowDataType); + } + + @Override + public void applyLimit(long limit) { + // partial limit push down to reduce the amount of data scanned + readOptions.setRowLimit(limit); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisJdbcLookupAdapter.java b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisJdbcLookupAdapter.java new file mode 100644 index 000000000..812adad80 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/java/org/apache/doris/flink/table/DorisJdbcLookupAdapter.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.doris.flink.table; + +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.LookupFunction; +import org.apache.flink.table.types.DataType; +import org.apache.flink.util.Collector; + +import org.apache.doris.flink.cfg.DorisLookupOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.deserialization.converter.DorisRowConverter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Adapter that bridges Flink 2.x {@link LookupFunction} to the existing {@link + * DorisRowDataJdbcLookupFunction} which is implemented against the older {@link + * org.apache.flink.table.functions.TableFunction} API and shared with Flink 1.15–1.20. + */ +public class DorisJdbcLookupAdapter extends LookupFunction { + + private final DorisRowDataJdbcLookupFunction delegate; + private final DorisRowConverter keyConverter; + private final int[] keyIndex; + + public DorisJdbcLookupAdapter( + DorisOptions options, + DorisLookupOptions lookupOptions, + String[] selectFields, + DataType[] fieldTypes, + String[] conditionFields, + int[] keyIndex) { + this.delegate = + new DorisRowDataJdbcLookupFunction( + options, + lookupOptions, + selectFields, + fieldTypes, + conditionFields, + keyIndex); + this.keyIndex = keyIndex; + DataType[] keyTypes = new DataType[keyIndex.length]; + for (int i = 0; i < keyIndex.length; i++) { + keyTypes[i] = fieldTypes[keyIndex[i]]; + } + this.keyConverter = new DorisRowConverter(keyTypes); + } + + @Override + public void open(FunctionContext context) throws Exception { + super.open(context); + delegate.open(context); + } + + @Override + public Collection lookup(RowData keyRow) throws IOException { + final List results = new ArrayList<>(); + delegate.setCollector( + new Collector() { + @Override + public void collect(RowData record) { + results.add(record); + } + + @Override + public void close() {} + }); + + delegate.eval(keyRow); + return results; + } + + @Override + public void close() throws Exception { + try { + delegate.close(); + } finally { + super.close(); + } + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-doris-connector/flink-doris-connector-flink2/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 000000000..5863c8bf2 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +org.apache.doris.flink.table.DorisDynamicTableFactory +org.apache.doris.flink.catalog.DorisCatalogFactory \ No newline at end of file diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/main/resources/log4j2.properties b/flink-doris-connector/flink-doris-connector-flink2/src/main/resources/log4j2.properties new file mode 100644 index 000000000..591598b95 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c [%t] %x - %m%n diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java new file mode 100644 index 000000000..b7a9e411d --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.catalog; + +import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.table.factories.FactoryUtil; + +import org.apache.doris.flink.table.DorisConfigOptions; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class DorisCatalogFactoryTest { + + @Test + public void testCreateCatalog() { + final Map options = new HashMap<>(); + options.put("type", DorisConfigOptions.IDENTIFIER); + options.put("jdbc-url", "jdbc:mysql://127.0.0.1:9030"); + options.put("fenodes", "127.0.0.1:8030"); + options.put("default-database", "doris_db"); + options.put("username", "root"); + options.put("password", ""); + options.put("doris.request.query.timeout", "3600s"); + options.put("sink.enable-2pc", "false"); + options.put("sink.properties.format", "json"); + options.put("sink.properties.read_json_by_line", "true"); + options.put("table.properties.replication_num", "1"); + + Catalog catalog = + FactoryUtil.createCatalog( + "test_catalog", + options, + null, + Thread.currentThread().getContextClassLoader()); + assertTrue(catalog instanceof DorisCatalog); + DorisCatalog dorisCatalog = (DorisCatalog) catalog; + assertEquals("test_catalog", dorisCatalog.getName()); + assertEquals("doris_db", dorisCatalog.getDefaultDatabase()); + assertEquals( + "jdbc:mysql://127.0.0.1:9030", dorisCatalog.getConnectionOptions().getJdbcUrl()); + assertEquals("127.0.0.1:8030", dorisCatalog.getConnectionOptions().getFenodes()); + assertEquals("root", dorisCatalog.getConnectionOptions().getUsername()); + assertEquals("", dorisCatalog.getConnectionOptions().getPassword()); + + Map properties = dorisCatalog.getProperties(); + assertEquals(10, properties.size()); + assertEquals("3600s", properties.get("doris.request.query.timeout")); + assertEquals("false", properties.get("sink.enable-2pc")); + assertEquals("json", properties.get("sink.properties.format")); + assertEquals("true", properties.get("sink.properties.read_json_by_line")); + assertEquals("1", properties.get("table.properties.replication_num")); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java new file mode 100644 index 000000000..5fd4268e2 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.catalog; + +import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException; +import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException; +import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException; +import org.apache.flink.table.catalog.exceptions.TableNotExistException; + +import org.apache.doris.flink.cfg.DorisConnectionOptions; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class DorisCatalogTest { + + private DorisCatalog catalog; + + @Before + public void setup() + throws DatabaseAlreadyExistException, + TableAlreadyExistException, + TableNotExistException, + DatabaseNotExistException { + DorisConnectionOptions connectionOptions = + new DorisConnectionOptions.DorisConnectionOptionsBuilder() + .withFenodes("127.0.0.1:8030") + .withJdbcUrl("jdbc:mysql://127.0.0.1:8030") + .withUsername("root") + .withPassword("xxxxx") + .build(); + + Map props = new HashMap<>(); + catalog = new DorisCatalog("catalog_test", connectionOptions, "test", props); + } + + @Test(expected = Exception.class) + public void testQueryFenodes() { + catalog.queryFenodes(); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java new file mode 100644 index 000000000..e5a6de64d --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink; + +import org.apache.flink.api.common.TaskInfo; +import org.apache.flink.api.connector.sink2.WriterInitContext; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.sink.batch.DorisBatchWriterAdapter; +import org.apache.doris.flink.sink.copy.DorisCopyWriterAdapter; +import org.apache.doris.flink.sink.writer.DorisAbstractWriter; +import org.apache.doris.flink.sink.writer.DorisWriterAdapter; +import org.apache.doris.flink.sink.writer.WriteMode; +import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer; +import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockedStatic; + +import java.util.Collections; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +public class DorisSinkTest { + + private MockedStatic backendUtilMockedStatic; + + @Before + public void setUp() throws Exception { + backendUtilMockedStatic = mockStatic(BackendUtil.class); + backendUtilMockedStatic.when(() -> BackendUtil.tryHttpConnection(any())).thenReturn(true); + } + + @Test + public void testDorisSink() { + DorisOptions dorisOptions = OptionUtils.buildDorisOptions(); + DorisReadOptions dorisReadOptions = OptionUtils.buildDorisReadOptions(); + DorisRecordSerializer serializer = new SimpleStringSerializer(); + WriterInitContext initContext = mock(WriterInitContext.class); + TaskInfo taskInfo = mock(TaskInfo.class); + when(initContext.getTaskInfo()).thenReturn(taskInfo); + + DorisExecutionOptions dorisExecutionOptions = + DorisExecutionOptions.builder().disable2PC().build(); + DorisSink dorisSink = + new DorisSink( + dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer); + DorisAbstractWriter dorisAbstractWriter = + dorisSink.getDorisAbstractWriter(initContext, Collections.emptyList()); + Assert.assertTrue(dorisAbstractWriter instanceof DorisWriterAdapter); + + dorisExecutionOptions = + DorisExecutionOptions.builder().setBatchMode(true).disable2PC().build(); + dorisSink = + new DorisSink( + dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer); + dorisAbstractWriter = + dorisSink.getDorisAbstractWriter(initContext, Collections.emptyList()); + Assert.assertTrue(dorisAbstractWriter instanceof DorisBatchWriterAdapter); + + dorisExecutionOptions = + DorisExecutionOptions.builder().disable2PC().setWriteMode(WriteMode.COPY).build(); + dorisSink = + new DorisSink( + dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer); + dorisAbstractWriter = + dorisSink.getDorisAbstractWriter(initContext, Collections.emptyList()); + Assert.assertTrue(dorisAbstractWriter instanceof DorisCopyWriterAdapter); + } + + @After + public void after() { + if (backendUtilMockedStatic != null) { + backendUtilMockedStatic.close(); + } + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java new file mode 100644 index 000000000..b511a3098 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.sink.batch; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer; +import org.junit.Assert; +import org.junit.Test; + +public class TestDorisBatchSink { + + @Test + public void testBuild() { + DorisBatchSink.Builder builder = DorisBatchSink.builder(); + DorisOptions.Builder dorisBuilder = DorisOptions.builder(); + dorisBuilder + .setFenodes("127.0.0.1:8030") + .setTableIdentifier("db.tbl") + .setUsername("root") + .setPassword(""); + DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder(); + DorisBatchSink build = + builder.setDorisExecutionOptions(executionBuilder.build()) + .setSerializer(new SimpleStringSerializer()) + .setDorisOptions(dorisBuilder.build()) + .build(); + + DorisReadOptions expected = DorisReadOptions.builder().build(); + DorisReadOptions actual = build.getDorisReadOptions(); + Assert.assertEquals(expected, actual); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java new file mode 100644 index 000000000..e2056afe2 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.table; + +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.factories.utils.FactoryMocks; +import org.apache.flink.table.legacy.api.TableSchema; + +import org.apache.doris.flink.cfg.DorisExecutionOptions; +import org.apache.doris.flink.cfg.DorisLookupOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_BATCH_SIZE_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_EXEC_MEM_LIMIT_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_REQUEST_RETRIES_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_TABLET_SIZE_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.FLIGHT_SQL_PORT_DEFAULT; +import static org.apache.doris.flink.cfg.ConfigurationOptions.USE_FLIGHT_SQL_DEFAULT; +import static org.apache.doris.flink.utils.FactoryMocks.SCHEMA; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class DorisDynamicTableFactoryTest { + + @Test + public void testDorisSourceProperties() { + Map properties = getAllOptions(); + properties.put("doris.request.query.timeout", "21600s"); + properties.put("doris.request.tablet.size", "1"); + properties.put("doris.batch.size", "4064"); + properties.put("doris.exec.mem.limit", "8192mb"); + properties.put("doris.deserialize.arrow.async", "false"); + properties.put("doris.deserialize.queue.size", "64"); + properties.put("doris.thrift.max.message.size", Integer.MAX_VALUE + ""); + + properties.put("lookup.cache.max-rows", "100"); + properties.put("lookup.cache.ttl", "20s"); + properties.put("lookup.max-retries", "1"); + properties.put("lookup.jdbc.async", "true"); + properties.put("lookup.jdbc.read.batch.size", "16"); + properties.put("lookup.jdbc.read.batch.queue-size", "16"); + properties.put("lookup.jdbc.read.thread-size", "1"); + properties.put("source.use-flight-sql", "true"); + properties.put("source.flight-sql-port", "-1"); + DynamicTableSource actual = FactoryMocks.createTableSource(SCHEMA, properties); + DorisOptions options = + DorisOptions.builder() + .setTableIdentifier("db.tbl") + .setFenodes("127.0.0.1:8030") + .setBenodes("127.0.0.1:8040") + .setAutoRedirect(true) + .setUsername("root") + .setPassword("") + .setJdbcUrl("jdbc:mysql://127.0.0.1:9030") + .build(); + DorisLookupOptions lookupOptions = + DorisLookupOptions.builder() + .setCacheExpireMs(20000) + .setCacheMaxSize(100) + .setAsync(true) + .setJdbcReadBatchQueueSize(16) + .setJdbcReadBatchSize(16) + .setJdbcReadThreadSize(1) + .setMaxRetryTimes(1) + .build(); + + final DorisReadOptions.Builder readOptionBuilder = DorisReadOptions.builder(); + readOptionBuilder + .setDeserializeArrowAsync(DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT) + .setDeserializeQueueSize(DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT) + .setExecMemLimit(DORIS_EXEC_MEM_LIMIT_DEFAULT) + .setRequestQueryTimeoutS(DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT) + .setRequestBatchSize(DORIS_BATCH_SIZE_DEFAULT) + .setRequestConnectTimeoutMs(DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT) + .setRequestReadTimeoutMs(DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT) + .setRequestRetries(DORIS_REQUEST_RETRIES_DEFAULT) + .setRequestTabletSize(DORIS_TABLET_SIZE_DEFAULT) + .setUseFlightSql(USE_FLIGHT_SQL_DEFAULT) + .setFlightSqlPort(FLIGHT_SQL_PORT_DEFAULT) + .setThriftMaxMessageSize(DORIS_THRIFT_MAX_MESSAGE_SIZE_DEFAULT); + DorisDynamicTableSource expected = + new DorisDynamicTableSource( + options, + readOptionBuilder.build(), + lookupOptions, + TableSchema.fromResolvedSchema(SCHEMA), + SCHEMA.toPhysicalRowDataType()); + + assertEquals(actual, expected); + } + + @Test + public void testDorisSinkProperties() { + Map properties = getAllOptions(); + properties.put("doris.request.query.timeout", "21600s"); + properties.put("doris.request.tablet.size", "1"); + properties.put("doris.batch.size", "4064"); + properties.put("doris.exec.mem.limit", "8192mb"); + properties.put("doris.deserialize.arrow.async", "false"); + properties.put("doris.deserialize.queue.size", "64"); + + properties.put("sink.label-prefix", "abc"); + properties.put("sink.properties.format", "json"); + properties.put("sink.properties.read_json_by_line", "true"); + properties.put("sink.enable-delete", "true"); + properties.put("sink.enable-2pc", "true"); + properties.put("sink.buffer-size", "1MB"); + properties.put("sink.buffer-count", "3"); + properties.put("sink.max-retries", "1"); + properties.put("sink.check-interval", "10s"); + properties.put("sink.use-cache", "true"); + properties.put("sink.enable.batch-mode", "true"); + properties.put("sink.flush.queue-size", "2"); + properties.put("sink.buffer-flush.max-rows", "10000"); + properties.put("sink.buffer-flush.max-bytes", "10MB"); + properties.put("sink.buffer-flush.interval", "10s"); + properties.put("sink.ignore.update-before", "true"); + properties.put("sink.ignore.commit-error", "false"); + properties.put("sink.parallelism", "1"); + + DynamicTableSink actual = FactoryMocks.createTableSink(SCHEMA, properties); + DorisOptions options = + DorisOptions.builder() + .setTableIdentifier("db.tbl") + .setFenodes("127.0.0.1:8030") + .setBenodes("127.0.0.1:8040") + .setAutoRedirect(true) + .setUsername("root") + .setPassword("") + .setJdbcUrl("jdbc:mysql://127.0.0.1:9030") + .build(); + + Properties prop = new Properties(); + prop.put("format", "json"); + prop.put("read_json_by_line", "true"); + DorisExecutionOptions executionOptions = + DorisExecutionOptions.builder() + .setLabelPrefix("abc") + .setBufferCount(3) + .setBufferSize(1024 * 1024) + .setStreamLoadProp(prop) + .setMaxRetries(1) + .setCheckInterval(10000) + .setBatchMode(true) + .enable2PC() + .setBufferFlushIntervalMs(10000) + .setBufferFlushMaxBytes(10 * 1024 * 1024) + .setBufferFlushMaxRows(10000) + .setFlushQueueSize(2) + .setUseCache(true) + .setIgnoreCommitError(false) + .build(); + + final DorisReadOptions.Builder readOptionBuilder = DorisReadOptions.builder(); + readOptionBuilder + .setDeserializeArrowAsync(DORIS_DESERIALIZE_ARROW_ASYNC_DEFAULT) + .setDeserializeQueueSize(DORIS_DESERIALIZE_QUEUE_SIZE_DEFAULT) + .setExecMemLimit(DORIS_EXEC_MEM_LIMIT_DEFAULT) + .setRequestQueryTimeoutS(DORIS_REQUEST_QUERY_TIMEOUT_S_DEFAULT) + .setRequestBatchSize(DORIS_BATCH_SIZE_DEFAULT) + .setRequestConnectTimeoutMs(DORIS_REQUEST_CONNECT_TIMEOUT_MS_DEFAULT) + .setRequestReadTimeoutMs(DORIS_REQUEST_READ_TIMEOUT_MS_DEFAULT) + .setRequestRetries(DORIS_REQUEST_RETRIES_DEFAULT) + .setRequestTabletSize(DORIS_TABLET_SIZE_DEFAULT) + .setUseFlightSql(USE_FLIGHT_SQL_DEFAULT) + .setFlightSqlPort(FLIGHT_SQL_PORT_DEFAULT); + DorisDynamicTableSink expected = + new DorisDynamicTableSink( + options, + readOptionBuilder.build(), + executionOptions, + TableSchema.fromResolvedSchema(SCHEMA), + 1); + + assertEquals(expected, expected); + assertNotEquals(expected, null); + assertEquals(actual, expected); + + options.setTableIdentifier("xxxxx"); + DorisDynamicTableSink expected2 = + new DorisDynamicTableSink( + options, + readOptionBuilder.build(), + executionOptions, + TableSchema.fromResolvedSchema(SCHEMA), + 1); + assertNotEquals(actual, expected2); + options.setTableIdentifier("db.tbl"); + + readOptionBuilder.setExecMemLimit(1L); + DorisDynamicTableSink expected3 = + new DorisDynamicTableSink( + options, + readOptionBuilder.build(), + executionOptions, + TableSchema.fromResolvedSchema(SCHEMA), + 1); + assertNotEquals(actual, expected3); + readOptionBuilder.setExecMemLimit(DORIS_EXEC_MEM_LIMIT_DEFAULT); + + executionOptions.setEnable2PC(false); + DorisDynamicTableSink expected4 = + new DorisDynamicTableSink( + options, + readOptionBuilder.build(), + executionOptions, + TableSchema.fromResolvedSchema(SCHEMA), + 1); + assertNotEquals(actual, expected4); + executionOptions.setEnable2PC(true); + + // DynamicTableSink actual2 = FactoryMocks.createTableSink(SCHEMA, new HashMap<>()); + // assertNotEquals(actual2, expected); + // In Flink 2.x, calling createTableSink with no options throws ValidationException + assertThrows( + ValidationException.class, + () -> FactoryMocks.createTableSink(SCHEMA, new HashMap<>())); + } + + private Map getAllOptions() { + Map options = new HashMap<>(); + options.put("connector", DorisConfigOptions.IDENTIFIER); + options.put("fenodes", "127.0.0.1:8030"); + options.put("benodes", "127.0.0.1:8040"); + options.put("jdbc-url", "jdbc:mysql://127.0.0.1:9030"); + options.put("table.identifier", "db.tbl"); + options.put("username", "root"); + options.put("password", ""); + options.put("auto-redirect", "true"); + options.put("doris.request.retries", "3"); + options.put("doris.request.connect.timeout", "60s"); + options.put("doris.request.read.timeout", "60s"); + return options; + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java new file mode 100644 index 000000000..110c57371 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java @@ -0,0 +1,323 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.table; + +import org.apache.flink.api.common.io.InputFormat; +import org.apache.flink.api.connector.source.Source; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.connector.source.InputFormatProvider; +import org.apache.flink.table.connector.source.ScanTableSource; +import org.apache.flink.table.connector.source.SourceProvider; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.expressions.CallExpression; +import org.apache.flink.table.expressions.FieldReferenceExpression; +import org.apache.flink.table.expressions.ResolvedExpression; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.legacy.api.TableSchema; +import org.apache.flink.table.runtime.connector.source.ScanRuntimeProviderContext; +import org.apache.flink.table.types.DataType; + +import org.apache.doris.flink.cfg.DorisLookupOptions; +import org.apache.doris.flink.cfg.DorisReadOptions; +import org.apache.doris.flink.rest.PartitionDefinition; +import org.apache.doris.flink.rest.RestService; +import org.apache.doris.flink.sink.OptionUtils; +import org.apache.doris.flink.source.DorisSource; +import org.apache.doris.flink.source.enumerator.PendingSplitsCheckpoint; +import org.apache.doris.flink.source.split.DorisSourceSplit; +import org.apache.doris.flink.utils.FactoryMocks; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedStatic; + +import java.time.LocalDate; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; + +import static org.apache.doris.flink.utils.FactoryMocks.SCHEMA; +import static org.apache.flink.table.expressions.ApiExpressionUtils.valueLiteral; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockStatic; + +public class DorisDynamicTableSourceTest { + + @Test + public void testDorisUseNewApi() { + DorisReadOptions.Builder builder = OptionUtils.dorisReadOptionsBuilder(); + builder.setUseOldApi(false); + final DorisDynamicTableSource actualDorisSource = + new DorisDynamicTableSource( + OptionUtils.buildDorisOptions(), + builder.build(), + DorisLookupOptions.builder().build(), + TableSchema.fromResolvedSchema(SCHEMA), + FactoryMocks.PHYSICAL_DATA_TYPE); + ScanTableSource.ScanRuntimeProvider provider = + actualDorisSource.getScanRuntimeProvider(ScanRuntimeProviderContext.INSTANCE); + assertDorisSource(provider); + } + + @Test + public void testDorisUseNewApiDefault() { + final DorisDynamicTableSource actualDorisSource = + new DorisDynamicTableSource( + OptionUtils.buildDorisOptions(), + OptionUtils.buildDorisReadOptions(), + DorisLookupOptions.builder().build(), + TableSchema.fromResolvedSchema(SCHEMA), + FactoryMocks.PHYSICAL_DATA_TYPE); + ScanTableSource.ScanRuntimeProvider provider = + actualDorisSource.getScanRuntimeProvider(ScanRuntimeProviderContext.INSTANCE); + assertDorisSource(provider); + } + + @Test + public void testDorisUseOldApi() { + DorisReadOptions.Builder builder = OptionUtils.dorisReadOptionsBuilder(); + builder.setUseOldApi(true); + MockedStatic restServiceMockedStatic = mockStatic(RestService.class); + restServiceMockedStatic + .when(() -> RestService.findPartitions(any(), any(), any())) + .thenReturn( + Collections.singletonList( + new PartitionDefinition("", "", "", new HashSet<>(), ""))); + final DorisDynamicTableSource actualDorisSource = + new DorisDynamicTableSource( + OptionUtils.buildDorisOptions(), + builder.build(), + DorisLookupOptions.builder().build(), + TableSchema.fromResolvedSchema(SCHEMA), + FactoryMocks.PHYSICAL_DATA_TYPE); + ScanTableSource.ScanRuntimeProvider provider = + actualDorisSource.getScanRuntimeProvider(ScanRuntimeProviderContext.INSTANCE); + assertDorisInputFormat(provider); + restServiceMockedStatic.close(); + } + + private void assertDorisInputFormat(ScanTableSource.ScanRuntimeProvider provider) { + assertThat(provider, instanceOf(InputFormatProvider.class)); + final InputFormatProvider inputFormatProvider = (InputFormatProvider) provider; + + InputFormat inputFormat = + (InputFormat) + inputFormatProvider.createInputFormat(); + assertThat(inputFormat, instanceOf(DorisRowDataInputFormat.class)); + } + + private void assertDorisSource(ScanTableSource.ScanRuntimeProvider provider) { + assertThat(provider, instanceOf(SourceProvider.class)); + final SourceProvider sourceProvider = (SourceProvider) provider; + + Source source = + (Source) + sourceProvider.createSource(); + assertThat(source, instanceOf(DorisSource.class)); + } + + @Test + public void testApplyProjection() { + int[][] projectionArray = new int[2][1]; + projectionArray[0][0] = 0; + projectionArray[1][0] = 2; + DataType projectionDataType = + ResolvedSchema.of( + Column.physical("a", DataTypes.STRING()), + Column.physical("c", DataTypes.BOOLEAN())) + .toPhysicalRowDataType(); + DorisReadOptions readOptions = OptionUtils.dorisReadOptionsBuilder().build(); + final DorisDynamicTableSource actualDorisSource = + new DorisDynamicTableSource( + OptionUtils.buildDorisOptions(), + readOptions, + DorisLookupOptions.builder().build(), + TableSchema.fromResolvedSchema(SCHEMA), + FactoryMocks.PHYSICAL_DATA_TYPE); + actualDorisSource.applyProjection(projectionArray, projectionDataType); + Assert.assertEquals(readOptions.getReadFields(), "`a`, `c`"); + } + + @Test + public void testFilter() { + DorisReadOptions readOptions = OptionUtils.dorisReadOptionsBuilder().build(); + final DorisDynamicTableSource actualDorisSource = + new DorisDynamicTableSource( + OptionUtils.buildDorisOptions(), + readOptions, + DorisLookupOptions.builder().build(), + TableSchema.fromResolvedSchema(SCHEMA), + FactoryMocks.PHYSICAL_DATA_TYPE); + ResolvedExpression aRef = new FieldReferenceExpression("a", DataTypes.STRING(), 0, 2); + ResolvedExpression aRefCharLength = + CallExpression.anonymous( + BuiltInFunctionDefinitions.CHAR_LENGTH, + Collections.singletonList(aRef), + DataTypes.INT()); + ResolvedExpression aExp = + CallExpression.anonymous( + BuiltInFunctionDefinitions.LESS_THAN, + Arrays.asList(aRefCharLength, valueLiteral(10)), + DataTypes.BOOLEAN()); + actualDorisSource.applyFilters(Collections.singletonList(aExp)); + Assert.assertTrue(actualDorisSource.getResolvedFilterQuery().isEmpty()); + + ResolvedExpression a1Ref = new FieldReferenceExpression("a", DataTypes.STRING(), 0, 2); + ResolvedExpression a1Exp = + CallExpression.anonymous( + BuiltInFunctionDefinitions.EQUALS, + Arrays.asList(a1Ref, valueLiteral("doris")), + DataTypes.BOOLEAN()); + actualDorisSource.applyFilters(Arrays.asList(a1Exp)); + assertEquals(Arrays.asList("(a = 'doris')"), actualDorisSource.getResolvedFilterQuery()); + + ResolvedExpression b1Ref = new FieldReferenceExpression("b", DataTypes.INT(), 0, 2); + ResolvedExpression b1Exp = + CallExpression.anonymous( + BuiltInFunctionDefinitions.EQUALS, + Arrays.asList(b1Ref, valueLiteral(1)), + DataTypes.BOOLEAN()); + actualDorisSource.getResolvedFilterQuery().clear(); + actualDorisSource.applyFilters(Arrays.asList(b1Exp)); + assertEquals(Arrays.asList("(b = 1)"), actualDorisSource.getResolvedFilterQuery()); + + actualDorisSource.getResolvedFilterQuery().clear(); + actualDorisSource.applyFilters( + Arrays.asList( + b1Exp, + CallExpression.anonymous( + BuiltInFunctionDefinitions.NOT_EQUALS, + Arrays.asList(b1Ref, valueLiteral(1)), + DataTypes.BOOLEAN()), + CallExpression.anonymous( + BuiltInFunctionDefinitions.GREATER_THAN, + Arrays.asList(b1Ref, valueLiteral(1)), + DataTypes.BOOLEAN()), + CallExpression.anonymous( + BuiltInFunctionDefinitions.GREATER_THAN_OR_EQUAL, + Arrays.asList(b1Ref, valueLiteral(1)), + DataTypes.BOOLEAN()), + CallExpression.anonymous( + BuiltInFunctionDefinitions.LESS_THAN, + Arrays.asList(b1Ref, valueLiteral(1)), + DataTypes.BOOLEAN()), + CallExpression.anonymous( + BuiltInFunctionDefinitions.LESS_THAN_OR_EQUAL, + Arrays.asList(b1Ref, valueLiteral(1)), + DataTypes.BOOLEAN()))); + assertEquals( + Arrays.asList("(b = 1)", "(b <> 1)", "(b > 1)", "(b >= 1)", "(b < 1)", "(b <= 1)"), + actualDorisSource.getResolvedFilterQuery()); + + actualDorisSource.getResolvedFilterQuery().clear(); + actualDorisSource.applyFilters( + Arrays.asList( + CallExpression.anonymous( + BuiltInFunctionDefinitions.OR, + Arrays.asList(a1Exp, b1Exp), + DataTypes.BOOLEAN()))); + assertEquals( + Arrays.asList("((a = 'doris') OR (b = 1))"), + actualDorisSource.getResolvedFilterQuery()); + + actualDorisSource.getResolvedFilterQuery().clear(); + actualDorisSource.applyFilters( + Arrays.asList( + CallExpression.anonymous( + BuiltInFunctionDefinitions.AND, + Arrays.asList(a1Exp, b1Exp), + DataTypes.BOOLEAN()))); + assertEquals( + Arrays.asList("((a = 'doris') AND (b = 1))"), + actualDorisSource.getResolvedFilterQuery()); + + ResolvedExpression aLikeExp = + CallExpression.anonymous( + BuiltInFunctionDefinitions.LIKE, + Arrays.asList(a1Ref, valueLiteral("d")), + DataTypes.BOOLEAN()); + actualDorisSource.getResolvedFilterQuery().clear(); + actualDorisSource.applyFilters(Collections.singletonList(aLikeExp)); + assertEquals(Arrays.asList("(a LIKE 'd')"), actualDorisSource.getResolvedFilterQuery()); + + ResolvedExpression aInExp = + CallExpression.anonymous( + BuiltInFunctionDefinitions.IN, + Arrays.asList(a1Ref, valueLiteral("doris")), + DataTypes.BOOLEAN()); + actualDorisSource.getResolvedFilterQuery().clear(); + actualDorisSource.applyFilters(Collections.singletonList(aInExp)); + assertTrue(actualDorisSource.getResolvedFilterQuery().isEmpty()); + + ResolvedExpression aNoNullExp = + CallExpression.anonymous( + BuiltInFunctionDefinitions.IS_NOT_NULL, + Arrays.asList(a1Ref), + DataTypes.BOOLEAN()); + actualDorisSource.getResolvedFilterQuery().clear(); + actualDorisSource.applyFilters(Collections.singletonList(aNoNullExp)); + assertEquals(Arrays.asList("(a IS NOT NULL)"), actualDorisSource.getResolvedFilterQuery()); + + ResolvedExpression aNullExp = + CallExpression.anonymous( + BuiltInFunctionDefinitions.IS_NULL, + Arrays.asList(a1Ref), + DataTypes.BOOLEAN()); + actualDorisSource.getResolvedFilterQuery().clear(); + actualDorisSource.applyFilters(Collections.singletonList(aNullExp)); + assertEquals(Arrays.asList("(a IS NULL)"), actualDorisSource.getResolvedFilterQuery()); + } + + @Test + public void testFilterDate() { + DorisReadOptions readOptions = OptionUtils.dorisReadOptionsBuilder().build(); + final DorisDynamicTableSource actualDorisSource = + new DorisDynamicTableSource( + OptionUtils.buildDorisOptions(), + readOptions, + DorisLookupOptions.builder().build(), + TableSchema.fromResolvedSchema(FactoryMocks.SCHEMA_DT), + FactoryMocks.SCHEMA_DT.toPhysicalRowDataType()); + ResolvedExpression dRef = new FieldReferenceExpression("d", DataTypes.DATE(), 0, 2); + ResolvedExpression dExp = + CallExpression.anonymous( + BuiltInFunctionDefinitions.EQUALS, + Arrays.asList(dRef, valueLiteral(LocalDate.of(2021, 1, 1))), + DataTypes.BOOLEAN()); + actualDorisSource.applyFilters(Arrays.asList(dExp)); + assertEquals( + Arrays.asList("(d = '2021-01-01')"), actualDorisSource.getResolvedFilterQuery()); + + ResolvedExpression eRef = new FieldReferenceExpression("e", DataTypes.TIMESTAMP(), 0, 2); + ResolvedExpression eExp = + CallExpression.anonymous( + BuiltInFunctionDefinitions.EQUALS, + Arrays.asList(eRef, valueLiteral("2021-01-01 11:12:13")), + DataTypes.BOOLEAN()); + actualDorisSource.getResolvedFilterQuery().clear(); + actualDorisSource.applyFilters(Arrays.asList(eExp)); + assertEquals( + Arrays.asList("(e = '2021-01-01 11:12:13')"), + actualDorisSource.getResolvedFilterQuery()); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java new file mode 100644 index 000000000..efc80917f --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.utils; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.Schema.UnresolvedColumn; +import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.types.AtomicDataType; +import org.apache.flink.table.types.logical.VarCharType; + +import java.util.ArrayList; +import java.util.Map; + +public class CatalogUtil { + public static CatalogTable createTable(Schema tableSchema, Map options) { + return CatalogTable.newBuilder() + .comment("FlinkTable") + .partitionKeys(new ArrayList<>()) + .schema(tableSchema) + .options(options) + .build(); + } + + public static Schema getTableSchema() { + return Schema.newBuilder() + .column("id", new AtomicDataType(new VarCharType(false, 128))) + .column("c_boolean", DataTypes.BOOLEAN()) + .column("c_char", DataTypes.CHAR(1)) + .column("c_date", DataTypes.DATE()) + .column("c_datetime", DataTypes.TIMESTAMP(0)) + .column("c_decimal", DataTypes.DECIMAL(10, 2)) + .column("c_double", DataTypes.DOUBLE()) + .column("c_float", DataTypes.FLOAT()) + .column("c_int", DataTypes.INT()) + .column("c_bigint", DataTypes.BIGINT()) + .column("c_largeint", DataTypes.STRING()) + .column("c_smallint", DataTypes.SMALLINT()) + .column("c_string", DataTypes.STRING()) + .column("c_tinyint", DataTypes.TINYINT()) + .column("c_array", DataTypes.ARRAY(DataTypes.INT())) + .column("c_map", DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING())) + .column("c_row", DataTypes.ROW()) + .column("c_varbinary", DataTypes.VARBINARY(16)) + .primaryKey("id") + .build(); + } + + public static String[] getColumns() { + return getTableSchema().getColumns().stream() + .map(UnresolvedColumn::getName) + .toArray(String[]::new); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java new file mode 100644 index 000000000..ca7aefbb1 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.utils; + +import org.apache.flink.api.common.state.CheckpointListener; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.runtime.state.FunctionInitializationContext; +import org.apache.flink.runtime.state.FunctionSnapshotContext; +import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; +import org.apache.flink.streaming.api.functions.source.legacy.RichParallelSourceFunction; +import org.apache.flink.util.SerializableObject; + +import org.apache.doris.flink.sink.batch.RecordWithMeta; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.Iterator; + +/* + * MockMultiTableSource is a mock source for testing multi-table source + * */ +public class MockMultiTableSource extends RichParallelSourceFunction + implements CheckpointedFunction, CheckpointListener { + private static final Logger LOG = LoggerFactory.getLogger(MockMultiTableSource.class); + private final Object blocker = new SerializableObject(); + private transient ListState state; + private transient ListState tableNumState; + private Long id = 0L; + private int numEventsTotal; + private int failCheckpointId = -1; + private volatile boolean running = true; + private volatile long waitNextCheckpoint = 0L; + private volatile long lastCheckpointConfirmed = 0L; + private String database; + private String tableName; + private int tableNum; + private int addTableCheckpointId; + + public MockMultiTableSource( + int numEventsTotal, + int failCheckpointId, + String database, + String tableName, + int tableNum, + int addTableCheckpointId) { + this.numEventsTotal = numEventsTotal; + this.failCheckpointId = failCheckpointId; + this.database = database; + this.tableName = tableName; + this.tableNum = tableNum; + this.addTableCheckpointId = addTableCheckpointId; + } + + @Override + public void run(SourceContext ctx) throws Exception { + int taskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); + while (this.running && id < this.numEventsTotal) { + id = id + 1; + for (int i = 1; i <= tableNum; i++) { + String record = id + "," + taskId; + RecordWithMeta output = new RecordWithMeta(database, tableName + i, record); + ctx.collect(output); + } + + // Wait for the checkpoint to complete before sending the next record + waitNextCheckpoint = lastCheckpointConfirmed + 1; + synchronized (this.blocker) { + while (this.lastCheckpointConfirmed < waitNextCheckpoint) { + this.blocker.wait(); + } + } + } + } + + @Override + public void cancel() { + this.running = false; + } + + @Override + public void snapshotState(FunctionSnapshotContext context) throws Exception { + state.update(Collections.singletonList(id)); + if (context.getCheckpointId() == addTableCheckpointId) { + tableNum++; + } + tableNumState.update(Collections.singletonList(tableNum)); + if (failCheckpointId > 0 && context.getCheckpointId() == failCheckpointId) { + throw new RuntimeException( + "Trigger fail for testing, checkpointId = " + context.getCheckpointId()); + } + LOG.info( + "snapshot state to id={}, tableNum={} for checkpoint {}", + id, + tableNum, + context.getCheckpointId()); + } + + @Override + public void initializeState(FunctionInitializationContext context) throws Exception { + state = + context.getOperatorStateStore() + .getListState( + new ListStateDescriptor<>("id", TypeInformation.of(Long.class))); + tableNumState = + context.getOperatorStateStore() + .getListState( + new ListStateDescriptor<>( + "tableNum", TypeInformation.of(Integer.class))); + if (context.isRestored()) { + Iterator iterator = state.get().iterator(); + while (iterator.hasNext()) { + id += iterator.next(); + } + Iterator tableNumIterator = tableNumState.get().iterator(); + while (tableNumIterator.hasNext()) { + tableNum = tableNumIterator.next(); + } + } + LOG.info("restore state from id {}, tableNum {}", id, tableNum); + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception { + synchronized (blocker) { + this.lastCheckpointConfirmed = checkpointId; + blocker.notifyAll(); + } + LOG.info("checkpoint {} finished", checkpointId); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/MockSource.java b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/MockSource.java new file mode 100644 index 000000000..05826d409 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/MockSource.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.utils; + +import org.apache.flink.api.common.state.CheckpointListener; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.runtime.state.FunctionInitializationContext; +import org.apache.flink.runtime.state.FunctionSnapshotContext; +import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; +import org.apache.flink.streaming.api.functions.source.legacy.RichParallelSourceFunction; +import org.apache.flink.util.SerializableObject; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.Iterator; + +/* + * MockSource is a mock source for testing + * */ +public class MockSource extends RichParallelSourceFunction + implements CheckpointedFunction, CheckpointListener { + private static final Logger LOG = LoggerFactory.getLogger(MockSource.class); + private final Object blocker = new SerializableObject(); + private transient ListState state; + private Long id = 0L; + private int numEventsTotal; + private int failCheckpointId = -1; + private volatile boolean running = true; + private volatile long waitNextCheckpoint = 0L; + private volatile long lastCheckpointConfirmed = 0L; + + public MockSource(int numEventsTotal) { + this.numEventsTotal = numEventsTotal; + } + + public MockSource(int numEventsTotal, int failCheckpointId) { + this.numEventsTotal = numEventsTotal; + this.failCheckpointId = failCheckpointId; + } + + @Override + public void run(SourceContext ctx) throws Exception { + + int taskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); + while (this.running && id < this.numEventsTotal) { + String record = ++id + "," + taskId; + ctx.collect(record); + // Wait for the checkpoint to complete before sending the next record + waitNextCheckpoint = lastCheckpointConfirmed + 1; + synchronized (this.blocker) { + while (this.lastCheckpointConfirmed < waitNextCheckpoint) { + this.blocker.wait(); + } + } + } + } + + @Override + public void cancel() { + this.running = false; + } + + @Override + public void snapshotState(FunctionSnapshotContext context) throws Exception { + state.update(Collections.singletonList(id)); + if (failCheckpointId > 0 && context.getCheckpointId() % failCheckpointId == 0) { + throw new RuntimeException( + "Trigger fail for testing, checkpointId = " + context.getCheckpointId()); + } + LOG.info("snapshot state to {} for checkpoint {}", id, context.getCheckpointId()); + } + + @Override + public void initializeState(FunctionInitializationContext context) throws Exception { + state = + context.getOperatorStateStore() + .getListState( + new ListStateDescriptor<>("id", TypeInformation.of(Long.class))); + if (context.isRestored()) { + Iterator iterator = state.get().iterator(); + while (iterator.hasNext()) { + id += iterator.next(); + } + } + LOG.info("restore state from {}", id); + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception { + synchronized (blocker) { + this.lastCheckpointConfirmed = checkpointId; + blocker.notifyAll(); + } + LOG.info("checkpoint {} finished", checkpointId); + } +} diff --git a/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java new file mode 100644 index 000000000..adc70f5f5 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-flink2/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.utils; + +import org.apache.flink.streaming.api.functions.source.legacy.SourceFunction; + +public interface MockSourceFunction extends SourceFunction {} diff --git a/flink-doris-connector/flink-doris-connector-it/pom.xml b/flink-doris-connector/flink-doris-connector-it/pom.xml new file mode 100644 index 000000000..f6700dc5e --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-it/pom.xml @@ -0,0 +1,166 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector-parent + ${revision} + ../pom.xml + + + flink-doris-connector-it + Flink Doris Connector Integration Tests + Integration test cases for Flink Doris Connector. + + + + org.apache.doris + flink-doris-connector-${flink.major.version} + ${revision} + test + + + org.apache.doris + flink-doris-connector-${flink.major.version} + ${revision} + tests + test + + + org.apache.doris + flink-doris-connector-base + ${revision} + tests + test + + + org.apache.flink + flink-table-runtime + test + + + org.apache.flink + flink-table-planner-loader + test + + + org.apache.flink + flink-table-api-java-bridge + test + + + org.apache.flink + flink-connector-base + test + + + + + org.apache.flink + flink-test-utils + + + org.apache.flink + flink-connector-test-utils + + + org.apache.flink + flink-runtime-web + + + org.apache.flink + flink-table-common + ${flink.version} + test-jar + test + + + + + junit + junit + + + org.hamcrest + hamcrest-core + + + org.mockito + mockito-core + + + org.awaitility + awaitility + + + + + org.testcontainers + testcontainers + + + org.testcontainers + mysql + + + mysql + mysql-connector-java + + + com.google.protobuf + protobuf-java + + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-1.2-api + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + + + + + diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java similarity index 88% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java index fe6f72d17..e57081cfb 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java @@ -17,24 +17,19 @@ package org.apache.doris.flink.catalog; -import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Schema; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.catalog.CatalogBaseTable; import org.apache.flink.table.catalog.CatalogDatabase; import org.apache.flink.table.catalog.CatalogDatabaseImpl; import org.apache.flink.table.catalog.CatalogTable; -import org.apache.flink.table.catalog.CatalogTableImpl; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException; import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException; import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException; import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException; import org.apache.flink.table.catalog.exceptions.TableNotExistException; -import org.apache.flink.table.types.AtomicDataType; -import org.apache.flink.table.types.logical.VarCharType; import org.apache.flink.types.Row; import org.apache.flink.types.RowKind; import org.apache.flink.util.CollectionUtil; @@ -43,6 +38,7 @@ import org.apache.doris.flink.cfg.DorisConnectionOptions; import org.apache.doris.flink.container.AbstractITCaseService; import org.apache.doris.flink.table.DorisConfigOptions; +import org.apache.doris.flink.utils.CatalogUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; @@ -70,29 +66,6 @@ public class DorisCatalogITCase extends AbstractITCaseService { private static final String TEST_TABLE_SINK = "t_all_types_sink"; private static final String TEST_TABLE_SINK_GROUPBY = "t_all_types_sink_groupby"; - private static final TableSchema TABLE_SCHEMA = - TableSchema.builder() - .field("id", new AtomicDataType(new VarCharType(false, 128))) - .field("c_boolean", DataTypes.BOOLEAN()) - .field("c_char", DataTypes.CHAR(1)) - .field("c_date", DataTypes.DATE()) - .field("c_datetime", DataTypes.TIMESTAMP(0)) - .field("c_decimal", DataTypes.DECIMAL(10, 2)) - .field("c_double", DataTypes.DOUBLE()) - .field("c_float", DataTypes.FLOAT()) - .field("c_int", DataTypes.INT()) - .field("c_bigint", DataTypes.BIGINT()) - .field("c_largeint", DataTypes.STRING()) - .field("c_smallint", DataTypes.SMALLINT()) - .field("c_string", DataTypes.STRING()) - .field("c_tinyint", DataTypes.TINYINT()) - .field("c_array", DataTypes.ARRAY(DataTypes.INT())) - .field("c_map", DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING())) - .field("c_row", DataTypes.ROW()) - .field("c_varbinary", DataTypes.VARBINARY(16)) - .primaryKey("id") - .build(); - private static final List ALL_TYPES_ROWS = Lists.newArrayList( Row.ofKind( @@ -133,8 +106,10 @@ public class DorisCatalogITCase extends AbstractITCaseService { @Before public void setup() - throws DatabaseAlreadyExistException, TableAlreadyExistException, - TableNotExistException, DatabaseNotExistException { + throws DatabaseAlreadyExistException, + TableAlreadyExistException, + TableNotExistException, + DatabaseNotExistException { DorisConnectionOptions connectionOptions = new DorisConnectionOptions.DorisConnectionOptionsBuilder() .withFenodes(getFenodes()) @@ -247,7 +222,7 @@ public void testGetTable() throws TableNotExistException { CatalogBaseTable table = catalog.getTable(new ObjectPath(TEST_DB, TEST_TABLE)); Schema actual = table.getUnresolvedSchema(); assertEquals( - TABLE_SCHEMA.getFieldNames(), + CatalogUtil.getColumns(), actual.getColumns().stream().map(Schema.UnresolvedColumn::getName).toArray()); } @@ -281,16 +256,15 @@ public void testCreateTableDbNoExists() @Test public void testCreateTable() throws TableAlreadyExistException, DatabaseNotExistException { - CatalogTableImpl catalogTable = - new CatalogTableImpl( - TABLE_SCHEMA, + CatalogTable catalogTable = + CatalogUtil.createTable( + CatalogUtil.getTableSchema(), new HashMap() { { put("connector", "doris-1"); put("table.properties.replication_num", "1"); } - }, - "FlinkTable"); + }); catalog.createTable( new ObjectPath(TEST_DB, "create_table_wrong_connector"), catalogTable, true); boolean exists = @@ -399,14 +373,13 @@ private static CatalogDatabase createDb() { } private static CatalogTable createTable() { - return new CatalogTableImpl( - TABLE_SCHEMA, + return CatalogUtil.createTable( + CatalogUtil.getTableSchema(), new HashMap() { { put("connector", DorisConfigOptions.IDENTIFIER); put("table.properties.replication_num", "1"); } - }, - "FlinkTable"); + }); } } diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/CatalogExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/CatalogExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/CatalogExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/CatalogExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java similarity index 88% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java index debc81406..f51370773 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java @@ -19,12 +19,8 @@ import org.apache.flink.api.common.RuntimeExecutionMode; import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.common.restartstrategy.RestartStrategies; -import org.apache.flink.api.common.time.Time; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStreamSource; -import org.apache.flink.streaming.api.environment.CheckpointConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.doris.flink.cfg.DorisExecutionOptions; @@ -53,13 +49,13 @@ public class DorisIntranetAccessSinkExample { public static void main(String[] args) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); + // env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setRuntimeMode(RuntimeExecutionMode.BATCH); env.enableCheckpointing(10000); - env.getCheckpointConfig() - .enableExternalizedCheckpoints( - CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); - env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5, Time.milliseconds(30000))); + // env.getCheckpointConfig() + // .enableExternalizedCheckpoints( + // CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); + // env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5, Time.milliseconds(30000))); DorisSink.Builder builder = DorisSink.builder(); final DorisReadOptions.Builder readOptionBuilder = DorisReadOptions.builder(); diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java similarity index 98% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java index 7a111b258..bad354e0f 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java @@ -19,7 +19,6 @@ import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.doris.flink.cfg.DorisExecutionOptions; import org.apache.doris.flink.cfg.DorisOptions; @@ -28,6 +27,7 @@ import org.apache.doris.flink.sink.batch.DorisBatchSink; import org.apache.doris.flink.sink.writer.WriteMode; import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer; +import org.apache.doris.flink.utils.MockSourceFunction; import java.util.Arrays; import java.util.Properties; @@ -81,7 +81,7 @@ public static void main(String[] args) throws Exception { .setDorisOptions(dorisBuilder.build()); env.addSource( - new SourceFunction() { + new MockSourceFunction() { private Long id = 0L; @Override diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java similarity index 89% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java index bcb169656..bf88b3adb 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java @@ -19,12 +19,8 @@ import org.apache.flink.api.common.RuntimeExecutionMode; import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.common.restartstrategy.RestartStrategies; -import org.apache.flink.api.common.time.Time; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStreamSource; -import org.apache.flink.streaming.api.environment.CheckpointConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.doris.flink.cfg.DorisExecutionOptions; @@ -84,13 +80,8 @@ public static void JSONFormatWrite() throws Exception { public static void CSVFormatWrite() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setRuntimeMode(RuntimeExecutionMode.BATCH); env.enableCheckpointing(10000); - env.getCheckpointConfig() - .enableExternalizedCheckpoints( - CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); - env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5, Time.milliseconds(30000))); DorisSink.Builder builder = DorisSink.builder(); Properties properties = new Properties(); properties.setProperty("column_separator", ","); diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java similarity index 87% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java index e2b8dae9f..40f5b8718 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java @@ -17,12 +17,10 @@ package org.apache.doris.flink.example; -import org.apache.flink.api.common.restartstrategy.RestartStrategies; -import org.apache.flink.api.common.time.Time; +import org.apache.flink.configuration.CheckpointingOptions; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.environment.CheckpointConfig; +import org.apache.flink.configuration.RestartStrategyOptions; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction; import org.apache.doris.flink.cfg.DorisExecutionOptions; import org.apache.doris.flink.cfg.DorisOptions; @@ -30,7 +28,9 @@ import org.apache.doris.flink.sink.DorisSink; import org.apache.doris.flink.sink.batch.RecordWithMeta; import org.apache.doris.flink.sink.writer.serializer.RecordWithMetaSerializer; +import org.apache.doris.flink.utils.MockSourceFunction; +import java.time.Duration; import java.util.Properties; import java.util.UUID; @@ -38,13 +38,15 @@ public class DorisSinkStreamMultiTableExample { public static void main(String[] args) throws Exception { Configuration config = new Configuration(); // config.setString("execution.savepoint.path","/tmp/checkpoint/chk-6"); + config.set(CheckpointingOptions.CHECKPOINT_STORAGE, "filesystem"); + config.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, "file:///tmp/checkpoints"); + config.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay"); + config.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 3); + config.set( + RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY, + Duration.ofSeconds(10000)); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config); env.setParallelism(1); - env.getCheckpointConfig().setCheckpointStorage("file:///tmp/checkpoint/"); - env.getCheckpointConfig() - .enableExternalizedCheckpoints( - CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); - env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5, Time.milliseconds(10000))); env.enableCheckpointing(10000); DorisSink.Builder builder = DorisSink.builder(); final DorisReadOptions.Builder readOptionBuilder = DorisReadOptions.builder(); @@ -78,7 +80,7 @@ public static void main(String[] args) throws Exception { // stringDataStreamSource.sinkTo(builder.build()); env.addSource( - new ParallelSourceFunction() { + new MockSourceFunction() { private Long id = 1000000L; @Override diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java similarity index 65% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java index 90e461cfc..c19518905 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java @@ -33,25 +33,17 @@ public static void main(String[] args) throws Exception { // register a table in the catalog tEnv.executeSql( "CREATE TABLE doris_source (" - + "bigint_1 BIGINT," - + "char_1 STRING," - + "date_1 STRING," - + "datetime_1 STRING," - + "decimal_1 DECIMAL(5,2)," - + "double_1 DOUBLE," - + "float_1 FLOAT ," - + "int_1 INT ," - + "largeint_1 STRING, " - + "smallint_1 SMALLINT, " - + "tinyint_1 TINYINT, " - + "varchar_1 STRING " - + ") " - + "WITH (\n" - + " 'connector' = 'doris',\n" - + " 'fenodes' = 'FE_IP:8030',\n" - + " 'table.identifier' = 'db.table',\n" - + " 'username' = 'root',\n" - + " 'password' = ''\n" + + " id INT," + + " name STRING," + + " age INT" + + " ) " + + " WITH (" + + " 'connector' = 'doris'," + + " 'fenodes' = '127.0.0.1:8030'," + + " 'table.identifier' = 'test.student'," + + " 'username' = 'root'," + + " 'password' = ''," + + " 'sink.label-prefix' = 'doris_labe3l'" + ")"); // define a dynamic aggregating query diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java similarity index 96% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java index 9e82c6d41..1f63aa2fe 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java @@ -20,11 +20,12 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; +import org.apache.doris.flink.utils.MockSourceFunction; + import java.util.UUID; import static org.apache.flink.table.api.Expressions.$; @@ -40,7 +41,7 @@ public static void main(String[] args) throws Exception { DataStreamSource> source = env.addSource( - new SourceFunction>() { + new MockSourceFunction>() { private Integer id = 1; @Override diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java similarity index 99% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java index d14666a6c..401fd1ec3 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java @@ -19,7 +19,6 @@ import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Schema; import org.apache.flink.table.api.Table; @@ -31,6 +30,7 @@ import org.apache.doris.flink.container.AbstractITCaseService; import org.apache.doris.flink.container.ContainerUtils; import org.apache.doris.flink.table.DorisConfigOptions; +import org.apache.doris.flink.utils.MockSourceFunction; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -224,7 +224,7 @@ public void testLookupCache() throws Exception { DataStreamSource mockSource = env.addSource( - new SourceFunction() { + new MockSourceFunction() { @Override public void run(SourceContext ctx) throws Exception { diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkFailoverITCase.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/sink/DorisSinkFailoverITCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkFailoverITCase.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/sink/DorisSinkFailoverITCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java similarity index 97% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java index 76256543c..3d1345b88 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java @@ -19,8 +19,9 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.RuntimeExecutionMode; -import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestartStrategyOptions; import org.apache.flink.core.execution.JobClient; import org.apache.flink.runtime.minicluster.RpcServiceSharing; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; @@ -484,10 +485,13 @@ public void testTableGzFormat() throws Exception { public void testJobManagerFailoverSink() throws Exception { LOG.info("start to test JobManagerFailoverSink."); initializeFailoverTable(TABLE_CSV_JM, DataModel.DUPLICATE); - StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + Configuration conf = new Configuration(); + conf.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay"); + conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 3); + conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY, Duration.ofSeconds(0)); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf); env.setParallelism(DEFAULT_PARALLELISM); env.enableCheckpointing(10000); - env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0)); DorisSink.Builder builder = DorisSink.builder(); final DorisReadOptions.Builder readOptionBuilder = DorisReadOptions.builder(); @@ -549,10 +553,13 @@ public void testJobManagerFailoverSink() throws Exception { public void testTaskManagerFailoverSink() throws Exception { LOG.info("start to test TaskManagerFailoverSink."); initializeFailoverTable(TABLE_CSV_TM, DataModel.DUPLICATE); - StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + Configuration conf = new Configuration(); + conf.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay"); + conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 3); + conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY, Duration.ofSeconds(0)); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf); env.setParallelism(DEFAULT_PARALLELISM); env.enableCheckpointing(10000); - env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0)); DorisSink.Builder builder = DorisSink.builder(); final DorisReadOptions.Builder readOptionBuilder = DorisReadOptions.builder(); diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java similarity index 98% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java index f41469902..918165cc3 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java @@ -22,7 +22,6 @@ import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.doris.flink.cfg.DorisExecutionOptions; import org.apache.doris.flink.cfg.DorisOptions; @@ -32,6 +31,7 @@ import org.apache.doris.flink.sink.batch.RecordWithMeta; import org.apache.doris.flink.sink.writer.serializer.RecordWithMetaSerializer; import org.apache.doris.flink.utils.MockMultiTableSource; +import org.apache.doris.flink.utils.MockSourceFunction; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -113,7 +113,7 @@ public void testTableNotExistCornerCase() throws Exception { DataStreamSource mockSource = env.addSource( - new SourceFunction() { + new MockSourceFunction() { @Override public void run(SourceContext ctx) throws Exception { RecordWithMeta record3 = @@ -167,6 +167,9 @@ public void cancel() {} "select id,task_id from %s.%s ", DATABASE, TABLE_MULTI_CSV_NO_EXIST_TBL); List expected = Arrays.asList("1,3"); + // wait data VISIBLE + Thread.sleep(5000); + if (!batchMode) { ContainerUtils.checkResult( getDorisQueryConnection(), LOG, expected, queryRes, 2, false); diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java similarity index 98% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java index 31a94598e..a0834863c 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java +++ b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java @@ -20,7 +20,8 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.RuntimeExecutionMode; import org.apache.flink.api.common.eventtime.WatermarkStrategy; -import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestartStrategyOptions; import org.apache.flink.runtime.minicluster.RpcServiceSharing; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -48,6 +49,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -220,7 +222,7 @@ public void testTableSource() throws Exception { } } String[] expectedFilter = new String[] {"+I[doris, 18]"}; - checkResultInAnyOrder("testTableSource", expectedFilter, actualFilter.toArray()); + checkResultInAnyOrder("testTableSourceTest", expectedFilter, actualFilter.toArray()); } @Test @@ -675,10 +677,13 @@ public void testTableSourceFilterWithUnionAllNotEqualFilter() throws Exception { public void testJobManagerFailoverSource() throws Exception { LOG.info("start to test JobManagerFailoverSource."); initializeTableWithData(TABLE_CSV_JM, DataModel.DUPLICATE); - StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + Configuration conf = new Configuration(); + conf.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay"); + conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 3); + conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY, Duration.ofSeconds(0)); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf); env.setParallelism(DEFAULT_PARALLELISM); env.enableCheckpointing(200L); - env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0)); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); String sourceDDL = String.format( @@ -760,10 +765,13 @@ private static List getExpectedData() { public void testTaskManagerFailoverSource() throws Exception { LOG.info("start to test TaskManagerFailoverSource."); initializeTableWithData(TABLE_CSV_TM, DataModel.UNIQUE); - StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + Configuration conf = new Configuration(); + conf.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay"); + conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 3); + conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY, Duration.ofSeconds(0)); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf); env.setParallelism(DEFAULT_PARALLELISM); env.enableCheckpointing(200L); - env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0)); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); String sourceDDL = String.format( diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java b/flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java rename to flink-doris-connector/flink-doris-connector-it/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java diff --git a/flink-doris-connector/flink-doris-connector-it/src/test/resources/log4j2-test.properties b/flink-doris-connector/flink-doris-connector-it/src/test/resources/log4j2-test.properties new file mode 100644 index 000000000..22b66ef92 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-it/src/test/resources/log4j2-test.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.test.ref = TestLogger + +appender.testlogger.name = TestLogger +appender.testlogger.type = CONSOLE +appender.testlogger.layout.type = PatternLayout +appender.testlogger.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c [%t] %x - %m%n diff --git a/flink-doris-connector/pom.xml b/flink-doris-connector/pom.xml index c584ebbf2..6789e3c34 100644 --- a/flink-doris-connector/pom.xml +++ b/flink-doris-connector/pom.xml @@ -26,9 +26,10 @@ under the License. 23 org.apache.doris - flink-doris-connector-${flink.major.version} + flink-doris-connector-parent ${revision} - Flink Doris Connector + pom + Flink Doris Connector Parent https://doris.apache.org/ @@ -67,10 +68,8 @@ under the License. - 25.1.0-SNAPSHOT - 1.18.0 - 1.18 - 3.2.1 + 26.0.0-SNAPSHOT + 3.5.0 flink-python 0.16.0 3.10.1 @@ -79,9 +78,11 @@ under the License. UTF-8 github 1.0.1 + 8 + 8 8.14 2.17 - 2.4.2 + 2.27.1 4.5.13 1.15 2.13.3 @@ -99,334 +100,216 @@ under the License. 0.12.0 - - - org.apache.doris - thrift-service - ${thrift-service.version} - - - org.apache.flink - flink-table-planner-loader - ${flink.version} - provided - - - - org.apache.flink - flink-connector-base - ${flink.version} - provided - - - - org.apache.flink - flink-table-api-java-bridge - ${flink.version} - provided - - - - org.apache.flink - flink-table-runtime - ${flink.version} - provided - - - - org.apache.flink - ${flink.python.id} - ${flink.version} - provided - - - - org.apache.thrift - libthrift - ${libthrift.version} - - - httpclient - org.apache.httpcomponents - - - httpcore - org.apache.httpcomponents - - - - - - org.apache.httpcomponents - httpclient - ${httpcomponents.version} - - - commons-codec - commons-codec - - - - - - commons-codec - commons-codec - ${commons-codec.version} - - - org.apache.arrow.adbc - adbc-driver-flight-sql - ${adbc.version} - - - org.apache.arrow - flight-sql-jdbc-core - - - org.apache.arrow - flight-sql-jdbc-driver - - - org.apache.arrow - arrow-vector - - - org.apache.arrow.adbc - adbc-driver-manager - - - - - org.apache.arrow - arrow-vector - ${arrow.version} - - - org.apache.arrow - arrow-memory-netty - ${arrow.version} - runtime - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - - - - com.fasterxml.jackson.core - jackson-annotations - ${fasterxml.version} - - - - com.fasterxml.jackson.core - jackson-core - ${fasterxml.version} - - - - com.fasterxml.jackson.core - jackson-databind - ${fasterxml.version} - - - com.google.guava - guava - ${guava.version} - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-api - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - test - - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - test - - - - org.apache.flink - flink-sql-connector-mysql-cdc - ${flink.sql.cdc.version} - provided - - - flink-shaded-guava - org.apache.flink - - - - - org.apache.flink - flink-sql-connector-oracle-cdc - ${flink.sql.cdc.version} - provided - - - flink-shaded-guava - org.apache.flink - - - - - org.apache.flink - flink-sql-connector-postgres-cdc - ${flink.sql.cdc.version} - provided - - - flink-shaded-guava - org.apache.flink - - - - - org.apache.flink - flink-sql-connector-sqlserver-cdc - ${flink.sql.cdc.version} - provided - - - flink-shaded-guava - org.apache.flink - - - - - org.apache.flink - flink-sql-connector-db2-cdc - ${flink.sql.cdc.version} - provided - - - flink-shaded-guava - org.apache.flink - - - - - org.apache.flink - flink-sql-connector-mongodb-cdc - - ${flink.sql.cdc.version} - provided - - - flink-shaded-guava - org.apache.flink - - - - - - mysql - mysql-connector-java - ${mysql.driver.version} - test - - - com.oracle.ojdbc - ojdbc8 - ${ojdbc.version} - provided - - - - org.apache.flink - flink-runtime-web - ${flink.version} - test - - - - org.hamcrest - hamcrest-core - ${hamcrest.version} - test - - - - org.mockito - mockito-core - ${mockito.version} - test - - - org.mockito - mockito-inline - ${mockito.version} - test - - - - junit - junit - ${junit.version} - test - - - org.testcontainers - testcontainers - ${testcontainers.version} - test - - - org.testcontainers - mysql - ${testcontainers.version} - test - - - org.apache.flink - flink-table-common - ${flink.version} - test-jar - test - - - org.apache.flink - flink-test-utils - ${flink.version} - test - - - org.apache.flink - flink-connector-test-utils - ${flink.version} - test - - - com.github.jsqlparser - jsqlparser - ${jsqlparser.version} - - + + flink-doris-connector-base + flink-doris-connector-it + + + + + + org.apache.doris + thrift-service + ${thrift-service.version} + + + + + org.apache.flink + flink-table-planner-loader + ${flink.version} + provided + + + org.apache.flink + flink-connector-base + ${flink.version} + provided + + + org.apache.flink + flink-table-api-java-bridge + ${flink.version} + provided + + + org.apache.flink + flink-table-runtime + ${flink.version} + provided + + + org.apache.flink + ${flink.python.id} + ${flink.version} + provided + + + + + org.apache.doris + flink-doris-connector-base + ${revision} + + + + org.apache.thrift + libthrift + ${libthrift.version} + + + org.apache.httpcomponents + httpclient + ${httpcomponents.version} + + + commons-codec + commons-codec + ${commons-codec.version} + + + org.apache.arrow.adbc + adbc-driver-flight-sql + ${adbc.version} + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${fasterxml.version} + + + com.fasterxml.jackson.core + jackson-core + ${fasterxml.version} + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.version} + + + com.google.guava + guava + ${guava.version} + + + com.github.jsqlparser + jsqlparser + ${jsqlparser.version} + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + test + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + test + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + test + + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + test + + + junit + junit + ${junit.version} + test + + + org.hamcrest + hamcrest-core + ${hamcrest.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.mockito + mockito-inline + ${mockito.version} + test + + + org.apache.flink + flink-runtime-web + ${flink.version} + test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + + + org.testcontainers + mysql + ${testcontainers.version} + test + + + org.apache.flink + flink-table-common + ${flink.version} + test-jar + test + + + org.apache.flink + flink-test-utils + ${flink.version} + test + + + org.apache.flink + flink-connector-test-utils + ${flink.version} + test + + + org.awaitility + awaitility + 4.2.0 + test + + + mysql + mysql-connector-java + ${mysql.driver.version} + test + + + @@ -445,65 +328,13 @@ under the License. - - org.apache.maven.plugins - maven-shade-plugin - 3.4.1 - - - - org.apache.arrow - org.apache.doris.shaded.org.apache.arrow - - - io.netty - org.apache.doris.shaded.io.netty - - - com.fasterxml.jackson - org.apache.doris.shaded.com.fasterxml.jackson - - - org.apache.commons.codec - org.apache.doris.shaded.org.apache.commons.codec - - - com.google - org.apache.doris.shaded.com.google - - - org.apache.thrift - org.apache.doris.shaded.org.apache.thrift - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - shade - - package - - - - org.apache.maven.plugins maven-compiler-plugin ${maven-compiler-plugin.version} - 8 - 8 + ${maven.compiler.source} + ${maven.compiler.target} @@ -513,7 +344,6 @@ under the License. ${maven-javadoc-plugin.version} true - 8 false @@ -581,6 +411,7 @@ under the License. **/*.java --> + ${test.jvm.args} @@ -594,7 +425,7 @@ under the License. - 1.7 + 1.24.0 @@ -652,10 +483,77 @@ under the License. maven-compiler-plugin - + + flink1 + + true + + + 1.20.0 + 1.20 + 8 + 8 + + + flink-doris-connector-flink1 + + + + flink2 + + 2.2.0 + 2.2 + 17 + 17 + + + flink-doris-connector-flink2 + + + + + java-8-target + + [1.8,11) + + + + + + com.diffplug.spotless + spotless-maven-plugin + + true + + + + + + + + java-17-target + + [17,) + + + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.sql/java.sql=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + + + + + custom-env @@ -695,6 +593,7 @@ under the License. + apache-release @@ -727,5 +626,4 @@ under the License. - diff --git a/tools/releasing/deploy_staging_jars.sh b/tools/releasing/deploy_staging_jars_for_jdk17.sh similarity index 65% rename from tools/releasing/deploy_staging_jars.sh rename to tools/releasing/deploy_staging_jars_for_jdk17.sh index a66d2a1ce..4cdf10a72 100644 --- a/tools/releasing/deploy_staging_jars.sh +++ b/tools/releasing/deploy_staging_jars_for_jdk17.sh @@ -39,27 +39,27 @@ fi ########################### -cd ${PROJECT_ROOT}/flink-doris-connector - -echo "Deploying to repository.apache.org" - -echo "Deploying Flink 1.15..." -${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.15.0 -Dflink.major.version=1.15 -Dflink.python.id=flink-python_2.12 -DskipTests=true +# Sanity check to ensure that current JDK is version 17 +JAVA_VERSION=$(java -version 2>&1 | awk -F'"' '/version/ {print $2}') +if [[ "${JAVA_VERSION}" != 17* ]]; then + echo "Error: This script requires JDK 17, but found version '${JAVA_VERSION}'." + echo "Please switch to JDK 17 before running this script." + exit 1 +fi +echo "JDK version check passed: ${JAVA_VERSION}" -echo "Deploying Flink 1.16..." -${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.16.0 -Dflink.major.version=1.16 -DskipTests=true +cd ${PROJECT_ROOT}/flink-doris-connector -echo "Deploying Flink 1.17..." -${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.17.0 -Dflink.major.version=1.17 -DskipTests=true +echo "Deploying to repository.apache.org for Flink 2.x" -echo "Deploying Flink 1.18..." -${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.18.0 -Dflink.major.version=1.18 -DskipTests=true +echo "Deploying Flink 2.0..." +${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-flink2 -am -Pflink2 -Dflink.version=2.0.0 -Dflink.major.version=2.0 -DskipTests=true -echo "Deploying Flink 1.19..." -${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.19.0 -Dflink.major.version=1.19 -DskipTests=true +echo "Deploying Flink 2.1..." +${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-flink2 -am -Pflink2 -Dflink.version=2.1.0 -Dflink.major.version=2.1 -DskipTests=true -echo "Deploying Flink 1.20..." -${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.20.0 -Dflink.major.version=1.20 -DskipTests=true +echo "Deploying Flink 2.2..." +${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-flink2 -am -Pflink2 -Dflink.version=2.2.0 -Dflink.major.version=2.2 -DskipTests=true -echo "Deploy jar finished." +echo "Deploy jar with jdk17 finished." cd ${CURR_DIR} \ No newline at end of file diff --git a/tools/releasing/deploy_staging_jars_for_jdk8.sh b/tools/releasing/deploy_staging_jars_for_jdk8.sh new file mode 100644 index 000000000..7e01d7bb0 --- /dev/null +++ b/tools/releasing/deploy_staging_jars_for_jdk8.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +############################################################## +# This script is deploy stage jars to repository.apache.org +############################################################## + +MVN=${MVN:-mvn} +CUSTOM_OPTIONS=${CUSTOM_OPTIONS:-} + +# fail immediately +set -o errexit +set -o nounset + +CURR_DIR=`pwd` +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +PROJECT_ROOT="${BASE_DIR}/../../" + +# Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root +if [ ! -f ${PROJECT_ROOT}/LICENSE.txt ]; then + echo "Project root path ${PROJECT_ROOT} is not valid; script may be in the wrong directory." + exit 1 +fi + +########################### + +# Sanity check to ensure that current JDK is version 1.8 +JAVA_VERSION=$(java -version 2>&1 | awk -F'"' '/version/ {print $2}') +if [[ "${JAVA_VERSION}" != 1.8* ]]; then + echo "Error: This script requires JDK 1.8, but found version '${JAVA_VERSION}'." + echo "Please switch to JDK 1.8 before running this script." + exit 1 +fi +echo "JDK version check passed: ${JAVA_VERSION}" + +cd ${PROJECT_ROOT}/flink-doris-connector + +echo "Deploying to repository.apache.org for Flink 1.x" + +echo "Deploying Flink 1.15..." +${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-flink1 -am -Pflink1 -Dflink.version=1.15.0 -Dflink.major.version=1.15 -Dflink.python.id=flink-python_2.12 -DskipTests=true + +echo "Deploying Flink 1.16..." +${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-flink1 -am -Pflink1 -Dflink.version=1.16.0 -Dflink.major.version=1.16 -DskipTests=true + +echo "Deploying Flink 1.17..." +${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-flink1 -am -Pflink1 -Dflink.version=1.17.0 -Dflink.major.version=1.17 -DskipTests=true + +echo "Deploying Flink 1.18..." +${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-flink1 -am -Pflink1 -Dflink.version=1.18.0 -Dflink.major.version=1.18 -DskipTests=true + +echo "Deploying Flink 1.19..." +${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-flink1 -am -Pflink1 -Dflink.version=1.19.0 -Dflink.major.version=1.19 -DskipTests=true + +echo "Deploying Flink 1.20..." +${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-flink1 -am -Pflink1 -Dflink.version=1.20.0 -Dflink.major.version=1.20 -DskipTests=true + +echo "Deploy jar with jdk1.8 finished." +cd ${CURR_DIR} \ No newline at end of file diff --git a/tools/releasing/update_branch_version.sh b/tools/releasing/update_branch_version.sh index 7022f3f81..d0474f550 100644 --- a/tools/releasing/update_branch_version.sh +++ b/tools/releasing/update_branch_version.sh @@ -42,8 +42,15 @@ if [ ! -f ${PROJECT_ROOT}/LICENSE.txt ]; then fi cd ${PROJECT_ROOT}/flink-doris-connector -mvn versions:set -DgenerateBackupPoms=false -DnewVersion=${RELEASE_VERSION} -mvn versions:set-property -DgenerateBackupPoms=false -Dproperty=revision -DnewVersion=${RELEASE_VERSION} + +# update version +mvn versions:set -DgenerateBackupPoms=false -DnewVersion=${RELEASE_VERSION} -Pflink1,flink2 + +# versions:set because of artifactId duplicate problem cannot identify flink1, manually update +sed -i '' "s/\${revision}/${RELEASE_VERSION}/g" flink-doris-connector-flink1/pom.xml + +# update revision property in parent pom +mvn versions:set-property -DgenerateBackupPoms=false -Dproperty=revision -DnewVersion=${RELEASE_VERSION} -Pflink1 git commit -am "[release] Update version to ${RELEASE_VERSION}"