chore: release version 0.7.4

.github/workflows/2pc-integration-tests.yml

@@ -0,0 +1,21 @@
+name: 2PC Integration Tests
+
+on:
+  push:
+    branches:
+      - "master"
+    paths:
+      - '**/*.go'
+      - '**/*.java'
+  pull_request:
+    branches:
+      - "*"
+    paths:
+      - '**/*.go'
+      - '**/*.java'
+
+jobs:
+  twopc-integration-tests:
+    uses: ./.github/workflows/integration-tests-runner.yml
+    with:
+      test_type: 2pc

.github/workflows/integration-tests-runner.yml

@@ -0,0 +1,179 @@
+name: Integration Test Runner
+
+on:
+  workflow_call:
+    inputs:
+      test_type:
+        description: 'Type of tests to run: integration or 2pc'
+        required: true
+        type: string
+
+env:
+  MYSQL_ROOT_PASSWORD: root1234
+
+jobs:
+  run-tests:
+    name: ${{ inputs.test_type == '2pc' && '2PC Integration Tests' || 'Integration Tests' }}
+    environment: integration_tests
+    runs-on: 32gb-runner
+    timeout-minutes: 45
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: "go.mod"
+
+      - name: Set up Java for Maven
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: Install DB2 CLI Driver
+        run: |
+          go run github.com/ibmdb/go_ibm_db/installer@v0.5.4
+          cp -r "$(go env GOPATH)/pkg/mod/github.com/ibmdb/clidriver" ./clidriver
+          IBM_DB_HOME="$(pwd)/clidriver"
+          echo "IBM_DB_HOME=$IBM_DB_HOME" >> $GITHUB_ENV
+          echo "CGO_CFLAGS=-I$IBM_DB_HOME/include" >> $GITHUB_ENV
+          echo "CGO_LDFLAGS=-L$IBM_DB_HOME/lib -Wl,-rpath,$IBM_DB_HOME/lib" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=$IBM_DB_HOME/lib" >> $GITHUB_ENV
+
+      - name: Start Test Infrastructure
+        run: |
+          docker compose -f ./drivers/mysql/docker-compose.yml up -d
+          docker compose -f ./drivers/postgres/docker-compose.yml up -d
+          docker compose -f ./drivers/mongodb/docker-compose.yml up -d
+          docker compose -f ./drivers/oracle/docker-compose.yml up -d
+          docker compose -f ./drivers/db2/docker-compose.yml up -d
+          docker compose -f ./drivers/mssql/docker-compose.yml up -d
+          docker compose -f ./destination/iceberg/local-test/docker-compose.yml up minio mc postgres spark-iceberg -d
+
+      - name: Start Kafka
+        if: inputs.test_type == 'integration'
+        run: docker compose -f ./drivers/kafka/docker-compose.yml up -d
+
+      - name: Wait for MySQL
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 5
+          max_attempts: 30
+          retry_wait_seconds: 5
+          command: |
+            docker exec olake_mysql-test mysql -h localhost -u root -p${{ env.MYSQL_ROOT_PASSWORD }} -e "SELECT 1"
+
+      - name: Wait for PostgreSQL
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 5
+          max_attempts: 30
+          retry_wait_seconds: 5
+          command: |
+            docker exec olake_postgres-test psql -h localhost -U postgres -d postgres -c "SELECT 1"
+
+      - name: Wait for MongoDB
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 5
+          max_attempts: 30
+          retry_wait_seconds: 5
+          command: |
+            docker exec primary_mongo mongosh --host localhost --port 27017 -u mongodb -p secure_password123 --authenticationDatabase admin --eval "db.adminCommand('ping')"
+
+      - name: Wait for Oracle
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 5
+          max_attempts: 30
+          retry_wait_seconds: 5
+          command: |
+            docker exec oracle-23c bash -c "echo 'SELECT 1 FROM dual;' | sqlplus -s system/secret1234@//localhost:1521/ORCL"
+
+      - name: Wait for DB2
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 10
+          max_attempts: 30
+          retry_wait_seconds: 25
+          command: |
+            docker exec db2-test bash -c "su - db2inst1 -c 'db2 connect to TESTDB'"
+
+      - name: Wait for Kafka
+        if: inputs.test_type == 'integration'
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 5
+          max_attempts: 30
+          retry_wait_seconds: 5
+          command: |
+            docker exec kafkaJson kafka-topics --bootstrap-server localhost:9092 --list
+            docker exec kafkaAvro kafka-topics --bootstrap-server localhost:9092 --list
+            curl -f http://localhost:8081/subjects
+
+      - name: Wait for MSSQL
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 5
+          max_attempts: 30
+          retry_wait_seconds: 5
+          command: |
+            docker exec olake-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Password!123' -C -Q "SELECT 1"
+
+      - name: Initialize MSSQL Database
+        run: |
+          docker exec olake-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Password!123' -C -d master -i /docker-entrypoint-initdb.d/01-init.sql
+
+      - name: Set up Data Directories
+        run: |
+          sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/postgres-data
+          sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/minio-data
+          sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/ivy-cache
+          sudo chown -R 999:999 /home/runner/work/olake/olake/destination/iceberg/local-test/data
+          sudo chmod -R 777 /home/runner/work/olake/olake/destination/iceberg/local-test/data
+
+      - name: Install Go Dependencies
+        run: go mod download
+
+      - name: Build Project
+        run: go build -v ./...
+
+      - name: Cache Maven dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: Build Iceberg Sink
+        working-directory: ./destination/iceberg/olake-iceberg-java-writer
+        run: mvn clean package -DskipTests
+
+      - name: Run Integration Tests
+        if: inputs.test_type == 'integration'
+        run: |
+          go test -v -p 7 ./drivers/kafka/internal/... ./drivers/mysql/internal/... ./drivers/postgres/internal/... ./drivers/mongodb/internal/... ./drivers/oracle/internal/... ./drivers/db2/internal/... ./drivers/mssql/internal/... -timeout 0 -run 'Integration'
+
+      - name: Run 2PC Integration Tests
+        if: inputs.test_type == '2pc'
+        run: |
+          go test -v -p 6 ./drivers/mysql/internal/... ./drivers/postgres/internal/... ./drivers/mongodb/internal/... ./drivers/mssql/internal/... ./drivers/oracle/internal/... ./drivers/db2/internal/... -timeout 0 -run '2PC'
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker compose -f ./destination/iceberg/local-test/docker-compose.yml down
+          docker compose -f ./drivers/mysql/docker-compose.yml down
+          docker compose -f ./drivers/postgres/docker-compose.yml down
+          docker compose -f ./drivers/oracle/docker-compose.yml down
+          docker compose -f ./drivers/mongodb/docker-compose.yml down
+          docker compose -f ./drivers/db2/docker-compose.yml down
+          docker compose -f ./drivers/mssql/docker-compose.yml down
+
+      - name: Cleanup Kafka
+        if: always() && inputs.test_type == 'integration'
+        run: docker compose -f ./drivers/kafka/docker-compose.yml down

.github/workflows/integration-tests.yml

@@ -1,4 +1,5 @@
 name: Integration Tests
+
 on:
   push:
     branches:
@@ -13,159 +14,8 @@ on:
       - '**/*.go'
       - '**/*.java'
 
-env:
-  MYSQL_ROOT_PASSWORD: root1234
-
 jobs:
   integration-tests:
-    environment: integration_tests
-    runs-on: 32gb-runner
-    timeout-minutes: 45
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version-file: "go.mod"
-
-      - name: Set up Java for Maven
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'temurin'
-          java-version: '17'
-
-      - name: Install DB2 CLI Driver
-        run: |
-          go run github.com/ibmdb/go_ibm_db/installer@v0.5.4
-          cp -r "$(go env GOPATH)/pkg/mod/github.com/ibmdb/clidriver" ./clidriver
-          IBM_DB_HOME="$(pwd)/clidriver"
-          echo "IBM_DB_HOME=$IBM_DB_HOME" >> $GITHUB_ENV
-          echo "CGO_CFLAGS=-I$IBM_DB_HOME/include" >> $GITHUB_ENV
-          echo "CGO_LDFLAGS=-L$IBM_DB_HOME/lib -Wl,-rpath,$IBM_DB_HOME/lib" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=$IBM_DB_HOME/lib" >> $GITHUB_ENV
-
-      - name: Start Test Infrastructure
-        run: |
-          docker compose -f ./drivers/mysql/docker-compose.yml up -d
-          docker compose -f ./drivers/postgres/docker-compose.yml up -d
-          docker compose -f ./drivers/mongodb/docker-compose.yml up -d
-          docker compose -f ./drivers/oracle/docker-compose.yml up -d
-          docker compose -f ./drivers/db2/docker-compose.yml up -d
-          docker compose -f ./drivers/mssql/docker-compose.yml up -d
-          docker compose -f ./drivers/kafka/docker-compose.yml up -d
-          docker compose -f ./destination/iceberg/local-test/docker-compose.yml up minio mc postgres spark-iceberg -d
-
-      - name: Wait for MySQL
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 5
-          max_attempts: 30
-          retry_wait_seconds: 5
-          command: |
-            docker exec olake_mysql-test mysql -h localhost -u root -p${{ env.MYSQL_ROOT_PASSWORD }} -e "SELECT 1"
-
-      - name: Wait for PostgreSQL
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 5
-          max_attempts: 30
-          retry_wait_seconds: 5
-          command: |
-            docker exec olake_postgres-test psql -h localhost -U postgres -d postgres -c "SELECT 1"
-
-      - name: Wait for MongoDB
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 5
-          max_attempts: 30
-          retry_wait_seconds: 5
-          command: |
-            docker exec primary_mongo mongosh --host localhost --port 27017 -u mongodb -p secure_password123 --authenticationDatabase admin --eval "db.adminCommand('ping')"
-
-      - name: Wait for Oracle
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 5
-          max_attempts: 30
-          retry_wait_seconds: 5
-          command: |
-            docker exec oracle-23c bash -c "echo 'SELECT 1 FROM dual;' | sqlplus -s system/secret1234@//localhost:1521/ORCL"
-
-      - name: Wait for DB2
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 30
-          retry_wait_seconds: 25
-          command: |
-            docker exec db2-test bash -c "su - db2inst1 -c 'db2 connect to TESTDB'"
-
-      - name: Wait for Kafka
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 5
-          max_attempts: 30
-          retry_wait_seconds: 5
-          command: |
-            docker exec kafkaJson kafka-topics --bootstrap-server localhost:9092 --list
-            docker exec kafkaAvro kafka-topics --bootstrap-server localhost:9092 --list
-            curl -f http://localhost:8081/subjects
-
-      - name: Wait for MSSQL
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 5
-          max_attempts: 30
-          retry_wait_seconds: 5
-          command: |
-            docker exec olake-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Password!123' -C -Q "SELECT 1"
-
-      - name: Initialize MSSQL Database
-        run: |
-          docker exec olake-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Password!123' -C -d master -i /docker-entrypoint-initdb.d/01-init.sql
-
-
-      - name: Set up Data Directories
-        run: |
-            sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/postgres-data
-            sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/minio-data
-            sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/ivy-cache
-            sudo chown -R 999:999 /home/runner/work/olake/olake/destination/iceberg/local-test/data
-            sudo chmod -R 777 /home/runner/work/olake/olake/destination/iceberg/local-test/data
-
-      - name: Install Go Dependencies
-        run: go mod download
-
-      - name: Build Project
-        run: go build -v ./...
-
-      - name: Cache Maven dependencies
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
-
-      - name: Build Iceberg Sink
-        working-directory: ./destination/iceberg/olake-iceberg-java-writer
-        run: mvn clean package -DskipTests
-
-      - name: Run Integration Tests
-        run: |  
-          go test -v -p 7 ./drivers/kafka/internal/... ./drivers/mysql/internal/... ./drivers/postgres/internal/... ./drivers/mongodb/internal/... ./drivers/oracle/internal/... ./drivers/db2/internal/... ./drivers/mssql/internal/... -timeout 0 -run 'Integration'
-
-      - name: Cleanup
-        if: always()
-        run: |
-          docker compose -f ./destination/iceberg/local-test/docker-compose.yml down
-          docker compose -f ./drivers/mysql/docker-compose.yml down
-          docker compose -f ./drivers/postgres/docker-compose.yml down
-          docker compose -f ./drivers/oracle/docker-compose.yml down
-          docker compose -f ./drivers/mongodb/docker-compose.yml down
-          docker compose -f ./drivers/db2/docker-compose.yml down
-          docker compose -f ./drivers/mssql/docker-compose.yml down
-          docker compose -f ./drivers/kafka/docker-compose.yml down
+    uses: ./.github/workflows/integration-tests-runner.yml
+    with:
+      test_type: integration