test: integration test kafka #3261
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests | |
| on: | |
| push: | |
| branches: | |
| - "master" | |
| paths: | |
| - '**/*.go' | |
| - '**/*.java' | |
| pull_request: | |
| branches: | |
| - "*" | |
| paths: | |
| - '**/*.go' | |
| - '**/*.java' | |
| env: | |
| MYSQL_ROOT_PASSWORD: root1234 | |
| jobs: | |
| integration-tests: | |
| environment: integration_tests | |
| runs-on: 32gb-runner | |
| timeout-minutes: 45 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version-file: "go.mod" | |
| - name: Set up Java for Maven | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| - name: Install DB2 CLI Driver | |
| run: | | |
| go run github.com/ibmdb/go_ibm_db/installer@v0.5.4 | |
| echo "IBM_DB_HOME=$(go env GOPATH)/pkg/mod/github.com/ibmdb/clidriver" >> $GITHUB_ENV | |
| IBM_DB_HOME=$(go env GOPATH)/pkg/mod/github.com/ibmdb/clidriver | |
| echo "CGO_CFLAGS=-I$IBM_DB_HOME/include" >> $GITHUB_ENV | |
| echo "CGO_LDFLAGS=-L$IBM_DB_HOME/lib -Wl,-rpath,$IBM_DB_HOME/lib" >> $GITHUB_ENV | |
| echo "LD_LIBRARY_PATH=$IBM_DB_HOME/lib" >> $GITHUB_ENV | |
| - name: Start Test Infrastructure | |
| run: | | |
| docker compose -f ./drivers/mysql/docker-compose.yml up -d | |
| docker compose -f ./drivers/postgres/docker-compose.yml up -d | |
| docker compose -f ./drivers/mongodb/docker-compose.yml up -d | |
| docker compose -f ./drivers/oracle/docker-compose.yml up -d | |
| docker compose -f ./drivers/db2/docker-compose.yml up -d | |
| docker compose -f ./drivers/mssql/docker-compose.yml up -d | |
| docker compose -f ./drivers/kafka/docker-compose.yml up -d | |
| docker compose -f ./destination/iceberg/local-test/docker-compose.yml up minio mc postgres spark-iceberg -d | |
| - name: Wait for MySQL | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 5 | |
| max_attempts: 30 | |
| retry_wait_seconds: 5 | |
| command: | | |
| docker exec olake_mysql-test mysql -h localhost -u root -p${{ env.MYSQL_ROOT_PASSWORD }} -e "SELECT 1" | |
| - name: Wait for PostgreSQL | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 5 | |
| max_attempts: 30 | |
| retry_wait_seconds: 5 | |
| command: | | |
| docker exec olake_postgres-test psql -h localhost -U postgres -d postgres -c "SELECT 1" | |
| - name: Wait for MongoDB | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 5 | |
| max_attempts: 30 | |
| retry_wait_seconds: 5 | |
| command: | | |
| docker exec primary_mongo mongosh --host localhost --port 27017 -u mongodb -p secure_password123 --authenticationDatabase admin --eval "db.adminCommand('ping')" | |
| - name: Wait for Oracle | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 5 | |
| max_attempts: 30 | |
| retry_wait_seconds: 5 | |
| command: | | |
| docker exec oracle-23c bash -c "echo 'SELECT 1 FROM dual;' | sqlplus -s system/secret1234@//localhost:1521/ORCL" | |
| - name: Wait for DB2 | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 30 | |
| retry_wait_seconds: 25 | |
| command: | | |
| docker exec db2-test bash -c "su - db2inst1 -c 'db2 connect to TESTDB'" | |
| - name: Wait for Kafka | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 5 | |
| max_attempts: 30 | |
| retry_wait_seconds: 5 | |
| command: | | |
| docker exec kafka kafka-topics --bootstrap-server localhost:9092 --list | |
| - name: Wait for MSSQL | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 5 | |
| max_attempts: 30 | |
| retry_wait_seconds: 5 | |
| command: | | |
| docker exec olake-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Password!123' -C -Q "SELECT 1" | |
| - name: Initialize MSSQL Database | |
| run: | | |
| docker exec olake-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Password!123' -C -d master -i /docker-entrypoint-initdb.d/01-init.sql | |
| - name: Set up Data Directories | |
| run: | | |
| sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/postgres-data | |
| sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/minio-data | |
| sudo mkdir -p /home/runner/work/olake/olake/destination/iceberg/local-test/data/ivy-cache | |
| sudo chown -R 999:999 /home/runner/work/olake/olake/destination/iceberg/local-test/data | |
| sudo chmod -R 777 /home/runner/work/olake/olake/destination/iceberg/local-test/data | |
| - name: Install Go Dependencies | |
| run: go mod download | |
| - name: Build Project | |
| run: go build -v ./... | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| - name: Build Iceberg Sink | |
| working-directory: ./destination/iceberg/olake-iceberg-java-writer | |
| run: mvn clean package -DskipTests | |
| - name: Run Integration Tests | |
| run: | | |
| go test -v -p 7 ./drivers/kafka/internal/... ./drivers/mysql/internal/... ./drivers/postgres/internal/... ./drivers/mongodb/internal/... ./drivers/oracle/internal/... ./drivers/db2/internal/... ./drivers/mssql/internal/... -timeout 0 -run 'Integration' | |
| - name: Cleanup | |
| if: always() | |
| run: | | |
| docker compose -f ./destination/iceberg/local-test/docker-compose.yml down | |
| docker compose -f ./drivers/mysql/docker-compose.yml down | |
| docker compose -f ./drivers/postgres/docker-compose.yml down | |
| docker compose -f ./drivers/oracle/docker-compose.yml down | |
| docker compose -f ./drivers/mongodb/docker-compose.yml down | |
| docker compose -f ./drivers/db2/docker-compose.yml down | |
| docker compose -f ./drivers/mssql/docker-compose.yml down | |
| docker compose -f ./drivers/kafka/docker-compose.yml down |