handle clickhouse znode error #21036
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Flow build and test | |
| on: | |
| pull_request: | |
| branches: [main] | |
| # Ignore changes to files that don't require builds or validations. | |
| # If more complex build avoidance is required, or for other events than pull_request, push, and pull_request_target, | |
| # replace this with a gate step in the job. | |
| # TODO: Re-enable build avoidance once we figure out how not to block the merge on the flow checks. | |
| #paths-ignore: &ignore_paths | |
| # - '.claude/**' | |
| # - 'README.md' | |
| push: | |
| branches: [main] | |
| #paths-ignore: *ignore_paths | |
| pull_request_target: | |
| branches: [main] | |
| #paths-ignore: *ignore_paths | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| flow_test: | |
| name: flow_test (${{ matrix.runner }}, ${{ matrix.db-version.pg }}, ${{ matrix.db-version.mysql }}, ${{ matrix.db-version.mongo }}, ${{ matrix.db-version.ch }}) | |
| if: | | |
| (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || | |
| (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) || | |
| github.event_name == 'push' | |
| environment: ${{ (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) && 'external-contributor' || null }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: [ubuntu-latest-16-cores] | |
| db-version: [ | |
| {pg: 16, mysql: 'mysql-gtid', mariadb: 'maria-11', mongo: '6.0', ch: 'lts'}, | |
| {pg: 17, mysql: 'mysql-pos', mariadb: 'maria-12', mongo: '7.0', ch: 'stable'}, | |
| {pg: 18, mysql: 'mysql-gtid', mariadb: 'maria-13', mongo: '8.0', ch: 'latest'}, | |
| ] | |
| # Per-version container settings consumed by the MySQL and MariaDB steps. | |
| # Wrapped in a single-item list because matrix values must be arrays; it stays | |
| # a single shared value (no extra jobs). | |
| version-configs: | |
| - mysql: | |
| mysql-gtid: | |
| img: 'mysql:9.5' | |
| env: ['MYSQL_ROOT_PASSWORD=cipass'] | |
| parameters: [] | |
| mysql-pos: | |
| img: 'mysql:5.7' | |
| env: ['MYSQL_ROOT_PASSWORD=cipass'] | |
| parameters: ['--log_bin=mysql-bin', '--server-id=1', '--bind-address=::'] | |
| mariadb: | |
| maria-11: | |
| img: 'mariadb:lts-ubi9' | |
| env: ['MARIADB_ROOT_PASSWORD=cipass'] | |
| parameters: ['--log-bin=maria', '--gtid-strict-mode=ON'] | |
| maria-12: | |
| img: 'mariadb:12-ubi' | |
| env: ['MARIADB_ROOT_PASSWORD=cipass'] | |
| parameters: ['--log-bin=maria', '--gtid-strict-mode=ON'] | |
| maria-13: | |
| img: 'mariadb:13.0-ubi-rc' | |
| env: ['MARIADB_ROOT_PASSWORD=cipass'] | |
| parameters: ['--log-bin=maria', '--gtid-strict-mode=ON'] | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 30 | |
| services: | |
| catalog: | |
| image: imresamu/postgis:${{ matrix.db-version.pg }}-3.5-alpine | |
| ports: | |
| - 5432:5432 | |
| env: | |
| PGUSER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_INITDB_ARGS: --locale=C.UTF-8 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| catalog2: | |
| image: imresamu/postgis:${{ matrix.db-version.pg }}-3.5-alpine | |
| ports: | |
| - 5437:5432 | |
| env: | |
| PGUSER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_INITDB_ARGS: --locale=C.UTF-8 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| redpanda: | |
| image: redpandadata/redpanda@sha256:3e72ea35731a893bdd3b6d283eb554416377e23766434fc85dd7941540673639 | |
| ports: | |
| - 9092:9092 | |
| - 9644:9644 | |
| elasticsearch: | |
| image: elasticsearch:9.4.2@sha256:e182d810a437e30ad7d0221fc153910a06d9be5b05e57e59e7983467b89e41e2 | |
| ports: | |
| - 9200:9200 | |
| env: | |
| discovery.type: single-node | |
| xpack.security.enabled: false | |
| xpack.security.enrollment.enabled: false | |
| otelcol: | |
| image: otel/opentelemetry-collector-contrib:0.155.0@sha256:4935caa35e9a4cb387e35732e8fb22b2b5759af8d12e7043357f03837f6e8df5 | |
| ports: | |
| - 4317:4317 | |
| toxiproxy: | |
| image: ghcr.io/shopify/toxiproxy:2.12.0@sha256:9378ed52a28bc50edc1350f936f518f31fa95f0d15917d6eb40b8e376d1a214e | |
| ports: | |
| - 18474:8474 | |
| - 9902:9902 | |
| - 9904:9904 | |
| - 9903:9903 | |
| - 10001:10001 | |
| - 12001:12001 | |
| - 12002:12002 | |
| - 12003:12003 | |
| - 12004:12004 | |
| - 12005:12005 | |
| - 14001:14001 | |
| - 14002:14002 | |
| - 14003:14003 | |
| openssh: | |
| image: linuxserver/openssh-server:latest@sha256:67d4c3a1402179a6579aa217a38b52ced557eb8a0c17a8e32fe986a4549fdee4 | |
| ports: | |
| - 2222:2222 | |
| env: | |
| PUID: 1000 | |
| PGID: 1000 | |
| TZ: Etc/UTC | |
| USER_NAME: testuser | |
| USER_PASSWORD: testpass | |
| PASSWORD_ACCESS: true | |
| DOCKER_MODS: linuxserver/mods:openssh-server-ssh-tunnel | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.ref }} | |
| - name: generate or hydrate protos | |
| uses: ./.github/actions/genprotos | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 | |
| with: | |
| go-version: '1.26.4' | |
| cache-dependency-path: flow/go.sum | |
| - name: install lib-geos and pg_dump | |
| run: | | |
| # No need to update man pages on package install | |
| sudo apt-get remove --purge man-db | |
| # Add PGDG apt repo for latest PostgreSQL client packages | |
| sudo install -d /usr/share/postgresql-common/pgdg | |
| sudo curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ | |
| --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc | |
| echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ | |
| https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | |
| | sudo tee /etc/apt/sources.list.d/pgdg.list | |
| sudo apt-get update | |
| sudo apt-get install -y libgeos-dev | |
| # pg_dump must be >= the server major version; install v18 so it | |
| # can dump PG 16, 17, and 18 (backward compatible). | |
| sudo apt-get install -y postgresql-client-18 | |
| echo /usr/lib/postgresql/18/bin >> $GITHUB_PATH | |
| - name: install retry tool | |
| run: | | |
| sudo apt-get install -y retry | |
| - run: go mod download | |
| working-directory: ./flow | |
| - name: setup gcp service account | |
| id: gcp-service-account | |
| uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3 | |
| with: | |
| name: "bq_service_account.json" | |
| json: ${{ secrets.GCP_GH_CI_PKEY }} | |
| - name: setup snowflake credentials | |
| id: sf-credentials | |
| uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3 | |
| with: | |
| name: "snowflake_creds.json" | |
| json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }} | |
| - name: setup GCS credentials | |
| id: gcs-credentials | |
| uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3 | |
| with: | |
| name: "gcs_creds.json" | |
| json: ${{ secrets.GCS_CREDS }} | |
| - name: setup Eventhubs credentials | |
| id: eventhubs-credentials | |
| uses: jsdaniell/create-json@b8e77fa01397ca39cc4a6198cc29a3be5481afef # v1.2.3 | |
| with: | |
| name: "eh_creds.json" | |
| json: ${{ secrets.EH_CREDS }} | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6 | |
| id: setup-aws | |
| with: | |
| audience: sts.amazonaws.com | |
| aws-region: us-west-2 | |
| role-to-assume: ${{ secrets.FLOW_TESTS_AWS_ROLE_ARN }} | |
| mask-aws-account-id: true | |
| output-credentials: true | |
| - name: MySQL | |
| env: | |
| DB_IMG: ${{ matrix.version-configs.mysql[matrix.db-version.mysql].img }} | |
| DB_ENV: ${{ join(matrix.version-configs.mysql[matrix.db-version.mysql].env, ' -e ') }} | |
| DB_PARAMS: ${{ join(matrix.version-configs.mysql[matrix.db-version.mysql].parameters, ' ') }} | |
| run: | | |
| docker run -d --rm --name mysql --network ${{ job.container.network }} -p 3306:3306 \ | |
| -e $DB_ENV $DB_IMG $DB_PARAMS | |
| - name: MariaDB | |
| env: | |
| DB_IMG: ${{ matrix.version-configs.mariadb[matrix.db-version.mariadb].img }} | |
| DB_ENV: ${{ join(matrix.version-configs.mariadb[matrix.db-version.mariadb].env, ' -e ') }} | |
| DB_PARAMS: ${{ join(matrix.version-configs.mariadb[matrix.db-version.mariadb].parameters, ' ') }} | |
| run: | | |
| docker run -d --rm --name mariadb --network ${{ job.container.network }} -p 3316:3306 \ | |
| -e $DB_ENV $DB_IMG $DB_PARAMS | |
| - name: Mongo | |
| run: | | |
| echo "starting mongoDB..." | |
| docker run -d --rm --name mongo -p 27017:27017 mongo:${{ matrix.db-version.mongo }} \ | |
| bash -c 'openssl rand -base64 756 > /data/mongo.key && chmod 400 /data/mongo.key && mongod --replSet rs0 --oplogMinRetentionHours 24 --bind_ip_all --keyFile /data/mongo.key' | |
| until docker exec mongo mongosh --eval 'db.runCommand({ ping: 1 })' &> /dev/null; do | |
| echo "waiting for MongoDB to be ready..." | |
| sleep 2 | |
| done | |
| echo "initialize replica set" | |
| docker exec mongo mongosh --eval 'rs.initiate({ | |
| _id: "rs0", | |
| members: [{ _id: 0, host: "localhost:27017" }] | |
| })' | |
| echo "create admin user for writing data to mongo" | |
| docker exec mongo mongosh --eval ' | |
| db = db.getSiblingDB("admin"); | |
| db.createUser({ | |
| user: "admin", | |
| pwd: "admin", | |
| roles: ["root"] | |
| })' | |
| echo "create non-admin user for reading data from changestream" | |
| docker exec mongo mongosh -u admin -p admin --eval ' | |
| db = db.getSiblingDB("admin"); | |
| db.createUser({ | |
| user: "csuser", | |
| pwd: "cspass", | |
| roles: ["readAnyDatabase", "clusterMonitor"] | |
| })' | |
| - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 | |
| id: cache-minio | |
| with: | |
| path: ./minio | |
| key: ${{ runner.os }}-minio | |
| - name: Install MinIO Server | |
| if: steps.cache-minio.outputs.cache-hit != 'true' | |
| run: | | |
| curl -O https://dl.min.io/server/minio/release/linux-amd64/minio && chmod +x minio | |
| - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 | |
| id: cache-minio-client | |
| with: | |
| path: ./mc | |
| key: ${{ runner.os }}-minio-client | |
| - name: Install MinIO Client | |
| if: steps.cache-minio-client.outputs.cache-hit != 'true' | |
| run: | | |
| curl -O https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x mc | |
| - name: MinIO | |
| run: > | |
| mkdir -p certs minio-data && | |
| openssl genrsa -out certs/cert.key 2048 && | |
| openssl req -new -key certs/cert.key -out certs/cert.csr -subj /CN=minio.local && | |
| openssl x509 -req -days 3650 -in certs/cert.csr -signkey certs/cert.key -out certs/cert.crt && | |
| chown -R 1001 certs && | |
| ./minio server ./minio-data --certs-dir ./certs --address :9999 & | |
| sleep 2 && | |
| ./mc alias set myminiopeerdb http://localhost:9999 minio miniosecret && | |
| ./mc mb myminiopeerdb/peerdb | |
| env: | |
| MINIO_ROOT_USER: minio | |
| MINIO_ROOT_PASSWORD: miniosecret | |
| AWS_EC2_METADATA_DISABLED: true | |
| - name: Generate ClickHouse TLS certificates | |
| run: | | |
| mkdir -p ch-certs | |
| # CA | |
| openssl genrsa -out ch-certs/ca.key 2048 | |
| openssl req -new -x509 -key ch-certs/ca.key -out ch-certs/ca.crt -days 3650 -subj "/CN=ClickHouse-CA" | |
| # Server cert (CN=localhost, SAN for TLS 1.3) | |
| openssl genrsa -out ch-certs/server.key 2048 | |
| openssl req -new -key ch-certs/server.key -out ch-certs/server.csr -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" | |
| openssl x509 -req -days 3650 -in ch-certs/server.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/server.crt -copy_extensions copyall | |
| # Client cert for mTLS (CN=peerdb-client), using cert-manager naming convention | |
| openssl genrsa -out ch-certs/tls.key 2048 | |
| openssl req -new -key ch-certs/tls.key -out ch-certs/client.csr -subj "/CN=peerdb-client" | |
| openssl x509 -req -days 3650 -in ch-certs/client.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/tls.crt | |
| - name: create postgres extensions, increase logical replication limits, and setup catalog database | |
| # Also enables certificate-based client authentication (mTLS) on the catalog postgres instance. | |
| run: > | |
| docker exec "${{ job.services.catalog.id }}" apk add --no-cache build-base git && | |
| docker exec "${{ job.services.catalog.id }}" git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git /tmp/pgvector && | |
| docker exec "${{ job.services.catalog.id }}" sh -c 'cd /tmp/pgvector && make with_llvm=no && make with_llvm=no install' && | |
| docker exec "${{ job.services.catalog.id }}" psql -U postgres -c "CREATE EXTENSION hstore;CREATE EXTENSION vector;" | |
| -c "ALTER SYSTEM SET wal_level=logical;" | |
| -c "ALTER SYSTEM SET max_replication_slots=192;" | |
| -c "ALTER SYSTEM SET max_wal_senders=256;" | |
| -c "ALTER SYSTEM SET max_connections=2048;" && | |
| (cat ./nexus/catalog/migrations/V{?,??}__* | docker exec -i "${{ job.services.catalog.id }}" psql -U postgres) && | |
| docker cp volumes/pg-tls "${{ job.services.catalog.id }}:/tmp/pg-tls" && | |
| docker exec "${{ job.services.catalog.id }}" sh -c 'install -d -o postgres -g postgres /var/lib/postgresql/tls && install -m 644 -o postgres -g postgres /tmp/pg-tls/server.crt /var/lib/postgresql/tls/server.crt && install -m 600 -o postgres -g postgres /tmp/pg-tls/server.key /var/lib/postgresql/tls/server.key && install -m 644 -o postgres -g postgres /tmp/pg-tls/client.crt /var/lib/postgresql/tls/ca.crt' && | |
| docker exec "${{ job.services.catalog.id }}" psql -U postgres -c "ALTER SYSTEM SET ssl=on;" -c "ALTER SYSTEM SET ssl_cert_file='/var/lib/postgresql/tls/server.crt';" -c "ALTER SYSTEM SET ssl_key_file='/var/lib/postgresql/tls/server.key';" -c "ALTER SYSTEM SET ssl_ca_file='/var/lib/postgresql/tls/ca.crt';" && | |
| docker restart "${{ job.services.catalog.id }}" | |
| env: | |
| PGPASSWORD: postgres | |
| - name: prepare secondary postgres for cross-cluster schema-dump tests | |
| run: > | |
| docker exec "${{ job.services.catalog2.id }}" psql -U postgres | |
| -c "ALTER SYSTEM SET wal_level=logical;" | |
| -c "ALTER SYSTEM SET max_replication_slots=192;" | |
| -c "ALTER SYSTEM SET max_wal_senders=256;" | |
| -c "ALTER SYSTEM SET max_connections=2048;" && | |
| docker restart "${{ job.services.catalog2.id }}" | |
| env: | |
| PGPASSWORD: postgres | |
| - name: set ClickHouse version | |
| id: ch-version | |
| run: | | |
| if [ "${{ matrix.db-version.ch }}" = "lts" ]; then | |
| echo "ch_version=v25.8.28.1-lts" >> $GITHUB_OUTPUT | |
| elif [ "${{ matrix.db-version.ch }}" = "stable" ]; then | |
| echo "ch_version=v26.3.16.16-lts" >> $GITHUB_OUTPUT | |
| elif [ "${{ matrix.db-version.ch }}" = "latest" ]; then | |
| # note: latest tag does not always reflect the latest version (could be an update on an lts), | |
| # but that is okay as we are only using it to invalidate the cache. | |
| echo "ch_version=$(curl -s https://api.github.com/repos/ClickHouse/ClickHouse/releases/latest | jq -r .tag_name)" >> $GITHUB_OUTPUT | |
| fi | |
| - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 | |
| id: cache-clickhouse | |
| with: | |
| path: ./clickhouse | |
| key: ${{ runner.os }}-clickhouse-${{ steps.ch-version.outputs.ch_version }} | |
| - name: Install ClickHouse | |
| if: steps.cache-clickhouse.outputs.cache-hit != 'true' | |
| run: | | |
| if [[ "${{ matrix.db-version.ch }}" = 'latest' ]]; then | |
| curl https://clickhouse.com | sh | |
| else | |
| VERSION="${{ steps.ch-version.outputs.ch_version }}" | |
| VERSION_NUM=$(echo ${VERSION#v} | cut -d'-' -f1) | |
| curl -sL https://github.com/ClickHouse/ClickHouse/releases/download/$VERSION/clickhouse-common-static-$VERSION_NUM-amd64.tgz | tar -xzf - | |
| mv "clickhouse-common-static-$VERSION_NUM/usr/bin/clickhouse" ./clickhouse | |
| rm -rf "clickhouse-common-static-$VERSION_NUM" | |
| chmod +x clickhouse | |
| fi | |
| - name: Run ClickHouse | |
| run: | | |
| ./clickhouse --version | |
| cat > config1.xml <<EOF | |
| <clickhouse> | |
| <profiles><default></default></profiles> | |
| <users> | |
| <default> | |
| <password></password> | |
| <networks> | |
| <ip>::/0</ip> | |
| </networks> | |
| <profile>default</profile> | |
| <quota>default</quota> | |
| <access_management>1</access_management> | |
| <named_collection_control>1</named_collection_control> | |
| </default> | |
| <peerdb_tls> | |
| <ssl_certificates> | |
| <common_name>peerdb-client</common_name> | |
| </ssl_certificates> | |
| <networks> | |
| <ip>::/0</ip> | |
| </networks> | |
| <profile>default</profile> | |
| <quota>default</quota> | |
| <access_management>1</access_management> | |
| </peerdb_tls> | |
| </users> | |
| <logger><level>none</level></logger> | |
| <path>var/lib/clickhouse</path> | |
| <tmp_path>var/lib/clickhouse/tmp</tmp_path> | |
| <user_files_path>var/lib/clickhouse/user_files</user_files_path> | |
| <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path> | |
| <tcp_port>9000</tcp_port> | |
| <tcp_port_secure>9440</tcp_port_secure> | |
| <http_port remove="1"/> | |
| <postgresql_port remove="1"/> | |
| <mysql_port remove="1"/> | |
| <openSSL> | |
| <server> | |
| <certificateFile>../ch-certs/server.crt</certificateFile> | |
| <privateKeyFile>../ch-certs/server.key</privateKeyFile> | |
| <caConfig>../ch-certs/ca.crt</caConfig> | |
| <verificationMode>relaxed</verificationMode> | |
| <cacheSessions>true</cacheSessions> | |
| <disableProtocols>sslv2,sslv3</disableProtocols> | |
| <preferServerCiphers>true</preferServerCiphers> | |
| </server> | |
| </openSSL> | |
| <macros> | |
| <shard>1</shard> | |
| <replica>1</replica> | |
| </macros> | |
| <zookeeper> | |
| <node> | |
| <host>localhost</host> | |
| <port>2181</port> | |
| </node> | |
| </zookeeper> | |
| <distributed_ddl> | |
| <path>/clickhouse/task_queue/ddl</path> | |
| </distributed_ddl> | |
| <remote_servers> | |
| <cicluster> | |
| <shard> | |
| <replica> | |
| <host>localhost</host> | |
| <port>9000</port> | |
| </replica> | |
| </shard> | |
| <shard> | |
| <replica> | |
| <host>localhost</host> | |
| <port>9001</port> | |
| </replica> | |
| </shard> | |
| </cicluster> | |
| </remote_servers> | |
| </clickhouse> | |
| EOF | |
| cat > config2.xml <<EOF | |
| <clickhouse> | |
| <profiles><default></default></profiles> | |
| <users> | |
| <default> | |
| <password></password> | |
| <networks> | |
| <ip>::/0</ip> | |
| </networks> | |
| <profile>default</profile> | |
| <quota>default</quota> | |
| <access_management>1</access_management> | |
| <named_collection_control>1</named_collection_control> | |
| </default> | |
| <peerdb_tls> | |
| <ssl_certificates> | |
| <common_name>peerdb-client</common_name> | |
| </ssl_certificates> | |
| <networks> | |
| <ip>::/0</ip> | |
| </networks> | |
| <profile>default</profile> | |
| <quota>default</quota> | |
| <access_management>1</access_management> | |
| </peerdb_tls> | |
| </users> | |
| <logger><level>none</level></logger> | |
| <path>var/lib/clickhouse</path> | |
| <tmp_path>var/lib/clickhouse/tmp</tmp_path> | |
| <user_files_path>var/lib/clickhouse/user_files</user_files_path> | |
| <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path> | |
| <tcp_port>9001</tcp_port> | |
| <tcp_port_secure>9441</tcp_port_secure> | |
| <http_port remove="1"/> | |
| <postgresql_port remove="1"/> | |
| <mysql_port remove="1"/> | |
| <openSSL> | |
| <server> | |
| <certificateFile>../ch-certs/server.crt</certificateFile> | |
| <privateKeyFile>../ch-certs/server.key</privateKeyFile> | |
| <caConfig>../ch-certs/ca.crt</caConfig> | |
| <verificationMode>relaxed</verificationMode> | |
| <cacheSessions>true</cacheSessions> | |
| <disableProtocols>sslv2,sslv3</disableProtocols> | |
| <preferServerCiphers>true</preferServerCiphers> | |
| </server> | |
| </openSSL> | |
| <macros> | |
| <shard>2</shard> | |
| <replica>1</replica> | |
| </macros> | |
| <zookeeper> | |
| <node> | |
| <host>localhost</host> | |
| <port>2181</port> | |
| </node> | |
| </zookeeper> | |
| <distributed_ddl> | |
| <path>/clickhouse/task_queue/ddl</path> | |
| </distributed_ddl> | |
| <remote_servers> | |
| <cicluster> | |
| <shard> | |
| <replica> | |
| <host>localhost</host> | |
| <port>9000</port> | |
| </replica> | |
| </shard> | |
| <shard> | |
| <replica> | |
| <host>localhost</host> | |
| <port>9001</port> | |
| </replica> | |
| </shard> | |
| </cicluster> | |
| </remote_servers> | |
| </clickhouse> | |
| EOF | |
| cat > config-keeper.xml <<EOF | |
| <clickhouse> | |
| <keeper_server> | |
| <tcp_port>2181</tcp_port> | |
| <server_id>1</server_id> | |
| <log_storage_path>var/lib/clickhouse/coordination/log</log_storage_path> | |
| <snapshot_storage_path>var/lib/clickhouse/coordination/snapshots</snapshot_storage_path> | |
| <raft_configuration> | |
| <server> | |
| <id>1</id> | |
| <hostname>localhost</hostname> | |
| <port>9234</port> | |
| </server> | |
| </raft_configuration> | |
| </keeper_server> | |
| </clickhouse> | |
| EOF | |
| mkdir chkeep ch1 ch2 | |
| (cd chkeep && ../clickhouse keeper -C ../config-keeper.xml) & | |
| while true; do | |
| if echo "ruok" | nc -w 3 127.0.0.1 2181 2>/dev/null | grep -q "imok"; then | |
| break | |
| fi | |
| echo "Waiting for keeper..." | |
| sleep 1 | |
| done | |
| sleep 5 | |
| (cd ch1 && ../clickhouse server -C ../config1.xml) & | |
| (cd ch2 && ../clickhouse server -C ../config2.xml) & | |
| - name: Install Temporal CLI | |
| uses: temporalio/setup-temporal@1059a504f87e7fa2f385e3fa40d1aa7e62f1c6ca # v0 | |
| - name: Setup AWS CA Certs | |
| env: | |
| URL: https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem | |
| run: | | |
| curl -fsSL -o aws-global-bundle.pem "$URL" | |
| sudo csplit -b '%02d.crt' -s -z -f /usr/local/share/ca-certificates/aws-global-split-- aws-global-bundle.pem '/-----BEGIN CERTIFICATE-----/' '{*}' | |
| sudo update-ca-certificates | |
| - name: Install gotestsum | |
| run: | | |
| go install gotest.tools/gotestsum@latest | |
| - name: run tests | |
| run: | | |
| mkdir coverage | |
| mkdir -p ../logs | |
| temporal server start-dev --namespace default --headless > ../logs/temporal.log 2>&1 & | |
| go build -cover -ldflags="-s -w" -o peer-flow | |
| temporal operator search-attribute create --name MirrorName --type Text --namespace default | |
| ./peer-flow worker > ../logs/peer-flow-worker.log 2>&1 & | |
| ./peer-flow snapshot-worker > ../logs/peer-flow-snapshot-worker.log 2>&1 & | |
| ./peer-flow api --port 8112 --gateway-port 8113 > ../logs/peer-flow-api.log 2>&1 & | |
| gotestsum --format standard-quiet --no-color --junitfile ../logs/test-results.xml -- -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 32 ./... -timeout 1200s -args -test.gocoverdir="$PWD/coverage" | |
| killall peer-flow | |
| sleep 1 | |
| go tool covdata textfmt -i=coverage -o ../coverage.out | |
| working-directory: ./flow | |
| env: | |
| GOCOVERDIR: coverage | |
| AWS_ENDPOINT_URL_S3: http://localhost:9999 | |
| AWS_ACCESS_KEY_ID: minio | |
| AWS_SECRET_ACCESS_KEY: miniosecret | |
| AWS_REGION: us-east-1 | |
| AWS_ENDPOINT_URL_S3_TLS: https://localhost:9998 | |
| PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio | |
| PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret | |
| PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1 | |
| PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999 | |
| PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdb | |
| PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio | |
| PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret | |
| PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_REGION: us-east-1 | |
| PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999 | |
| PEERDB_SNOWFLAKE_AWS_S3_BUCKET_NAME: peerdb | |
| TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json | |
| TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json | |
| TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json | |
| TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json | |
| TEST_EH_CREDS: ${{ github.workspace }}/eh_creds.json | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| PEERDB_CATALOG_HOST: localhost | |
| PEERDB_CATALOG_PORT: 5432 | |
| PEERDB_CATALOG_USER: postgres | |
| PEERDB_CATALOG_PASSWORD: postgres | |
| PEERDB_CATALOG_DATABASE: postgres | |
| PG_HOST: localhost | |
| PG_PORT: 5432 | |
| PG_USER: postgres | |
| PG_PASSWORD: postgres | |
| PG_DATABASE: postgres | |
| PG2_HOST: localhost | |
| PG2_PORT: 5437 | |
| PG2_USER: postgres | |
| PG2_PASSWORD: postgres | |
| PG2_DATABASE: postgres | |
| # Mutual-TLS client-certificate auth test (catalog has TLS enabled above) | |
| PG_MTLS_ROOT_CA_PATH: ${{ github.workspace }}/volumes/pg-tls/server.crt | |
| PG_MTLS_CLIENT_CERT_PATH: ${{ github.workspace }}/volumes/pg-tls/client.crt | |
| PG_MTLS_CLIENT_KEY_PATH: ${{ github.workspace }}/volumes/pg-tls/client.key | |
| PEERDB_SWITCHBOARD_ENABLED: "true" | |
| PEERDB_QUEUE_FORCE_TOPIC_CREATION: "true" | |
| ELASTICSEARCH_TEST_ADDRESS: http://localhost:9200 | |
| CI_PG_VERSION: ${{ matrix.db-version.pg }} | |
| CI_MYSQL_HOST: localhost | |
| CI_MYSQL_PORT: 3306 | |
| CI_MYSQL_ROOT_PASSWORD: cipass | |
| CI_MYSQL_VERSION: ${{ matrix.db-version.mysql }} | |
| CI_SSH_MYSQL_HOST: mysql | |
| CI_MARIADB_HOST: localhost | |
| CI_MARIADB_PORT: 3316 | |
| CI_MARIADB_ROOT_PASSWORD: cipass | |
| CI_MARIADB_VERSION: ${{ matrix.db-version.mariadb }} | |
| CI_MONGO_ADMIN_URI: mongodb://localhost:27017 | |
| CI_MONGO_ADMIN_USERNAME: "admin" | |
| CI_MONGO_ADMIN_PASSWORD: "admin" | |
| CI_MONGO_URI: mongodb://localhost:27017 | |
| CI_MONGO_USERNAME: "csuser" | |
| CI_MONGO_PASSWORD: "cspass" | |
| SSH_POSTGRES_HOST: catalog | |
| ENABLE_OTEL_METRICS: ${{ (matrix.db-version.pg == '17' || matrix.db-version.mysql == 'mysql-pos') && 'true' || 'false' }} | |
| OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: http://localhost:4317 | |
| OTEL_EXPORTER_OTLP_METRICS_PROTOCOL: grpc | |
| PEERDB_OTEL_METRICS_NAMESPACE: 'peerdb_ci_tests.' | |
| PEERDB_OTEL_TEMPORAL_METRICS_EXPORT_LIST: '__ALL__' | |
| PEERDB_OTEL_METRICS_PANIC_ON_EXPORT_FAILURE: 'true' | |
| # Below are used to test RDS IAM Auth for Postgres and MySQL | |
| FLOW_TESTS_RDS_IAM_AUTH_AWS_ACCESS_KEY_ID: ${{ steps.setup-aws.outputs.aws-access-key-id }} | |
| FLOW_TESTS_RDS_IAM_AUTH_AWS_SECRET_ACCESS_KEY: ${{ steps.setup-aws.outputs.aws-secret-access-key }} | |
| FLOW_TESTS_RDS_IAM_AUTH_AWS_SESSION_TOKEN: ${{ steps.setup-aws.outputs.aws-session-token }} | |
| FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES }} | |
| FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES_PROXY: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_POSTGRES_PROXY }} | |
| FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL }} | |
| FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL_PROXY: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_HOST_MYSQL_PROXY }} | |
| FLOW_TESTS_RDS_IAM_AUTH_USERNAME_POSTGRES: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_USERNAME_POSTGRES }} | |
| FLOW_TESTS_RDS_IAM_AUTH_USERNAME_MYSQL: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_USERNAME_MYSQL }} | |
| FLOW_TESTS_RDS_IAM_AUTH_ASSUME_ROLE: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_ASSUME_ROLE }} | |
| FLOW_TESTS_RDS_IAM_AUTH_CHAINED_ROLE: ${{ secrets.FLOW_TESTS_RDS_IAM_AUTH_CHAINED_ROLE }} | |
| # For ClickHouse S3 IAM Role based tests | |
| FLOW_TESTS_AWS_S3_BUCKET_NAME: ${{ secrets.FLOW_TESTS_AWS_S3_BUCKET_NAME }} | |
| FLOW_TESTS_AWS_ACCESS_KEY_ID: ${{ steps.setup-aws.outputs.aws-access-key-id }} | |
| FLOW_TESTS_AWS_SECRET_ACCESS_KEY: ${{ steps.setup-aws.outputs.aws-secret-access-key }} | |
| FLOW_TESTS_AWS_SESSION_TOKEN: ${{ steps.setup-aws.outputs.aws-session-token }} | |
| # ClickHouse TLS/mTLS test certificates | |
| PEERDB_CLICKHOUSE_TLS_PORT: "9440" | |
| PEERDB_CLICKHOUSE_TLS_CERT_DIR: ${{ github.workspace }}/ch-certs | |
| - name: Upload peer-flow logs and test results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: peer-flow-logs-pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-ma${{ matrix.db-version.mariadb }}-mo${{ matrix.db-version.mongo }} | |
| path: logs/ | |
| retention-days: 30 | |
| - name: Upload test results to Codecov | |
| if: success() || failure() | |
| uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6 | |
| with: | |
| report_type: test_results | |
| files: logs/test-results.xml | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Ingest tests results for analysis | |
| if: success() || failure() | |
| uses: ./.github/actions/ingest-test-results | |
| with: | |
| combination-id: "pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-ma${{ matrix.db-version.mariadb }}-mo${{ matrix.db-version.mongo }}-ch${{ matrix.db-version.ch }}" | |
| o11y-api-key-id: ${{ secrets.CI_O11Y_TARGET_API_KEY_ID }} | |
| o11y-api-key-secret: ${{ secrets.CI_O11Y_TARGET_API_KEY_SECRET }} | |
| o11y-query-endpoint: ${{ secrets.CI_O11Y_TARGET_QUERY_ENDPOINT }} | |
| - name: Check all present test functions in the source branch were exercised in CI | |
| # Compares the top-level tests that actually ran in CI (from the | |
| # normalized results produced by the ingest step above) against every top-level | |
| # test defined under ./flow, and fails if the branch defines tests CI never ran. | |
| # Only runs for pull request builds; skipped for non-PR builds (e.g. push to main). | |
| # Parens matter: && binds tighter than || in GitHub Actions expressions. | |
| if: | | |
| (success() || failure()) && | |
| (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') | |
| working-directory: ./flow | |
| run: | | |
| set -uo pipefail | |
| tmp="${RUNNER_TEMP:-/tmp}" | |
| ndjson="../logs/normalized-test-results.ndjson" | |
| if [ ! -s "$ndjson" ]; then | |
| echo "::error::No normalized test results at ${ndjson} (results ingestion disabled or no tests ran); failing check." | |
| exit 1 | |
| fi | |
| # (run-tests): top-level test functions observed in CI results, with subtests | |
| # collapsed onto their parent (TestFoo/sub -> TestFoo). | |
| # This collapse is necessary because the branch test enumeration below only lists top-level tests functions. | |
| jq -r '.test' "$ndjson" | { grep '^Test' || true; } | awk -F '/' '{ print $1 }' | sort -u > "${tmp}/run-tests.txt" | |
| if [ ! -s "${tmp}/run-tests.txt" ]; then | |
| echo "::error::No 'Test*' entries found in CI results; failing check." | |
| exit 1 | |
| fi | |
| # (tests-in-branch): every top-level test function defined in the source branch. | |
| if ! go test -list '.*' ./... > "${tmp}/go-list.txt" 2> "${tmp}/go-list.err"; then | |
| echo "::error::'go test -list' failed (likely a compilation error already surfaced by the test step); failing check." | |
| cat "${tmp}/go-list.err" | |
| exit 1 | |
| fi | |
| { grep '^Test' "${tmp}/go-list.txt" || true; } | sort -u > "${tmp}/tests-in-branch.txt" | |
| # Tests defined in the branch but never observed in CI results. | |
| missing="$(comm -13 "${tmp}/run-tests.txt" "${tmp}/tests-in-branch.txt")" | |
| if [ -z "$missing" ]; then | |
| echo "All source-branch tests were exercised in CI." | |
| exit 0 | |
| fi | |
| echo "The following tests in the source branch have not been run in CI:" | |
| printf '%s\n' "$missing" | |
| exit 1 |