mongo: partition snapshots for numeric and string _id collections

.claude/skills/tilt/SKILL.md

@@ -86,7 +86,9 @@ All e2e tests also depend on core PeerDB services: `flow-api`, `flow-worker`, `c
 |----------|---------|----------------------|
 | `connector_postgres` | `connectors/postgres` | postgres |
 | `connector_mongo` | `connectors/mongo` | mongodb |
-| `connector_mysql` | `connectors/mysql` | mysql-gtid |
+| `connector_mysql-gtid` | `connectors/mysql` | mysql-gtid |
+| `connector_mysql-pos` | `connectors/mysql` | mysql-pos |
+| `connector_mariadb` | `connectors/mysql` | mariadb |
 | `connector_clickhouse` | `connectors/clickhouse` | clickhouse |
 
 Connector tests only depend on `catalog` + their specific DB.

.env.example

@@ -14,11 +14,15 @@ CI_CLICKHOUSE_NATIVE_PORT=11000
 CI_MYSQL_HOST=host.docker.internal
 CI_MYSQL_GTID_PORT=3306
 CI_MYSQL_POS_PORT=3307
-CI_MARIADB_PORT=3308
 CI_MYSQL_ROOT_PASSWORD=cipass
 CI_MYSQL_GTID_VERSION=mysql-gtid
 CI_MYSQL_POS_VERSION=mysql-pos
-CI_MARIADB_VERSION=maria
+CI_SSH_MYSQL_HOST=host.docker.internal
+
+CI_MARIADB_HOST=host.docker.internal
+CI_MARIADB_PORT=3308
+CI_MARIADB_ROOT_PASSWORD=cipass
+CI_MARIADB_VERSION=maria-11
 
 CI_MYSQL_VERSION=mysql-gtid
 CI_MYSQL_PORT=3306 # Default is gtid
@@ -46,6 +50,7 @@ AWS_ACCESS_KEY_ID=_peerdb_minioadmin
 AWS_SECRET_ACCESS_KEY=_peerdb_minioadmin
 AWS_ENDPOINT_URL_S3=http://host.docker.internal:9001
 AWS_EC2_METADATA_DISABLED="true"
+FLOW_TESTS_RDS_IAM_AUTH_SKIP=true
 
 MINIO_ROOT_USER=_peerdb_minioadmin
 MINIO_ROOT_PASSWORD=_peerdb_minioadmin

.github/actions/genprotos/action.yml

@@ -5,7 +5,7 @@ runs:
   steps:
     - name: check cache
       id: cache
-      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+      uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
       with:
         path: |
           ./flow/generated

.github/actions/ingest-test-results/action.yml

@@ -0,0 +1,74 @@
+name: 'Ingest test results'
+description: >-
+  Normalize JUnit test results and ingest them into the CI observability store
+  for analysis. No-op when the o11y target secrets are not configured. Must run
+  from the workspace root (uses .github/scripts/test-results.sh) and requires the
+  `retry` CLI on PATH.
+
+inputs:
+  combination-id:
+    description: >-
+      Identifier for the combination/compatibility this run covers (stored as
+      compatibility_matrix_id). e.g. "pg17-my8-mo7-ch24" or "tilt-TestApiPg".
+    required: true
+  test-results-file:
+    description: Path to the JUnit XML test results file (relative to the workspace root).
+    required: false
+    default: logs/test-results.xml
+  o11y-api-key-id:
+    description: API key id for the o11y ingestion endpoint. Ingestion is skipped when empty.
+    required: false
+    default: ''
+  o11y-api-key-secret:
+    description: API key secret for the o11y ingestion endpoint. Ingestion is skipped when empty.
+    required: false
+    default: ''
+  o11y-query-endpoint:
+    description: Query endpoint the normalized results are POSTed to. Ingestion is skipped when empty.
+    required: false
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Ingest tests results for analysis
+      shell: bash
+      env:
+        WORKFLOW_RUN_ID: ${{ github.run_id }}
+        WORKFLOW_RUN_LINK: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        WORKFLOW_HEAD_BRANCH: ${{ github.head_ref || github.ref_name }}
+        WORKFLOW_RUN_NUMBER: ${{ github.run_attempt }}
+        COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        SUMMARY_HEADER: "x-clickhouse-summary"
+        COMBINATION_ID: ${{ inputs.combination-id }}
+        TEST_RESULTS_FILE: ${{ inputs.test-results-file }}
+        O11Y_API_KEY_ID: ${{ inputs.o11y-api-key-id }}
+        O11Y_API_KEY_SECRET: ${{ inputs.o11y-api-key-secret }}
+        O11Y_QUERY_ENDPOINT: ${{ inputs.o11y-query-endpoint }}
+      run: |
+        if [ -z "$O11Y_API_KEY_ID" ] \
+          || [ -z "$O11Y_API_KEY_SECRET" ] \
+          || [ -z "$O11Y_QUERY_ENDPOINT" ]; then
+            echo "Secrets not configured; Test results ingestion is disabled."
+            exit 0
+        fi
+        .github/scripts/test-results.sh "$TEST_RESULTS_FILE" > logs/normalized-test-results.ndjson
+        if [ ! -s logs/normalized-test-results.ndjson ]; then
+            echo "Failed to extract test results"
+            exit 1
+        fi
+        # Retry upload with exponential backoff
+        retry --times=5 --delay "1,2,4,8,16,32,64,128" -- \
+        curl --fail-with-body -i -H "Content-Type: application/octet-stream" \
+          --user "$O11Y_API_KEY_ID:$O11Y_API_KEY_SECRET" \
+          "$O11Y_QUERY_ENDPOINT" \
+          --data-binary @logs/normalized-test-results.ndjson \
+          | tee upload.log
+        WRITTEN_ROWS=$(cat upload.log | grep $SUMMARY_HEADER | sed "s/$SUMMARY_HEADER: //" | jq '.written_rows' -r)
+        if [ -z "$WRITTEN_ROWS" ] || [ "$WRITTEN_ROWS" -eq 0 ]; then
+            echo "No results uploaded. Query client logs:"
+            cat upload.log
+            exit 2
+        fi
+        echo "Number of ingested test results: $WRITTEN_ROWS"
+        exit 0

.github/workflows/auto-retry-flaky-tests.yml

@@ -27,7 +27,7 @@ jobs:
 
       - name: Detect flaky test failures
         id: detect
-        uses: anthropics/claude-code-action@4d7e1f0cd85743fdc93b1c8040ab54395da024e2 # main
+        uses: anthropics/claude-code-action@a92e7c70a4da9793dc164451d829089dc057a464 # main
         with:
           allowed_bots: 'renovate'
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}

.github/workflows/claude-code-review-dependencies-updates.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Run Claude Code Review
         id: claude-review
-        uses: anthropics/claude-code-action@4d7e1f0cd85743fdc93b1c8040ab54395da024e2 # v1
+        uses: anthropics/claude-code-action@a92e7c70a4da9793dc164451d829089dc057a464 # v1
         with:
           allowed_bots: 'renovate'
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}

.github/workflows/claude-code-review.yml

@@ -28,10 +28,13 @@ jobs:
 
       - name: Run Claude Code Review
         id: claude-review
-        uses: anthropics/claude-code-action@4d7e1f0cd85743fdc93b1c8040ab54395da024e2 # v1
+        uses: anthropics/claude-code-action@a92e7c70a4da9793dc164451d829089dc057a464 # v1
         with:
           allowed_bots: 'renovate'
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+          # Fork PRs get an environment-scoped OIDC token that the Claude App token exchange rejects (401),
+          # so pass github_token only there; internal PRs keep the App token flow.
+          github_token: ${{ (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) && github.token || '' }}
           claude_args: |
             --model claude-opus-4-6
           plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'

.github/workflows/claude.yml

@@ -40,7 +40,7 @@ jobs:
 
       - name: Run Claude Code
         id: claude
-        uses: anthropics/claude-code-action@4d7e1f0cd85743fdc93b1c8040ab54395da024e2 # v1
+        uses: anthropics/claude-code-action@a92e7c70a4da9793dc164451d829089dc057a464 # v1
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
 

.github/workflows/cleanup.yml

@@ -12,7 +12,7 @@ jobs:
       - name: checkout sources
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
 
-      - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
+      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
         with:
           go-version: '1.26.4'
           cache-dependency-path: e2e_cleanup/go.sum

.github/workflows/flow.yml

@@ -23,6 +23,7 @@ permissions:
 
 jobs:
   flow_test:
+    name: flow_test (${{ matrix.runner }}, ${{ matrix.db-version.pg }}, ${{ matrix.db-version.mysql }}, ${{ matrix.db-version.mongo }}, ${{ matrix.db-version.ch }})
     if: |
       (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
       (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) ||
@@ -33,10 +34,36 @@ jobs:
       matrix:
         runner: [ubuntu-latest-16-cores]
         db-version: [
-          {pg: 16, mysql: 'mysql-gtid', mongo: '6.0', ch: 'lts'},
-          {pg: 17, mysql: 'mysql-pos', mongo: '7.0', ch: 'stable'},
-          {pg: 18, mysql: 'maria', mongo: '8.0', ch: 'latest'},
+          {pg: 16, mysql: 'mysql-gtid', mariadb: 'maria-11', mongo: '6.0', ch: 'lts'},
+          {pg: 17, mysql: 'mysql-pos', mariadb: 'maria-12', mongo: '7.0', ch: 'stable'},
+          {pg: 18, mysql: 'mysql-gtid', mariadb: 'maria-13', mongo: '8.0', ch: 'latest'},
         ]
+        # Per-version container settings consumed by the MySQL and MariaDB steps.
+        # Wrapped in a single-item list because matrix values must be arrays; it stays
+        # a single shared value (no extra jobs).
+        version-configs:
+          - mysql:
+              mysql-gtid:
+                img: 'mysql:9.5'
+                env: ['MYSQL_ROOT_PASSWORD=cipass']
+                parameters: []
+              mysql-pos:
+                img: 'mysql:5.7'
+                env: ['MYSQL_ROOT_PASSWORD=cipass']
+                parameters: ['--log_bin=mysql-bin', '--server-id=1', '--bind-address=::']
+            mariadb:
+              maria-11:
+                img: 'mariadb:lts-ubi9'
+                env: ['MARIADB_ROOT_PASSWORD=cipass']
+                parameters: ['--log-bin=maria', '--gtid-strict-mode=ON']
+              maria-12:
+                img: 'mariadb:12-ubi'
+                env: ['MARIADB_ROOT_PASSWORD=cipass']
+                parameters: ['--log-bin=maria', '--gtid-strict-mode=ON']
+              maria-13:
+                img: 'mariadb:13.0-ubi-rc'
+                env: ['MARIADB_ROOT_PASSWORD=cipass']
+                parameters: ['--log-bin=maria', '--gtid-strict-mode=ON']
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     services:
@@ -69,20 +96,20 @@ jobs:
           --health-timeout 5s
           --health-retries 5
       redpanda:
-        image: redpandadata/redpanda@sha256:394cee6e2076a23f9b7b01f5d68228d52429fe683fd087de11c8a68ff5287037
+        image: redpandadata/redpanda@sha256:3e72ea35731a893bdd3b6d283eb554416377e23766434fc85dd7941540673639
         ports:
           - 9092:9092
           - 9644:9644
       elasticsearch:
-        image: elasticsearch:9.4.2@sha256:715c5d552328d83dcee6082bd5125327332a649f8e0aedb5fe0553d7be75bf21
+        image: elasticsearch:9.4.2@sha256:e182d810a437e30ad7d0221fc153910a06d9be5b05e57e59e7983467b89e41e2
         ports:
           - 9200:9200
         env:
           discovery.type: single-node
           xpack.security.enabled: false
           xpack.security.enrollment.enabled: false
       otelcol:
-        image: otel/opentelemetry-collector-contrib:0.153.0@sha256:93aad750175cbf1a973ae1c5886c3371f4d800f61be25cdd26870b8441ffe9fa
+        image: otel/opentelemetry-collector-contrib:0.154.0@sha256:b3079f45e19bdb7326bf49cdddce6cf60dfd865138db39f2733ea48ab17bc4cb
         ports:
           - 4317:4317
       toxiproxy:
@@ -102,7 +129,7 @@ jobs:
           - 14002:14002
           - 14003:14003
       openssh:
-        image: linuxserver/openssh-server:latest@sha256:d7ba00e0f66af202beebfa5cd96bad6698aa1543d5196e4d85dba340ea40d092
+        image: linuxserver/openssh-server:latest@sha256:67d4c3a1402179a6579aa217a38b52ced557eb8a0c17a8e32fe986a4549fdee4
         ports:
           - 2222:2222
         env:
@@ -121,7 +148,7 @@ jobs:
       - name: generate or hydrate protos
         uses: ./.github/actions/genprotos
 
-      - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
+      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
         with:
           go-version: '1.26.4'
           cache-dependency-path: flow/go.sum
@@ -180,7 +207,7 @@ jobs:
           json: ${{ secrets.EH_CREDS }}
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
+        uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6
         id: setup-aws
         with:
           audience: sts.amazonaws.com
@@ -190,14 +217,22 @@ jobs:
           output-credentials: true
 
       - name: MySQL
+        env:
+          DB_IMG: ${{ matrix.version-configs.mysql[matrix.db-version.mysql].img }}
+          DB_ENV: ${{ join(matrix.version-configs.mysql[matrix.db-version.mysql].env, ' -e ') }}
+          DB_PARAMS: ${{ join(matrix.version-configs.mysql[matrix.db-version.mysql].parameters, ' ') }}
         run: |
-          if [ "${{ matrix.db-version.mysql }}" = "mysql-gtid" ]; then
-            docker run -d --rm --name mysql --network ${{ job.container.network }} -p 3306:3306 -e MYSQL_ROOT_PASSWORD=cipass mysql:9.5
-          elif [ "${{ matrix.db-version.mysql }}" = "mysql-pos" ]; then
-            docker run -d --rm --name mysql --network ${{ job.container.network }} -p 3306:3306 -e MYSQL_ROOT_PASSWORD=cipass mysql:5.7 --log_bin=mysql-bin --server-id=1 --bind-address=::
-          elif [ "${{ matrix.db-version.mysql }}" = "maria" ]; then
-            docker run -d --rm --name mariadb --network ${{ job.container.network }} -p 3306:3306 -e MARIADB_ROOT_PASSWORD=cipass mariadb:lts-ubi9@sha256:55a81b2d791d2ff8ad33fef413d9e45e0ac57a951127e0cfc69a8e59f922ba6e --log-bin=maria
-          fi
+          docker run -d --rm --name mysql --network ${{ job.container.network }} -p 3306:3306 \
+            -e $DB_ENV $DB_IMG $DB_PARAMS
+
+      - name: MariaDB
+        env:
+          DB_IMG: ${{ matrix.version-configs.mariadb[matrix.db-version.mariadb].img }}
+          DB_ENV: ${{ join(matrix.version-configs.mariadb[matrix.db-version.mariadb].env, ' -e ') }}
+          DB_PARAMS: ${{ join(matrix.version-configs.mariadb[matrix.db-version.mariadb].parameters, ' ') }}
+        run: |
+          docker run -d --rm --name mariadb --network ${{ job.container.network }} -p 3316:3306 \
+            -e $DB_ENV $DB_IMG $DB_PARAMS
 
       - name: Mongo
         run: |
@@ -234,7 +269,7 @@ jobs:
               roles: ["readAnyDatabase", "clusterMonitor"]
             })'
 
-      - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+      - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
         id: cache-minio
         with:
           path: ./minio
@@ -245,7 +280,7 @@ jobs:
         run: |
           curl -O https://dl.min.io/server/minio/release/linux-amd64/minio && chmod +x minio
 
-      - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+      - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
         id: cache-minio-client
         with:
           path: ./mc
@@ -326,7 +361,7 @@ jobs:
             echo "ch_version=$(curl -s https://api.github.com/repos/ClickHouse/ClickHouse/releases/latest | jq -r .tag_name)" >> $GITHUB_OUTPUT
           fi
 
-      - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+      - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
         id: cache-clickhouse
         with:
           path: ./clickhouse
@@ -608,7 +643,15 @@ jobs:
           PEERDB_QUEUE_FORCE_TOPIC_CREATION: "true"
           ELASTICSEARCH_TEST_ADDRESS: http://localhost:9200
           CI_PG_VERSION: ${{ matrix.db-version.pg }}
+          CI_MYSQL_HOST: localhost
+          CI_MYSQL_PORT: 3306
+          CI_MYSQL_ROOT_PASSWORD: cipass
           CI_MYSQL_VERSION: ${{ matrix.db-version.mysql }}
+          CI_SSH_MYSQL_HOST: mysql
+          CI_MARIADB_HOST: localhost
+          CI_MARIADB_PORT: 3316
+          CI_MARIADB_ROOT_PASSWORD: cipass
+          CI_MARIADB_VERSION: ${{ matrix.db-version.mariadb }}
           CI_MONGO_ADMIN_URI: mongodb://localhost:27017
           CI_MONGO_ADMIN_USERNAME: "admin"
           CI_MONGO_ADMIN_PASSWORD: "admin"
@@ -647,7 +690,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: peer-flow-logs-pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-mo${{ matrix.db-version.mongo }}
+          name: peer-flow-logs-pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-ma${{ matrix.db-version.mariadb }}-mo${{ matrix.db-version.mongo }}
           path: logs/
           retention-days: 30
 
@@ -664,38 +707,9 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
       - name: Ingest tests results for analysis
         if: success() || failure()
-        env:
-          WORKFLOW_RUN_ID: ${{ github.run_id }}
-          WORKFLOW_RUN_LINK: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          WORKFLOW_HEAD_BRANCH: ${{ github.head_ref || github.ref_name }}
-          WORKFLOW_RUN_NUMBER: ${{ github.run_attempt }}
-          COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
-          SUMMARY_HEADER: "x-clickhouse-summary"
-          COMBINATION_ID: "pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-mo${{ matrix.db-version.mongo }}-ch${{ matrix.db-version.ch }}"
-        run: |
-          if [ -z "${{ secrets.CI_O11Y_TARGET_API_KEY_ID }}" ] \
-            || [ -z "${{ secrets.CI_O11Y_TARGET_API_KEY_SECRET }}" ] \
-            || [ -z "${{ secrets.CI_O11Y_TARGET_QUERY_ENDPOINT }}" ]; then
-              echo "Secrets not configured; Test results ingestion is disabled."
-              exit 0
-          fi
-          .github/scripts/test-results.sh logs/test-results.xml > logs/normalized-test-results.ndjson
-          if [ ! -s logs/normalized-test-results.ndjson ]; then
-              echo "Failed to extract test results"
-              exit 1
-          fi
-          # Retry upload with exponential backoff
-          retry --times=5 --delay "1,2,4,8,16,32,64,128" -- \
-          curl --fail-with-body -i -H "Content-Type: application/octet-stream" \
-            --user "${{ secrets.CI_O11Y_TARGET_API_KEY_ID }}:${{ secrets.CI_O11Y_TARGET_API_KEY_SECRET }}" \
-            "${{ secrets.CI_O11Y_TARGET_QUERY_ENDPOINT }}" \
-            --data-binary @logs/normalized-test-results.ndjson \
-            | tee upload.log
-          WRITTEN_ROWS=$(cat upload.log | grep $SUMMARY_HEADER | sed "s/$SUMMARY_HEADER: //" | jq '.written_rows' -r)
-          if [ -z "$WRITTEN_ROWS" ] || [ "$WRITTEN_ROWS" -eq 0 ]; then
-              echo "No results uploaded. Query client logs:"
-              cat upload.log
-              exit 2
-          fi
-          echo "Number of ingested test results: $WRITTEN_ROWS"
-          exit 0
+        uses: ./.github/actions/ingest-test-results
+        with:
+          combination-id: "pg${{ matrix.db-version.pg }}-my${{ matrix.db-version.mysql }}-ma${{ matrix.db-version.mariadb }}-mo${{ matrix.db-version.mongo }}-ch${{ matrix.db-version.ch }}"
+          o11y-api-key-id: ${{ secrets.CI_O11Y_TARGET_API_KEY_ID }}
+          o11y-api-key-secret: ${{ secrets.CI_O11Y_TARGET_API_KEY_SECRET }}
+          o11y-query-endpoint: ${{ secrets.CI_O11Y_TARGET_QUERY_ENDPOINT }}