Merge branch 'main' into add-vscode-remote-go-debugger-in-tilt

Author: masterashu

.env.example

@@ -29,10 +29,11 @@ PG_USER=postgres
 PG_PASSWORD=postgres
 PG_DATABASE=postgres
 
-# Set to 1 for enabling go debugger in dockerfile and Tilt Setup
-DOCKER_GO_DEBUG_FLOW_API=
-DOCKER_GO_DEBUG_FLOW_WORKER=
-DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER=
+PG2_HOST=host.docker.internal
+PG2_PORT=5437
+PG2_USER=postgres
+PG2_PASSWORD=postgres
+PG2_DATABASE=postgres
 
 PEERDB_CATALOG_HOST=host.docker.internal
 PEERDB_CATALOG_PORT=9901
@@ -49,6 +50,11 @@ AWS_EC2_METADATA_DISABLED="true"
 MINIO_ROOT_USER=_peerdb_minioadmin
 MINIO_ROOT_PASSWORD=_peerdb_minioadmin
 
+# Set to 1 for enabling go debugger in dockerfile and Tilt Setup
+DOCKER_GO_DEBUG_FLOW_API=
+DOCKER_GO_DEBUG_FLOW_WORKER=
+DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER=
+
 TZ='UTC'
 
 ## Port Overrides for parallel tilt and docker-compose runs.

.github/CODEOWNERS

@@ -0,0 +1,47 @@
+# Default owner
+*                                                       @PeerDB-io/db-integrations
+
+# Non-CH, non-PG destinations
+flow/connectors/bigquery/                               @PeerDB-io/eng
+flow/connectors/elasticsearch/                          @PeerDB-io/eng
+flow/connectors/eventhub/                               @PeerDB-io/eng
+flow/connectors/kafka/                                  @PeerDB-io/eng
+flow/connectors/pubsub/                                 @PeerDB-io/eng
+flow/connectors/s3/                                     @PeerDB-io/eng
+flow/connectors/snowflake/                              @PeerDB-io/eng
+
+ui/components/PeerForms/BigqueryConfig.tsx              @PeerDB-io/eng
+ui/components/PeerForms/SnowflakeForm.tsx               @PeerDB-io/eng
+ui/components/PeerForms/S3Form.tsx                      @PeerDB-io/eng
+ui/components/PeerForms/ElasticsearchConfigForm.tsx     @PeerDB-io/eng
+ui/components/PeerForms/Eventhubs/                      @PeerDB-io/eng
+ui/components/PeerForms/KafkaConfig.tsx                 @PeerDB-io/eng
+ui/components/PeerForms/PubSubConfig.tsx                @PeerDB-io/eng
+ui/app/peers/create/[peerType]/helpers/bq.ts            @PeerDB-io/eng
+ui/app/peers/create/[peerType]/helpers/sf.ts            @PeerDB-io/eng
+ui/app/peers/create/[peerType]/helpers/s3.ts            @PeerDB-io/eng
+ui/app/peers/create/[peerType]/helpers/es.ts            @PeerDB-io/eng
+ui/app/peers/create/[peerType]/helpers/eh.ts            @PeerDB-io/eng
+ui/app/peers/create/[peerType]/helpers/ka.ts            @PeerDB-io/eng
+ui/app/peers/create/[peerType]/helpers/ps.ts            @PeerDB-io/eng
+ui/app/mirrors/create/cdc/eventhubsCallout.tsx          @PeerDB-io/eng
+
+flow/e2e/bigquery.go                                    @PeerDB-io/eng
+flow/e2e/bigquery_qrep_test.go                          @PeerDB-io/eng
+flow/e2e/bigquery_test.go                               @PeerDB-io/eng
+flow/e2e/elasticsearch*.go                              @PeerDB-io/eng
+flow/e2e/eventhub*.go                                   @PeerDB-io/eng
+flow/e2e/kafka*.go                                      @PeerDB-io/eng
+flow/e2e/pubsub*.go                                     @PeerDB-io/eng
+flow/e2e/s3_*.go                                        @PeerDB-io/eng
+flow/e2e/snowflake*.go                                  @PeerDB-io/eng
+
+# PG destination
+flow/connectors/postgres/postgres_destination.go        @PeerDB-io/ch-postgres
+flow/connectors/postgres/qrep_destination.go            @PeerDB-io/ch-postgres
+flow/connectors/postgres/normalize_stmt_generator.go    @PeerDB-io/ch-postgres
+flow/connectors/postgres/sink_pg.go                     @PeerDB-io/ch-postgres
+flow/model/pg_items.go                                  @PeerDB-io/ch-postgres
+flow/pkg/postgres/dest_validation.go                    @PeerDB-io/ch-postgres
+
+flow/e2e/postgres*.go                                   @PeerDB-io/ch-postgres

.github/workflows/auto-retry-flaky-tests.yml

@@ -27,7 +27,7 @@ jobs:
 
       - name: Detect flaky test failures
         id: detect
-        uses: anthropics/claude-code-action@567fe954a4527e81f132d87d1bdbcc94f7737434 # main
+        uses: anthropics/claude-code-action@2cc1ac1331eac7a6a96d716dd204dd2888d0fcd2 # main
         with:
           allowed_bots: 'renovate'
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}

.github/workflows/claude-code-review.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: Run Claude Code Review
         id: claude-review
-        uses: anthropics/claude-code-action@567fe954a4527e81f132d87d1bdbcc94f7737434 # v1
+        uses: anthropics/claude-code-action@2cc1ac1331eac7a6a96d716dd204dd2888d0fcd2 # v1
         with:
           allowed_bots: 'renovate'
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}

.github/workflows/claude.yml

@@ -40,7 +40,7 @@ jobs:
 
       - name: Run Claude Code
         id: claude
-        uses: anthropics/claude-code-action@567fe954a4527e81f132d87d1bdbcc94f7737434 # v1
+        uses: anthropics/claude-code-action@2cc1ac1331eac7a6a96d716dd204dd2888d0fcd2 # v1
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
 

.github/workflows/codeql-analysis.yml

@@ -47,12 +47,12 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
+      uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7 # v4
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
+      uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7 # v4
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/flow.yml

@@ -54,13 +54,27 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      catalog2:
+        image: imresamu/postgis:${{ matrix.db-version.pg }}-3.5-alpine
+        ports:
+          - 5437:5432
+        env:
+          PGUSER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+          POSTGRES_INITDB_ARGS: --locale=C.UTF-8
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
       redpanda:
         image: redpandadata/redpanda@sha256:12ddbd8e5572210a48d39afecac44db61503ce1cbdae50ee9b1ff5998a8d91e1
         ports:
           - 9092:9092
           - 9644:9644
       elasticsearch:
-        image: elasticsearch:9.3.2@sha256:56282e41f1d9c8dc437e4c99d132cf9f154def73b6ea8a25ac969a28c2f529c6
+        image: elasticsearch:9.3.3@sha256:1d9ddbe28380c3305ef24c4364168030e4b8ce44834121f7c7dbdaedd2bb9cdf
         ports:
           - 9200:9200
         env:
@@ -87,7 +101,7 @@ jobs:
           - 49002:49002
           - 49003:49003
       openssh:
-        image: linuxserver/openssh-server:latest@sha256:742f577d4100f5ad3b38f270d722931bbe98b997444c13b1a2a838df12a9971e
+        image: linuxserver/openssh-server:latest@sha256:792bb9969a6751f9be2dcecdebd166f75c51187628f1be48ac9c9b3776561c16
         ports:
           - 2222:2222
         env:
@@ -111,11 +125,23 @@ jobs:
           go-version: '1.26.2'
           cache-dependency-path: flow/go.sum
 
-      - name: install lib-geos
+      - name: install lib-geos and pg_dump
         run: |
           # No need to update man pages on package install
           sudo apt-get remove --purge man-db
-          sudo apt-get install libgeos-dev
+          # Add PGDG apt repo for latest PostgreSQL client packages
+          sudo install -d /usr/share/postgresql-common/pgdg
+          sudo curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \
+            --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
+          echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \
+            https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
+            | sudo tee /etc/apt/sources.list.d/pgdg.list
+          sudo apt-get update
+          sudo apt-get install -y libgeos-dev
+          # pg_dump must be >= the server major version; install v18 so it
+          # can dump PG 16, 17, and 18 (backward compatible).
+          sudo apt-get install -y postgresql-client-18
+          echo /usr/lib/postgresql/18/bin >> $GITHUB_PATH
 
       - name: install retry tool
         run: |
@@ -245,6 +271,21 @@ jobs:
           MINIO_ROOT_PASSWORD: miniosecret
           AWS_EC2_METADATA_DISABLED: true
 
+      - name: Generate ClickHouse TLS certificates
+        run: |
+          mkdir -p ch-certs
+          # CA
+          openssl genrsa -out ch-certs/ca.key 2048
+          openssl req -new -x509 -key ch-certs/ca.key -out ch-certs/ca.crt -days 3650 -subj "/CN=ClickHouse-CA"
+          # Server cert (CN=localhost, SAN for TLS 1.3)
+          openssl genrsa -out ch-certs/server.key 2048
+          openssl req -new -key ch-certs/server.key -out ch-certs/server.csr -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1"
+          openssl x509 -req -days 3650 -in ch-certs/server.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/server.crt -copy_extensions copyall
+          # Client cert for mTLS (CN=peerdb-client), using cert-manager naming convention
+          openssl genrsa -out ch-certs/tls.key 2048
+          openssl req -new -key ch-certs/tls.key -out ch-certs/client.csr -subj "/CN=peerdb-client"
+          openssl x509 -req -days 3650 -in ch-certs/client.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/tls.crt
+
       - name: create postgres extensions, increase logical replication limits, and setup catalog database
         run: >
           docker exec "${{ job.services.catalog.id }}" apk add --no-cache build-base git &&
@@ -260,6 +301,17 @@ jobs:
         env:
           PGPASSWORD: postgres
 
+      - name: prepare secondary postgres for cross-cluster schema-dump tests
+        run: >
+          docker exec "${{ job.services.catalog2.id }}" psql -U postgres
+          -c "ALTER SYSTEM SET wal_level=logical;"
+          -c "ALTER SYSTEM SET max_replication_slots=192;"
+          -c "ALTER SYSTEM SET max_wal_senders=256;"
+          -c "ALTER SYSTEM SET max_connections=2048;" &&
+          docker restart "${{ job.services.catalog2.id }}"
+        env:
+          PGPASSWORD: postgres
+
       - name: set ClickHouse version
         id: ch-version
         run: |
@@ -310,16 +362,39 @@ jobs:
                 <access_management>1</access_management>
                 <named_collection_control>1</named_collection_control>
               </default>
+              <peerdb_tls>
+                <ssl_certificates>
+                  <common_name>peerdb-client</common_name>
+                </ssl_certificates>
+                <networks>
+                  <ip>::/0</ip>
+                </networks>
+                <profile>default</profile>
+                <quota>default</quota>
+                <access_management>1</access_management>
+              </peerdb_tls>
             </users>
             <logger><level>none</level></logger>
             <path>var/lib/clickhouse</path>
             <tmp_path>var/lib/clickhouse/tmp</tmp_path>
             <user_files_path>var/lib/clickhouse/user_files</user_files_path>
             <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path>
             <tcp_port>9000</tcp_port>
+            <tcp_port_secure>9440</tcp_port_secure>
             <http_port remove="1"/>
             <postgresql_port remove="1"/>
             <mysql_port remove="1"/>
+            <openSSL>
+              <server>
+                <certificateFile>../ch-certs/server.crt</certificateFile>
+                <privateKeyFile>../ch-certs/server.key</privateKeyFile>
+                <caConfig>../ch-certs/ca.crt</caConfig>
+                <verificationMode>relaxed</verificationMode>
+                <cacheSessions>true</cacheSessions>
+                <disableProtocols>sslv2,sslv3</disableProtocols>
+                <preferServerCiphers>true</preferServerCiphers>
+              </server>
+            </openSSL>
             <macros>
               <shard>1</shard>
               <replica>1</replica>
@@ -365,16 +440,39 @@ jobs:
                 <access_management>1</access_management>
                 <named_collection_control>1</named_collection_control>
               </default>
+              <peerdb_tls>
+                <ssl_certificates>
+                  <common_name>peerdb-client</common_name>
+                </ssl_certificates>
+                <networks>
+                  <ip>::/0</ip>
+                </networks>
+                <profile>default</profile>
+                <quota>default</quota>
+                <access_management>1</access_management>
+              </peerdb_tls>
             </users>
             <logger><level>none</level></logger>
             <path>var/lib/clickhouse</path>
             <tmp_path>var/lib/clickhouse/tmp</tmp_path>
             <user_files_path>var/lib/clickhouse/user_files</user_files_path>
             <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path>
             <tcp_port>9001</tcp_port>
+            <tcp_port_secure>9441</tcp_port_secure>
             <http_port remove="1"/>
             <postgresql_port remove="1"/>
             <mysql_port remove="1"/>
+            <openSSL>
+              <server>
+                <certificateFile>../ch-certs/server.crt</certificateFile>
+                <privateKeyFile>../ch-certs/server.key</privateKeyFile>
+                <caConfig>../ch-certs/ca.crt</caConfig>
+                <verificationMode>relaxed</verificationMode>
+                <cacheSessions>true</cacheSessions>
+                <disableProtocols>sslv2,sslv3</disableProtocols>
+                <preferServerCiphers>true</preferServerCiphers>
+              </server>
+            </openSSL>
             <macros>
               <shard>2</shard>
               <replica>1</replica>
@@ -500,6 +598,11 @@ jobs:
           PG_USER: postgres
           PG_PASSWORD: postgres
           PG_DATABASE: postgres
+          PG2_HOST: localhost
+          PG2_PORT: 5437
+          PG2_USER: postgres
+          PG2_PASSWORD: postgres
+          PG2_DATABASE: postgres
           PEERDB_SWITCHBOARD_ENABLED: "true"
           PEERDB_QUEUE_FORCE_TOPIC_CREATION: "true"
           ELASTICSEARCH_TEST_ADDRESS: http://localhost:9200
@@ -535,6 +638,9 @@ jobs:
           FLOW_TESTS_AWS_ACCESS_KEY_ID: ${{ steps.setup-aws.outputs.aws-access-key-id }}
           FLOW_TESTS_AWS_SECRET_ACCESS_KEY: ${{ steps.setup-aws.outputs.aws-secret-access-key }}
           FLOW_TESTS_AWS_SESSION_TOKEN: ${{ steps.setup-aws.outputs.aws-session-token }}
+          # ClickHouse TLS/mTLS test certificates
+          PEERDB_CLICKHOUSE_TLS_PORT: "9440"
+          PEERDB_CLICKHOUSE_TLS_CERT_DIR: ${{ github.workspace }}/ch-certs
 
       - name: Upload peer-flow logs and test results
         if: always()

.github/workflows/release-drafter.yml

@@ -15,4 +15,4 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-      - uses: release-drafter/release-drafter@5de93583980a40bd78603b6dfdcda5b4df377b32 # v7
+      - uses: release-drafter/release-drafter@563bf132657a13ded0b01fcb723c5a58cdd824e2 # v7

Tiltfile

@@ -130,6 +130,13 @@ local_resource(
     resource_deps=['postgres']
 )
 
+local_resource(
+    'provision-postgres2',
+    cmd='./local_provision_scripts/postgres.sh peerdb-postgres2',
+    labels=['Ancillary-DB-Provisioning'],
+    resource_deps=['postgres2']
+)
+
 # This is not defined as a resource as we need the file to be present
 # when `docker_compose` loads the configuration (next line).
 local('./generate-test-environment.sh')
@@ -167,6 +174,10 @@ dc_resource('postgres', labels=['Ancillary-DB'], links=[
     link('http://localhost:5432', 'PostgreSQL'),
 ], auto_init=False)
 
+dc_resource('postgres2', labels=['Ancillary-DB'], links=[
+    link('http://localhost:5437', 'PostgreSQL (secondary)'),
+], auto_init=False)
+
 local_resource(
     'all-test-resources',
     cmd=' '.join([

ancillary-docker-compose.yml

@@ -125,6 +125,27 @@ services:
       timeout: 10s
       retries: 5
 
+  postgres2:
+    container_name: peerdb-postgres2
+    image: ${POSTGRES_IMAGE}
+    restart: unless-stopped
+    volumes:
+      - postgres2_data:/var/lib/postgresql/data
+    ports:
+      - "${PG2_PORT}:5432"
+    environment:
+      PGUSER: ${PG2_USER}
+      POSTGRES_PASSWORD: ${PG2_PASSWORD}
+      POSTGRES_DB: ${PG2_DATABASE}
+      POSTGRES_INITDB_ARGS: --locale=C.UTF-8
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 2s
+      timeout: 10s
+      retries: 5
+
   dozzle:
     container_name: dozzle-compose-monitor
     image: amir20/dozzle:latest
@@ -185,6 +206,7 @@ services:
 
 volumes:
   postgres_data:
+  postgres2_data:
   clickhouse_data:
   mongo_data:
   mysql_gtid_data: