Skip to content

Commit 1b469a5

Browse files
pfcoperezpfcoperez
committed
Merge remote-tracking branch 'upstream/main' into DBI-793/connectors/prostgres/tls/mutual-authentication
2 parents b129d16 + 3cd6c9c commit 1b469a5

66 files changed

Lines changed: 4170 additions & 1851 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.env.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ CI_CLICKHOUSE_HOST=host.docker.internal
1111
CI_CLICKHOUSE_HTTP_PORT=11123
1212
CI_CLICKHOUSE_NATIVE_PORT=11000
1313

14+
CI_CLICKHOUSE_HTTP_PORT_02=13123
15+
CI_CLICKHOUSE_NATIVE_PORT_02=13000
16+
1417
CI_MYSQL_HOST=host.docker.internal
1518
CI_MYSQL_GTID_PORT=3306
1619
CI_MYSQL_POS_PORT=3307

‎.github/workflows/claude-code-review.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ jobs:
3232
with:
3333
allowed_bots: 'renovate'
3434
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
35+
# Fork PRs get an environment-scoped OIDC token that the Claude App token exchange rejects (401),
36+
# so pass github_token only there; internal PRs keep the App token flow.
37+
github_token: ${{ (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) && github.token || '' }}
3538
claude_args: |
3639
--model claude-opus-4-6
3740
plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'

‎.github/workflows/flow-api-client-tag.yml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
name: flow-api-client-tag
2+
3+
# Manually tags a release of the generated flow-api-client.
4+
#
5+
# The `flow-api-client` workflow keeps the long-lived `flow-api-client` branch in
6+
# sync with the generated proto Go files on every push to `main`. This workflow
7+
# publishes a consumable version by pushing a `flow-api-client/vX.Y.Z` tag that
8+
# points at the current tip of that branch.
9+
#
10+
# NOTE: GitHub Actions cannot dynamically pre-fill a workflow_dispatch input, so
11+
# the version box cannot be seeded with "latest + 1" in the UI. Instead, leave
12+
# the `version` input blank and the workflow auto-increments the patch level of
13+
# the latest existing flow-api-client/vX.Y.Z tag.
14+
15+
on:
16+
workflow_dispatch:
17+
inputs:
18+
version:
19+
description: >-
20+
flow-api-client version to publish (template X.Y.Z, tagged as
21+
flow-api-client/vX.Y.Z). Leave blank to auto-increment the patch level
22+
of the latest existing flow-api-client tag.
23+
Make sure your changes have been merged to the flow-api-client branch before running this workflow.
24+
You can check the history of completed syncs at https://github.com/PeerDB-io/peerdb/actions/workflows/flow-api-client.yml
25+
required: false
26+
type: string
27+
default: ''
28+
29+
concurrency:
30+
group: flow-api-client-tag
31+
cancel-in-progress: false
32+
33+
jobs:
34+
tag:
35+
runs-on: ubuntu-latest
36+
permissions:
37+
contents: write
38+
steps:
39+
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
40+
41+
- name: Fetch flow-api-client branch and existing tags
42+
shell: bash
43+
run: |
44+
set -euo pipefail
45+
# Latest state of the branch we will tag.
46+
git fetch --no-tags origin \
47+
'+refs/heads/flow-api-client:refs/remotes/origin/flow-api-client'
48+
# Only the flow-api-client/* tags (ignores unrelated / malformed tags).
49+
git fetch --no-tags origin \
50+
'+refs/tags/flow-api-client/*:refs/tags/flow-api-client/*'
51+
52+
- name: Resolve version
53+
id: resolve
54+
# Only needed to derive the next version; skipped entirely when an explicit version is given.
55+
if: ${{ inputs.version == '' }}
56+
shell: bash
57+
run: |
58+
set -euo pipefail
59+
60+
latest="$(git tag -l 'flow-api-client/v[0-9]*.[0-9]*.[0-9]*' \
61+
| sed 's|^flow-api-client/v||' \
62+
| grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \
63+
| sort --version-sort \
64+
| tail -1)"
65+
if [ -z "$latest" ]; then
66+
echo "::error::No existing flow-api-client/vX.Y.Z tag found; provide an explicit version input."
67+
exit 1
68+
fi
69+
70+
IFS=. read -r major minor patch <<< "$latest"
71+
version="${major}.${minor}.$((patch + 1))"
72+
echo "Latest tag flow-api-client/v${latest} -> next flow-api-client/v${version}"
73+
echo "version=${version}" >> "$GITHUB_OUTPUT"
74+
75+
- name: Create and push tag
76+
shell: bash
77+
env:
78+
INPUT_VERSION: ${{ inputs.version }}
79+
RESOLVED_VERSION: ${{ steps.resolve.outputs.version }}
80+
run: |
81+
set -euo pipefail
82+
83+
version="${INPUT_VERSION#v}"
84+
version="${version:-$RESOLVED_VERSION}"
85+
86+
if ! printf '%s' "$version" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
87+
echo "::error::Version '${version}' does not match the X.Y.Z template."
88+
exit 1
89+
fi
90+
91+
tag="flow-api-client/v${version}"
92+
if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then
93+
echo "::error::Tag ${tag} already exists."
94+
exit 1
95+
fi
96+
97+
git config user.name "github-actions[bot]"
98+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
99+
100+
target="$(git rev-parse refs/remotes/origin/flow-api-client)"
101+
echo "Tagging flow-api-client tip ${target} as ${tag}"
102+
103+
git tag -a "${tag}" "${target}" -m "${tag}"
104+
git push origin "refs/tags/${tag}"
105+
106+
{
107+
echo "### Published \`${tag}\`"
108+
echo ""
109+
echo "- Branch tip: \`${target}\`"
110+
} >> "$GITHUB_STEP_SUMMARY"

‎.github/workflows/flow.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -721,3 +721,51 @@ jobs:
721721
o11y-api-key-id: ${{ secrets.CI_O11Y_TARGET_API_KEY_ID }}
722722
o11y-api-key-secret: ${{ secrets.CI_O11Y_TARGET_API_KEY_SECRET }}
723723
o11y-query-endpoint: ${{ secrets.CI_O11Y_TARGET_QUERY_ENDPOINT }}
724+
725+
- name: Check all present test functions in the source branch were exercised in CI
726+
# Compares the top-level tests that actually ran in CI (from the
727+
# normalized results produced by the ingest step above) against every top-level
728+
# test defined under ./flow, and fails if the branch defines tests CI never ran.
729+
# Only runs for pull request builds; skipped for non-PR builds (e.g. push to main).
730+
# Parens matter: && binds tighter than || in GitHub Actions expressions.
731+
if: |
732+
(success() || failure()) &&
733+
(github.event_name == 'pull_request' || github.event_name == 'pull_request_target')
734+
working-directory: ./flow
735+
run: |
736+
set -uo pipefail
737+
tmp="${RUNNER_TEMP:-/tmp}"
738+
739+
ndjson="../logs/normalized-test-results.ndjson"
740+
if [ ! -s "$ndjson" ]; then
741+
echo "::error::No normalized test results at ${ndjson} (results ingestion disabled or no tests ran); failing check."
742+
exit 1
743+
fi
744+
745+
# (run-tests): top-level test functions observed in CI results, with subtests
746+
# collapsed onto their parent (TestFoo/sub -> TestFoo).
747+
# This collapse is necessary because the branch test enumeration below only lists top-level tests functions.
748+
jq -r '.test' "$ndjson" | { grep '^Test' || true; } | awk -F '/' '{ print $1 }' | sort -u > "${tmp}/run-tests.txt"
749+
if [ ! -s "${tmp}/run-tests.txt" ]; then
750+
echo "::error::No 'Test*' entries found in CI results; failing check."
751+
exit 1
752+
fi
753+
754+
# (tests-in-branch): every top-level test function defined in the source branch.
755+
if ! go test -list '.*' ./... > "${tmp}/go-list.txt" 2> "${tmp}/go-list.err"; then
756+
echo "::error::'go test -list' failed (likely a compilation error already surfaced by the test step); failing check."
757+
cat "${tmp}/go-list.err"
758+
exit 1
759+
fi
760+
{ grep '^Test' "${tmp}/go-list.txt" || true; } | sort -u > "${tmp}/tests-in-branch.txt"
761+
762+
# Tests defined in the branch but never observed in CI results.
763+
missing="$(comm -13 "${tmp}/run-tests.txt" "${tmp}/tests-in-branch.txt")"
764+
if [ -z "$missing" ]; then
765+
echo "All source-branch tests were exercised in CI."
766+
exit 0
767+
fi
768+
769+
echo "The following tests in the source branch have not been run in CI:"
770+
printf '%s\n' "$missing"
771+
exit 1

‎Tiltfile‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,13 @@ local_resource(
108108
resource_deps=['clickhouse']
109109
)
110110

111+
local_resource(
112+
'provision-clickhouse-cluster',
113+
cmd='./local_provision_scripts/clickhouse-cluster.sh',
114+
labels=['Ancillary-DB-Provisioning'],
115+
resource_deps=['provision-clickhouse', 'clickhouse-02', 'clickhouse-keeper'],
116+
)
117+
111118
local_resource(
112119
'provision-mysql-gtid',
113120
cmd='./local_provision_scripts/mysql.sh peerdb-mysql-gtid',
@@ -164,6 +171,13 @@ local_resource(
164171
resource_deps=['peerdb', 'provision-clickhouse'],
165172
)
166173

174+
local_resource(
175+
'setup-clickhouse-cluster-peer',
176+
cmd='./local_provision_scripts/setup-clickhouse-cluster-peer.sh',
177+
labels=['Setup-PeerDB-Peers'],
178+
resource_deps=['peerdb', 'provision-clickhouse-cluster'],
179+
)
180+
167181
local_resource(
168182
'setup-mongodb-peer',
169183
cmd='./local_provision_scripts/setup-mongodb-peer.sh',
@@ -209,6 +223,13 @@ dc_resource('clickhouse', labels=['Ancillary-DB'], links=[
209223
link('http://localhost:' + resolve_ancillary_env('CI_CLICKHOUSE_NATIVE_PORT', '9000'), 'ClickHouse TCP'),
210224
], auto_init=False)
211225

226+
dc_resource('clickhouse-keeper', labels=['Ancillary-DB'], auto_init=False)
227+
228+
dc_resource('clickhouse-02', labels=['Ancillary-DB'], links=[
229+
link('http://localhost:' + resolve_ancillary_env('CI_CLICKHOUSE_HTTP_PORT_02', '13123'), 'CH Node 2 HTTP'),
230+
link('http://localhost:' + resolve_ancillary_env('CI_CLICKHOUSE_NATIVE_PORT_02', '13000'), 'CH Node 2 TCP'),
231+
], auto_init=False)
232+
212233
dc_resource('mongodb', labels=['Ancillary-DB'], links=[
213234
link('http://localhost:' + resolve_ancillary_env('CI_MONGO_PORT', '27017'), 'MongoDB'),
214235
], auto_init=False)

‎ancillary-docker-compose.yml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,66 @@ services:
4242

4343
clickhouse:
4444
container_name: peerdb-clickhouse
45+
hostname: clickhouse
4546
image: ${CLICKHOUSE_IMAGE}
4647
restart: unless-stopped
4748
ports:
4849
- "${CI_CLICKHOUSE_HTTP_PORT}:8123"
4950
- "${CI_CLICKHOUSE_NATIVE_PORT}:9000"
5051
volumes:
5152
- clickhouse_data:/var/lib/clickhouse/
53+
- ./local_provision_scripts/clickhouse-cluster-config/node-common/config.d/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml:ro
54+
- ./local_provision_scripts/clickhouse-cluster-config/node-common/users.d/default-user.xml:/etc/clickhouse-server/users.d/default-user.xml:ro
55+
- ./local_provision_scripts/clickhouse-cluster-config/node1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml:ro
56+
ulimits:
57+
nofile:
58+
soft: 262144
59+
hard: 262144
60+
extra_hosts:
61+
- "host.docker.internal:host-gateway"
62+
healthcheck:
63+
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8123/ping"]
64+
interval: 2s
65+
timeout: 10s
66+
retries: 5
67+
68+
clickhouse-keeper:
69+
container_name: peerdb-clickhouse-keeper
70+
hostname: clickhouse-keeper
71+
image: ${CLICKHOUSE_IMAGE}
72+
restart: unless-stopped
73+
command: ["clickhouse", "keeper", "--config-file=/etc/clickhouse-keeper/keeper_config.xml"]
74+
volumes:
75+
- clickhouse_keeper_data:/var/lib/clickhouse/
76+
- ./local_provision_scripts/clickhouse-cluster-config/keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml:ro
77+
ulimits:
78+
nofile:
79+
soft: 262144
80+
hard: 262144
81+
extra_hosts:
82+
- "host.docker.internal:host-gateway"
83+
healthcheck:
84+
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/9181 && printf ruok >&3 && timeout 2 cat <&3 | grep -q imok"]
85+
interval: 2s
86+
timeout: 10s
87+
retries: 15
88+
89+
clickhouse-02:
90+
container_name: peerdb-clickhouse-02
91+
hostname: clickhouse-02
92+
image: ${CLICKHOUSE_IMAGE}
93+
restart: unless-stopped
94+
ports:
95+
- "${CI_CLICKHOUSE_HTTP_PORT_02}:8123"
96+
- "${CI_CLICKHOUSE_NATIVE_PORT_02}:9000"
97+
volumes:
98+
- clickhouse_02_data:/var/lib/clickhouse/
99+
- ./local_provision_scripts/clickhouse-cluster-config/node-common/config.d/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml:ro
100+
- ./local_provision_scripts/clickhouse-cluster-config/node-common/users.d/default-user.xml:/etc/clickhouse-server/users.d/default-user.xml:ro
101+
- ./local_provision_scripts/clickhouse-cluster-config/node2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml:ro
102+
depends_on:
103+
clickhouse-keeper:
104+
condition: service_healthy
52105
ulimits:
53106
nofile:
54107
soft: 262144
@@ -231,6 +284,8 @@ volumes:
231284
postgres_data:
232285
postgres2_data:
233286
clickhouse_data:
287+
clickhouse_keeper_data:
288+
clickhouse_02_data:
234289
mongo_data:
235290
mysql_gtid_data:
236291
mysql_pos_data:

‎docker-compose-dev.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ x-flow-worker-env: &flow-worker-env
4747
# https://cloud.google.com/storage/docs/authentication/managing-hmackeys
4848
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
4949
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
50+
# Temporary AWS Credentials for local IAM auth testing
51+
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-}
5052
# For GCS, set this to "auto" without the quotes
5153
AWS_REGION: ${AWS_REGION:-}
5254
# For GCS, set this as: https://storage.googleapis.com

‎e2e_cleanup/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.26.0
55
require (
66
cloud.google.com/go/bigquery v1.74.0
77
cloud.google.com/go/pubsub/v2 v2.3.0
8-
github.com/snowflakedb/gosnowflake/v2 v2.0.2
8+
github.com/snowflakedb/gosnowflake/v2 v2.1.0
99
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78
1010
google.golang.org/api v0.279.0
1111
)

‎e2e_cleanup/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -211,8 +211,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
211211
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
212212
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
213213
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
214-
github.com/snowflakedb/gosnowflake/v2 v2.0.2 h1:8UZo+v1T2Y9sgoPk3JYT3RatAUd9o6q6yjL40TyHluA=
215-
github.com/snowflakedb/gosnowflake/v2 v2.0.2/go.mod h1:c0hIqJ/dxgaMl7g1o8n4Ca3Mf5YCiiVx9igio/PNqC8=
214+
github.com/snowflakedb/gosnowflake/v2 v2.1.0 h1:rfjs6NAMnbLKCBYlOarqQX/UKgQVrXi43TZNHCP5/jw=
215+
github.com/snowflakedb/gosnowflake/v2 v2.1.0/go.mod h1:c0hIqJ/dxgaMl7g1o8n4Ca3Mf5YCiiVx9igio/PNqC8=
216216
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
217217
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
218218
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

‎flow/activities/flowable.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,17 @@ func (a *FlowableActivity) GetQRepPartitions(ctx context.Context,
596596
return nil, a.Alerter.LogFlowError(ctx, config.FlowJobName, shared.WrapError("failed to get partitions from source", err))
597597
}
598598
if len(partitions) > 0 {
599+
shouldOffload, err := internal.PeerDBOffloadPartitionRanges(ctx, config.Env)
600+
if err != nil {
601+
return nil, fmt.Errorf("failed to read offload partition ranges setting: %w", err)
602+
}
603+
if shouldOffload && config.InitialCopyOnly {
604+
if err := connmetadata.OffloadPartitionRanges(
605+
ctx, a.CatalogPool, config.ParentMirrorName, runUUID, partitions,
606+
); err != nil {
607+
return nil, fmt.Errorf("failed to offload partition ranges: %w", err)
608+
}
609+
}
599610
if err := monitoring.InitializeQRepRun(
600611
ctx,
601612
logger,
@@ -663,6 +674,10 @@ func (a *FlowableActivity) ReplicateQRepPartitions(ctx context.Context,
663674
return a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
664675
}
665676

677+
if err := connmetadata.RestoreOffloadedPartitionRanges(ctx, a.CatalogPool, runUUID, partitions.Partitions); err != nil {
678+
return fmt.Errorf("failed to rehydrate partition ranges: %w", err)
679+
}
680+
666681
for i, partition := range partitions.Partitions {
667682
partLogger := log.With(logger,
668683
slog.Int64("batchID", int64(partitions.BatchId)),
@@ -1623,6 +1638,10 @@ func (a *FlowableActivity) QRepHasNewRows(ctx context.Context,
16231638
if maxValue.(time.Time).After(x.TimestampRange.End.AsTime()) {
16241639
return true, nil
16251640
}
1641+
case *protos.PartitionRange_StringRange:
1642+
// checking for new rows is only possible for standalone QRepFlowWorkflow;
1643+
// this is a legacy feature and string partitioning is not supported
1644+
return false, errors.New("checking for new rows by a string partition range is not supported")
16261645
default:
16271646
return false, fmt.Errorf("unknown range type: %v", x)
16281647
}

0 commit comments

Comments
 (0)