From 2dca815fbc982472967727f485f79b8300f7cf4c Mon Sep 17 00:00:00 2001 From: Jeremy Yang Date: Wed, 24 Jun 2026 15:03:15 -0700 Subject: [PATCH 1/5] Deprecate non-ClickHouse / non-Postgres destinations: docs + UI Signal deprecation of the Snowflake, BigQuery, ElasticSearch, Kafka (incl. Confluent/Redpanda), Azure Event Hubs, Google Pub/Sub, and S3 *destination* connectors. No code is removed; they remain fully functional. Actively-maintained paths: Postgres -> ClickHouse / ClickHouse Cloud / Postgres. - README: deprecation notice (above the fold) + connector status matrix; BigQuery is listed as a maintained source and a deprecated destination. - docs/deprecated-connectors.md: pin + fork migration guide. - CONTRIBUTING.md: links to the migration guide. - UI: "Deprecated" badge in the peer-creation grid. BigQuery is dual-role, so it is marked destination-only ("Deprecated*" + footnote) since it is still a valid source peer. Refs PeerDB-io/peerdb#4465 Co-Authored-By: Claude Opus 4.8 (1M context) --- CONTRIBUTING.md | 9 ++++ README.md | 23 ++++++++- docs/deprecated-connectors.md | 90 ++++++++++++++++++++++++++++++++++ ui/app/api/peer-types/route.ts | 40 +++++++++------ ui/components/SelectSource.tsx | 87 ++++++++++++++++++++++++++------ 5 files changed, 217 insertions(+), 32 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 docs/deprecated-connectors.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..dfb9cd043 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,9 @@ +# Contributing to PeerDB + +Thanks for your interest in contributing to PeerDB! Bug reports, feature requests, and pull requests are all welcome. If you have a question, feel free to drop by our [Slack](https://slack.peerdb.io/). + +## Deprecated connectors + +Several destination connectors (Snowflake, BigQuery, ElasticSearch, Kafka including Confluent and Redpanda, Azure Event Hubs, and Google Pub/Sub) are deprecated and no longer actively maintained. They remain fully functional, and no code is being removed. + +If you depend on one of these connectors, see the [deprecated connectors migration guide](docs/deprecated-connectors.md) for how to pin to a release or fork the relevant connector code. diff --git a/README.md b/README.md index a753a465f..271c340a7 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ At PeerDB, we are building a fast, simple and the most cost effective way to str We support different modes of streaming - log based (CDC), cursor based (timestamp or integer) and XMIN based. Performance wise, we are 10x faster than existing tools. Features wise, we support native Postgres features such as comprehensive set of data-types incl. jsonb/arrays/geospatial, efficiently streaming TOAST columns, schema changes and so on. +> **Connector deprecation notice:** The Snowflake, BigQuery, ElasticSearch, Kafka (including Confluent and Redpanda variants), Azure Event Hubs, Google Pub/Sub, and S3 **destination** connectors are deprecated and no longer actively maintained. They remain fully functional in the current and all prior releases, and no code is being removed. The actively-maintained paths going forward are Postgres → ClickHouse, Postgres → ClickHouse Cloud, and Postgres → Postgres. This deprecation applies to the **destination** role only — BigQuery remains a supported **source**. See the [connector status matrix](#status) and the [deprecated connectors migration guide](docs/deprecated-connectors.md) for how to pin to a release or fork the relevant code. + ## Get started ```bash @@ -89,7 +91,26 @@ You can use Postgres’ ecosystem to manage your ETL — ## Status -We have expanded our connector ecosystem to support multiple source connectors beyond Postgres, including MySQL and MongoDB. You can check the status of connectors [here](https://docs.peerdb.io/sql/commands/supported-connectors) +We have expanded our connector ecosystem to support multiple source connectors beyond Postgres, including MySQL and MongoDB. You can check the status of connectors [here](https://docs.peerdb.io/sql/commands/supported-connectors). + +| Connector | Direction (source/destination) | Status | +| --- | --- | --- | +| Postgres | Source | **Actively maintained** | +| MySQL | Source | **Actively maintained** | +| MongoDB | Source | **Actively maintained** | +| BigQuery | Source | **Actively maintained** | +| ClickHouse | Destination | **Actively maintained** | +| ClickHouse Cloud | Destination | **Actively maintained** | +| Postgres | Destination | **Actively maintained** | +| S3 | Destination | **Deprecated** | +| Snowflake | Destination | **Deprecated** | +| BigQuery | Destination | **Deprecated** | +| ElasticSearch | Destination | **Deprecated** | +| Kafka | Destination | **Deprecated** | +| Redpanda | Destination | **Deprecated** | +| Confluent | Destination | **Deprecated** | +| Azure Event Hubs | Destination | **Deprecated** | +| Google Pub/Sub | Destination | **Deprecated** | ## Local End to End testing diff --git a/docs/deprecated-connectors.md b/docs/deprecated-connectors.md new file mode 100644 index 000000000..502b07f40 --- /dev/null +++ b/docs/deprecated-connectors.md @@ -0,0 +1,90 @@ +# Deprecated connectors migration guide + +The following **destination** connectors are deprecated and no longer actively maintained: + +- Snowflake +- BigQuery (destination role only — see below) +- ElasticSearch +- Kafka (including Confluent and Redpanda variants) +- Azure Event Hubs +- Google Pub/Sub +- S3 + +These connectors remain **fully functional** in the current and all prior releases. No code is being removed, and they will **not** be automatically removed from existing deployments. The actively-maintained paths going forward are Postgres → ClickHouse, Postgres → ClickHouse Cloud, and Postgres → Postgres. Sources (Postgres, MySQL, MongoDB) are not affected by this deprecation. + +> **BigQuery is deprecated only as a destination.** BigQuery remains a **supported source** for QRep mirrors and is not deprecated in that role. The migration guidance below applies to BigQuery's use as a *destination*. + +If you rely on a deprecated destination, you have two supported options to keep using it: pin to a release that includes it, or fork the relevant connector code. + +## How to pin to the last actively-maintained release + +Deprecated connectors continue to ship in current and prior releases, so the simplest way to keep using one is to pin your deployment to a specific release rather than tracking the latest mainline. + +- **Docker / docker-compose:** pin the PeerDB image tags to an explicit release tag instead of a moving tag like `latest`. In your `docker-compose.yml`, replace floating tags with the latest release tag that still includes the connector you depend on. +- **Helm chart:** pin the chart version and the image tags in your `values.yaml` to that same release. + +Pin to the latest release tag before any future removal so that you have the most recent fixes while retaining the connector. Once pinned, validate your pipelines and avoid upgrading past a release that no longer carries the connector you need. + +## How to fork + +Because no code is being removed, you can fork PeerDB and carry the relevant connector code yourself. The connectors are self-contained enough that forking and maintaining a single connector is practical: + +1. Fork the repository. +2. Keep the connector's flow code, its UI form and helper, and (optionally) its end-to-end tests so you can continue validating it. +3. Rebase your fork periodically onto upstream `main` to pick up fixes to shared infrastructure. + +The sections below list the exact code paths to carry for each deprecated connector. + +### Snowflake + +- `flow/connectors/snowflake/` +- `ui/components/PeerForms/SnowflakeForm.tsx` +- `ui/app/peers/create/[peerType]/helpers/sf.ts` +- `flow/e2e/snowflake*.go` + +### BigQuery (destination only) + +> Deprecated only as a **destination**. BigQuery is still a supported **source**, so keep this code if you use BigQuery as a source. + +- `flow/connectors/bigquery/` +- `ui/components/PeerForms/BigqueryConfig.tsx` +- `ui/app/peers/create/[peerType]/helpers/bq.ts` +- `flow/e2e/bigquery.go` +- `flow/e2e/bigquery_qrep_test.go` +- `flow/e2e/bigquery_test.go` + +### ElasticSearch + +- `flow/connectors/elasticsearch/` +- `ui/components/PeerForms/ElasticsearchConfigForm.tsx` +- `ui/app/peers/create/[peerType]/helpers/es.ts` +- `flow/e2e/elasticsearch*.go` + +### Azure Event Hubs + +- `flow/connectors/eventhub/` +- `ui/components/PeerForms/Eventhubs/` +- `ui/app/peers/create/[peerType]/helpers/eh.ts` +- `ui/app/mirrors/create/cdc/eventhubsCallout.tsx` +- `flow/e2e/eventhub*.go` + +### Kafka (including Confluent and Redpanda) + +- `flow/connectors/kafka/` +- `ui/components/PeerForms/KafkaConfig.tsx` +- `ui/app/peers/create/[peerType]/helpers/ka.ts` +- `flow/e2e/kafka*.go` + +### Google Pub/Sub + +- `flow/connectors/pubsub/` +- `ui/components/PeerForms/PubSubConfig.tsx` +- `ui/app/peers/create/[peerType]/helpers/ps.ts` +- `flow/e2e/pubsub*.go` + +### S3 + +- `flow/connectors/s3/` +- `ui/components/PeerForms/S3Form.tsx` +- `ui/app/peers/create/[peerType]/helpers/s3.ts` +- `flow/e2e/s3_*.go` diff --git a/ui/app/api/peer-types/route.ts b/ui/app/api/peer-types/route.ts index 66fc0956d..9332ca1fc 100644 --- a/ui/app/api/peer-types/route.ts +++ b/ui/app/api/peer-types/route.ts @@ -2,28 +2,36 @@ import { GetPeerDBClickHouseMode } from '@/peerdb-env/allowed_targets'; import { NextRequest } from 'next/server'; export const dynamic = 'force-dynamic'; +type PeerTypeItem = { + label: string; + url?: string; + deprecated?: boolean; + // When deprecation only applies to the connector's use as a destination + // (e.g. BigQuery is still a supported source), the badge reflects that. + deprecatedRole?: 'destination'; +}; +type PeerTypeCategory = [string, ...Array]; + export async function GET(request: NextRequest) { - const allWarehouseTypes = [ + const allWarehouseTypes: PeerTypeCategory = [ 'Warehouses', - 'SNOWFLAKE', - 'BIGQUERY', - 'S3', + { label: 'SNOWFLAKE', deprecated: true }, + // BigQuery remains a supported source; only its destination role is deprecated. + { label: 'BIGQUERY', deprecated: true, deprecatedRole: 'destination' }, + { label: 'S3', deprecated: true }, 'CLICKHOUSE', - 'ELASTICSEARCH', + { label: 'ELASTICSEARCH', deprecated: true }, ]; - const clickhouseWarehouseTypes = ['Targets', 'CLICKHOUSE']; - const queueTypes = [ + const clickhouseWarehouseTypes: PeerTypeCategory = ['Targets', 'CLICKHOUSE']; + const queueTypes: PeerTypeCategory = [ 'Queues', - 'REDPANDA', - 'CONFLUENT', - 'KAFKA', - 'EVENTHUBS', - 'PUBSUB', + { label: 'REDPANDA', deprecated: true }, + { label: 'CONFLUENT', deprecated: true }, + { label: 'KAFKA', deprecated: true }, + { label: 'EVENTHUBS', deprecated: true }, + { label: 'PUBSUB', deprecated: true }, ]; - const postgresTypes: [ - string, - ...Array, - ] = [ + const postgresTypes: PeerTypeCategory = [ 'Sources', 'MYSQL', 'POSTGRESQL', diff --git a/ui/components/SelectSource.tsx b/ui/components/SelectSource.tsx index 55811234c..9062e5fd8 100644 --- a/ui/components/SelectSource.tsx +++ b/ui/components/SelectSource.tsx @@ -1,7 +1,9 @@ 'use client'; import { fetcher } from '@/app/utils/swr'; import TitleCase from '@/app/utils/titlecase'; +import { Badge } from '@/lib/Badge'; import { Button } from '@/lib/Button/Button'; +import { Label } from '@/lib/Label'; import { ProgressCircle } from '@/lib/ProgressCircle'; import Image from 'next/image'; import Link from 'next/link'; @@ -10,9 +12,23 @@ import useSWR from 'swr'; import { DBTypeToImageMapping } from './PeerComponent'; // label corresponds to PeerType -function SourceLabel({ label, url }: { label: string; url?: string }) { +function SourceLabel({ + label, + url, + deprecated, + deprecatedRole, +}: { + label: string; + url?: string; + deprecated?: boolean; + deprecatedRole?: 'destination'; +}) { const theme = useStyledTheme(); const peerLogo = DBTypeToImageMapping(label); + // Uniform "Deprecated" badge; an asterisk flags connectors deprecated only + // as a destination (explained by the footnote below the category). + const deprecatedText = + deprecatedRole === 'destination' ? 'Deprecated*' : 'Deprecated'; return ( ); } @@ -65,23 +90,55 @@ export default function SelectSource() { paddingLeft: '10px', paddingRight: '10px', } as const; + + const footnoteStyle = { + flexBasis: '100%', + fontSize: '12px', + color: theme.colors.base.text.lowContrast, + } as const; const { data: dbTypes, isLoading } = useSWR< - [string, ...Array][] + [ + string, + ...Array< + | string + | { + label: string; + url?: string; + deprecated?: boolean; + deprecatedRole?: 'destination'; + } + >, + ][] >('/api/peer-types', fetcher); if (!dbTypes || isLoading) { return ; } - return dbTypes.map(([category, ...items]) => ( -
-
{category}
- {items.map((item, i) => - typeof item === 'string' ? ( - - ) : ( - - ) - )} -
- )); + return dbTypes.map(([category, ...items]) => { + const hasDestinationDeprecation = items.some( + (item) => + typeof item !== 'string' && item.deprecatedRole === 'destination' + ); + return ( +
+
{category}
+ {items.map((item, i) => + typeof item === 'string' ? ( + + ) : ( + + ) + )} + {hasDestinationDeprecation && ( +
* Deprecated as a destination only.
+ )} +
+ ); + }); } From 0e50c1c5b504c776f0c690badaee34a0dce338a3 Mon Sep 17 00:00:00 2001 From: Jeremy Yang Date: Wed, 24 Jun 2026 15:03:15 -0700 Subject: [PATCH 2/5] Add deprecated-connector GitHub automation - Issue chooser offers only the Bug report form, which includes a destination-connector dropdown and a deprecation notice. - labeler.yml: label PRs touching deprecated destination connectors. List BigQuery destination files explicitly so BigQuery *source* changes are never auto-labeled (a `!`-negation / all-globs-to-all-files block was found to silently disable the whole actions/labeler v5 config). - workflow: apply `deprecated` to bug reports that select a deprecated destination, label deprecated-connector PRs, and post a single de-duplicated notice. It never auto-closes. The `deprecated` label must be created once in repository settings. Refs PeerDB-io/peerdb#4465 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/ISSUE_TEMPLATE/bug_report.yml | 70 ++++++++++ .github/ISSUE_TEMPLATE/config.yml | 1 + .github/labeler.yml | 52 ++++++++ .github/workflows/deprecated-connector.yml | 147 +++++++++++++++++++++ 4 files changed, 270 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/labeler.yml create mode 100644 .github/workflows/deprecated-connector.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..e8303290a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,70 @@ +name: Bug report +description: Report a problem with PeerDB +title: "[Bug]: " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to file a bug report! Please fill out the fields below so we can reproduce and triage the issue. + + - type: textarea + id: description + attributes: + label: Description + description: A clear and concise description of what the bug is, including the expected behavior and what actually happened. + placeholder: When I run a CDC mirror to ..., I expected ... but instead ... + validations: + required: true + + - type: textarea + id: steps_to_reproduce + attributes: + label: Steps to reproduce + description: Step-by-step instructions to reproduce the behavior. + placeholder: | + 1. Create a peer with ... + 2. Create a mirror with ... + 3. Observe ... + validations: + required: true + + - type: input + id: version + attributes: + label: PeerDB version + description: The PeerDB version (or Docker image tag / commit SHA) you are running. + placeholder: e.g. v0.30.0 or stable-dev + validations: + required: true + + - type: markdown + attributes: + value: | + ### ⚠️ Deprecated destination connectors + + The following destination connectors are **deprecated**: **Snowflake**, **BigQuery**, **ElasticSearch**, **Kafka**, **Redpanda**, **Confluent**, **Azure Event Hubs**, **Google Pub/Sub**, and **S3**. + + These destinations are no longer actively maintained, but remain functional. Bugs affecting only these destinations are unlikely to be prioritized. Note: BigQuery is deprecated **only as a destination** — it remains a supported source. See the [deprecated connectors documentation](https://github.com/PeerDB-io/peerdb/blob/main/docs/deprecated-connectors.md) ([../../docs/deprecated-connectors.md](../../docs/deprecated-connectors.md)) for details and migration guidance. + + - type: dropdown + id: destination_connector + attributes: + label: Destination connector + description: Which destination connector is affected? (See the deprecation notice above.) + options: + - ClickHouse + - ClickHouse Cloud + - Postgres + - S3 + - Snowflake + - BigQuery + - ElasticSearch + - Kafka + - Redpanda + - Confluent + - Azure Event Hubs + - Google Pub/Sub + - Other / N/A + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..3ba13e0ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 000000000..650aa6f3e --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,52 @@ +# Configuration for actions/labeler@v5. +# Applies the `deprecated` label to PRs that touch deprecated destination connectors: +# Snowflake, BigQuery, ElasticSearch, Kafka (incl. Confluent/Redpanda), Azure Event Hubs, Google Pub/Sub. +# S3 and ClickHouse are intentionally excluded. +deprecated: + - changed-files: + - any-glob-to-any-file: + # Destination-only connectors — the whole connector is deprecated. + - flow/connectors/snowflake/** + - flow/connectors/elasticsearch/** + - flow/connectors/eventhub/** + - flow/connectors/kafka/** + - flow/connectors/pubsub/** + - flow/connectors/s3/** + # End-to-end tests + - flow/e2e/snowflake*.go + - flow/e2e/elasticsearch*.go + - flow/e2e/eventhub*.go + - flow/e2e/kafka*.go + - flow/e2e/pubsub*.go + - flow/e2e/s3_*.go + # UI peer forms + - ui/components/PeerForms/SnowflakeForm.tsx + - ui/components/PeerForms/ElasticsearchConfigForm.tsx + - ui/components/PeerForms/KafkaConfig.tsx + - ui/components/PeerForms/PubSubConfig.tsx + - ui/components/PeerForms/S3Form.tsx + - ui/components/PeerForms/Eventhubs/** + # UI peer-create helpers. The literal `[` in the [peerType] dynamic + # route segment is escaped as `[[]` so minimatch treats it literally + # rather than as a character class. + - ui/app/peers/create/[[]peerType]/helpers/sf.ts + - ui/app/peers/create/[[]peerType]/helpers/es.ts + - ui/app/peers/create/[[]peerType]/helpers/eh.ts + - ui/app/peers/create/[[]peerType]/helpers/ka.ts + - ui/app/peers/create/[[]peerType]/helpers/ps.ts + - ui/app/peers/create/[[]peerType]/helpers/s3.ts + # BigQuery is dual-role: only its DESTINATION is deprecated, while its + # source is actively maintained and lives in the same directory. List + # the destination-specific files explicitly so BigQuery *source* + # changes (source.go, source_schema.go, qrep_object_pull.go, + # bigquery_source_test.go) are never auto-labeled as deprecated. + # (A `!`-negation / `all-globs-to-all-files` block was tried first but + # silently disables the whole labeler config in actions/labeler v5.) + - flow/connectors/bigquery/bigquery.go + - flow/connectors/bigquery/merge_stmt_generator.go + - flow/connectors/bigquery/qrep_avro_sync.go + - flow/connectors/bigquery/clustering.go + - flow/e2e/bigquery_test.go + - flow/e2e/bigquery_qrep_test.go + - ui/components/PeerForms/BigqueryConfig.tsx + - ui/app/peers/create/[[]peerType]/helpers/bq.ts diff --git a/.github/workflows/deprecated-connector.yml b/.github/workflows/deprecated-connector.yml new file mode 100644 index 000000000..899a35175 --- /dev/null +++ b/.github/workflows/deprecated-connector.yml @@ -0,0 +1,147 @@ +name: Deprecated connector + +# - Labels PRs that touch deprecated destination connectors (actions/labeler). +# - On bug reports, applies the `deprecated` label only when the selected +# destination connector (issue-form dropdown) is a deprecated one. +# - Posts a single informational notice on any issue/PR carrying the +# `deprecated` label. It never closes the issue or PR. +# +# NOTE: the `deprecated` label must be created once in the repository settings +# (Settings -> Labels) — labels are not defined in this repo. + +on: + pull_request_target: + types: [opened, synchronize, reopened] + pull_request: + types: [labeled] + issues: + types: [opened, edited, labeled] + +permissions: + contents: read + +jobs: + # Apply path-based labels (incl. `deprecated`) to PRs per .github/labeler.yml. + label: + if: github.event_name == 'pull_request_target' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0 + with: + sync-labels: false + + # On a new/edited bug report, apply `deprecated` only when the chosen + # destination connector (issue-form dropdown) is a deprecated one. Issues + # that select an actively-maintained destination are left untouched. + triage-issue: + if: github.event_name == 'issues' && (github.event.action == 'opened' || github.event.action == 'edited') + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const issue = context.payload.issue; + const body = issue.body || ''; + // Extract the "Destination connector" dropdown value from the form body. + const match = body.match(/###\s*Destination connector\s*\r?\n+\s*(.+)/); + const selection = match ? match[1].trim() : ''; + const deprecated = new Set([ + 'Snowflake', 'BigQuery', 'ElasticSearch', 'Kafka', 'Redpanda', + 'Confluent', 'Azure Event Hubs', 'Google Pub/Sub', 'S3', + ]); + if (!deprecated.has(selection)) { + core.info(`Destination "${selection || '(none)'}" is not a deprecated connector; not labeling.`); + return; + } + if ((issue.labels || []).some((l) => (l.name || l) === 'deprecated')) { + core.info('Issue already labeled deprecated.'); + return; + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + labels: ['deprecated'], + }); + core.info(`Labeled issue #${issue.number} deprecated (destination: ${selection}).`); + + # Post a one-time notice when the `deprecated` label is present on an issue/PR. + # Depends on the prior jobs so it observes labels they applied within this same + # run (a GITHUB_TOKEN label write does not itself emit a `labeled` event). + # `if: always()` keeps it running when an upstream job is skipped. + notice: + needs: [label, triage-issue] + if: always() + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const marker = ''; + const number = + context.eventName === 'issues' + ? context.payload.issue?.number + : context.payload.pull_request?.number; + if (!number) { + core.info('No issue/PR number in payload; nothing to do.'); + return; + } + + // Re-fetch labels so we observe any applied by the label/triage jobs + // earlier in this run (rather than the stale event payload). + const { data: labels } = await github.rest.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: number, + }); + if (!labels.some((l) => l.name === 'deprecated')) { + core.info('No deprecated label present; nothing to do.'); + return; + } + + // Guard against duplicate comments using the hidden marker. + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: number, + }); + if (comments.some((c) => c.body && c.body.includes(marker))) { + core.info('Deprecated-connector notice already posted; skipping.'); + return; + } + + const body = [ + marker, + '### ⚠️ Deprecated destination connector', + '', + 'This issue or pull request relates to a **deprecated** destination connector ' + + '(Snowflake, BigQuery, ElasticSearch, Kafka, Redpanda, Confluent, Azure Event Hubs, Google Pub/Sub, or S3).', + '', + 'These destinations are **no longer actively maintained**, but remain functional. ' + + 'We are unlikely to prioritize new work here.', + '', + '> Note: BigQuery is deprecated **only as a destination** — it remains a supported source.', + '', + 'If you depend on one of these connectors, we recommend:', + '', + '- **Pin** to a known-good PeerDB version so behavior stays stable.', + '- **Fork** the repository if you need to carry your own changes.', + '', + 'See the [deprecated connectors documentation](https://github.com/PeerDB-io/peerdb/blob/main/docs/deprecated-connectors.md) for details and migration guidance.', + ].join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: number, + body, + }); + core.info(`Posted deprecated-connector notice on #${number}.`); From 21ff46eda1dd9dd165ea4d29c823abab112087fa Mon Sep 17 00:00:00 2001 From: Jeremy Yang Date: Wed, 24 Jun 2026 15:43:22 -0700 Subject: [PATCH 3/5] Address review (ilidemi) - labeler.yml: header comment lists S3 as deprecated and notes ClickHouse/Postgres as the excluded (maintained) destinations; add the BigQuery destination QRep file (qrep.go) to the tracked paths. - CONTRIBUTING.md: add S3 to the deprecated list; note BigQuery is deprecated only as a destination. - bug_report.yml: drop the duplicate migration-guide link. - feature_request.yml: new issue-form template so feature requests have a home while the chooser stays curated (blank issues remain disabled). - Issue forms: add an optional "Source connector" dropdown and make the destination dropdown optional, so source-side issues aren't forced to pick a destination (GitHub issue forms have no conditional fields). The destination dropdown keeps its exact label so the deprecation-triage workflow still parses it. Refs PeerDB-io/peerdb#4465 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/ISSUE_TEMPLATE/bug_report.yml | 22 +++++-- .github/ISSUE_TEMPLATE/feature_request.yml | 77 ++++++++++++++++++++++ .github/labeler.yml | 5 +- CONTRIBUTING.md | 2 +- 4 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index e8303290a..9c6d7d3ac 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -38,6 +38,20 @@ body: validations: required: true + - type: dropdown + id: source_connector + attributes: + label: Source connector + description: If this issue relates to a source connector, select it here. Leave blank if not applicable. + options: + - Postgres + - MySQL + - MongoDB + - BigQuery + - Other + validations: + required: false + - type: markdown attributes: value: | @@ -45,13 +59,13 @@ body: The following destination connectors are **deprecated**: **Snowflake**, **BigQuery**, **ElasticSearch**, **Kafka**, **Redpanda**, **Confluent**, **Azure Event Hubs**, **Google Pub/Sub**, and **S3**. - These destinations are no longer actively maintained, but remain functional. Bugs affecting only these destinations are unlikely to be prioritized. Note: BigQuery is deprecated **only as a destination** — it remains a supported source. See the [deprecated connectors documentation](https://github.com/PeerDB-io/peerdb/blob/main/docs/deprecated-connectors.md) ([../../docs/deprecated-connectors.md](../../docs/deprecated-connectors.md)) for details and migration guidance. + These destinations are no longer actively maintained, but remain functional. Bugs affecting only these destinations are unlikely to be prioritized. Note: BigQuery is deprecated **only as a destination** — it remains a supported source. See the [deprecated connectors documentation](https://github.com/PeerDB-io/peerdb/blob/main/docs/deprecated-connectors.md) for details and migration guidance. - type: dropdown id: destination_connector attributes: label: Destination connector - description: Which destination connector is affected? (See the deprecation notice above.) + description: If this issue relates to a destination connector, select it here (see the deprecation notice above). Leave blank if not applicable. options: - ClickHouse - ClickHouse Cloud @@ -65,6 +79,6 @@ body: - Confluent - Azure Event Hubs - Google Pub/Sub - - Other / N/A + - Other validations: - required: true + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..c2bc38e66 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,77 @@ +name: Feature request +description: Suggest a feature or improvement for PeerDB +title: "[Feature]: " +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Thanks for suggesting an improvement! Please describe the problem you are trying to solve and what you would like to happen. + + - type: textarea + id: problem + attributes: + label: Problem / motivation + description: What problem are you trying to solve, and what is the use case? + placeholder: When replicating ..., I need ... but currently ... + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed solution + description: Describe what you would like to happen. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Any alternative solutions or workarounds you have considered. + validations: + required: false + + - type: dropdown + id: source_connector + attributes: + label: Source connector + description: If this request relates to a source connector, select it here. Leave blank if not applicable. + options: + - Postgres + - MySQL + - MongoDB + - BigQuery + - Other + validations: + required: false + + - type: markdown + attributes: + value: | + ### ⚠️ Deprecated destination connectors + + The following destination connectors are **deprecated** and no longer actively maintained: **Snowflake**, **BigQuery**, **ElasticSearch**, **Kafka**, **Redpanda**, **Confluent**, **Azure Event Hubs**, **Google Pub/Sub**, and **S3**. New features for these destinations are unlikely to be prioritized. Note: BigQuery is deprecated **only as a destination** — it remains a supported source. See the [deprecated connectors documentation](https://github.com/PeerDB-io/peerdb/blob/main/docs/deprecated-connectors.md). + + - type: dropdown + id: destination_connector + attributes: + label: Destination connector + description: If this request relates to a destination connector, select it here (see the deprecation notice above). Leave blank if not applicable. + options: + - ClickHouse + - ClickHouse Cloud + - Postgres + - S3 + - Snowflake + - BigQuery + - ElasticSearch + - Kafka + - Redpanda + - Confluent + - Azure Event Hubs + - Google Pub/Sub + - Other + validations: + required: false diff --git a/.github/labeler.yml b/.github/labeler.yml index 650aa6f3e..da032aa64 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,7 +1,7 @@ # Configuration for actions/labeler@v5. # Applies the `deprecated` label to PRs that touch deprecated destination connectors: -# Snowflake, BigQuery, ElasticSearch, Kafka (incl. Confluent/Redpanda), Azure Event Hubs, Google Pub/Sub. -# S3 and ClickHouse are intentionally excluded. +# Snowflake, BigQuery, ElasticSearch, Kafka (incl. Confluent/Redpanda), Azure Event Hubs, Google Pub/Sub, S3. +# ClickHouse and Postgres (the actively-maintained destinations) are intentionally excluded. deprecated: - changed-files: - any-glob-to-any-file: @@ -44,6 +44,7 @@ deprecated: # silently disables the whole labeler config in actions/labeler v5.) - flow/connectors/bigquery/bigquery.go - flow/connectors/bigquery/merge_stmt_generator.go + - flow/connectors/bigquery/qrep.go - flow/connectors/bigquery/qrep_avro_sync.go - flow/connectors/bigquery/clustering.go - flow/e2e/bigquery_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dfb9cd043..af4fe1b7c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,6 @@ Thanks for your interest in contributing to PeerDB! Bug reports, feature request ## Deprecated connectors -Several destination connectors (Snowflake, BigQuery, ElasticSearch, Kafka including Confluent and Redpanda, Azure Event Hubs, and Google Pub/Sub) are deprecated and no longer actively maintained. They remain fully functional, and no code is being removed. +Several destination connectors (Snowflake, BigQuery, ElasticSearch, Kafka including Confluent and Redpanda, Azure Event Hubs, Google Pub/Sub, and S3) are deprecated and no longer actively maintained. They remain fully functional, and no code is being removed. (BigQuery is deprecated only as a destination — it remains a supported source.) If you depend on one of these connectors, see the [deprecated connectors migration guide](docs/deprecated-connectors.md) for how to pin to a release or fork the relevant connector code. From 1e94bbf27f70e930b6b29d501a568371aa804435 Mon Sep 17 00:00:00 2001 From: Jeremy Yang Date: Thu, 25 Jun 2026 13:16:37 -0700 Subject: [PATCH 4/5] Deprecate QRep and XMin mirror types Mark the Query Replication (QRep) and XMin mirror types as deprecated; CDC is the recommended, actively-maintained mirror type. They remain fully functional and no code is currently being removed. - ui/app/api/mirror-types: flag QRep and XMin cards as deprecated. - ui/app/mirrors/create/mirrorcards: render a "Deprecated" badge on those cards in the create-mirror flow. - README: add a mirror-type deprecation notice; soften "no code is being removed" to "currently" across the connector/mirror notices. - CONTRIBUTING.md, docs/deprecated-connectors.md: same "currently" wording. Refs PeerDB-io/peerdb#4465 Co-Authored-By: Claude Opus 4.8 (1M context) --- CONTRIBUTING.md | 2 +- README.md | 6 ++++-- docs/deprecated-connectors.md | 4 ++-- ui/app/api/mirror-types/route.ts | 2 ++ ui/app/mirrors/create/mirrorcards.tsx | 23 ++++++++++++++++++++++- 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af4fe1b7c..61b13bfdd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,6 @@ Thanks for your interest in contributing to PeerDB! Bug reports, feature request ## Deprecated connectors -Several destination connectors (Snowflake, BigQuery, ElasticSearch, Kafka including Confluent and Redpanda, Azure Event Hubs, Google Pub/Sub, and S3) are deprecated and no longer actively maintained. They remain fully functional, and no code is being removed. (BigQuery is deprecated only as a destination — it remains a supported source.) +Several destination connectors (Snowflake, BigQuery, ElasticSearch, Kafka including Confluent and Redpanda, Azure Event Hubs, Google Pub/Sub, and S3) are deprecated and no longer actively maintained. They remain fully functional, and no code is currently being removed. (BigQuery is deprecated only as a destination — it remains a supported source.) If you depend on one of these connectors, see the [deprecated connectors migration guide](docs/deprecated-connectors.md) for how to pin to a release or fork the relevant connector code. diff --git a/README.md b/README.md index 271c340a7..623a0b593 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,9 @@ At PeerDB, we are building a fast, simple and the most cost effective way to str We support different modes of streaming - log based (CDC), cursor based (timestamp or integer) and XMIN based. Performance wise, we are 10x faster than existing tools. Features wise, we support native Postgres features such as comprehensive set of data-types incl. jsonb/arrays/geospatial, efficiently streaming TOAST columns, schema changes and so on. -> **Connector deprecation notice:** The Snowflake, BigQuery, ElasticSearch, Kafka (including Confluent and Redpanda variants), Azure Event Hubs, Google Pub/Sub, and S3 **destination** connectors are deprecated and no longer actively maintained. They remain fully functional in the current and all prior releases, and no code is being removed. The actively-maintained paths going forward are Postgres → ClickHouse, Postgres → ClickHouse Cloud, and Postgres → Postgres. This deprecation applies to the **destination** role only — BigQuery remains a supported **source**. See the [connector status matrix](#status) and the [deprecated connectors migration guide](docs/deprecated-connectors.md) for how to pin to a release or fork the relevant code. +> **Mirror type deprecation notice:** CDC is the recommended and actively-maintained mirror type. The **Query Replication (QRep)** and **XMIN** mirror types are deprecated and no longer actively maintained; they remain fully functional, and no code is currently being removed. + +> **Connector deprecation notice:** The Snowflake, BigQuery, ElasticSearch, Kafka (including Confluent and Redpanda variants), Azure Event Hubs, Google Pub/Sub, and S3 **destination** connectors are deprecated and no longer actively maintained. They remain fully functional in the current and all prior releases, and no code is currently being removed. The actively-maintained paths going forward are Postgres → ClickHouse, Postgres → ClickHouse Cloud, and Postgres → Postgres. This deprecation applies to the **destination** role only — BigQuery remains a supported **source**. See the [connector status matrix](#status) and the [deprecated connectors migration guide](docs/deprecated-connectors.md) for how to pin to a release or fork the relevant code. ## Get started @@ -67,7 +69,7 @@ PeerDB is an ETL/ELT tool built for PostgreSQL. We implement multiple Postgres n **For reliability**, we have mechanisms in place for fault tolerance - state management, automatic retries, handling idempotency and consistency and so on (). Configurable batching and parallelism prevent out of memory (OOMs) and crashes. -**From a feature richness standpoint**, we support efficient syncing of tables with large (TOAST) columns. We support multiple streaming modes - log based (CDC), cursor based (timestamp or integer) and XMIN based. We provide rich data-type mapping and plan to support every possible type (incl. Custom types) that Postgres supports to the best extent possible on the target data-store. +**From a feature richness standpoint**, we support efficient syncing of tables with large (TOAST) columns. We support multiple streaming modes - log based (CDC), cursor based (timestamp or integer) and XMIN based (the cursor-based / Query Replication and XMIN modes are deprecated; CDC is recommended). We provide rich data-type mapping and plan to support every possible type (incl. Custom types) that Postgres supports to the best extent possible on the target data-store. ### Now available natively in ClickHouse Cloud (Generally Available) diff --git a/docs/deprecated-connectors.md b/docs/deprecated-connectors.md index 502b07f40..1febd3193 100644 --- a/docs/deprecated-connectors.md +++ b/docs/deprecated-connectors.md @@ -10,7 +10,7 @@ The following **destination** connectors are deprecated and no longer actively m - Google Pub/Sub - S3 -These connectors remain **fully functional** in the current and all prior releases. No code is being removed, and they will **not** be automatically removed from existing deployments. The actively-maintained paths going forward are Postgres → ClickHouse, Postgres → ClickHouse Cloud, and Postgres → Postgres. Sources (Postgres, MySQL, MongoDB) are not affected by this deprecation. +These connectors remain **fully functional** in the current and all prior releases. No code is currently being removed, and they will **not** be automatically removed from existing deployments. The actively-maintained paths going forward are Postgres → ClickHouse, Postgres → ClickHouse Cloud, and Postgres → Postgres. Sources (Postgres, MySQL, MongoDB) are not affected by this deprecation. > **BigQuery is deprecated only as a destination.** BigQuery remains a **supported source** for QRep mirrors and is not deprecated in that role. The migration guidance below applies to BigQuery's use as a *destination*. @@ -27,7 +27,7 @@ Pin to the latest release tag before any future removal so that you have the mos ## How to fork -Because no code is being removed, you can fork PeerDB and carry the relevant connector code yourself. The connectors are self-contained enough that forking and maintaining a single connector is practical: +Because the connector code is still present, you can fork PeerDB and carry the relevant connector code yourself. The connectors are self-contained enough that forking and maintaining a single connector is practical: 1. Fork the repository. 2. Keep the connector's flow code, its UI form and helper, and (optionally) its end-to-end tests so you can continue validating it. diff --git a/ui/app/api/mirror-types/route.ts b/ui/app/api/mirror-types/route.ts index 792e97691..97d402b60 100644 --- a/ui/app/api/mirror-types/route.ts +++ b/ui/app/api/mirror-types/route.ts @@ -15,6 +15,7 @@ export async function GET(_: NextRequest) { description: 'Query Replication allows you to specify a set of rows to be synced via a SELECT query. Useful for replicating views and tables without primary keys.', link: 'https://docs.peerdb.io/usecases/Streaming%20Query%20Replication/overview', + deprecated: true, }, ]; const xminCard = { @@ -22,6 +23,7 @@ export async function GET(_: NextRequest) { description: 'XMIN mode uses the xmin system column of PostgreSQL as a watermark column for replication.', link: 'https://docs.peerdb.io/sql/commands/create-mirror#xmin-query-replication', + deprecated: true, }; if (!GetPeerDBClickHouseMode()) { diff --git a/ui/app/mirrors/create/mirrorcards.tsx b/ui/app/mirrors/create/mirrorcards.tsx index e1bd07e00..65065d4d6 100644 --- a/ui/app/mirrors/create/mirrorcards.tsx +++ b/ui/app/mirrors/create/mirrorcards.tsx @@ -1,6 +1,7 @@ 'use client'; import { MirrorType } from '@/app/dto/MirrorsDTO'; import { fetcher } from '@/app/utils/swr'; +import { Badge } from '@/lib/Badge'; import { Label } from '@/lib/Label'; import { RowWithRadiobutton } from '@/lib/Layout'; import { ProgressCircle } from '@/lib/ProgressCircle'; @@ -22,6 +23,7 @@ export default function MirrorCards({ title: MirrorType; description: string; link: string; + deprecated?: boolean; }[] >('/api/mirror-types', fetcher); @@ -50,7 +52,26 @@ export default function MirrorCards({ -
{card.title}
+
+ {card.title} + {card.deprecated && ( + + + + )} +
} action={} From c402b9cc0464782a0e7840656bce5b3e75d06f3f Mon Sep 17 00:00:00 2001 From: Jeremy Yang Date: Mon, 6 Jul 2026 07:56:48 -0700 Subject: [PATCH 5/5] ci: re-trigger Lint UI after rebasing onto main Branch was behind main's prettier 3.8.3 / TypeScript 6.0.3 lockfile bump (#4536), causing a stale Prettier check failure. Rebasing picks up the updated formatting; this empty commit re-runs CI against the current tree. Co-Authored-By: Claude Opus 4.8 (1M context)