Skip to content

Commit 1e7a9a9

Browse files
mongo: partition snapshots for numeric and string _id collections (#4487)
## Summary Extends the MongoDB connector so initial snapshots **parallelize for numeric and string `_id` collections**, not just ObjectID. Previously any non-ObjectID `_id` fell back to a single serial full-table partition, making large naturally-keyed collections (app-store numeric IDs, package-name strings) too slow to snapshot inside the oplog retention window. Based on `v0.36.29` (our deployed release). It improves initial sync (MongoDb -> Clickhouse from 10 hours to 3 hours on 8mln collection). ## Approach `GetQRepPartitions` dispatches on the collection's min/max `_id` BSON type: - **ObjectID** — unchanged (uniform division of the 12-byte keyspace). - **Int32/Int64** — uniform min/max division, reusing `utils.PartitionHelper` to emit non-overlapping `IntPartitionRange` partitions (`$gte`/`$lte`). - **String** — sampling-based quantile boundaries via `$sample` (honours read preference), anchored by the real min/max, emitted as a new `StringPartitionRange` proto type with contiguous half-open ranges (`$gte`/`$lt`). - **Mixed-type / Double / empty** — safe full-table fallback. ## Changes - `protos/flow.proto` — add `StringPartitionRange` + `string_range` oneof field. - `connectors/mongo/qrep_partition.go` — type dispatcher; numeric + string builders; pure, unit-tested `computeStringBoundaries`. - `connectors/mongo/qrep.go` — `IntRange`/`StringRange` cases in `toRangeFilter`. - `connectors/utils/monitoring/monitoring.go` — handle `StringRange` in `addPartitionToQRepRun` (avoids the `unknown range type` error path). ## Testing - `go build ./...`, `go vet` clean. - New unit tests for `computeStringBoundaries` (even/clustered/sparse/duplicate/out-of-range/empty + contiguity), `toRangeFilter` (Int/String/ObjectID/unsupported), and `_id` type detection — all passing. ## Notes / limitations - Numeric uses uniform value division; monitor partition balance on first run for sparse ID spaces. - Double `_id` and mixed-type collections fall back to full-table by design. - Requires regenerating protobuf bindings (`./generate-protos.sh`) before building images, as `flow/generated` is gitignored. --------- Co-authored-by: Ilia Demianenko <ilia.demianenko@clickhouse.com>
1 parent 4ee56fc commit 1e7a9a9

4 files changed

Lines changed: 567 additions & 87 deletions

File tree

flow/connectors/mongo/qrep.go

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,17 +26,9 @@ func (c *MongoConnector) GetQRepPartitions(
2626
config *protos.QRepConfig,
2727
last *protos.QRepPartition,
2828
) ([]*protos.QRepPartition, error) {
29-
fullTablePartition := []*protos.QRepPartition{
30-
{
31-
PartitionId: utils.FullTablePartitionID,
32-
Range: nil,
33-
FullTablePartition: true,
34-
},
35-
}
36-
3729
if config.WatermarkColumn != DefaultDocumentKeyColumnName {
3830
c.logger.Warn("unexpected watermark column, falling back to full table partition")
39-
return fullTablePartition, nil
31+
return utils.FullTablePartition(), nil
4032
}
4133

4234
if config.NumRowsPerPartition <= 0 {
@@ -70,10 +62,10 @@ func (c *MongoConnector) GetQRepPartitions(
7062

7163
if adjustedPartitions.AdjustedNumPartitions <= 1 {
7264
c.logger.Info("[mongo] insufficient partitions for parallel snapshot, falling back to full table partition")
73-
return fullTablePartition, nil
65+
return utils.FullTablePartition(), nil
7466
}
7567

76-
return c.minMaxPartitions(ctx, collection, adjustedPartitions.AdjustedNumPartitions)
68+
return c.buildPartitions(ctx, collection, adjustedPartitions.AdjustedNumPartitions)
7769
}
7870

7971
func (c *MongoConnector) GetDefaultPartitionKeyForTables(
@@ -234,6 +226,29 @@ func toRangeFilter(watermarkColumn string, partitionRange *protos.PartitionRange
234226
bson.E{Key: "$lte", Value: endObjectID},
235227
}},
236228
}, nil
229+
case *protos.PartitionRange_IntRange:
230+
// Numeric _id: inclusive range. PartitionHelper builds non-overlapping
231+
// [start, end] integer ranges. Mongo compares int32/int64 numerically, so
232+
// int64 bounds match int32-typed _ids.
233+
return bson.D{
234+
bson.E{Key: watermarkColumn, Value: bson.D{
235+
bson.E{Key: "$gte", Value: r.IntRange.Start},
236+
bson.E{Key: "$lte", Value: r.IntRange.End},
237+
}},
238+
}, nil
239+
case *protos.PartitionRange_StringRange:
240+
// String _id: half-open [start, end) for all but the last partition;
241+
// the last partition is closed [start, end] (EndInclusive == true).
242+
endOp := "$lt"
243+
if r.StringRange.EndInclusive {
244+
endOp = "$lte"
245+
}
246+
return bson.D{
247+
bson.E{Key: watermarkColumn, Value: bson.D{
248+
bson.E{Key: "$gte", Value: r.StringRange.Start},
249+
bson.E{Key: endOp, Value: r.StringRange.End},
250+
}},
251+
}, nil
237252
default:
238253
return nil, fmt.Errorf("unsupported partition range type")
239254
}

0 commit comments

Comments
 (0)