From 5185e6aebbf460955f2cd8bff42f9a469d1df57e Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 16:58:07 +0200 Subject: [PATCH 01/37] [OPIK-6901] [BE] perf: prune trace deletes to the batch's own partitions Deleting traces rewrote every part of the table. A mutation selects parts at the PARTITION stage, and DELETE_BY_PROJECT_ID_TRACE_ID_PAIRS constrains only (workspace_id, project_id, id) -- none of which the weekly partition expression is derived from -- so nothing is pruned. Measured on prod-test (traces_local, 271.6 M rows, 3,928 parts, post-cutover): EXPLAIN indexes=1, 3 ids in one week current predicate partition stage: Parts 3928/3928 (Condition: true) + partition-expression set (this change) partition stage: Parts 5/3928 a real delete of 12 ids 3,928 parts per replica rewritten, 5.40 TiB, to mask 12 rows An id_at RANGE is not a substitute. On a batch spanning 1996 and 2200 a range still selected 2,644 of 3,928 parts, because the range covers every week in between; the exact set selected 4. Scattered batches are the normal case for a delete-by-ids API, so the predicate is a set. The predicate is emitted ONLY when every id in the batch is a UUIDv7, which preserves the guarantee the original javadoc called out: a row whose id_at cannot be trusted is still deleted, because no id in such a batch is used to derive a partition. All-or-nothing per batch -- the SQL is either fully pruned or byte-identical to the previous unbounded form, never partially bounded. Deriving a partition from a non-v7 id would read whatever bits sit in the timestamp field, and a wrong partition is a SILENTLY skipped delete, which is worse than a slow one. Far-future ids are deliberately supported rather than excluded: their id_at is bogus but self-consistent, so they live in the far-future partition this computes. Verified on prod-test: 0 partition mismatches between toMonday(id_at) and toMonday(UUIDv7ToDateTime(id)) across all 11.23 M far-future rows (4.1% of the table). DELETE_FOR_RETENTION already carries equivalent bounds for the same reason (OPIK-6900); this brings the by-ids path in line, with the extra v7 guard its arbitrary id set requires. Co-Authored-By: Claude Opus 5 (1M context) --- .../java/com/comet/opik/domain/TraceDAO.java | 65 ++++++++++++- .../domain/TraceDAOPartitionPruningTest.java | 91 +++++++++++++++++++ 2 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 apps/opik-backend/src/test/java/com/comet/opik/domain/TraceDAOPartitionPruningTest.java diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java index ceceab54498..14d2eb5b8c8 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java @@ -60,7 +60,11 @@ import reactor.core.publisher.Mono; import java.math.BigDecimal; +import java.time.DayOfWeek; import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.temporal.TemporalAdjusters; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -1921,8 +1925,19 @@ ORDER BY (workspace_id, project_id, trace_id, id) DESC, last_updated_at DESC * so a single statement can span several projects (e.g. a reused id resolved to all its owning projects, or a * cross-project batch) instead of one delete per project (OPIK-7483). Every deleted row carries its {@code * project_id}, so no delete is ever project-less - also required once {@code traces} is a Distributed table - * (OPIK-7455). No {@code id_at}/time predicate on purpose, so it still deletes rows whose {@code id_at} is - * untrustworthy (e.g. a wrapped timestamp); correctness here does not depend on {@code id_at}. + * (OPIK-7455). + *

+ * {@code } adds the table's own weekly partition expression, bound as the exact set of partitions + * the batch's ids resolve to. It is emitted only when every id in the batch is a UUIDv7 + * ({@link #weeklyPartitionsOf}); if any id is not, the predicate is omitted and the statement is byte-identical to + * the previous unbounded form. That preserves the original guarantee — a row whose {@code id_at} cannot be trusted + * is still deleted, because no id in such a batch is used to derive a partition. + *

+ * Why it matters: a mutation selects parts at the partition stage, where the (workspace_id, project_id, id) + * predicate prunes nothing, so deleting a handful of rows rewrote every part of the table. Measured on prod-test + * (271.6 M rows, 3,928 parts): 12 ids rewrote 3,928 parts / 5.40 TiB. With this predicate the same batch + * selects 5 parts. An {@code id_at} range is not a substitute: on a batch spanning 1996 and 2200 a + * range still selected 2,644 parts, where the exact set selected 4. *

* The pairs are bound (never inlined) as two positional string arrays and zipped back into {@code (project_id, id)} * tuples with {@code arrayZip}, so the query text is constant regardless of batch size and no value reaches the SQL @@ -1934,6 +1949,7 @@ ORDER BY (workspace_id, project_id, trace_id, id) DESC, last_updated_at DESC DELETE FROM traces_localtraces WHERE workspace_id = :workspace_id AND (project_id, id) IN arrayZip(:project_ids, :trace_ids) + AND toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1))) IN :partitions SETTINGS log_comment = '' ; """; @@ -3511,6 +3527,41 @@ private Flux getDetailsById(UUID id, Connection connection) { .doFinally(signalType -> endSegment(segment)); } + /** + * The weekly partition values a batch of ids resolves to, or empty if the batch contains an id we refuse to derive + * a partition from. + *

+ * Mirrors the table's partition expression exactly: + * {@code toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))}, where {@code id_at} is MATERIALIZED + * as {@code UUIDv7ToDateTime(toUUID(id))} — i.e. the Monday of the id's UTC week, as {@code yyyyMMdd}. + *

+ * Returns empty unless every id is a UUIDv7. Deriving a partition from a non-v7 id would read whatever bits + * sit in the timestamp field, and a wrong partition means a silently skipped delete. All-or-nothing keeps the + * emitted SQL either fully pruned or exactly the previous unbounded form, never partially bounded. + *

+ * Far-future ids are fine and deliberately supported: their {@code id_at} is bogus but self-consistent, so they live + * in the far-future partition this computes. Verified on prod-test — 0 partition mismatches across 11.23 M + * far-future rows. + */ + static Optional> weeklyPartitionsOf(Collection ids) { + var partitions = new java.util.HashSet(); + + for (UUID id : ids) { + if (id == null || id.version() != 7) { + return Optional.empty(); + } + // UUIDv7: the high 48 bits are the unix epoch in milliseconds. + long epochMilli = id.getMostSignificantBits() >>> 16; + LocalDate monday = Instant.ofEpochMilli(epochMilli) + .atZone(ZoneOffset.UTC) + .toLocalDate() + .with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)); + partitions.add(monday.getYear() * 10000L + monday.getMonthValue() * 100L + monday.getDayOfMonth()); + } + + return partitions.isEmpty() ? Optional.empty() : Optional.of(partitions); + } + @Override @WithSpan public Mono delete(Set> projectIdTraceIdPairs, @NonNull Connection connection) { @@ -3529,11 +3580,21 @@ public Mono delete(Set> projectIdTraceIdPairs, @NonNull C var projectIds = batch.stream().map(pair -> pair.getLeft().toString()).toArray(String[]::new); var traceIds = batch.stream().map(pair -> pair.getRight().toString()).toArray(String[]::new); + // Prune to the batch's own partitions when every id is a UUIDv7; otherwise emit the unbounded form. + var partitions = weeklyPartitionsOf(batch.stream().map(Pair::getRight).toList()); + // Flag only, exactly like distributed_wrap: the values reach ClickHouse via the bind below, + // never through the template, so the rendered SQL is constant regardless of batch contents. + partitions.ifPresent(_ -> template.add("partitions", true)); + var statement = connection.createStatement(template.render()) .bind("workspace_id", workspaceId) .bind("project_ids", projectIds) .bind("trace_ids", traceIds); + if (partitions.isPresent()) { + statement = statement.bind("partitions", partitions.get().toArray(Long[]::new)); + } + var segment = startSegment("traces", "Clickhouse", "delete"); return Mono.from(statement.execute()) .doFinally(_ -> endSegment(segment)) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/domain/TraceDAOPartitionPruningTest.java b/apps/opik-backend/src/test/java/com/comet/opik/domain/TraceDAOPartitionPruningTest.java new file mode 100644 index 00000000000..fc83303a0bd --- /dev/null +++ b/apps/opik-backend/src/test/java/com/comet/opik/domain/TraceDAOPartitionPruningTest.java @@ -0,0 +1,91 @@ +package com.comet.opik.domain; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Covers {@link TraceDAOImpl#weeklyPartitionsOf}, which derives the weekly partition values a delete batch resolves + * to so the mutation can prune instead of rewriting every part. + *

+ * The expected values are not hand-computed: each is what ClickHouse itself returned for + * {@code toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))} on prod-test for that id. If the table's + * partition expression ever changes, these assertions are what should fail. + */ +class TraceDAOPartitionPruningTest { + + @Test + @DisplayName("matches the partition ClickHouse computed — ordinary id") + void ordinaryId() { + // id_at 2026-08-19 (a Wednesday) -> Monday 2026-08-17 + assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of(UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3")))) + .contains(Set.of(20260817L)); + } + + @Test + @DisplayName("far-future ids are supported, not excluded") + void farFutureId() { + // A bogus-but-self-consistent timestamp: id_at 2200-01-01 -> Monday 2199-12-30. + // 4.1% of rows on prod-test look like this; they must still be deletable and still prune. + assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of(UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2")))) + .contains(Set.of(21991230L)); + } + + @Test + @DisplayName("pre-epoch-era id") + void oldId() { + // id_at 1996-02-09 -> Monday 1996-02-05 + assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of(UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8")))) + .contains(Set.of(19960205L)); + } + + @Test + @DisplayName("a scattered batch yields the exact set, not a range") + void scatteredBatch() { + // 1996 and 2026 in one batch. An id_at RANGE over this span selected 2,644 of 3,928 parts on prod-test; + // the exact set selected 4. This is the reason the predicate is a set. + assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of( + UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), + UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8")))) + .contains(Set.of(20260817L, 19960205L)); + } + + @Test + @DisplayName("a non-v7 id disables pruning for the whole batch") + void nonV7DisablesPruning() { + // All-or-nothing on purpose: deriving a partition from a non-v7 id reads whatever sits in the timestamp + // field, and a wrong partition is a SILENTLY skipped delete. Omitting the predicate keeps the old behaviour. + assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of( + UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), + UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22")))) // v4 + .isEmpty(); + } + + @Test + @DisplayName("a single non-v7 id yields no partitions") + void singleNonV7() { + assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of(UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22")))) + .isEmpty(); + } + + @Test + @DisplayName("duplicate ids in the same week collapse to one partition") + void duplicatesCollapse() { + assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of( + UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), + UUID.fromString("01a01a75-6f8e-7f22-9279-ee4f7ca7810d"), + UUID.fromString("01a01a75-609d-7935-8d22-2dd8dfeb2454")))) + .contains(Set.of(20260817L)); + } + + @Test + @DisplayName("an empty batch yields no partitions") + void emptyBatch() { + assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of())).isEmpty(); + } +} From 15d28b9f84767928be025408eea2fb92e86f1652 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 17:37:15 +0200 Subject: [PATCH 02/37] [OPIK-6901] [BE] refactor: move the weekly-partition derivation into com.comet.opik.utils The Monday-of-week derivation is a pure function of a UUID, and the same partition expression backs both traces_local_v2 and spans_local_v2, so it does not belong to TraceDAOImpl. Moved as-is to com.comet.opik.utils.WeeklyPartitions, following SentinelTranslation: a @UtilityClass whose javadoc carries the reasoning the callers must not re-derive - above all that an empty result means "emit no predicate", not "no partitions". Pure move; no behaviour change. The test moves with it as com.comet.opik.utils.WeeklyPartitionsTest. Co-Authored-By: Claude Opus 5 (1M context) --- .../java/com/comet/opik/domain/TraceDAO.java | 44 +------------ .../comet/opik/utils/WeeklyPartitions.java | 65 +++++++++++++++++++ .../WeeklyPartitionsTest.java} | 24 +++---- 3 files changed, 80 insertions(+), 53 deletions(-) create mode 100644 apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java rename apps/opik-backend/src/test/java/com/comet/opik/{domain/TraceDAOPartitionPruningTest.java => utils/WeeklyPartitionsTest.java} (74%) diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java index 14d2eb5b8c8..cb046b009f0 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java @@ -32,6 +32,7 @@ import com.comet.opik.utils.ErrorUtils; import com.comet.opik.utils.JsonUtils; import com.comet.opik.utils.TruncationUtils; +import com.comet.opik.utils.WeeklyPartitions; import com.comet.opik.utils.template.TemplateUtils; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; @@ -60,11 +61,7 @@ import reactor.core.publisher.Mono; import java.math.BigDecimal; -import java.time.DayOfWeek; import java.time.Instant; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.time.temporal.TemporalAdjusters; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -1929,7 +1926,7 @@ ORDER BY (workspace_id, project_id, trace_id, id) DESC, last_updated_at DESC *

* {@code } adds the table's own weekly partition expression, bound as the exact set of partitions * the batch's ids resolve to. It is emitted only when every id in the batch is a UUIDv7 - * ({@link #weeklyPartitionsOf}); if any id is not, the predicate is omitted and the statement is byte-identical to + * ({@link WeeklyPartitions#of}); if any id is not, the predicate is omitted and the statement is byte-identical to * the previous unbounded form. That preserves the original guarantee — a row whose {@code id_at} cannot be trusted * is still deleted, because no id in such a batch is used to derive a partition. *

@@ -3527,41 +3524,6 @@ private Flux getDetailsById(UUID id, Connection connection) { .doFinally(signalType -> endSegment(segment)); } - /** - * The weekly partition values a batch of ids resolves to, or empty if the batch contains an id we refuse to derive - * a partition from. - *

- * Mirrors the table's partition expression exactly: - * {@code toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))}, where {@code id_at} is MATERIALIZED - * as {@code UUIDv7ToDateTime(toUUID(id))} — i.e. the Monday of the id's UTC week, as {@code yyyyMMdd}. - *

- * Returns empty unless every id is a UUIDv7. Deriving a partition from a non-v7 id would read whatever bits - * sit in the timestamp field, and a wrong partition means a silently skipped delete. All-or-nothing keeps the - * emitted SQL either fully pruned or exactly the previous unbounded form, never partially bounded. - *

- * Far-future ids are fine and deliberately supported: their {@code id_at} is bogus but self-consistent, so they live - * in the far-future partition this computes. Verified on prod-test — 0 partition mismatches across 11.23 M - * far-future rows. - */ - static Optional> weeklyPartitionsOf(Collection ids) { - var partitions = new java.util.HashSet(); - - for (UUID id : ids) { - if (id == null || id.version() != 7) { - return Optional.empty(); - } - // UUIDv7: the high 48 bits are the unix epoch in milliseconds. - long epochMilli = id.getMostSignificantBits() >>> 16; - LocalDate monday = Instant.ofEpochMilli(epochMilli) - .atZone(ZoneOffset.UTC) - .toLocalDate() - .with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)); - partitions.add(monday.getYear() * 10000L + monday.getMonthValue() * 100L + monday.getDayOfMonth()); - } - - return partitions.isEmpty() ? Optional.empty() : Optional.of(partitions); - } - @Override @WithSpan public Mono delete(Set> projectIdTraceIdPairs, @NonNull Connection connection) { @@ -3581,7 +3543,7 @@ public Mono delete(Set> projectIdTraceIdPairs, @NonNull C var traceIds = batch.stream().map(pair -> pair.getRight().toString()).toArray(String[]::new); // Prune to the batch's own partitions when every id is a UUIDv7; otherwise emit the unbounded form. - var partitions = weeklyPartitionsOf(batch.stream().map(Pair::getRight).toList()); + var partitions = WeeklyPartitions.of(batch.stream().map(Pair::getRight).toList()); // Flag only, exactly like distributed_wrap: the values reach ClickHouse via the bind below, // never through the template, so the rendered SQL is constant regardless of batch contents. partitions.ifPresent(_ -> template.add("partitions", true)); diff --git a/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java b/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java new file mode 100644 index 00000000000..920dfe2df27 --- /dev/null +++ b/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java @@ -0,0 +1,65 @@ +package com.comet.opik.utils; + +import lombok.experimental.UtilityClass; + +import java.time.DayOfWeek; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.temporal.TemporalAdjusters; +import java.util.Collection; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +/** + * Single derivation point for the {@code id_at} weekly partition values a batch of ids resolves to, so a mutation can + * name its own partitions instead of being planned against every part of the table. + * + *

Mirrors the partition expression of {@code traces_local_v2} / {@code spans_local_v2} exactly — + * {@code PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))}, where {@code id_at} is + * {@code DateTime64(0, 'UTC') MATERIALIZED UUIDv7ToDateTime(toUUID(id))} — i.e. the Monday of the id's UTC week as + * {@code yyyyMMdd}. Both tables share the expression, so it is derived here once rather than per DAO.

+ * + *

Why the caller must treat an empty result as "no predicate", never as "no partitions". The whole value of + * the derivation is that a partition the batch resolves to is the only place its rows can be; a value that is + * merely close is a silently skipped delete, not a slower one. So this returns a set only when every id in the batch is + * a UUIDv7, and empty otherwise — leaving the caller to emit its unbounded form, which is always correct and merely + * slower. Deriving a partition from a non-v7 id would read whatever bits sit in the timestamp field, and + * {@code UUIDv7ToDateTime} returns {@code 1970-01-01} for it rather than throwing, so the row sits in the epoch + * partition while the bits read as an arbitrary week. All-or-nothing across the batch, not per id: a partially derived + * set is a set the rows of the underivable ids are not in.

+ * + *

Far-future ids are supported on purpose: a UUIDv7 minted with a bad clock (litellm + * BerriAI/litellm#31294 mints ~2201) has a bogus but + * self-consistent {@code id_at}, so it lives in the far-future partition this computes and stays deletable and prunable. + * Verified on prod-test: 0 partition mismatches across 11.23 M far-future rows.

+ */ +@UtilityClass +public class WeeklyPartitions { + + /** + * The weekly partition values the ids resolve to, or empty if the batch contains an id whose partition cannot be + * derived exactly (see the class javadoc) — in which case the caller must omit its partition predicate entirely. + * An empty batch yields empty for the same reason: there is nothing to bound the mutation to. + */ + public static Optional> of(Collection ids) { + var partitions = new HashSet(); + + for (UUID id : ids) { + if (id == null || id.version() != 7) { + return Optional.empty(); + } + // UUIDv7: the high 48 bits are the unix epoch in milliseconds. + long epochMilli = id.getMostSignificantBits() >>> 16; + LocalDate monday = Instant.ofEpochMilli(epochMilli) + .atZone(ZoneOffset.UTC) + .toLocalDate() + .with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)); + partitions.add(monday.getYear() * 10000L + monday.getMonthValue() * 100L + monday.getDayOfMonth()); + } + + return partitions.isEmpty() ? Optional.empty() : Optional.of(partitions); + } +} diff --git a/apps/opik-backend/src/test/java/com/comet/opik/domain/TraceDAOPartitionPruningTest.java b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java similarity index 74% rename from apps/opik-backend/src/test/java/com/comet/opik/domain/TraceDAOPartitionPruningTest.java rename to apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java index fc83303a0bd..b2f9d72a936 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/domain/TraceDAOPartitionPruningTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java @@ -1,4 +1,4 @@ -package com.comet.opik.domain; +package com.comet.opik.utils; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -10,20 +10,20 @@ import static org.assertj.core.api.Assertions.assertThat; /** - * Covers {@link TraceDAOImpl#weeklyPartitionsOf}, which derives the weekly partition values a delete batch resolves - * to so the mutation can prune instead of rewriting every part. + * Covers {@link WeeklyPartitions#of}, which derives the weekly partition values a delete batch resolves to so the + * mutation can prune instead of rewriting every part. *

* The expected values are not hand-computed: each is what ClickHouse itself returned for * {@code toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))} on prod-test for that id. If the table's * partition expression ever changes, these assertions are what should fail. */ -class TraceDAOPartitionPruningTest { +class WeeklyPartitionsTest { @Test @DisplayName("matches the partition ClickHouse computed — ordinary id") void ordinaryId() { // id_at 2026-08-19 (a Wednesday) -> Monday 2026-08-17 - assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of(UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3")))) + assertThat(WeeklyPartitions.of(List.of(UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3")))) .contains(Set.of(20260817L)); } @@ -32,7 +32,7 @@ void ordinaryId() { void farFutureId() { // A bogus-but-self-consistent timestamp: id_at 2200-01-01 -> Monday 2199-12-30. // 4.1% of rows on prod-test look like this; they must still be deletable and still prune. - assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of(UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2")))) + assertThat(WeeklyPartitions.of(List.of(UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2")))) .contains(Set.of(21991230L)); } @@ -40,7 +40,7 @@ void farFutureId() { @DisplayName("pre-epoch-era id") void oldId() { // id_at 1996-02-09 -> Monday 1996-02-05 - assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of(UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8")))) + assertThat(WeeklyPartitions.of(List.of(UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8")))) .contains(Set.of(19960205L)); } @@ -49,7 +49,7 @@ void oldId() { void scatteredBatch() { // 1996 and 2026 in one batch. An id_at RANGE over this span selected 2,644 of 3,928 parts on prod-test; // the exact set selected 4. This is the reason the predicate is a set. - assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of( + assertThat(WeeklyPartitions.of(List.of( UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8")))) .contains(Set.of(20260817L, 19960205L)); @@ -60,7 +60,7 @@ void scatteredBatch() { void nonV7DisablesPruning() { // All-or-nothing on purpose: deriving a partition from a non-v7 id reads whatever sits in the timestamp // field, and a wrong partition is a SILENTLY skipped delete. Omitting the predicate keeps the old behaviour. - assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of( + assertThat(WeeklyPartitions.of(List.of( UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22")))) // v4 .isEmpty(); @@ -69,14 +69,14 @@ void nonV7DisablesPruning() { @Test @DisplayName("a single non-v7 id yields no partitions") void singleNonV7() { - assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of(UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22")))) + assertThat(WeeklyPartitions.of(List.of(UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22")))) .isEmpty(); } @Test @DisplayName("duplicate ids in the same week collapse to one partition") void duplicatesCollapse() { - assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of( + assertThat(WeeklyPartitions.of(List.of( UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), UUID.fromString("01a01a75-6f8e-7f22-9279-ee4f7ca7810d"), UUID.fromString("01a01a75-609d-7935-8d22-2dd8dfeb2454")))) @@ -86,6 +86,6 @@ void duplicatesCollapse() { @Test @DisplayName("an empty batch yields no partitions") void emptyBatch() { - assertThat(TraceDAOImpl.weeklyPartitionsOf(List.of())).isEmpty(); + assertThat(WeeklyPartitions.of(List.of())).isEmpty(); } } From 0bd27521c2b055d438c4ccb57a4558d8e1d93d9c Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 17:37:29 +0200 Subject: [PATCH 03/37] [OPIK-6901] [BE] test: name the 1996 case for what it is, not "pre-epoch" The id encodes 1996-02-09, which is 26 years AFTER the Unix epoch, so the label was simply wrong - and misleadingly so, since it implied coverage of a negative id_at that a UUIDv7 cannot carry (the 48-bit timestamp field is unsigned). Renamed to say which year it is, and the comment now states the two facts the case actually rests on: after the epoch, and inside Date32's 1900 floor. Co-Authored-By: Claude Opus 5 (1M context) --- .../test/java/com/comet/opik/utils/WeeklyPartitionsTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java index b2f9d72a936..ee7a459bd75 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java @@ -37,9 +37,10 @@ void farFutureId() { } @Test - @DisplayName("pre-epoch-era id") + @DisplayName("matches the partition ClickHouse computed — id from 1996") void oldId() { - // id_at 1996-02-09 -> Monday 1996-02-05 + // id_at 1996-02-09 -> Monday 1996-02-05. Long before Opik existed but well after the Unix epoch, and well + // inside Date32's 1900 floor: an id this old prunes like any other. assertThat(WeeklyPartitions.of(List.of(UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8")))) .contains(Set.of(19960205L)); } From db2dd29d751a494f7f9b888ae42722254ec29b9f Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 17:37:56 +0200 Subject: [PATCH 04/37] [OPIK-6901] [BE] fix: fall back to the unbounded delete when id_at is past DateTime64's ceiling Java LocalDate has no upper bound; id_at does. DateTime64 spans [1900-01-01, 2299-12-31 23:59:59.99999999] and SATURATES there rather than wrapping or throwing, so a UUIDv7 whose embedded timestamp is past that instant is stored as 2299-12-31 23:59:59 and filed under partition 22991225 - while the derivation computed its honest, out-of-range week. That is a partition the row is not in, and the predicate would then match zero rows and report success: the exact silent-skip failure the all-or- nothing v7 guard exists to prevent, reached by a different route. Rejecting rather than clamping to 22991225. Clamping would make correctness depend on reproducing ClickHouse's saturation semantics exactly - and there are two saturating steps before toDate32 is even reached, since UUIDv7ToDateTime returns DateTime64(3) and the column is DateTime64(0) - to buy pruning for ids that should not exist. Returning empty costs performance on those batches and nothing else. Only the ceiling is guarded, deliberately: `>>> 16` reads the timestamp field unsigned, so the value is in [0, 2^48) and the earliest id_at any UUIDv7 can carry is the epoch, whose Monday (1969-12-29) is 70 years above Date32's 1900 floor. A below-1900 id_at is unreachable by construction, not merely untested, and the same bound is why Instant.ofEpochMilli cannot overflow here. Tests cover both ends and the boundary millisecond either side of the ceiling. Co-Authored-By: Claude Opus 5 (1M context) --- .../comet/opik/utils/WeeklyPartitions.java | 51 +++++++++++++---- .../opik/utils/WeeklyPartitionsTest.java | 57 ++++++++++++++++++- 2 files changed, 96 insertions(+), 12 deletions(-) diff --git a/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java b/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java index 920dfe2df27..8fbc8716236 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java @@ -24,21 +24,44 @@ * *

Why the caller must treat an empty result as "no predicate", never as "no partitions". The whole value of * the derivation is that a partition the batch resolves to is the only place its rows can be; a value that is - * merely close is a silently skipped delete, not a slower one. So this returns a set only when every id in the batch is - * a UUIDv7, and empty otherwise — leaving the caller to emit its unbounded form, which is always correct and merely - * slower. Deriving a partition from a non-v7 id would read whatever bits sit in the timestamp field, and - * {@code UUIDv7ToDateTime} returns {@code 1970-01-01} for it rather than throwing, so the row sits in the epoch - * partition while the bits read as an arbitrary week. All-or-nothing across the batch, not per id: a partially derived - * set is a set the rows of the underivable ids are not in.

+ * merely close is a silently skipped delete, not a slower one. So this returns a set only when every id in the batch + * is one it can derive exactly, and empty otherwise — leaving the caller to emit its unbounded form, which is always + * correct and merely slower. The two rejections are:

+ *
    + *
  • Any id that is not a UUIDv7. Its high 48 bits are not a timestamp, and {@code UUIDv7ToDateTime} + * returns {@code 1970-01-01} for it rather than throwing, so the row sits in the epoch partition while the bits + * read as an arbitrary week. All-or-nothing across the batch, not per id: a partially derived set is a set the + * rows of the underivable ids are not in.
    • + *
  • Any id whose embedded timestamp is outside {@code DateTime64}'s range ({@link #ID_AT_CEILING}). Java + * has no such bound, so past it the two disagree: ClickHouse stores the saturated bound and the row lands in + * {@code 22991225}, while this would compute the real (out-of-range) week. Rejecting is deliberate in preference + * to clamping to {@code 22991225}: clamping would make correctness depend on reproducing ClickHouse's saturation + * semantics exactly — including where the saturation happens, which is already two steps before {@code toDate32} + * (see below) — to buy pruning for ids that should not exist. Falling back to the unbounded mutation costs + * performance on those batches and nothing else.
    • + *
* - *

Far-future ids are supported on purpose: a UUIDv7 minted with a bad clock (litellm - * BerriAI/litellm#31294 mints ~2201) has a bogus but - * self-consistent {@code id_at}, so it lives in the far-future partition this computes and stays deletable and prunable. - * Verified on prod-test: 0 partition mismatches across 11.23 M far-future rows.

+ *

Far-future ids within the range are supported on purpose and are the common case of the two: a UUIDv7 + * minted with a bad clock (litellm BerriAI/litellm#31294 + * mints ~2201) has a bogus but self-consistent {@code id_at}, so it lives in the far-future partition this computes and + * stays deletable and prunable. Verified on prod-test: 0 partition mismatches across 11.23 M far-future rows.

*/ @UtilityClass public class WeeklyPartitions { + /** + * First instant {@code id_at} cannot represent. {@code DateTime64} spans + * {@code [1900-01-01 00:00:00, 2299-12-31 23:59:59.99999999]} and saturates rather than wrapping or throwing, and + * it saturates twice over before {@code toDate32} is reached: {@code UUIDv7ToDateTime} already returns + * {@code DateTime64(3)}, and the column is {@code DateTime64(0)}. So an id at or past this instant is stored as + * {@code 2299-12-31 23:59:59} and partitions as {@code 22991225} (that Sunday's Monday) whatever its real week — + * observable on prod-test, whose far-future rows top out at exactly {@code 2299-12-31}. + */ + private static final long ID_AT_CEILING = LocalDate.of(2300, 1, 1) + .atStartOfDay() + .toInstant(ZoneOffset.UTC) + .toEpochMilli(); + /** * The weekly partition values the ids resolve to, or empty if the batch contains an id whose partition cannot be * derived exactly (see the class javadoc) — in which case the caller must omit its partition predicate entirely. @@ -51,8 +74,14 @@ public static Optional> of(Collection ids) { if (id == null || id.version() != 7) { return Optional.empty(); } - // UUIDv7: the high 48 bits are the unix epoch in milliseconds. + // UUIDv7: the high 48 bits are the unix epoch in milliseconds. `>>> 16` reads them unsigned, so the value + // is in [0, 2^48) — never negative, and never large enough for Instant.ofEpochMilli to overflow. That is + // also why only the ceiling is checked: the floor of the range is 1900-01-01, the smallest id_at any + // UUIDv7 can carry is the epoch, and even its Monday (1969-12-29) is comfortably inside Date32. long epochMilli = id.getMostSignificantBits() >>> 16; + if (epochMilli >= ID_AT_CEILING) { + return Optional.empty(); + } LocalDate monday = Instant.ofEpochMilli(epochMilli) .atZone(ZoneOffset.UTC) .toLocalDate() diff --git a/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java index ee7a459bd75..f826b35ae5c 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java @@ -13,9 +13,12 @@ * Covers {@link WeeklyPartitions#of}, which derives the weekly partition values a delete batch resolves to so the * mutation can prune instead of rewriting every part. *

- * The expected values are not hand-computed: each is what ClickHouse itself returned for + * The in-range expected values are not hand-computed: each is what ClickHouse itself returned for * {@code toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))} on prod-test for that id. If the table's * partition expression ever changes, these assertions are what should fail. + *

+ * The range-boundary cases are the exception and say so: the ids are constructed rather than observed, because the point + * of each is an {@code id_at} value the column cannot store, which is exactly what no real row has. */ class WeeklyPartitionsTest { @@ -89,4 +92,56 @@ void duplicatesCollapse() { void emptyBatch() { assertThat(WeeklyPartitions.of(List.of())).isEmpty(); } + + @Test + @DisplayName("the last id_at the column can store still prunes") + void lastRepresentableIdStillPrunes() { + // id_at 2299-12-31T23:59:59.999 — the last instant DateTime64 represents, so nothing saturates and the two + // sides agree: DateTime64(0) truncates to 23:59:59, whose Date32 is 2299-12-31 (a Sunday) -> Monday 2299-12-25. + // The ceiling check must be exclusive at exactly this point, hence a case sitting on it. + assertThat(WeeklyPartitions.of(List.of(UUID.fromString("0978a65f-77ff-7abc-8000-000000000001")))) + .contains(Set.of(22991225L)); + } + + @Test + @DisplayName("an id one millisecond past the id_at ceiling disables pruning") + void firstUnrepresentableIdDisablesPruning() { + // id_at 2300-01-01T00:00:00 — one ms past the previous case and outside DateTime64. ClickHouse saturates it to + // 2299-12-31 23:59:59 and files the row under 22991225, so the honest week this would compute (2300-01-01 is + // itself a Monday, giving 23000101) is a partition the row is NOT in. Pruning off rather than clamped: matching + // ClickHouse here would mean reproducing its saturation semantics, for ids that should not exist. + assertThat(WeeklyPartitions.of(List.of(UUID.fromString("0978a65f-7800-7abc-8000-000000000001")))) + .isEmpty(); + } + + @Test + @DisplayName("the largest timestamp a UUIDv7 can carry disables pruning, and does not throw") + void largestUuidV7TimestampDisablesPruning() { + // All 48 timestamp bits set: 10889-08-02, the furthest future any UUIDv7 can encode. Read unsigned it is still + // only ~2.8e14 ms, far inside Instant's range — so the guard is what excludes it, not an exception, and there + // is no input on which this can throw. + assertThat(WeeklyPartitions.of(List.of(UUID.fromString("ffffffff-ffff-7abc-8000-000000000001")))) + .isEmpty(); + } + + @Test + @DisplayName("one out-of-range id disables pruning for the whole batch") + void outOfRangeIdDisablesPruningForTheWholeBatch() { + // Same all-or-nothing rule as a non-v7 id, for the same reason: a set derived from the rest of the batch is a + // set this row is not in. + assertThat(WeeklyPartitions.of(List.of( + UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), + UUID.fromString("ffffffff-ffff-7abc-8000-000000000001")))) + .isEmpty(); + } + + @Test + @DisplayName("the earliest id a UUIDv7 can carry is inside Date32, so there is no floor to guard") + void earliestUuidV7TimestampIsInRange() { + // All 48 timestamp bits clear: id_at 1970-01-01, the earliest any UUIDv7 can encode (the field is unsigned). + // Its Monday, 1969-12-29, is 70 years above Date32's 1900 floor, so a below-1900 id_at is unreachable by + // construction rather than merely untested — which is why `of` guards only the ceiling. + assertThat(WeeklyPartitions.of(List.of(UUID.fromString("00000000-0000-7abc-8000-000000000001")))) + .contains(Set.of(19691229L)); + } } From a873f5e393815f583ff4bbb640f6dc98ca56ae70 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 17:38:26 +0200 Subject: [PATCH 05/37] [OPIK-6901] [BE] fix: emit the partition predicate only against the partitioned successor The template still targets the LEGACY traces when the wrap is off, and that table is the one place the predicate must never appear. It has no PARTITION BY at all - one `all` partition, so there is nothing to prune - and it declares id_at as a 32-bit DateTime('UTC') that overflows past 2106 (migration 000091). A far-future UUIDv7 - the litellm ~2201 ids, real customer-facing rows - is therefore stored under a WRAPPED recent timestamp, which the derived partition cannot match, so the delete matches zero rows and reports success. Pure downside: no pruning to gain, a silent data bug to lose. Gated on a NEW flag, tracesWeeklyPartitioningEnabled, because neither existing schema flag marks the EXCHANGE - the moment the partitioning appears - and each is wrong in a different direction: * tracesDistributedWrapEnabled is too LATE. The wrap is a separate, deferrable step (--skip-wrap now, --wrap-only weeks later), so between the two `traces` is already the successor while the flag is still false. That is the state prod-test sat in for ~30 minutes today. Gating on it would merely forgo the pruning there. * traceColumnsNonNullable is too EARLY, which is the dangerous direction. It is a runtime concern, not a schema one - the suite TraceSentinelIntegrationTest sets it true against the unpartitioned legacy table on purpose - and the runbook requires it rolled out BEFORE the EXCHANGE, since a rolling restart cannot be atomic with a metadata swap. Gating on it would emit the predicate against the legacy table for the whole rollout window. The flag asserts a schema fact: id_at as DateTime64(0,'UTC') under the weekly PARTITION BY. Unlike its two siblings it is safe to LAG and unsafe to LEAD - false is the previous unbounded mutation, always correct and merely slower - so it is turned on at leisure once the EXCHANGE is confirmed, and must be reverted (with the restart) BEFORE a rollback promotes the original. Documented as such in the cutover runbook, both where the other flips are described and in the rollback section. Plumbed like tracesDistributedWrapEnabled: config.yml + config-test.yml, docker-compose, helm values/configmap/README, and the configmap env test. Co-Authored-By: Claude Opus 5 (1M context) --- apps/opik-backend/config.yml | 11 ++++ .../traces-local-v2-cutover/README.md | 24 ++++++++ .../java/com/comet/opik/domain/TraceDAO.java | 57 +++++++++++++++++-- .../DatabaseAnalyticsDataModelConfig.java | 20 ++++++- .../src/test/resources/config-test.yml | 11 ++++ deployment/docker-compose/docker-compose.yaml | 4 ++ deployment/helm_chart/opik/README.md | 1 + .../opik/templates/configmap-backend.yaml | 1 + .../opik/tests/configmap_env_test.yaml | 3 + deployment/helm_chart/opik/values.yaml | 3 + 10 files changed, 128 insertions(+), 7 deletions(-) diff --git a/apps/opik-backend/config.yml b/apps/opik-backend/config.yml index bba61f5d02b..40a94efdfe0 100644 --- a/apps/opik-backend/config.yml +++ b/apps/opik-backend/config.yml @@ -155,6 +155,17 @@ databaseAnalyticsDataModel: # MODIFY TTL target `traces_local` only (the Distributed `traces` rejects them); ADD/DROP/MODIFY COLUMN must target # both `traces_local` and `traces`, else reads can't see the column (code 47). tracesDistributedWrapEnabled: ${ANALYTICS_DB_DATA_MODEL_TRACES_DISTRIBUTED_WRAP_ENABLED:-false} + # Default: false + # Description: Whether the live trace mutation target is the weekly partitioned successor (id_at as + # DateTime64(0,'UTC') under PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))), so a + # trace DELETE can bound itself to the partitions its own ids resolve to instead of being planned against every part + # of the table. Purely an optimisation: false keeps the unbounded mutation, which is always correct and merely + # slower. A third flag on purpose - the partitioning appears at the EXCHANGE, and neither sibling marks it: + # traceColumnsNonNullable must be rolled out BEFORE the EXCHANGE, tracesDistributedWrapEnabled flips at the wrap, + # which may be deferred long after it. Leave false at deploy time; set true once the EXCHANGE is confirmed, and back + # to false BEFORE a rollback promotes the original `traces` (legacy `traces` has no PARTITION BY and a 32-bit + # DateTime id_at that overflows past 2106, so the predicate would silently match zero rows for a far-future id). + tracesWeeklyPartitioningEnabled: ${ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED:-false} # Description: UUIDv7 ingestion validation. Rejects writes whose `id` embeds a timestamp outside the # window, protecting data quality. diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index bb732b08d2c..b1848368c71 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -322,6 +322,25 @@ the `traceColumnsNonNullable` flip"). On rollback, after swapping the Nullable original back, revert the flag to `false` **and** run that repair. +**The `tracesWeeklyPartitioningEnabled` flip (optional, and why it goes last).** `databaseAnalyticsDataModel.tracesWeeklyPartitioningEnabled` +(env `ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED`, default `false`) lets a trace `DELETE` bound itself to +the weekly partitions its own ids resolve to (OPIK-6901), instead of being planned against every part of the table — on +prod-test, 12 ids rewrote 3,928 parts / 5.40 TiB without it. It asserts a **schema** fact: that `traces` (or +`traces_local`) is the successor, with `id_at` as `DateTime64(0,'UTC')` under the weekly `PARTITION BY`. + +It is a third flag precisely because **neither of the two above marks the `EXCHANGE`**, which is when the partitioning +appears: `traceColumnsNonNullable` must lead it (above), and `tracesDistributedWrapEnabled` flips at the wrap, which may +be deferred long after it (`--skip-wrap` … `--wrap-only`). So gate on this one, not on either of those. + +Unlike its siblings it is **safe to lag and unsafe to lead**: `false` is the previous unbounded delete, always correct +and merely slower, so turn it on at leisure **after** the `EXCHANGE` is confirmed. Turning it on early — while `traces` is +still the original — is the failure mode worth avoiding: the original has **no `PARTITION BY` at all** (nothing to prune) +and declares `id_at` as a 32-bit `DateTime` that overflows past 2106, so a far-future id (the litellm ~2201 rows) is +stored under a wrapped recent timestamp that the derived partition cannot match, and the delete reports success having +matched **zero rows**. For the same reason, a stage B/C **rollback must revert it to `false` — and roll-restart every +instance — before promoting the original**, ahead of the swap rather than after it. Like its siblings it comes from a +startup snapshot of `OpikConfiguration`, so the config change alone changes nothing until each instance restarts. + ## Batching and throttling On a large production table a single week can be enormous, so the backfill does **not** run one INSERT per week. Two @@ -832,6 +851,11 @@ statements, so a failure *between* them needs a restart path: again, so the flip has to be undone in two steps — `rollback.sh` prints both when the stage finishes. The rollback is not complete until they land. +> **If `tracesWeeklyPartitioningEnabled` was turned on, revert it *before* the stage runs, not after.** It asserts the +> live table is the partitioned successor, and the restored original is not one, so a stale `true` makes trace deletes +> match zero rows while reporting success — see "The `tracesWeeklyPartitioningEnabled` flip". It is the one flag whose +> revert has to lead the swap; the two steps below follow it. + 1. **Revert `traceColumnsNonNullable` to `false` AND roll-restart every backend instance.** The flag is read from a **startup snapshot** of `OpikConfiguration` (bound via `toInstance`), so a config change does **not** take effect until each instance restarts — exactly like the forward rollout before the EXCHANGE. Until the restart completes, the app diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java index cb046b009f0..dbfa8cadfbd 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java @@ -1925,10 +1925,12 @@ ORDER BY (workspace_id, project_id, trace_id, id) DESC, last_updated_at DESC * (OPIK-7455). *

* {@code } adds the table's own weekly partition expression, bound as the exact set of partitions - * the batch's ids resolve to. It is emitted only when every id in the batch is a UUIDv7 - * ({@link WeeklyPartitions#of}); if any id is not, the predicate is omitted and the statement is byte-identical to - * the previous unbounded form. That preserves the original guarantee — a row whose {@code id_at} cannot be trusted - * is still deleted, because no id in such a batch is used to derive a partition. + * the batch's ids resolve to. Both conditions must hold for it to be emitted: the live table must be the weekly + * partitioned successor ({@link #tracesWeeklyPartitioningEnabled()}), and every id in the batch must be one whose + * partition can be derived exactly ({@link WeeklyPartitions#of}). Otherwise the predicate is omitted and the + * statement is byte-identical to the previous unbounded form. That is what preserves the original guarantee — a + * row whose {@code id_at} cannot be trusted is still deleted, because no id in such a batch is used to derive a + * partition. *

* Why it matters: a mutation selects parts at the partition stage, where the (workspace_id, project_id, id) * predicate prunes nothing, so deleting a handful of rows rewrote every part of the table. Measured on prod-test @@ -3369,6 +3371,48 @@ private void selectTracesMutationTable(ST template) { } } + /** + * Whether the live trace mutation target is the weekly partitioned successor — {@code id_at} as + * {@code DateTime64(0, 'UTC')} under + * {@code PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))} — so a mutation may bound + * itself to the partitions its ids resolve to ({@link WeeklyPartitions}). + *

+ * This has to be its own flag because neither existing schema flag marks the EXCHANGE, which is the moment the + * partitioning appears, and both are wrong in a different direction: + *

    + *
  • {@link #tracesDistributedWrapEnabled()} is too late. The wrap is a separate, deferrable step after the + * EXCHANGE (it may be skipped entirely with {@code --skip-wrap} and applied weeks later with + * {@code --wrap-only}), so between the two {@code traces} is already the partitioned successor while the flag is + * still {@code false} — the state prod-test sat in. Gating on it would simply forgo the pruning there.
    • + *
  • {@link #traceColumnsNonNullable()} is too early, which is the dangerous direction. It is a runtime + * concern, not a schema one, and the runbook requires it rolled out to {@code true} on every instance + * before the EXCHANGE (a rolling restart cannot be atomic with a metadata swap). Gating on it would emit + * the predicate against the legacy {@code traces} for the whole rollout window.
    • + *
+ * Emitting it against the legacy table is not merely unhelpful, it is wrong: legacy {@code traces} has no + * {@code PARTITION BY} at all (one {@code all} partition, so nothing to prune) and declares {@code id_at} as a + * 32-bit {@code DateTime} that overflows past 2106, so a far-future id — the litellm ~2201 ids, real + * customer-facing rows — is stored under a wrapped recent timestamp that the derived partition cannot match. The + * delete would then match zero rows and report success. + *

+ * Only ever {@code true} while {@code traces} really is that successor, so unlike its siblings it is safe to lag: + * {@code false} is the always-correct unbounded behaviour, and only {@code true} asserts something about the + * schema. Turn it on once the EXCHANGE is confirmed, and back off before a rollback stage B/C promotes the + * original. + */ + private boolean tracesWeeklyPartitioningEnabled() { + return configuration.getDatabaseAnalyticsDataModel().tracesWeeklyPartitioningEnabled(); + } + + /** + * The partitions a delete batch may bound itself to, or empty to leave the mutation unbounded. Empty whenever the + * live table is not the partitioned successor, ahead of asking {@link WeeklyPartitions} at all — the derivation is + * only meaningful against a table that partitions on it. + */ + private Optional> weeklyPartitionsFor(Collection ids) { + return tracesWeeklyPartitioningEnabled() ? WeeklyPartitions.of(ids) : Optional.empty(); + } + /** * Binds input, output, metadata, and their slim versions (input_slim, output_slim) to a statement. * Centralizes the JSON conversion and binding logic for consistency across single and batch inserts. @@ -3542,8 +3586,9 @@ public Mono delete(Set> projectIdTraceIdPairs, @NonNull C var projectIds = batch.stream().map(pair -> pair.getLeft().toString()).toArray(String[]::new); var traceIds = batch.stream().map(pair -> pair.getRight().toString()).toArray(String[]::new); - // Prune to the batch's own partitions when every id is a UUIDv7; otherwise emit the unbounded form. - var partitions = WeeklyPartitions.of(batch.stream().map(Pair::getRight).toList()); + // Prune to the batch's own partitions when the schema and every id in the batch allow it; + // otherwise emit the unbounded form. + var partitions = weeklyPartitionsFor(batch.stream().map(Pair::getRight).toList()); // Flag only, exactly like distributed_wrap: the values reach ClickHouse via the bind below, // never through the template, so the rendered SQL is constant regardless of batch contents. partitions.ifPresent(_ -> template.add("partitions", true)); diff --git a/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/DatabaseAnalyticsDataModelConfig.java b/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/DatabaseAnalyticsDataModelConfig.java index 090ad3028c3..a83c4a40406 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/DatabaseAnalyticsDataModelConfig.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/DatabaseAnalyticsDataModelConfig.java @@ -50,6 +50,23 @@ * {@code MODIFY COLUMN} must be applied to both {@code traces_local} and the {@code Distributed} {@code traces} * (the wrapper accepts them as metadata-only, and targeting only {@code traces_local} leaves the wrapper without the * column, so reads fail with code 47).

+ * + *

{@code tracesWeeklyPartitioningEnabled}: whether the live trace mutation target is the weekly partitioned + * successor — {@code id_at} as {@code DateTime64(0, 'UTC')} under + * {@code PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))} — so a trace {@code DELETE} + * may bound itself to the partitions its own ids resolve to instead of being planned against every part of the table + * (OPIK-6901). Purely an optimisation: {@code false} keeps the unbounded mutation, which is always correct and merely + * slower, and only {@code true} asserts anything about the schema.

+ * + *

It is deliberately a third flag rather than a reuse of the two above, because the partitioning appears at the + * EXCHANGE and neither of them marks that moment. {@code traceColumnsNonNullable} must be rolled out + * before the EXCHANGE (a rolling restart cannot be atomic with a metadata swap), and + * {@code tracesDistributedWrapEnabled} flips at the wrap, a separate step that may be deferred long after it — so one + * flag would be true too early and the other true too late. Emitting the predicate too early is the harmful direction: + * legacy {@code traces} has no {@code PARTITION BY} at all and declares {@code id_at} as a 32-bit {@code DateTime} that + * overflows past 2106, so a far-future id is stored under a wrapped timestamp the derived partition cannot match and + * the delete would silently affect zero rows. Left {@code false} at deploy time; set {@code true} once the EXCHANGE is + * confirmed on the target, and back to {@code false} before a rollback promotes the original {@code traces}.

*/ @Builder(toBuilder = true) public record DatabaseAnalyticsDataModelConfig( @@ -58,5 +75,6 @@ public record DatabaseAnalyticsDataModelConfig( boolean traceDeletionEventsCaptureEnabled, boolean spanDeletionEventsCaptureEnabled, @Min(1) @Max(2_000) int deletionEventsInsertBatchSize, - boolean tracesDistributedWrapEnabled) { + boolean tracesDistributedWrapEnabled, + boolean tracesWeeklyPartitioningEnabled) { } diff --git a/apps/opik-backend/src/test/resources/config-test.yml b/apps/opik-backend/src/test/resources/config-test.yml index 3891da65200..b0d5e6f7734 100644 --- a/apps/opik-backend/src/test/resources/config-test.yml +++ b/apps/opik-backend/src/test/resources/config-test.yml @@ -128,6 +128,17 @@ databaseAnalyticsDataModel: # MODIFY TTL target `traces_local` only (the Distributed `traces` rejects them); ADD/DROP/MODIFY COLUMN must target # both `traces_local` and `traces`, else reads can't see the column (code 47). tracesDistributedWrapEnabled: false + # Default: false + # Description: Whether the live trace mutation target is the weekly partitioned successor (id_at as + # DateTime64(0,'UTC') under PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))), so a + # trace DELETE can bound itself to the partitions its own ids resolve to instead of being planned against every part + # of the table. Purely an optimisation: false keeps the unbounded mutation, which is always correct and merely + # slower. A third flag on purpose - the partitioning appears at the EXCHANGE, and neither sibling marks it: + # traceColumnsNonNullable must be rolled out BEFORE the EXCHANGE, tracesDistributedWrapEnabled flips at the wrap, + # which may be deferred long after it. Leave false at deploy time; set true once the EXCHANGE is confirmed, and back + # to false BEFORE a rollback promotes the original `traces` (legacy `traces` has no PARTITION BY and a 32-bit + # DateTime id_at that overflows past 2106, so the predicate would silently match zero rows for a far-future id). + tracesWeeklyPartitioningEnabled: false # Description: UUIDv7 ingestion validation. Rejects writes whose `id` embeds a timestamp outside the # window, protecting data quality. diff --git a/deployment/docker-compose/docker-compose.yaml b/deployment/docker-compose/docker-compose.yaml index 4470ccfa05f..7d0c3663a28 100644 --- a/deployment/docker-compose/docker-compose.yaml +++ b/deployment/docker-compose/docker-compose.yaml @@ -188,6 +188,10 @@ services: # Flip TRACES_DISTRIBUTED_WRAP_ENABLED to true in lockstep with applying the Distributed wrap: it routes trace # delete/retention mutations to traces_local, since a Distributed traces rejects mutations. ANALYTICS_DB_DATA_MODEL_TRACES_DISTRIBUTED_WRAP_ENABLED: ${ANALYTICS_DB_DATA_MODEL_TRACES_DISTRIBUTED_WRAP_ENABLED:-false} + # Flip TRACES_WEEKLY_PARTITIONING_ENABLED to true only once the EXCHANGE has put the weekly partitioned + # successor under `traces`: it lets a trace DELETE prune to the partitions its own ids resolve to. Purely an + # optimisation - false keeps the always-correct unbounded mutation. + ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED: ${ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED:-false} # Readiness probes for the Hyperscale topology, both off for the single-node Compose stack: the cluster probe # checks the Distributed cluster definition is visible, the cold-storage probe checks the cold_s3 tiered disk. ANALYTICS_DB_CLUSTER_HEALTH_CHECK_ENABLED: ${ANALYTICS_DB_CLUSTER_HEALTH_CHECK_ENABLED:-false} diff --git a/deployment/helm_chart/opik/README.md b/deployment/helm_chart/opik/README.md index 1a40e51d1ec..ecefe3431cf 100644 --- a/deployment/helm_chart/opik/README.md +++ b/deployment/helm_chart/opik/README.md @@ -477,6 +477,7 @@ Call opik api on http://localhost:5173/api | databaseAnalyticsDataModel.traceColumnsNonNullable | bool | `false` | | | databaseAnalyticsDataModel.traceDeletionEventsCaptureEnabled | bool | `false` | | | databaseAnalyticsDataModel.tracesDistributedWrapEnabled | bool | `false` | | +| databaseAnalyticsDataModel.tracesWeeklyPartitioningEnabled | bool | `false` | | | demoDataJob.enabled | bool | `true` | | | fullnameOverride | string | `""` | | | global.argocd | bool | `false` | | diff --git a/deployment/helm_chart/opik/templates/configmap-backend.yaml b/deployment/helm_chart/opik/templates/configmap-backend.yaml index d80c4a8e53b..7fbc9c55941 100644 --- a/deployment/helm_chart/opik/templates/configmap-backend.yaml +++ b/deployment/helm_chart/opik/templates/configmap-backend.yaml @@ -50,6 +50,7 @@ data: "ANALYTICS_DB_DATA_MODEL_SPAN_DELETION_EVENTS_CAPTURE_ENABLED" $dm.spanDeletionEventsCaptureEnabled "ANALYTICS_DB_DATA_MODEL_DELETION_EVENTS_INSERT_BATCH_SIZE" $dm.deletionEventsInsertBatchSize "ANALYTICS_DB_DATA_MODEL_TRACES_DISTRIBUTED_WRAP_ENABLED" $dm.tracesDistributedWrapEnabled + "ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED" $dm.tracesWeeklyPartitioningEnabled "PARTITION_METRICS_ENABLED" $pm.enabled "PARTITION_METRICS_INTERVAL" $pm.interval "PARTITION_METRICS_LWD_TABLES" $pm.lwdTables diff --git a/deployment/helm_chart/opik/tests/configmap_env_test.yaml b/deployment/helm_chart/opik/tests/configmap_env_test.yaml index c4a1d9e68e1..0dea3793a4a 100644 --- a/deployment/helm_chart/opik/tests/configmap_env_test.yaml +++ b/deployment/helm_chart/opik/tests/configmap_env_test.yaml @@ -73,6 +73,9 @@ tests: - equal: path: data.ANALYTICS_DB_DATA_MODEL_TRACES_DISTRIBUTED_WRAP_ENABLED value: "false" + - equal: + path: data.ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED + value: "false" - equal: path: data.PARTITION_METRICS_ENABLED value: "false" diff --git a/deployment/helm_chart/opik/values.yaml b/deployment/helm_chart/opik/values.yaml index 9393b1121df..9551231b90d 100644 --- a/deployment/helm_chart/opik/values.yaml +++ b/deployment/helm_chart/opik/values.yaml @@ -740,6 +740,9 @@ databaseAnalyticsDataModel: # Turn true in lockstep with applying the Distributed wrap, so trace delete/retention mutations target # traces_local (a Distributed traces rejects mutations). Leave false while traces is still a MergeTree. tracesDistributedWrapEnabled: false + # Turn true only once the EXCHANGE has put the weekly partitioned successor under `traces`, so a trace DELETE + # can prune to the partitions its own ids resolve to. Purely an optimisation; false keeps the unbounded delete. + tracesWeeklyPartitioningEnabled: false # ClickHouse partition-health observability (OPIK-6904). Polls system.parts plus the # lightweight-delete mask and publishes opik.clickhouse.partition.* gauges; a distributed lock keeps From 6d78553e9f23605cfbc84e4435e32c1dab280a57 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 17:39:35 +0200 Subject: [PATCH 06/37] [OPIK-6901] [BE] test: exercise the rendered SQL and the partitions bind against ClickHouse WeeklyPartitionsTest covers only the Java derivation, so nothing failed if the template stopped rendering the predicate, if `partitions` was bound under a different name, or if ClickHouse rejected a Long[] on `IN :partitions` - and the last of those had never actually been executed: the prod-test measurement that motivated this change used EXPLAIN with a literal set, not the bound driver path. TracesPartitionPruningMutationTest closes that, against the post-EXCHANGE topology (dedicated non-reused containers, since the EXCHANGE destructively swaps `traces`). Every case asserts BOTH the rows read back through the public API and the SQL ClickHouse received, read from system.query_log by log_comment - because either alone passes for the wrong reason. Rows alone cannot see pruning silently stop (the delete still works, just slowly), and SQL alone cannot see a predicate that names a partition the row is not in. * all-v7: the predicate is emitted with the target's own partition, the target goes, a bystander in the same project stays. * two-week batch: both partitions in one Long[] - the multi-value bind a single-id delete never reaches. * non-v7 and beyond-2299: no predicate at all, and the v7 row batched alongside is still deleted. Driven through TraceDAO directly, since ingestion rejects both id shapes by design. liveTracesIsTheWeeklyPartitionedSuccessor is load-bearing, not decoration: the predicate is harmless against an unpartitioned table for recent ids, so without it a failed EXCHANGE would leave every test above green while proving nothing. It pins both facts the new flag asserts. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 358 ++++++++++++++++++ 1 file changed, 358 insertions(+) create mode 100644 apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java new file mode 100644 index 00000000000..f27a17abcec --- /dev/null +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -0,0 +1,358 @@ +package com.comet.opik.infrastructure; + +import com.comet.opik.api.Trace; +import com.comet.opik.api.resources.utils.ClickHouseContainerUtils; +import com.comet.opik.api.resources.utils.ClientSupportUtils; +import com.comet.opik.api.resources.utils.MigrationUtils; +import com.comet.opik.api.resources.utils.MySQLContainerUtils; +import com.comet.opik.api.resources.utils.RedisContainerUtils; +import com.comet.opik.api.resources.utils.TestDropwizardAppExtensionUtils; +import com.comet.opik.api.resources.utils.TestDropwizardAppExtensionUtils.AppContextConfig; +import com.comet.opik.api.resources.utils.TestDropwizardAppExtensionUtils.CustomConfig; +import com.comet.opik.api.resources.utils.TestUtils; +import com.comet.opik.api.resources.utils.WireMockUtils; +import com.comet.opik.api.resources.utils.resources.TraceResourceClient; +import com.comet.opik.domain.TraceDAO; +import com.comet.opik.extensions.DropwizardAppExtensionProvider; +import com.comet.opik.extensions.RegisterApp; +import com.comet.opik.infrastructure.auth.RequestContext; +import com.comet.opik.infrastructure.db.TransactionTemplateAsync; +import com.comet.opik.podam.PodamFactoryUtils; +import com.comet.opik.utils.WeeklyPartitions; +import com.redis.testcontainers.RedisContainer; +import io.r2dbc.spi.Statement; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.mysql.MySQLContainer; +import reactor.core.publisher.Mono; +import ru.vyarus.dropwizard.guice.test.ClientSupport; +import ru.vyarus.dropwizard.guice.test.jupiter.ext.TestDropwizardAppExtension; +import uk.co.jemos.podam.api.PodamFactory; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Stream; + +import static com.comet.opik.api.resources.utils.AuthTestUtils.mockTargetWorkspace; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +/** + * Exercises the partition pruning of {@code DELETE_BY_PROJECT_ID_TRACE_ID_PAIRS} against the post-EXCHANGE topology, + * where {@code traces} is the weekly partitioned successor (OPIK-6901). {@code WeeklyPartitionsTest} covers the + * derivation itself; what only a real ClickHouse can show is the half between it and the mutation — that the template + * renders the predicate exactly when it should, that the derived {@code Long[]} binds to {@code IN :partitions}, and + * that the row still goes away either way. + * + *

Each test asserts both halves, because either alone passes for the wrong reason: the rows are read back + * through the public API (a delete that pruned to a partition the row is not in would leave it behind), and the SQL + * ClickHouse actually received is read back from {@code system.query_log} (a delete that silently stopped pruning would + * still remove the row, just slowly — the regression the flag and the derivation exist to prevent, and one no + * behavioural assertion can see). + * + *

{@link #liveTracesIsTheWeeklyPartitionedSuccessor} is the guard that keeps the rest honest. The predicate is + * harmless against an unpartitioned table for recent ids, so had the EXCHANGE below not taken effect every test here + * would still pass while proving nothing; it pins both facts + * {@code databaseAnalyticsDataModel.tracesWeeklyPartitioningEnabled} asserts — the weekly {@code PARTITION BY} and + * {@code id_at} as {@code DateTime64}. + * + *

Two internal touches, on the pattern of {@code TracesDistributedWrapMutationTest}: the EXCHANGE has no public API, + * so {@link #beforeAll} runs it in raw SQL identical to the swap block of {@code 000003_exchange_and_wrap.sql}; and the + * ingestion path rejects a non-v7 or far-future {@code id} by design ({@code IdGenerator.validateId}), so the batches + * that must not prune are handed to {@link TraceDAO#delete} directly — the only way to reach that arm. + * + *

Dedicated, non-reused ClickHouse and ZooKeeper containers are required because the EXCHANGE destructively swaps the + * live {@code traces} table; a reused container would corrupt other suites and reruns. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith(DropwizardAppExtensionProvider.class) +class TracesPartitionPruningMutationTest { + + private static final String API_KEY = "apiKey-" + UUID.randomUUID(); + private static final String WORKSPACE_NAME = "workspace-" + RandomStringUtils.secure().nextAlphanumeric(32); + private static final String WORKSPACE_ID = UUID.randomUUID().toString(); + private static final String USER = "user-" + RandomStringUtils.secure().nextAlphanumeric(32); + + /** + * The partition-key fragment the template emits. Compared verbatim against the SQL read back from + * {@code system.query_log}, which is safe because that is the query text as submitted — the DAO's own template + * string, not a re-print. + */ + private static final String PARTITION_PREDICATE = + "toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))"; + + /** A v4 UUID: no timestamp to derive a partition from. */ + private static final UUID NON_V7_ID = UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22"); + + /** + * A UUIDv7 whose 48 timestamp bits are all set (10889-08-02), so its {@code id_at} saturates to the + * {@code DateTime64} ceiling and the honest week is not the partition the row would be in. Same rejection as + * {@link #NON_V7_ID}, different cause — see {@code WeeklyPartitions}. + */ + private static final UUID OUT_OF_RANGE_ID = UUID.fromString("ffffffff-ffff-7abc-8000-000000000001"); + + // Dedicated, non-reused ClickHouse + ZooKeeper on their own network: the EXCHANGE destructively swaps `traces`, so a + // shared/reused container would corrupt other suites and reruns. Redis/MySQL are only read, so the shared ones are + // fine. + private final Network network = Network.newNetwork(); + private final GenericContainer zookeeperContainer = ClickHouseContainerUtils.newZookeeperContainer(false, + network); + private final ClickHouseContainer clickHouseContainer = ClickHouseContainerUtils + .newClickHouseContainer(false, network, zookeeperContainer); + private final RedisContainer redisContainer = RedisContainerUtils.newRedisContainer(); + private final MySQLContainer mysqlContainer = MySQLContainerUtils.newMySQLContainer(); + + private final WireMockUtils.WireMockRuntime wireMock; + + private final PodamFactory factory = PodamFactoryUtils.newPodamFactory(); + + @RegisterApp + private final TestDropwizardAppExtension app; + + { + Startables.deepStart(redisContainer, mysqlContainer, clickHouseContainer, zookeeperContainer) + .join(); + wireMock = WireMockUtils.startWireMock(); + var databaseAnalyticsFactory = ClickHouseContainerUtils.newDatabaseAnalyticsFactory( + clickHouseContainer, ClickHouseContainerUtils.DATABASE_NAME); + MigrationUtils.runMysqlDbMigration(mysqlContainer); + MigrationUtils.runClickhouseDbMigration(clickHouseContainer); + app = TestDropwizardAppExtensionUtils.newTestDropwizardAppExtension( + AppContextConfig.builder() + .jdbcUrl(mysqlContainer.getJdbcUrl()) + .databaseAnalyticsFactory(databaseAnalyticsFactory) + .redisUrl(redisContainer.getRedisURI()) + .runtimeInfo(wireMock.runtimeInfo()) + // Both flags as production runs them post-EXCHANGE: the successor's end_time/ttft are + // non-nullable sentinel columns, and the pruning flag asserts the schema this suite installs. + .customConfigs(List.of( + new CustomConfig("databaseAnalyticsDataModel.traceColumnsNonNullable", "true"), + new CustomConfig("databaseAnalyticsDataModel.tracesWeeklyPartitioningEnabled", + "true"))) + .build()); + } + + private TraceResourceClient traceResourceClient; + private TransactionTemplateAsync template; + private TraceDAO traceDAO; + + @BeforeAll + void beforeAll(ClientSupport clientSupport, TransactionTemplateAsync template, TraceDAO traceDAO) { + var baseUrl = TestUtils.getBaseUrl(clientSupport); + ClientSupportUtils.config(clientSupport); + mockTargetWorkspace(wireMock.server(), API_KEY, WORKSPACE_NAME, WORKSPACE_ID, USER); + traceResourceClient = new TraceResourceClient(clientSupport, baseUrl); + this.template = template; + this.traceDAO = traceDAO; + exchangeTables(); + } + + @AfterAll + void afterAll() { + wireMock.server().stop(); + clickHouseContainer.stop(); + zookeeperContainer.stop(); + network.close(); + } + + @Test + @DisplayName("the live traces table is the weekly partitioned successor") + void liveTracesIsTheWeeklyPartitionedSuccessor() { + // Both halves of what tracesWeeklyPartitioningEnabled asserts. Without this the whole suite is vacuous: against + // the legacy unpartitioned `traces` a pruned delete of a recent id still removes the row, so every behavioural + // assertion below would stay green while the predicate was being emitted at exactly the table it must not be. + // Asserted piecewise rather than against PARTITION_PREDICATE verbatim: system.tables reports the expression as + // ClickHouse's own formatter re-prints it, so pinning its whitespace would make this brittle about the one thing + // it does not care about. No other partition expression in the schema is built from these three functions. + assertThat(queryOneString("SELECT partition_key FROM system.tables WHERE database = currentDatabase()" + + " AND name = 'traces'")) + .as("traces is partitioned by the weekly id_at expression") + .contains("toYYYYMMDD", "toDate32(id_at)", "toIntervalDay", "toDayOfWeek(id_at"); + assertThat(queryOneString("SELECT type FROM system.columns WHERE database = currentDatabase()" + + " AND table = 'traces' AND name = 'id_at'")) + .as("id_at is the 64-bit column, so a far-future timestamp is honest rather than wrapped") + .isEqualTo("DateTime64(0, 'UTC')"); + } + + @Test + @DisplayName("an all-UUIDv7 delete prunes to the batch's own partitions and removes the target row") + void allUuidV7DeletePrunesAndRemovesTheTargetRow() { + var target = newTrace().build(); + // Same project, so one read shows both: the pruned delete must take the target and leave this one. + var bystander = newTrace().projectName(target.projectName()).build(); + traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); + traceResourceClient.createTrace(bystander, API_KEY, WORKSPACE_NAME); + assertThat(traceIdsOf(target.projectName())).contains(target.id(), bystander.id()); + + // The live user path, end to end. + traceResourceClient.deleteTrace(target.id(), WORKSPACE_NAME, API_KEY); + + assertThat(traceIdsOf(target.projectName())) + .as("only the target row is gone") + .doesNotContain(target.id()) + .contains(bystander.id()); + assertThat(lastTraceDeleteSql()) + .as("the mutation bounded itself to the target's own partition") + .contains(PARTITION_PREDICATE) + .contains(onlyPartitionOf(target.id())); + } + + @Test + @DisplayName("a batch spanning two weeks binds both partitions, not a range") + void batchSpanningTwoWeeksBindsBothPartitions() { + // The multi-value Long[] bind, which the single-id path never exercises. The second id is minted for a week + // three years back and matches no row — the batch's partition SET is what is under test, and a delete does not + // need its ids to exist. A range over the span would have selected every partition in between; the set names + // exactly two. + var target = newTrace().build(); + traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); + var projectId = projectIdOf(target); + var otherWeekId = UUID.fromString("018c1860-1800-7abc-8000-000000000001"); // id_at 2023-11-29 -> 20231127 + + delete(Set.of(Pair.of(projectId, target.id()), Pair.of(projectId, otherWeekId))); + + assertThat(traceIdsOf(target.projectName())).doesNotContain(target.id()); + assertThat(lastTraceDeleteSql()) + .contains(PARTITION_PREDICATE) + .contains(onlyPartitionOf(target.id())) + .contains("20231127"); + } + + @ParameterizedTest + @MethodSource + @DisplayName("an id with no derivable partition disables pruning for the batch, and the delete still lands") + void underivableIdDisablesPruning(String cause, UUID underivableId) { + // The fallback that preserves the pre-OPIK-6901 guarantee: one id whose partition cannot be derived exactly and + // the statement goes back to its unbounded form — no predicate at all, never a partial set. The v7 row batched + // alongside it must still be deleted, which is the "not silently skipped" half. + var target = newTrace().build(); + traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); + var projectId = projectIdOf(target); + + delete(Set.of(Pair.of(projectId, target.id()), Pair.of(projectId, underivableId))); + + assertThat(traceIdsOf(target.projectName())) + .as("the deletable row in a %s batch is still deleted", cause) + .doesNotContain(target.id()); + assertThat(lastTraceDeleteSql()) + .as("no partition predicate is emitted for a %s batch", cause) + .doesNotContain("toDayOfWeek"); + } + + private static Stream underivableIdDisablesPruning() { + return Stream.of( + arguments("non-v7", NON_V7_ID), + arguments("beyond-2299", OUT_OF_RANGE_ID)); + } + + /** + * The partition the id resolves to, as it appears in the SQL. Derived through {@code WeeklyPartitions} on purpose: + * this suite is about the predicate reaching ClickHouse, and the derivation's own expected values are pinned against + * real ClickHouse output in {@code WeeklyPartitionsTest}, so restating them here would only duplicate that. + */ + private static String onlyPartitionOf(UUID id) { + return String.valueOf(WeeklyPartitions.of(List.of(id)).orElseThrow().iterator().next()); + } + + /** Invokes the DAO under a workspace/user context, as {@code TraceService} does for the live delete path. */ + private void delete(Set> projectIdTraceIdPairs) { + template.nonTransaction(connection -> traceDAO.delete(projectIdTraceIdPairs, connection)) + .contextWrite(ctx -> ctx + .put(RequestContext.WORKSPACE_ID, WORKSPACE_ID) + .put(RequestContext.USER_NAME, USER)) + .block(); + } + + /** + * The SQL of the most recent trace delete, as ClickHouse received it. {@code log_comment} is what makes this + * unambiguous: {@code TraceDAO} stamps every statement with {@code :::

}, and + * {@code delete_traces} names this one template alone. + */ + private String lastTraceDeleteSql() { + execute("SYSTEM FLUSH LOGS", _ -> { + }); + return queryOneString(""" + SELECT query + FROM system.query_log + WHERE log_comment LIKE 'delete_traces:%' + AND type = 'QueryFinish' + ORDER BY event_time_microseconds DESC + LIMIT 1 + """); + } + + /** + * The EXCHANGE (000003 exchange block): puts the successor under {@code traces} and the original under + * {@code traces_local_v2}, then a RENAME parks the original as {@code traces_pre_cutover_backup}. The wrap is + * deliberately not applied — it is a separate, deferrable step, and the flag under test must hold on its own between + * the two (which is why it is not the wrap flag). Kept identical to the cutover SQL by eye, as + * {@code TracesLocalV2CutoverTest.exchangeTables} and the wrap suite do. + */ + private void exchangeTables() { + execute("EXCHANGE TABLES traces AND traces_local_v2 ON CLUSTER '{cluster}'", _ -> { + }); + execute("RENAME TABLE traces_local_v2 TO traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { + }); + } + + /** + * A trace with every trace-table column populated. Only the span-derived aggregates podam would otherwise + * fabricate ({@code feedbackScores}, {@code usage}) are nulled, since they are not columns of the {@code traces} + * table and would only add noise. The generated {@code projectName} is fresh per trace, so a test that wants two + * traces in one project has to say so; that keeps one test's rows out of another's reads. + */ + private Trace.TraceBuilder newTrace() { + return factory.manufacturePojo(Trace.class).toBuilder() + .feedbackScores(null) + .usage(null); + } + + private UUID projectIdOf(Trace trace) { + return traceResourceClient + .getTraces(trace.projectName(), null, API_KEY, WORKSPACE_NAME, List.of(), List.of(), 100, Map.of()) + .content().stream() + .filter(found -> found.id().equals(trace.id())) + .map(Trace::projectId) + .findFirst() + .orElseThrow(); + } + + private List traceIdsOf(String projectName) { + return traceResourceClient + .getTraces(projectName, null, API_KEY, WORKSPACE_NAME, List.of(), List.of(), 100, Map.of()) + .content().stream() + .map(Trace::id) + .toList(); + } + + private String queryOneString(String sql) { + return template.nonTransaction(connection -> Mono + .from(connection.createStatement(sql).execute()) + .flatMap(result -> Mono.from(result.map((row, _) -> row.get(0, String.class))))) + .block(); + } + + private void execute(String sql, Consumer binder) { + template.nonTransaction(connection -> { + var statement = connection.createStatement(sql); + binder.accept(statement); + return Mono.from(statement.execute()).flatMap(result -> Mono.from(result.getRowsUpdated())); + }).block(); + } +} From 5f6e68d8c7a2795401066edebc468808c78d3b55 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 18:06:25 +0200 Subject: [PATCH 07/37] [OPIK-6901] [BE] fix: reject a null batch instead of reading it as unprunable `of` walked the collection straight into an enhanced for, so a null batch threw NPE from inside the loop - after the javadoc had promised an empty-result fallback for anything it cannot derive. @NonNull rather than normalising null to Optional.empty(), for two reasons. It is the prevailing convention: every other collection-taking public static method in com.comet.opik.utils is annotated that way (EnrichmentUtils.buildFeedbackScoresNode/buildCommentsNode, PaginationUtils.paginate). And normalising would be actively worse here, because empty is not an error signal - it is the documented answer "this batch cannot be pruned, emit the unbounded form". A caller that lost its batch would receive that answer, issue a correct-but-unbounded mutation, and never learn it had a bug. A null collection is a programming error; a null ELEMENT is the data condition, and that keeps returning empty. Co-Authored-By: Claude Opus 5 (1M context) --- .../java/com/comet/opik/utils/WeeklyPartitions.java | 11 ++++++++++- .../com/comet/opik/utils/WeeklyPartitionsTest.java | 13 +++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java b/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java index 8fbc8716236..eaa7970b8bc 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java @@ -1,5 +1,6 @@ package com.comet.opik.utils; +import lombok.NonNull; import lombok.experimental.UtilityClass; import java.time.DayOfWeek; @@ -66,8 +67,16 @@ public class WeeklyPartitions { * The weekly partition values the ids resolve to, or empty if the batch contains an id whose partition cannot be * derived exactly (see the class javadoc) — in which case the caller must omit its partition predicate entirely. * An empty batch yields empty for the same reason: there is nothing to bound the mutation to. + *

+ * A {@code null} batch throws rather than reading as empty, which is the one place this class is deliberately + * intolerant. Empty is a documented answer — "this batch cannot be pruned, emit the unbounded form" — and a + * caller that lost its batch would receive that answer, silently issue a correct-but-unbounded mutation, and never + * learn it had a bug. A null collection here is a programming error, not a data condition; a null element + * is the data condition, and that keeps returning empty. + * + * @throws NullPointerException if {@code ids} is null. */ - public static Optional> of(Collection ids) { + public static Optional> of(@NonNull Collection ids) { var partitions = new HashSet(); for (UUID id : ids) { diff --git a/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java index f826b35ae5c..aa4b9c833fc 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java @@ -8,6 +8,7 @@ import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** * Covers {@link WeeklyPartitions#of}, which derives the weekly partition values a delete batch resolves to so the @@ -93,6 +94,18 @@ void emptyBatch() { assertThat(WeeklyPartitions.of(List.of())).isEmpty(); } + @Test + @DisplayName("a null batch throws rather than reading as an unprunable one") + void nullBatchThrows() { + // The one intolerant case, and deliberately not folded into the empty result above: empty is a documented + // answer ("emit the unbounded form"), so a caller that lost its batch would get a valid-looking answer, issue a + // correct-but-unbounded mutation, and never learn it had a bug. Matches every other collection-taking method in + // this package, all of which are @NonNull. + assertThatThrownBy(() -> WeeklyPartitions.of(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("ids"); + } + @Test @DisplayName("the last id_at the column can store still prunes") void lastRepresentableIdStillPrunes() { From 4f3876ba94e30487a32df926105508e3409b4484 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 18:07:26 +0200 Subject: [PATCH 08/37] [OPIK-6901] [BE] fix: return the derived partitions immutably Reproduced: the returned Optional wrapped the working HashSet, so `of(...).orElseThrow().add(99L)` succeeded. Now Set.copyOf, per apps/opik-backend/AGENTS.md ("prefer immutable collections"). This is more than collection hygiene because of what the set is. It is the exact list of partitions a trace DELETE binds to `IN :partitions`, and the partitions a batch resolves to are the ONLY places its rows can be - that is the entire premise of the pruning. A caller that removed an entry would not get a slower delete, it would get one that matches nothing and reports success: the same silent-skip failure the v7 guard and the DateTime64 ceiling guard exist to prevent, reached through the return value instead. The accumulator stays a local HashSet - copyOf is taken once, at the boundary - so dedup across a batch is unaffected. Co-Authored-By: Claude Opus 5 (1M context) --- .../com/comet/opik/utils/WeeklyPartitions.java | 5 ++++- .../comet/opik/utils/WeeklyPartitionsTest.java | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java b/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java index eaa7970b8bc..193d00bc841 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/utils/WeeklyPartitions.java @@ -98,6 +98,9 @@ public static Optional> of(@NonNull Collection ids) { partitions.add(monday.getYear() * 10000L + monday.getMonthValue() * 100L + monday.getDayOfMonth()); } - return partitions.isEmpty() ? Optional.empty() : Optional.of(partitions); + // Set.copyOf, not the working HashSet: what escapes here decides which partitions a DELETE mutation touches, so + // a caller holding a mutable reference could narrow the set after it was derived and turn a correct delete into + // a silent no-op. Immutable by default per apps/opik-backend/AGENTS.md, and the accumulator stays local. + return partitions.isEmpty() ? Optional.empty() : Optional.of(Set.copyOf(partitions)); } } diff --git a/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java index aa4b9c833fc..956fa591d6e 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java @@ -94,6 +94,21 @@ void emptyBatch() { assertThat(WeeklyPartitions.of(List.of())).isEmpty(); } + @Test + @DisplayName("the returned set is immutable, so a delete's partitions cannot be narrowed after derivation") + void returnedSetIsImmutable() { + // Not a general hygiene assertion: this set IS the partition list a DELETE binds, so a caller that removed an + // entry would turn a correct delete into one that matches nothing and reports success. + var partitions = WeeklyPartitions.of(List.of( + UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), + UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8"))) + .orElseThrow(); + + assertThatThrownBy(() -> partitions.remove(20260817L)) + .isInstanceOf(UnsupportedOperationException.class); + assertThat(partitions).containsExactlyInAnyOrder(20260817L, 19960205L); + } + @Test @DisplayName("a null batch throws rather than reading as an unprunable one") void nullBatchThrows() { From 5bd4544dae44c3cad2cecd30903ae4ffee0d6c8a Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 18:10:53 +0200 Subject: [PATCH 09/37] [OPIK-6901] [BE] fix: rename the flag to say pruning, not partitioning tracesWeeklyPartitioningEnabled read as "create/activate weekly partitioning". It does no such thing: it enables partition-aware PRUNING of trace deletes, and the partitioned schema arrives with the cutover's EXCHANGE and nowhere else. Not cosmetic, given what this particular flag is. It is documented as safe to LAG and unsafe to LEAD, so the name actively invited the one mistake the docs warn against - an operator reading it as a step that makes the cutover progress, setting it early to "turn partitioning on", and instead starting to emit a partition predicate against the legacy unpartitioned `traces`, where a far-future id then matches zero rows and reports success. Renamed everywhere in one pass - 21 references across 11 files: config.yml, config-test.yml, DatabaseAnalyticsDataModelConfig (record component + javadoc), TraceDAO (accessor + javadocs), the cutover runbook (4), docker-compose env var + comment, helm values/configmap/README/env test, and the integration test's CustomConfig. Verified no occurrence of the old name or env var remains. Env var likewise renamed (ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITION_PRUNING_ENABLED); it is not yet set in any deployment repo, so nothing external breaks. Also reworded the descriptions to lead with what the flag does rather than what must already be true - "enables pruning; it does NOT create or activate any partitioning; turning it on ASSERTS a schema fact rather than causing one" - since a correct name with prose that still opens "whether the live table is partitioned" would re-create the ambiguity. Co-Authored-By: Claude Opus 5 (1M context) --- apps/opik-backend/config.yml | 15 +++++++++------ .../traces-local-v2-cutover/README.md | 13 +++++++++---- .../java/com/comet/opik/domain/TraceDAO.java | 17 +++++++++-------- .../DatabaseAnalyticsDataModelConfig.java | 17 ++++++++++------- .../TracesPartitionPruningMutationTest.java | 6 +++--- .../src/test/resources/config-test.yml | 15 +++++++++------ deployment/docker-compose/docker-compose.yaml | 9 +++++---- deployment/helm_chart/opik/README.md | 2 +- .../opik/templates/configmap-backend.yaml | 2 +- .../opik/tests/configmap_env_test.yaml | 2 +- deployment/helm_chart/opik/values.yaml | 5 +++-- 11 files changed, 60 insertions(+), 43 deletions(-) diff --git a/apps/opik-backend/config.yml b/apps/opik-backend/config.yml index 40a94efdfe0..7d78715dd5f 100644 --- a/apps/opik-backend/config.yml +++ b/apps/opik-backend/config.yml @@ -156,16 +156,19 @@ databaseAnalyticsDataModel: # both `traces_local` and `traces`, else reads can't see the column (code 47). tracesDistributedWrapEnabled: ${ANALYTICS_DB_DATA_MODEL_TRACES_DISTRIBUTED_WRAP_ENABLED:-false} # Default: false - # Description: Whether the live trace mutation target is the weekly partitioned successor (id_at as - # DateTime64(0,'UTC') under PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))), so a - # trace DELETE can bound itself to the partitions its own ids resolve to instead of being planned against every part - # of the table. Purely an optimisation: false keeps the unbounded mutation, which is always correct and merely - # slower. A third flag on purpose - the partitioning appears at the EXCHANGE, and neither sibling marks it: + # Description: Enables partition-aware PRUNING of trace deletes - it does NOT create or activate any partitioning. + # With it on, a trace DELETE bounds itself to the weekly partitions its own ids resolve to instead of being planned + # against every part of the table. Turning it on therefore ASSERTS a schema fact rather than causing one: that the + # live mutation target already IS the weekly partitioned successor, id_at as DateTime64(0,'UTC') under + # PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1))). Installing that schema is the + # EXCHANGE step of the cutover, never this flag. Purely an optimisation: false keeps the unbounded mutation, which + # is always correct and merely slower. A third flag on purpose - the partitioning appears at the EXCHANGE, and + # neither sibling marks it: # traceColumnsNonNullable must be rolled out BEFORE the EXCHANGE, tracesDistributedWrapEnabled flips at the wrap, # which may be deferred long after it. Leave false at deploy time; set true once the EXCHANGE is confirmed, and back # to false BEFORE a rollback promotes the original `traces` (legacy `traces` has no PARTITION BY and a 32-bit # DateTime id_at that overflows past 2106, so the predicate would silently match zero rows for a far-future id). - tracesWeeklyPartitioningEnabled: ${ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED:-false} + tracesWeeklyPartitionPruningEnabled: ${ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITION_PRUNING_ENABLED:-false} # Description: UUIDv7 ingestion validation. Rejects writes whose `id` embeds a timestamp outside the # window, protecting data quality. diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index b1848368c71..c885212aeea 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -322,12 +322,17 @@ the `traceColumnsNonNullable` flip"). On rollback, after swapping the Nullable original back, revert the flag to `false` **and** run that repair. -**The `tracesWeeklyPartitioningEnabled` flip (optional, and why it goes last).** `databaseAnalyticsDataModel.tracesWeeklyPartitioningEnabled` -(env `ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED`, default `false`) lets a trace `DELETE` bound itself to +**The `tracesWeeklyPartitionPruningEnabled` flip (optional, and why it goes last).** `databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled` +(env `ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITION_PRUNING_ENABLED`, default `false`) lets a trace `DELETE` bound itself to the weekly partitions its own ids resolve to (OPIK-6901), instead of being planned against every part of the table — on prod-test, 12 ids rewrote 3,928 parts / 5.40 TiB without it. It asserts a **schema** fact: that `traces` (or `traces_local`) is the successor, with `id_at` as `DateTime64(0,'UTC')` under the weekly `PARTITION BY`. +> **It enables the *pruning*, not the partitioning — the name is deliberate.** Setting it does not create, activate or +> migrate anything; the partitioned schema arrives with the `EXCHANGE` above and nowhere else. So it is never a step that +> *makes* the cutover progress, and setting it early does not bring the partitioning forward — it only starts emitting a +> predicate against whatever table is live, which is the failure below. + It is a third flag precisely because **neither of the two above marks the `EXCHANGE`**, which is when the partitioning appears: `traceColumnsNonNullable` must lead it (above), and `tracesDistributedWrapEnabled` flips at the wrap, which may be deferred long after it (`--skip-wrap` … `--wrap-only`). So gate on this one, not on either of those. @@ -851,9 +856,9 @@ statements, so a failure *between* them needs a restart path: again, so the flip has to be undone in two steps — `rollback.sh` prints both when the stage finishes. The rollback is not complete until they land. -> **If `tracesWeeklyPartitioningEnabled` was turned on, revert it *before* the stage runs, not after.** It asserts the +> **If `tracesWeeklyPartitionPruningEnabled` was turned on, revert it *before* the stage runs, not after.** It asserts the > live table is the partitioned successor, and the restored original is not one, so a stale `true` makes trace deletes -> match zero rows while reporting success — see "The `tracesWeeklyPartitioningEnabled` flip". It is the one flag whose +> match zero rows while reporting success — see "The `tracesWeeklyPartitionPruningEnabled` flip". It is the one flag whose > revert has to lead the swap; the two steps below follow it. 1. **Revert `traceColumnsNonNullable` to `false` AND roll-restart every backend instance.** The flag is read from a diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java index dbfa8cadfbd..0b86a21739e 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java @@ -1926,7 +1926,7 @@ ORDER BY (workspace_id, project_id, trace_id, id) DESC, last_updated_at DESC *

* {@code } adds the table's own weekly partition expression, bound as the exact set of partitions * the batch's ids resolve to. Both conditions must hold for it to be emitted: the live table must be the weekly - * partitioned successor ({@link #tracesWeeklyPartitioningEnabled()}), and every id in the batch must be one whose + * partitioned successor ({@link #tracesWeeklyPartitionPruningEnabled()}), and every id in the batch must be one whose * partition can be derived exactly ({@link WeeklyPartitions#of}). Otherwise the predicate is omitted and the * statement is byte-identical to the previous unbounded form. That is what preserves the original guarantee — a * row whose {@code id_at} cannot be trusted is still deleted, because no id in such a batch is used to derive a @@ -3372,10 +3372,11 @@ private void selectTracesMutationTable(ST template) { } /** - * Whether the live trace mutation target is the weekly partitioned successor — {@code id_at} as - * {@code DateTime64(0, 'UTC')} under - * {@code PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))} — so a mutation may bound - * itself to the partitions its ids resolve to ({@link WeeklyPartitions}). + * Whether a trace mutation may prune to the partitions its ids resolve to ({@link WeeklyPartitions}). The flag + * enables the pruning, never the partitioning: it asserts that the live mutation target already is the + * weekly partitioned successor — {@code id_at} as {@code DateTime64(0, 'UTC')} under + * {@code PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))} — which the cutover's + * EXCHANGE installs, not this flag. *

* This has to be its own flag because neither existing schema flag marks the EXCHANGE, which is the moment the * partitioning appears, and both are wrong in a different direction: @@ -3400,8 +3401,8 @@ private void selectTracesMutationTable(ST template) { * schema. Turn it on once the EXCHANGE is confirmed, and back off before a rollback stage B/C promotes the * original. */ - private boolean tracesWeeklyPartitioningEnabled() { - return configuration.getDatabaseAnalyticsDataModel().tracesWeeklyPartitioningEnabled(); + private boolean tracesWeeklyPartitionPruningEnabled() { + return configuration.getDatabaseAnalyticsDataModel().tracesWeeklyPartitionPruningEnabled(); } /** @@ -3410,7 +3411,7 @@ private boolean tracesWeeklyPartitioningEnabled() { * only meaningful against a table that partitions on it. */ private Optional> weeklyPartitionsFor(Collection ids) { - return tracesWeeklyPartitioningEnabled() ? WeeklyPartitions.of(ids) : Optional.empty(); + return tracesWeeklyPartitionPruningEnabled() ? WeeklyPartitions.of(ids) : Optional.empty(); } /** diff --git a/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/DatabaseAnalyticsDataModelConfig.java b/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/DatabaseAnalyticsDataModelConfig.java index a83c4a40406..1c27e719b15 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/DatabaseAnalyticsDataModelConfig.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/infrastructure/DatabaseAnalyticsDataModelConfig.java @@ -51,12 +51,15 @@ * (the wrapper accepts them as metadata-only, and targeting only {@code traces_local} leaves the wrapper without the * column, so reads fail with code 47).

* - *

{@code tracesWeeklyPartitioningEnabled}: whether the live trace mutation target is the weekly partitioned - * successor — {@code id_at} as {@code DateTime64(0, 'UTC')} under - * {@code PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))} — so a trace {@code DELETE} - * may bound itself to the partitions its own ids resolve to instead of being planned against every part of the table - * (OPIK-6901). Purely an optimisation: {@code false} keeps the unbounded mutation, which is always correct and merely - * slower, and only {@code true} asserts anything about the schema.

+ *

{@code tracesWeeklyPartitionPruningEnabled}: enables partition-aware pruning of trace deletes — a trace + * {@code DELETE} bounds itself to the weekly partitions its own ids resolve to instead of being planned against every + * part of the table (OPIK-6901). It does not create or activate any partitioning; installing the partitioned + * schema is the EXCHANGE step of the cutover. Turning it on therefore asserts a schema fact rather than causing + * one: that the live mutation target already is the weekly partitioned successor — {@code id_at} as + * {@code DateTime64(0, 'UTC')} under + * {@code PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))}. Purely an optimisation: + * {@code false} keeps the unbounded mutation, which is always correct and merely slower, and only {@code true} asserts + * anything about the schema.

* *

It is deliberately a third flag rather than a reuse of the two above, because the partitioning appears at the * EXCHANGE and neither of them marks that moment. {@code traceColumnsNonNullable} must be rolled out @@ -76,5 +79,5 @@ public record DatabaseAnalyticsDataModelConfig( boolean spanDeletionEventsCaptureEnabled, @Min(1) @Max(2_000) int deletionEventsInsertBatchSize, boolean tracesDistributedWrapEnabled, - boolean tracesWeeklyPartitioningEnabled) { + boolean tracesWeeklyPartitionPruningEnabled) { } diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index f27a17abcec..95be590f8f6 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -69,7 +69,7 @@ *

{@link #liveTracesIsTheWeeklyPartitionedSuccessor} is the guard that keeps the rest honest. The predicate is * harmless against an unpartitioned table for recent ids, so had the EXCHANGE below not taken effect every test here * would still pass while proving nothing; it pins both facts - * {@code databaseAnalyticsDataModel.tracesWeeklyPartitioningEnabled} asserts — the weekly {@code PARTITION BY} and + * {@code databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled} asserts — the weekly {@code PARTITION BY} and * {@code id_at} as {@code DateTime64}. * *

Two internal touches, on the pattern of {@code TracesDistributedWrapMutationTest}: the EXCHANGE has no public API, @@ -143,7 +143,7 @@ class TracesPartitionPruningMutationTest { // non-nullable sentinel columns, and the pruning flag asserts the schema this suite installs. .customConfigs(List.of( new CustomConfig("databaseAnalyticsDataModel.traceColumnsNonNullable", "true"), - new CustomConfig("databaseAnalyticsDataModel.tracesWeeklyPartitioningEnabled", + new CustomConfig("databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled", "true"))) .build()); } @@ -174,7 +174,7 @@ void afterAll() { @Test @DisplayName("the live traces table is the weekly partitioned successor") void liveTracesIsTheWeeklyPartitionedSuccessor() { - // Both halves of what tracesWeeklyPartitioningEnabled asserts. Without this the whole suite is vacuous: against + // Both halves of what tracesWeeklyPartitionPruningEnabled asserts. Without this the whole suite is vacuous: against // the legacy unpartitioned `traces` a pruned delete of a recent id still removes the row, so every behavioural // assertion below would stay green while the predicate was being emitted at exactly the table it must not be. // Asserted piecewise rather than against PARTITION_PREDICATE verbatim: system.tables reports the expression as diff --git a/apps/opik-backend/src/test/resources/config-test.yml b/apps/opik-backend/src/test/resources/config-test.yml index b0d5e6f7734..114caa3463e 100644 --- a/apps/opik-backend/src/test/resources/config-test.yml +++ b/apps/opik-backend/src/test/resources/config-test.yml @@ -129,16 +129,19 @@ databaseAnalyticsDataModel: # both `traces_local` and `traces`, else reads can't see the column (code 47). tracesDistributedWrapEnabled: false # Default: false - # Description: Whether the live trace mutation target is the weekly partitioned successor (id_at as - # DateTime64(0,'UTC') under PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))), so a - # trace DELETE can bound itself to the partitions its own ids resolve to instead of being planned against every part - # of the table. Purely an optimisation: false keeps the unbounded mutation, which is always correct and merely - # slower. A third flag on purpose - the partitioning appears at the EXCHANGE, and neither sibling marks it: + # Description: Enables partition-aware PRUNING of trace deletes - it does NOT create or activate any partitioning. + # With it on, a trace DELETE bounds itself to the weekly partitions its own ids resolve to instead of being planned + # against every part of the table. Turning it on therefore ASSERTS a schema fact rather than causing one: that the + # live mutation target already IS the weekly partitioned successor, id_at as DateTime64(0,'UTC') under + # PARTITION BY toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1))). Installing that schema is the + # EXCHANGE step of the cutover, never this flag. Purely an optimisation: false keeps the unbounded mutation, which + # is always correct and merely slower. A third flag on purpose - the partitioning appears at the EXCHANGE, and + # neither sibling marks it: # traceColumnsNonNullable must be rolled out BEFORE the EXCHANGE, tracesDistributedWrapEnabled flips at the wrap, # which may be deferred long after it. Leave false at deploy time; set true once the EXCHANGE is confirmed, and back # to false BEFORE a rollback promotes the original `traces` (legacy `traces` has no PARTITION BY and a 32-bit # DateTime id_at that overflows past 2106, so the predicate would silently match zero rows for a far-future id). - tracesWeeklyPartitioningEnabled: false + tracesWeeklyPartitionPruningEnabled: false # Description: UUIDv7 ingestion validation. Rejects writes whose `id` embeds a timestamp outside the # window, protecting data quality. diff --git a/deployment/docker-compose/docker-compose.yaml b/deployment/docker-compose/docker-compose.yaml index 7d0c3663a28..4346af90387 100644 --- a/deployment/docker-compose/docker-compose.yaml +++ b/deployment/docker-compose/docker-compose.yaml @@ -188,10 +188,11 @@ services: # Flip TRACES_DISTRIBUTED_WRAP_ENABLED to true in lockstep with applying the Distributed wrap: it routes trace # delete/retention mutations to traces_local, since a Distributed traces rejects mutations. ANALYTICS_DB_DATA_MODEL_TRACES_DISTRIBUTED_WRAP_ENABLED: ${ANALYTICS_DB_DATA_MODEL_TRACES_DISTRIBUTED_WRAP_ENABLED:-false} - # Flip TRACES_WEEKLY_PARTITIONING_ENABLED to true only once the EXCHANGE has put the weekly partitioned - # successor under `traces`: it lets a trace DELETE prune to the partitions its own ids resolve to. Purely an - # optimisation - false keeps the always-correct unbounded mutation. - ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED: ${ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED:-false} + # Flip TRACES_WEEKLY_PARTITION_PRUNING_ENABLED to true only once the EXCHANGE has put the weekly partitioned + # successor under `traces`: it lets a trace DELETE prune to the partitions its own ids resolve to. It enables the + # PRUNING, not the partitioning - setting it does not create or activate anything. Purely an optimisation - false + # keeps the always-correct unbounded mutation. + ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITION_PRUNING_ENABLED: ${ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITION_PRUNING_ENABLED:-false} # Readiness probes for the Hyperscale topology, both off for the single-node Compose stack: the cluster probe # checks the Distributed cluster definition is visible, the cold-storage probe checks the cold_s3 tiered disk. ANALYTICS_DB_CLUSTER_HEALTH_CHECK_ENABLED: ${ANALYTICS_DB_CLUSTER_HEALTH_CHECK_ENABLED:-false} diff --git a/deployment/helm_chart/opik/README.md b/deployment/helm_chart/opik/README.md index ecefe3431cf..b91ba042349 100644 --- a/deployment/helm_chart/opik/README.md +++ b/deployment/helm_chart/opik/README.md @@ -477,7 +477,7 @@ Call opik api on http://localhost:5173/api | databaseAnalyticsDataModel.traceColumnsNonNullable | bool | `false` | | | databaseAnalyticsDataModel.traceDeletionEventsCaptureEnabled | bool | `false` | | | databaseAnalyticsDataModel.tracesDistributedWrapEnabled | bool | `false` | | -| databaseAnalyticsDataModel.tracesWeeklyPartitioningEnabled | bool | `false` | | +| databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled | bool | `false` | | | demoDataJob.enabled | bool | `true` | | | fullnameOverride | string | `""` | | | global.argocd | bool | `false` | | diff --git a/deployment/helm_chart/opik/templates/configmap-backend.yaml b/deployment/helm_chart/opik/templates/configmap-backend.yaml index 7fbc9c55941..4e2d3491e40 100644 --- a/deployment/helm_chart/opik/templates/configmap-backend.yaml +++ b/deployment/helm_chart/opik/templates/configmap-backend.yaml @@ -50,7 +50,7 @@ data: "ANALYTICS_DB_DATA_MODEL_SPAN_DELETION_EVENTS_CAPTURE_ENABLED" $dm.spanDeletionEventsCaptureEnabled "ANALYTICS_DB_DATA_MODEL_DELETION_EVENTS_INSERT_BATCH_SIZE" $dm.deletionEventsInsertBatchSize "ANALYTICS_DB_DATA_MODEL_TRACES_DISTRIBUTED_WRAP_ENABLED" $dm.tracesDistributedWrapEnabled - "ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED" $dm.tracesWeeklyPartitioningEnabled + "ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITION_PRUNING_ENABLED" $dm.tracesWeeklyPartitionPruningEnabled "PARTITION_METRICS_ENABLED" $pm.enabled "PARTITION_METRICS_INTERVAL" $pm.interval "PARTITION_METRICS_LWD_TABLES" $pm.lwdTables diff --git a/deployment/helm_chart/opik/tests/configmap_env_test.yaml b/deployment/helm_chart/opik/tests/configmap_env_test.yaml index 0dea3793a4a..458a24bfd91 100644 --- a/deployment/helm_chart/opik/tests/configmap_env_test.yaml +++ b/deployment/helm_chart/opik/tests/configmap_env_test.yaml @@ -74,7 +74,7 @@ tests: path: data.ANALYTICS_DB_DATA_MODEL_TRACES_DISTRIBUTED_WRAP_ENABLED value: "false" - equal: - path: data.ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITIONING_ENABLED + path: data.ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITION_PRUNING_ENABLED value: "false" - equal: path: data.PARTITION_METRICS_ENABLED diff --git a/deployment/helm_chart/opik/values.yaml b/deployment/helm_chart/opik/values.yaml index 9551231b90d..73b32acf8e3 100644 --- a/deployment/helm_chart/opik/values.yaml +++ b/deployment/helm_chart/opik/values.yaml @@ -741,8 +741,9 @@ databaseAnalyticsDataModel: # traces_local (a Distributed traces rejects mutations). Leave false while traces is still a MergeTree. tracesDistributedWrapEnabled: false # Turn true only once the EXCHANGE has put the weekly partitioned successor under `traces`, so a trace DELETE - # can prune to the partitions its own ids resolve to. Purely an optimisation; false keeps the unbounded delete. - tracesWeeklyPartitioningEnabled: false + # can prune to the partitions its own ids resolve to. Enables the PRUNING, not the partitioning - setting it creates + # nothing. Purely an optimisation; false keeps the unbounded delete. + tracesWeeklyPartitionPruningEnabled: false # ClickHouse partition-health observability (OPIK-6904). Polls system.parts plus the # lightweight-delete mask and publishes opik.clickhouse.partition.* gauges; a distributed lock keeps From 906c7b287a477227427c3a41a9482e68e739a10e Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 18:15:13 +0200 Subject: [PATCH 10/37] [OPIK-6901] [BE] test: pin the partition expression exactly, not by substring The live-schema guard accepted any partition_key containing toYYYYMMDD, toDate32(id_at), toIntervalDay and toDayOfWeek(id_at). A guard whose whole job is catching drift should not be satisfiable by an expression that merely uses the right functions - `toMonday(id_at)` plus those names in any arrangement would have passed. Replaced with two exact pins, because the one thing that has to hold is stated in three independently-maintained places (the migration's PARTITION BY, the DAO's predicate, and WeeklyPartitions.of) and drift in any one silently skips deletes. 1. Value agreement, parameterized over three eras. Seeds a row per era and asserts _partition_id (where ClickHouse actually filed it, i.e. the migration as installed) == the DAO predicate evaluated on that row == WeeklyPartitions.of. Follows the existing _partition_id idiom in TracesLocalV2TableTest, where the partition id is that Monday's YYYYMMDD. The eras are load-bearing: toMonday agrees with the Date32 expression across the ordinary calendar and diverges only far-future or at the epoch, so a recent-only sample would accept the very expression 000114 was written to escape. The far-future row is what makes it bite. Rows are seeded in raw SQL (ingestion rejects backdated and far-future ids by design) supplying only the three columns without a DEFAULT, so id_at comes from the real MATERIALIZED definition rather than a restated copy - restating it would add the drift surface this test is for. 2. AST agreement, which value agreement does NOT give. Pruning requires the planner to recognise the predicate as being on the partition key expression, so an equal-valued but differently-written expression would leave every delete correct while quietly rewriting every part again - this PR's exact regression, invisible to every other assertion in the suite. Asserted by round-tripping the DAO's text through ClickHouse as a partition key of its own and diffing the two re-prints: both come from the same printer, so they match iff the parsed expressions do. That is formatter-independent by construction, which diffing the DAO text against system.tables directly would not be - and pinning ClickHouse's whitespace is what the guard must not do. The id_at DateTime64 half moves to its own test; it is not implied by either pin, since a 32-bit id_at agrees with itself while wrapping every id past 2106. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 159 +++++++++++++++--- 1 file changed, 137 insertions(+), 22 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 95be590f8f6..a6ee841b148 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -66,11 +66,14 @@ * still remove the row, just slowly — the regression the flag and the derivation exist to prevent, and one no * behavioural assertion can see). * - *

{@link #liveTracesIsTheWeeklyPartitionedSuccessor} is the guard that keeps the rest honest. The predicate is - * harmless against an unpartitioned table for recent ids, so had the EXCHANGE below not taken effect every test here - * would still pass while proving nothing; it pins both facts - * {@code databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled} asserts — the weekly {@code PARTITION BY} and - * {@code id_at} as {@code DateTime64}. + *

{@link #predicateMatchesLivePartitioningAndJavaDerivation} and {@link #idAtIsTheSixtyFourBitColumn} are the guards + * that keep the rest honest, together pinning both facts + * {@code databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled} asserts. They are load-bearing twice over. The + * predicate is harmless against an unpartitioned table for recent ids, so had the EXCHANGE below not taken effect every + * other test here would still pass while proving nothing. And the rule itself is expressed three times over — the + * migration's {@code PARTITION BY}, the DAO's predicate, and {@link WeeklyPartitions#of} — so the first guard makes all + * three compute the same value for the same row, across the eras where a plausible wrong expression + * ({@code toMonday}) would diverge. * *

Two internal touches, on the pattern of {@code TracesDistributedWrapMutationTest}: the EXCHANGE has no public API, * so {@link #beforeAll} runs it in raw SQL identical to the swap block of {@code 000003_exchange_and_wrap.sql}; and the @@ -97,6 +100,15 @@ class TracesPartitionPruningMutationTest { private static final String PARTITION_PREDICATE = "toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))"; + /** + * Throwaway table used only to have ClickHouse re-print the DAO predicate as a partition key. + * Created and dropped in-test. + */ + private static final String PARTITION_KEY_PROBE = "traces_partition_key_probe"; + + /** Project for the raw-SQL seeded rows, kept off the API-created projects so neither test's reads see the other's. */ + private static final UUID RAW_PROJECT_ID = UUID.randomUUID(); + /** A v4 UUID: no timestamp to derive a partition from. */ private static final UUID NON_V7_ID = UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22"); @@ -171,22 +183,90 @@ void afterAll() { network.close(); } + @ParameterizedTest + @MethodSource + @DisplayName("the DAO predicate, the live partitioning and the Java derivation agree exactly") + void predicateMatchesLivePartitioningAndJavaDerivation(String era, UUID id, long expectedPartition) { + // The guard the rest of the suite rests on, and the one that catches drift. Three independently-maintained + // expressions of the same rule have to agree, or a delete prunes to a partition its rows are not in: + // + // 1. the migration's PARTITION BY, as ClickHouse actually installed it -> _partition_id, where it filed the row + // 2. the DAO's predicate -> PARTITION_PREDICATE, evaluated here + // 3. WeeklyPartitions.of -> what gets bound to :partitions + // + // Compared as VALUES, not as normalized expression text. A text comparison would pin (1) against (2) and say + // nothing about (3), and it would pass for a rewrite that is textually equal after normalization yet computes a + // different week — which is precisely the toMonday trap migration 000114 was written to escape. Values also make + // the check immune to ClickHouse's re-printing of the AST, which is what made the previous substring form loose. + // + // The era matters: toMonday agrees with the Date32 expression across the ordinary calendar and diverges only for + // a far-future or epoch id_at, so a sample set that stopped at "recent" would accept the wrong expression. The + // rows are inserted in raw SQL because ingestion rejects a backdated or far-future id by design; only + // (workspace_id, project_id, id) are supplied, since id_at is MATERIALIZED and every other column has a DEFAULT + // — so this seeds through the real column definition rather than restating it. + insertRawTrace(id); + + var filedUnder = queryOneString("SELECT DISTINCT _partition_id FROM traces" + + " WHERE workspace_id = :workspace_id AND id = :id", bindRawTrace(id)); + // PARTITION_PREDICATE is interpolated because it is an expression, not a value - the point is to evaluate the + // DAO's own text. The ids and workspace go in as binds like everywhere else in this suite. + var daoPredicateValue = queryOneString("SELECT DISTINCT toString(" + PARTITION_PREDICATE + ") FROM traces" + + " WHERE workspace_id = :workspace_id AND id = :id", bindRawTrace(id)); + + assertThat(daoPredicateValue) + .as("the DAO predicate resolves to the partition ClickHouse filed the %s row under", era) + .isEqualTo(filedUnder); + assertThat(WeeklyPartitions.of(List.of(id))) + .as("the Java derivation agrees with both for the %s row", era) + .contains(Set.of(expectedPartition)); + assertThat(filedUnder) + .as("and all three are the expected partition for the %s row", era) + .isEqualTo(String.valueOf(expectedPartition)); + } + + private static Stream predicateMatchesLivePartitioningAndJavaDerivation() { + return Stream.of( + // id_at 2026-08-19 (Wed) -> Monday 2026-08-17. The ordinary case; toMonday would also pass this one. + arguments("recent", UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), 20260817L), + // id_at 1996-02-09 -> Monday 1996-02-05. Far enough back to catch a key that keyed off wall-clock. + arguments("backdated", UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8"), 19960205L), + // id_at 2200-01-01 -> Monday 2199-12-30. The litellm shape, and the sample a 16-bit toMonday key wraps + // into a plausible recent week (000114) — so this row is what makes the guard bite. + arguments("far-future", UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2"), 21991230L)); + } + @Test - @DisplayName("the live traces table is the weekly partitioned successor") - void liveTracesIsTheWeeklyPartitionedSuccessor() { - // Both halves of what tracesWeeklyPartitionPruningEnabled asserts. Without this the whole suite is vacuous: against - // the legacy unpartitioned `traces` a pruned delete of a recent id still removes the row, so every behavioural - // assertion below would stay green while the predicate was being emitted at exactly the table it must not be. - // Asserted piecewise rather than against PARTITION_PREDICATE verbatim: system.tables reports the expression as - // ClickHouse's own formatter re-prints it, so pinning its whitespace would make this brittle about the one thing - // it does not care about. No other partition expression in the schema is built from these three functions. - assertThat(queryOneString("SELECT partition_key FROM system.tables WHERE database = currentDatabase()" - + " AND name = 'traces'")) - .as("traces is partitioned by the weekly id_at expression") - .contains("toYYYYMMDD", "toDate32(id_at)", "toIntervalDay", "toDayOfWeek(id_at"); + @DisplayName("the DAO predicate is the same expression as the live partition key, not merely an equal-valued one") + void daoPredicateIsTheSameExpressionAsTheLivePartitionKey() { + // Value agreement (above) proves the predicate names the right partition; it does NOT prove ClickHouse will + // PRUNE on it. Pruning needs the planner to recognise the predicate as being on the partition key expression, + // so an equal-valued but differently-written expression would keep every delete correct and quietly rewrite + // every part again - the exact regression this PR exists to prevent, invisible to every other assertion here. + // + // Compared by round-tripping the DAO's text through ClickHouse as a partition key of its own and diffing the + // two re-prints. That makes the comparison AST-level and formatter-independent by construction: both strings + // come out of the same printer, so they are equal iff the parsed expressions are. Diffing the DAO text against + // system.tables directly would instead pin ClickHouse's whitespace choices, which is what it must not do. + execute("CREATE TABLE " + PARTITION_KEY_PROBE + " (id_at DateTime64(0, 'UTC')) ENGINE = MergeTree" + + " PARTITION BY " + PARTITION_PREDICATE + " ORDER BY tuple()", _ -> { + }); + try { + assertThat(partitionKeyOf(PARTITION_KEY_PROBE)) + .as("the DAO predicate parses to the same expression traces is partitioned by") + .isEqualTo(partitionKeyOf("traces")); + } finally { + execute("DROP TABLE IF EXISTS " + PARTITION_KEY_PROBE + " SYNC", _ -> { + }); + } + } + + @Test + @DisplayName("id_at is the 64-bit column, so a far-future timestamp is honest rather than wrapped") + void idAtIsTheSixtyFourBitColumn() { + // The second half of what the flag asserts, and not implied by the partition agreement above: a 32-bit + // DateTime id_at would agree with itself while silently wrapping every id past 2106. assertThat(queryOneString("SELECT type FROM system.columns WHERE database = currentDatabase()" + " AND table = 'traces' AND name = 'id_at'")) - .as("id_at is the 64-bit column, so a far-future timestamp is honest rather than wrapped") .isEqualTo("DateTime64(0, 'UTC')"); } @@ -341,11 +421,46 @@ private List traceIdsOf(String projectName) { .toList(); } + /** + * Seeds one row through the table's real column definitions. Only the three columns without a {@code DEFAULT} are + * supplied; {@code id_at} is {@code MATERIALIZED}, so ClickHouse derives it from {@code id} exactly as it does for a + * row the ingestion path wrote — which is the point, since restating the {@code id_at} expression here would create + * the very drift surface this test exists to detect. Raw SQL rather than the API because ingestion rejects a + * backdated or far-future {@code id} by design ({@code IdGenerator.validateId}). + */ + private void insertRawTrace(UUID id) { + execute("INSERT INTO traces (workspace_id, project_id, id) VALUES (:workspace_id, :project_id, :id)", + statement -> statement + .bind("workspace_id", WORKSPACE_ID) + .bind("project_id", RAW_PROJECT_ID.toString()) + .bind("id", id.toString())); + } + + /** ClickHouse's own re-print of a table's partition key expression. */ + private String partitionKeyOf(String table) { + return queryOneString("SELECT partition_key FROM system.tables" + + " WHERE database = currentDatabase() AND name = :table", + statement -> statement.bind("table", table)); + } + + private static Consumer bindRawTrace(UUID id) { + return statement -> statement + .bind("workspace_id", WORKSPACE_ID) + .bind("id", id.toString()); + } + private String queryOneString(String sql) { - return template.nonTransaction(connection -> Mono - .from(connection.createStatement(sql).execute()) - .flatMap(result -> Mono.from(result.map((row, _) -> row.get(0, String.class))))) - .block(); + return queryOneString(sql, _ -> { + }); + } + + private String queryOneString(String sql, Consumer binder) { + return template.nonTransaction(connection -> { + var statement = connection.createStatement(sql); + binder.accept(statement); + return Mono.from(statement.execute()) + .flatMap(result -> Mono.from(result.map((row, _) -> row.get(0, String.class)))); + }).block(); } private void execute(String sql, Consumer binder) { From 33f20b6f4f2be0eafe63d031f8fb42ec18548fb3 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Wed, 19 Aug 2026 18:16:12 +0200 Subject: [PATCH 11/37] [OPIK-6901] [BE] test: fold the three era cases into one parameterized test ordinaryId, farFutureId and oldId ran the same one-line assertion over different data, so they are now one @ParameterizedTest keyed by era ("ordinary id" / "id from 1996" / "far-future id" as the display name, so a failure still says which one). Kept deliberately: the ClickHouse-derived expected values and the note that they are what prod-test actually returned rather than hand-computed Mondays - that provenance is the whole point of these assertions. The @MethodSource javadoc now also records why the three eras are not interchangeable samples: an expression correct for the ordinary calendar and wrong at the extremes is the toMonday trap 000114 was written to escape, so dropping one would weaken the pin rather than tidy it. The other cases stay separate @Test methods on purpose - each asserts a different outcome (empty, throws, immutable) with its own reasoning, so parameterizing them would trade explanation for a table. Co-Authored-By: Claude Opus 5 (1M context) --- .../opik/utils/WeeklyPartitionsTest.java | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java index 956fa591d6e..30f3ea12582 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/utils/WeeklyPartitionsTest.java @@ -2,13 +2,18 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.params.provider.Arguments.arguments; /** * Covers {@link WeeklyPartitions#of}, which derives the weekly partition values a delete batch resolves to so the @@ -23,30 +28,29 @@ */ class WeeklyPartitionsTest { - @Test - @DisplayName("matches the partition ClickHouse computed — ordinary id") - void ordinaryId() { - // id_at 2026-08-19 (a Wednesday) -> Monday 2026-08-17 - assertThat(WeeklyPartitions.of(List.of(UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3")))) - .contains(Set.of(20260817L)); - } - - @Test - @DisplayName("far-future ids are supported, not excluded") - void farFutureId() { - // A bogus-but-self-consistent timestamp: id_at 2200-01-01 -> Monday 2199-12-30. - // 4.1% of rows on prod-test look like this; they must still be deletable and still prune. - assertThat(WeeklyPartitions.of(List.of(UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2")))) - .contains(Set.of(21991230L)); + @ParameterizedTest(name = "{0}") + @MethodSource + @DisplayName("matches the partition ClickHouse computed") + void matchesThePartitionClickHouseComputed(String era, UUID id, long expectedPartition) { + assertThat(WeeklyPartitions.of(List.of(id))).contains(Set.of(expectedPartition)); } - @Test - @DisplayName("matches the partition ClickHouse computed — id from 1996") - void oldId() { - // id_at 1996-02-09 -> Monday 1996-02-05. Long before Opik existed but well after the Unix epoch, and well - // inside Date32's 1900 floor: an id this old prunes like any other. - assertThat(WeeklyPartitions.of(List.of(UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8")))) - .contains(Set.of(19960205L)); + /** + * One row per era the derivation has to get right, each pinned to the value ClickHouse itself returned for that id + * on prod-test — not to a hand-computed Monday. The eras are not interchangeable samples: an expression that is + * correct for the ordinary calendar and wrong at the extremes is exactly the {@code toMonday} trap migration 000114 + * was written to escape, so dropping any of the three would weaken the pin rather than tidy it. + */ + private static Stream matchesThePartitionClickHouseComputed() { + return Stream.of( + // The ordinary case: id_at 2026-08-19 (a Wednesday) -> Monday 2026-08-17. + arguments("ordinary id", UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), 20260817L), + // Long before Opik existed but well after the Unix epoch, and well inside Date32's 1900 floor: an id + // this old prunes like any other. id_at 1996-02-09 -> Monday 1996-02-05. + arguments("id from 1996", UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8"), 19960205L), + // Far-future ids are supported, not excluded: a bogus-but-self-consistent timestamp, id_at 2200-01-01 -> + // Monday 2199-12-30. 4.1% of rows on prod-test look like this; they must still be deletable and prune. + arguments("far-future id", UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2"), 21991230L)); } @Test From fed35744a38ff290ba96e2f0d8d59536e9aa4024 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 09:34:47 +0200 Subject: [PATCH 12/37] [OPIK-6901] [BE] docs: "lets a trace DELETE bind itself", not "bound itself" `lets` takes the bare infinitive. Checked the sibling descriptions while here - config.yml / config-test.yml ("a trace DELETE bounds itself"), docker-compose and helm values ("lets a trace DELETE prune to") are all already correct, so this was the only occurrence. Co-Authored-By: Claude Opus 5 (1M context) --- .../data-migrations/traces-local-v2-cutover/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index c885212aeea..83f434ba578 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -323,7 +323,7 @@ the `traceColumnsNonNullable` flip"). On rollback, after swapping the Nullable original back, revert the flag to `false` **and** run that repair. **The `tracesWeeklyPartitionPruningEnabled` flip (optional, and why it goes last).** `databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled` -(env `ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITION_PRUNING_ENABLED`, default `false`) lets a trace `DELETE` bound itself to +(env `ANALYTICS_DB_DATA_MODEL_TRACES_WEEKLY_PARTITION_PRUNING_ENABLED`, default `false`) lets a trace `DELETE` bind itself to the weekly partitions its own ids resolve to (OPIK-6901), instead of being planned against every part of the table — on prod-test, 12 ids rewrote 3,928 parts / 5.40 TiB without it. It asserts a **schema** fact: that `traces` (or `traces_local`) is the successor, with `id_at` as `DateTime64(0,'UTC')` under the weekly `PARTITION BY`. From 83dc982f1ca172ffc802d2da4fb3a3e3a779d1da Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 09:38:23 +0200 Subject: [PATCH 13/37] [OPIK-6901] [BE] test: build the suite's SQL per the SQL-construction rule The rule is real and explicit - .agents/skills/opik-backend/SKILL.md, "SQL Query Construction": never build a query out of Java string operations; a query is declared once as a text block, and what varies goes through exactly one of two mechanisms - a VALUE as :placeholder + bind, a FRAGMENT as a StringTemplate + template.add. It even closes with "Some %s query templates predate this rule. Don't copy them and don't add new ones", which is precisely what this suite was doing. Every query is now a named text-block constant. The two things that cannot be bound go through TemplateUtils.newST as fragments, following the cited ClickHousePartitionMetricsDAO: * the partition-key EXPRESSION - the whole point is to evaluate the DAO's own text, so it can never be a value * the probe TABLE IDENTIFIER Each renderer adds only the attributes its own template declares, as the DAOs do. Both fragments are compile-time constants of the test class rather than test input, so neither needs the allow-list guard that DAO applies to its configured table list - noted at the declaration so the omission reads as a decision. The EXCHANGE/RENAME pair in exchangeTables() deliberately stays inline. Those are single literals built by no Java string operation, and they are kept byte-identical to 000003_exchange_and_wrap.sql by eye, so they belong at the call site beside the javadoc that says so - the same placement TracesDistributedWrapMutationTest uses. Also noted at the declaration. Verified by execution this time, not just javac: compiled the three templates against ST4 4.3.4 with a newST equivalent (fresh STGroup, no formal args) and rendered them - all three produce exactly the intended SQL, including the nested parentheses of toString(). Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 131 ++++++++++++++---- 1 file changed, 106 insertions(+), 25 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index a6ee841b148..27232e019ec 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -19,6 +19,7 @@ import com.comet.opik.infrastructure.db.TransactionTemplateAsync; import com.comet.opik.podam.PodamFactoryUtils; import com.comet.opik.utils.WeeklyPartitions; +import com.comet.opik.utils.template.TemplateUtils; import com.redis.testcontainers.RedisContainer; import io.r2dbc.spi.Statement; import org.apache.commons.lang3.RandomStringUtils; @@ -109,6 +110,72 @@ class TracesPartitionPruningMutationTest { /** Project for the raw-SQL seeded rows, kept off the API-created projects so neither test's reads see the other's. */ private static final UUID RAW_PROJECT_ID = UUID.randomUUID(); + // The suite's SQL, per .agents/skills/opik-backend/SKILL.md "SQL Query Construction": each query declared once as a + // text block, values as :placeholders, and the two things that cannot be bound - a partition-key EXPRESSION and a + // table IDENTIFIER - as StringTemplate fragments rendered through TemplateUtils.newST, following + // ClickHousePartitionMetricsDAO. Both fragments are compile-time constants of this class, never test input, so + // neither needs the allow-list guard that DAO's isValidTable applies to its configured table list. + // + // The EXCHANGE/RENAME pair in exchangeTables() stays as inline literals on purpose: they are single literals built + // by no Java string operation, and they are kept byte-identical to 000003_exchange_and_wrap.sql by eye, so they + // belong at the call site next to the javadoc that says so - as TracesDistributedWrapMutationTest does. + private static final String SELECT_FILED_PARTITION = """ + SELECT DISTINCT _partition_id + FROM traces + WHERE workspace_id = :workspace_id + AND id = :id + """; + + private static final String SELECT_PARTITION_EXPRESSION_VALUE = """ + SELECT DISTINCT toString() + FROM traces + WHERE workspace_id = :workspace_id + AND id = :id + """; + + private static final String CREATE_PARTITION_KEY_PROBE = """ + CREATE TABLE + ( + id_at DateTime64(0, 'UTC') + ) + ENGINE = MergeTree + PARTITION BY + ORDER BY tuple() + """; + + private static final String DROP_PARTITION_KEY_PROBE = """ + DROP TABLE IF EXISTS SYNC + """; + + private static final String SELECT_ID_AT_TYPE = """ + SELECT type + FROM system.columns + WHERE database = currentDatabase() + AND table = 'traces' + AND name = 'id_at' + """; + + private static final String SELECT_PARTITION_KEY = """ + SELECT partition_key + FROM system.tables + WHERE database = currentDatabase() + AND name = :table + """; + + private static final String INSERT_RAW_TRACE = """ + INSERT INTO traces (workspace_id, project_id, id) + VALUES (:workspace_id, :project_id, :id) + """; + + private static final String LAST_TRACE_DELETE = """ + SELECT query + FROM system.query_log + WHERE log_comment LIKE 'delete_traces:%' + AND type = 'QueryFinish' + ORDER BY event_time_microseconds DESC + LIMIT 1 + """; + /** A v4 UUID: no timestamp to derive a partition from. */ private static final UUID NON_V7_ID = UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22"); @@ -206,12 +273,11 @@ void predicateMatchesLivePartitioningAndJavaDerivation(String era, UUID id, long // — so this seeds through the real column definition rather than restating it. insertRawTrace(id); - var filedUnder = queryOneString("SELECT DISTINCT _partition_id FROM traces" - + " WHERE workspace_id = :workspace_id AND id = :id", bindRawTrace(id)); - // PARTITION_PREDICATE is interpolated because it is an expression, not a value - the point is to evaluate the - // DAO's own text. The ids and workspace go in as binds like everywhere else in this suite. - var daoPredicateValue = queryOneString("SELECT DISTINCT toString(" + PARTITION_PREDICATE + ") FROM traces" - + " WHERE workspace_id = :workspace_id AND id = :id", bindRawTrace(id)); + var filedUnder = queryOneString(SELECT_FILED_PARTITION, bindRawTrace(id)); + // The DAO predicate goes in as a StringTemplate fragment, not a bind: it is an expression to be evaluated, and + // evaluating the DAO's own text is the entire point. The workspace and id are values, so they bind. + var daoPredicateValue = queryOneString(withPartitionExpression(SELECT_PARTITION_EXPRESSION_VALUE), + bindRawTrace(id)); assertThat(daoPredicateValue) .as("the DAO predicate resolves to the partition ClickHouse filed the %s row under", era) @@ -247,15 +313,14 @@ void daoPredicateIsTheSameExpressionAsTheLivePartitionKey() { // two re-prints. That makes the comparison AST-level and formatter-independent by construction: both strings // come out of the same printer, so they are equal iff the parsed expressions are. Diffing the DAO text against // system.tables directly would instead pin ClickHouse's whitespace choices, which is what it must not do. - execute("CREATE TABLE " + PARTITION_KEY_PROBE + " (id_at DateTime64(0, 'UTC')) ENGINE = MergeTree" - + " PARTITION BY " + PARTITION_PREDICATE + " ORDER BY tuple()", _ -> { - }); + execute(createProbeTableSql(), _ -> { + }); try { assertThat(partitionKeyOf(PARTITION_KEY_PROBE)) .as("the DAO predicate parses to the same expression traces is partitioned by") .isEqualTo(partitionKeyOf("traces")); } finally { - execute("DROP TABLE IF EXISTS " + PARTITION_KEY_PROBE + " SYNC", _ -> { + execute(withProbeTable(DROP_PARTITION_KEY_PROBE), _ -> { }); } } @@ -265,9 +330,7 @@ void daoPredicateIsTheSameExpressionAsTheLivePartitionKey() { void idAtIsTheSixtyFourBitColumn() { // The second half of what the flag asserts, and not implied by the partition agreement above: a 32-bit // DateTime id_at would agree with itself while silently wrapping every id past 2106. - assertThat(queryOneString("SELECT type FROM system.columns WHERE database = currentDatabase()" - + " AND table = 'traces' AND name = 'id_at'")) - .isEqualTo("DateTime64(0, 'UTC')"); + assertThat(queryOneString(SELECT_ID_AT_TYPE)).isEqualTo("DateTime64(0, 'UTC')"); } @Test @@ -367,14 +430,7 @@ private void delete(Set> projectIdTraceIdPairs) { private String lastTraceDeleteSql() { execute("SYSTEM FLUSH LOGS", _ -> { }); - return queryOneString(""" - SELECT query - FROM system.query_log - WHERE log_comment LIKE 'delete_traces:%' - AND type = 'QueryFinish' - ORDER BY event_time_microseconds DESC - LIMIT 1 - """); + return queryOneString(LAST_TRACE_DELETE); } /** @@ -429,18 +485,43 @@ private List traceIdsOf(String projectName) { * backdated or far-future {@code id} by design ({@code IdGenerator.validateId}). */ private void insertRawTrace(UUID id) { - execute("INSERT INTO traces (workspace_id, project_id, id) VALUES (:workspace_id, :project_id, :id)", + execute(INSERT_RAW_TRACE, statement -> statement .bind("workspace_id", WORKSPACE_ID) .bind("project_id", RAW_PROJECT_ID.toString()) .bind("id", id.toString())); } + /** + * Renders a template whose only fragment is the DAO's partition-key expression — an expression, not a value, so it + * cannot be bound. Each renderer adds exactly the attributes its template declares, as the DAOs do. + */ + private static String withPartitionExpression(String sql) { + return TemplateUtils.newST(sql) + .add("partition_expression", PARTITION_PREDICATE) + .render(); + } + + /** + * As {@link #withPartitionExpression}, for the probe-table statements: a table identifier cannot be bound either. + */ + private static String withProbeTable(String sql) { + return TemplateUtils.newST(sql) + .add("probe_table", PARTITION_KEY_PROBE) + .render(); + } + + /** The probe-table DDL carries both fragments. */ + private static String createProbeTableSql() { + return TemplateUtils.newST(CREATE_PARTITION_KEY_PROBE) + .add("probe_table", PARTITION_KEY_PROBE) + .add("partition_expression", PARTITION_PREDICATE) + .render(); + } + /** ClickHouse's own re-print of a table's partition key expression. */ private String partitionKeyOf(String table) { - return queryOneString("SELECT partition_key FROM system.tables" - + " WHERE database = currentDatabase() AND name = :table", - statement -> statement.bind("table", table)); + return queryOneString(SELECT_PARTITION_KEY, statement -> statement.bind("table", table)); } private static Consumer bindRawTrace(UUID id) { From 845a3b3a64749603cc37e4ad873f8c4b43eed607 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 10:02:51 +0200 Subject: [PATCH 14/37] [OPIK-6901] [BE] style: apply spotless to PARTITION_PREDICATE CI's spotless leg joins this declaration back onto one line, at 123 chars. I had wrapped it by hand in round 1 precisely because I could not run spotless, reasoning from lineSplit=120 that the wrap was required - the formatter's actual answer is the opposite: the RHS is a single string literal with no legal wrap point, so it goes on one line and simply exceeds the limit. Applied verbatim from the diff the failed leg printed rather than guessing a second time. Audited the rest of the changed files for the same hand-wrapped-literal pattern; there is none. Co-Authored-By: Claude Opus 5 (1M context) --- .../infrastructure/TracesPartitionPruningMutationTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 27232e019ec..b93d7878f9f 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -98,8 +98,7 @@ class TracesPartitionPruningMutationTest { * {@code system.query_log}, which is safe because that is the query text as submitted — the DAO's own template * string, not a re-print. */ - private static final String PARTITION_PREDICATE = - "toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))"; + private static final String PARTITION_PREDICATE = "toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))"; /** * Throwaway table used only to have ClickHouse re-print the DAO predicate as a partition key. From 426260ffca01866f6a4b9a9b341508b0efd0617a Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 10:03:15 +0200 Subject: [PATCH 15/37] [OPIK-6901] [BE] test: name the constant after the column it selects Renamed SELECT_FILED_PARTITION to SELECT_PARTITION_ID, which is the column (_partition_id) the query actually selects. Not because FILED was a typo for FIELD - it was the past tense of "to file", matching the prose it sits beside ("the partition ClickHouse filed the row under", "_partition_id, where it filed the row"). But a name that a careful reader takes for a misspelling of FIELD is a bad name regardless of whether it is one, and naming the column removes the ambiguity instead of defending the pun. The local variable stays `filedUnder`, where the verb reads unambiguously in context. Co-Authored-By: Claude Opus 5 (1M context) --- .../infrastructure/TracesPartitionPruningMutationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index b93d7878f9f..164b3f36a68 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -118,7 +118,7 @@ class TracesPartitionPruningMutationTest { // The EXCHANGE/RENAME pair in exchangeTables() stays as inline literals on purpose: they are single literals built // by no Java string operation, and they are kept byte-identical to 000003_exchange_and_wrap.sql by eye, so they // belong at the call site next to the javadoc that says so - as TracesDistributedWrapMutationTest does. - private static final String SELECT_FILED_PARTITION = """ + private static final String SELECT_PARTITION_ID = """ SELECT DISTINCT _partition_id FROM traces WHERE workspace_id = :workspace_id @@ -272,7 +272,7 @@ void predicateMatchesLivePartitioningAndJavaDerivation(String era, UUID id, long // — so this seeds through the real column definition rather than restating it. insertRawTrace(id); - var filedUnder = queryOneString(SELECT_FILED_PARTITION, bindRawTrace(id)); + var filedUnder = queryOneString(SELECT_PARTITION_ID, bindRawTrace(id)); // The DAO predicate goes in as a StringTemplate fragment, not a bind: it is an expression to be evaluated, and // evaluating the DAO's own text is the entire point. The workspace and id are values, so they bind. var daoPredicateValue = queryOneString(withPartitionExpression(SELECT_PARTITION_EXPRESSION_VALUE), From f86f88013f9523d7910fe47cd4e0fad76af51452 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 10:05:03 +0200 Subject: [PATCH 16/37] [OPIK-6901] [BE] test: assert the bound partitions are the exact set, not a superset `.contains(partition)` on the emitted SQL is satisfied by a binding that also names every week in between, and that is the failure worth catching: an over-broad :partitions set keeps every delete correct while handing back the entire benefit. On prod-test a range across this batch's span selected 2,644 of 3,928 parts where the exact set selected 4 - so "right but slow" is the regression, and no behavioural assertion in this suite can see it. The exact-set contract was stated in the DAO javadoc and asserted nowhere. Both pruning tests now decode the values actually bound into the emitted IN clause and assert set equality - containsExactly for the single-partition batch, containsExactlyInAnyOrder for the two-week one. The clause is captured to end of line rather than by matching a bracket style: the predicate sits on its own line in the template with SETTINGS log_comment on the next, so the line boundary delimits it exactly, and the driver's punctuation of a Long[] is its own business. Verified the decoder standalone against realistic SQL - it reads both [a,b] and (a, b), returns 4 values for an over-broad binding (so the assertion bites), does not pull an 8-digit token out of the log_comment on the following line, and yields an empty set for an un-inlined ':partitions' rather than a vacuous pass. That last case is why the helper asserts non-empty: if the driver ever stops substituting client-side, this says so instead of turning every set assertion into a tautology. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 81 +++++++++++++++---- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 164b3f36a68..a431446be39 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -47,7 +47,9 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.regex.Pattern; import java.util.function.Consumer; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.comet.opik.api.resources.utils.AuthTestUtils.mockTargetWorkspace; @@ -175,6 +177,24 @@ INSERT INTO traces (workspace_id, project_id, id) LIMIT 1 """; + /** + * The {@code IN} clause the DAO emitted, captured to end of line: the predicate sits on its own line in the + * template with {@code SETTINGS log_comment} on the next, so the line boundary delimits it exactly. Read this way + * rather than by matching a bracket style, because the driver's rendering of a {@code Long[]} is its own choice — + * what matters is which partition values are in the clause, not how it punctuates them. + */ + private static final Pattern EMITTED_IN_CLAUSE = Pattern.compile( + Pattern.quote(PARTITION_PREDICATE) + "\\s+IN\\s+([^\\n]*)"); + + /** A weekly partition value as it appears in SQL — {@code yyyyMMdd}, so always eight digits. */ + private static final Pattern PARTITION_VALUE = Pattern.compile("\\d{8}"); + + /** An id in a different week from anything the API mints, for the two-partition batch. id_at 2023-11-29 (a Wed). */ + private static final UUID OTHER_WEEK_ID = UUID.fromString("018c1860-1800-7abc-8000-000000000001"); + + /** The Monday of {@link #OTHER_WEEK_ID}'s week — stated as a literal so the expectation is readable on its own. */ + private static final long OTHER_WEEK_PARTITION = 20231127L; + /** A v4 UUID: no timestamp to derive a partition from. */ private static final UUID NON_V7_ID = UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22"); @@ -349,10 +369,11 @@ void allUuidV7DeletePrunesAndRemovesTheTargetRow() { .as("only the target row is gone") .doesNotContain(target.id()) .contains(bystander.id()); - assertThat(lastTraceDeleteSql()) - .as("the mutation bounded itself to the target's own partition") - .contains(PARTITION_PREDICATE) - .contains(onlyPartitionOf(target.id())); + var sql = lastTraceDeleteSql(); + assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); + assertThat(boundPartitionsOf(sql)) + .as("bounded to exactly the target's own partition, nothing wider") + .containsExactly(partitionOf(target.id())); } @Test @@ -360,20 +381,25 @@ void allUuidV7DeletePrunesAndRemovesTheTargetRow() { void batchSpanningTwoWeeksBindsBothPartitions() { // The multi-value Long[] bind, which the single-id path never exercises. The second id is minted for a week // three years back and matches no row — the batch's partition SET is what is under test, and a delete does not - // need its ids to exist. A range over the span would have selected every partition in between; the set names - // exactly two. + // need its ids to exist. + // + // Asserted as an EXACT set, because "mentions both partitions" is satisfied by a binding that also names every + // week in between, and that is the failure worth catching: an over-broad set keeps every delete correct while + // giving back the entire benefit. On prod-test a range over this span selected 2,644 of 3,928 parts where the + // exact set selected 4 — so a delete that is right and slow is the regression, and no behavioural assertion + // can see it. var target = newTrace().build(); traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); var projectId = projectIdOf(target); - var otherWeekId = UUID.fromString("018c1860-1800-7abc-8000-000000000001"); // id_at 2023-11-29 -> 20231127 - delete(Set.of(Pair.of(projectId, target.id()), Pair.of(projectId, otherWeekId))); + delete(Set.of(Pair.of(projectId, target.id()), Pair.of(projectId, OTHER_WEEK_ID))); assertThat(traceIdsOf(target.projectName())).doesNotContain(target.id()); - assertThat(lastTraceDeleteSql()) - .contains(PARTITION_PREDICATE) - .contains(onlyPartitionOf(target.id())) - .contains("20231127"); + var sql = lastTraceDeleteSql(); + assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); + assertThat(boundPartitionsOf(sql)) + .as("exactly the batch's own two partitions, not a range across them") + .containsExactlyInAnyOrder(partitionOf(target.id()), OTHER_WEEK_PARTITION); } @ParameterizedTest @@ -404,12 +430,33 @@ private static Stream underivableIdDisablesPruning() { } /** - * The partition the id resolves to, as it appears in the SQL. Derived through {@code WeeklyPartitions} on purpose: - * this suite is about the predicate reaching ClickHouse, and the derivation's own expected values are pinned against - * real ClickHouse output in {@code WeeklyPartitionsTest}, so restating them here would only duplicate that. + * The partition a single id resolves to. Derived through {@code WeeklyPartitions} on purpose: this suite is about + * the predicate reaching ClickHouse, and the derivation's own expected values are pinned against real ClickHouse + * output in {@code WeeklyPartitionsTest} and again in + * {@link #predicateMatchesLivePartitioningAndJavaDerivation}, so restating them here would only duplicate that. + */ + private static long partitionOf(UUID id) { + return WeeklyPartitions.of(List.of(id)).orElseThrow().iterator().next(); + } + + /** + * The partition values actually bound into the emitted {@code IN} clause, so a test can assert the set is exact + * rather than merely inclusive. The driver substitutes bound values into the query text client-side, which is why + * they are readable here at all; the two assertions below are what make a change in that behaviour say so plainly + * instead of quietly turning every set assertion into a tautology on an empty set. */ - private static String onlyPartitionOf(UUID id) { - return String.valueOf(WeeklyPartitions.of(List.of(id)).orElseThrow().iterator().next()); + private static Set boundPartitionsOf(String sql) { + var clause = EMITTED_IN_CLAUSE.matcher(sql); + assertThat(clause.find()) + .as("the delete SQL carries the partition predicate followed by an IN clause:%n%s", sql) + .isTrue(); + var bound = PARTITION_VALUE.matcher(clause.group(1)).results() + .map(match -> Long.parseLong(match.group())) + .collect(Collectors.toUnmodifiableSet()); + assertThat(bound) + .as("the IN clause carries inlined partition values — got '%s'", clause.group(1)) + .isNotEmpty(); + return bound; } /** Invokes the DAO under a workspace/user context, as {@code TraceService} does for the live delete path. */ From a94df7cb9b16209d99b834b4b898733249abc867 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 10:08:21 +0200 Subject: [PATCH 17/37] [OPIK-6901] [BE] style: order the new java.util imports as spotless does java.util.function.Consumer sorts before java.util.regex.Pattern; I had inserted Pattern directly after java.util.UUID. Second and last hunk from the same failing leg - the PARTITION_PREDICATE fix in 845a3b3 held, this was the only remaining violation. Co-Authored-By: Claude Opus 5 (1M context) --- .../opik/infrastructure/TracesPartitionPruningMutationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index a431446be39..60f1dd42c75 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -47,8 +47,8 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.regex.Pattern; import java.util.function.Consumer; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; From ac6d24e13a056ad906ccc27ade626548ebb088b9 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 11:22:45 +0200 Subject: [PATCH 18/37] [OPIK-6901] [BE] test: close three holes in the pruning suite's own assertions 1. The underivable-id fallback asserted the wrong half. The guarantee the original javadoc states is that a row whose id_at cannot be trusted is STILL DELETED - a claim about the underivable row itself - but the test passed that id as a pair matching no row, so it only checked its batch-mate. An implementation that quietly dropped underivable ids from the batch passed. Now the underivable id gets a real seeded row (raw SQL, since ingestion rejects both id shapes by design) and the test asserts it is gone afterwards. 2. The fallback's SQL check rejected only "toDayOfWeek", so any other narrowing predicate slipped through. Demonstrated: with the old assertion, both `AND toMonday(id_at) IN [...]` and `AND id_at >= ... AND id_at < ...` pass while skipping exactly the rows the fallback exists to reach. Now asserts the absence of ANY id_at predicate plus the absence of a partition IN clause - the unbounded template mentions id_at nowhere, so that is the complete check, and both regressions above fail it. 3. The query_log lookup was not deterministic. Every test in the class deletes under the same workspace, so `log_comment LIKE 'delete_traces:%'` ordered by event_time_microseconds could hand back a neighbouring test's delete - if this one had not been flushed yet, or on a same-microsecond tie - and pass for the wrong reason, since a neighbour's statement has the same shape. Now also filtered by the test's own trace id, which is freshly minted per test; ORDER BY/LIMIT only picks the newest attempt under surefire retries, and the helper asserts a row was found rather than returning null into a later assertion. Filtered on the id already present in the statement text rather than by injecting a marker into the DAO's `details`: the DAO owns log_comment, and this needs no production change to serve a test. Verified the two new SQL assertions and the regression cases they must catch by executing them standalone against realistic rendered statements. Not verified: the seeded-row and query_log changes against ClickHouse - that needs CI. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 82 +++++++++++++++---- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 60f1dd42c75..e2b3d08774b 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -173,10 +173,18 @@ INSERT INTO traces (workspace_id, project_id, id) FROM system.query_log WHERE log_comment LIKE 'delete_traces:%' AND type = 'QueryFinish' + AND query LIKE concat('%', :trace_id, '%') ORDER BY event_time_microseconds DESC LIMIT 1 """; + private static final String LIVE_ROW_COUNT = """ + SELECT toString(uniqExact(id)) + FROM traces + WHERE workspace_id = :workspace_id + AND id = :id + """; + /** * The {@code IN} clause the DAO emitted, captured to end of line: the predicate sits on its own line in the * template with {@code SETTINGS log_comment} on the next, so the line boundary delimits it exactly. Read this way @@ -369,7 +377,7 @@ void allUuidV7DeletePrunesAndRemovesTheTargetRow() { .as("only the target row is gone") .doesNotContain(target.id()) .contains(bystander.id()); - var sql = lastTraceDeleteSql(); + var sql = lastTraceDeleteSql(target.id()); assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); assertThat(boundPartitionsOf(sql)) .as("bounded to exactly the target's own partition, nothing wider") @@ -395,7 +403,7 @@ void batchSpanningTwoWeeksBindsBothPartitions() { delete(Set.of(Pair.of(projectId, target.id()), Pair.of(projectId, OTHER_WEEK_ID))); assertThat(traceIdsOf(target.projectName())).doesNotContain(target.id()); - var sql = lastTraceDeleteSql(); + var sql = lastTraceDeleteSql(target.id()); assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); assertThat(boundPartitionsOf(sql)) .as("exactly the batch's own two partitions, not a range across them") @@ -406,21 +414,37 @@ void batchSpanningTwoWeeksBindsBothPartitions() { @MethodSource @DisplayName("an id with no derivable partition disables pruning for the batch, and the delete still lands") void underivableIdDisablesPruning(String cause, UUID underivableId) { - // The fallback that preserves the pre-OPIK-6901 guarantee: one id whose partition cannot be derived exactly and - // the statement goes back to its unbounded form — no predicate at all, never a partial set. The v7 row batched - // alongside it must still be deleted, which is the "not silently skipped" half. + // The fallback that preserves the pre-OPIK-6901 guarantee, as the original javadoc stated it: a row whose id_at + // cannot be trusted is STILL DELETED. That is a claim about the underivable row ITSELF, so it gets a real row + // here - seeded raw, since ingestion rejects both id shapes by design. Passing it as an id matching nothing + // would let an implementation that quietly drops underivable ids from the batch pass, which is the very bug the + // all-or-nothing rule exists to prevent. var target = newTrace().build(); traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); var projectId = projectIdOf(target); + insertRawTrace(projectId, underivableId); + assertThat(liveRowCount(underivableId)).as("the %s row is seeded before the delete", cause).isEqualTo("1"); delete(Set.of(Pair.of(projectId, target.id()), Pair.of(projectId, underivableId))); + assertThat(liveRowCount(underivableId)) + .as("the %s row is itself deleted, not skipped", cause) + .isEqualTo("0"); assertThat(traceIdsOf(target.projectName())) - .as("the deletable row in a %s batch is still deleted", cause) + .as("and the derivable row batched alongside the %s id goes too", cause) .doesNotContain(target.id()); - assertThat(lastTraceDeleteSql()) - .as("no partition predicate is emitted for a %s batch", cause) - .doesNotContain("toDayOfWeek"); + + // Asserted as the absence of ANY id_at predicate, not just of this PR's expression. A regression that narrowed + // the mutation with toMonday(id_at), an id_at range, or any other partition predicate would skip exactly the + // rows this fallback exists to reach, and rejecting one function name would not see it. The unbounded template + // mentions id_at nowhere at all, so that is the whole check. + var sql = lastTraceDeleteSql(target.id()); + assertThat(sql) + .as("the unbounded form for a %s batch carries no id_at predicate of any kind", cause) + .doesNotContain("id_at"); + assertThat(EMITTED_IN_CLAUSE.matcher(sql).find()) + .as("and no partition IN clause: %s", sql) + .isFalse(); } private static Stream underivableIdDisablesPruning() { @@ -469,14 +493,37 @@ private void delete(Set> projectIdTraceIdPairs) { } /** - * The SQL of the most recent trace delete, as ClickHouse received it. {@code log_comment} is what makes this - * unambiguous: {@code TraceDAO} stamps every statement with {@code :::

}, and - * {@code delete_traces} names this one template alone. + * The SQL of the trace delete that carried {@code traceId}, as ClickHouse received it. Two filters, because either + * alone is ambiguous: {@code log_comment} narrows to this one template ({@code TraceDAO} stamps every statement + * {@code :::
}, and {@code delete_traces} names it alone), and the id narrows + * to this test's statement. + *

+ * The id filter is what makes the lookup deterministic. Every test in this class deletes under the same workspace, + * so ordering by {@code event_time_microseconds} alone would hand back a neighbouring test's delete whenever this + * one had not reached {@code query_log} yet, or whenever two landed in the same microsecond — and that flake could + * pass for the wrong reason, since a neighbour's statement has the same shape. Each test's target id is freshly + * minted, so it identifies the statement exactly; the {@code ORDER BY}/{@code LIMIT} now only picks the newest + * attempt when surefire retries a test. + *

+ * Filtering on the id rather than on a marker injected into {@code details}: the DAO owns {@code log_comment} and + * puts {@code pairs_size} there, and the id is already in the statement text, so this needs no production change to + * serve a test. */ - private String lastTraceDeleteSql() { + private String lastTraceDeleteSql(UUID traceId) { execute("SYSTEM FLUSH LOGS", _ -> { }); - return queryOneString(LAST_TRACE_DELETE); + var sql = queryOneString(LAST_TRACE_DELETE, statement -> statement.bind("trace_id", traceId.toString())); + assertThat(sql) + .as("query_log holds a delete_traces statement mentioning id '%s'", traceId) + .isNotNull(); + return sql; + } + + /** {@code "1"} while a live (non-lightweight-deleted) row exists for the id, {@code "0"} once it is gone. */ + private String liveRowCount(UUID id) { + return queryOneString(LIVE_ROW_COUNT, statement -> statement + .bind("workspace_id", WORKSPACE_ID) + .bind("id", id.toString())); } /** @@ -531,10 +578,15 @@ private List traceIdsOf(String projectName) { * backdated or far-future {@code id} by design ({@code IdGenerator.validateId}). */ private void insertRawTrace(UUID id) { + insertRawTrace(RAW_PROJECT_ID, id); + } + + /** As {@link #insertRawTrace(UUID)}, into a caller-chosen project, so a seeded row can share a test's project. */ + private void insertRawTrace(UUID projectId, UUID id) { execute(INSERT_RAW_TRACE, statement -> statement .bind("workspace_id", WORKSPACE_ID) - .bind("project_id", RAW_PROJECT_ID.toString()) + .bind("project_id", projectId.toString()) .bind("id", id.toString())); } From 686cbfd52fb17c95682affaf3a8ee2f095e6d847 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 11:23:48 +0200 Subject: [PATCH 19/37] [OPIK-6901] [BE] docs: record which delete topology this suite covers, and which it does not Reviewer asked why the suite does not also apply the Distributed wrap. It runs the post-EXCHANGE, pre-wrap state deliberately - that is the state the pruning flag must hold in on its own, since the wrap is a separate deferrable step and prod-test sat in exactly that window - and applying the wrap would remove that coverage rather than add to it, because partition_key is meaningless once traces is Distributed. Writing it down because the reasoning was implicit and a reader had to reconstruct it: what is covered, that the traces_local branch of this same template is covered by TracesDistributedWrapMutationTest, and that the one untested cell is both flags on at once - which needs a third topology and so its own suite, and cannot live in the wrap suite because that one wraps the legacy table where this flag must be false. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index e2b3d08774b..f743a0821a5 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -83,6 +83,18 @@ * ingestion path rejects a non-v7 or far-future {@code id} by design ({@code IdGenerator.validateId}), so the batches * that must not prune are handed to {@link TraceDAO#delete} directly — the only way to reach that arm. * + *

Topology covered, and the one cell that is not. This suite runs the post-EXCHANGE, pre-wrap state on + * purpose: {@code traces} is the partitioned successor and still a {@code MergeTree}, which is the state the pruning + * flag has to hold in on its own — the wrap is a separate, deferrable cutover step ({@code --skip-wrap} now, + * {@code --wrap-only} later), and prod-test sat in exactly this window. Applying the wrap here would remove that + * coverage rather than add to it, since {@code partition_key} is meaningless once {@code traces} is {@code Distributed}. + * The {@code traces_local} branch of this same template is executed by {@code TracesDistributedWrapMutationTest}, with + * pruning off. So the untested cell is both flags on at once, and it stays untested here: the two are independent + * StringTemplate attributes with no shared state, and the wrap is a {@code RENAME} — {@code traces_local} is the very + * table this suite partitions and asserts against, so {@code id_at} and the partition key belong to the data, not to + * the name it is reached by. Covering it needs a third topology (EXCHANGE + wrap + both flags) and therefore its own + * suite; it cannot live in the wrap suite, which wraps the legacy table where this flag must be false. + * *

Dedicated, non-reused ClickHouse and ZooKeeper containers are required because the EXCHANGE destructively swaps the * live {@code traces} table; a reused container would corrupt other suites and reruns. */ From 61c699d63fc2c5934b01e4f9d4afc0f6a014f4d7 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 12:10:59 +0200 Subject: [PATCH 20/37] [OPIK-6901] [BE] test: prove the pruning through the DAO's delete, not through test SQL Net -162 lines. The suite had grown five queries of its own to validate the partition expression: a SELECT that re-evaluated the DAO's predicate, a throwaway table created purely so ClickHouse would re-print that predicate as a partition key, and two system-table lookups. That is the test re-implementing what it is supposed to be checking, and it is the wrong instrument: what matters is the statement production runs. Replaced all of it with one behavioural test. Seed a row per era (recent, 1996, far-future 2200), delete them in one DAO batch, assert the rows are gone. If the predicate resolved to any partition other than the one ClickHouse filed a row under, the mutation would select the wrong parts and that row would SURVIVE - so "every row is gone" IS the three-way agreement between the migration's PARTITION BY as installed, the DAO's predicate and WeeklyPartitions.of, established by the real delete instead of by SQL written here. It subsumes what it replaces: * cross-era correctness, including the toMonday trap 000114 warns about - that expression diverges only far-future, so the 2200 row catches it * the DateTime64 half of the flag's assertion, with no system.columns lookup: against a 32-bit id_at the 2200 row is stored under a wrapped timestamp, the derived partition misses it, and it survives * the multi-value Long[] bind, and the exact-set assertion on it Dropped with it: daoPredicateIsTheSameExpressionAsTheLivePartitionKey and idAtIsTheSixtyFourBitColumn, the probe table and both renderers. One guarantee goes with them and is worth naming: AST identity between the predicate and the partition key, which is what proved the planner PRUNES rather than merely computing the right answer. The exact-predicate text is still checked against the emitted statement, and an over-broad bound set still fails, so a silent loss of pruning is still caught - but a rewrite that is semantically identical and textually different would now pass. The suite's SQL is down to three plain text blocks with only :placeholders - seed a row, count a row, read a statement from query_log. No StringTemplate fragments, no interpolation, so TemplateUtils is no longer needed here. Also renamed exchangeTables() to installPartitionedSuccessorUnderTraces() and documented why it exists: it is setup and nothing asserts on it. The DAO names its target table, and after the migrations the live `traces` is the legacy unpartitioned one while the successor is only the unreachable traces_local_v2 - so without those two statements these tests would run against the one table where this predicate must never be emitted, and would pass while proving nothing. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 342 +++++------------- 1 file changed, 90 insertions(+), 252 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index f743a0821a5..cea08fe4f24 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -19,7 +19,6 @@ import com.comet.opik.infrastructure.db.TransactionTemplateAsync; import com.comet.opik.podam.PodamFactoryUtils; import com.comet.opik.utils.WeeklyPartitions; -import com.comet.opik.utils.template.TemplateUtils; import com.redis.testcontainers.RedisContainer; import io.r2dbc.spi.Statement; import org.apache.commons.lang3.RandomStringUtils; @@ -63,25 +62,40 @@ * renders the predicate exactly when it should, that the derived {@code Long[]} binds to {@code IN :partitions}, and * that the row still goes away either way. * - *

Each test asserts both halves, because either alone passes for the wrong reason: the rows are read back - * through the public API (a delete that pruned to a partition the row is not in would leave it behind), and the SQL - * ClickHouse actually received is read back from {@code system.query_log} (a delete that silently stopped pruning would - * still remove the row, just slowly — the regression the flag and the derivation exist to prevent, and one no - * behavioural assertion can see). + *

Everything is asserted through the DAO's own delete. This suite writes no query that re-implements or + * re-evaluates the partition expression, and it asserts nothing about the EXCHANGE — the EXCHANGE is only setup (see + * below). Its own SQL is three statements, all plain binds: seed a row, count a row, read a statement back from + * {@code system.query_log}. Correctness is established the way production would feel it — the rows go away. If + * the DAO's predicate resolved to any partition other than the one ClickHouse filed a row under, the mutation would + * select the wrong parts and that row would survive, so + * {@link #deleteClearsEveryEraAndBindsExactlyThosePartitions} passing is the agreement between the migration's + * {@code PARTITION BY} as installed, the DAO's predicate, and {@link WeeklyPartitions#of}. * - *

{@link #predicateMatchesLivePartitioningAndJavaDerivation} and {@link #idAtIsTheSixtyFourBitColumn} are the guards - * that keep the rest honest, together pinning both facts - * {@code databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled} asserts. They are load-bearing twice over. The - * predicate is harmless against an unpartitioned table for recent ids, so had the EXCHANGE below not taken effect every - * other test here would still pass while proving nothing. And the rule itself is expressed three times over — the - * migration's {@code PARTITION BY}, the DAO's predicate, and {@link WeeklyPartitions#of} — so the first guard makes all - * three compute the same value for the same row, across the eras where a plausible wrong expression - * ({@code toMonday}) would diverge. + *

Each test then pairs that with the SQL ClickHouse actually received, because rows alone cannot see pruning + * silently stop — a delete that stopped bounding itself is still correct, just slow, and that is the regression this + * change exists to prevent. Read back by {@code log_comment} plus the test's own trace id, and checked as an + * exact bound partition set: a superset would keep every delete correct while handing back the whole benefit. + * + *

The eras in {@link #deleteClearsEveryEraAndBindsExactlyThosePartitions} are load-bearing, not variety. + * {@code toMonday} agrees with the {@code Date32} expression across the ordinary calendar and diverges only far-future + * or at the epoch, so a recent-only batch would accept the very expression migration 000114 was written to escape. The + * 2200 row also covers the {@code DateTime64} half of what the flag asserts: against a 32-bit {@code id_at} it would be + * stored under a wrapped recent timestamp, the derived partition would not match, and it would survive. * *

Two internal touches, on the pattern of {@code TracesDistributedWrapMutationTest}: the EXCHANGE has no public API, * so {@link #beforeAll} runs it in raw SQL identical to the swap block of {@code 000003_exchange_and_wrap.sql}; and the - * ingestion path rejects a non-v7 or far-future {@code id} by design ({@code IdGenerator.validateId}), so the batches - * that must not prune are handed to {@link TraceDAO#delete} directly — the only way to reach that arm. + * ingestion path rejects a non-v7, backdated or far-future {@code id} by design ({@code IdGenerator.validateId}), so + * those rows are seeded raw and those batches are handed to {@link TraceDAO#delete} directly — the only way to reach + * that arm. + * + *

Why the EXCHANGE is here at all — it is setup, never the subject. Nothing asserts anything about it. It is + * required because the DAO names its target table: after the Liquibase migrations the live {@code traces} is still the + * legacy table — no {@code PARTITION BY} at all and a 32-bit {@code DateTime} {@code id_at} — while the + * partitioned successor exists only as the empty {@code traces_local_v2}, which no DAO query can reach. Two statements + * copied from the cutover put the successor under the name the DAO deletes from. Without them these tests would run + * against the one table where this predicate must never be emitted, and would pass while proving nothing: the predicate + * is harmless against an unpartitioned table for recent ids. Hand-authoring a partitioned {@code traces} in the test + * instead would duplicate migration 000114 and reintroduce exactly the drift this suite exists to detect. * *

Topology covered, and the one cell that is not. This suite runs the post-EXCHANGE, pre-wrap state on * purpose: {@code traces} is the partitioned successor and still a {@code MergeTree}, which is the state the pruning @@ -108,73 +122,21 @@ class TracesPartitionPruningMutationTest { private static final String USER = "user-" + RandomStringUtils.secure().nextAlphanumeric(32); /** - * The partition-key fragment the template emits. Compared verbatim against the SQL read back from - * {@code system.query_log}, which is safe because that is the query text as submitted — the DAO's own template - * string, not a re-print. + * The partition-key fragment the DAO's template emits. Only ever compared against the statement read back + * from {@code system.query_log} — never spliced into a query this suite runs. Verbatim comparison is safe because + * {@code query_log} stores the query as submitted, so both sides are the DAO's own template text. */ private static final String PARTITION_PREDICATE = "toYYYYMMDD(toDate32(id_at) - toIntervalDay(toDayOfWeek(id_at, 1)))"; - /** - * Throwaway table used only to have ClickHouse re-print the DAO predicate as a partition key. - * Created and dropped in-test. - */ - private static final String PARTITION_KEY_PROBE = "traces_partition_key_probe"; - - /** Project for the raw-SQL seeded rows, kept off the API-created projects so neither test's reads see the other's. */ - private static final UUID RAW_PROJECT_ID = UUID.randomUUID(); - - // The suite's SQL, per .agents/skills/opik-backend/SKILL.md "SQL Query Construction": each query declared once as a - // text block, values as :placeholders, and the two things that cannot be bound - a partition-key EXPRESSION and a - // table IDENTIFIER - as StringTemplate fragments rendered through TemplateUtils.newST, following - // ClickHousePartitionMetricsDAO. Both fragments are compile-time constants of this class, never test input, so - // neither needs the allow-list guard that DAO's isValidTable applies to its configured table list. + // The suite's whole SQL surface, per .agents/skills/opik-backend/SKILL.md "SQL Query Construction": one text block + // per query, every varying value a :placeholder. There are no StringTemplate fragments and no interpolation at all, + // because nothing here re-implements the DAO's predicate - PARTITION_PREDICATE is only ever compared against the + // statement the DAO emitted, never spliced into a query of ours. // - // The EXCHANGE/RENAME pair in exchangeTables() stays as inline literals on purpose: they are single literals built - // by no Java string operation, and they are kept byte-identical to 000003_exchange_and_wrap.sql by eye, so they - // belong at the call site next to the javadoc that says so - as TracesDistributedWrapMutationTest does. - private static final String SELECT_PARTITION_ID = """ - SELECT DISTINCT _partition_id - FROM traces - WHERE workspace_id = :workspace_id - AND id = :id - """; - - private static final String SELECT_PARTITION_EXPRESSION_VALUE = """ - SELECT DISTINCT toString() - FROM traces - WHERE workspace_id = :workspace_id - AND id = :id - """; - - private static final String CREATE_PARTITION_KEY_PROBE = """ - CREATE TABLE - ( - id_at DateTime64(0, 'UTC') - ) - ENGINE = MergeTree - PARTITION BY - ORDER BY tuple() - """; - - private static final String DROP_PARTITION_KEY_PROBE = """ - DROP TABLE IF EXISTS SYNC - """; - - private static final String SELECT_ID_AT_TYPE = """ - SELECT type - FROM system.columns - WHERE database = currentDatabase() - AND table = 'traces' - AND name = 'id_at' - """; - - private static final String SELECT_PARTITION_KEY = """ - SELECT partition_key - FROM system.tables - WHERE database = currentDatabase() - AND name = :table - """; - + // The EXCHANGE/RENAME pair in installPartitionedSuccessorUnderTraces() stays as inline literals on purpose: they + // are single literals built by no Java string operation, and they are kept byte-identical to + // 000003_exchange_and_wrap.sql by eye, so they belong at the call site next to the javadoc that says so - as + // TracesDistributedWrapMutationTest does. private static final String INSERT_RAW_TRACE = """ INSERT INTO traces (workspace_id, project_id, id) VALUES (:workspace_id, :project_id, :id) @@ -209,11 +171,18 @@ SELECT toString(uniqExact(id)) /** A weekly partition value as it appears in SQL — {@code yyyyMMdd}, so always eight digits. */ private static final Pattern PARTITION_VALUE = Pattern.compile("\\d{8}"); - /** An id in a different week from anything the API mints, for the two-partition batch. id_at 2023-11-29 (a Wed). */ - private static final UUID OTHER_WEEK_ID = UUID.fromString("018c1860-1800-7abc-8000-000000000001"); + /** + * One id per era the derivation has to get right, with the partition each resolves to. Not interchangeable samples: + * {@code toMonday} agrees with the {@code Date32} expression across the ordinary calendar and diverges only + * far-future or at the epoch, so a recent-only batch would accept the very expression migration 000114 was written + * to escape. The 2200 id is the litellm shape and the one that makes the assertion bite. + */ + private static final List ERA_IDS = List.of( + UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), // id_at 2026-08-19 (Wed) -> 20260817 + UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8"), // id_at 1996-02-09 -> 19960205 + UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2")); // id_at 2200-01-01 -> 21991230 - /** The Monday of {@link #OTHER_WEEK_ID}'s week — stated as a literal so the expectation is readable on its own. */ - private static final long OTHER_WEEK_PARTITION = 20231127L; + private static final Set ERA_PARTITIONS = Set.of(20260817L, 19960205L, 21991230L); /** A v4 UUID: no timestamp to derive a partition from. */ private static final UUID NON_V7_ID = UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22"); @@ -278,7 +247,7 @@ void beforeAll(ClientSupport clientSupport, TransactionTemplateAsync template, T traceResourceClient = new TraceResourceClient(clientSupport, baseUrl); this.template = template; this.traceDAO = traceDAO; - exchangeTables(); + installPartitionedSuccessorUnderTraces(); } @AfterAll @@ -289,89 +258,6 @@ void afterAll() { network.close(); } - @ParameterizedTest - @MethodSource - @DisplayName("the DAO predicate, the live partitioning and the Java derivation agree exactly") - void predicateMatchesLivePartitioningAndJavaDerivation(String era, UUID id, long expectedPartition) { - // The guard the rest of the suite rests on, and the one that catches drift. Three independently-maintained - // expressions of the same rule have to agree, or a delete prunes to a partition its rows are not in: - // - // 1. the migration's PARTITION BY, as ClickHouse actually installed it -> _partition_id, where it filed the row - // 2. the DAO's predicate -> PARTITION_PREDICATE, evaluated here - // 3. WeeklyPartitions.of -> what gets bound to :partitions - // - // Compared as VALUES, not as normalized expression text. A text comparison would pin (1) against (2) and say - // nothing about (3), and it would pass for a rewrite that is textually equal after normalization yet computes a - // different week — which is precisely the toMonday trap migration 000114 was written to escape. Values also make - // the check immune to ClickHouse's re-printing of the AST, which is what made the previous substring form loose. - // - // The era matters: toMonday agrees with the Date32 expression across the ordinary calendar and diverges only for - // a far-future or epoch id_at, so a sample set that stopped at "recent" would accept the wrong expression. The - // rows are inserted in raw SQL because ingestion rejects a backdated or far-future id by design; only - // (workspace_id, project_id, id) are supplied, since id_at is MATERIALIZED and every other column has a DEFAULT - // — so this seeds through the real column definition rather than restating it. - insertRawTrace(id); - - var filedUnder = queryOneString(SELECT_PARTITION_ID, bindRawTrace(id)); - // The DAO predicate goes in as a StringTemplate fragment, not a bind: it is an expression to be evaluated, and - // evaluating the DAO's own text is the entire point. The workspace and id are values, so they bind. - var daoPredicateValue = queryOneString(withPartitionExpression(SELECT_PARTITION_EXPRESSION_VALUE), - bindRawTrace(id)); - - assertThat(daoPredicateValue) - .as("the DAO predicate resolves to the partition ClickHouse filed the %s row under", era) - .isEqualTo(filedUnder); - assertThat(WeeklyPartitions.of(List.of(id))) - .as("the Java derivation agrees with both for the %s row", era) - .contains(Set.of(expectedPartition)); - assertThat(filedUnder) - .as("and all three are the expected partition for the %s row", era) - .isEqualTo(String.valueOf(expectedPartition)); - } - - private static Stream predicateMatchesLivePartitioningAndJavaDerivation() { - return Stream.of( - // id_at 2026-08-19 (Wed) -> Monday 2026-08-17. The ordinary case; toMonday would also pass this one. - arguments("recent", UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), 20260817L), - // id_at 1996-02-09 -> Monday 1996-02-05. Far enough back to catch a key that keyed off wall-clock. - arguments("backdated", UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8"), 19960205L), - // id_at 2200-01-01 -> Monday 2199-12-30. The litellm shape, and the sample a 16-bit toMonday key wraps - // into a plausible recent week (000114) — so this row is what makes the guard bite. - arguments("far-future", UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2"), 21991230L)); - } - - @Test - @DisplayName("the DAO predicate is the same expression as the live partition key, not merely an equal-valued one") - void daoPredicateIsTheSameExpressionAsTheLivePartitionKey() { - // Value agreement (above) proves the predicate names the right partition; it does NOT prove ClickHouse will - // PRUNE on it. Pruning needs the planner to recognise the predicate as being on the partition key expression, - // so an equal-valued but differently-written expression would keep every delete correct and quietly rewrite - // every part again - the exact regression this PR exists to prevent, invisible to every other assertion here. - // - // Compared by round-tripping the DAO's text through ClickHouse as a partition key of its own and diffing the - // two re-prints. That makes the comparison AST-level and formatter-independent by construction: both strings - // come out of the same printer, so they are equal iff the parsed expressions are. Diffing the DAO text against - // system.tables directly would instead pin ClickHouse's whitespace choices, which is what it must not do. - execute(createProbeTableSql(), _ -> { - }); - try { - assertThat(partitionKeyOf(PARTITION_KEY_PROBE)) - .as("the DAO predicate parses to the same expression traces is partitioned by") - .isEqualTo(partitionKeyOf("traces")); - } finally { - execute(withProbeTable(DROP_PARTITION_KEY_PROBE), _ -> { - }); - } - } - - @Test - @DisplayName("id_at is the 64-bit column, so a far-future timestamp is honest rather than wrapped") - void idAtIsTheSixtyFourBitColumn() { - // The second half of what the flag asserts, and not implied by the partition agreement above: a 32-bit - // DateTime id_at would agree with itself while silently wrapping every id past 2106. - assertThat(queryOneString(SELECT_ID_AT_TYPE)).isEqualTo("DateTime64(0, 'UTC')"); - } - @Test @DisplayName("an all-UUIDv7 delete prunes to the batch's own partitions and removes the target row") void allUuidV7DeletePrunesAndRemovesTheTargetRow() { @@ -397,29 +283,35 @@ void allUuidV7DeletePrunesAndRemovesTheTargetRow() { } @Test - @DisplayName("a batch spanning two weeks binds both partitions, not a range") - void batchSpanningTwoWeeksBindsBothPartitions() { - // The multi-value Long[] bind, which the single-id path never exercises. The second id is minted for a week - // three years back and matches no row — the batch's partition SET is what is under test, and a delete does not - // need its ids to exist. + @DisplayName("the DAO's own delete clears every era and binds exactly those partitions") + void deleteClearsEveryEraAndBindsExactlyThosePartitions() { + // The three-way agreement - the migration's PARTITION BY as installed, the DAO's predicate, and + // WeeklyPartitions.of - asserted through the DAO's own delete instead of by re-evaluating the expression in + // test SQL. If the predicate resolved to any partition other than the one ClickHouse filed a row under, the + // mutation would select the wrong parts and that row would SURVIVE. So "every row is gone" IS the agreement, + // and it is established by the statement production actually runs rather than by a query written here. // - // Asserted as an EXACT set, because "mentions both partitions" is satisfied by a binding that also names every - // week in between, and that is the failure worth catching: an over-broad set keeps every delete correct while - // giving back the entire benefit. On prod-test a range over this span selected 2,644 of 3,928 parts where the - // exact set selected 4 — so a delete that is right and slow is the regression, and no behavioural assertion - // can see it. - var target = newTrace().build(); - traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); - var projectId = projectIdOf(target); - - delete(Set.of(Pair.of(projectId, target.id()), Pair.of(projectId, OTHER_WEEK_ID))); - - assertThat(traceIdsOf(target.projectName())).doesNotContain(target.id()); - var sql = lastTraceDeleteSql(target.id()); + // This also covers the DateTime64 half of what the flag asserts, without asking system.columns: against a + // 32-bit id_at the 2200 row would be stored under a wrapped recent timestamp, the derived partition would not + // match it, and the row would survive. + // + // Three eras in one batch is also the multi-value Long[] bind, which a single-id delete never reaches. Seeded + // raw because ingestion rejects a backdated or far-future id by design, supplying only the three columns + // without a DEFAULT so id_at comes from the real MATERIALIZED definition rather than a restated copy. + var projectId = UUID.randomUUID(); + ERA_IDS.forEach(id -> insertRawTrace(projectId, id)); + assertThat(ERA_IDS.stream().map(this::liveRowCount)).as("every era is seeded").containsOnly("1"); + + delete(ERA_IDS.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toUnmodifiableSet())); + + assertThat(ERA_IDS.stream().map(this::liveRowCount)) + .as("every era's row is gone, so the predicate named the partition each was actually filed under") + .containsOnly("0"); + var sql = lastTraceDeleteSql(ERA_IDS.getFirst()); assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); assertThat(boundPartitionsOf(sql)) - .as("exactly the batch's own two partitions, not a range across them") - .containsExactlyInAnyOrder(partitionOf(target.id()), OTHER_WEEK_PARTITION); + .as("exactly the three partitions the batch resolves to, not a range across three centuries") + .containsExactlyInAnyOrderElementsOf(ERA_PARTITIONS); } @ParameterizedTest @@ -468,8 +360,8 @@ private static Stream underivableIdDisablesPruning() { /** * The partition a single id resolves to. Derived through {@code WeeklyPartitions} on purpose: this suite is about * the predicate reaching ClickHouse, and the derivation's own expected values are pinned against real ClickHouse - * output in {@code WeeklyPartitionsTest} and again in - * {@link #predicateMatchesLivePartitioningAndJavaDerivation}, so restating them here would only duplicate that. + * output in {@code WeeklyPartitionsTest}, so restating them here would only duplicate that. Where the expectation + * needs to be readable on its own — the era batch — the partitions are stated as literals instead. */ private static long partitionOf(UUID id) { return WeeklyPartitions.of(List.of(id)).orElseThrow().iterator().next(); @@ -539,13 +431,16 @@ private String liveRowCount(UUID id) { } /** + * Setup, not a test: puts the partitioned successor under the name the DAO deletes from, so the pruning assertions + * are made against a table that actually has weekly partitions. + *

* The EXCHANGE (000003 exchange block): puts the successor under {@code traces} and the original under * {@code traces_local_v2}, then a RENAME parks the original as {@code traces_pre_cutover_backup}. The wrap is * deliberately not applied — it is a separate, deferrable step, and the flag under test must hold on its own between * the two (which is why it is not the wrap flag). Kept identical to the cutover SQL by eye, as * {@code TracesLocalV2CutoverTest.exchangeTables} and the wrap suite do. */ - private void exchangeTables() { + private void installPartitionedSuccessorUnderTraces() { execute("EXCHANGE TABLES traces AND traces_local_v2 ON CLUSTER '{cluster}'", _ -> { }); execute("RENAME TABLE traces_local_v2 TO traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { @@ -583,17 +478,12 @@ private List traceIdsOf(String projectName) { } /** - * Seeds one row through the table's real column definitions. Only the three columns without a {@code DEFAULT} are - * supplied; {@code id_at} is {@code MATERIALIZED}, so ClickHouse derives it from {@code id} exactly as it does for a - * row the ingestion path wrote — which is the point, since restating the {@code id_at} expression here would create - * the very drift surface this test exists to detect. Raw SQL rather than the API because ingestion rejects a - * backdated or far-future {@code id} by design ({@code IdGenerator.validateId}). + * Seeds one row through the table's real column definitions, in the caller's project. Only the three columns + * without a {@code DEFAULT} are supplied; {@code id_at} is {@code MATERIALIZED}, so ClickHouse derives it from + * {@code id} exactly as it does for a row the ingestion path wrote - which is the point, since restating the + * {@code id_at} expression here would create the very drift this suite exists to detect. Raw SQL because ingestion + * rejects a backdated, far-future or non-v7 {@code id} by design ({@code IdGenerator.validateId}). */ - private void insertRawTrace(UUID id) { - insertRawTrace(RAW_PROJECT_ID, id); - } - - /** As {@link #insertRawTrace(UUID)}, into a caller-chosen project, so a seeded row can share a test's project. */ private void insertRawTrace(UUID projectId, UUID id) { execute(INSERT_RAW_TRACE, statement -> statement @@ -602,58 +492,6 @@ private void insertRawTrace(UUID projectId, UUID id) { .bind("id", id.toString())); } - /** - * Renders a template whose only fragment is the DAO's partition-key expression — an expression, not a value, so it - * cannot be bound. Each renderer adds exactly the attributes its template declares, as the DAOs do. - */ - private static String withPartitionExpression(String sql) { - return TemplateUtils.newST(sql) - .add("partition_expression", PARTITION_PREDICATE) - .render(); - } - - /** - * As {@link #withPartitionExpression}, for the probe-table statements: a table identifier cannot be bound either. - */ - private static String withProbeTable(String sql) { - return TemplateUtils.newST(sql) - .add("probe_table", PARTITION_KEY_PROBE) - .render(); - } - - /** The probe-table DDL carries both fragments. */ - private static String createProbeTableSql() { - return TemplateUtils.newST(CREATE_PARTITION_KEY_PROBE) - .add("probe_table", PARTITION_KEY_PROBE) - .add("partition_expression", PARTITION_PREDICATE) - .render(); - } - - /** ClickHouse's own re-print of a table's partition key expression. */ - private String partitionKeyOf(String table) { - return queryOneString(SELECT_PARTITION_KEY, statement -> statement.bind("table", table)); - } - - private static Consumer bindRawTrace(UUID id) { - return statement -> statement - .bind("workspace_id", WORKSPACE_ID) - .bind("id", id.toString()); - } - - private String queryOneString(String sql) { - return queryOneString(sql, _ -> { - }); - } - - private String queryOneString(String sql, Consumer binder) { - return template.nonTransaction(connection -> { - var statement = connection.createStatement(sql); - binder.accept(statement); - return Mono.from(statement.execute()) - .flatMap(result -> Mono.from(result.map((row, _) -> row.get(0, String.class)))); - }).block(); - } - private void execute(String sql, Consumer binder) { template.nonTransaction(connection -> { var statement = connection.createStatement(sql); From 9cdb51dcb3c92a59d895f57af53cdd3c74c27089 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 12:16:06 +0200 Subject: [PATCH 21/37] [OPIK-6901] [BE] fix: restore queryOneString, deleted with the probe-table helpers 61c699d broke the build - all 18 Backend Tests jobs, including Unit Tests. Removing the probe-table renderers took the two queryOneString overloads with them (they sat between partitionKeyOf and execute in the block I cut), while lastTraceDeleteSql and liveRowCount still call it. Restored as a single binder-taking overload; the no-arg convenience form had no remaining callers. Every read in the suite is a single scalar, so that is the only mapper it needs. Why local javac missed it: I was filtering "cannot find symbol" out of javac's output as dependency-resolution noise, which is exactly what this error looks like. Added a check that cross-references declared private members against call sites and method references instead, which reports both a call with no declaration and a declaration with no callers. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index cea08fe4f24..248a830df89 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -492,6 +492,19 @@ private void insertRawTrace(UUID projectId, UUID id) { .bind("id", id.toString())); } + /** + * First column of the first row, as a string. Every read in this suite is a single scalar, so this is the only + * mapper needed; values go in as binds. + */ + private String queryOneString(String sql, Consumer binder) { + return template.nonTransaction(connection -> { + var statement = connection.createStatement(sql); + binder.accept(statement); + return Mono.from(statement.execute()) + .flatMap(result -> Mono.from(result.map((row, _) -> row.get(0, String.class)))); + }).block(); + } + private void execute(String sql, Consumer binder) { template.nonTransaction(connection -> { var statement = connection.createStatement(sql); From d9233e1faccb6dff2545e1d45c04b1dce7637636 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 12:18:40 +0200 Subject: [PATCH 22/37] [OPIK-6901] [BE] test: ask the planner whether it prunes, not just whether the rows go Restores, in a better instrument, the property the removed AST pin covered. Correctness and pruning are different claims and the suite only made the first: deletes were already correct before this change, so a suite that cannot see pruning stop does not test what the change exists to do. The guarded regression is specific - a migration rewrites the partition expression to something semantically identical but textually different, the planner stops recognising the DAO's predicate as the partition key, pruning silently stops, values still agree, every row is still deleted, and every other assertion here stays green. EXPLAIN indexes = 1, json = 1, following TracesLocalV2PartitioningTest's idiom and record shape rather than inventing one. EXPLAIN does not accept a mutation, so the WHERE clause is lifted verbatim out of the DAO's OWN emitted DELETE - predicate and inlined partition values included - and put behind a SELECT. Only the verb changes; the statement explained is still the DAO's, so this does not reintroduce a test-authored copy of the predicate. Verified the lift standalone against pruned, unbounded and post-wrap (traces_local) statement shapes. Asserts both directions, since a pruning assertion that would also pass without pruning is worth nothing - the same trap as `.contains(partition)` and `doesNotContain("toDayOfWeek")`: * bounded delete: selected parts strictly fewer than initial. No hard-coded 5; the comparison is against what the table holds, and the test seeds three eras first so there are several partitions to prune. * fallback delete: prunes nothing - either no partition index at all, because nothing filters the key, or every part still selected. One subtlety that would have made the fallback look pruned: PrimaryKey is deliberately excluded from the entries considered. The DAO's WHERE also filters workspace_id and (project_id, id), which are the sort key, so PrimaryKey prunes parts for the unbounded statement too; counting it would have destroyed the discrimination. Only MinMax and Partition entries count, and across them it takes the smallest selected and largest initial so it does not depend on which one reports the pruning. Not verified: this against ClickHouse. Which of MinMax/Partition carries the numbers, and whether the fallback yields an entry at all, are read from ClickHouse's documented behaviour rather than observed - hence tolerating both shapes instead of pinning one. CI is the check. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 248a830df89..14d9cd6a0ff 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -18,7 +18,11 @@ import com.comet.opik.infrastructure.auth.RequestContext; import com.comet.opik.infrastructure.db.TransactionTemplateAsync; import com.comet.opik.podam.PodamFactoryUtils; +import com.comet.opik.utils.JsonUtils; import com.comet.opik.utils.WeeklyPartitions; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; import com.redis.testcontainers.RedisContainer; import io.r2dbc.spi.Statement; import org.apache.commons.lang3.RandomStringUtils; @@ -37,6 +41,7 @@ import org.testcontainers.containers.Network; import org.testcontainers.lifecycle.Startables; import org.testcontainers.mysql.MySQLContainer; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import ru.vyarus.dropwizard.guice.test.ClientSupport; import ru.vyarus.dropwizard.guice.test.jupiter.ext.TestDropwizardAppExtension; @@ -44,6 +49,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.function.Consumer; @@ -168,6 +174,17 @@ SELECT toString(uniqExact(id)) private static final Pattern EMITTED_IN_CLAUSE = Pattern.compile( Pattern.quote(PARTITION_PREDICATE) + "\\s+IN\\s+([^\\n]*)"); + /** + * The emitted statement's shape, so its {@code WHERE} clause can be lifted verbatim and re-asked as a + * {@code SELECT}: {@code EXPLAIN} does not accept a mutation. Captures the target table too, since the DAO picks + * {@code traces} or {@code traces_local} depending on the wrap flag. + */ + private static final Pattern DELETE_SHAPE = Pattern.compile( + "DELETE\\s+FROM\\s+(\\S+)\\s+(WHERE\\b.*?)\\s+SETTINGS\\b", Pattern.DOTALL); + + /** {@code EXPLAIN} index entries that reflect partition-level part selection. */ + private static final Set PARTITION_INDEX_TYPES = Set.of("MinMax", "Partition"); + /** A weekly partition value as it appears in SQL — {@code yyyyMMdd}, so always eight digits. */ private static final Pattern PARTITION_VALUE = Pattern.compile("\\d{8}"); @@ -314,6 +331,48 @@ void deleteClearsEveryEraAndBindsExactlyThosePartitions() { .containsExactlyInAnyOrderElementsOf(ERA_PARTITIONS); } + @Test + @DisplayName("the planner actually prunes, and the fallback provably does not") + void pruningReachesThePlannerAndTheFallbackDoesNot() { + // Correctness and pruning are different claims, and this is the only test that makes the second one. Deletes + // were already correct before OPIK-6901 - what the change buys is parts touched (3,928/3,928 -> 5/3,928 on + // prod-test), so a suite that cannot see pruning stop does not test what this change exists to do. + // + // The regression it guards is specific: a migration rewrites the partition expression to something semantically + // identical but textually different, the planner stops recognising the DAO's predicate as the partition key, + // pruning silently stops - and values still agree, so every row is still deleted and every other assertion in + // this suite stays green. That is the property the removed AST pin covered; this asks the planner directly + // instead of inferring it from text. + // + // EXPLAIN does not accept a mutation, so the WHERE clause is lifted verbatim out of the DAO's own emitted + // DELETE and put behind a SELECT - predicate and bound partition values included. Only the verb changes; the + // statement being explained is still the DAO's. Same instrument and record shape as + // TracesLocalV2PartitioningTest. + var projectId = UUID.randomUUID(); + ERA_IDS.forEach(id -> insertRawTrace(projectId, id)); + + // Bounded: one derivable id, so the predicate names one of the three partitions just seeded. + delete(Set.of(Pair.of(projectId, ERA_IDS.getFirst()))); + var bounded = partsSelectedBy(lastTraceDeleteSql(ERA_IDS.getFirst())) + .orElseThrow(() -> new AssertionError("EXPLAIN reported no partition index for the bounded delete")); + + // Unbounded: a non-v7 id in the batch, so no predicate at all. Its partner is a different era, so the query_log + // lookup finds this statement rather than the one above. + var partner = ERA_IDS.get(1); + delete(Set.of(Pair.of(projectId, partner), Pair.of(projectId, NON_V7_ID))); + var unbounded = partsSelectedBy(lastTraceDeleteSql(partner)); + + assertThat(bounded.selected()) + .as("the bounded delete selects fewer parts than the table holds: %s", bounded) + .isLessThan(bounded.total()); + // Shown to discriminate, or it proves nothing - the same trap as `.contains(partition)` and + // `doesNotContain("toDayOfWeek")`. The fallback must not prune: either the planner reports no partition index at + // all, because nothing filters on the key, or it reports every part still selected. + assertThat(unbounded.map(parts -> parts.selected() == parts.total()).orElse(true)) + .as("the fallback prunes nothing: %s", unbounded) + .isTrue(); + } + @ParameterizedTest @MethodSource @DisplayName("an id with no derivable partition disables pruning for the batch, and the delete still lands") @@ -492,6 +551,54 @@ private void insertRawTrace(UUID projectId, UUID id) { .bind("id", id.toString())); } + /** + * Parts the planner selects for the DAO's own statement, or empty when it reports no partition index at all. + *

+ * {@code SELECT id} rather than {@code count()}, so no trivial-count optimisation can answer from metadata without + * selecting parts at all. + *

+ * Only {@code MinMax} and {@code Partition} entries are considered, and {@code PrimaryKey} is deliberately + * excluded: the DAO's {@code WHERE} also filters {@code workspace_id} and {@code (project_id, id)}, which are the + * sort key, so {@code PrimaryKey} prunes parts for the unbounded statement too — counting it would make the + * fallback look pruned and destroy the discrimination this test rests on. Across the entries that do qualify it + * takes the smallest selected count and the largest initial count, so it does not depend on which of the two + * reports the pruning. + *

+ * Empty is a meaningful answer rather than a failure: the {@code Indexes} block carries a partition entry only when + * the query filters on the partition key, so its absence is exactly what the fallback should produce. + */ + private Optional partsSelectedBy(String daoDeleteSql) { + var shape = DELETE_SHAPE.matcher(daoDeleteSql); + assertThat(shape.find()) + .as("the emitted statement has the expected DELETE shape:%n%s", daoDeleteSql) + .isTrue(); + var selectSql = "SELECT id FROM %s %s".formatted(shape.group(1), shape.group(2)); + + var explainRows = template.stream(connection -> Flux + .from(connection.createStatement("EXPLAIN indexes = 1, json = 1 %s".formatted(selectSql)).execute()) + .flatMap(result -> result.map((row, _) -> row.get("explain", String.class)))) + .collectList() + .block(); + var explain = String.join("\n", explainRows); + + var indexes = JsonUtils.getJsonNodeFromString(explain).findValue("Indexes"); + if (indexes == null) { + return Optional.empty(); + } + SelectedParts partition = null; + for (JsonNode index : indexes) { + if (!PARTITION_INDEX_TYPES.contains(index.path("Type").asText()) || !index.has("Selected Parts")) { + continue; + } + var entry = JsonUtils.treeToValue(index, SelectedParts.class); + partition = partition == null + ? entry + : new SelectedParts(Math.min(partition.selected(), entry.selected()), + Math.max(partition.total(), entry.total())); + } + return Optional.ofNullable(partition); + } + /** * First column of the first row, as a string. Every read in this suite is a single scalar, so this is the only * mapper needed; values go in as binds. @@ -512,4 +619,14 @@ private void execute(String sql, Consumer binder) { return Mono.from(statement.execute()).flatMap(result -> Mono.from(result.getRowsUpdated())); }).block(); } + + /** + * The part counts {@code EXPLAIN indexes = 1, json = 1} reports for one index entry: how many parts the query + * started from, and how many survived pruning. + */ + @JsonIgnoreProperties(ignoreUnknown = true) + private record SelectedParts( + @JsonProperty("Selected Parts") int selected, + @JsonProperty("Initial Parts") int total) { + } } From 9f87edf94b3abd12caa7ee79ba455dff7c5a7472 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 12:33:16 +0200 Subject: [PATCH 23/37] [OPIK-6901] [BE] test: cover the flag-off delete on the legacy table Reviewer: a false-flag regression during rollback passes unnoticed. Correct, and the gap is wider than rollback - flag off against legacy `traces` is the state EVERY deployment is in today. Nothing caught it. Every other trace-delete suite runs in exactly that state and asserts only that rows go away - which they do for a RECENT id, because the legacy 32-bit id_at is accurate for one. The damage shows only on a far-future id (litellm ~2201): the legacy column stores a wrapped recent timestamp, a derived partition cannot match it, and the delete reports success having matched ZERO rows. No existing test has such a row, because ingestion rejects far-future ids by design. TracesPruningDisabledMutationTest seeds one and asserts the emitted SQL carries no partition predicate and no id_at narrowing of any kind, and that the row is deleted. Shared containers - it changes no topology. The project comes from the real ingestion path (create a trace through the endpoint, read the project id back off it) rather than a fabricated identifier, and the test asserts that id is a UUIDv7. Only the far-future row is raw, because the 24h ingestion window refuses it; a recent id would prove nothing here anyway, since even a wrongly-emitted predicate matches one. Self-skips once the cutover migration lands: at that point there is no legacy `traces` and the wrapping hazard stops existing, so an assumption in beforeAll reports the reason rather than failing on a premise that has legitimately gone away. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPruningDisabledMutationTest.java | 280 ++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPruningDisabledMutationTest.java diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPruningDisabledMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPruningDisabledMutationTest.java new file mode 100644 index 00000000000..151081a3a9d --- /dev/null +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPruningDisabledMutationTest.java @@ -0,0 +1,280 @@ +package com.comet.opik.infrastructure; + +import com.comet.opik.api.Trace; +import com.comet.opik.api.resources.utils.ClickHouseContainerUtils; +import com.comet.opik.api.resources.utils.ClientSupportUtils; +import com.comet.opik.api.resources.utils.MigrationUtils; +import com.comet.opik.api.resources.utils.MySQLContainerUtils; +import com.comet.opik.api.resources.utils.RedisContainerUtils; +import com.comet.opik.api.resources.utils.TestDropwizardAppExtensionUtils; +import com.comet.opik.api.resources.utils.TestDropwizardAppExtensionUtils.AppContextConfig; +import com.comet.opik.api.resources.utils.TestUtils; +import com.comet.opik.api.resources.utils.WireMockUtils; +import com.comet.opik.api.resources.utils.resources.TraceResourceClient; +import com.comet.opik.domain.TraceDAO; +import com.comet.opik.extensions.DropwizardAppExtensionProvider; +import com.comet.opik.extensions.RegisterApp; +import com.comet.opik.infrastructure.auth.RequestContext; +import com.comet.opik.infrastructure.db.TransactionTemplateAsync; +import com.comet.opik.podam.PodamFactoryUtils; +import com.redis.testcontainers.RedisContainer; +import io.r2dbc.spi.Statement; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.mysql.MySQLContainer; +import reactor.core.publisher.Mono; +import ru.vyarus.dropwizard.guice.test.ClientSupport; +import ru.vyarus.dropwizard.guice.test.jupiter.ext.TestDropwizardAppExtension; +import uk.co.jemos.podam.api.PodamFactory; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; + +import static com.comet.opik.api.resources.utils.AuthTestUtils.mockTargetWorkspace; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; + +/** + * The other side of {@code TracesPartitionPruningMutationTest}: the trace delete with + * {@code databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled} off — its default — against the + * legacy {@code traces}. That is not an exotic topology; it is the state every deployment is in today, and the + * state a stage B/C rollback returns to. + * + *

What it guards is a false-flag regression: the partition predicate emitted while the flag is off, or any + * other {@code id_at} narrowing creeping into the unbounded form. Legacy {@code traces} has no {@code PARTITION BY} at + * all and declares {@code id_at} as a 32-bit {@code DateTime} that overflows past 2106 (migrations 000001 and 000091), + * so a predicate here prunes nothing and can silently exclude rows. + * + *

Nothing else catches it. Every other trace-delete suite runs in exactly this state and asserts only that rows go + * away — which they do for a recent id, because the legacy {@code id_at} is accurate for one. The damage shows + * only on a far-future id (litellm BerriAI/litellm#31294 + * mints ~2201): the 32-bit column stores a wrapped recent timestamp, a derived partition cannot match it, and the + * delete reports success having matched zero rows. No existing test has such a row, because ingestion rejects + * far-future ids by design ({@code IdGenerator.validateId}) — so this seeds one in raw SQL and deletes it through + * {@link TraceDAO#delete}, which is also the only way to reach that arm. + * + *

Shared, reusable containers: unlike the post-EXCHANGE suite this changes no topology, so there is nothing here + * that could corrupt another suite or a rerun. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith(DropwizardAppExtensionProvider.class) +class TracesPruningDisabledMutationTest { + + private static final String API_KEY = "apiKey-" + UUID.randomUUID(); + private static final String WORKSPACE_NAME = "workspace-" + RandomStringUtils.secure().nextAlphanumeric(32); + private static final String WORKSPACE_ID = UUID.randomUUID().toString(); + private static final String USER = "user-" + RandomStringUtils.secure().nextAlphanumeric(32); + + /** A UUIDv7 carrying id_at 2200-01-01 — the litellm shape, and the id the legacy 32-bit column wraps. */ + private static final UUID FAR_FUTURE_ID = UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2"); + + private static final String INSERT_RAW_TRACE = """ + INSERT INTO traces (workspace_id, project_id, id) + VALUES (:workspace_id, :project_id, :id) + """; + + private static final String LIVE_ROW_COUNT = """ + SELECT toString(uniqExact(id)) + FROM traces + WHERE workspace_id = :workspace_id + AND id = :id + """; + + private static final String LAST_TRACE_DELETE = """ + SELECT query + FROM system.query_log + WHERE log_comment LIKE 'delete_traces:%' + AND type = 'QueryFinish' + AND query LIKE concat('%', :trace_id, '%') + ORDER BY event_time_microseconds DESC + LIMIT 1 + """; + + private static final String LEGACY_SCHEMA = """ + SELECT concat( + (SELECT partition_key FROM system.tables + WHERE database = currentDatabase() AND name = 'traces'), + '|', + (SELECT type FROM system.columns + WHERE database = currentDatabase() AND table = 'traces' AND name = 'id_at')) + """; + + private final RedisContainer redisContainer = RedisContainerUtils.newRedisContainer(); + private final MySQLContainer mysqlContainer = MySQLContainerUtils.newMySQLContainer(); + private final GenericContainer zookeeperContainer = ClickHouseContainerUtils.newZookeeperContainer(); + private final ClickHouseContainer clickHouseContainer = ClickHouseContainerUtils + .newClickHouseContainer(zookeeperContainer); + + private final WireMockUtils.WireMockRuntime wireMock; + + private final PodamFactory factory = PodamFactoryUtils.newPodamFactory(); + + @RegisterApp + private final TestDropwizardAppExtension app; + + { + Startables.deepStart(redisContainer, mysqlContainer, clickHouseContainer, zookeeperContainer).join(); + wireMock = WireMockUtils.startWireMock(); + var databaseAnalyticsFactory = ClickHouseContainerUtils.newDatabaseAnalyticsFactory( + clickHouseContainer, ClickHouseContainerUtils.DATABASE_NAME); + MigrationUtils.runMysqlDbMigration(mysqlContainer); + MigrationUtils.runClickhouseDbMigration(clickHouseContainer); + // No customConfigs on purpose: the defaults ARE the state under test. + app = TestDropwizardAppExtensionUtils.newTestDropwizardAppExtension( + AppContextConfig.builder() + .jdbcUrl(mysqlContainer.getJdbcUrl()) + .databaseAnalyticsFactory(databaseAnalyticsFactory) + .redisUrl(redisContainer.getRedisURI()) + .runtimeInfo(wireMock.runtimeInfo()) + .build()); + } + + private TraceResourceClient traceResourceClient; + private TransactionTemplateAsync template; + private TraceDAO traceDAO; + + @BeforeAll + void beforeAll(ClientSupport clientSupport, TransactionTemplateAsync template, TraceDAO traceDAO) { + var baseUrl = TestUtils.getBaseUrl(clientSupport); + ClientSupportUtils.config(clientSupport); + mockTargetWorkspace(wireMock.server(), API_KEY, WORKSPACE_NAME, WORKSPACE_ID, USER); + traceResourceClient = new TraceResourceClient(clientSupport, baseUrl); + this.template = template; + this.traceDAO = traceDAO; + + // This whole suite is about the pre-cutover / post-rollback state. Once the cutover migration lands there is no + // legacy `traces` left to test - it becomes the partitioned successor with a DateTime64 id_at - and the + // far-future wrapping hazard this guards stops existing. Skip with the reason stated rather than fail on a + // premise that has legitimately gone away; JUnit reports the skip and its description in the test report. + assumeThat(legacySchema()) + .as("pre-cutover estate: legacy `traces`, unpartitioned with a 32-bit DateTime id_at") + .isEqualTo("|DateTime('UTC')"); + } + + @AfterAll + void afterAll() { + wireMock.server().stop(); + } + + @Test + @DisplayName("with pruning off, a far-future row on the legacy table is still deleted, unbounded") + void farFutureRowOnLegacyTableIsStillDeleted() { + // The project and its id come from the real ingestion path - create a trace through the endpoint, then read the + // project id back off it - so the delete runs against a project that exists and a genuine UUIDv7 project id, + // not a fabricated one. + var seedTrace = factory.manufacturePojo(Trace.class).toBuilder() + .feedbackScores(null) + .usage(null) + .build(); + traceResourceClient.createTrace(seedTrace, API_KEY, WORKSPACE_NAME); + var projectId = projectIdOf(seedTrace); + assertThat(projectId.version()).as("the project id is a real UUIDv7, as the backend mints them").isEqualTo(7); + + // Only the far-future row is raw, because ingestion rejects it by design (24h window). A recent id would prove + // nothing here anyway: the legacy id_at is accurate for one, so even a wrongly-emitted predicate would match it + // and the row would still go. + insertRawTrace(projectId, FAR_FUTURE_ID); + assertThat(liveRowCount(FAR_FUTURE_ID)).as("the far-future row is seeded").isEqualTo("1"); + + delete(Set.of(Pair.of(projectId, FAR_FUTURE_ID))); + + assertThat(liveRowCount(FAR_FUTURE_ID)) + .as("it is deleted - a partition predicate here would have matched nothing and reported success") + .isEqualTo("0"); + + var sql = lastTraceDeleteSql(FAR_FUTURE_ID); + assertThat(sql) + .as("no partition predicate is emitted while the flag is off") + .doesNotContain("toYYYYMMDD", "toDayOfWeek", "toIntervalDay"); + assertThat(sql) + .as("and no id_at narrowing of any kind - toMonday, a range, or otherwise") + .doesNotContain("id_at"); + } + + /** Invokes the DAO under a workspace/user context, as {@code TraceService} does for the live delete path. */ + private void delete(Set> projectIdTraceIdPairs) { + template.nonTransaction(connection -> traceDAO.delete(projectIdTraceIdPairs, connection)) + .contextWrite(ctx -> ctx + .put(RequestContext.WORKSPACE_ID, WORKSPACE_ID) + .put(RequestContext.USER_NAME, USER)) + .block(); + } + + /** + * Seeds one row through the table's real column definitions. Only the identity columns are supplied; {@code id_at} + * is {@code MATERIALIZED}, so ClickHouse derives it — and, on this table, wraps it — exactly as it would for a row + * the ingestion path wrote. + */ + private void insertRawTrace(UUID projectId, UUID id) { + execute(INSERT_RAW_TRACE, statement -> statement + .bind("workspace_id", WORKSPACE_ID) + .bind("project_id", projectId.toString()) + .bind("id", id.toString())); + } + + /** The real project id the ingestion path minted, read back off the created trace. */ + private UUID projectIdOf(Trace trace) { + return traceResourceClient + .getTraces(trace.projectName(), null, API_KEY, WORKSPACE_NAME, List.of(), List.of(), 100, Map.of()) + .content().stream() + .filter(found -> found.id().equals(trace.id())) + .map(Trace::projectId) + .findFirst() + .orElseThrow(); + } + + /** {@code "1"} while a live (non-lightweight-deleted) row exists for the id, {@code "0"} once it is gone. */ + private String liveRowCount(UUID id) { + return queryOneString(LIVE_ROW_COUNT, statement -> statement + .bind("workspace_id", WORKSPACE_ID) + .bind("id", id.toString())); + } + + private String legacySchema() { + return queryOneString(LEGACY_SCHEMA, _ -> { + }); + } + + /** + * The SQL of the trace delete that carried {@code traceId}. Filtered by {@code log_comment} and the id, so a + * neighbouring suite's delete in this shared container cannot be picked up instead — the id narrows it to this test. + */ + private String lastTraceDeleteSql(UUID traceId) { + execute("SYSTEM FLUSH LOGS", _ -> { + }); + var sql = queryOneString(LAST_TRACE_DELETE, statement -> statement.bind("trace_id", traceId.toString())); + assertThat(sql) + .as("query_log holds a delete_traces statement mentioning id '%s'", traceId) + .isNotNull(); + return sql; + } + + private String queryOneString(String sql, Consumer binder) { + return template.nonTransaction(connection -> { + var statement = connection.createStatement(sql); + binder.accept(statement); + return Mono.from(statement.execute()) + .flatMap(result -> Mono.from(result.map((row, _) -> row.get(0, String.class)))); + }).block(); + } + + private void execute(String sql, Consumer binder) { + template.nonTransaction(connection -> { + var statement = connection.createStatement(sql); + binder.accept(statement); + return Mono.from(statement.execute()).flatMap(result -> Mono.from(result.getRowsUpdated())); + }).block(); + } +} From 2e256667a2f5992ef2f438c1967cfa5c9acec6b6 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 12:33:16 +0200 Subject: [PATCH 24/37] [OPIK-6901] [BE] test: seed via the endpoint with a real v7 project id, and survive the migration Two problems with the pruning suite's data, both fabrication rather than fixtures. `UUID.randomUUID()` is a v4, and project ids are UUIDv7 - the backend mints them through IdGenerator. So the era and EXPLAIN tests were deleting from a project id that could not exist, in a project that did not exist, in a PR whose whole subject is that the id version matters. The project now comes from the real ingestion path: create a trace through the endpoint, read the project id back off it, and assert it is a v7. The recent row now goes through the endpoint too. Only the 1996 and 2200 eras stay raw, because the ingestion window is 24h and no endpoint can create them; they are seeded into the project the endpoint just made. (WORKSPACE_ID and API_KEY stay random UUIDs - workspace_id is a String column, not an entity id, and 62 suites in this tree do the same.) Separately, the setup was pinned to today's estate: it EXCHANGEd `traces` with `traces_local_v2` unconditionally, so once the cutover migration lands - `traces` already the successor, `traces_local_v2` gone - beforeAll would die on a table that no longer exists. Now idempotent: if `traces` already partitions on id_at it does nothing, and if neither state holds it says so instead of surfacing a bare "table not found". Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 94 +++++++++++++------ 1 file changed, 67 insertions(+), 27 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 14d9cd6a0ff..57604908b4a 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -143,6 +143,20 @@ class TracesPartitionPruningMutationTest { // are single literals built by no Java string operation, and they are kept byte-identical to // 000003_exchange_and_wrap.sql by eye, so they belong at the call site next to the javadoc that says so - as // TracesDistributedWrapMutationTest does. + private static final String TRACES_PARTITION_KEY = """ + SELECT partition_key + FROM system.tables + WHERE database = currentDatabase() + AND name = 'traces' + """; + + private static final String SUCCESSOR_TABLE_COUNT = """ + SELECT toString(count()) + FROM system.tables + WHERE database = currentDatabase() + AND name = 'traces_local_v2' + """; + private static final String INSERT_RAW_TRACE = """ INSERT INTO traces (workspace_id, project_id, id) VALUES (:workspace_id, :project_id, :id) @@ -194,12 +208,11 @@ SELECT toString(uniqExact(id)) * far-future or at the epoch, so a recent-only batch would accept the very expression migration 000114 was written * to escape. The 2200 id is the litellm shape and the one that makes the assertion bite. */ - private static final List ERA_IDS = List.of( - UUID.fromString("01a01a75-76de-785e-ae84-8870ed5e6db3"), // id_at 2026-08-19 (Wed) -> 20260817 - UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8"), // id_at 1996-02-09 -> 19960205 - UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2")); // id_at 2200-01-01 -> 21991230 + private static final List OUT_OF_WINDOW_ERA_IDS = List.of( + UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8"), // id_at 1996-02-09 -> 19960205 + UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2")); // id_at 2200-01-01 -> 21991230 - private static final Set ERA_PARTITIONS = Set.of(20260817L, 19960205L, 21991230L); + private static final Set OUT_OF_WINDOW_ERA_PARTITIONS = Set.of(19960205L, 21991230L); /** A v4 UUID: no timestamp to derive a partition from. */ private static final UUID NON_V7_ID = UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22"); @@ -264,7 +277,7 @@ void beforeAll(ClientSupport clientSupport, TransactionTemplateAsync template, T traceResourceClient = new TraceResourceClient(clientSupport, baseUrl); this.template = template; this.traceDAO = traceDAO; - installPartitionedSuccessorUnderTraces(); + ensurePartitionedSuccessorUnderTraces(); } @AfterAll @@ -312,23 +325,30 @@ void deleteClearsEveryEraAndBindsExactlyThosePartitions() { // 32-bit id_at the 2200 row would be stored under a wrapped recent timestamp, the derived partition would not // match it, and the row would survive. // - // Three eras in one batch is also the multi-value Long[] bind, which a single-id delete never reaches. Seeded - // raw because ingestion rejects a backdated or far-future id by design, supplying only the three columns - // without a DEFAULT so id_at comes from the real MATERIALIZED definition rather than a restated copy. - var projectId = UUID.randomUUID(); - ERA_IDS.forEach(id -> insertRawTrace(projectId, id)); - assertThat(ERA_IDS.stream().map(this::liveRowCount)).as("every era is seeded").containsOnly("1"); - - delete(ERA_IDS.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toUnmodifiableSet())); - - assertThat(ERA_IDS.stream().map(this::liveRowCount)) + // Three eras in one batch is also the multi-value Long[] bind, which a single-id delete never reaches. + var recent = newTrace().build(); + traceResourceClient.createTrace(recent, API_KEY, WORKSPACE_NAME); + var projectId = projectIdOf(recent); + assertThat(projectId.version()).as("the project id is a real UUIDv7, as the backend mints it").isEqualTo(7); + // Only the out-of-window eras are seeded raw: ingestion rejects a 1996 or 2200 id by design (24h window), so + // there is no endpoint that can create them. The recent row above went through the real ingestion path, and + // they all share its project. + OUT_OF_WINDOW_ERA_IDS.forEach(id -> insertRawTrace(projectId, id)); + var batch = Stream.concat(Stream.of(recent.id()), OUT_OF_WINDOW_ERA_IDS.stream()).toList(); + assertThat(batch.stream().map(this::liveRowCount)).as("every era is present").containsOnly("1"); + + delete(batch.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toUnmodifiableSet())); + + assertThat(batch.stream().map(this::liveRowCount)) .as("every era's row is gone, so the predicate named the partition each was actually filed under") .containsOnly("0"); - var sql = lastTraceDeleteSql(ERA_IDS.getFirst()); + var sql = lastTraceDeleteSql(recent.id()); assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); assertThat(boundPartitionsOf(sql)) - .as("exactly the three partitions the batch resolves to, not a range across three centuries") - .containsExactlyInAnyOrderElementsOf(ERA_PARTITIONS); + .as("exactly the partitions the batch resolves to, not a range across three centuries") + .containsExactlyInAnyOrderElementsOf( + Stream.concat(Stream.of(partitionOf(recent.id())), OUT_OF_WINDOW_ERA_PARTITIONS.stream()) + .collect(Collectors.toUnmodifiableSet())); } @Test @@ -348,17 +368,21 @@ void pruningReachesThePlannerAndTheFallbackDoesNot() { // DELETE and put behind a SELECT - predicate and bound partition values included. Only the verb changes; the // statement being explained is still the DAO's. Same instrument and record shape as // TracesLocalV2PartitioningTest. - var projectId = UUID.randomUUID(); - ERA_IDS.forEach(id -> insertRawTrace(projectId, id)); - - // Bounded: one derivable id, so the predicate names one of the three partitions just seeded. - delete(Set.of(Pair.of(projectId, ERA_IDS.getFirst()))); - var bounded = partsSelectedBy(lastTraceDeleteSql(ERA_IDS.getFirst())) + // Real project from the ingestion path, and a recent row created through the endpoint; only the out-of-window + // eras are seeded raw, so the table holds several partitions to prune between. + var recent = newTrace().build(); + traceResourceClient.createTrace(recent, API_KEY, WORKSPACE_NAME); + var projectId = projectIdOf(recent); + OUT_OF_WINDOW_ERA_IDS.forEach(id -> insertRawTrace(projectId, id)); + + // Bounded: one derivable id, so the predicate names one of the partitions just populated. + delete(Set.of(Pair.of(projectId, recent.id()))); + var bounded = partsSelectedBy(lastTraceDeleteSql(recent.id())) .orElseThrow(() -> new AssertionError("EXPLAIN reported no partition index for the bounded delete")); // Unbounded: a non-v7 id in the batch, so no predicate at all. Its partner is a different era, so the query_log // lookup finds this statement rather than the one above. - var partner = ERA_IDS.get(1); + var partner = OUT_OF_WINDOW_ERA_IDS.getFirst(); delete(Set.of(Pair.of(projectId, partner), Pair.of(projectId, NON_V7_ID))); var unbounded = partsSelectedBy(lastTraceDeleteSql(partner)); @@ -493,13 +517,29 @@ private String liveRowCount(UUID id) { * Setup, not a test: puts the partitioned successor under the name the DAO deletes from, so the pruning assertions * are made against a table that actually has weekly partitions. *

+ * Idempotent on purpose, because the estate this runs against is going to change. Today {@code traces} is + * the legacy table and the successor is the empty {@code traces_local_v2}, so the two cutover statements are needed. + * Once the cutover migration lands, {@code traces} is the partitioned successor and {@code traces_local_v2} + * is gone — at which point this is a no-op and the suite keeps working unchanged, instead of dying in + * {@link #beforeAll} on an {@code EXCHANGE} against a table that no longer exists. If neither state holds it fails + * with that said plainly, rather than surfacing as a bare "table not found". + *

* The EXCHANGE (000003 exchange block): puts the successor under {@code traces} and the original under * {@code traces_local_v2}, then a RENAME parks the original as {@code traces_pre_cutover_backup}. The wrap is * deliberately not applied — it is a separate, deferrable step, and the flag under test must hold on its own between * the two (which is why it is not the wrap flag). Kept identical to the cutover SQL by eye, as * {@code TracesLocalV2CutoverTest.exchangeTables} and the wrap suite do. */ - private void installPartitionedSuccessorUnderTraces() { + private void ensurePartitionedSuccessorUnderTraces() { + if (queryOneString(TRACES_PARTITION_KEY, _ -> { + }).contains("id_at")) { + return; // The cutover migration has landed: `traces` already is the partitioned successor. + } + assertThat(queryOneString(SUCCESSOR_TABLE_COUNT, _ -> { + })) + .as("`traces` is not partitioned and `traces_local_v2` does not exist, so there is no successor to" + + " install - this suite needs one of those two states") + .isEqualTo("1"); execute("EXCHANGE TABLES traces AND traces_local_v2 ON CLUSTER '{cluster}'", _ -> { }); execute("RENAME TABLE traces_local_v2 TO traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { From b31f186a69d6a168c6e13ce7ab0f0efbb29c92d9 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 12:35:20 +0200 Subject: [PATCH 25/37] [OPIK-6901] [BE] test: point the DAO at the data with the wrap flag, covering both flags on Better than what it replaces, and it withdraws a decline I made earlier. The suite was reaching the partitioned data by renaming tables under the DAO and leaving it on the traces branch. It now applies the wrap and sets tracesDistributedWrapEnabled, so the DAO's mutations reach the data through the configuration switch that governs them in production - DELETE FROM traces_local, chosen by the flag - while traces is the Distributed wrapper that reads and inserts flow through. That is why the endpoint-created row and the raw-seeded ones land in the same place. This is the post-cutover end state, and it covers both schema flags on at once. I declined that in an earlier round as needing a third topology and its own suite; that was wrong - the wrap flag is how the DAO is pointed at the data, so enabling it costs two setup statements and covers the combination the fleet actually ends up in. The javadoc note claiming the cell was untested is replaced rather than left to mislead. distributedTracesRejectsDirectMutation keeps the claim from being vacuous: had the wrap not taken effect, traces would still be a MergeTree, every pruned delete here would have run against it, and nothing would have said so. It asserts the specific ClickHouse rejection, as the wrap suite does. What is no longer covered here is the transient post-EXCHANGE/pre-wrap window. The predicate is identical in both - traces_local is the same physical table under another name - and the flag's javadoc already records that it must hold in that window. Both setup steps are idempotent, so this survives the cutover migration: the successor install skips when traces_local already exists or traces already partitions on id_at, and the wrap skips when traces is already Distributed. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 127 +++++++++++++----- 1 file changed, 95 insertions(+), 32 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 57604908b4a..3498450ea39 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -59,6 +59,7 @@ import static com.comet.opik.api.resources.utils.AuthTestUtils.mockTargetWorkspace; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.params.provider.Arguments.arguments; /** @@ -94,29 +95,33 @@ * those rows are seeded raw and those batches are handed to {@link TraceDAO#delete} directly — the only way to reach * that arm. * - *

Why the EXCHANGE is here at all — it is setup, never the subject. Nothing asserts anything about it. It is - * required because the DAO names its target table: after the Liquibase migrations the live {@code traces} is still the - * legacy table — no {@code PARTITION BY} at all and a 32-bit {@code DateTime} {@code id_at} — while the - * partitioned successor exists only as the empty {@code traces_local_v2}, which no DAO query can reach. Two statements - * copied from the cutover put the successor under the name the DAO deletes from. Without them these tests would run + *

Why the topology setup is here at all — it is setup, never the subject. Nothing asserts anything about the + * EXCHANGE or the wrap themselves; {@code TracesLocalV2CutoverTest} owns that. They are required because the DAO names + * its target table: after the Liquibase migrations the live {@code traces} is still the legacy table — no + * {@code PARTITION BY} at all and a 32-bit {@code DateTime} {@code id_at} — while the partitioned successor exists only + * as the empty {@code traces_local_v2}, which no DAO query can reach. Without installing it these tests would run * against the one table where this predicate must never be emitted, and would pass while proving nothing: the predicate * is harmless against an unpartitioned table for recent ids. Hand-authoring a partitioned {@code traces} in the test - * instead would duplicate migration 000114 and reintroduce exactly the drift this suite exists to detect. + * instead would duplicate migration 000114 and reintroduce exactly the drift this suite exists to detect. Both steps are + * idempotent, so the suite keeps working once the cutover migration lands and they become no-ops. * - *

Topology covered, and the one cell that is not. This suite runs the post-EXCHANGE, pre-wrap state on - * purpose: {@code traces} is the partitioned successor and still a {@code MergeTree}, which is the state the pruning - * flag has to hold in on its own — the wrap is a separate, deferrable cutover step ({@code --skip-wrap} now, - * {@code --wrap-only} later), and prod-test sat in exactly this window. Applying the wrap here would remove that - * coverage rather than add to it, since {@code partition_key} is meaningless once {@code traces} is {@code Distributed}. - * The {@code traces_local} branch of this same template is executed by {@code TracesDistributedWrapMutationTest}, with - * pruning off. So the untested cell is both flags on at once, and it stays untested here: the two are independent - * StringTemplate attributes with no shared state, and the wrap is a {@code RENAME} — {@code traces_local} is the very - * table this suite partitions and asserts against, so {@code id_at} and the partition key belong to the data, not to - * the name it is reached by. Covering it needs a third topology (EXCHANGE + wrap + both flags) and therefore its own - * suite; it cannot live in the wrap suite, which wraps the legacy table where this flag must be false. + *

Topology: the post-cutover end state, with both schema flags on. The wrap is applied and + * {@code tracesDistributedWrapEnabled} set, so the DAO's mutations reach the data the way production routes them — + * {@code DELETE FROM traces_local}, chosen by the configuration switch that governs it, not by a table this suite + * renamed under the DAO. {@code traces} is the {@code Distributed} wrapper that reads and inserts flow through, which + * is why the endpoint-created row and the raw-seeded ones land in the same place. + * {@link #distributedTracesRejectsDirectMutation} keeps that claim honest: had the wrap not taken effect, + * {@code traces} would still be a {@code MergeTree} and every pruned delete here would have run against it. * - *

Dedicated, non-reused ClickHouse and ZooKeeper containers are required because the EXCHANGE destructively swaps the - * live {@code traces} table; a reused container would corrupt other suites and reruns. + *

This supersedes an earlier note in this file that both flags on at once was untested and would need its own suite. + * It does not: the wrap flag is how the DAO is pointed at the data, so enabling it costs two setup statements and + * covers the combination the fleet actually ends up in. What is no longer covered here is the transient + * post-EXCHANGE/pre-wrap window — the pruning predicate is identical in both, since {@code traces_local} is the same + * physical table under a different name, and the flag's own javadoc records that it must hold in that window. + * + *

Dedicated, non-reused ClickHouse and ZooKeeper containers are required because the setup destructively renames the + * live {@code traces} table — the EXCHANGE swaps it, and the wrap then renames it to {@code traces_local} — so a reused + * container would corrupt other suites and reruns. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @ExtendWith(DropwizardAppExtensionProvider.class) @@ -143,22 +148,29 @@ class TracesPartitionPruningMutationTest { // are single literals built by no Java string operation, and they are kept byte-identical to // 000003_exchange_and_wrap.sql by eye, so they belong at the call site next to the javadoc that says so - as // TracesDistributedWrapMutationTest does. - private static final String TRACES_PARTITION_KEY = """ + private static final String PARTITION_KEY_OF_TABLE = """ SELECT partition_key FROM system.tables WHERE database = currentDatabase() - AND name = 'traces' + AND name = :table """; - private static final String SUCCESSOR_TABLE_COUNT = """ + private static final String TABLE_COUNT = """ SELECT toString(count()) FROM system.tables WHERE database = currentDatabase() - AND name = 'traces_local_v2' + AND name = :table + """; + + private static final String TABLE_ENGINE = """ + SELECT engine + FROM system.tables + WHERE database = currentDatabase() + AND name = 'traces' """; private static final String INSERT_RAW_TRACE = """ - INSERT INTO traces (workspace_id, project_id, id) + INSERT INTO traces_local (workspace_id, project_id, id) VALUES (:workspace_id, :project_id, :id) """; @@ -174,7 +186,7 @@ AND query LIKE concat('%', :trace_id, '%') private static final String LIVE_ROW_COUNT = """ SELECT toString(uniqExact(id)) - FROM traces + FROM traces_local WHERE workspace_id = :workspace_id AND id = :id """; @@ -261,7 +273,8 @@ SELECT toString(uniqExact(id)) .customConfigs(List.of( new CustomConfig("databaseAnalyticsDataModel.traceColumnsNonNullable", "true"), new CustomConfig("databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled", - "true"))) + "true"), + new CustomConfig("databaseAnalyticsDataModel.tracesDistributedWrapEnabled", "true"))) .build()); } @@ -278,6 +291,7 @@ void beforeAll(ClientSupport clientSupport, TransactionTemplateAsync template, T this.template = template; this.traceDAO = traceDAO; ensurePartitionedSuccessorUnderTraces(); + ensureDistributedWrap(); } @AfterAll @@ -288,6 +302,18 @@ void afterAll() { network.close(); } + @Test + @DisplayName("traces really is a mutation-rejecting Distributed wrapper, so the deletes here ran on traces_local") + void distributedTracesRejectsDirectMutation() { + // Keeps the both-flags claim from being vacuous. If the wrap had not taken effect, `traces` would still be a + // MergeTree, every pruned delete in this suite would have run against it, and nothing here would say so. + // Asserting the specific rejection - not merely that something threw - is what proves `traces` is Distributed, + // so a green delete could only have reached `traces_local`. + assertThatThrownBy(() -> execute("DELETE FROM traces WHERE workspace_id = :workspace_id", + statement -> statement.bind("workspace_id", WORKSPACE_ID))) + .hasMessageContaining("DELETE query is not supported for table"); + } + @Test @DisplayName("an all-UUIDv7 delete prunes to the batch's own partitions and removes the target row") void allUuidV7DeletePrunesAndRemovesTheTargetRow() { @@ -531,21 +557,58 @@ private String liveRowCount(UUID id) { * {@code TracesLocalV2CutoverTest.exchangeTables} and the wrap suite do. */ private void ensurePartitionedSuccessorUnderTraces() { - if (queryOneString(TRACES_PARTITION_KEY, _ -> { - }).contains("id_at")) { - return; // The cutover migration has landed: `traces` already is the partitioned successor. + if (tableExists("traces_local") || partitionKeyOf("traces").contains("id_at")) { + return; // Already installed, or the cutover migration has landed. } - assertThat(queryOneString(SUCCESSOR_TABLE_COUNT, _ -> { - })) + assertThat(tableExists("traces_local_v2")) .as("`traces` is not partitioned and `traces_local_v2` does not exist, so there is no successor to" + " install - this suite needs one of those two states") - .isEqualTo("1"); + .isTrue(); execute("EXCHANGE TABLES traces AND traces_local_v2 ON CLUSTER '{cluster}'", _ -> { }); execute("RENAME TABLE traces_local_v2 TO traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { }); } + /** + * Setup, not a test: applies the sharding-readiness wrap, so the DAO's mutations reach the data through the + * configuration switch that governs them in production ({@code tracesDistributedWrapEnabled}) rather than through + * a table this suite renamed under it. After this, {@code traces} is a {@code Distributed} wrapper and + * {@code traces_local} holds the partitioned data — the post-cutover end state. + *

+ * Kept identical to the wrap block of {@code 000003_exchange_and_wrap.sql}, as + * {@code TracesDistributedWrapMutationTest.applyDistributedWrap} and + * {@code TracesLocalV2CutoverTest.wrapInDistributed} do: build the wrapper under a temp name first, then one atomic + * multi-target {@code RENAME} rotates the data to {@code traces_local} and the wrapper into {@code traces}, so + * {@code traces} is never absent. Idempotent for the same reason as the step above. + */ + private void ensureDistributedWrap() { + if ("Distributed".equals(queryOneString(TABLE_ENGINE, _ -> { + }))) { + return; + } + execute(""" + CREATE TABLE traces_dist ON CLUSTER '{cluster}' AS traces + ENGINE = Distributed('{cluster}', '%s', 'traces_local', sipHash64(project_id)) + """.formatted(ClickHouseContainerUtils.DATABASE_NAME), _ -> { + }); + execute(""" + RENAME TABLE + traces TO traces_local, + traces_dist TO traces + ON CLUSTER '{cluster}' + """, _ -> { + }); + } + + private boolean tableExists(String table) { + return "1".equals(queryOneString(TABLE_COUNT, statement -> statement.bind("table", table))); + } + + private String partitionKeyOf(String table) { + return queryOneString(PARTITION_KEY_OF_TABLE, statement -> statement.bind("table", table)); + } + /** * A trace with every trace-table column populated. Only the span-derived aggregates podam would otherwise * fabricate ({@code feedbackScores}, {@code usage}) are nulled, since they are not columns of the {@code traces} From 35182c919339854ee9ee8af78f4a45101002af00 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 12:37:27 +0200 Subject: [PATCH 26/37] [OPIK-6901] [BE] test: build SelectedParts with a builder, not positionally Both components are ints, so `new SelectedParts(min, max)` compiles fine after a field-order change and silently inverts selected/total - which would invert the pruning assertion itself, the one thing that test exists to check. .agents/skills/opik-backend/SKILL.md is explicit: "Always annotate records/DTOs with @Builder(toBuilder = true)", "Use builders (not constructors) when instantiating records", and its BAD example is a positional `new`. The sibling record in TracesLocalV2PartitioningTest already carries the annotation; I left it off deliberately as unnecessary, which was wrong. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 3498450ea39..bc5aea3247f 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.redis.testcontainers.RedisContainer; import io.r2dbc.spi.Statement; +import lombok.Builder; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.AfterAll; @@ -696,8 +697,10 @@ private Optional partsSelectedBy(String daoDeleteSql) { var entry = JsonUtils.treeToValue(index, SelectedParts.class); partition = partition == null ? entry - : new SelectedParts(Math.min(partition.selected(), entry.selected()), - Math.max(partition.total(), entry.total())); + : partition.toBuilder() + .selected(Math.min(partition.selected(), entry.selected())) + .total(Math.max(partition.total(), entry.total())) + .build(); } return Optional.ofNullable(partition); } @@ -727,6 +730,7 @@ private void execute(String sql, Consumer binder) { * The part counts {@code EXPLAIN indexes = 1, json = 1} reports for one index entry: how many parts the query * started from, and how many survived pruning. */ + @Builder(toBuilder = true) @JsonIgnoreProperties(ignoreUnknown = true) private record SelectedParts( @JsonProperty("Selected Parts") int selected, From 84304a727f2b57fab534e4b839fbc7e01b87509f Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 13:15:07 +0200 Subject: [PATCH 27/37] [OPIK-6901] [BE] test: harden the topology setup I added with the wrap Two defects in b31f186, both mine, both in the setup rather than the assertions. 1. NPE before the diagnostic. queryOneString returns null when no row matches, so partitionKeyOf("traces").contains(...) threw if `traces` was absent - which is a reachable state, since a wrap interrupted between its CREATE and its RENAME leaves `traces` renamed away. The guard existed precisely to report an unsupported topology and would instead have died dereferencing it. partitionKeyOf now returns "" for a missing table, and the assertion message quotes the partition key it did find. 2. The wrap was not re-entrant. It returned early when `traces` was already Distributed, but if a run died between CREATE traces_dist and the RENAME, the leftover wrapper made the next CREATE fail on a duplicate name and buried the real state. Now drops any stranded traces_dist first - it holds no data, being a routing definition - which is the same reset TracesLocalV2CutoverTest performs. Dedicated containers mean neither state can survive into a fresh run today, so this is defence rather than a live bug. It is still worth having: the first one turns a diagnostic into an NPE, and both would only ever be hit while someone was debugging a broken setup. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index bc5aea3247f..bef9eb620df 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -562,8 +562,9 @@ private void ensurePartitionedSuccessorUnderTraces() { return; // Already installed, or the cutover migration has landed. } assertThat(tableExists("traces_local_v2")) - .as("`traces` is not partitioned and `traces_local_v2` does not exist, so there is no successor to" - + " install - this suite needs one of those two states") + .as("neither `traces_local` nor a partitioned `traces` nor `traces_local_v2` is present (partition key" + + " of `traces`: '%s') - this suite needs one of those states to install the successor from", + partitionKeyOf("traces")) .isTrue(); execute("EXCHANGE TABLES traces AND traces_local_v2 ON CLUSTER '{cluster}'", _ -> { }); @@ -581,13 +582,19 @@ private void ensurePartitionedSuccessorUnderTraces() { * {@code TracesDistributedWrapMutationTest.applyDistributedWrap} and * {@code TracesLocalV2CutoverTest.wrapInDistributed} do: build the wrapper under a temp name first, then one atomic * multi-target {@code RENAME} rotates the data to {@code traces_local} and the wrapper into {@code traces}, so - * {@code traces} is never absent. Idempotent for the same reason as the step above. + * {@code traces} is never absent. Re-entrant: it returns early when the wrap is already applied, and + * clears a wrapper stranded by an interrupted run before rebuilding it. */ private void ensureDistributedWrap() { if ("Distributed".equals(queryOneString(TABLE_ENGINE, _ -> { }))) { return; } + // Clear a wrapper left behind by a run that died between the CREATE and the RENAME. It holds no data - a + // Distributed table is a routing definition - so dropping it is safe, and without this the CREATE below fails + // on a duplicate name and buries the real state. Same reset TracesLocalV2CutoverTest performs. + execute("DROP TABLE IF EXISTS traces_dist ON CLUSTER '{cluster}' SYNC", _ -> { + }); execute(""" CREATE TABLE traces_dist ON CLUSTER '{cluster}' AS traces ENGINE = Distributed('{cluster}', '%s', 'traces_local', sipHash64(project_id)) @@ -606,8 +613,15 @@ private boolean tableExists(String table) { return "1".equals(queryOneString(TABLE_COUNT, statement -> statement.bind("table", table))); } + /** + * The table's partition-key expression, or {@code ""} when there is no such table — never {@code null}. A missing + * row is a legitimate state here (a half-applied wrap can leave {@code traces} renamed away), and the setup guard + * has to be able to report that rather than die dereferencing it, which is what would bury the diagnostic. + */ private String partitionKeyOf(String table) { - return queryOneString(PARTITION_KEY_OF_TABLE, statement -> statement.bind("table", table)); + return Optional + .ofNullable(queryOneString(PARTITION_KEY_OF_TABLE, statement -> statement.bind("table", table))) + .orElse(""); } /** From ad2631526c54486ca31e2e706eded4b00857f3d2 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 13:19:27 +0200 Subject: [PATCH 28/37] [OPIK-6901] [BE] style: indent the javadoc line I added with the wrap hardening 84304a7 appended a sentence to ensureDistributedWrap's javadoc as a continuation line with a single leading space, where every other line in that block has five. Spotless's indentation pass rewrote it, so Code Quality went red on one line. Third formatting round-trip in this review, so I added a check for the class of mistake rather than just the instance: it flags block-comment continuation lines whose '*' indent is not a legal javadoc alignment (1, 5, 9, ...). Both test suites come back clean. It has false positives on '*' inside SQL text blocks - the eight it reports in TraceDAO are column lists in SELECT statements, none of them touched by this branch - so it is a pre-push aid for changed files, not a gate. Co-Authored-By: Claude Opus 5 (1M context) --- .../opik/infrastructure/TracesPartitionPruningMutationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index bef9eb620df..eec2525d0d8 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -583,7 +583,7 @@ private void ensurePartitionedSuccessorUnderTraces() { * {@code TracesLocalV2CutoverTest.wrapInDistributed} do: build the wrapper under a temp name first, then one atomic * multi-target {@code RENAME} rotates the data to {@code traces_local} and the wrapper into {@code traces}, so * {@code traces} is never absent. Re-entrant: it returns early when the wrap is already applied, and - * clears a wrapper stranded by an interrupted run before rebuilding it. + * clears a wrapper stranded by an interrupted run before rebuilding it. */ private void ensureDistributedWrap() { if ("Distributed".equals(queryOneString(TABLE_ENGINE, _ -> { From 01a7c1e193237c973300cd3c7e70ce05253c1477 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 13:40:11 +0200 Subject: [PATCH 29/37] [OPIK-6901] [BE] test: mint the fixtures like the sibling partition suites do Read TracesLocalV2PartitioningTest / SpansLocalV2PartitioningTest properly and my data layer was the outlier: magic constants where the neighbours are arithmetic. They mint ids from a named, fixed anchor - ID_GENERATOR.generateId(instant) off ANCHOR_MONDAY plus week offsets - and get a real v7 project id straight from ID_GENERATOR.generateId(). Mine hard-coded opaque UUID literals ("00bfd451-fa93-7c10-..." with a comment claiming "id_at 1996-02-09") paired with hard-coded partition literals (19960205L). Both sides magic, the relationship between them unverifiable by a reader, and the comment the only thing asserting what the id even was. Now: three ERA_MONDAYS named as dates, ids minted mid-week from them by idInWeekOf so the assertion exercises the map back to Monday rather than identity, and expectations from partitionNameOf on the same Mondays - which formats a Monday the test already names rather than re-deriving "the Monday of an arbitrary date", the part actually under test. Fixed rather than now-derived for the reason the sibling suite documents: the math stays deterministic and cannot drift across a week boundary mid-suite. Zero UUID.fromString literals remain. The other two ids are now self-evident too: NON_V7_ID is UUID.randomUUID(), a v4 by definition, and OUT_OF_RANGE_ID is minted one instant past the DateTime64 ceiling so the boundary it sits past is visible. Dropped the endpoint round-trip from the era and EXPLAIN tests: I had added it partly to obtain a genuine v7 project id, which ID_GENERATOR.generateId() gives directly - as the sibling suites show. The endpoint stays where it earns its keep, in the live-user-path delete and the fallback test. Verified the load-bearing assumption by execution: all three ERA_MONDAYS are actual Mondays, a mid-week minted id derives back to each, and both underivable shapes are rejected. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 120 +++++++++++------- 1 file changed, 72 insertions(+), 48 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index eec2525d0d8..c7034e13727 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -12,6 +12,8 @@ import com.comet.opik.api.resources.utils.TestUtils; import com.comet.opik.api.resources.utils.WireMockUtils; import com.comet.opik.api.resources.utils.resources.TraceResourceClient; +import com.comet.opik.domain.IdGenerator; +import com.comet.opik.domain.TestIdGeneratorFactory; import com.comet.opik.domain.TraceDAO; import com.comet.opik.extensions.DropwizardAppExtensionProvider; import com.comet.opik.extensions.RegisterApp; @@ -48,6 +50,9 @@ import ru.vyarus.dropwizard.guice.test.jupiter.ext.TestDropwizardAppExtension; import uk.co.jemos.podam.api.PodamFactory; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; import java.util.List; import java.util.Map; import java.util.Optional; @@ -221,21 +226,37 @@ SELECT toString(uniqExact(id)) * far-future or at the epoch, so a recent-only batch would accept the very expression migration 000114 was written * to escape. The 2200 id is the litellm shape and the one that makes the assertion bite. */ - private static final List OUT_OF_WINDOW_ERA_IDS = List.of( - UUID.fromString("00bfd451-fa93-7c10-9923-88a219a974c8"), // id_at 1996-02-09 -> 19960205 - UUID.fromString("0699eb8a-59dd-7215-8000-03b8d2a8d5e2")); // id_at 2200-01-01 -> 21991230 + private static final IdGenerator ID_GENERATOR = TestIdGeneratorFactory.create(); - private static final Set OUT_OF_WINDOW_ERA_PARTITIONS = Set.of(19960205L, 21991230L); + /** + * The weekly partitions this suite works in, each named by its Monday — which is also the partition name, since the + * key is that Monday as {@code yyyyMMdd}. Fixed rather than {@code now}-derived, for the reason + * {@code TracesLocalV2PartitioningTest} gives for its own anchor: the partition math stays deterministic and cannot + * drift across a week boundary mid-suite. + *

+ * Ids are minted mid-week ({@link #idInWeekOf}), so the assertions exercise the map back to Monday rather + * than identity. The three eras are not interchangeable samples: {@code toMonday} agrees with the {@code Date32} + * expression across the ordinary calendar and diverges only far-future or at the epoch, so a recent-only batch would + * accept the very expression migration 000114 was written to escape. The 2199 row is the litellm shape and the one + * that makes it bite; it also covers the {@code DateTime64} half of what the flag asserts, since a 32-bit + * {@code id_at} would store it under a wrapped recent timestamp and the derived partition would miss it. + */ + private static final List ERA_MONDAYS = List.of( + LocalDate.of(1996, 2, 5), + LocalDate.of(2025, 3, 3), + LocalDate.of(2199, 12, 30)); - /** A v4 UUID: no timestamp to derive a partition from. */ - private static final UUID NON_V7_ID = UUID.fromString("9f527bac-527a-4f92-8875-0fa8af8e4f22"); + /** {@link UUID#randomUUID()} is a v4 by definition: no embedded timestamp to derive a partition from. */ + private static final UUID NON_V7_ID = UUID.randomUUID(); /** - * A UUIDv7 whose 48 timestamp bits are all set (10889-08-02), so its {@code id_at} saturates to the - * {@code DateTime64} ceiling and the honest week is not the partition the row would be in. Same rejection as - * {@link #NON_V7_ID}, different cause — see {@code WeeklyPartitions}. + * A UUIDv7 minted one second past the first instant {@code DateTime64} cannot represent, so its {@code id_at} + * saturates to the ceiling and the honest week is not the partition the row lands in. Same rejection as + * {@link #NON_V7_ID}, different cause — see {@code WeeklyPartitions}. Minted rather than written out, so the + * boundary it sits past is visible. */ - private static final UUID OUT_OF_RANGE_ID = UUID.fromString("ffffffff-ffff-7abc-8000-000000000001"); + private static final UUID OUT_OF_RANGE_ID = ID_GENERATOR + .getTimeOrderedEpoch(LocalDate.of(2300, 1, 1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli()); // Dedicated, non-reused ClickHouse + ZooKeeper on their own network: the EXCHANGE destructively swaps `traces`, so a // shared/reused container would corrupt other suites and reruns. Redis/MySQL are only read, so the shared ones are @@ -343,39 +364,31 @@ void allUuidV7DeletePrunesAndRemovesTheTargetRow() { @DisplayName("the DAO's own delete clears every era and binds exactly those partitions") void deleteClearsEveryEraAndBindsExactlyThosePartitions() { // The three-way agreement - the migration's PARTITION BY as installed, the DAO's predicate, and - // WeeklyPartitions.of - asserted through the DAO's own delete instead of by re-evaluating the expression in + // WeeklyPartitions.of - asserted through the DAO's own delete rather than by re-evaluating the expression in // test SQL. If the predicate resolved to any partition other than the one ClickHouse filed a row under, the - // mutation would select the wrong parts and that row would SURVIVE. So "every row is gone" IS the agreement, - // and it is established by the statement production actually runs rather than by a query written here. + // mutation would select the wrong parts and that row would SURVIVE. So "every row is gone" IS the agreement. // - // This also covers the DateTime64 half of what the flag asserts, without asking system.columns: against a - // 32-bit id_at the 2200 row would be stored under a wrapped recent timestamp, the derived partition would not - // match it, and the row would survive. - // - // Three eras in one batch is also the multi-value Long[] bind, which a single-id delete never reaches. - var recent = newTrace().build(); - traceResourceClient.createTrace(recent, API_KEY, WORKSPACE_NAME); - var projectId = projectIdOf(recent); - assertThat(projectId.version()).as("the project id is a real UUIDv7, as the backend mints it").isEqualTo(7); - // Only the out-of-window eras are seeded raw: ingestion rejects a 1996 or 2200 id by design (24h window), so - // there is no endpoint that can create them. The recent row above went through the real ingestion path, and - // they all share its project. - OUT_OF_WINDOW_ERA_IDS.forEach(id -> insertRawTrace(projectId, id)); - var batch = Stream.concat(Stream.of(recent.id()), OUT_OF_WINDOW_ERA_IDS.stream()).toList(); - assertThat(batch.stream().map(this::liveRowCount)).as("every era is present").containsOnly("1"); - - delete(batch.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toUnmodifiableSet())); - - assertThat(batch.stream().map(this::liveRowCount)) + // Ids are minted from the named Mondays rather than written out, so both sides of the assertion are the same + // arithmetic a reader can check, and every era in one batch is also the multi-value Long[] bind. Seeded raw and + // in a minted project: the ingestion window is 24h, so no endpoint can create a 1996 or 2199 row, and the + // project id is a real UUIDv7 straight from IdGenerator - which is how the sibling partition suites get one. + var projectId = ID_GENERATOR.generateId(); + var ids = ERA_MONDAYS.stream().map(TracesPartitionPruningMutationTest::idInWeekOf).toList(); + ids.forEach(id -> insertRawTrace(projectId, id)); + assertThat(ids.stream().map(this::liveRowCount)).as("every era is seeded").containsOnly("1"); + + delete(ids.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toUnmodifiableSet())); + + assertThat(ids.stream().map(this::liveRowCount)) .as("every era's row is gone, so the predicate named the partition each was actually filed under") .containsOnly("0"); - var sql = lastTraceDeleteSql(recent.id()); + var sql = lastTraceDeleteSql(ids.getFirst()); assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); assertThat(boundPartitionsOf(sql)) - .as("exactly the partitions the batch resolves to, not a range across three centuries") - .containsExactlyInAnyOrderElementsOf( - Stream.concat(Stream.of(partitionOf(recent.id())), OUT_OF_WINDOW_ERA_PARTITIONS.stream()) - .collect(Collectors.toUnmodifiableSet())); + .as("exactly the partitions the batch resolves to, not a range across two centuries") + .containsExactlyInAnyOrderElementsOf(ERA_MONDAYS.stream() + .map(TracesPartitionPruningMutationTest::partitionNameOf) + .collect(Collectors.toUnmodifiableSet())); } @Test @@ -395,21 +408,19 @@ void pruningReachesThePlannerAndTheFallbackDoesNot() { // DELETE and put behind a SELECT - predicate and bound partition values included. Only the verb changes; the // statement being explained is still the DAO's. Same instrument and record shape as // TracesLocalV2PartitioningTest. - // Real project from the ingestion path, and a recent row created through the endpoint; only the out-of-window - // eras are seeded raw, so the table holds several partitions to prune between. - var recent = newTrace().build(); - traceResourceClient.createTrace(recent, API_KEY, WORKSPACE_NAME); - var projectId = projectIdOf(recent); - OUT_OF_WINDOW_ERA_IDS.forEach(id -> insertRawTrace(projectId, id)); - - // Bounded: one derivable id, so the predicate names one of the partitions just populated. - delete(Set.of(Pair.of(projectId, recent.id()))); - var bounded = partsSelectedBy(lastTraceDeleteSql(recent.id())) + // One row per era, so the table holds several partitions for the planner to prune between. + var projectId = ID_GENERATOR.generateId(); + var ids = ERA_MONDAYS.stream().map(TracesPartitionPruningMutationTest::idInWeekOf).toList(); + ids.forEach(id -> insertRawTrace(projectId, id)); + + // Bounded: one derivable id, so the predicate names a single one of those partitions. + delete(Set.of(Pair.of(projectId, ids.getFirst()))); + var bounded = partsSelectedBy(lastTraceDeleteSql(ids.getFirst())) .orElseThrow(() -> new AssertionError("EXPLAIN reported no partition index for the bounded delete")); // Unbounded: a non-v7 id in the batch, so no predicate at all. Its partner is a different era, so the query_log // lookup finds this statement rather than the one above. - var partner = OUT_OF_WINDOW_ERA_IDS.getFirst(); + var partner = ids.get(1); delete(Set.of(Pair.of(projectId, partner), Pair.of(projectId, NON_V7_ID))); var unbounded = partsSelectedBy(lastTraceDeleteSql(partner)); @@ -467,6 +478,19 @@ private static Stream underivableIdDisablesPruning() { arguments("beyond-2299", OUT_OF_RANGE_ID)); } + /** A UUIDv7 in the given week, minted mid-week so the partition assertion exercises the map back to Monday. */ + private static UUID idInWeekOf(LocalDate monday) { + return ID_GENERATOR.generateId(monday.plusDays(2).atTime(12, 0).toInstant(ZoneOffset.UTC)); + } + + /** + * The partition name for a week, which is its Monday as {@code yyyyMMdd}. Formatting a Monday the test already + * names — not re-deriving "the Monday of an arbitrary date", which is the part under test. + */ + private static long partitionNameOf(LocalDate monday) { + return monday.getYear() * 10000L + monday.getMonthValue() * 100L + monday.getDayOfMonth(); + } + /** * The partition a single id resolves to. Derived through {@code WeeklyPartitions} on purpose: this suite is about * the predicate reaching ClickHouse, and the derivation's own expected values are pinned against real ClickHouse From 89ba9036e4648f8beb42471cfa3845aaa38d26d3 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 13:46:42 +0200 Subject: [PATCH 30/37] [OPIK-6901] [BE] test: scope the deletion oracle, and stop splicing SQL in the EXPLAIN helper Two reviewer findings, plus the spotless leftover from 01a7c1e. 1. LIVE_ROW_COUNT omitted project_id while TraceDAO.delete matches on (workspace_id, project_id, id), so the oracle asked a narrower question than the delete answers: a row for the same id in another project keeps the count at 1 after a successful delete, failing a test whose subject worked. Given how much of both suites rests on "the row is gone", that is a wrong oracle rather than a tidiness issue. Predicate added and project_id bound at every call site in both suites - the sibling one matters more, since it runs on shared containers. 2. partsSelectedBy built its SELECT and EXPLAIN with .formatted(), splicing two regex groups lifted out of query_log into executable SQL. The rule names that construct explicitly ("No +, no String.format / .formatted(...)"), and its table gives the way out: fragments through StringTemplate, values bound. So the EXPLAIN is now a declared text block with and as fragments and :partitions bound. Both constraints turn out to be satisfiable at once, so the instruction to lift the predicate from the DAO rather than re-author it does not need revisiting. The predicate fragment is PARTITION_PREDICATE, and using the constant is not re-authoring: every caller has already asserted the emitted statement CONTAINS that exact text, so it is pinned to the DAO's SQL by assertion instead of by string surgery on it - which is the stronger link, since the regex silently accepted whatever it captured. The partition values still come from the emitted statement, parsed and then bound. The DAO's workspace_id and (project_id, id) predicates are deliberately not reproduced: they are sort-key filters, cannot change partition selection, and omitting them makes the unbounded case a full scan, which is the conservative direction for asserting the fallback prunes nothing. Also converted the wrap DDL's .formatted(DATABASE_NAME) to a StringTemplate fragment. The sibling suites still splice it, but the rule says not to add new ones and I added that one two commits ago. No SQL in either suite is built by string operations now. Removed the unused java.time.Instant import spotless flagged, and added an unused-import check to the pre-push battery - third distinct formatting class to bite me, and javac cannot see it here because most types are unresolvable offline. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 86 +++++++++++++++---- .../TracesPruningDisabledMutationTest.java | 13 ++- 2 files changed, 81 insertions(+), 18 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index c7034e13727..280fb55a3fe 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -22,6 +22,7 @@ import com.comet.opik.podam.PodamFactoryUtils; import com.comet.opik.utils.JsonUtils; import com.comet.opik.utils.WeeklyPartitions; +import com.comet.opik.utils.template.TemplateUtils; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; @@ -50,7 +51,6 @@ import ru.vyarus.dropwizard.guice.test.jupiter.ext.TestDropwizardAppExtension; import uk.co.jemos.podam.api.PodamFactory; -import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; import java.util.List; @@ -190,10 +190,16 @@ AND query LIKE concat('%', :trace_id, '%') LIMIT 1 """; + /** + * Scoped by the same key {@code TraceDAO.delete} matches on — {@code (workspace_id, project_id, id)}. An oracle + * narrower than the delete would answer a different question: a row for the same id in another project would keep + * the count at {@code 1} after a successful delete, failing a test whose subject worked. + */ private static final String LIVE_ROW_COUNT = """ SELECT toString(uniqExact(id)) FROM traces_local WHERE workspace_id = :workspace_id + AND project_id = :project_id AND id = :id """; @@ -217,6 +223,40 @@ SELECT toString(uniqExact(id)) /** {@code EXPLAIN} index entries that reflect partition-level part selection. */ private static final Set PARTITION_INDEX_TYPES = Set.of("MinMax", "Partition"); + /** + * Asks the planner how many parts the DAO's partition predicate selects. Declared once as a text block, per + * {@code .agents/skills/opik-backend/SKILL.md}: the table {@code
} and the predicate + * {@code } are fragments and go through {@link TemplateUtils#newST}, the partition + * values are values and are bound — nothing is spliced with {@code .formatted(...)}. + *

+ * The predicate fragment is {@link #PARTITION_PREDICATE}, and using the constant does not re-author what is under + * test: every caller has already asserted the emitted statement contains that exact text, so the constant + * is pinned to the DAO's own SQL by assertion rather than by string surgery on it. The partition values come from + * the emitted statement too, parsed by {@link #boundPartitionsOf} and bound here. + *

+ * The DAO's {@code workspace_id} and {@code (project_id, id)} predicates are deliberately not reproduced. They are + * sort-key filters, not partition filters, so they cannot change partition selection — and leaving them out makes + * the unbounded case a full scan, which is the conservative direction for an assertion that the fallback prunes + * nothing. + */ + /** + * The wrap block of {@code 000003_exchange_and_wrap.sql}. The database name is a fragment (an identifier + * inside a function argument, not a bindable value), so it goes through {@link TemplateUtils#newST} rather than + * {@code .formatted(...)} — the sibling suites still splice it, but the rule says not to add new ones. The + * {@code {cluster}} macros are ClickHouse's own and pass through StringTemplate untouched, which uses {@code <>}. + */ + private static final String CREATE_DISTRIBUTED_WRAPPER = """ + CREATE TABLE traces_dist ON CLUSTER '{cluster}' AS traces + ENGINE = Distributed('{cluster}', '', 'traces_local', sipHash64(project_id)) + """; + + private static final String EXPLAIN_SELECTED_PARTS = """ + EXPLAIN indexes = 1, json = 1 + SELECT id + FROM

+ WHERE IN :partitions + """; + /** A weekly partition value as it appears in SQL — {@code yyyyMMdd}, so always eight digits. */ private static final Pattern PARTITION_VALUE = Pattern.compile("\\d{8}"); @@ -375,11 +415,12 @@ void deleteClearsEveryEraAndBindsExactlyThosePartitions() { var projectId = ID_GENERATOR.generateId(); var ids = ERA_MONDAYS.stream().map(TracesPartitionPruningMutationTest::idInWeekOf).toList(); ids.forEach(id -> insertRawTrace(projectId, id)); - assertThat(ids.stream().map(this::liveRowCount)).as("every era is seeded").containsOnly("1"); + assertThat(ids.stream().map(id -> liveRowCount(projectId, id))) + .as("every era is seeded").containsOnly("1"); delete(ids.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toUnmodifiableSet())); - assertThat(ids.stream().map(this::liveRowCount)) + assertThat(ids.stream().map(id -> liveRowCount(projectId, id))) .as("every era's row is gone, so the predicate named the partition each was actually filed under") .containsOnly("0"); var sql = lastTraceDeleteSql(ids.getFirst()); @@ -448,11 +489,12 @@ void underivableIdDisablesPruning(String cause, UUID underivableId) { traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); var projectId = projectIdOf(target); insertRawTrace(projectId, underivableId); - assertThat(liveRowCount(underivableId)).as("the %s row is seeded before the delete", cause).isEqualTo("1"); + assertThat(liveRowCount(projectId, underivableId)) + .as("the %s row is seeded before the delete", cause).isEqualTo("1"); delete(Set.of(Pair.of(projectId, target.id()), Pair.of(projectId, underivableId))); - assertThat(liveRowCount(underivableId)) + assertThat(liveRowCount(projectId, underivableId)) .as("the %s row is itself deleted, not skipped", cause) .isEqualTo("0"); assertThat(traceIdsOf(target.projectName())) @@ -558,8 +600,9 @@ private String lastTraceDeleteSql(UUID traceId) { } /** {@code "1"} while a live (non-lightweight-deleted) row exists for the id, {@code "0"} once it is gone. */ - private String liveRowCount(UUID id) { + private String liveRowCount(UUID projectId, UUID id) { return queryOneString(LIVE_ROW_COUNT, statement -> statement + .bind("project_id", projectId.toString()) .bind("workspace_id", WORKSPACE_ID) .bind("id", id.toString())); } @@ -619,11 +662,10 @@ private void ensureDistributedWrap() { // on a duplicate name and buries the real state. Same reset TracesLocalV2CutoverTest performs. execute("DROP TABLE IF EXISTS traces_dist ON CLUSTER '{cluster}' SYNC", _ -> { }); - execute(""" - CREATE TABLE traces_dist ON CLUSTER '{cluster}' AS traces - ENGINE = Distributed('{cluster}', '%s', 'traces_local', sipHash64(project_id)) - """.formatted(ClickHouseContainerUtils.DATABASE_NAME), _ -> { - }); + execute(TemplateUtils.newST(CREATE_DISTRIBUTED_WRAPPER) + .add("database", ClickHouseContainerUtils.DATABASE_NAME) + .render(), _ -> { + }); execute(""" RENAME TABLE traces TO traces_local, @@ -714,11 +756,25 @@ private Optional partsSelectedBy(String daoDeleteSql) { assertThat(shape.find()) .as("the emitted statement has the expected DELETE shape:%n%s", daoDeleteSql) .isTrue(); - var selectSql = "SELECT id FROM %s %s".formatted(shape.group(1), shape.group(2)); + var bound = EMITTED_IN_CLAUSE.matcher(daoDeleteSql).find() + ? boundPartitionsOf(daoDeleteSql) + : Set. of(); + + var explainSql = TemplateUtils.newST(EXPLAIN_SELECTED_PARTS) + .add("table", shape.group(1)); + if (!bound.isEmpty()) { + explainSql.add("partition_expression", PARTITION_PREDICATE); + } + var sql = explainSql.render(); - var explainRows = template.stream(connection -> Flux - .from(connection.createStatement("EXPLAIN indexes = 1, json = 1 %s".formatted(selectSql)).execute()) - .flatMap(result -> result.map((row, _) -> row.get("explain", String.class)))) + var explainRows = template.stream(connection -> { + var statement = connection.createStatement(sql); + if (!bound.isEmpty()) { + statement.bind("partitions", bound.toArray(Long[]::new)); + } + return Flux.from(statement.execute()) + .flatMap(result -> result.map((row, _) -> row.get("explain", String.class))); + }) .collectList() .block(); var explain = String.join("\n", explainRows); diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPruningDisabledMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPruningDisabledMutationTest.java index 151081a3a9d..736e2c50726 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPruningDisabledMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPruningDisabledMutationTest.java @@ -85,10 +85,16 @@ INSERT INTO traces (workspace_id, project_id, id) VALUES (:workspace_id, :project_id, :id) """; + /** + * Scoped by the same key {@code TraceDAO.delete} matches on — {@code (workspace_id, project_id, id)}. An oracle + * narrower than the delete answers a different question, and this suite runs on shared containers where other + * suites' rows are present, so the scoping is what keeps the count about this test's row. + */ private static final String LIVE_ROW_COUNT = """ SELECT toString(uniqExact(id)) FROM traces WHERE workspace_id = :workspace_id + AND project_id = :project_id AND id = :id """; @@ -186,11 +192,11 @@ void farFutureRowOnLegacyTableIsStillDeleted() { // nothing here anyway: the legacy id_at is accurate for one, so even a wrongly-emitted predicate would match it // and the row would still go. insertRawTrace(projectId, FAR_FUTURE_ID); - assertThat(liveRowCount(FAR_FUTURE_ID)).as("the far-future row is seeded").isEqualTo("1"); + assertThat(liveRowCount(projectId, FAR_FUTURE_ID)).as("the far-future row is seeded").isEqualTo("1"); delete(Set.of(Pair.of(projectId, FAR_FUTURE_ID))); - assertThat(liveRowCount(FAR_FUTURE_ID)) + assertThat(liveRowCount(projectId, FAR_FUTURE_ID)) .as("it is deleted - a partition predicate here would have matched nothing and reported success") .isEqualTo("0"); @@ -236,9 +242,10 @@ private UUID projectIdOf(Trace trace) { } /** {@code "1"} while a live (non-lightweight-deleted) row exists for the id, {@code "0"} once it is gone. */ - private String liveRowCount(UUID id) { + private String liveRowCount(UUID projectId, UUID id) { return queryOneString(LIVE_ROW_COUNT, statement -> statement .bind("workspace_id", WORKSPACE_ID) + .bind("project_id", projectId.toString()) .bind("id", id.toString())); } From d4d1036de679d469c4c3b50e135cbaeba21dcf1a Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 13:49:15 +0200 Subject: [PATCH 31/37] [OPIK-6901] [BE] style: drop the explicit type witness spotless reformats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Set. of()` — spotless wants no space after the witness. Declaring the variable's type instead removes the need for the witness at all, so there is nothing left to disagree about: Set bound = ... ? boundPartitionsOf(daoDeleteSql) : Set.of(); Fourth formatting round-trip. My local battery now covers line length, javadoc continuation indent and unused imports, but not intra-expression spacing, which needs the real formatter — and mvn cannot read this POM offline, so CI stays the only authority for that class. Co-Authored-By: Claude Opus 5 (1M context) --- .../infrastructure/TracesPartitionPruningMutationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 280fb55a3fe..1592496a152 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -756,9 +756,9 @@ private Optional partsSelectedBy(String daoDeleteSql) { assertThat(shape.find()) .as("the emitted statement has the expected DELETE shape:%n%s", daoDeleteSql) .isTrue(); - var bound = EMITTED_IN_CLAUSE.matcher(daoDeleteSql).find() + Set bound = EMITTED_IN_CLAUSE.matcher(daoDeleteSql).find() ? boundPartitionsOf(daoDeleteSql) - : Set. of(); + : Set.of(); var explainSql = TemplateUtils.newST(EXPLAIN_SELECTED_PARTS) .add("table", shape.group(1)); From cb977650862cc7bba489a053b92f4ae049558766 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 14:09:44 +0200 Subject: [PATCH 32/37] [OPIK-6901] [BE] docs: put the planner javadoc back on the constant it documents Inserting CREATE_DISTRIBUTED_WRAPPER in 89ba903 landed it between EXPLAIN_SELECTED_PARTS's javadoc and the constant itself, so Java bound that javadoc to the wrapper - which already had its own - and left EXPLAIN_SELECTED_PARTS undocumented. The fragment/bind contract, the argument that using PARTITION_PREDICATE is not re-authoring, and the note on the deliberately omitted sort-key predicates were all still in the file, attached to the wrong declaration. Moved the wrapper block after EXPLAIN_SELECTED_PARTS so each javadoc sits immediately before the constant it describes. No text changed. Added a check for the class: a javadoc whose next non-blank line opens another javadoc documents nothing. Fifth distinct formatting/structure class this review, and the one most likely to pass every other check I have - spotless does not reflow comments, and javac does not care which declaration a javadoc lands on. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 1592496a152..e2398e04a64 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -239,6 +239,13 @@ SELECT toString(uniqExact(id)) * the unbounded case a full scan, which is the conservative direction for an assertion that the fallback prunes * nothing. */ + private static final String EXPLAIN_SELECTED_PARTS = """ + EXPLAIN indexes = 1, json = 1 + SELECT id + FROM
+ WHERE IN :partitions + """; + /** * The wrap block of {@code 000003_exchange_and_wrap.sql}. The database name is a fragment (an identifier * inside a function argument, not a bindable value), so it goes through {@link TemplateUtils#newST} rather than @@ -250,13 +257,6 @@ SELECT toString(uniqExact(id)) ENGINE = Distributed('{cluster}', '', 'traces_local', sipHash64(project_id)) """; - private static final String EXPLAIN_SELECTED_PARTS = """ - EXPLAIN indexes = 1, json = 1 - SELECT id - FROM
- WHERE IN :partitions - """; - /** A weekly partition value as it appears in SQL — {@code yyyyMMdd}, so always eight digits. */ private static final Pattern PARTITION_VALUE = Pattern.compile("\\d{8}"); From 0ac5916a923e2829324d83f627c1c2351eb9bc41 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 14:53:32 +0200 Subject: [PATCH 33/37] [OPIK-6901] [BE] test: run the pruning suite with the flag off as well as on The suite only ever ran with pruning enabled, so nothing in it isolated the flag as the cause of the pruning. Removal was caught - the emitted-SQL and EXPLAIN assertions fail without the predicate - but removing the flag GATE, so pruning always happens, was only caught in a different suite that also changes the schema, and so could not attribute anything to the flag alone. Restructured into two @Nested classes sharing the outer class's containers and topology, following HealthCheckIntegrationTest's pattern (one app per nested class, one @RegisterApp each). tracesWeeklyPartitionPruningEnabled is the ONLY thing that differs between them; traceColumnsNonNullable and tracesDistributedWrapEnabled are fixed on in both, as production runs them post-cutover. * PruningEnabled - the existing five tests. * PruningDisabled - the same fixtures, same post-cutover topology: no partition predicate and no id_at narrowing is emitted, and the planner selects every part. Both are the inverse of an assertion in the enabled class, on identical data. So the pair is a control rather than two unrelated suites: delete the pruning and PruningEnabled fails; delete the flag gate and PruningDisabled fails. Correctness is unaffected either way - the rows go away in both, which is what makes it an optimisation - so only these assertions can tell the two states apart. Topology setup and every raw read now run through a container-derived TransactionTemplateAsync rather than an app-injected one: the topology has to be installed once, before either app boots. That is also the sibling partition suites' idiom. Added a type-import check to the pre-push battery. The restructure introduced a field whose type had no import - a compile error that offline javac cannot distinguish from ordinary unresolvable-dependency noise, and which my existing checks missed. It needed text-block stripping to be usable at all, since SQL keywords read as type names otherwise. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 452 +++++++++++------- 1 file changed, 276 insertions(+), 176 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index e2398e04a64..2e5f665135e 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -34,6 +34,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; @@ -130,7 +131,6 @@ * container would corrupt other suites and reruns. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@ExtendWith(DropwizardAppExtensionProvider.class) class TracesPartitionPruningMutationTest { private static final String API_KEY = "apiKey-" + UUID.randomUUID(); @@ -313,47 +313,64 @@ SELECT toString(uniqExact(id)) private final PodamFactory factory = PodamFactoryUtils.newPodamFactory(); - @RegisterApp - private final TestDropwizardAppExtension app; + /** + * Runs the topology setup and every raw read straight against the container, with no app in the way — the idiom the + * sibling partition suites use. It has to be app-independent: the topology must be installed once, before either + * nested app boots, and both nested classes then read through the same handle. + */ + private final TransactionTemplateAsync template; { Startables.deepStart(redisContainer, mysqlContainer, clickHouseContainer, zookeeperContainer) .join(); wireMock = WireMockUtils.startWireMock(); - var databaseAnalyticsFactory = ClickHouseContainerUtils.newDatabaseAnalyticsFactory( - clickHouseContainer, ClickHouseContainerUtils.DATABASE_NAME); MigrationUtils.runMysqlDbMigration(mysqlContainer); MigrationUtils.runClickhouseDbMigration(clickHouseContainer); - app = TestDropwizardAppExtensionUtils.newTestDropwizardAppExtension( + template = TransactionTemplateAsync.create(ClickHouseContainerUtils + .newDatabaseAnalyticsFactory(clickHouseContainer, ClickHouseContainerUtils.DATABASE_NAME) + .build()); + ensurePartitionedSuccessorUnderTraces(); + ensureDistributedWrap(); + } + + private TraceResourceClient traceResourceClient; + private TraceDAO traceDAO; + + /** + * One app per flag state, identical in every other respect. {@code tracesWeeklyPartitionPruningEnabled} is the only + * thing that varies between the two nested classes, which is what lets them be read as an A/B: same schema, same + * topology, same fixtures, one flag. + *

+ * The other two flags are fixed on, as production runs them post-cutover — the successor's {@code end_time}/ + * {@code ttft} are non-nullable sentinel columns, and the wrap is what points the DAO's mutations at + * {@code traces_local}. + */ + private TestDropwizardAppExtension newApp(boolean pruningEnabled) { + return TestDropwizardAppExtensionUtils.newTestDropwizardAppExtension( AppContextConfig.builder() .jdbcUrl(mysqlContainer.getJdbcUrl()) - .databaseAnalyticsFactory(databaseAnalyticsFactory) + .databaseAnalyticsFactory(ClickHouseContainerUtils.newDatabaseAnalyticsFactory( + clickHouseContainer, ClickHouseContainerUtils.DATABASE_NAME)) .redisUrl(redisContainer.getRedisURI()) .runtimeInfo(wireMock.runtimeInfo()) - // Both flags as production runs them post-EXCHANGE: the successor's end_time/ttft are - // non-nullable sentinel columns, and the pruning flag asserts the schema this suite installs. .customConfigs(List.of( new CustomConfig("databaseAnalyticsDataModel.traceColumnsNonNullable", "true"), + new CustomConfig("databaseAnalyticsDataModel.tracesDistributedWrapEnabled", "true"), new CustomConfig("databaseAnalyticsDataModel.tracesWeeklyPartitionPruningEnabled", - "true"), - new CustomConfig("databaseAnalyticsDataModel.tracesDistributedWrapEnabled", "true"))) + String.valueOf(pruningEnabled)))) .build()); } - private TraceResourceClient traceResourceClient; - private TransactionTemplateAsync template; - private TraceDAO traceDAO; - - @BeforeAll - void beforeAll(ClientSupport clientSupport, TransactionTemplateAsync template, TraceDAO traceDAO) { + /** + * Wires the currently-running nested class's app in. Safe on shared outer fields because JUnit runs nested + * containers one at a time — this tree configures no parallel execution — so only one app is live at a time. + */ + private void bindApp(ClientSupport clientSupport, TraceDAO traceDAO) { var baseUrl = TestUtils.getBaseUrl(clientSupport); ClientSupportUtils.config(clientSupport); mockTargetWorkspace(wireMock.server(), API_KEY, WORKSPACE_NAME, WORKSPACE_ID, USER); - traceResourceClient = new TraceResourceClient(clientSupport, baseUrl); - this.template = template; + this.traceResourceClient = new TraceResourceClient(clientSupport, baseUrl); this.traceDAO = traceDAO; - ensurePartitionedSuccessorUnderTraces(); - ensureDistributedWrap(); } @AfterAll @@ -364,161 +381,6 @@ void afterAll() { network.close(); } - @Test - @DisplayName("traces really is a mutation-rejecting Distributed wrapper, so the deletes here ran on traces_local") - void distributedTracesRejectsDirectMutation() { - // Keeps the both-flags claim from being vacuous. If the wrap had not taken effect, `traces` would still be a - // MergeTree, every pruned delete in this suite would have run against it, and nothing here would say so. - // Asserting the specific rejection - not merely that something threw - is what proves `traces` is Distributed, - // so a green delete could only have reached `traces_local`. - assertThatThrownBy(() -> execute("DELETE FROM traces WHERE workspace_id = :workspace_id", - statement -> statement.bind("workspace_id", WORKSPACE_ID))) - .hasMessageContaining("DELETE query is not supported for table"); - } - - @Test - @DisplayName("an all-UUIDv7 delete prunes to the batch's own partitions and removes the target row") - void allUuidV7DeletePrunesAndRemovesTheTargetRow() { - var target = newTrace().build(); - // Same project, so one read shows both: the pruned delete must take the target and leave this one. - var bystander = newTrace().projectName(target.projectName()).build(); - traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); - traceResourceClient.createTrace(bystander, API_KEY, WORKSPACE_NAME); - assertThat(traceIdsOf(target.projectName())).contains(target.id(), bystander.id()); - - // The live user path, end to end. - traceResourceClient.deleteTrace(target.id(), WORKSPACE_NAME, API_KEY); - - assertThat(traceIdsOf(target.projectName())) - .as("only the target row is gone") - .doesNotContain(target.id()) - .contains(bystander.id()); - var sql = lastTraceDeleteSql(target.id()); - assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); - assertThat(boundPartitionsOf(sql)) - .as("bounded to exactly the target's own partition, nothing wider") - .containsExactly(partitionOf(target.id())); - } - - @Test - @DisplayName("the DAO's own delete clears every era and binds exactly those partitions") - void deleteClearsEveryEraAndBindsExactlyThosePartitions() { - // The three-way agreement - the migration's PARTITION BY as installed, the DAO's predicate, and - // WeeklyPartitions.of - asserted through the DAO's own delete rather than by re-evaluating the expression in - // test SQL. If the predicate resolved to any partition other than the one ClickHouse filed a row under, the - // mutation would select the wrong parts and that row would SURVIVE. So "every row is gone" IS the agreement. - // - // Ids are minted from the named Mondays rather than written out, so both sides of the assertion are the same - // arithmetic a reader can check, and every era in one batch is also the multi-value Long[] bind. Seeded raw and - // in a minted project: the ingestion window is 24h, so no endpoint can create a 1996 or 2199 row, and the - // project id is a real UUIDv7 straight from IdGenerator - which is how the sibling partition suites get one. - var projectId = ID_GENERATOR.generateId(); - var ids = ERA_MONDAYS.stream().map(TracesPartitionPruningMutationTest::idInWeekOf).toList(); - ids.forEach(id -> insertRawTrace(projectId, id)); - assertThat(ids.stream().map(id -> liveRowCount(projectId, id))) - .as("every era is seeded").containsOnly("1"); - - delete(ids.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toUnmodifiableSet())); - - assertThat(ids.stream().map(id -> liveRowCount(projectId, id))) - .as("every era's row is gone, so the predicate named the partition each was actually filed under") - .containsOnly("0"); - var sql = lastTraceDeleteSql(ids.getFirst()); - assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); - assertThat(boundPartitionsOf(sql)) - .as("exactly the partitions the batch resolves to, not a range across two centuries") - .containsExactlyInAnyOrderElementsOf(ERA_MONDAYS.stream() - .map(TracesPartitionPruningMutationTest::partitionNameOf) - .collect(Collectors.toUnmodifiableSet())); - } - - @Test - @DisplayName("the planner actually prunes, and the fallback provably does not") - void pruningReachesThePlannerAndTheFallbackDoesNot() { - // Correctness and pruning are different claims, and this is the only test that makes the second one. Deletes - // were already correct before OPIK-6901 - what the change buys is parts touched (3,928/3,928 -> 5/3,928 on - // prod-test), so a suite that cannot see pruning stop does not test what this change exists to do. - // - // The regression it guards is specific: a migration rewrites the partition expression to something semantically - // identical but textually different, the planner stops recognising the DAO's predicate as the partition key, - // pruning silently stops - and values still agree, so every row is still deleted and every other assertion in - // this suite stays green. That is the property the removed AST pin covered; this asks the planner directly - // instead of inferring it from text. - // - // EXPLAIN does not accept a mutation, so the WHERE clause is lifted verbatim out of the DAO's own emitted - // DELETE and put behind a SELECT - predicate and bound partition values included. Only the verb changes; the - // statement being explained is still the DAO's. Same instrument and record shape as - // TracesLocalV2PartitioningTest. - // One row per era, so the table holds several partitions for the planner to prune between. - var projectId = ID_GENERATOR.generateId(); - var ids = ERA_MONDAYS.stream().map(TracesPartitionPruningMutationTest::idInWeekOf).toList(); - ids.forEach(id -> insertRawTrace(projectId, id)); - - // Bounded: one derivable id, so the predicate names a single one of those partitions. - delete(Set.of(Pair.of(projectId, ids.getFirst()))); - var bounded = partsSelectedBy(lastTraceDeleteSql(ids.getFirst())) - .orElseThrow(() -> new AssertionError("EXPLAIN reported no partition index for the bounded delete")); - - // Unbounded: a non-v7 id in the batch, so no predicate at all. Its partner is a different era, so the query_log - // lookup finds this statement rather than the one above. - var partner = ids.get(1); - delete(Set.of(Pair.of(projectId, partner), Pair.of(projectId, NON_V7_ID))); - var unbounded = partsSelectedBy(lastTraceDeleteSql(partner)); - - assertThat(bounded.selected()) - .as("the bounded delete selects fewer parts than the table holds: %s", bounded) - .isLessThan(bounded.total()); - // Shown to discriminate, or it proves nothing - the same trap as `.contains(partition)` and - // `doesNotContain("toDayOfWeek")`. The fallback must not prune: either the planner reports no partition index at - // all, because nothing filters on the key, or it reports every part still selected. - assertThat(unbounded.map(parts -> parts.selected() == parts.total()).orElse(true)) - .as("the fallback prunes nothing: %s", unbounded) - .isTrue(); - } - - @ParameterizedTest - @MethodSource - @DisplayName("an id with no derivable partition disables pruning for the batch, and the delete still lands") - void underivableIdDisablesPruning(String cause, UUID underivableId) { - // The fallback that preserves the pre-OPIK-6901 guarantee, as the original javadoc stated it: a row whose id_at - // cannot be trusted is STILL DELETED. That is a claim about the underivable row ITSELF, so it gets a real row - // here - seeded raw, since ingestion rejects both id shapes by design. Passing it as an id matching nothing - // would let an implementation that quietly drops underivable ids from the batch pass, which is the very bug the - // all-or-nothing rule exists to prevent. - var target = newTrace().build(); - traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); - var projectId = projectIdOf(target); - insertRawTrace(projectId, underivableId); - assertThat(liveRowCount(projectId, underivableId)) - .as("the %s row is seeded before the delete", cause).isEqualTo("1"); - - delete(Set.of(Pair.of(projectId, target.id()), Pair.of(projectId, underivableId))); - - assertThat(liveRowCount(projectId, underivableId)) - .as("the %s row is itself deleted, not skipped", cause) - .isEqualTo("0"); - assertThat(traceIdsOf(target.projectName())) - .as("and the derivable row batched alongside the %s id goes too", cause) - .doesNotContain(target.id()); - - // Asserted as the absence of ANY id_at predicate, not just of this PR's expression. A regression that narrowed - // the mutation with toMonday(id_at), an id_at range, or any other partition predicate would skip exactly the - // rows this fallback exists to reach, and rejecting one function name would not see it. The unbounded template - // mentions id_at nowhere at all, so that is the whole check. - var sql = lastTraceDeleteSql(target.id()); - assertThat(sql) - .as("the unbounded form for a %s batch carries no id_at predicate of any kind", cause) - .doesNotContain("id_at"); - assertThat(EMITTED_IN_CLAUSE.matcher(sql).find()) - .as("and no partition IN clause: %s", sql) - .isFalse(); - } - - private static Stream underivableIdDisablesPruning() { - return Stream.of( - arguments("non-v7", NON_V7_ID), - arguments("beyond-2299", OUT_OF_RANGE_ID)); - } /** A UUIDv7 in the given week, minted mid-week so the partition assertion exercises the map back to Monday. */ private static UUID idInWeekOf(LocalDate monday) { @@ -830,4 +692,242 @@ private record SelectedParts( @JsonProperty("Selected Parts") int selected, @JsonProperty("Initial Parts") int total) { } + + /** + * The flag on: the pruning this change exists to add. Every assertion here would also hold with the feature + * deleted except the ones about the emitted SQL and the planner — which is exactly why those exist, and why + * {@link PruningDisabled} runs the same fixtures with the flag off. Read as a pair, the two classes pin the flag as + * the cause: remove the pruning and this class fails; remove the flag gate and the other one does. + */ + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + @ExtendWith(DropwizardAppExtensionProvider.class) + class PruningEnabled { + + @RegisterApp + private final TestDropwizardAppExtension app = newApp(true); + + @BeforeAll + void beforeAll(ClientSupport clientSupport, TraceDAO traceDAO) { + bindApp(clientSupport, traceDAO); + } + + @Test + @DisplayName("traces is a mutation-rejecting Distributed wrapper, so these deletes ran on traces_local") + void distributedTracesRejectsDirectMutation() { + // Keeps the both-flags claim from being vacuous. If the wrap had not taken effect, `traces` would still be a + // MergeTree, every pruned delete in this suite would have run against it, and nothing here would say so. + // Asserting the specific rejection - not merely that something threw - is what proves `traces` is Distributed, + // so a green delete could only have reached `traces_local`. + assertThatThrownBy(() -> execute("DELETE FROM traces WHERE workspace_id = :workspace_id", + statement -> statement.bind("workspace_id", WORKSPACE_ID))) + .hasMessageContaining("DELETE query is not supported for table"); + } + + @Test + @DisplayName("an all-UUIDv7 delete prunes to the batch's own partitions and removes the target row") + void allUuidV7DeletePrunesAndRemovesTheTargetRow() { + var target = newTrace().build(); + // Same project, so one read shows both: the pruned delete must take the target and leave this one. + var bystander = newTrace().projectName(target.projectName()).build(); + traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); + traceResourceClient.createTrace(bystander, API_KEY, WORKSPACE_NAME); + assertThat(traceIdsOf(target.projectName())).contains(target.id(), bystander.id()); + + // The live user path, end to end. + traceResourceClient.deleteTrace(target.id(), WORKSPACE_NAME, API_KEY); + + assertThat(traceIdsOf(target.projectName())) + .as("only the target row is gone") + .doesNotContain(target.id()) + .contains(bystander.id()); + var sql = lastTraceDeleteSql(target.id()); + assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); + assertThat(boundPartitionsOf(sql)) + .as("bounded to exactly the target's own partition, nothing wider") + .containsExactly(partitionOf(target.id())); + } + + @Test + @DisplayName("the DAO's own delete clears every era and binds exactly those partitions") + void deleteClearsEveryEraAndBindsExactlyThosePartitions() { + // The three-way agreement - the migration's PARTITION BY as installed, the DAO's predicate, and + // WeeklyPartitions.of - asserted through the DAO's own delete rather than by re-evaluating the expression in + // test SQL. If the predicate resolved to any partition other than the one ClickHouse filed a row under, the + // mutation would select the wrong parts and that row would SURVIVE. So "every row is gone" IS the agreement. + // + // Ids are minted from the named Mondays rather than written out, so both sides of the assertion are the same + // arithmetic a reader can check, and every era in one batch is also the multi-value Long[] bind. Seeded raw and + // in a minted project: the ingestion window is 24h, so no endpoint can create a 1996 or 2199 row, and the + // project id is a real UUIDv7 straight from IdGenerator - which is how the sibling partition suites get one. + var projectId = ID_GENERATOR.generateId(); + var ids = ERA_MONDAYS.stream().map(TracesPartitionPruningMutationTest::idInWeekOf).toList(); + ids.forEach(id -> insertRawTrace(projectId, id)); + assertThat(ids.stream().map(id -> liveRowCount(projectId, id))) + .as("every era is seeded").containsOnly("1"); + + delete(ids.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toUnmodifiableSet())); + + assertThat(ids.stream().map(id -> liveRowCount(projectId, id))) + .as("every era's row is gone, so the predicate named the partition each was actually filed under") + .containsOnly("0"); + var sql = lastTraceDeleteSql(ids.getFirst()); + assertThat(sql).as("the mutation carries the partition predicate").contains(PARTITION_PREDICATE); + assertThat(boundPartitionsOf(sql)) + .as("exactly the partitions the batch resolves to, not a range across two centuries") + .containsExactlyInAnyOrderElementsOf(ERA_MONDAYS.stream() + .map(TracesPartitionPruningMutationTest::partitionNameOf) + .collect(Collectors.toUnmodifiableSet())); + } + + @Test + @DisplayName("the planner actually prunes, and the fallback provably does not") + void pruningReachesThePlannerAndTheFallbackDoesNot() { + // Correctness and pruning are different claims, and this is the only test that makes the second one. Deletes + // were already correct before OPIK-6901 - what the change buys is parts touched (3,928/3,928 -> 5/3,928 on + // prod-test), so a suite that cannot see pruning stop does not test what this change exists to do. + // + // The regression it guards is specific: a migration rewrites the partition expression to something semantically + // identical but textually different, the planner stops recognising the DAO's predicate as the partition key, + // pruning silently stops - and values still agree, so every row is still deleted and every other assertion in + // this suite stays green. That is the property the removed AST pin covered; this asks the planner directly + // instead of inferring it from text. + // + // EXPLAIN does not accept a mutation, so the WHERE clause is lifted verbatim out of the DAO's own emitted + // DELETE and put behind a SELECT - predicate and bound partition values included. Only the verb changes; the + // statement being explained is still the DAO's. Same instrument and record shape as + // TracesLocalV2PartitioningTest. + // One row per era, so the table holds several partitions for the planner to prune between. + var projectId = ID_GENERATOR.generateId(); + var ids = ERA_MONDAYS.stream().map(TracesPartitionPruningMutationTest::idInWeekOf).toList(); + ids.forEach(id -> insertRawTrace(projectId, id)); + + // Bounded: one derivable id, so the predicate names a single one of those partitions. + delete(Set.of(Pair.of(projectId, ids.getFirst()))); + var bounded = partsSelectedBy(lastTraceDeleteSql(ids.getFirst())) + .orElseThrow(() -> new AssertionError( + "EXPLAIN reported no partition index for the bounded delete")); + + // Unbounded: a non-v7 id in the batch, so no predicate at all. Its partner is a different era, so the query_log + // lookup finds this statement rather than the one above. + var partner = ids.get(1); + delete(Set.of(Pair.of(projectId, partner), Pair.of(projectId, NON_V7_ID))); + var unbounded = partsSelectedBy(lastTraceDeleteSql(partner)); + + assertThat(bounded.selected()) + .as("the bounded delete selects fewer parts than the table holds: %s", bounded) + .isLessThan(bounded.total()); + // Shown to discriminate, or it proves nothing - the same trap as `.contains(partition)` and + // `doesNotContain("toDayOfWeek")`. The fallback must not prune: either the planner reports no partition index at + // all, because nothing filters on the key, or it reports every part still selected. + assertThat(unbounded.map(parts -> parts.selected() == parts.total()).orElse(true)) + .as("the fallback prunes nothing: %s", unbounded) + .isTrue(); + } + + @ParameterizedTest + @MethodSource + @DisplayName("an id with no derivable partition disables pruning for the batch, and the delete still lands") + void underivableIdDisablesPruning(String cause, UUID underivableId) { + // The fallback that preserves the pre-OPIK-6901 guarantee, as the original javadoc stated it: a row whose id_at + // cannot be trusted is STILL DELETED. That is a claim about the underivable row ITSELF, so it gets a real row + // here - seeded raw, since ingestion rejects both id shapes by design. Passing it as an id matching nothing + // would let an implementation that quietly drops underivable ids from the batch pass, which is the very bug the + // all-or-nothing rule exists to prevent. + var target = newTrace().build(); + traceResourceClient.createTrace(target, API_KEY, WORKSPACE_NAME); + var projectId = projectIdOf(target); + insertRawTrace(projectId, underivableId); + assertThat(liveRowCount(projectId, underivableId)) + .as("the %s row is seeded before the delete", cause).isEqualTo("1"); + + delete(Set.of(Pair.of(projectId, target.id()), Pair.of(projectId, underivableId))); + + assertThat(liveRowCount(projectId, underivableId)) + .as("the %s row is itself deleted, not skipped", cause) + .isEqualTo("0"); + assertThat(traceIdsOf(target.projectName())) + .as("and the derivable row batched alongside the %s id goes too", cause) + .doesNotContain(target.id()); + + // Asserted as the absence of ANY id_at predicate, not just of this PR's expression. A regression that narrowed + // the mutation with toMonday(id_at), an id_at range, or any other partition predicate would skip exactly the + // rows this fallback exists to reach, and rejecting one function name would not see it. The unbounded template + // mentions id_at nowhere at all, so that is the whole check. + var sql = lastTraceDeleteSql(target.id()); + assertThat(sql) + .as("the unbounded form for a %s batch carries no id_at predicate of any kind", cause) + .doesNotContain("id_at"); + assertThat(EMITTED_IN_CLAUSE.matcher(sql).find()) + .as("and no partition IN clause: %s", sql) + .isFalse(); + } + + private static Stream underivableIdDisablesPruning() { + return Stream.of( + arguments("non-v7", NON_V7_ID), + arguments("beyond-2299", OUT_OF_RANGE_ID)); + } + } + + /** + * The flag off, against the same post-cutover topology and the same fixtures — so the only difference + * from {@link PruningEnabled} is the flag itself. That is what makes the pair a control rather than two unrelated + * suites: the sibling {@code TracesPruningDisabledMutationTest} also runs with pruning off, but against the legacy + * table, so it varies the schema at the same time and cannot attribute anything to the flag alone. + *

+ * Both assertions here are the inverse of one in {@link PruningEnabled}, on identical data: no partition predicate + * is emitted, and the planner selects every part. Delete the flag gate so pruning always happens and these fail; + * delete the pruning and the other class fails. Correctness is unaffected either way, which is the point — the rows + * go away in both, so only these assertions can tell the two states apart. + */ + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + @ExtendWith(DropwizardAppExtensionProvider.class) + class PruningDisabled { + + @RegisterApp + private final TestDropwizardAppExtension app = newApp(false); + + @BeforeAll + void beforeAll(ClientSupport clientSupport, TraceDAO traceDAO) { + bindApp(clientSupport, traceDAO); + } + + @Test + @DisplayName("the same batch still clears every era, and emits no partition predicate") + void deleteStillClearsEveryEraWithoutPruning() { + var projectId = ID_GENERATOR.generateId(); + var ids = ERA_MONDAYS.stream().map(TracesPartitionPruningMutationTest::idInWeekOf).toList(); + ids.forEach(id -> insertRawTrace(projectId, id)); + + delete(ids.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toUnmodifiableSet())); + + assertThat(ids.stream().map(id -> liveRowCount(projectId, id))) + .as("the delete is still correct with the flag off - that is what makes it an optimisation") + .containsOnly("0"); + var sql = lastTraceDeleteSql(ids.getFirst()); + assertThat(sql) + .as("and carries no id_at narrowing of any kind") + .doesNotContain("id_at"); + assertThat(EMITTED_IN_CLAUSE.matcher(sql).find()) + .as("nor a partition IN clause: %s", sql) + .isFalse(); + } + + @Test + @DisplayName("the planner selects every part - the enabled class's assertion, inverted on the same data") + void plannerPrunesNothingWithoutPruning() { + var projectId = ID_GENERATOR.generateId(); + var ids = ERA_MONDAYS.stream().map(TracesPartitionPruningMutationTest::idInWeekOf).toList(); + ids.forEach(id -> insertRawTrace(projectId, id)); + + delete(Set.of(Pair.of(projectId, ids.getFirst()))); + + var parts = partsSelectedBy(lastTraceDeleteSql(ids.getFirst())); + assertThat(parts.map(selected -> selected.selected() == selected.total()).orElse(true)) + .as("no partition pruning with the flag off: %s", parts) + .isTrue(); + } + } } From 2963f3b4815c52a8cba8f2040c5a0539fb2a9382 Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 14:57:03 +0200 Subject: [PATCH 34/37] [OPIK-6901] [BE] style: collapse the double blank line the restructure left Lifting the tests into nested classes left two blank lines where the old afterAll used to end. Spotless collapses them. Added the check, which is the cheapest of the lot and should have been there first: consecutive blank lines are a guaranteed round-trip, since spotless always collapses them and nothing else I run looks at them. Seven pre-push checks now - line length, javadoc indent, orphaned javadoc, unused imports, unimported types, self-referential symbols, blank-line runs. Co-Authored-By: Claude Opus 5 (1M context) --- .../opik/infrastructure/TracesPartitionPruningMutationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 2e5f665135e..0b110d9376f 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -381,7 +381,6 @@ void afterAll() { network.close(); } - /** A UUIDv7 in the given week, minted mid-week so the partition assertion exercises the map back to Monday. */ private static UUID idInWeekOf(LocalDate monday) { return ID_GENERATOR.generateId(monday.plusDays(2).atTime(12, 0).toInstant(ZoneOffset.UTC)); From 58d8dc59e37ac7af5e9a19bb1d4e93fcc7c31cbb Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 16:04:37 +0200 Subject: [PATCH 35/37] [OPIK-6901] [BE] test: cover multi-chunk pruning, and stop routing DAO deletes through a raw container handle Two reviewer findings, and the second one exposed a defect of mine. 1. ensureDistributedWrap returned on engine == 'Distributed' alone, so a wrapper over another database or another local table would satisfy the guard, block the rebuild, and route reads and inserts away from the table the assertions then inspect. Now checks engine_full for this database and 'traces_local'. Validates and fails rather than dropping and recreating: a byte-compare of the whole engine expression would pin ClickHouse's re-printing, and silently repairing a wrapper someone else left behind hides the surprise instead of reporting it. 2. Multi-chunk pruning was untested. ANALYTICS_DELETE_BATCH_SIZE is 10000 and the derivation sits inside the concatMap, so all-or-nothing is a PER-STATEMENT guarantee, not per-request - a chunk with an underivable id loses its pruning while its siblings keep theirs. Every prior test passed one chunk's worth of pairs and could not see that. New test sends 10,002: chunk one all-derivable, chunk two carrying NON_V7_ID, asserting both chunks' real rows are deleted and that chunk two emitted no id_at predicate. Writing it surfaced two things worth more than the test: * MY DEFECT. It died on "Code: 62. Max query size exceeded ... position 262122". ClickHouseContainerUtils.newDatabaseAnalyticsFactory sets no queryParameters, while config.yml and config-test.yml both carry custom_http_params=max_query_size=100000000. Moving this suite's template to a container-derived handle for the nested-class restructure routed the DAO's deletes through it, so every delete here ran on non-production connection settings - confirmed on the live container: max_query_size = 262144, the default. A full chunk inlines to ~762 KiB and dies at 256 KiB. Now split: appTemplate (app-injected) for anything the DAO executes, template (container-derived) only for the pre-app topology install and this suite's raw seeds and reads, with javadoc on both saying they are not interchangeable. * A LIMIT ON THE ORACLE. With that fixed the statement ran, and the test then failed asserting the first chunk's predicate - because log_queries_cut_to_length is 100000 and a 762 KiB statement is truncated in query_log long before the predicate at its tail. That assertion was on a string the server never kept, so it is gone and the helper's javadoc now records the limit. A derivable chunk's pruning is covered by the single-chunk tests; what only this test shows is that the chunks are derived independently. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 144 ++++++++++++++++-- 1 file changed, 131 insertions(+), 13 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 0b110d9376f..5e9b76d5d96 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -54,6 +54,8 @@ import java.time.LocalDate; import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -65,6 +67,7 @@ import java.util.stream.Stream; import static com.comet.opik.api.resources.utils.AuthTestUtils.mockTargetWorkspace; +import static com.comet.opik.infrastructure.FilterUtils.ANALYTICS_DELETE_BATCH_SIZE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -168,8 +171,8 @@ SELECT toString(count()) AND name = :table """; - private static final String TABLE_ENGINE = """ - SELECT engine + private static final String TABLE_ENGINE_FULL = """ + SELECT engine_full FROM system.tables WHERE database = currentDatabase() AND name = 'traces' @@ -190,6 +193,21 @@ AND query LIKE concat('%', :trace_id, '%') LIMIT 1 """; + /** + * The newest {@code delete_traces} statement carrying exactly {@code pairs_size} pairs. A request larger than + * {@link com.comet.opik.infrastructure.FilterUtils#ANALYTICS_DELETE_BATCH_SIZE} is chunked by the DAO into one + * statement per chunk, and pruning is derived per chunk — so a test that reads only one statement cannot see + * the second chunk at all. + */ + private static final String DELETE_BY_PAIR_COUNT = """ + SELECT query + FROM system.query_log + WHERE log_comment LIKE concat('delete_traces:%pairs_size=', :pairs_size) + AND type = 'QueryFinish' + ORDER BY event_time_microseconds DESC + LIMIT 1 + """; + /** * Scoped by the same key {@code TraceDAO.delete} matches on — {@code (workspace_id, project_id, id)}. An oracle * narrower than the delete would answer a different question: a row for the same id in another project would keep @@ -314,9 +332,13 @@ SELECT toString(uniqExact(id)) private final PodamFactory factory = PodamFactoryUtils.newPodamFactory(); /** - * Runs the topology setup and every raw read straight against the container, with no app in the way — the idiom the - * sibling partition suites use. It has to be app-independent: the topology must be installed once, before either - * nested app boots, and both nested classes then read through the same handle. + * Runs the topology setup and this suite's own raw reads and seeds straight against the container, with no app in + * the way — the idiom the sibling partition suites use. It has to be app-independent: the topology must be + * installed once, before either nested app boots, and both nested classes then read through the same handle. + *

+ * Never use this for anything the DAO executes — see {@link #appTemplate}. It carries none of the production + * {@code queryParameters}, so a statement the real connection would accept can fail here for reasons production + * would never hit. */ private final TransactionTemplateAsync template; @@ -336,6 +358,15 @@ SELECT toString(uniqExact(id)) private TraceResourceClient traceResourceClient; private TraceDAO traceDAO; + /** + * The app's own connection handle, used for anything the DAO executes. It is not interchangeable with + * {@link #template}: the app builds its factory from {@code config-test.yml}, which carries the production + * {@code queryParameters} — including {@code max_query_size=100000000}. The container-derived handle sets none of + * them, so a full-size delete chunk (10,000 pairs inline to ~762 KiB of SQL) dies on ClickHouse's 256 KiB default. + * Routing the DAO through the container handle silently ran every delete in this suite on non-production settings. + */ + private TransactionTemplateAsync appTemplate; + /** * One app per flag state, identical in every other respect. {@code tracesWeeklyPartitionPruningEnabled} is the only * thing that varies between the two nested classes, which is what lets them be read as an A/B: same schema, same @@ -365,12 +396,13 @@ private TestDropwizardAppExtension newApp(boolean pruningEnabled) { * Wires the currently-running nested class's app in. Safe on shared outer fields because JUnit runs nested * containers one at a time — this tree configures no parallel execution — so only one app is live at a time. */ - private void bindApp(ClientSupport clientSupport, TraceDAO traceDAO) { + private void bindApp(ClientSupport clientSupport, TraceDAO traceDAO, TransactionTemplateAsync appTemplate) { var baseUrl = TestUtils.getBaseUrl(clientSupport); ClientSupportUtils.config(clientSupport); mockTargetWorkspace(wireMock.server(), API_KEY, WORKSPACE_NAME, WORKSPACE_ID, USER); this.traceResourceClient = new TraceResourceClient(clientSupport, baseUrl); this.traceDAO = traceDAO; + this.appTemplate = appTemplate; } @AfterAll @@ -426,7 +458,7 @@ private static Set boundPartitionsOf(String sql) { /** Invokes the DAO under a workspace/user context, as {@code TraceService} does for the live delete path. */ private void delete(Set> projectIdTraceIdPairs) { - template.nonTransaction(connection -> traceDAO.delete(projectIdTraceIdPairs, connection)) + appTemplate.nonTransaction(connection -> traceDAO.delete(projectIdTraceIdPairs, connection)) .contextWrite(ctx -> ctx .put(RequestContext.WORKSPACE_ID, WORKSPACE_ID) .put(RequestContext.USER_NAME, USER)) @@ -460,6 +492,27 @@ private String lastTraceDeleteSql(UUID traceId) { return sql; } + /** + * The newest {@code delete_traces} statement that carried exactly {@code pairsSize} pairs. Chunks are identified by + * their pair count rather than by a contained id, because the point is to inspect a specific chunk of one + * request — and the DAO stamps each chunk's size into its {@code log_comment}. + *

+ * The returned text is truncated for large statements. ClickHouse caps {@code query_log.query} at + * {@code log_queries_cut_to_length} (100,000 bytes by default), and a full 10,000-pair chunk inlines to ~762 KiB, + * so anything at the tail of such a statement — the partition predicate included — is simply absent. Only assert on + * the text of statements small enough to be recorded whole. + */ + private String deleteSqlForChunkOf(int pairsSize) { + execute("SYSTEM FLUSH LOGS", _ -> { + }); + var sql = queryOneString(DELETE_BY_PAIR_COUNT, + statement -> statement.bind("pairs_size", String.valueOf(pairsSize))); + assertThat(sql) + .as("query_log holds a delete_traces statement with pairs_size=%s", pairsSize) + .isNotBlank(); + return sql; + } + /** {@code "1"} while a live (non-lightweight-deleted) row exists for the id, {@code "0"} once it is gone. */ private String liveRowCount(UUID projectId, UUID id) { return queryOneString(LIVE_ROW_COUNT, statement -> statement @@ -514,8 +567,17 @@ private void ensurePartitionedSuccessorUnderTraces() { * clears a wrapper stranded by an interrupted run before rebuilding it. */ private void ensureDistributedWrap() { - if ("Distributed".equals(queryOneString(TABLE_ENGINE, _ -> { - }))) { + // Accepting any Distributed table would let a wrapper pointing at another database or another local table + // block the rebuild and silently route reads and inserts elsewhere - so the check is on the definition, not the + // engine name. Matched on the two parts that decide where rows actually go (the database and the local target) + // rather than the whole string, which ClickHouse re-prints and which would make this brittle about spacing. + var engineFull = Optional.ofNullable(queryOneString(TABLE_ENGINE_FULL, _ -> { + })).orElse(""); + if (engineFull.startsWith("Distributed")) { + assertThat(engineFull) + .as("`traces` is already Distributed but not over this database's traces_local: %s", engineFull) + .contains("'" + ClickHouseContainerUtils.DATABASE_NAME + "'") + .contains("'traces_local'"); return; } // Clear a wrapper left behind by a run that died between the CREATE and the RENAME. It holds no data - a @@ -707,8 +769,8 @@ class PruningEnabled { private final TestDropwizardAppExtension app = newApp(true); @BeforeAll - void beforeAll(ClientSupport clientSupport, TraceDAO traceDAO) { - bindApp(clientSupport, traceDAO); + void beforeAll(ClientSupport clientSupport, TraceDAO traceDAO, TransactionTemplateAsync appTemplate) { + bindApp(clientSupport, traceDAO, appTemplate); } @Test @@ -824,6 +886,62 @@ void pruningReachesThePlannerAndTheFallbackDoesNot() { .isTrue(); } + @Test + @DisplayName("a request spanning two chunks prunes each chunk on its own") + void requestSpanningTwoChunksPrunesEachChunkIndependently() { + // The DAO chunks a request at ANALYTICS_DELETE_BATCH_SIZE and derives partitions PER CHUNK, inside the + // concatMap - so "all-or-nothing" is a per-statement guarantee, not a per-request one. Every other test + // here passes a handful of pairs, which is one chunk, so none of them can see that. + // + // What this pins: chunk one is all-derivable and must prune; chunk two carries a non-v7 id and must fall + // back to the unbounded form; and the real rows in BOTH chunks must be deleted either way. A refactor that + // hoisted the derivation out of the lambda would make the whole request unbounded - safe, but it would + // silently give back the pruning on every large delete, and only this test would notice. + var projectId = ID_GENERATOR.generateId(); + var firstChunkRow = idInWeekOf(ERA_MONDAYS.getFirst()); + var secondChunkRow = idInWeekOf(ERA_MONDAYS.getLast()); + insertRawTrace(projectId, firstChunkRow); + insertRawTrace(projectId, secondChunkRow); + + // Chunk one: the real row plus filler, all derivable, exactly ANALYTICS_DELETE_BATCH_SIZE pairs. Filler ids + // match no row - a delete does not need its ids to exist, and the chunk boundary is what is under test. + var ordered = new ArrayList(); + ordered.add(firstChunkRow); + while (ordered.size() < ANALYTICS_DELETE_BATCH_SIZE) { + ordered.add(idInWeekOf(ERA_MONDAYS.getFirst())); + } + // Chunk two: the second real row and a non-v7 id, so this chunk alone loses its pruning. + ordered.add(secondChunkRow); + ordered.add(NON_V7_ID); + + delete(ordered.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toCollection( + LinkedHashSet::new))); + + assertThat(liveRowCount(projectId, firstChunkRow)) + .as("the row in the pruned chunk is deleted") + .isEqualTo("0"); + assertThat(liveRowCount(projectId, secondChunkRow)) + .as("and so is the row in the chunk that fell back to unbounded") + .isEqualTo("0"); + + // Two statements, with the sizes the chunking implies - this is what shows the request was split at all. + // The full chunk's own text is NOT inspectable: query_log truncates at log_queries_cut_to_length (100,000 + // bytes on this server) and a 10,000-pair statement inlines to ~762 KiB, so the recorded text stops long + // before the partition predicate at the statement's tail. Asserting on it would be asserting on a string + // the server never kept. That a derivable chunk prunes is covered by the single-chunk tests; what only this + // test can show is that the two chunks are derived INDEPENDENTLY - which the second chunk's shape proves, + // since it lost its pruning while the first still executed and deleted its row. + deleteSqlForChunkOf(ANALYTICS_DELETE_BATCH_SIZE); + + var secondChunkSql = deleteSqlForChunkOf(2); + assertThat(secondChunkSql) + .as("the chunk carrying the non-v7 id emits no id_at predicate of any kind") + .doesNotContain("id_at"); + assertThat(EMITTED_IN_CLAUSE.matcher(secondChunkSql).find()) + .as("nor a partition IN clause: %s", secondChunkSql) + .isFalse(); + } + @ParameterizedTest @MethodSource @DisplayName("an id with no derivable partition disables pruning for the batch, and the delete still lands") @@ -889,8 +1007,8 @@ class PruningDisabled { private final TestDropwizardAppExtension app = newApp(false); @BeforeAll - void beforeAll(ClientSupport clientSupport, TraceDAO traceDAO) { - bindApp(clientSupport, traceDAO); + void beforeAll(ClientSupport clientSupport, TraceDAO traceDAO, TransactionTemplateAsync appTemplate) { + bindApp(clientSupport, traceDAO, appTemplate); } @Test From 6b2419d9476f8fb9838e63fbc46a4d274df1dd9e Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 16:16:58 +0200 Subject: [PATCH 36/37] [OPIK-6901] [BE] test: cover the post-cutover state, where the EXCHANGE is not needed at all The setup was already idempotent, but nothing exercised the branch that makes it so. In today's estate a fresh container always takes the INSTALL path in both steps, so both early returns are dead code - and they are precisely the path that becomes the ONLY path once the cutover migration lands and the migrations hand us `traces_local` partitioned with the Distributed `traces` over it. The EXCHANGE stops being needed then, and the first day that code matters is the wrong day to discover it was wrong. topologySetupIsANoOpOnceTheEstateProvidesIt re-runs both steps against the topology they already installed, which is that same shape: `traces_local` exists and `traces` is Distributed over it. It asserts the wrapper and the partitioned data are untouched, and then that a pruned delete still works - so a no-op setup cannot quietly leave the suite asserting against something no longer partitioned. Two states are covered now: WITH the swap, which every other test needs today, and WITHOUT it, which is the estate after the cutover. Not a tautology, and the comment says why: a step that failed to early-return would THROW rather than quietly repeat. The EXCHANGE needs `traces_local_v2`, which the install renamed to `traces_pre_cutover_backup`; the wrap ends in a RENAME onto `traces_local`, which by then exists. The class javadoc's idempotency claim now names this test instead of asserting it in prose. Verified locally: 10 tests, 0 failures. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 5e9b76d5d96..2539750af70 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -113,7 +113,11 @@ * against the one table where this predicate must never be emitted, and would pass while proving nothing: the predicate * is harmless against an unpartitioned table for recent ids. Hand-authoring a partitioned {@code traces} in the test * instead would duplicate migration 000114 and reintroduce exactly the drift this suite exists to detect. Both steps are - * idempotent, so the suite keeps working once the cutover migration lands and they become no-ops. + * idempotent, so the suite keeps working once the cutover migration lands and they become no-ops — and that is + * asserted rather than assumed, by {@link PruningEnabled#topologySetupIsANoOpOnceTheEstateProvidesIt}. Two states + * are therefore covered: with the swap, which every other test here needs today, and without it, + * which is what the estate will look like once the migrations create {@code traces_local} partitioned with the + * {@code Distributed} {@code traces} over it and this suite's {@code EXCHANGE} stops being needed at all. * *

Topology: the post-cutover end state, with both schema flags on. The wrap is applied and * {@code tracesDistributedWrapEnabled} set, so the DAO's mutations reach the data the way production routes them — @@ -886,6 +890,52 @@ void pruningReachesThePlannerAndTheFallbackDoesNot() { .isTrue(); } + @Test + @DisplayName("the topology setup is a no-op once the estate provides it - the path it takes post-cutover") + void topologySetupIsANoOpOnceTheEstateProvidesIt() { + // Today this suite installs the topology itself, so both setup steps take their INSTALL path and their + // early returns are dead code. Once the cutover migration lands, the migrations will provide + // `traces_local` partitioned with the Distributed `traces` over it: the EXCHANGE and the wrap are no + // longer needed, and that early return becomes the ONLY path either step takes. Nothing would exercise it + // until the day it becomes load-bearing, which is the wrong day to find out it was wrong. + // + // Re-running the setup against the topology it already installed IS that shape - `traces_local` exists and + // `traces` is Distributed over it, which is what the migration will hand us. So this covers the second of + // the two states: with the swap (every other test here) and without it (this one). + // Not a tautology: if either step failed to early-return it would THROW, not quietly repeat itself. The + // EXCHANGE needs `traces_local_v2`, which the install renamed to `traces_pre_cutover_backup`; and the wrap + // ends in a RENAME onto `traces_local`, which by now exists. So a non-idempotent step fails loudly here. + var tracesEngineBefore = queryOneString(TABLE_ENGINE_FULL, _ -> { + }); + var localPartitionKeyBefore = partitionKeyOf("traces_local"); + + ensurePartitionedSuccessorUnderTraces(); + ensureDistributedWrap(); + + assertThat(queryOneString(TABLE_ENGINE_FULL, _ -> { + })) + .as("re-running the setup left the Distributed wrapper untouched") + .isEqualTo(tracesEngineBefore); + assertThat(partitionKeyOf("traces_local")) + .as("and left the partitioned data untouched") + .isEqualTo(localPartitionKeyBefore); + + // Not just survivable - still testing what it claims. A pruned delete on the untouched topology, so a + // no-op setup cannot quietly leave the suite asserting against something that is no longer partitioned. + var projectId = ID_GENERATOR.generateId(); + var id = idInWeekOf(ERA_MONDAYS.getFirst()); + insertRawTrace(projectId, id); + + delete(Set.of(Pair.of(projectId, id))); + + assertThat(liveRowCount(projectId, id)) + .as("the row is still deleted after a no-op setup") + .isEqualTo("0"); + assertThat(lastTraceDeleteSql(id)) + .as("and the delete is still pruned") + .contains(PARTITION_PREDICATE); + } + @Test @DisplayName("a request spanning two chunks prunes each chunk on its own") void requestSpanningTwoChunksPrunesEachChunkIndependently() { From 63a00f0c4878607a45e7445fd83ce718b7c8ce9f Mon Sep 17 00:00:00 2001 From: Thiago Hora Date: Thu, 20 Aug 2026 16:58:14 +0200 Subject: [PATCH 37/37] [OPIK-6901] [BE] test: make the chunk and idempotence tests able to fail Two reviewer findings, both of them the same defect I have been fixing elsewhere in this review: an assertion that would pass whether or not the behaviour it names exists. 1. The multi-chunk test asserted nothing about the first chunk, so hoisting weeklyPartitionsFor over the request would leave BOTH statements unbounded, delete the rows anyway, and still satisfy chunk two's "no predicate" assertions. My own comment claimed only this test would notice that refactor, which was false. Fixed by inverting the arrangement rather than adding an assertion, because the first chunk can never be inspected: Lists.partition sizes chunks [BATCH_SIZE, remainder], so the first is always full, and query_log truncates at log_queries_cut_to_length (100,000 bytes) while a 10,000-pair statement inlines to ~762 KiB - its tail, where the predicate sits, is never recorded. EXPLAIN cannot help either, since it reads the same truncated text. So the non-v7 id goes in the first chunk and the derivable ids in the readable remainder, which is asserted to carry the predicate and to be bounded to exactly its own two weeks. Verified by mutation test, not by argument: patching the DAO to derive over projectIdTraceIdPairs instead of batch - precisely the refactor in question - turns the test red on the named assertion, and reverting turns it green again. 2. The idempotence test captured only engine_full and partition_key, then seeded its row AFTER both helpers ran. A table recreated from the same DDL reports identical metadata while being empty, so a step that rebuilt the topology instead of skipping it satisfied every assertion, and the row inserted afterwards landed happily in the fresh table. The test could not tell "skipped" from "rebuilt" - the one thing it exists to check. Now a row is created through the ingestion path BEFORE the setup re-runs and asserted still visible afterwards via traceIdsOf, the production getTraces path. Reading it back that way also shows the Distributed wrapper still ROUTES, which no metadata check can establish and which is what a misrouted rebuild would break. Verified locally: 10 tests, 0 failures. Co-Authored-By: Claude Opus 5 (1M context) --- .../TracesPartitionPruningMutationTest.java | 70 ++++++++++++------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java index 2539750af70..98285e7d2e9 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesPartitionPruningMutationTest.java @@ -902,9 +902,21 @@ void topologySetupIsANoOpOnceTheEstateProvidesIt() { // Re-running the setup against the topology it already installed IS that shape - `traces_local` exists and // `traces` is Distributed over it, which is what the migration will hand us. So this covers the second of // the two states: with the swap (every other test here) and without it (this one). - // Not a tautology: if either step failed to early-return it would THROW, not quietly repeat itself. The - // EXCHANGE needs `traces_local_v2`, which the install renamed to `traces_pre_cutover_backup`; and the wrap - // ends in a RENAME onto `traces_local`, which by now exists. So a non-idempotent step fails loudly here. + // A row that already exists BEFORE the setup runs, created through the ingestion path so the check reads it + // back the way production does. Metadata on its own cannot see data loss: a table recreated from the same + // DDL reports the same engine_full and partition_key while being empty, so a step that rebuilt the topology + // rather than skipping it would satisfy every metadata assertion here. The surviving row is the assertion + // that distinguishes "skipped" from "rebuilt", and reading it through the wrapper also shows routing is + // intact rather than merely that the wrapper exists. + var existing = newTrace().build(); + traceResourceClient.createTrace(existing, API_KEY, WORKSPACE_NAME); + assertThat(traceIdsOf(existing.projectName())) + .as("the pre-existing row is readable before the setup re-runs") + .contains(existing.id()); + + // Not a tautology either: if either step failed to early-return it would THROW, not quietly repeat itself. + // The EXCHANGE needs `traces_local_v2`, which the install renamed to `traces_pre_cutover_backup`; and the + // wrap ends in a RENAME onto `traces_local`, which by now exists. So a non-idempotent step fails loudly. var tracesEngineBefore = queryOneString(TABLE_ENGINE_FULL, _ -> { }); var localPartitionKeyBefore = partitionKeyOf("traces_local"); @@ -912,12 +924,15 @@ void topologySetupIsANoOpOnceTheEstateProvidesIt() { ensurePartitionedSuccessorUnderTraces(); ensureDistributedWrap(); + assertThat(traceIdsOf(existing.projectName())) + .as("the row that existed before the setup is still there, and still routed through the wrapper") + .contains(existing.id()); assertThat(queryOneString(TABLE_ENGINE_FULL, _ -> { })) .as("re-running the setup left the Distributed wrapper untouched") .isEqualTo(tracesEngineBefore); assertThat(partitionKeyOf("traces_local")) - .as("and left the partitioned data untouched") + .as("and left the partitioned table's key untouched") .isEqualTo(localPartitionKeyBefore); // Not just survivable - still testing what it claims. A pruned delete on the untouched topology, so a @@ -943,53 +958,58 @@ void requestSpanningTwoChunksPrunesEachChunkIndependently() { // concatMap - so "all-or-nothing" is a per-statement guarantee, not a per-request one. Every other test // here passes a handful of pairs, which is one chunk, so none of them can see that. // - // What this pins: chunk one is all-derivable and must prune; chunk two carries a non-v7 id and must fall - // back to the unbounded form; and the real rows in BOTH chunks must be deleted either way. A refactor that - // hoisted the derivation out of the lambda would make the whole request unbounded - safe, but it would - // silently give back the pruning on every large delete, and only this test would notice. + // The non-v7 id goes in the FIRST chunk and the derivable ids in the second, which is the only arrangement + // that can actually discriminate. Chunks are sized [BATCH_SIZE, remainder], so the first is always full and + // never inspectable: query_log truncates at log_queries_cut_to_length (100,000 bytes here) and a + // 10,000-pair statement inlines to ~762 KiB, so its tail - where the predicate sits - is not recorded. + // Only the remainder chunk is small enough to read back, so the assertion that matters has to live there. + // + // That makes the test bite on the refactor it exists to catch. Hoisting weeklyPartitionsFor out of the + // lambda, so the whole request is derived once, would let the non-v7 id in chunk one strip pruning from + // chunk two as well - and chunk two's predicate is exactly what is asserted below. Removing the pruning + // outright fails the same assertion. With the arrangement reversed, both regressions passed. var projectId = ID_GENERATOR.generateId(); var firstChunkRow = idInWeekOf(ERA_MONDAYS.getFirst()); var secondChunkRow = idInWeekOf(ERA_MONDAYS.getLast()); insertRawTrace(projectId, firstChunkRow); insertRawTrace(projectId, secondChunkRow); - // Chunk one: the real row plus filler, all derivable, exactly ANALYTICS_DELETE_BATCH_SIZE pairs. Filler ids + // Chunk one: a real row, the non-v7 id, and filler up to exactly ANALYTICS_DELETE_BATCH_SIZE. Filler ids // match no row - a delete does not need its ids to exist, and the chunk boundary is what is under test. var ordered = new ArrayList(); ordered.add(firstChunkRow); + ordered.add(NON_V7_ID); while (ordered.size() < ANALYTICS_DELETE_BATCH_SIZE) { ordered.add(idInWeekOf(ERA_MONDAYS.getFirst())); } - // Chunk two: the second real row and a non-v7 id, so this chunk alone loses its pruning. + // Chunk two: the remainder, all derivable, in two different weeks so the bound set is exact rather than + // trivially a single value. + var secondChunkCompanion = idInWeekOf(ERA_MONDAYS.get(1)); ordered.add(secondChunkRow); - ordered.add(NON_V7_ID); + ordered.add(secondChunkCompanion); delete(ordered.stream().map(id -> Pair.of(projectId, id)).collect(Collectors.toCollection( LinkedHashSet::new))); assertThat(liveRowCount(projectId, firstChunkRow)) - .as("the row in the pruned chunk is deleted") + .as("the row in the chunk that fell back to unbounded is deleted") .isEqualTo("0"); assertThat(liveRowCount(projectId, secondChunkRow)) - .as("and so is the row in the chunk that fell back to unbounded") + .as("and so is the row in the chunk that pruned") .isEqualTo("0"); - // Two statements, with the sizes the chunking implies - this is what shows the request was split at all. - // The full chunk's own text is NOT inspectable: query_log truncates at log_queries_cut_to_length (100,000 - // bytes on this server) and a 10,000-pair statement inlines to ~762 KiB, so the recorded text stops long - // before the partition predicate at the statement's tail. Asserting on it would be asserting on a string - // the server never kept. That a derivable chunk prunes is covered by the single-chunk tests; what only this - // test can show is that the two chunks are derived INDEPENDENTLY - which the second chunk's shape proves, - // since it lost its pruning while the first still executed and deleted its row. + // The full chunk's statement is only checked to exist - that is what shows the request was split at all. + // Nothing about its text can be asserted, for the truncation reason above. deleteSqlForChunkOf(ANALYTICS_DELETE_BATCH_SIZE); var secondChunkSql = deleteSqlForChunkOf(2); assertThat(secondChunkSql) - .as("the chunk carrying the non-v7 id emits no id_at predicate of any kind") - .doesNotContain("id_at"); - assertThat(EMITTED_IN_CLAUSE.matcher(secondChunkSql).find()) - .as("nor a partition IN clause: %s", secondChunkSql) - .isFalse(); + .as("the all-derivable chunk prunes even though an earlier chunk could not") + .contains(PARTITION_PREDICATE); + assertThat(boundPartitionsOf(secondChunkSql)) + .as("bounded to exactly its own two weeks") + .containsExactlyInAnyOrder(partitionNameOf(ERA_MONDAYS.getLast()), + partitionNameOf(ERA_MONDAYS.get(1))); } @ParameterizedTest