From 37144381a1922492efcfd45079807cebad29ca7d Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Mon, 6 Oct 2025 17:19:07 -0700 Subject: [PATCH 1/4] Add snapshot construction spec --- .../models/SnapshotConstructionSpec.java | 42 +++++++++++++++ .../benchmarks/models/WorkloadSpec.java | 5 +- .../SnapshotConstructionRunner.java | 49 ++++++++++++++++++ .../specs/read_latest/expected_data.parquet | Bin 0 -> 459 bytes .../specs/{latest => read_latest}/spec.json | 0 .../specs/read_v0/expected_data.parquet | Bin 0 -> 996 bytes .../specs/{v0 => read_v0}/spec.json | 0 .../specs/snapshot_latest/spec.json | 3 ++ 8 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/SnapshotConstructionSpec.java create mode 100644 kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java create mode 100644 kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_latest/expected_data.parquet rename kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/{latest => read_latest}/spec.json (100%) create mode 100644 kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_v0/expected_data.parquet rename kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/{v0 => read_v0}/spec.json (100%) create mode 100644 kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/snapshot_latest/spec.json diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/SnapshotConstructionSpec.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/SnapshotConstructionSpec.java new file mode 100644 index 00000000000..41758e69adb --- /dev/null +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/SnapshotConstructionSpec.java @@ -0,0 +1,42 @@ +package io.delta.kernel.defaults.benchmarks.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.delta.kernel.defaults.benchmarks.workloadrunners.SnapshotConstructionRunner; +import io.delta.kernel.defaults.benchmarks.workloadrunners.WorkloadRunner; +import io.delta.kernel.engine.Engine; + +public class SnapshotConstructionSpec extends WorkloadSpec { + + /** The snapshot version to read. If null, the latest version will be read. From spec file. */ + @JsonProperty("version") + private Long version; + + /** Expected data file for validating the read data result. From spec file. */ + @JsonProperty("expected_protocol_and_metadata") + private String expectedProtocolAndMetadata; + + // Default constructor for Jackson + public SnapshotConstructionSpec() { + super("snapshot_construction"); + } + + // Copy constructor + public SnapshotConstructionSpec( + TableInfo tableInfo, String caseName, Long version, String expectedProtocolAndMetadata) { + super("snapshot_construction"); + this.tableInfo = tableInfo; + this.version = version; + this.caseName = caseName; + this.expectedProtocolAndMetadata = expectedProtocolAndMetadata; + } + + /** @return the snapshot version to read, or null if the latest version should be read. */ + public Long getVersion() { + return version; + } + + @Override + public WorkloadRunner getRunner(Engine engine) { + return new SnapshotConstructionRunner(this, engine); + } +} diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WorkloadSpec.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WorkloadSpec.java index 1d1ed0ba71c..89ff203c242 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WorkloadSpec.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WorkloadSpec.java @@ -33,7 +33,10 @@ * field in the JSON. */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -@JsonSubTypes({@JsonSubTypes.Type(value = ReadSpec.class, name = "read")}) +@JsonSubTypes({ + @JsonSubTypes.Type(value = ReadSpec.class, name = "read"), + @JsonSubTypes.Type(value = SnapshotConstructionSpec.class, name = "snapshot_construction"), +}) public abstract class WorkloadSpec { /** * The type of workload (e.g., "read"). This is used by Jackson's polymorphic deserialization to diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java new file mode 100644 index 00000000000..ca008e7429f --- /dev/null +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java @@ -0,0 +1,49 @@ +package io.delta.kernel.defaults.benchmarks.workloadrunners; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.SnapshotBuilder; +import io.delta.kernel.TableManager; +import io.delta.kernel.defaults.benchmarks.models.SnapshotConstructionSpec; +import io.delta.kernel.defaults.benchmarks.models.WorkloadSpec; +import io.delta.kernel.engine.Engine; +import org.openjdk.jmh.infra.Blackhole; + +public class SnapshotConstructionRunner extends WorkloadRunner { + + private final SnapshotConstructionSpec workloadSpec; + private final Engine engine; + + public SnapshotConstructionRunner(SnapshotConstructionSpec spec, Engine engine) { + this.workloadSpec = spec; + this.engine = engine; + } + + @Override + public String getName() { + return "snapshot_construction"; + } + + @Override + public WorkloadSpec getWorkloadSpec() { + return this.workloadSpec; + } + + @Override + public void setup() throws Exception { + /* No setup needed for snapshot construction */ + } + + @Override + public void executeAsBenchmark(Blackhole blackhole) throws Exception { + blackhole.consume(this.execute()); + } + + private Snapshot execute() { + String workloadTableRoot = workloadSpec.getTableInfo().getTableRoot(); + SnapshotBuilder builder = TableManager.loadSnapshot(workloadTableRoot); + if (workloadSpec.getVersion() != null) { + builder.atVersion(workloadSpec.getVersion()); + } + return builder.build(engine); + } +} diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_latest/expected_data.parquet b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_latest/expected_data.parquet new file mode 100644 index 0000000000000000000000000000000000000000..db8d5c59bad0f6b68cd0e8a23ba397505fbf6c30 GIT binary patch literal 459 zcmZvZ&q~8U5XNU>5;sc~JO+Gq16 z`UcJ>1_V2>vpbvb_s#AmySzIipbIi5BzfVJ4{)3~03Z=L=jf${U6eLeHr1@Vr|b`K z8g)Q`xX;`Ude^LUdD!sU2^#Qr3%Fh(x|?Aj1e%G5t~9|x|IoCUGUS9e=b-nhW%*nc zJ1$2PwT~^o(}-XwdKolR&dz|l3ecQVa*Y^1Q5=zDb5`V<6F}i{yZlCdQ`~i`dK-~ZU literal 0 HcmV?d00001 diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/latest/spec.json b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_latest/spec.json similarity index 100% rename from kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/latest/spec.json rename to kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_latest/spec.json diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_v0/expected_data.parquet b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_v0/expected_data.parquet new file mode 100644 index 0000000000000000000000000000000000000000..dfa1224494cc96bd2deb44f777ccfc6165c03594 GIT binary patch literal 996 zcmb7D&x_MQ6rQBrY_=3d*f%5)qy)EgrG|$7*lsEAdK1}0@g%siOfu88n*P`{RiyOb zS@6F713ZcT6CU=kh&K^0UiS?mecejX8;WwJh)9x>4sFJ$!|(H- z->?3KF+Ek3^u^wWenZ)B&n&ZVUQnv{@PK$jbkW*YjU_)2>JM4E07` zPlNG1eJhvd7aOXW{+$WUhd2>0iu~31M!a{~+bhY|7S^bj=w@Lm?u^ Date: Tue, 14 Oct 2025 15:17:47 -0700 Subject: [PATCH 2/4] remove unnecessary files --- .../specs/read_latest/expected_data.parquet | Bin 459 -> 0 bytes .../specs/read_v0/expected_data.parquet | Bin 996 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_latest/expected_data.parquet delete mode 100644 kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_v0/expected_data.parquet diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_latest/expected_data.parquet b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_latest/expected_data.parquet deleted file mode 100644 index db8d5c59bad0f6b68cd0e8a23ba397505fbf6c30..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 459 zcmZvZ&q~8U5XNU>5;sc~JO+Gq16 z`UcJ>1_V2>vpbvb_s#AmySzIipbIi5BzfVJ4{)3~03Z=L=jf${U6eLeHr1@Vr|b`K z8g)Q`xX;`Ude^LUdD!sU2^#Qr3%Fh(x|?Aj1e%G5t~9|x|IoCUGUS9e=b-nhW%*nc zJ1$2PwT~^o(}-XwdKolR&dz|l3ecQVa*Y^1Q5=zDb5`V<6F}i{yZlCdQ`~i`dK-~ZU diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_v0/expected_data.parquet b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/read_v0/expected_data.parquet deleted file mode 100644 index dfa1224494cc96bd2deb44f777ccfc6165c03594..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 996 zcmb7D&x_MQ6rQBrY_=3d*f%5)qy)EgrG|$7*lsEAdK1}0@g%siOfu88n*P`{RiyOb zS@6F713ZcT6CU=kh&K^0UiS?mecejX8;WwJh)9x>4sFJ$!|(H- z->?3KF+Ek3^u^wWenZ)B&n&ZVUQnv{@PK$jbkW*YjU_)2>JM4E07` zPlNG1eJhvd7aOXW{+$WUhd2>0iu~31M!a{~+bhY|7S^bj=w@Lm?u^ Date: Mon, 27 Oct 2025 09:37:11 -0700 Subject: [PATCH 3/4] add header --- .../models/SnapshotConstructionSpec.java | 16 ++++++++++++++++ .../SnapshotConstructionRunner.java | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/SnapshotConstructionSpec.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/SnapshotConstructionSpec.java index 41758e69adb..c6a9e853fdf 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/SnapshotConstructionSpec.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/SnapshotConstructionSpec.java @@ -1,3 +1,19 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.delta.kernel.defaults.benchmarks.models; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java index ca008e7429f..a3f82eaa979 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java @@ -1,3 +1,19 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.delta.kernel.defaults.benchmarks.workloadrunners; import io.delta.kernel.Snapshot; From ec353417c2e3b4c0b44e5b61c54adf740f8692ef Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Mon, 27 Oct 2025 14:38:17 -0700 Subject: [PATCH 4/4] add doc --- .../workloadrunners/SnapshotConstructionRunner.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java index a3f82eaa979..ea6e5f22132 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/SnapshotConstructionRunner.java @@ -29,8 +29,13 @@ public class SnapshotConstructionRunner extends WorkloadRunner { private final SnapshotConstructionSpec workloadSpec; private final Engine engine; - public SnapshotConstructionRunner(SnapshotConstructionSpec spec, Engine engine) { - this.workloadSpec = spec; + /** + * Construct a SnapshotConstructionRunner from the workload spec and engine. + * @param workloadSpec The snapshot_construction workload specification. + * @param engine The engine to use for executing the workload. + */ + public SnapshotConstructionRunner(SnapshotConstructionSpec workloadSpec, Engine engine) { + this.workloadSpec = workloadSpec; this.engine = engine; }