diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/AbstractBenchmarkState.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/AbstractBenchmarkState.java index 6e3bcfc6fbb..9869dbb10f9 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/AbstractBenchmarkState.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/AbstractBenchmarkState.java @@ -76,6 +76,17 @@ public void setupInvocation() throws Exception { runner.setup(); } + /** + * Teardown method that runs after each benchmark invocation. This calls the {@link + * WorkloadRunner#cleanup()} to clean up any state created during execution. + * + * @throws Exception If any error occurs during cleanup. + */ + @TearDown(Level.Invocation) + public void teardownInvocation() throws Exception { + runner.cleanup(); + } + /** * Returns an instance of the desired engine based on the provided engine name. * diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/BenchmarkUtils.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/BenchmarkUtils.java index 9fa0b976936..1cb533e9727 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/BenchmarkUtils.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/BenchmarkUtils.java @@ -115,11 +115,10 @@ private static List loadSpecsFromTable(Path tableDir) { validateTableStructure(tableDir); Path tableInfoPath = tableDir.resolve(TABLE_INFO_FILE_NAME); - Path deltaDir = tableDir.resolve(DELTA_DIR_NAME); Path specsDir = tableDir.resolve(SPECS_DIR_NAME); TableInfo tableInfo = - TableInfo.fromJsonPath(tableInfoPath.toString(), deltaDir.toAbsolutePath().toString()); + TableInfo.fromJsonPath(tableInfoPath.toString(), tableDir.toAbsolutePath().toString()); return findSpecDirectories(specsDir).stream() .map(specDir -> loadSingleSpec(specDir, tableInfo)) diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/WorkloadBenchmark.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/WorkloadBenchmark.java index 6806a573d8f..f64717a1c3b 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/WorkloadBenchmark.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/WorkloadBenchmark.java @@ -50,7 +50,15 @@ public static class DefaultBenchmarkState extends AbstractBenchmarkState { @Override protected Engine getEngine(String engineName) { if (engineName.equals("default")) { - return DefaultEngine.create(new Configuration()); + return DefaultEngine.create( + new Configuration() { + { + // Set the batch size. This is required for writes. + set("delta.kernel.default.parquet.reader.batch-size", "1024"); + set("delta.kernel.default.json.reader.batch-size", "1024"); + set("delta.kernel.default.parquet.writer.targetMaxFileSize", "10485760"); // 1 MB + } + }); } else { throw new IllegalArgumentException("Unsupported engine: " + engineName); } @@ -102,8 +110,8 @@ public static void main(String[] args) throws RunnerException, IOException { // TODO: In the future, this can be extended to support multiple engines. .param("engineName", "default") .forks(1) - .warmupIterations(3) // Proper warmup for production benchmarks - .measurementIterations(5) // Proper measurement iterations for production benchmarks + .warmupIterations(3) + .measurementIterations(5) .warmupTime(TimeValue.seconds(1)) .measurementTime(TimeValue.seconds(1)) .addProfiler(KernelMetricsProfiler.class) diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/TableInfo.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/TableInfo.java index 647b52e5eb9..6f3d598cd5c 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/TableInfo.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/TableInfo.java @@ -16,10 +16,12 @@ package io.delta.kernel.defaults.benchmarks.models; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; -import org.codehaus.jackson.map.ObjectMapper; +import java.nio.file.Paths; /** * Represents metadata about a Delta table used in benchmark workloads. @@ -60,23 +62,9 @@ public class TableInfo { @JsonProperty("engine_info") public String engineInfo; - /** The root path where the Delta table is stored. */ - @JsonProperty("table_root") - public String tableRoot; - - /** @return the absolute path to the root of the table */ - public String getTableRoot() { - return tableRoot; - } - - /** - * Sets the root path of the Delta table. - * - * @param tableRoot the absolute path to the root of the table - */ - public void setTableRoot(String tableRoot) { - this.tableRoot = tableRoot; - } + /** The resolved absolute path to the root of the table. */ + @JsonProperty("table_info_path") + private String tableInfoPath; /** * Default constructor for Jackson deserialization. @@ -86,6 +74,21 @@ public void setTableRoot(String tableRoot) { */ public TableInfo() {} + /** Resolves the table root path based on the table type and location configuration. */ + @JsonIgnore + public String getResolvedTableRoot() { + // Default to "relative" if tableType is null or "relative" + return Paths.get(tableInfoPath, "delta").toAbsolutePath().toString(); + } + + public String getTableInfoPath() { + return tableInfoPath; + } + + public void setTableInfoPath(String tableInfoDirectory) { + this.tableInfoPath = tableInfoDirectory; + } + /** * Creates a TableInfo instance by reading from a JSON file at the specified path. * @@ -94,15 +97,16 @@ public TableInfo() {} * separately with the absolute path. * * @param jsonPath the path to the JSON file containing the TableInfo metadata - * @param tableRoot the absolute path to the root of the Delta table + * @param tableInfoPath the directory containing the table_info.json file (used for relative path + * resolution) * @return a TableInfo instance populated from the JSON file and table root path * @throws RuntimeException if there is an error reading or parsing the JSON file */ - public static TableInfo fromJsonPath(String jsonPath, String tableRoot) { + public static TableInfo fromJsonPath(String jsonPath, String tableInfoPath) { ObjectMapper mapper = new ObjectMapper(); try { TableInfo info = mapper.readValue(new File(jsonPath), TableInfo.class); - info.setTableRoot(tableRoot); + info.setTableInfoPath(tableInfoPath); return info; } catch (IOException e) { throw new RuntimeException("Failed to read TableInfo from JSON file: " + jsonPath, e); diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WorkloadSpec.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WorkloadSpec.java index 1d1ed0ba71c..861de96187b 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WorkloadSpec.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WorkloadSpec.java @@ -33,7 +33,10 @@ * field in the JSON. */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -@JsonSubTypes({@JsonSubTypes.Type(value = ReadSpec.class, name = "read")}) +@JsonSubTypes({ + @JsonSubTypes.Type(value = ReadSpec.class, name = "read"), + @JsonSubTypes.Type(value = WriteSpec.class, name = "write") +}) public abstract class WorkloadSpec { /** * The type of workload (e.g., "read"). This is used by Jackson's polymorphic deserialization to diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WriteSpec.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WriteSpec.java new file mode 100644 index 00000000000..2a321c34dc0 --- /dev/null +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/models/WriteSpec.java @@ -0,0 +1,118 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.kernel.defaults.benchmarks.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.delta.kernel.defaults.benchmarks.workloadrunners.WorkloadRunner; +import io.delta.kernel.defaults.benchmarks.workloadrunners.WriteRunner; +import io.delta.kernel.engine.Engine; +import java.util.Collections; +import java.util.List; + +/** + * Workload specification for write benchmarks. Defines test cases for writing to Delta tables with + * one or more commits containing add/remove actions. + * + *

Usage

+ * + *

To run this workload, use {@link WorkloadSpec#getRunner(Engine)} to get the appropriate {@link + * WriteRunner}. + * + * @see WriteRunner + */ +public class WriteSpec extends WorkloadSpec { + + /** + * Container for a single commit's configuration. + * + *

Each commit references a file containing the Delta log JSON actions (add/remove files) to be + * committed. + */ + public static class CommitSpec { + /** + * Path to the commit file containing Delta log JSON actions. The path is relative to the spec + * directory (where spec.json is located). + * + *

Example: "commit_a.json" + */ + @JsonProperty("data_files_path") + private String dataFilesPath; + + /** Default constructor for Jackson. */ + public CommitSpec() {} + + public CommitSpec(String dataFilesPath) { + this.dataFilesPath = dataFilesPath; + } + + public String getDataFilesPath() { + return dataFilesPath; + } + } + + /** + * List of commits to execute in sequence. Each commit contains a reference to a file with Delta + * log JSON actions. All commits are executed as part of the timed benchmark. + */ + @JsonProperty("commits") + private List commits; + + // Default constructor for Jackson + public WriteSpec() { + super("write"); + } + + /** + * Gets the list of commits to execute. + * + * @return list of commit specifications + */ + public List getCommits() { + return commits != null ? commits : Collections.emptyList(); + } + + /** @return the full name of this workload, derived from table name, case name, and type. */ + @Override + public String getFullName() { + return this.tableInfo.name + "/" + caseName + "/write"; + } + + @Override + public WorkloadRunner getRunner(Engine engine) { + return new WriteRunner(this, engine); + } + + /** + * Generates workload variants from this test case specification. + * + *

Currently, WriteSpec generates a single variant (itself). In the future, this could be + * extended to generate variants for different write patterns or configurations. + * + * @return list of WriteSpec variants, each representing a separate workload execution + */ + @Override + public List getWorkloadVariants() { + return Collections.singletonList(this); + } + + @Override + public String toString() { + return String.format( + "Write{caseName='%s', commits=%d, tableInfo='%s'}", + caseName, getCommits().size(), tableInfo); + } +} diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/ReadMetadataRunner.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/ReadMetadataRunner.java index 8b264b11665..7a2e4ee4b31 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/ReadMetadataRunner.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/ReadMetadataRunner.java @@ -56,8 +56,8 @@ public ReadMetadataRunner(ReadSpec workloadSpec, Engine engine) { } @Override - public void setup() { - String workloadTableRoot = workloadSpec.getTableInfo().getTableRoot(); + public void setup() throws Exception { + String workloadTableRoot = workloadSpec.getTableInfo().getResolvedTableRoot(); SnapshotBuilder builder = TableManager.loadSnapshot(workloadTableRoot); if (workloadSpec.getVersion() != null) { builder.atVersion(workloadSpec.getVersion()); diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/WorkloadRunner.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/WorkloadRunner.java index 5107bab2a99..d3916413eb5 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/WorkloadRunner.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/WorkloadRunner.java @@ -61,6 +61,19 @@ public WorkloadRunner() {} */ public abstract void executeAsBenchmark(Blackhole blackhole) throws Exception; + /** + * Cleans up any state created during benchmark execution. For write workloads, this removes added + * files and reverts table state. For read workloads, this is typically a no-op. + * + *

This method is called after each benchmark invocation to ensure a clean state for the next + * run. + * + * @throws Exception if any error occurs during cleanup. + */ + public void cleanup() throws Exception { + // Default implementation: no-op for read workloads + } + // TODO: Add executeAsTest() method for correctness validation // public abstract void executeAsTest() throws Exception; } diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/WriteRunner.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/WriteRunner.java new file mode 100644 index 00000000000..35955fba5e6 --- /dev/null +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/benchmarks/workloadrunners/WriteRunner.java @@ -0,0 +1,256 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.kernel.defaults.benchmarks.workloadrunners; + +import static io.delta.kernel.internal.util.Utils.toCloseableIterator; + +import io.delta.kernel.*; +import io.delta.kernel.data.ColumnarBatch; +import io.delta.kernel.data.Row; +import io.delta.kernel.defaults.benchmarks.models.WorkloadSpec; +import io.delta.kernel.defaults.benchmarks.models.WriteSpec; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.internal.actions.AddFile; +import io.delta.kernel.internal.actions.RemoveFile; +import io.delta.kernel.internal.actions.SingleAction; +import io.delta.kernel.transaction.UpdateTableTransactionBuilder; +import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterator; +import io.delta.kernel.utils.FileStatus; +import java.io.File; +import java.io.IOException; +import java.util.*; +import org.openjdk.jmh.infra.Blackhole; + +/** + * A WorkloadRunner that executes write workloads as benchmarks. This runner performs one or more + * commits to a Delta table and measures the performance of those commits. + * + *

The runner executes commits specified in the {@link WriteSpec}, where each commit contains a + * set of Delta log actions (add/remove files) defined in external JSON files. + * + *

If run as a benchmark using {@link #executeAsBenchmark(Blackhole)}, this measures the time to + * execute all commits. Setup (loading commit files) and cleanup (reverting changes) are not + * included in the benchmark timing. + */ +public class WriteRunner extends WorkloadRunner { + private final Engine engine; + private final WriteSpec workloadSpec; + private List> commitActions; + private List committedVersions; + private Snapshot currentSnapshot; + private long originalVersion; + private Set initialDeltaLogFiles; + + /** + * Constructs the WriteRunner from the workload spec and engine. + * + * @param workloadSpec The write workload specification. + * @param engine The engine to use for executing the workload. + */ + public WriteRunner(WriteSpec workloadSpec, Engine engine) { + this.workloadSpec = workloadSpec; + this.engine = engine; + this.committedVersions = new ArrayList<>(); + } + + @Override + public void setup() throws Exception { + commitActions = new ArrayList<>(); + + String tableRoot = workloadSpec.getTableInfo().getResolvedTableRoot(); + + // Get the current snapshot + SnapshotBuilder builder = TableManager.loadSnapshot(tableRoot); + currentSnapshot = builder.build(engine); + + // Capture initial listing of delta log files + initialDeltaLogFiles = captureFileListing(); + + // Load and parse all commit files + for (WriteSpec.CommitSpec commitSpec : workloadSpec.getCommits()) { + String commitFilePath = + workloadSpec.getTableInfo().getTableInfoPath() + + "/specs/" + + workloadSpec.getCaseName() + + "/" + + commitSpec.getDataFilesPath(); + List actions = parseCommitFile(commitFilePath); + commitActions.add(actions); + } + } + + /** @return the name of this workload. */ + @Override + public String getName() { + return "write"; + } + + /** @return The workload specification used to create this runner. */ + @Override + public WorkloadSpec getWorkloadSpec() { + return workloadSpec; + } + + /** + * Executes the write workload as a benchmark, consuming results via the provided Blackhole. + * + *

This method executes all commits specified in the workload spec in sequence. The timing + * includes only the commit execution, not the setup or cleanup. We reuse the post-commit snapshot + * from each transaction to avoid reloading from disk, which makes the benchmark more efficient + * and realistic. + * + * @param blackhole The Blackhole to consume results and avoid dead code elimination. + */ + @Override + public void executeAsBenchmark(Blackhole blackhole) throws Exception { + // Execute all commits in sequence (timed) + for (List actions : commitActions) { + // Create transaction from table (first iteration) or from post-commit snapshot (subsequent) + UpdateTableTransactionBuilder txnBuilder = + currentSnapshot.buildUpdateTableTransaction("Delta-Kernel-Benchmarks", Operation.WRITE); + + // Build and commit the transaction + Transaction txn = txnBuilder.build(engine); + + // Convert actions list to CloseableIterable + CloseableIterator actionsIter = toCloseableIterator(actions.iterator()); + io.delta.kernel.utils.CloseableIterable dataActions = + io.delta.kernel.utils.CloseableIterable.inMemoryIterable(actionsIter); + + TransactionCommitResult result = txn.commit(engine, dataActions); + + long version = result.getVersion(); + committedVersions.add(version); + blackhole.consume(version); + + // Use the post-commit snapshot for the next transaction + // Post-commit snapshot should always be present unless there was a conflict + currentSnapshot = + result + .getPostCommitSnapshot() + .orElseThrow( + () -> + new IllegalStateException( + "Post-commit snapshot not available. This indicates a conflict occurred during " + + "the benchmark, which should not happen. Ensure no other processes are writing " + + "to the table: " + + workloadSpec.getTableInfo().getResolvedTableRoot())); + } + } + + /** + * Parses a Delta log JSON file and returns a list of action Rows. + * + *

Uses Kernel's built-in JsonHandler to read Delta log format JSON files. + * + * @param commitFilePath path to the commit file containing Delta log JSON actions + * @return list of Row objects representing the actions (AddFile, RemoveFile, etc.) + * @throws IOException if there's an error reading or parsing the file + */ + private List parseCommitFile(String commitFilePath) throws IOException { + /** Schema for reading commit files with both "add" and "remove" actions. */ + final StructType COMMIT_FILE_SCHEMA = + new StructType() + .add("add", AddFile.FULL_SCHEMA, true /* nullable */) + .add("remove", RemoveFile.FULL_SCHEMA, true /* nullable */); + + List actions = new ArrayList<>(); + + File file = new File(commitFilePath); + if (!file.exists()) { + throw new IOException("Commit file not found: " + commitFilePath); + } + + // Create a FileStatus for the commit file + FileStatus fileStatus = FileStatus.of(commitFilePath, file.length(), file.lastModified()); + + // Use Kernel's JsonHandler to read the file + try (CloseableIterator fileIter = + toCloseableIterator(Collections.singletonList(fileStatus).iterator()); + CloseableIterator batchIter = + engine.getJsonHandler().readJsonFiles(fileIter, COMMIT_FILE_SCHEMA, Optional.empty())) { + + while (batchIter.hasNext()) { + ColumnarBatch batch = batchIter.next(); + + // Process each row in the batch + try (CloseableIterator rowIter = batch.getRows()) { + while (rowIter.hasNext()) { + Row singleActionRow = rowIter.next(); + + // Extract the actual action Row and wrap it in SingleAction format + // Check if this row has an "add" action + if (!singleActionRow.isNullAt(COMMIT_FILE_SCHEMA.indexOf("add"))) { + Row addRow = singleActionRow.getStruct(COMMIT_FILE_SCHEMA.indexOf("add")); + // Wrap in SingleAction format for commit + actions.add(SingleAction.createAddFileSingleAction(addRow)); + } + // Check if this row has a "remove" action + else if (!singleActionRow.isNullAt(COMMIT_FILE_SCHEMA.indexOf("remove"))) { + Row removeRow = singleActionRow.getStruct(COMMIT_FILE_SCHEMA.indexOf("remove")); + // Wrap in SingleAction format for commit + actions.add(SingleAction.createRemoveFileSingleAction(removeRow)); + } else { + // Throw an error if the action is not recognized (not "add" or "remove") + throw new IOException( + "Unrecognized action in commit file row: " + singleActionRow.toString()); + } + } + } + } + } + + return actions; + } + + /** Cleans up the state created during benchmark execution by reverting all committed changes. */ + @Override + public void cleanup() throws Exception { + // Delete any files that weren't present initially + Set currentFiles = captureFileListing(); + for (String filePath : currentFiles) { + if (!initialDeltaLogFiles.contains(filePath)) { + engine.getFileSystemClient().delete(filePath); + } + } + + committedVersions.clear(); + } + + /** @return a set of all file paths in the the `_delta_log/` directory of the table. */ + private Set captureFileListing() throws IOException { + // Construct path prefix for all files in `_delta_log/`. The prefix is for file with name `0` + // because the filesystem client lists all _sibling_ files in the directory with a path greater + // than `0`. + String deltaLogPathPrefix = + new io.delta.kernel.internal.fs.Path( + workloadSpec.getTableInfo().getResolvedTableRoot(), "_delta_log/0") + .toUri() + .getPath(); + + Set files = new HashSet<>(); + try (CloseableIterator filesIter = + engine.getFileSystemClient().listFrom(deltaLogPathPrefix)) { + while (filesIter.hasNext()) { + FileStatus file = filesIter.next(); + files.add(file.getPath()); + } + } + return files; + } +} diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/add_and_remove/commit_add.json b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/add_and_remove/commit_add.json new file mode 100644 index 00000000000..1f37b2b9871 --- /dev/null +++ b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/add_and_remove/commit_add.json @@ -0,0 +1 @@ +{"add":{"path":"part-00000-a9daef62-5a40-43c5-ac63-3ad4a7d749ae-c000.snappy.parquet","partitionValues":{},"size":1024,"modificationTime":1712091404545,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"j\",\"number\":10,\"a_float\":10.10},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} \ No newline at end of file diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/add_and_remove/commit_remove.json b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/add_and_remove/commit_remove.json new file mode 100644 index 00000000000..bd9089e46f4 --- /dev/null +++ b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/add_and_remove/commit_remove.json @@ -0,0 +1 @@ +{"remove":{"path":"part-00000-a9daef62-5a40-43c5-ac63-3ad4a7d749ae-c000.snappy.parquet","partitionValues":{},"size":1024,"modificationTime":1712091404545,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"j\",\"number\":10,\"a_float\":10.10},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} \ No newline at end of file diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/add_and_remove/spec.json b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/add_and_remove/spec.json new file mode 100644 index 00000000000..3b8c404532c --- /dev/null +++ b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/add_and_remove/spec.json @@ -0,0 +1,8 @@ +{ + "type": "write", + "commits": [ + {"data_files_path": "commit_add.json"}, + {"data_files_path": "commit_remove.json"} + ] +} + diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/single_add/commit_add.json b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/single_add/commit_add.json new file mode 100644 index 00000000000..1f37b2b9871 --- /dev/null +++ b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/single_add/commit_add.json @@ -0,0 +1 @@ +{"add":{"path":"part-00000-a9daef62-5a40-43c5-ac63-3ad4a7d749ae-c000.snappy.parquet","partitionValues":{},"size":1024,"modificationTime":1712091404545,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"j\",\"number\":10,\"a_float\":10.10},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} \ No newline at end of file diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/single_add/spec.json b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/single_add/spec.json new file mode 100644 index 00000000000..207c7ce20a2 --- /dev/null +++ b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/specs/single_add/spec.json @@ -0,0 +1,7 @@ +{ + "type": "write", + "commits": [ + {"data_files_path": "commit_add.json"} + ] +} + diff --git a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/table_info.json b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/table_info.json index ad470d337d1..94637d8fc33 100644 --- a/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/table_info.json +++ b/kernel/kernel-defaults/src/test/resources/workload_specs/basic_append/table_info.json @@ -1,5 +1,5 @@ { "name": "basic_append", "description": "A basic table with two append writes.", - "engineInfo": "Apache-Spark/3.5.1 Delta-Lake/3.1.0" + "engine_info": "Apache-Spark/3.5.1 Delta-Lake/3.1.0" } \ No newline at end of file