reduce image size, consolidate iceberg file location (#1038)

Author: nabuskey

data-stacks/spark-on-eks/benchmarks/tpcds-iceberg-benchmarks/Dockerfile-comet-iceberg

@@ -102,7 +102,20 @@ EOF
 RUN sbt -Dsbt.override.build.repos=true -batch clean assembly
 
 # ============================================================
-### Stage 3 – Runtime image (Graviton4 / ARM64)
+### Stage 3 – Build TPC-DS toolkit
+# ============================================================
+FROM ubuntu:22.04 AS tpcds_builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc libc6-dev make flex bison git ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN git clone -b v4.0.0 https://github.com/heyujiao99/tpcds-kit.git /opt/tpcds-kit && \
+    cd /opt/tpcds-kit/tools && \
+    make OS=LINUX CFLAGS="-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -DYYDEBUG -DLINUX -g -Wall -fcommon"
+
+# ============================================================
+### Stage 4 – Final runtime image (Graviton4 / ARM64)
 # ============================================================
 FROM apache/spark:3.5.8-scala2.12-java17-python3-ubuntu
 
@@ -115,26 +128,19 @@ ARG SCALA_BINARY=2.12
 ARG SPARK_UID=185
 ARG SPARK_GID=185
 
-ENV DEBIAN_FRONTEND=noninteractive
 ENV SPARK_HOME=/opt/spark
 ENV PATH=$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH
-# JDK path from apache/spark base image (Eclipse Temurin)
 ENV JAVA_HOME=/opt/java/openjdk
 
 USER root
 WORKDIR ${SPARK_HOME}
 
-# System dependencies for TPC-DS toolkit compilation
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gcc make flex bison git wget curl ca-certificates && \
-    rm -rf /var/lib/apt/lists/*
-
 # Non-root Spark user — UID 185 matches Kubernetes runAsUser convention
 RUN (groupadd -g ${SPARK_GID} spark 2>/dev/null || true) && \
     (useradd -m -u ${SPARK_UID} -g ${SPARK_GID} -s /bin/bash spark 2>/dev/null || true)
 
-# ── Hadoop upgrade to 3.4.3 ──────────────────────────────────────────────────
-# Remove all bundled Hadoop + legacy AWS SDK jars to prevent version conflicts
+# ── Hadoop upgrade + dependency JARs ─────────────────────────────────────────
+# Remove bundled Hadoop + legacy AWS SDK jars, then download replacements
 RUN cd ${SPARK_HOME}/jars && \
     find . -maxdepth 1 -type f -name 'hadoop-*.jar' -delete || true && \
     find . -maxdepth 1 -type f -name 'aws-java-sdk-*.jar' -delete || true && \
@@ -152,28 +158,17 @@ RUN cd ${SPARK_HOME}/jars && \
     wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-hadoop-cloud_${SCALA_BINARY}/${SPARK_VERSION}/spark-hadoop-cloud_${SCALA_BINARY}-${SPARK_VERSION}.jar && \
     # XML deps for jackson-dataformat-xml
     wget -q https://repo1.maven.org/maven2/com/fasterxml/woodstox/woodstox-core/6.5.1/woodstox-core-6.5.1.jar && \
-    wget -q https://repo1.maven.org/maven2/org/codehaus/woodstox/stax2-api/4.2.2/stax2-api-4.2.2.jar
-
-# ── DataFusion Comet 0.14.0 ──────────────────────────────────────────────────
-RUN wget -q \
-    https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.5_${SCALA_BINARY}/${COMET_VERSION}/comet-spark-spark3.5_${SCALA_BINARY}-${COMET_VERSION}.jar \
-    -P ${SPARK_HOME}/jars/
-
-# ── Apache Iceberg 1.10.1 ────────────────────────────────────────────────────
-# iceberg-spark-runtime: Spark/Iceberg integration (catalog, DPP, read/write)
-# iceberg-aws-bundle:    Glue catalog implementation + S3FileIO
-RUN wget -q \
-    https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.5_${SCALA_BINARY}/${ICEBERG_VERSION}/iceberg-spark-runtime-3.5_${SCALA_BINARY}-${ICEBERG_VERSION}.jar \
-    -P ${SPARK_HOME}/jars/ && \
-    wget -q \
-    https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
-    -P ${SPARK_HOME}/jars/
-
-# ── TPC-DS toolkit v4.0.0 ────────────────────────────────────────────────────
-RUN git clone -b v4.0.0 https://github.com/heyujiao99/tpcds-kit.git /opt/tpcds-kit && \
-    cd /opt/tpcds-kit/tools && \
-    make OS=LINUX CFLAGS="-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -DYYDEBUG -DLINUX -g -Wall -fcommon" && \
-    install -m 0755 dsdgen dsqgen /usr/local/bin
+    wget -q https://repo1.maven.org/maven2/org/codehaus/woodstox/stax2-api/4.2.2/stax2-api-4.2.2.jar && \
+    # DataFusion Comet
+    wget -q https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.5_${SCALA_BINARY}/${COMET_VERSION}/comet-spark-spark3.5_${SCALA_BINARY}-${COMET_VERSION}.jar && \
+    # Apache Iceberg (spark-runtime + aws-bundle)
+    wget -q https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.5_${SCALA_BINARY}/${ICEBERG_VERSION}/iceberg-spark-runtime-3.5_${SCALA_BINARY}-${ICEBERG_VERSION}.jar && \
+    wget -q https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
+
+# ── TPC-DS toolkit binaries ──────────────────────────────────────────────────
+COPY --from=tpcds_builder /opt/tpcds-kit /opt/tpcds-kit
+COPY --from=tpcds_builder /opt/tpcds-kit/tools/dsdgen /usr/local/bin/dsdgen
+COPY --from=tpcds_builder /opt/tpcds-kit/tools/dsqgen /usr/local/bin/dsqgen
 
 # ── Benchmark JARs from build stages ─────────────────────────────────────────
 COPY --from=spark_sql_perf_builder /build/spark-sql-perf/target/scala-2.12/*.jar \
@@ -182,8 +177,7 @@ COPY --from=benchmark_builder /build/spark-k8s-benchmarks/target/scala-2.12/*.ja
     ${SPARK_HOME}/jars/
 
 # ── Parquet-to-Iceberg conversion script ─────────────────────────────────────
-RUN mkdir -p /opt/scripts
-COPY convert_parquet_to_iceberg.py /opt/scripts/convert_parquet_to_iceberg.py
+COPY --chown=spark:spark convert_parquet_to_iceberg.py /opt/scripts/convert_parquet_to_iceberg.py
 
 # ── Permissions ──────────────────────────────────────────────────────────────
 RUN chown -R spark:spark ${SPARK_HOME} && \
@@ -193,4 +187,4 @@ RUN chown -R spark:spark ${SPARK_HOME} && \
 USER ${SPARK_UID}
 WORKDIR ${SPARK_HOME}
 
-ENTRYPOINT ["/opt/entrypoint.sh"]
+ENTRYPOINT ["/opt/entrypoint.sh"]

data-stacks/spark-on-eks/benchmarks/tpcds-iceberg-benchmarks/convert_parquet_to_iceberg.py

@@ -13,20 +13,20 @@
 
 Usage (via SparkApplication YAML — all args come from SparkApplication.spec.arguments):
   spark-submit convert_parquet_to_iceberg.py \\
-      --src-bucket  <S3_BUCKET>              \\
-      --src-prefix  TPCDS-TEST-3TB           \\
-      --glue-db     tpcds_3tb                \\
-      --warehouse   s3a://<S3_BUCKET>/TPCDS-TEST-3TB-ICEBERG \\
-      --region      us-west-2                \\
-      [--table      <single_table>]          # optional: convert one table only
-
-S3 layout (three dedicated prefixes — no cross-contamination between benchmark runs):
-  Parquet-only data : s3://<bucket>/TPCDS-TEST-{N}TB/{table}/           (--src-prefix)
-  Iceberg data files: s3://<bucket>/TPCDS-TEST-{N}TB-ICEBERG-DATA/{table}/ (--data-path)
-  Iceberg metadata  : s3://<bucket>/TPCDS-TEST-{N}TB-ICEBERG-WH/           (--warehouse)
+      --src-bucket      <S3_BUCKET>              \\
+      --src-prefix      TPCDS-TEST-3TB           \\
+      --glue-db         tpcds_3tb                \\
+      --iceberg-prefix  TPCDS-TEST-3TB-ICEBERG   \\
+      --region          us-west-2                \\
+      [--table          <single_table>]          # optional: convert one table only
+
+S3 layout (single Iceberg prefix with data/ and metadata/ subdirectories):
+  Parquet-only data : s3://<bucket>/TPCDS-TEST-{N}TB/{table}/                  (--src-prefix)
+  Iceberg data files: s3://<bucket>/TPCDS-TEST-{N}TB-ICEBERG/data/{table}/     (derived)
+  Iceberg metadata  : s3://<bucket>/TPCDS-TEST-{N}TB-ICEBERG/metadata/         (derived)
 
   To run at a different scale (3TB, 10TB), change --src-prefix, --glue-db,
-  --warehouse, and --data-path in the SparkApplication YAML — no image rebuild needed.
+  and --iceberg-prefix in the SparkApplication YAML — no image rebuild needed.
 
 The job is idempotent — it checks existing Glue tables and skips any already converted.
 
@@ -81,15 +81,14 @@
 def parse_args():
     parser = argparse.ArgumentParser(description="Convert TPC-DS Parquet → Iceberg (Glue catalog)")
     # ── Scale-specific args — only these change per benchmark scale in the YAML ─
-    # Scale | --src-prefix         | --glue-db   | --warehouse                          | --data-path
-    # 1TB   | TPCDS-TEST-1TB       | tpcds_1tb   | s3a://<b>/TPCDS-TEST-1TB-ICEBERG-WH  | s3a://<b>/TPCDS-TEST-1TB-ICEBERG-DATA
-    # 3TB   | TPCDS-TEST-3TB       | tpcds_3tb   | s3a://<b>/TPCDS-TEST-3TB-ICEBERG-WH  | s3a://<b>/TPCDS-TEST-3TB-ICEBERG-DATA
-    # 10TB  | TPCDS-TEST-10TB      | tpcds_10tb  | s3a://<b>/TPCDS-TEST-10TB-ICEBERG-WH | s3a://<b>/TPCDS-TEST-10TB-ICEBERG-DATA
-    parser.add_argument("--src-bucket",  required=True,  help="Source S3 bucket name (no s3a:// prefix)")
-    parser.add_argument("--src-prefix",  required=True,  help="S3 prefix for source Parquet, e.g. TPCDS-TEST-3TB")
-    parser.add_argument("--glue-db",     required=True,  help="Target Glue database name, e.g. tpcds_3tb")
-    parser.add_argument("--warehouse",   required=True,  help="Iceberg metadata warehouse, e.g. s3a://<bucket>/TPCDS-TEST-3TB-ICEBERG-WH")
-    parser.add_argument("--data-path",   required=True,  help="Iceberg data file root, e.g. s3a://<bucket>/TPCDS-TEST-3TB-ICEBERG-DATA")
+    # Scale | --src-prefix         | --glue-db   | --iceberg-prefix
+    # 1TB   | TPCDS-TEST-1TB       | tpcds_1tb   | TPCDS-TEST-1TB-ICEBERG
+    # 3TB   | TPCDS-TEST-3TB       | tpcds_3tb   | TPCDS-TEST-3TB-ICEBERG
+    # 10TB  | TPCDS-TEST-10TB      | tpcds_10tb  | TPCDS-TEST-10TB-ICEBERG
+    parser.add_argument("--src-bucket",     required=True,  help="Source S3 bucket name (no s3a:// prefix)")
+    parser.add_argument("--src-prefix",     required=True,  help="S3 prefix for source Parquet, e.g. TPCDS-TEST-3TB")
+    parser.add_argument("--glue-db",        required=True,  help="Target Glue database name, e.g. tpcds_3tb")
+    parser.add_argument("--iceberg-prefix", required=True,  help="S3 prefix for Iceberg output, e.g. TPCDS-TEST-3TB-ICEBERG")
     # ── Fixed args ───────────────────────────────────────────────────────────────
     parser.add_argument("--region",      default="us-west-2",  help="AWS region for Glue")
     parser.add_argument("--table",       default=None,         help="Convert a single table only (optional)")
@@ -115,10 +114,10 @@ def convert_table(spark, table_name, partition_col, src_path, glue_db, data_path
     CTAS: read Parquet from src_path, write as partitioned Iceberg table in Glue.
     Uses CREATE OR REPLACE so re-running is safe (drops old Iceberg snapshot).
 
-    Three fully separate S3 prefixes (set via YAML args — no image rebuild needed):
-      src_path  → --src-prefix  (raw Parquet, read-only source)
-      data_path → --data-path   (Iceberg data files, e.g. TPCDS-TEST-3TB-ICEBERG-DATA)
-      warehouse → --warehouse   (Iceberg metadata/snapshots, e.g. TPCDS-TEST-3TB-ICEBERG-WH)
+    S3 layout (derived from --iceberg-prefix):
+      src_path  → --src-prefix           (raw Parquet, read-only source)
+      data_path → <iceberg-prefix>/data  (Iceberg data files)
+      warehouse → <iceberg-prefix>/metadata (Iceberg metadata/snapshots)
     """
     full_table = f"glue_catalog.{glue_db}.{table_name}"
 
@@ -152,6 +151,8 @@ def main():
     args = parse_args()
 
     src_base = f"s3a://{args.src_bucket}/{args.src_prefix}"
+    warehouse = f"s3a://{args.src_bucket}/{args.iceberg_prefix}/metadata"
+    data_path = f"s3a://{args.src_bucket}/{args.iceberg_prefix}/data"
 
     spark = (
         SparkSession.builder
@@ -163,7 +164,7 @@ def main():
                 "org.apache.iceberg.spark.SparkCatalog")
         .config("spark.sql.catalog.glue_catalog.catalog-impl",
                 "org.apache.iceberg.aws.glue.GlueCatalog")
-        .config("spark.sql.catalog.glue_catalog.warehouse", args.warehouse)
+        .config("spark.sql.catalog.glue_catalog.warehouse", warehouse)
         .config("spark.sql.catalog.glue_catalog.io-impl",
                 "org.apache.iceberg.aws.s3.S3FileIO")
         .config("spark.sql.catalog.glue_catalog.glue.region", args.region)
@@ -204,7 +205,7 @@ def main():
 
         src_path = f"{src_base}/{table_name}"
         try:
-            convert_table(spark, table_name, partition_col, src_path, args.glue_db, args.data_path)
+            convert_table(spark, table_name, partition_col, src_path, args.glue_db, data_path)
             converted += 1
         except Exception as exc:
             print(f"[fail] {table_name}: {exc}")

data-stacks/spark-on-eks/benchmarks/tpcds-iceberg-benchmarks/tpcds-benchmark-iceberg-comet.yaml

@@ -9,8 +9,8 @@
 #   3. Image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/spark-benchmark-native:3.5.8-tpcds4
 #
 # S3 layout:
-#   Iceberg data     : s3://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG-DATA/
-#   Iceberg metadata : s3://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG-WH/
+#   Iceberg data     : s3://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG/data/
+#   Iceberg metadata : s3://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG/metadata/
 #   Results          : s3://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-RESULT-COMET-ICEBERG-R8G/
 #
 # Usage:
@@ -29,15 +29,15 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/spark-benchmark-native:3.5.8-tpcds4"
+  image: "public.ecr.aws/data-on-eks/spark-benchmark:3.5.8-tpcds4"
   imagePullPolicy: Always
   sparkVersion: "3.5.8"
   mainClass: com.k8s.spark.benchmark.BenchmarkSQL
   mainApplicationFile: "local:///opt/spark/jars/spark-k8s-benchmarks-assembly-1.0.0.jar"
   arguments:
     # arg[0] tpcdsDataDir — Iceberg data files root (used for TPCDSTables init;
     #         format=iceberg bypasses path-based table creation, uses icebergDatabase instead)
-    - "s3a://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG-DATA"
+    - "s3a://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG/data"
     # arg[1] Results location
     - "s3a://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-RESULT-COMET-ICEBERG-R8G"
     # arg[2] Path to TPC-DS kit (dsdgen/dsqgen) in the image
@@ -169,7 +169,7 @@ spec:
     # Primary catalog — glue_catalog.tpcds_${SCALE}tb.<table>
     "spark.sql.catalog.glue_catalog": "org.apache.iceberg.spark.SparkCatalog"
     "spark.sql.catalog.glue_catalog.catalog-impl": "org.apache.iceberg.aws.glue.GlueCatalog"
-    "spark.sql.catalog.glue_catalog.warehouse": "s3a://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG-WH"
+    "spark.sql.catalog.glue_catalog.warehouse": "s3a://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG/metadata"
     "spark.sql.catalog.glue_catalog.io-impl": "org.apache.iceberg.aws.s3.S3FileIO"
     "spark.sql.catalog.glue_catalog.glue.region": "${AWS_REGION}"
     # SparkSessionCatalog v2 bridge — allows DPP to work with default catalog tables

data-stacks/spark-on-eks/benchmarks/tpcds-iceberg-benchmarks/tpcds-data-generation-3tb.yaml

@@ -32,7 +32,7 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/spark-benchmark-native:3.5.8-tpcds4"
+  image: "public.ecr.aws/data-on-eks/spark-benchmark:3.5.8-tpcds4"
   imagePullPolicy: Always
   sparkVersion: "3.5.8"
   mainClass: com.k8s.spark.benchmark.DataGeneration

data-stacks/spark-on-eks/benchmarks/tpcds-iceberg-benchmarks/tpcds-parquet-to-iceberg.yaml

@@ -12,8 +12,8 @@
 #
 # Scale-derived paths (auto-resolved from SCALE):
 #   Source Parquet  : s3://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB/
-#   Iceberg data    : s3://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG-DATA/
-#   Iceberg metadata: s3://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG-WH/
+#   Iceberg data    : s3://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG/data/
+#   Iceberg metadata: s3://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG/metadata/
 #   Glue database   : tpcds_${SCALE}tb
 #
 # Usage:
@@ -41,7 +41,7 @@ metadata:
 spec:
   type: Python
   mode: cluster
-  image: "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/spark-benchmark-native:3.5.8-tpcds4"
+  image: "public.ecr.aws/data-on-eks/spark-benchmark:3.5.8-tpcds4"
   imagePullPolicy: Always
   sparkVersion: "3.5.8"
   mainApplicationFile: "local:///opt/scripts/convert_parquet_to_iceberg.py"
@@ -52,10 +52,8 @@ spec:
     - "TPCDS-TEST-${SCALE}TB"
     - "--glue-db"
     - "tpcds_${SCALE}tb"
-    - "--warehouse"
-    - "s3a://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG-WH"
-    - "--data-path"
-    - "s3a://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG-DATA"
+    - "--iceberg-prefix"
+    - "TPCDS-TEST-${SCALE}TB-ICEBERG"
     - "--region"
     - "${AWS_REGION}"
   restartPolicy:
@@ -135,7 +133,7 @@ spec:
     "spark.sql.extensions": "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"
     "spark.sql.catalog.glue_catalog": "org.apache.iceberg.spark.SparkCatalog"
     "spark.sql.catalog.glue_catalog.catalog-impl": "org.apache.iceberg.aws.glue.GlueCatalog"
-    "spark.sql.catalog.glue_catalog.warehouse": "s3a://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG-WH"
+    "spark.sql.catalog.glue_catalog.warehouse": "s3a://${S3_BUCKET}/TPCDS-TEST-${SCALE}TB-ICEBERG/metadata"
     "spark.sql.catalog.glue_catalog.io-impl": "org.apache.iceberg.aws.s3.S3FileIO"
     "spark.sql.catalog.glue_catalog.glue.region": "${AWS_REGION}"
     "spark.sql.catalog.spark_catalog": "org.apache.iceberg.spark.SparkSessionCatalog"
@@ -165,7 +163,7 @@ spec:
     # shuffle.partitions: set high enough to avoid OOM on large fact tables;
     # AQE will auto-coalesce small partitions at runtime.
     # Rule of thumb: ~800 × SCALE (1TB→800, 3TB→2400, 10TB→8000)
-    "spark.sql.shuffle.partitions": "800"
+    "spark.sql.shuffle.partitions": "2400"
     # Large Iceberg writes — no per-file record cap (let Iceberg manage file sizing)
     "spark.sql.files.maxRecordsPerFile": "0"