feat: dsdgen compatibility

Author: clflushopt

.gitignore

@@ -3,4 +3,5 @@ target/
 __old/
 Cargo.lock
 .idea
-.venv/
+.venv/
+tpcds/

.gitmodules

@@ -8,6 +8,9 @@
 # Test fixtures (generated).
 #/tests/fixtures/
 
+# Python cache.
+scripts/__pycache__/
+
 # Stuff I need to remember
 NEXT_STEPS.md
 ISSUES.md

tpcdsgen/.gitignore

@@ -0,0 +1 @@
+3.14

tpcdsgen/.python-version

@@ -0,0 +1,82 @@
+------
+-- return_reasons
+------
+-- values           weights
+-- -----------------------
+--  1. reason       1-6. not sure... none are ever used
+------
+Package was damaged: 1, 0, 0, 0, 0, 0
+Stopped working: 1, 0, 0, 0, 0, 0
+Did not get it on time: 1, 0, 0, 0, 0, 0
+Not the product that was ordred: 1, 0, 0, 0, 0, 0
+Parts missing: 1, 0, 0, 0, 0, 0
+Does not work with a product that I have: 1, 0, 0, 0, 0, 0
+Gift exchange: 1, 0, 0, 0, 0, 0
+Did not like the color: 1, 0, 0, 0, 0, 0
+Did not like the model: 1, 0, 0, 0, 0, 0
+Did not like the make: 1, 0, 0, 0, 0, 0
+Did not like the warranty: 1, 0, 0, 0, 0, 0
+No service location in my area: 1, 0, 0, 0, 0, 0
+Found a better price in a store: 1, 0, 0, 0, 0, 0
+Found a better extended warranty in a store: 1, 0, 0, 0, 0, 0
+Not working any more: 1, 0, 0, 0, 0, 0
+Did not fit: 1, 0, 0, 0, 0, 0
+Wrong size: 1, 0, 0, 0, 0, 0
+Lost my job: 1, 0, 0, 0, 0, 0
+unauthoized purchase: 1, 0, 0, 0, 0, 0
+duplicate purchase: 1, 0, 0, 0, 0, 0
+its is a boy: 1, 0, 0, 0, 0, 0
+it is a girl: 1, 0, 0, 0, 0, 0
+reason 23: 1, 0, 0, 0, 0, 0
+reason 24: 1, 0, 0, 0, 0, 0
+reason 25: 1, 0, 0, 0, 0, 0
+reason 26: 1, 0, 0, 0, 0, 0
+reason 27: 1, 0, 0, 0, 0, 0
+reason 28: 1, 0, 0, 0, 0, 0
+reason 29: 1, 0, 0, 0, 0, 0
+reason 30: 1, 0, 0, 0, 0, 0
+reason 31: 1, 0, 0, 0, 0, 0
+reason 32: 1, 0, 0, 0, 0, 0
+reason 33: 1, 0, 0, 0, 0, 0
+reason 34: 1, 0, 0, 0, 0, 0
+reason 35: 1, 0, 0, 0, 0, 0
+reason 36: 1, 1, 0, 0, 0, 0
+reason 37: 1, 1, 0, 0, 0, 0
+reason 38: 1, 1, 0, 0, 0, 0
+reason 39: 1, 1, 0, 0, 0, 0
+reason 40: 1, 1, 0, 0, 0, 0
+reason 41: 1, 1, 0, 0, 0, 0
+reason 42: 1, 1, 0, 0, 0, 0
+reason 43: 1, 1, 0, 0, 0, 0
+reason 44: 1, 1, 0, 0, 0, 0
+reason 45: 1, 1, 0, 0, 0, 0
+reason 46: 1, 1, 1, 0, 0, 0
+reason 47: 1, 1, 1, 0, 0, 0
+reason 48: 1, 1, 1, 0, 0, 0
+reason 49: 1, 1, 1, 0, 0, 0
+reason 50: 1, 1, 1, 0, 0, 0
+reason 51: 1, 1, 1, 0, 0, 0
+reason 52: 1, 1, 1, 0, 0, 0
+reason 53: 1, 1, 1, 0, 0, 0
+reason 54: 1, 1, 1, 0, 0, 0
+reason 55: 1, 1, 1, 0, 0, 0
+reason 56: 1, 1, 1, 1, 0, 0
+reason 57: 1, 1, 1, 1, 0, 0
+reason 58: 1, 1, 1, 1, 0, 0
+reason 59: 1, 1, 1, 1, 0, 0
+reason 60: 1, 1, 1, 1, 0, 0
+reason 61: 1, 1, 1, 1, 0, 0
+reason 62: 1, 1, 1, 1, 0, 0
+reason 63: 1, 1, 1, 1, 0, 0
+reason 64: 1, 1, 1, 1, 0, 0
+reason 65: 1, 1, 1, 1, 0, 0
+reason 66: 1, 1, 1, 1, 1, 0
+reason 67: 1, 1, 1, 1, 1, 0
+reason 68: 1, 1, 1, 1, 1, 0
+reason 69: 1, 1, 1, 1, 1, 0
+reason 70: 1, 1, 1, 1, 1, 0
+reason 71: 1, 1, 1, 1, 1, 1
+reason 72: 1, 1, 1, 1, 1, 1
+reason 73: 1, 1, 1, 1, 1, 1
+reason 74: 1, 1, 1, 1, 1, 1
+reason 75: 1, 1, 1, 1, 1, 1

tpcdsgen/data/return_reasons_c.dst

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from tpcdsgen!")
+
+
+if __name__ == "__main__":
+    main()

tpcdsgen/main.py

@@ -0,0 +1,10 @@
+[project]
+name = "tpcdsgen"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.14"
+dependencies = [
+    "datafusion>=53.0.0",
+    "pyarrow>=24.0.0",
+]

tpcdsgen/pyproject.toml

@@ -161,7 +161,7 @@ Compares Rust-generated output for a single table against the Java reference fix
 [INFO] Comparing outputs...
 [INFO] Java fixture: 6 rows, 4.0K
 [INFO] Rust output:  6 rows, 4.0K
-[SUCCESS] ✓ call_center: Outputs match exactly (6 rows)
+[SUCCESS] ✓ call_center: MD5 match (6 rows, cc9aabc63eb8603bd7330b6735ed0961)
 [INFO] =========================================
 ```
 
@@ -210,7 +210,7 @@ Runs comparison tests for all tables that have been ported to Rust. This is the
 
 [INFO] Testing: call_center
 ...
-[SUCCESS] ✓ call_center: Outputs match exactly (6 rows)
+[SUCCESS] ✓ call_center: MD5 match (6 rows, cc9aabc63eb8603bd7330b6735ed0961)
 ...
 
 [INFO] =========================================
@@ -324,15 +324,6 @@ These scripts are designed to be CI-friendly:
 
 Exit codes make it easy to fail CI on mismatches.
 
----
-
-## TODOs
-
-- [ ] Support multiple scale factors (scale-10, scale-100)
-- [ ] MD5 hash validation (faster than full diff for large tables)
-
----
-
 ## Notes
 
 - **Fixtures are gitignored** - They're generated artifacts, not source code

tpcdsgen/scripts/README.md

@@ -1,12 +1,12 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # TPC-DS Benchmark Script
 # Measures generation time for all tables at scale factors 1, 10, and 100
 #
 # Usage: ./scripts/benchmark.sh [--no-output] [--json] [--scales "1 10 100"]
 #
 
-set -e
+set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
@@ -84,9 +84,6 @@ else
     CLEANUP_DIR=false
 fi
 
-# Results file for comparison
-RESULTS_FILE="$PROJECT_DIR/benchmark_results_$(date +%Y%m%d_%H%M%S).txt"
-
 echo "=========================================="
 echo "TPC-DS Rust Generator Benchmark"
 echo "=========================================="

tpcdsgen/scripts/benchmark.sh

@@ -109,7 +109,7 @@ find_java_jar() {
     local jar_pattern="$JAVA_DIR/target/tpcds-*-jar-with-dependencies.jar"
     local jar_file
 
-    jar_file=$(ls $jar_pattern 2>/dev/null | head -1)
+    jar_file=$(find "$JAVA_DIR/target" -name "tpcds-*-jar-with-dependencies.jar" 2>/dev/null | head -1)
 
     if [[ -z "$jar_file" ]]; then
         return 1
@@ -132,7 +132,7 @@ clone_java_repo() {
         if [[ -d "$JAVA_DIR/.git" ]]; then
             log_info "Existing git repository found, pulling latest changes..."
             cd "$JAVA_DIR"
-            git pull origin master || log_warn "Failed to pull latest changes"
+            git pull || log_warn "Failed to pull latest changes"
             cd - >/dev/null
             return 0
         else