refactor(storage): remove native storage format

.github/actions/test_cloud_sqllogic_standalone_linux/action.yml

@@ -9,10 +9,6 @@ inputs:
     description: "logic test handlers, mysql,http,hybrid"
     required: true
     default: ""
-  storage-format:
-    description: "storage format for databend query to test"
-    required: true
-    default: all
 
 runs:
   using: "composite"
@@ -22,7 +18,6 @@ runs:
         artifacts: sqllogictests,meta,query
 
     - name: Run sqllogic Tests with Standalone mode
-      if: inputs.storage-format == 'all' || inputs.storage-format == 'parquet'
       shell: bash
       env:
         TEST_HANDLERS: ${{ inputs.handlers }}

.github/actions/test_ee_sqllogic_standalone_linux/action.yml

@@ -9,10 +9,6 @@ inputs:
     description: "logic test handlers, mysql,http,hybrid"
     required: true
     default: ""
-  storage-format:
-    description: "storage format for databend query to test"
-    required: true
-    default: all
 
 runs:
   using: "composite"
@@ -22,7 +18,6 @@ runs:
         artifacts: sqllogictests,meta,query
 
     - name: Run sqllogic Tests with Standalone mode
-      if: inputs.storage-format == 'all' || inputs.storage-format == 'parquet'
       shell: bash
       env:
         TEST_HANDLERS: ${{ inputs.handlers }}

.github/actions/test_sqllogic_standalone_linux/action.yml

@@ -13,10 +13,6 @@ inputs:
     description: "logic test parallel"
     required: false
     default: ""
-  storage-format:
-    description: "storage format for databend query to test"
-    required: true
-    default: all
   enable_table_meta_cache:
     description: "Enable table meta cache"
     required: false
@@ -30,20 +26,9 @@ runs:
         artifacts: sqllogictests,meta,query
 
     - name: Run sqllogic Tests with Standalone mode
-      if: inputs.storage-format == 'all' || inputs.storage-format == 'parquet'
       shell: bash
       env:
         TEST_HANDLERS: ${{ inputs.handlers }}
         TEST_PARALLEL: ${{ inputs.parallel }}
         CACHE_ENABLE_TABLE_META_CACHE: ${{ inputs.enable_table_meta_cache}}
       run: bash ./scripts/ci/ci-run-sqllogic-tests.sh ${{ inputs.dirs }}
-
-    - name: Run native sqllogic Tests with Standalone mode
-      if: inputs.storage-format == 'all' || inputs.storage-format == 'native'
-      shell: bash
-      env:
-        TEST_HANDLERS: ${{ inputs.handlers }}
-        TEST_PARALLEL: ${{ inputs.parallel }}
-        TEST_EXT_ARGS: '--skip_file tpcds_spill_1.test,tpcds_spill_2.test,tpcds_spill_3.test'
-        CACHE_ENABLE_TABLE_META_CACHE: ${{ inputs.enable_table_meta_cache}}
-      run: bash ./scripts/ci/ci-run-sqllogic-tests-native.sh ${{ inputs.dirs }}

.github/actions/test_sqllogic_standalone_linux_minio/action.yml

@@ -9,10 +9,6 @@ inputs:
     description: "logic test handlers, mysql,http,hybrid"
     required: true
     default: ""
-  storage-format:
-    description: "storage format for databend query to test"
-    required: true
-    default: all
 
 runs:
   using: "composite"
@@ -24,15 +20,7 @@ runs:
     - uses: ./.github/actions/setup_minio
 
     - name: Run sqllogic Tests with Standalone mode
-      if: inputs.storage-format == 'all' || inputs.storage-format == 'parquet'
       shell: bash
       env:
         TEST_HANDLERS: ${{ inputs.handlers }}
       run: bash ./scripts/ci/ci-run-sqllogic-tests-minio.sh ${{ inputs.dirs }}
-
-    - name: Run native sqllogic Tests with Standalone mode
-      if: inputs.storage-format == 'all' || inputs.storage-format == 'native'
-      shell: bash
-      env:
-        TEST_HANDLERS: ${{ inputs.handlers }}
-      run: bash ./scripts/ci/ci-run-sqllogic-tests-native-minio.sh ${{ inputs.dirs }}

.github/workflows/reuse.sqllogic.yml

@@ -74,7 +74,6 @@ jobs:
           dirs: ${{ matrix.tests.dirs }}
           handlers: ${{ matrix.handler }}
           parallel: ${{ matrix.tests.parallel }}
-          storage-format: all
       - name: Upload failure
         if: failure()
         uses: ./.github/actions/artifact_failure
@@ -105,7 +104,6 @@ jobs:
           dirs: udf_server
           handlers: http,hybrid
           parallel: 1
-          storage-format: all
       - name: Upload failure
         if: failure() || cancelled()
         uses: ./.github/actions/artifact_failure
@@ -134,7 +132,6 @@ jobs:
         with:
           dirs: task
           handlers: http,hybrid
-          storage-format: all
       - name: Upload failure
         if: failure() || cancelled()
         uses: ./.github/actions/artifact_failure
@@ -156,22 +153,18 @@ jobs:
         handler:
           - "hybrid"
           - "http"
-        format:
-          - "native"
-          - "parquet"
     steps:
       - uses: actions/checkout@v6
       - uses: ./.github/actions/test_sqllogic_standalone_linux_minio
         timeout-minutes: 15
         with:
           dirs: ${{ matrix.dirs }}
           handlers: ${{ matrix.handler }}
-          storage-format: ${{ matrix.format }}
       - name: Upload failure
         if: failure()
         uses: ./.github/actions/artifact_failure
         with:
-          name: test-sqllogic-standalone-minio-${{ matrix.dirs }}-${{ matrix.handler }}-${{ matrix.format }}
+          name: test-sqllogic-standalone-minio-${{ matrix.dirs }}-${{ matrix.handler }}
 
   standalone_minio_with_bendsave:
     runs-on:
@@ -350,7 +343,6 @@ jobs:
         with:
           dirs: ${{ matrix.dirs }}
           handlers: ${{ matrix.handler }}
-          storage-format: all
           enable_table_meta_cache: false
       - name: Upload failure
         if: failure()
@@ -365,12 +357,6 @@ jobs:
       - Linux
       - 2c
       - "${{ inputs.runner_provider }}"
-    strategy:
-      fail-fast: false
-      matrix:
-        format:
-          - "parquet"
-          - "native"
     steps:
       - uses: actions/checkout@v6
       - uses: ./.github/actions/setup_license
@@ -382,9 +368,8 @@ jobs:
         with:
           dirs: ee
           handlers: http,hybrid
-          storage-format: ${{ matrix.format }}
       - name: Upload failure
         if: failure()
         uses: ./.github/actions/artifact_failure
         with:
-          name: test-ee-sqllogic-standalone-ee-${{ matrix.format }}
+          name: test-ee-sqllogic-standalone-ee

Cargo.toml

@@ -97,7 +97,6 @@ databend-common-meta-schema-api-test-suite = { path = "src/meta/schema-api-test-
 databend-common-meta-schema-api-tests = { path = "src/meta/schema-api-tests" }
 databend-common-meta-store = { path = "src/meta/store" }
 databend-common-metrics = { path = "src/common/metrics" }
-databend-common-native = { path = "src/common/native" }
 databend-common-pipeline = { path = "src/query/pipeline" }
 databend-common-pipeline-transforms = { path = "src/query/pipeline/transforms" }
 databend-common-proto-conv = { path = "src/meta/proto-conv" }

benchmark/benchmark_local.sh

@@ -61,7 +61,7 @@ export BENDSQL_DSN="databend://root:@localhost:8000/${BENCHMARK_DATASET}?sslmode
 
 # Create table
 if [[ -f "${BENCHMARK_DATASET}/create_local.sql" ]]; then
-    echo "Creating table for benchmark with native storage format..."
+    echo "Creating table for benchmark..."
     bendsql <"${BENCHMARK_DATASET}/create_local.sql"
 fi
 

benchmark/benchmark_local_merge_into.sh

@@ -59,7 +59,7 @@ export BENDSQL_DSN="databend://root:@localhost:8000/${BENCHMARK_DATASET}?sslmode
 echo "CREATE DATABASE ${BENCHMARK_DATASET};" | bendsql
 
 # Load the data
-echo "Creating table for benchmark with native storage format..."
+echo "Creating table for benchmark..."
 bendsql <"${BENCHMARK_DATASET}/create_local.sql"
 
 # Detect instance type

benchmark/hits/create_local.sql

@@ -104,4 +104,4 @@ CREATE TRANSIENT TABLE hits (
     RefererHash BIGINT NOT NULL,
     URLHash BIGINT NOT NULL,
     CLID INTEGER NOT NULL
-) storage_format = 'native' compression = 'lz4';
+) storage_format = 'parquet' compression = 'lz4';

benchmark/old/tpch/tpch.sh

@@ -10,11 +10,7 @@ EXPORT DATABASE '/tmp/tpch_1/' (FORMAT CSV, DELIMITER '|');
 
 mv /tmp/tpch_1/ "$(pwd)/data/"
 
-if [[ $2 == native ]]; then
-  echo "native"
-  sh ./load_data.sh "storage_format = 'native' compression = 'lz4'"
-else
-  echo "fuse"
-  sh ./load_data.sh ""
-fi
+# `parquet` is the only supported storage format; an empty option string lets
+# the table fall back to it.
+sh ./load_data.sh ""
 

benchmark/tpch/create_local.sql

@@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS customer (
     c_acctbal DECIMAL(15, 2) not null,
     c_mktsegment STRING not null,
     c_comment STRING not null
-) CLUSTER BY (c_custkey) storage_format = 'native' compression = 'lz4';
+) CLUSTER BY (c_custkey) storage_format = 'parquet' compression = 'lz4';
 
 CREATE TABLE IF NOT EXISTS lineitem (
     l_orderkey BIGINT not null,
@@ -26,14 +26,14 @@ CREATE TABLE IF NOT EXISTS lineitem (
     l_shipinstruct STRING not null,
     l_shipmode STRING not null,
     l_comment STRING not null
-) CLUSTER BY(l_shipdate, l_orderkey) storage_format = 'native' compression = 'lz4';
+) CLUSTER BY(l_shipdate, l_orderkey) storage_format = 'parquet' compression = 'lz4';
 
 CREATE TABLE IF NOT EXISTS nation (
     n_nationkey INTEGER not null,
     n_name STRING not null,
     n_regionkey INTEGER not null,
     n_comment STRING
-) CLUSTER BY (n_nationkey) storage_format = 'native' compression = 'lz4';
+) CLUSTER BY (n_nationkey) storage_format = 'parquet' compression = 'lz4';
 
 CREATE TABLE IF NOT EXISTS orders (
     o_orderkey BIGINT not null,
@@ -45,15 +45,15 @@ CREATE TABLE IF NOT EXISTS orders (
     o_clerk STRING not null,
     o_shippriority INTEGER not null,
     o_comment STRING not null
-) CLUSTER BY (o_orderkey, o_orderdate) storage_format = 'native' compression = 'lz4';
+) CLUSTER BY (o_orderkey, o_orderdate) storage_format = 'parquet' compression = 'lz4';
 
 CREATE TABLE IF NOT EXISTS partsupp (
     ps_partkey BIGINT not null,
     ps_suppkey BIGINT not null,
     ps_availqty BIGINT not null,
     ps_supplycost DECIMAL(15, 2) not null,
     ps_comment STRING not null
-) CLUSTER BY (ps_partkey) storage_format = 'native' compression = 'lz4';
+) CLUSTER BY (ps_partkey) storage_format = 'parquet' compression = 'lz4';
 
 CREATE TABLE IF NOT EXISTS part (
     p_partkey BIGINT not null,
@@ -65,13 +65,13 @@ CREATE TABLE IF NOT EXISTS part (
     p_container STRING not null,
     p_retailprice DECIMAL(15, 2) not null,
     p_comment STRING not null
-) CLUSTER BY (p_partkey) storage_format = 'native' compression = 'lz4';
+) CLUSTER BY (p_partkey) storage_format = 'parquet' compression = 'lz4';
 
 CREATE TABLE IF NOT EXISTS region (
     r_regionkey INTEGER not null,
     r_name STRING not null,
     r_comment STRING
-) CLUSTER BY (r_regionkey) storage_format = 'native' compression = 'lz4';
+) CLUSTER BY (r_regionkey) storage_format = 'parquet' compression = 'lz4';
 
 CREATE TABLE IF NOT EXISTS supplier (
     s_suppkey BIGINT not null,
@@ -81,4 +81,4 @@ CREATE TABLE IF NOT EXISTS supplier (
     s_phone STRING not null,
     s_acctbal DECIMAL(15, 2) not null,
     s_comment STRING not null
-) CLUSTER BY (s_suppkey) storage_format = 'native' compression = 'lz4';
+) CLUSTER BY (s_suppkey) storage_format = 'parquet' compression = 'lz4';