Test for s3

Author: matthewmturner

.github/actions/setup-rust/action.yml

@@ -37,4 +37,4 @@ runs:
       shell: bash
       run: |
         cargo install taplo-cli --locked
-        cargo install cargo-machete
+        cargo install cargo-machete just

.github/workflows/test.yml

@@ -72,6 +72,31 @@ jobs:
           # test to only be run against the server spun up in that test.  With parallelism tests 
           # can connec to server in different test which breaks determinism.
           cargo test --features=flightsql extension_cases::flightsql -- --test-threads=1
+  test-cli:
+    name: App / CLI
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Rust Toolchain
+        uses: ./.github/actions/setup-rust
+      - name: Start LocalStack
+        uses: LocalStack/setup-localstack@v0.2.3
+        with:
+          image-tag: 'latest'
+          install-awslocal: 'true'
+          configuration: DEBUG=1
+      - name: Run Tests against LocalStack
+        run: |
+          awslocal s3 mb s3://test
+          awslocal s3 mv data/aggregate_test_100.csv s3://test/
+          awslocal s3 mb s3://tpch-db
+          echo "Test Execution complete!"
+      - name: Run CLI tests
+        run: |
+          cargo test cli_cases
   test-s3:
     name: Extension / S3
     runs-on: ubuntu-latest

Cargo.toml

@@ -81,7 +81,7 @@ url = "2.5.2"
 
 # When addding a new feature, also add it to the features tested list in CI (`.github/workflows/rust.yml`)
 [features]
-default = ["functions-parquet"]
+default = ["functions-parquet", "s3"]
 deltalake = ["datafusion-app/deltalake"]
 flightsql = [
   "datafusion-app/flightsql",

justfile

@@ -36,3 +36,7 @@ bench-http-custom file:
     echo "Running bench on $custom_bench_path"
     echo ""
     oha --urls-from-file "$custom_bench_path"
+
+setup-test-env:
+    localstack start -d
+    awslocal s3api create-bucket --bucket tmp --acl public-read

tests/cli_cases/tpch.rs

@@ -19,6 +19,7 @@
 
 use crate::config::TestConfigBuilder;
 use assert_cmd::Command;
+use object_store::{aws::AmazonS3Builder, path::Path, ObjectStore};
 
 #[test]
 fn test_custom_config() {
@@ -50,3 +51,55 @@ fn test_custom_config() {
         assert!(metadata.len() > 1000);
     })
 }
+
+#[tokio::test]
+async fn test_custom_config_with_s3() {
+    let mut config_builder = TestConfigBuilder::default();
+    config_builder.with_db_path("s3://tpch-db/db/");
+    let bucket = "tpch-db";
+    let endpoint = "http://localhost:4566";
+    let access_key = "LSIAQAAAAAAVNCBMPNSG";
+    let secret = "5555555555555555555555555555555555555555";
+    let allow_http = true;
+    config_builder.with_s3_object_store(
+        "cli",
+        "s3",
+        bucket,
+        "s3://tpch-db",
+        endpoint,
+        access_key,
+        secret,
+        allow_http,
+    );
+    let config = config_builder.build("my_config.toml");
+
+    Command::cargo_bin("dft")
+        .unwrap()
+        .arg("--config")
+        .arg(config.path)
+        .arg("generate-tpch")
+        .assert()
+        .success();
+
+    let s3 = AmazonS3Builder::new()
+        .with_bucket_name(bucket)
+        .with_endpoint(endpoint)
+        .with_access_key_id(access_key)
+        .with_secret_access_key(secret)
+        .with_allow_http(allow_http)
+        .build()
+        .unwrap();
+
+    let r = s3
+        .list_with_delimiter(Some(&Path::parse("db/tables/dft/tpch/").unwrap()))
+        .await
+        .unwrap();
+
+    let needed_dirs = [
+        "customer", "orders", "lineitem", "nation", "part", "partsupp", "region", "supplier",
+    ];
+
+    for prefix in r.common_prefixes {
+        assert!(needed_dirs.contains(&prefix.to_string().as_str()));
+    }
+}