[pull] trunk from spiceai:trunk

.github/actions/configure-apt/action.yml

@@ -34,11 +34,9 @@ runs:
         #
         # Verified on https://launchpad.net/ubuntu/+archivemirrors :
         #   - mirror.pit.teraswitch.com       (Pittsburgh, PA - Teraswitch)
-        #   - mirror.cs.pitt.edu              (Pittsburgh, PA - U. of Pittsburgh)
         #   - mirrors.edge.kernel.org         (global kernel.org CDN)
         ARCHIVE_MIRRORS=(
           "http://mirror.pit.teraswitch.com/ubuntu/"
-          "http://mirror.cs.pitt.edu/ubuntu/archive/"
           "http://mirrors.edge.kernel.org/ubuntu/"
           "http://archive.ubuntu.com/ubuntu/"
         )

.github/workflows/integration.yml

@@ -150,47 +150,59 @@ jobs:
         with:
           name: integration-test-archive
           path: ./integration.tar.zst
-          retention-days: 1
+          retention-days: 3
+          # Archive is already zstd-compressed; skip artifact zip compression.
+          compression-level: 0
 
       - name: Upload retention OOM test archive
         if: needs.check_changes.outputs.relevant_changes == 'true'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: integration-retention-oom-test-archive
           path: ./retention_oom.tar.zst
-          retention-days: 1
+          retention-days: 3
+          # Archive is already zstd-compressed; skip artifact zip compression.
+          compression-level: 0
 
       - name: Upload AWS SDK test archive
         if: needs.check_changes.outputs.relevant_changes == 'true'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: integration-aws-sdk-test-archive
           path: ./integration_aws_sdk.tar.zst
-          retention-days: 1
+          retention-days: 3
+          # Archive is already zstd-compressed; skip artifact zip compression.
+          compression-level: 0
 
       - name: Upload AWS SDK credential bridge test archive
         if: needs.check_changes.outputs.relevant_changes == 'true'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: integration-aws-sdk-credential-bridge-test-archive
           path: ./integration_aws_sdk_credential_bridge.tar.zst
-          retention-days: 1
+          retention-days: 3
+          # Archive is already zstd-compressed; skip artifact zip compression.
+          compression-level: 0
 
       - name: Upload partition table test archive
         if: needs.check_changes.outputs.relevant_changes == 'true'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: integration-partition-table-test-archive
           path: ./partition_table_test.tar.zst
-          retention-days: 1
+          retention-days: 3
+          # Archive is already zstd-compressed; skip artifact zip compression.
+          compression-level: 0
 
       - name: Upload data components test archive
         if: needs.check_changes.outputs.relevant_changes == 'true'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: integration-data-components-test-archive
           path: ./data_components_hadoop.tar.zst
-          retention-days: 1
+          retention-days: 3
+          # Archive is already zstd-compressed; skip artifact zip compression.
+          compression-level: 0
 
   test:
     name: Integration Tests (part ${{ matrix.label }})

.github/workflows/integration_llms.yml

@@ -116,7 +116,9 @@ jobs:
         with:
           name: llms-integration-test-binary
           path: ./llms_integration_test
-          retention-days: 1
+          retention-days: 3
+          # Test binary is already a compact native binary; skip artifact zip compression.
+          compression-level: 0
 
   test:
     runs-on: ${{ matrix.target.runner }}

.github/workflows/integration_models.yml

@@ -134,7 +134,9 @@ jobs:
         with:
           name: ai-integration-test-archive
           path: ./integration.tar.zst
-          retention-days: 1
+          retention-days: 3
+          # Archive is already zstd-compressed; skip artifact zip compression.
+          compression-level: 0
 
       - name: Kill spiceio
         if: always() && runner.os == 'macOS' && steps.setup-spiceio.outputs.pid != ''
@@ -445,7 +447,9 @@ jobs:
         with:
           name: ai-integration-test-archive-extended
           path: ./integration-extended.tar.zst
-          retention-days: 1
+          retention-days: 3
+          # Archive is already zstd-compressed; skip artifact zip compression.
+          compression-level: 0
 
       - name: Kill spiceio
         if: always() && runner.os == 'macOS' && steps.setup-spiceio.outputs.pid != ''

.github/workflows/pr.yml

@@ -58,7 +58,7 @@ jobs:
   # `on.pull_requests.paths-ignore`. Instead, we conditionally run the jobs (which report
   # a success when skipped).
   check_changes:
-    runs-on: spiceai-dev-runners
+    runs-on: spiceai-macos
     outputs:
       relevant_changes: ${{ steps.check.outputs.relevant_changes }}
     steps:
@@ -79,8 +79,10 @@ jobs:
   # Run Rust linting
   lint-rust:
     name: Rust Lint
-    runs-on: spiceai-dev-runners
+    runs-on: spiceai-macos
     needs: check_changes
+    env:
+      HAS_SPICEIO_SECRET: ${{ secrets.UNAS_SMB_PASS != '' }}
 
     steps:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
@@ -89,7 +91,7 @@ jobs:
           fetch-depth: 1
 
       - name: Cache Cargo registry
-        if: needs.check_changes.outputs.relevant_changes == 'true'
+        if: needs.check_changes.outputs.relevant_changes == 'true' && runner.os == 'Linux'
         uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
         with:
           path: |
@@ -104,11 +106,23 @@ jobs:
         if: needs.check_changes.outputs.relevant_changes == 'true'
         uses: ./.github/actions/setup-rust
 
+      - name: Set up spiceio
+        if: needs.check_changes.outputs.relevant_changes == 'true' && runner.os == 'macOS' && env.HAS_SPICEIO_SECRET == 'true'
+        id: setup-spiceio
+        uses: spiceai/spiceio/.github/actions/setup@18e298c86831c6969a0ff16ee827fe2fd816df7d # v0.5.1
+        with:
+          smb-url: smb://runner@192.168.3.148/ai_platform_dev
+          smb-pass: ${{ secrets.UNAS_SMB_PASS }}
+          token: ${{ github.token }}
+          bucket: sccache
+          region: us-west-1
+
       - name: Set up sccache
         if: needs.check_changes.outputs.relevant_changes == 'true'
         uses: ./.github/actions/setup-sccache
         with:
           minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
+          spiceio_endpoint: ${{ steps.setup-spiceio.outputs.endpoint }}
 
       - name: Install Protoc
         if: needs.check_changes.outputs.relevant_changes == 'true'
@@ -143,6 +157,18 @@ jobs:
         if: ${{ needs.check_changes.outputs.relevant_changes == 'true' && env.SCCACHE_SETUP == 'true' }}
         run: sccache --show-stats
 
+      - name: Kill spiceio
+        if: always() && runner.os == 'macOS' && steps.setup-spiceio.outputs.pid != ''
+        run: kill ${{ steps.setup-spiceio.outputs.pid }} || true
+
+      - name: Upload spiceio logs
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: spiceio-lint-logs
+          path: ${{ runner.temp }}/spiceio.log
+          if-no-files-found: ignore
+
   # Build and test
   build-test:
     name: Build and Test
@@ -156,7 +182,7 @@ jobs:
           fetch-depth: 1
 
       - name: Cache Cargo registry
-        if: needs.check_changes.outputs.relevant_changes == 'true'
+        if: needs.check_changes.outputs.relevant_changes == 'true' && runner.os == 'Linux'
         uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
         with:
           path: |
@@ -236,6 +262,44 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }}
         run: cargo build -p testoperator --all-features --locked
 
+      # The install-* targets hardcode target/debug/ and target/release/ paths, so the workflow-level
+      # RUST_PROFILE=lint must be overridden to match (otherwise binaries land in target/lint/).
+      - name: make install-dev
+        if: needs.check_changes.outputs.relevant_changes == 'true'
+        env:
+          RUSTC_WRAPPER: ${{ env.RUSTC_WRAPPER }}
+          RUST_PROFILE: dev
+          AWS_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }}
+        run: make install-dev
+
+      - name: make install
+        if: needs.check_changes.outputs.relevant_changes == 'true'
+        env:
+          RUSTC_WRAPPER: ${{ env.RUSTC_WRAPPER }}
+          RUST_PROFILE: release
+          AWS_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }}
+        run: make install
+
+      - name: make install-cli
+        if: needs.check_changes.outputs.relevant_changes == 'true'
+        env:
+          RUSTC_WRAPPER: ${{ env.RUSTC_WRAPPER }}
+          RUST_PROFILE: release
+          AWS_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }}
+        run: make install-cli
+
+      - name: make install-runtime
+        if: needs.check_changes.outputs.relevant_changes == 'true'
+        env:
+          RUSTC_WRAPPER: ${{ env.RUSTC_WRAPPER }}
+          RUST_PROFILE: release
+          AWS_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }}
+        run: make install-runtime
+
       - name: Show sccache stats
         if: ${{ needs.check_changes.outputs.relevant_changes == 'true' && env.SCCACHE_SETUP == 'true' }}
         run: sccache --show-stats