ci.yaml

name: CI

on:
  push:
    tags:
      - "v*.*.*"
    branches:
      - main
      - dev
  pull_request:

env:
  CARGO_TERM_COLOR: always
  ENVIRONMENT: ${{ github.event_name == 'push' && github.ref_type == 'tag' && (!contains(github.ref, '-dev.')) && 'prod' || 'dev' }}

jobs:
  tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-is
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
          components: rustfmt, clippy
      - uses: davidB/rust-cargo-make@v1
      - name: Check formatting and clippy rules
        run: cargo make check
      - name: Run tests
        run: cargo make unit-tests
  build-all:
    runs-on: ubuntu-latest
    if: "!startsWith(github.ref, 'refs/tags/v')"
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-is
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - uses: davidB/rust-cargo-make@v1
      - uses: cargo-bins/cargo-binstall@main
      - name: Build all
        run: cargo make build
  build-test-components:
    runs-on: blacksmith-8vcpu-ubuntu-2404
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-is
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - uses: davidB/rust-cargo-make@v1
      - uses: cargo-bins/cargo-binstall@main
      - name: Install cargo-component
        run: cargo binstall --force cargo-component@0.21.1
      - name: Install Golem CLI
        run: |
          set -e
          echo "Installing Golem CLI"
          sudo curl -L https://github.com/golemcloud/golem/releases/download/v1.5.1-rc1/golem-cli-x86_64-unknown-linux-gnu -o ./golem-cli
          sudo chmod +x ./golem-cli
          sudo mv ./golem-cli /usr/local/bin/golem-cli
          echo "Installed Golem CLI"
      - name: Install wac-cli
        run: cargo binstall --force wac-cli --locked --no-confirm
      - name: Build all test components
        run: cargo make release-build-test-components
  golem-ai-exec-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-ollama-integration
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - uses: davidB/rust-cargo-make@v1
      - uses: cargo-bins/cargo-binstall@main
      - name: Install tools
        run: |
          set -e
          cargo binstall --force cargo-component@0.21.1
          cargo binstall wac-cli --locked --force --no-confirm
      - name: Install and Run latest Golem Server
        run: |
          set -e
          echo "Installing Golem server"
          sudo curl -L https://github.com/golemcloud/golem/releases/download/v1.5.0-rc5/golem-x86_64-unknown-linux-gnu -o ./golem
          sudo chmod +x ./golem
          sudo mv ./golem /usr/local/bin/golem
          golem --version
          golem profile switch local
          nohup golem server run >golem-server.log 2>&1 &
          echo "Golem server started."
      - name: Build and test golem:exec implementations
        run: |
          set -eo pipefail
          cargo make --cwd exec release-build
          cd test/exec
          golem profile config local set-format json
          export GOLEM_PRESET=release
          golem deploy --yes
          golem agent invoke 'ExecJsTest("test-1")' test01 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecJsTest("test-2")' test02 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecJsTest("test-3")' test03 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecJsTest("test-4")' test04 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecJsTest("test-5")' test05 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecJsTest("test-6")' test06 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecJsTest("test-7")' test07 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecJsTest("test-8")' test08 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecJsTest("test-9")' test09 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecJsTest("test-10")' test10 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecJsTest("test-11")' test11 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecPyTest("test-1")' test1 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecPyTest("test-2")' test2 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecPyTest("test-3")' test3 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecPyTest("test-4")' test4 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecPyTest("test-5")' test5 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecPyTest("test-6")' test6 --no-stream | jq -e '(.result_wave[0]=="true")'
          golem agent invoke 'ExecPyTest("test-7")' test7 --no-stream | jq -e '(.result_wave[0]=="true")'

  ollama-integration-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-ollama-integration
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - uses: davidB/rust-cargo-make@v1
      - uses: cargo-bins/cargo-binstall@main
      - name: Install tools
        run: |
          set -e
          cargo binstall --force cargo-component@0.21.1
          cargo binstall wac-cli --locked --force --no-confirm
      - name: Start Ollama in Docker
        run: |
          set -e
          docker run -d --name ollama -p 11434:11434 ollama/ollama:latest
          timeout 60 bash -c 'until curl -f http://localhost:11434/api/version; do sleep 2; done'
          echo "Pulling Qwen3:1.7b"
          docker exec ollama ollama pull qwen3:1.7b
          echo "Pulling Gemma3:4b"
          docker exec ollama ollama pull gemma3:4b
          echo "Verifying models are available"
          docker exec ollama ollama list | grep -q "qwen3:1.7b" || exit 1
          docker exec ollama ollama list | grep -q "gemma3:4b" || exit 1
          echo "Ollama setup completed."
      - name: Install and Run latest Golem Server
        run: |
          set -e
          echo "Installing Golem server"
          sudo curl -L https://github.com/golemcloud/golem/releases/download/v1.5.0-rc5/golem-x86_64-unknown-linux-gnu -o ./golem
          sudo chmod +x ./golem
          sudo mv ./golem /usr/local/bin/golem
          golem --version
          golem profile switch local
          nohup golem server run >golem-server.log 2>&1 &
          echo "Golem server started."
      - name: Build and test Ollama integration
        run: |
          set -eo pipefail
          cargo make --cwd llm build-ollama
          cd test/llm
          golem deploy --preset ollama-debug --yes
          golem agent new -e GOLEM_OLLAMA_BASE_URL=http://localhost:11434 'LlmTest("ollama-1")'
          golem agent invoke 'LlmTest("ollama-1")' test1 | grep -v "ERROR: "
          golem agent invoke 'LlmTest("ollama-1")' test2 | grep -v "ERROR: "
          golem agent invoke 'LlmTest("ollama-1")' test3 | grep -v "ERROR: "
          golem agent invoke 'LlmTest("ollama-1")' test4 | grep -v "ERROR: "
          golem agent invoke 'LlmTest("ollama-1")' test5 | grep -v "ERROR: "
          golem agent invoke 'LlmTest("ollama-1")' test6 | grep -v "ERROR: "
          golem agent invoke 'LlmTest("ollama-1")' test7 | grep -v "ERROR: "
          golem agent invoke 'LlmTest("ollama-1")' test8 | grep -v "ERROR: "
          golem agent invoke 'LlmTest("ollama-1")' test9 | grep -v "ERROR: "

  graph-integration-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-graph-integration
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - uses: davidB/rust-cargo-make@v1
      - uses: cargo-bins/cargo-binstall@main
      - name: Install tools
        run: |
          set -e
          cargo binstall --force cargo-component@0.21.1
          cargo binstall wac-cli --locked --force --no-confirm
      - name: Start graph-providers in Docker
        run: |
          set -e
          echo "Setting up graph database providers..."

          # ArangoDB
          echo "Starting ArangoDB..."
          docker run -d --name arangodb -e ARANGO_NO_AUTH=1 -p 8529:8529 -v arangodb_data:/var/lib/arangodb3 -v arangodb_apps:/var/lib/arangodb3-apps arangodb/arangodb:latest
          timeout 60 bash -c 'until curl -f http://localhost:8529/_api/version; do sleep 2; done'
          echo "Creating test database..."
          curl -X POST http://localhost:8529/_api/database \
            -H "Content-Type: application/json" \
            -d '{"name": "test"}' || echo "Database might already exist"
          echo "ArangoDB is ready"

          # JanusGraph with Cassandra
          echo "Starting Cassandra for JanusGraph..."
          docker network create janus-net || true
          docker run -d --name cassandra --network janus-net -p 9042:9042 cassandra:3.11
          echo "Waiting for Cassandra to be ready..."
          timeout 60 bash -c 'until docker exec cassandra cqlsh -e "DESCRIBE KEYSPACES" > /dev/null 2>&1; do sleep 5; done'

          echo "Starting JanusGraph..."
          docker run -d --name janusgraph --network janus-net \
            -e storage.backend=cassandra \
            -e storage.hostname=cassandra \
            -e gremlinserver.channelizer=org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer \
            -e GREMLIN_OPTS="-Dtx.max-commit-time=60000 -Dstorage.cassandra.read-consistency-level=ONE -Dstorage.cassandra.write-consistency-level=ONE" \
            -p 8182:8182 janusgraph/janusgraph:latest
          echo "JanusGraph is ready"

          # Setup Neo4j
          echo "Starting Neo4j..."
          docker run -d --name neo4j \
            --publish=7474:7474 --publish=7687:7687 \
            --volume=neo4j_data:/data \
            --env=NEO4J_AUTH=neo4j/password \
            neo4j:latest
          echo "Waiting for Neo4j to be ready..."
          timeout 120 bash -c 'until curl -f http://localhost:7474/ > /dev/null 2>&1; do sleep 5; done'
          echo "Neo4j is ready"

          echo "All graph database providers are ready!"
      - name: Install and Run latest Golem Server
        run: |
          set -e
          echo "Installing Golem server"
          sudo curl -L https://github.com/golemcloud/golem/releases/download/v1.5.0-rc5/golem-x86_64-unknown-linux-gnu -o ./golem
          sudo chmod +x ./golem
          sudo mv ./golem /usr/local/bin/golem
          golem --version
          golem profile switch local
          nohup golem server run >golem-server.log 2>&1 &
          echo "Golem server started."
      - name: Build and test graph integration
        run: |
          set -eo pipefail
          cargo make --cwd graph release-build-arangodb
          cargo make --cwd graph release-build-janusgraph
          cargo make --cwd graph release-build-neo4j
          cd test/graph

          echo "Testing ArangoDB integration..."
          golem deploy --preset arangodb-release --yes
          golem agent new -e ARANGODB_HOST=localhost -e ARANGODB_USER="" -e ARANGODB_PASSWORD="" -e ARANGODB_PORT="8529" -e ARANGODB_DATABASE="test" 'GraphTest("arangodb-1")'
          golem agent invoke 'GraphTest("arangodb-1")' test1 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("arangodb-1")' test2 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("arangodb-1")' test3 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("arangodb-1")' test4 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("arangodb-1")' test5 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("arangodb-1")' test6 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("arangodb-1")' test7 | grep -v "ERROR: "
          echo "ArangoDB tests completed successfully"
          golem clean -P arangodb-release

          echo "Testing JanusGraph integration..."
          golem deploy --preset janusgraph-release --yes
          sleep 10
          golem agent new -e JANUSGRAPH_HOST=localhost -e JANUSGRAPH_USER="" -e JANUSGRAPH_PASSWORD="" -e JANUSGRAPH_PORT="8182" 'GraphTest("janusgraph-1")'
          golem agent invoke 'GraphTest("janusgraph-1")' test1 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("janusgraph-1")' test2 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("janusgraph-1")' test3 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("janusgraph-1")' test4 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("janusgraph-1")' test5 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("janusgraph-1")' test6 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("janusgraph-1")' test7 | grep -v "ERROR: "
          echo "JanusGraph tests completed successfully"
          golem clean -P janusgraph-release

          echo "Testing Neo4j integration..."
          golem deploy --preset neo4j-release --yes
          sleep 10
          golem agent new -e NEO4J_HOST=localhost -e NEO4J_USER=neo4j -e NEO4J_PASSWORD=password -e NEO4J_PORT="7474" -e NEO4J_DATABASE=neo4j 'GraphTest("neo4j-1")'
          golem agent invoke 'GraphTest("neo4j-1")' test1 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("neo4j-1")' test2 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("neo4j-1")' test3 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("neo4j-1")' test4 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("neo4j-1")' test5 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("neo4j-1")' test6 | grep -v "ERROR: "
          golem agent invoke 'GraphTest("neo4j-1")' test7 | grep -v "ERROR: "
          echo "Neo4j tests completed successfully"
          golem clean -P neo4j-release

          echo "All graph integration tests completed successfully!"

  vector-integration-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-vector-integration
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - uses: davidB/rust-cargo-make@v1
      - uses: cargo-bins/cargo-binstall@main
      - name: Install tools
        run: |
          set -e
          cargo binstall --force cargo-component@0.21.1
          cargo binstall wac-cli --locked --force --no-confirm
      - name: Start vector-providers in Docker
        run: |
          set -e
          echo "Setting up vector database providers..."

          # Milvus
          echo "Starting Milvus..."
          cd /home/runner
          mkdir milvus-setup && cd milvus-setup
          curl -sfL https://raw.githubusercontent.com/milvus-io/milvus/2.6/scripts/standalone_embed.sh -o standalone_embed.sh
          sleep 30
          bash standalone_embed.sh start
          cd $GITHUB_WORKSPACE
          echo "Milvus is ready"

          # PGVECTOR
          echo "Starting PGVECTOR..."
          docker pull pgvector/pgvector:0.8.1-pg18-trixie
          sleep 30
          docker run -d \
            --name pgvector-db \
            -p 3000:5432 \
            -e POSTGRES_USER=postgres \
            -e POSTGRES_PASSWORD=mysecretpassword \
            -e POSTGRES_DB=mydatabase \
            pgvector/pgvector:0.8.1-pg18-trixie

          echo "Waiting for pgvector to be ready..."

          until docker exec pgvector-db pg_isready -U postgres -d mydatabase -q; do
            echo "Waiting for pgvector database to be ready..."
            sleep 2
          done

          echo "Database is ready. Creating extension..."
          docker exec pgvector-db psql -U postgres -d mydatabase -c "CREATE EXTENSION IF NOT EXISTS vector;"
          echo "PGVECTOR is ready"

          # Qdrant
          echo "Starting Qdrant..."
          docker pull qdrant/qdrant

          docker run -d --name qdrant-db -p 6333:6333 -p 6334:6334 qdrant/qdrant

          echo "Waiting for Qdrant to be ready..."
          sleep 10
          echo "Qdrant is ready"

          echo "All Vector database providers are ready!"
      - name: Install and Run latest Golem Server
        run: |
          set -e
          echo "Installing Golem server"
          sudo curl -L https://github.com/golemcloud/golem/releases/download/v1.5.0-rc5/golem-x86_64-unknown-linux-gnu -o ./golem
          sudo chmod +x ./golem
          sudo mv ./golem /usr/local/bin/golem
          golem --version
          golem profile switch local
          nohup golem server run >golem-server.log 2>&1 &
          echo "Golem server started."
      - name: Build and test vector integration
        run: |
          set -eo pipefail
          cargo make --cwd vector release-build-milvus
          cargo make --cwd vector release-build-pgvector
          cargo make --cwd vector release-build-qdrant
          cd test/vector

          echo "Testing milvus integration..."
          export GOLEM_PRESET=milvus-release
          golem deploy --yes
          golem agent new -e MILVUS_URI=http://127.0.0.1:19530 -e MILVUS_DATABASE=default 'VectorTest("milvus-1")'
          golem agent invoke 'VectorTest("milvus-1")' test1 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("milvus-1")' test2 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("milvus-1")' test3 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("milvus-1")' test4 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("milvus-1")' test5 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("milvus-1")' test6 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("milvus-1")' test7 | grep -v "ERROR: "
          echo "Milvus tests completed successfully"
          golem clean

          # FIXME: reenable after golem-cli bump
          # echo "Testing pgvector integration..."
          # export GOLEM_PRESET=pgvector-release
          # golem deploy --yes
          # sleep 10
          # golem agent new -e PGVECTOR_CONNECTION_STRING=postgresql://postgres:mysecretpassword@localhost:3000/mydatabase 'VectorTest("pgvector-1")'
          # golem agent invoke 'VectorTest("pgvector-1")' test1 | grep -v "ERROR: "
          # golem agent invoke 'VectorTest("pgvector-1")' test2 | grep -v "ERROR: "
          # golem agent invoke 'VectorTest("pgvector-1")' test3 | grep -v "ERROR: "
          # golem agent invoke 'VectorTest("pgvector-1")' test4 | grep -v "ERROR: "
          # golem agent invoke 'VectorTest("pgvector-1")' test5 | grep -v "ERROR: "
          # golem agent invoke 'VectorTest("pgvector-1")' test6 | grep -v "ERROR: "
          # golem agent invoke 'VectorTest("pgvector-1")' test7 | grep -v "ERROR: "
          # echo "pgvector tests completed successfully"
          # golem clean

          echo "Testing qdrant integration..."
          export GOLEM_PRESET=qdrant-release
          golem deploy --yes
          sleep 10
          golem agent new -e QDRANT_URL=http://127.0.0.1:6333 'VectorTest("qdrant-1")'
          golem agent invoke 'VectorTest("qdrant-1")' test1 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("qdrant-1")' test2 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("qdrant-1")' test3 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("qdrant-1")' test4 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("qdrant-1")' test5 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("qdrant-1")' test6 | grep -v "ERROR: "
          golem agent invoke 'VectorTest("qdrant-1")' test7 | grep -v "ERROR: "
          echo "Qdrant tests completed successfully"
          golem clean

          echo "All vector integration tests completed successfully!"

  publish-all:
    needs:
      - tests
      - build-test-components
      - ollama-integration-tests
      - graph-integration-tests
      - golem-ai-exec-tests
      - vector-integration-tests
    runs-on: ubuntu-latest
    permissions:
      contents: write
    if: "startsWith(github.ref, 'refs/tags/v')"
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-is
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - uses: davidB/rust-cargo-make@v1
      - uses: cargo-bins/cargo-binstall@main
      - name: Build all components in release
        run: cargo make release-build
      - name: Publish crates to crates.io
        env:
          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
        run: |
          set -e

          # Extract version from tag (strip leading 'v')
          VERSION="${GITHUB_REF_NAME#v}"
          echo "Publishing version: $VERSION"

          # Install cargo-edit for set-version
          cargo binstall --force cargo-edit --no-confirm

          # Set version on all workspace crates
          cargo set-version --workspace "$VERSION"

          PUBLISH_FLAGS="--no-verify --allow-dirty"

          # Publish core crates first (no internal deps)
          echo "Publishing core crates..."
          for crate in \
            llm/llm \
            embed/embed \
            websearch/websearch \
            search/search \
            graph/graph \
            video/video \
            exec/exec \
            stt/stt \
            tts/tts \
            vector/vector; do
            echo "Publishing $crate..."
            cargo publish -p "$(grep '^name' "$crate/Cargo.toml" | head -1 | sed 's/.*"\(.*\)"/\1/')" $PUBLISH_FLAGS
            sleep 30
          done

          # Publish provider crates (depend on core crates)
          echo "Publishing provider crates..."
          for crate in \
            llm/anthropic \
            llm/bedrock \
            llm/grok \
            llm/ollama \
            llm/openai \
            llm/openrouter \
            embed/cohere \
            embed/hugging-face \
            embed/openai \
            embed/voyageai \
            websearch/brave \
            websearch/google \
            websearch/serper \
            websearch/tavily \
            search/algolia \
            search/elasticsearch \
            search/meilisearch \
            search/opensearch \
            search/typesense \
            graph/arangodb \
            graph/janusgraph \
            graph/neo4j \
            video/kling \
            video/runway \
            video/stability \
            video/veo \
            stt/aws \
            stt/azure \
            stt/deepgram \
            stt/google \
            stt/whisper \
            tts/aws \
            tts/deepgram \
            tts/elevenlabs \
            tts/google \
            vector/milvus \
            vector/pgvector \
            vector/pinecone \
            vector/qdrant; do
            echo "Publishing $crate..."
            cargo publish -p "$(grep '^name' "$crate/Cargo.toml" | head -1 | sed 's/.*"\(.*\)"/\1/')" $PUBLISH_FLAGS
            sleep 30
          done

          echo "All crates published successfully!"