diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 7c88ef03bb7..6f8ee9baed2 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -3,10 +3,10 @@ description: "This action sets up Docker Buildx and authenticates to registries" inputs: ghcr-username: description: "Github Container Registry username" - required: false + required: true ghcr-password: description: "Github Container Registry password" - required: false + required: true dockerhub-username: description: "DockerHub username" required: true @@ -26,7 +26,6 @@ runs: uses: docker/setup-buildx-action@v2 - name: Log in to the Github Container registry uses: docker/login-action@v2.1.0 - if: ${{ inputs.ghcr-username != '' }} with: registry: ghcr.io username: ${{ inputs.ghcr-username }} @@ -36,6 +35,3 @@ runs: with: username: ${{ inputs.dockerhub-username }} password: ${{ inputs.dockerhub-password }} - - uses: useblacksmith/build-push-action@v1.1 - with: - setup-only: true diff --git a/.github/workflows/_build_release_container.yml b/.github/workflows/_build_release_container.yml index 782cd1c2ed1..19a87c0c389 100644 --- a/.github/workflows/_build_release_container.yml +++ b/.github/workflows/_build_release_container.yml @@ -66,8 +66,7 @@ jobs: uses: useblacksmith/build-push-action@v1.1 with: context: . - file: rust/Dockerfile - target: cli + file: rust/cli/Dockerfile platforms: ${{ env.PLATFORMS }} push: true build-args: | diff --git a/.github/workflows/_go-tests.yml b/.github/workflows/_go-tests.yml index cff19e384a5..317da1af997 100644 --- a/.github/workflows/_go-tests.yml +++ b/.github/workflows/_go-tests.yml @@ -15,11 +15,9 @@ jobs: uses: actions/checkout@v4 - name: Setup uses: ./.github/actions/go - - name: Set up Docker - uses: ./.github/actions/docker + - uses: useblacksmith/build-push-action@v1.1 with: - dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} - dockerhub-password: ${{ secrets.DOCKERHUB_TOKEN }} + setup-only: true - uses: ./.github/actions/tilt - run: bin/cluster-test.sh bash -c 'cd go && make test' - name: Save service logs to artifact diff --git a/.github/workflows/_python-tests.yml b/.github/workflows/_python-tests.yml index 167eaabc364..a3c1d215e86 100644 --- a/.github/workflows/_python-tests.yml +++ b/.github/workflows/_python-tests.yml @@ -179,11 +179,9 @@ jobs: # languages: python # api_key: ${{ secrets.DD_API_KEY }} # site: ${{ vars.DD_SITE }} - - name: Set up Docker - uses: ./.github/actions/docker + - uses: useblacksmith/build-push-action@v1.1 with: - dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} - dockerhub-password: ${{ secrets.DOCKERHUB_TOKEN }} + setup-only: true - uses: ./.github/actions/tilt - name: Test run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.test-globs }}"' --durations 10 diff --git a/.github/workflows/_rust-tests.yml b/.github/workflows/_rust-tests.yml index 8baaa2a0247..4a759d344e3 100644 --- a/.github/workflows/_rust-tests.yml +++ b/.github/workflows/_rust-tests.yml @@ -76,11 +76,9 @@ jobs: uses: ./.github/actions/rust with: github-token: ${{ github.token }} - - name: Set up Docker - uses: ./.github/actions/docker + - uses: useblacksmith/build-push-action@v1.1 with: - dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} - dockerhub-password: ${{ secrets.DOCKERHUB_TOKEN }} + setup-only: true - name: Start services in Tilt uses: ./.github/actions/tilt - name: Build CLI diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5b2f8c80f71..2604ce3e5c0 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -191,7 +191,6 @@ jobs: needs: change-detection if: contains(fromJson(needs.change-detection.outputs.tests-to-run), 'go') uses: ./.github/workflows/_go-tests.yml - secrets: inherit check-title: name: Check PR Title diff --git a/Tiltfile b/Tiltfile index 197cfdc116f..e3729fda78c 100644 --- a/Tiltfile +++ b/Tiltfile @@ -40,7 +40,7 @@ else: if config.tilt_subcommand == "ci": custom_build( 'rust-log-service', - 'docker buildx build --load -t $EXPECTED_REF --target=log_service -f ./rust/Dockerfile .', + 'docker buildx build --load -t $EXPECTED_REF -f ./rust/log-service/Dockerfile .', ['./rust/', './idl/', './Cargo.toml', './Cargo.lock'] ) else: @@ -48,8 +48,7 @@ else: 'rust-log-service', '.', only=["rust/", "idl/", "Cargo.toml", "Cargo.lock"], - dockerfile='./rust/Dockerfile', - target='log_service' + dockerfile='./rust/log-service/Dockerfile', ) if config.tilt_subcommand == "ci": @@ -85,7 +84,7 @@ else: if config.tilt_subcommand == "ci": custom_build( 'rust-frontend-service', - 'docker buildx build --load -t $EXPECTED_REF -f ./rust/Dockerfile --target cli . ', + 'docker buildx build --load -t $EXPECTED_REF -f ./rust/cli/Dockerfile . ', ['./rust/', './idl/', './Cargo.toml', './Cargo.lock'] ) else: @@ -93,14 +92,13 @@ else: 'rust-frontend-service', '.', only=["rust/", "idl/", "Cargo.toml", "Cargo.lock"], - dockerfile='./rust/Dockerfile', - target='cli' + dockerfile='./rust/cli/Dockerfile', ) if config.tilt_subcommand == "ci": custom_build( 'query-service', - 'docker buildx build --load -t $EXPECTED_REF --target query_service -f ./rust/Dockerfile .', + 'docker buildx build --load -t $EXPECTED_REF --target query_service -f ./rust/worker/Dockerfile .', ['./rust/', './idl/', './Cargo.toml', './Cargo.lock'] ) else: @@ -108,14 +106,14 @@ else: 'query-service', '.', only=["rust/", "idl/", "Cargo.toml", "Cargo.lock"], - dockerfile='./rust/Dockerfile', + dockerfile='./rust/worker/Dockerfile', target='query_service' ) if config.tilt_subcommand == "ci": custom_build( 'compaction-service', - 'docker buildx build --load -t $EXPECTED_REF --target compaction_service -f ./rust/Dockerfile .', + 'docker buildx build --load -t $EXPECTED_REF --target compaction_service -f ./rust/worker/Dockerfile .', ['./rust/', './idl/', './Cargo.toml', './Cargo.lock'] ) else: @@ -123,14 +121,14 @@ else: 'compaction-service', '.', only=["rust/", "idl/", "Cargo.toml", "Cargo.lock"], - dockerfile='./rust/Dockerfile', + dockerfile='./rust/worker/Dockerfile', target='compaction_service' ) if config.tilt_subcommand == "ci": custom_build( 'garbage-collector', - 'docker buildx build --load -t $EXPECTED_REF --target garbage_collector -f ./rust/Dockerfile .', + 'docker buildx build --load -t $EXPECTED_REF --target garbage_collector -f ./rust/garbage_collector/Dockerfile .', ['./rust/', './idl/', './Cargo.toml', './Cargo.lock'] ) else: @@ -138,7 +136,7 @@ else: 'garbage-collector', '.', only=["rust/", "idl/", "Cargo.toml", "Cargo.lock"], - dockerfile='./rust/Dockerfile', + dockerfile='./rust/garbage_collector/Dockerfile', target='garbage_collector' ) diff --git a/docker-compose.yml b/docker-compose.yml index 5608192cdc7..d6d68b4b531 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,7 @@ services: image: server build: context: . - dockerfile: rust/Dockerfile - target: cli + dockerfile: rust/cli/Dockerfile # Uncomment below to build in release mode # args: # RELEASE_MODE: "1" diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 52e8885de2e..556550a209a 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,8 @@ [toolchain] -# Keep this version in sync with rust/Dockerfile. +# Keep this version in sync with: +# - rust/cli/Dockerfile +# - rust/load/Dockerfile +# - rust/log/Dockerfile +# - rust/worker/Dockerfile channel = "1.81.0" components = ["rustfmt", "clippy"] diff --git a/rust/Dockerfile b/rust/Dockerfile deleted file mode 100644 index 75adb3b2e7f..00000000000 --- a/rust/Dockerfile +++ /dev/null @@ -1,81 +0,0 @@ -FROM rust:1.81.0 AS builder - -ARG RELEASE_MODE= - -WORKDIR /chroma - -ENV PROTOC_ZIP=protoc-25.1-linux-x86_64.zip -RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v25.1/$PROTOC_ZIP \ - && unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \ - && unzip -o $PROTOC_ZIP -d /usr/local 'include/*' \ - && rm -f $PROTOC_ZIP - -COPY idl/ idl/ -COPY Cargo.toml Cargo.toml -COPY Cargo.lock Cargo.lock -COPY rust/ rust/ - -# Skip building these as they're not needed by images (and if Python bindings are built, the final binaries are unnecessarily linked against Python). -ENV EXCLUDED_PACKAGES="chromadb_rust_bindings chromadb-js-bindings chroma-benchmark " - -RUN --mount=type=cache,sharing=locked,target=/chroma/target/ \ - --mount=type=cache,sharing=locked,target=/usr/local/cargo/registry/ \ - if [ "$RELEASE_MODE" = "1" ]; then cargo build --workspace $(printf -- '--exclude %s ' $EXCLUDED_PACKAGES) --release; else cargo build --workspace $(printf -- '--exclude %s ' $EXCLUDED_PACKAGES); fi && \ - # Move CLI binary - if [ "$RELEASE_MODE" = "1" ]; then mv target/release/chroma ./chroma; else mv target/debug/chroma ./chroma; fi && \ - # Move garbage collector binary - if [ "$RELEASE_MODE" = "1" ]; then mv target/release/garbage_collector_service ./garbage_collector_service; else mv target/debug/garbage_collector_service ./garbage_collector_service; fi && \ - # Move load service binaries - if [ "$RELEASE_MODE" = "1" ]; then mv target/release/chroma-load-start ./chroma-load-start; else mv target/debug/chroma-load-start ./chroma-load-start; fi && \ - if [ "$RELEASE_MODE" = "1" ]; then mv target/release/chroma-load ./chroma-load; else mv target/debug/chroma-load ./chroma-load; fi && \ - # Move log service binary - if [ "$RELEASE_MODE" = "1" ]; then mv target/release/log_service ./log_service; else mv target/debug/log_service ./log_service; fi && \ - # Move query service binary - if [ "$RELEASE_MODE" = "1" ]; then mv target/release/query_service ./query_service; else mv target/debug/query_service ./query_service; fi && \ - # Move compaction service binary - if [ "$RELEASE_MODE" = "1" ]; then mv target/release/compaction_service ./compaction_service; else mv target/debug/compaction_service ./compaction_service; fi - - -FROM debian:stable-slim AS runner - -RUN apt-get update && apt-get install -y dumb-init libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* - -FROM runner AS cli - -COPY --from=builder /chroma/rust/frontend/sample_configs/docker_single_node.yaml /config.yaml -COPY --from=builder /chroma/rust/frontend/sample_configs/tilt_config.yaml /tilt_config.yaml -COPY --from=builder /chroma/chroma /usr/local/bin/chroma - -EXPOSE 8000 - -ENTRYPOINT [ "dumb-init", "--", "chroma" ] -CMD [ "run", "/config.yaml" ] - -FROM runner AS garbage_collector -COPY --from=builder /chroma/garbage_collector_service . -ENTRYPOINT [ "./garbage_collector_service" ] - -FROM runner AS load_service -COPY --from=builder /chroma/chroma-load . -COPY --from=builder /chroma/chroma-load-start . -ENTRYPOINT [ "./chroma-load" ] - -FROM runner AS log_service -COPY --from=builder /chroma/log_service . -ENTRYPOINT [ "./log_service" ] - -FROM runner AS query_service -COPY --from=builder /chroma/rust/worker/chroma_config.yaml . -# NOTE(rescrv): We need the tilt config in the docker file. This is temporary. -COPY --from=builder /chroma/rust/worker/tilt_config.yaml . - -COPY --from=builder /chroma/query_service . -ENTRYPOINT [ "./query_service" ] - -FROM runner AS compaction_service -COPY --from=builder /chroma/rust/worker/chroma_config.yaml . -# NOTE(rescrv): We need the tilt config in the docker file. This is temporary. -COPY --from=builder /chroma/rust/worker/tilt_config.yaml . - -COPY --from=builder /chroma/compaction_service . -ENTRYPOINT [ "./compaction_service" ]