chore(test-suite): speed up local build/test iterations (#1736)

* chore(test-suite): speed up local builds

* chore(test-suite): split local build cache by service

* feat(build): add local cargo profile for faster local builds

- Add 'local' profile to coprocessor and kms-connector Cargo.toml
  (opt-level=1, no LTO, 16 codegen-units for faster compilation)
- Update all Dockerfiles to accept CARGO_PROFILE build arg
- Update compose files to pass CARGO_PROFILE and RUST_IMAGE_VERSION
- Add .dockerignore to reduce build context size
- Fix deploy script bash 3.2 compatibility (macOS)
- Add DOCKER_BUILD_PROVENANCE=false to disable attestations
- Add kms-connector Cargo patch for monorepo path resolution
- Add kms-connector-db-migration to local cache services list

* perf(kms-connector): add target cache and cargo fetch for faster builds

- Add BuildKit cache mount for target/ directory to enable incremental builds
- Add cargo fetch step to cache dependency resolution separately
- Copy binaries to /tmp before prod stage (cache mounts don't persist)

* feat(coprocessor): add BuildKit target cache mount for incremental builds

Add target directory cache mount to all coprocessor Dockerfiles to enable
incremental Rust compilation caching, consistent with kms-connector pattern.

Cache mounts are not committed to image layers, so binaries must be copied
to /tmp before the final COPY --from stage can access them.

* chore: remove superfluous CARGO_PROFILE in prod stage

* chore: rename --no-compile to --no-hardhat-compile

* fix(test-suite): use repo root as build context for host-sc

The host-contracts Dockerfile expects repo root as build context
(COPY host-contracts/... paths), but compose file incorrectly
specified host-contracts/ directory as context since initial
creation in commit 9a92155.

This was never caught because CI uses repo root context and
local deploys without --build pull pre-built images.

Author: enitrat

.gitignore

@@ -37,6 +37,7 @@ logs/
 
 # Build output & temporary directories (generic)
 .cache/
+.buildx-cache/
 temp/
 tmp/
 
@@ -85,4 +86,4 @@ mochawesome-report/
 # Rust specific
 #-------------------------------------------------------------------------------
 # Build artifacts directory
-target/
+target/

coprocessor/fhevm-engine/Cargo.toml

@@ -101,3 +101,9 @@ overflow-checks = false
 [profile.release]
 opt-level = 3
 lto = "fat"
+
+[profile.local]
+inherits = "release"
+opt-level = 1
+lto = false
+codegen-units = 16

coprocessor/fhevm-engine/gw-listener/Dockerfile

@@ -15,6 +15,8 @@ RUN npm install && \
 # Stage 1: Build GW Listener
 FROM ghcr.io/zama-ai/fhevm/gci/rust-glibc:1.91.0 AS builder
 
+ARG CARGO_PROFILE=release
+
 USER root
 
 WORKDIR /app
@@ -30,16 +32,21 @@ COPY .git/HEAD ./coprocessor/fhevm-engine/BUILD_ID
 WORKDIR /app/coprocessor/fhevm-engine
 
 # Build gw_listener binary
+# NOTE: We use a cache mount for the target directory to enable incremental compilation.
+# Because cache mounts are NOT committed to the image layer, we must copy the binary
+# to a non-mounted path (/tmp) during the same RUN instruction for COPY --from to work.
 RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/app/coprocessor/fhevm-engine/target,sharing=locked \
     cargo fetch && \
-    SQLX_OFFLINE=true BUILD_ID=$(cat BUILD_ID) cargo build --release -p gw-listener
+    SQLX_OFFLINE=true BUILD_ID=$(cat BUILD_ID) cargo build --profile=${CARGO_PROFILE} -p gw-listener && \
+    cp target/${CARGO_PROFILE}/gw_listener /tmp/gw_listener
 
 # Stage 2: Runtime image
 FROM cgr.dev/zama.ai/glibc-dynamic:15.2.0 AS prod
 
 COPY --from=builder /etc/group /etc/group
 COPY --from=builder /etc/passwd /etc/passwd
-COPY --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/gw_listener /usr/local/bin/gw_listener
+COPY --from=builder --chown=fhevm:fhevm /tmp/gw_listener /usr/local/bin/gw_listener
 
 USER fhevm:fhevm
 

coprocessor/fhevm-engine/host-listener/Dockerfile

@@ -15,6 +15,8 @@ RUN npm install && HARDHAT_NETWORK=hardhat npm run deploy:emptyProxies && npx ha
 # Stage 1: Build Host Listener
 FROM ghcr.io/zama-ai/fhevm/gci/rust-glibc:1.91.0 AS builder
 
+ARG CARGO_PROFILE=release
+
 USER root
 
 WORKDIR /app
@@ -29,17 +31,23 @@ COPY .git/HEAD ./coprocessor/fhevm-engine/BUILD_ID
 WORKDIR /app/coprocessor/fhevm-engine
 
 # Build host_listener binary
+# NOTE: We use a cache mount for the target directory to enable incremental compilation.
+# Because cache mounts are NOT committed to the image layer, we must copy the binary
+# to a non-mounted path (/tmp) during the same RUN instruction for COPY --from to work.
 RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/app/coprocessor/fhevm-engine/target,sharing=locked \
     cargo fetch && \
-    SQLX_OFFLINE=true BUILD_ID=$(cat BUILD_ID) cargo build --release -p host-listener
+    SQLX_OFFLINE=true BUILD_ID=$(cat BUILD_ID) cargo build --profile=${CARGO_PROFILE} -p host-listener && \
+    cp target/${CARGO_PROFILE}/host_listener /tmp/host_listener && \
+    cp target/${CARGO_PROFILE}/host_listener_poller /tmp/host_listener_poller
 
 # Stage 2: Runtime image
 FROM cgr.dev/zama.ai/glibc-dynamic:15.2.0 AS prod
 
 COPY --from=builder /etc/group /etc/group
 COPY --from=builder /etc/passwd /etc/passwd
-COPY --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/host_listener /usr/local/bin/host_listener
-COPY --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/host_listener_poller /usr/local/bin/host_listener_poller
+COPY --from=builder --chown=fhevm:fhevm /tmp/host_listener /usr/local/bin/host_listener
+COPY --from=builder --chown=fhevm:fhevm /tmp/host_listener_poller /usr/local/bin/host_listener_poller
 
 USER fhevm:fhevm
 

coprocessor/fhevm-engine/sns-worker/Dockerfile

@@ -1,6 +1,8 @@
 # Stage 1: Build SNS Worker
 FROM ghcr.io/zama-ai/fhevm/gci/rust-glibc:1.91.0 AS builder
 
+ARG CARGO_PROFILE=release
+
 USER root
 
 WORKDIR /app
@@ -12,16 +14,21 @@ COPY gateway-contracts/rust_bindings ./gateway-contracts/rust_bindings
 WORKDIR /app/coprocessor/fhevm-engine
 
 # Build sns_executor binary
+# NOTE: We use a cache mount for the target directory to enable incremental compilation.
+# Because cache mounts are NOT committed to the image layer, we must copy the binary
+# to a non-mounted path (/tmp) during the same RUN instruction for COPY --from to work.
 RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/app/coprocessor/fhevm-engine/target,sharing=locked \
     cargo fetch && \
-    SQLX_OFFLINE=true cargo build --release -p sns-worker
+    SQLX_OFFLINE=true cargo build --profile=${CARGO_PROFILE} -p sns-worker && \
+    cp target/${CARGO_PROFILE}/sns_worker /tmp/sns_worker
 
 # Stage 2: Runtime image
 FROM cgr.dev/zama.ai/glibc-dynamic:15.2.0 AS prod
 
 COPY --from=builder /etc/group /etc/group
 COPY --from=builder /etc/passwd /etc/passwd
-COPY --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/sns_worker /usr/local/bin/sns_worker
+COPY --from=builder --chown=fhevm:fhevm /tmp/sns_worker /usr/local/bin/sns_worker
 
 USER fhevm:fhevm
 

coprocessor/fhevm-engine/stress-test-generator/Dockerfile

@@ -21,6 +21,8 @@ RUN cp .env.example .env \
 # Stage 1: Build Stress-Tool
 FROM ghcr.io/zama-ai/fhevm/gci/rust-glibc:1.91.0 AS builder
 
+ARG CARGO_PROFILE=release
+
 USER root
 
 WORKDIR /app
@@ -34,16 +36,21 @@ COPY --from=contract_builder /app/host-contracts/artifacts/contracts /app/host-c
 WORKDIR /app/coprocessor/fhevm-engine
 
 # Build stress_generator binary
+# NOTE: We use a cache mount for the target directory to enable incremental compilation.
+# Because cache mounts are NOT committed to the image layer, we must copy the binary
+# to a non-mounted path (/tmp) during the same RUN instruction for COPY --from to work.
 RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/app/coprocessor/fhevm-engine/target,sharing=locked \
     cargo fetch && \
-    SQLX_OFFLINE=true cargo build --release -p stress-test-generator
+    SQLX_OFFLINE=true cargo build --profile=${CARGO_PROFILE} -p stress-test-generator && \
+    cp target/${CARGO_PROFILE}/stress_generator /tmp/stress_generator
 
 # Stage 2: Runtime image
 FROM cgr.dev/chainguard/glibc-dynamic:latest AS prod
 
 COPY --from=builder /etc/group /etc/group
 COPY --from=builder /etc/passwd /etc/passwd
-COPY --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/stress_generator /usr/local/bin/stress_generator
+COPY --from=builder --chown=fhevm:fhevm /tmp/stress_generator /usr/local/bin/stress_generator
 USER fhevm:fhevm
 
 CMD ["/usr/local/bin/stress_generator"]

coprocessor/fhevm-engine/tfhe-worker/Dockerfile

@@ -1,6 +1,8 @@
 # Stage 1: Build TFHE Worker
 FROM ghcr.io/zama-ai/fhevm/gci/rust-glibc:1.91.0 AS builder
 
+ARG CARGO_PROFILE=release
+
 USER root
 
 WORKDIR /app
@@ -12,16 +14,21 @@ COPY gateway-contracts/rust_bindings ./gateway-contracts/rust_bindings
 WORKDIR /app/coprocessor/fhevm-engine
 
 # Build tfhe_worker binary
+# NOTE: We use a cache mount for the target directory to enable incremental compilation.
+# Because cache mounts are NOT committed to the image layer, we must copy the binary
+# to a non-mounted path (/tmp) during the same RUN instruction for COPY --from to work.
 RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/app/coprocessor/fhevm-engine/target,sharing=locked \
     cargo fetch && \
-    SQLX_OFFLINE=true cargo build --release -p tfhe-worker
+    SQLX_OFFLINE=true cargo build --profile=${CARGO_PROFILE} -p tfhe-worker && \
+    cp target/${CARGO_PROFILE}/tfhe_worker /tmp/tfhe_worker
 
 # Stage 2: Runtime image
 FROM cgr.dev/zama.ai/glibc-dynamic:15.2.0 AS prod
 
 COPY --from=builder /etc/group /etc/group
 COPY --from=builder /etc/passwd /etc/passwd
-COPY --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/tfhe_worker /usr/local/bin/tfhe_worker
+COPY --from=builder --chown=fhevm:fhevm /tmp/tfhe_worker /usr/local/bin/tfhe_worker
 
 USER fhevm:fhevm
 

coprocessor/fhevm-engine/transaction-sender/Dockerfile

@@ -1,6 +1,8 @@
 # Stage 1: Build Transaction Sender
 FROM ghcr.io/zama-ai/fhevm/gci/rust-glibc:1.91.0 AS builder
 
+ARG CARGO_PROFILE=release
+
 USER root
 
 WORKDIR /app
@@ -12,16 +14,21 @@ COPY gateway-contracts/rust_bindings ./gateway-contracts/rust_bindings
 WORKDIR /app/coprocessor/fhevm-engine
 
 # Build transaction_sender binary
+# NOTE: We use a cache mount for the target directory to enable incremental compilation.
+# Because cache mounts are NOT committed to the image layer, we must copy the binary
+# to a non-mounted path (/tmp) during the same RUN instruction for COPY --from to work.
 RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/app/coprocessor/fhevm-engine/target,sharing=locked \
     cargo fetch && \
-    SQLX_OFFLINE=true cargo build --release -p transaction-sender
+    SQLX_OFFLINE=true cargo build --profile=${CARGO_PROFILE} -p transaction-sender && \
+    cp target/${CARGO_PROFILE}/transaction_sender /tmp/transaction_sender
 
 # Stage 2: Runtime image
 FROM cgr.dev/zama.ai/glibc-dynamic:15.2.0 AS prod
 
 COPY --from=builder /etc/group /etc/group
 COPY --from=builder /etc/passwd /etc/passwd
-COPY --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/transaction_sender /usr/local/bin/transaction_sender
+COPY --from=builder --chown=fhevm:fhevm /tmp/transaction_sender /usr/local/bin/transaction_sender
 
 USER fhevm:fhevm
 

coprocessor/fhevm-engine/zkproof-worker/Dockerfile

@@ -1,6 +1,8 @@
 # Stage 1: Build ZK Proof Worker
 FROM ghcr.io/zama-ai/fhevm/gci/rust-glibc:1.91.0 AS builder
 
+ARG CARGO_PROFILE=release
+
 USER root
 
 WORKDIR /app
@@ -12,16 +14,21 @@ COPY gateway-contracts/rust_bindings ./gateway-contracts/rust_bindings
 WORKDIR /app/coprocessor/fhevm-engine
 
 # Build zkproof_worker binary
+# NOTE: We use a cache mount for the target directory to enable incremental compilation.
+# Because cache mounts are NOT committed to the image layer, we must copy the binary
+# to a non-mounted path (/tmp) during the same RUN instruction for COPY --from to work.
 RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/app/coprocessor/fhevm-engine/target,sharing=locked \
     cargo fetch && \
-    SQLX_OFFLINE=true cargo build --release -p zkproof-worker
+    SQLX_OFFLINE=true cargo build --profile=${CARGO_PROFILE} -p zkproof-worker && \
+    cp target/${CARGO_PROFILE}/zkproof_worker /tmp/zkproof_worker
 
 # Stage 2: Runtime image
 FROM cgr.dev/zama.ai/glibc-dynamic:15.2.0 AS prod
 
 COPY --from=builder /etc/group /etc/group
 COPY --from=builder /etc/passwd /etc/passwd
-COPY --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/zkproof_worker /usr/local/bin/zkproof_worker
+COPY --from=builder --chown=fhevm:fhevm /tmp/zkproof_worker /usr/local/bin/zkproof_worker
 
 USER fhevm:fhevm
 

kms-connector/Cargo.toml

@@ -102,3 +102,9 @@ serial_test = "3.2.0"
 testcontainers = "=0.24.0"
 toml = { version = "=0.9.8", default-features = true }
 tracing-test = { version = "=0.2.5", default-features = false }
+
+[profile.local]
+inherits = "release"
+opt-level = 1
+lto = false
+codegen-units = 16