Skip to content
test

Feat: Non-determinism errors #7778

Sign in to view logs

Feat: Non-determinism errors

Feat: Non-determinism errors #7778

Workflow file for this run

.github/workflows/test.yml at 78d5355
name: test
on:
pull_request:
paths-ignore:
- "sdks/typescript/**"
- "sdks/python/**"
- "frontend/**"
- "examples/**"
jobs:
generate:
runs-on: ubicloud-standard-4
env:
DATABASE_URL: postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet?sslmode=disable
steps:
- uses: actions/checkout@v6
- name: Install Protoc
uses: arduino/setup-protoc@v3
with:
version: "29.3"
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Task
uses: arduino/setup-task@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.25"
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 10.16.1
run_install: false
- name: Compose
run: docker compose up -d
- name: Generate
run: |
go run ./cmd/hatchet-migrate
task pre-commit-install
task generate-all
- name: Check for diff
run: git diff --exit-code
- name: Teardown
run: docker compose down
unit:
runs-on: ubicloud-standard-8
steps:
- uses: actions/checkout@v6
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.25"
- name: Go deps
run: go mod download
- name: Generate
run: go generate ./...
- name: Test
run: go test $(go list ./... | grep -v "quickstart") -v -failfast
integration:
runs-on: ubicloud-standard-8
env:
DATABASE_URL: postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet?sslmode=disable
steps:
- uses: actions/checkout@v6
- name: Install Task
uses: arduino/setup-task@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.25"
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 10.16.1
run_install: false
- name: Install Atlas
run: |
curl -sSf https://atlasgo.sh | sh
- name: Compose
run: docker compose up -d
- name: Go deps
run: go mod download
- name: Generate
run: |
go run ./cmd/hatchet-migrate
task generate-certs
task generate-local-encryption-keys
- name: Test
run: go test -tags integration $(go list ./... | grep -v "quickstart") -v -failfast
- name: Teardown
run: docker compose down
e2e:
runs-on: ubicloud-standard-8
timeout-minutes: 30
env:
DATABASE_URL: postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet?sslmode=disable
steps:
- uses: actions/checkout@v6
- name: Install Task
uses: arduino/setup-task@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.25"
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 10.16.1
run_install: false
- name: Install Atlas
run: |
curl -sSf https://atlasgo.sh | sh
- name: Compose
run: docker compose up -d
- name: Go deps
run: go mod download
- name: Prepare
run: |
cat > .env <<EOF
HATCHET_CLIENT_TENANT_ID=707d0855-80ab-4e1f-a156-f1c4546cbf52
DATABASE_URL="postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet"
HATCHET_CLIENT_TLS_ROOT_CA_FILE=./hack/dev/certs/ca.cert
HATCHET_CLIENT_TLS_SERVER_NAME="cluster"
SERVER_TLS_CERT_FILE=./hack/dev/certs/cluster.pem
SERVER_TLS_KEY_FILE=./hack/dev/certs/cluster.key
SERVER_TLS_ROOT_CA_FILE=./hack/dev/certs/ca.cert
SERVER_ENCRYPTION_MASTER_KEYSET_FILE=./hack/dev/encryption-keys/master.key
SERVER_ENCRYPTION_JWT_PRIVATE_KEYSET_FILE=./hack/dev/encryption-keys/private_ec256.key
SERVER_ENCRYPTION_JWT_PUBLIC_KEYSET_FILE=./hack/dev/encryption-keys/public_ec256.key
SERVER_LOGGER_LEVEL=warn
SERVER_LOGGER_FORMAT=console
DATABASE_LOGGER_LEVEL=warn
DATABASE_LOGGER_FORMAT=console
DATABASE_URL='postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet'
SERVER_PORT=8080
SERVER_URL=http://localhost:8080
SERVER_AUTH_COOKIE_SECRETS="kPpegRDNpofgkUsr HoWe67haMOF5qnaB"
SERVER_AUTH_COOKIE_DOMAIN=app.dev.hatchet-tools.com
SERVER_AUTH_COOKIE_INSECURE=false
SERVER_AUTH_SET_EMAIL_VERIFIED=true
EOF
- name: Generate
run: |
go run ./cmd/hatchet-migrate
task generate-certs
task generate-local-encryption-keys
- name: Run engine
run: |
set -a
. .env
set +a
go run ./cmd/hatchet-admin quickstart
go run ./cmd/hatchet-engine --config ./generated/ &
go run ./cmd/hatchet-api --config ./generated/ &
sleep 30
- name: Test
run: |
go test -tags e2e ./... -p 1 -v -failfast
- name: Teardown
run: docker compose down
e2e-pgmq:
runs-on: ubicloud-standard-8
timeout-minutes: 30
env:
DATABASE_URL: postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet?sslmode=disable
steps:
- uses: actions/checkout@v6
- name: Install Task
uses: arduino/setup-task@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.25"
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 10.16.1
run_install: false
- name: Install Atlas
run: |
curl -sSf https://atlasgo.sh | sh
- name: Compose
run: docker compose up -d
- name: Go deps
run: go mod download
- name: Prepare
run: |
cat > .env <<EOF
HATCHET_CLIENT_TENANT_ID=707d0855-80ab-4e1f-a156-f1c4546cbf52
DATABASE_URL="postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet"
HATCHET_CLIENT_TLS_ROOT_CA_FILE=./hack/dev/certs/ca.cert
HATCHET_CLIENT_TLS_SERVER_NAME="cluster"
SERVER_TLS_CERT_FILE=./hack/dev/certs/cluster.pem
SERVER_TLS_KEY_FILE=./hack/dev/certs/cluster.key
SERVER_TLS_ROOT_CA_FILE=./hack/dev/certs/ca.cert
SERVER_ENCRYPTION_MASTER_KEYSET_FILE=./hack/dev/encryption-keys/master.key
SERVER_ENCRYPTION_JWT_PRIVATE_KEYSET_FILE=./hack/dev/encryption-keys/private_ec256.key
SERVER_ENCRYPTION_JWT_PUBLIC_KEYSET_FILE=./hack/dev/encryption-keys/public_ec256.key
SERVER_LOGGER_LEVEL=warn
SERVER_LOGGER_FORMAT=console
DATABASE_LOGGER_LEVEL=warn
DATABASE_LOGGER_FORMAT=console
DATABASE_URL='postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet'
SERVER_PORT=8080
SERVER_URL=http://localhost:8080
SERVER_AUTH_COOKIE_SECRETS="kPpegRDNpofgkUsr HoWe67haMOF5qnaB"
SERVER_AUTH_COOKIE_DOMAIN=app.dev.hatchet-tools.com
SERVER_AUTH_COOKIE_INSECURE=false
SERVER_AUTH_SET_EMAIL_VERIFIED=true
SERVER_TASKQUEUE_KIND=postgres
EOF
- name: Generate
run: |
go run ./cmd/hatchet-migrate
task generate-certs
task generate-local-encryption-keys
- name: Run engine
run: |
set -a
. .env
set +a
go run ./cmd/hatchet-admin quickstart
go run ./cmd/hatchet-engine --config ./generated/ &
go run ./cmd/hatchet-api --config ./generated/ &
sleep 30
- name: Test
run: |
go test -tags e2e ./... -p 1 -v -failfast
- name: Teardown
run: docker compose down
load:
runs-on: ubicloud-standard-8
timeout-minutes: 30
strategy:
matrix:
migrate-strategy: ["latest"]
rabbitmq-enabled: ["true"]
pg-version: ["17-alpine"]
optimistic-scheduling: ["true", "false"]
steps:
- uses: actions/checkout@v6
- name: Install Task
uses: arduino/setup-task@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.25"
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 10.16.1
run_install: false
- name: Go deps
run: go mod download
- name: Test
run: |
# Disable gzip compression for load tests to reduce CPU overhead
# Compression adds overhead without benefit for 0kb payloads
HATCHET_CLIENT_DISABLE_GZIP_COMPRESSION=true go test -tags load ./... -p 5 -v -race -failfast -timeout 20m
env:
TESTING_MATRIX_MIGRATE: ${{ matrix.migrate-strategy }}
TESTING_MATRIX_RABBITMQ_ENABLED: ${{ matrix.rabbitmq-enabled }}
TESTING_MATRIX_PG_VERSION: ${{ matrix.pg-version }}
TESTING_MATRIX_OPTIMISTIC_SCHEDULING: ${{ matrix.optimistic-scheduling }}
load-online-migrate:
runs-on: ubicloud-standard-8
timeout-minutes: 30
env:
DATABASE_URL: postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet?sslmode=disable
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
fetch-tags: true
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.25"
- name: Compose
run: docker compose up -d
- name: Determine latest stable release tag
run: |
LATEST_TAG=$(git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -1)
if [ -z "$LATEST_TAG" ]; then
echo "ERROR: No stable release tag found"
exit 1
fi
echo "Latest stable tag: $LATEST_TAG"
echo "LATEST_TAG=$LATEST_TAG" >> $GITHUB_ENV
- name: Pull old release images
run: |
docker pull ghcr.io/hatchet-dev/hatchet/hatchet-migrate:${{ env.LATEST_TAG }}
docker pull ghcr.io/hatchet-dev/hatchet/hatchet-admin:${{ env.LATEST_TAG }}
docker pull ghcr.io/hatchet-dev/hatchet/hatchet-engine:${{ env.LATEST_TAG }}
docker pull ghcr.io/hatchet-dev/hatchet/hatchet-loadtest:${{ env.LATEST_TAG }}
- name: Run old migrations
run: |
docker run --rm --network host \
-e DATABASE_URL="${{ env.DATABASE_URL }}" \
ghcr.io/hatchet-dev/hatchet/hatchet-migrate:${{ env.LATEST_TAG }}
- name: Setup config and seed database
run: |
mkdir -p generated
docker run --rm --network host \
-v ${{ github.workspace }}/generated:/hatchet/generated \
-e DATABASE_URL="${{ env.DATABASE_URL }}" \
-e SERVER_GRPC_PORT=7077 \
-e SERVER_GRPC_BROADCAST_ADDRESS=localhost:7077 \
-e SERVER_GRPC_INSECURE=true \
-e SERVER_AUTH_COOKIE_DOMAIN=localhost \
-e SERVER_AUTH_COOKIE_INSECURE=true \
ghcr.io/hatchet-dev/hatchet/hatchet-admin:${{ env.LATEST_TAG }} \
/hatchet/hatchet-admin quickstart --skip certs --generated-config-dir /hatchet/generated
- name: Generate API token
run: |
TOKEN=$(docker run --rm --network host \
-v ${{ github.workspace }}/generated:/hatchet/generated \
-e DATABASE_URL="${{ env.DATABASE_URL }}" \
-e SERVER_GRPC_PORT=7077 \
-e SERVER_GRPC_BROADCAST_ADDRESS=localhost:7077 \
-e SERVER_GRPC_INSECURE=true \
-e SERVER_AUTH_COOKIE_DOMAIN=localhost \
-e SERVER_AUTH_COOKIE_INSECURE=true \
ghcr.io/hatchet-dev/hatchet/hatchet-admin:${{ env.LATEST_TAG }} \
/hatchet/hatchet-admin token create --config /hatchet/generated)
echo "HATCHET_CLIENT_TOKEN=$TOKEN" >> $GITHUB_ENV
- name: Start old engine
run: |
docker run -d --name hatchet-engine --network host \
-v ${{ github.workspace }}/generated:/hatchet/generated \
-e DATABASE_URL="${{ env.DATABASE_URL }}" \
-e SERVER_GRPC_PORT=7077 \
-e SERVER_GRPC_BROADCAST_ADDRESS=localhost:7077 \
-e SERVER_GRPC_INSECURE=true \
-e SERVER_AUTH_COOKIE_DOMAIN=localhost \
-e SERVER_AUTH_COOKIE_INSECURE=true \
-e SERVER_MSGQUEUE_KIND=postgres \
-e SERVER_LOGGER_LEVEL=warn \
-e SERVER_LOGGER_FORMAT=console \
-e DATABASE_LOGGER_LEVEL=warn \
-e DATABASE_LOGGER_FORMAT=console \
ghcr.io/hatchet-dev/hatchet/hatchet-engine:${{ env.LATEST_TAG }} \
/hatchet/hatchet-engine --config /hatchet/generated
echo "Waiting 30s for engine to start..."
sleep 30
- name: Start old load test
run: |
docker run -d --name hatchet-loadtest --network host \
-e HATCHET_CLIENT_TOKEN="${{ env.HATCHET_CLIENT_TOKEN }}" \
-e HATCHET_CLIENT_TLS_STRATEGY=none \
-e HATCHET_CLIENT_HOST_PORT=localhost:7077 \
ghcr.io/hatchet-dev/hatchet/hatchet-loadtest:${{ env.LATEST_TAG }} \
/hatchet/hatchet-load-test loadtest -e 10 -d 240s -w 60s -s 100
- name: Wait then apply new migrations
run: |
echo "Waiting 30s for load test to get started..."
sleep 30
echo "Applying new migrations from current branch..."
go run ./cmd/hatchet-migrate
echo "New migrations applied successfully"
- name: Wait for load test to complete
run: |
echo "Waiting for load test container to finish..."
docker wait hatchet-loadtest
EXIT_CODE=$(docker inspect hatchet-loadtest --format='{{.State.ExitCode}}')
echo "Load test exited with code: $EXIT_CODE"
if [ "$EXIT_CODE" != "0" ]; then
echo "=== Load test logs ==="
docker logs hatchet-loadtest
echo "=== Engine logs ==="
docker logs hatchet-engine
exit 1
fi
echo "Load test passed"
- name: Teardown
if: always()
run: |
docker rm -f hatchet-loadtest hatchet-engine 2>/dev/null || true
docker compose down
load-deadlock:
runs-on: ubicloud-standard-8
timeout-minutes: 30
strategy:
matrix:
migrate-strategy: ["latest"]
rabbitmq-enabled: ["true"]
pg-version: ["17-alpine"]
optimistic-scheduling: ["true", "false"]
steps:
- uses: actions/checkout@v6
- name: Install Task
uses: arduino/setup-task@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.25"
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 10.16.1
run_install: false
- name: Go deps
run: go mod download
- name: Add go-deadlock dependency
run: go get github.com/sasha-s/go-deadlock@v0.3.6
- name: Patch sync imports to use go-deadlock (sed)
shell: bash
run: |
set -euo pipefail
# Replace ONLY the stdlib "sync" import with an alias that preserves `sync.X` call sites.
# - `import "sync"` -> `import sync "github.com/sasha-s/go-deadlock"`
# - within import blocks: `"sync"` -> `sync "github.com/sasha-s/go-deadlock"`
# NOTE: use `-i''` (no backup) for portability across GNU/BSD sed.
find . -name '*.go' -not -path './vendor/*' -print0 | xargs -0 sed -i'' -E \
-e 's/^([[:space:]]*)import[[:space:]]+"sync"[[:space:]]*$/\1import sync "github.com\/sasha-s\/go-deadlock"/' \
-e 's/^([[:space:]]*)"sync"[[:space:]]*$/\1sync "github.com\/sasha-s\/go-deadlock"/'
# Keep formatting/import grouping consistent after rewriting.
find . -name '*.go' -not -path './vendor/*' -print0 | xargs -0 gofmt -w
# Evidence in CI logs that rewriting happened (or not).
echo "Changed Go files (after patch):"
git diff --name-only -- '*.go' || true
echo ""
echo "Contents of pkg/scheduling/v1/scheduler.go after patch:"
echo "----"
cat pkg/scheduling/v1/scheduler.go
echo "----"
- name: Test (deadlock-instrumented)
run: |
# Disable gzip compression for load tests to reduce CPU overhead
# Compression adds overhead without benefit for 0kb payloads
HATCHET_CLIENT_DISABLE_GZIP_COMPRESSION=true go test -tags load ./... -p 5 -v -race -failfast -timeout 20m
env:
# This job adds go-deadlock + -race overhead; relax perf threshold to avoid flakes.
HATCHET_LOADTEST_AVERAGE_DURATION_THRESHOLD: 1s
# Give the engine a bit more time to come up under instrumentation.
HATCHET_LOADTEST_STARTUP_SLEEP: 30s
TESTING_MATRIX_MIGRATE: ${{ matrix.migrate-strategy }}
TESTING_MATRIX_RABBITMQ_ENABLED: ${{ matrix.rabbitmq-enabled }}
TESTING_MATRIX_PG_VERSION: ${{ matrix.pg-version }}
TESTING_MATRIX_OPTIMISTIC_SCHEDULING: ${{ matrix.optimistic-scheduling }}
rampup:
runs-on: ubicloud-standard-8
timeout-minutes: 30
strategy:
matrix:
migrate-strategy: ["latest"]
rabbitmq-enabled: ["true"]
pg-version: ["17-alpine"]
steps:
- uses: actions/checkout@v6
- name: Install Task
uses: arduino/setup-task@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.25"
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 10.16.1
run_install: false
- name: Go deps
run: go mod download
- name: Test
run: |
go test -tags rampup ./... -p 5 -v -race -failfast -timeout 20m
env:
TESTING_MATRIX_MIGRATE: ${{ matrix.migrate-strategy }}
TESTING_MATRIX_RABBITMQ_ENABLED: ${{ matrix.rabbitmq-enabled }}
TESTING_MATRIX_PG_VERSION: ${{ matrix.pg-version }}