feat: ClickHouse Dictionary SDK support (TS + Python + E2E) (#4022) #6486
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release CLI | |
| on: | |
| push: | |
| branches: | |
| - "**" # All branches | |
| paths: | |
| - ".github/workflows/release-cli.yaml" | |
| - "apps/moose-cli-npm/**" | |
| - "apps/framework-cli/**" | |
| - "apps/create-moose-app/**" | |
| - "apps/moose-console/**" | |
| - "packages/**" | |
| - "templates/**" | |
| - "pnpm-lock.yaml" | |
| workflow_dispatch: | |
| inputs: | |
| dry-run: | |
| type: boolean | |
| description: "Run the release in dry-run mode" | |
| required: true | |
| default: true | |
| jobs: | |
| version: | |
| concurrency: release | |
| if: ${{ ! contains(github.event.head_commit.message , '[no-release]') && ! startsWith(github.ref, 'refs/heads/gh-readonly-queue/') && github.actor != 'dependabot[bot]' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: "write" | |
| id-token: "write" | |
| outputs: | |
| version: ${{ steps.version.outputs.VERSION }} | |
| python_version: ${{ steps.version.outputs.PYTHON_VERSION }} | |
| is_ci_build: ${{ steps.version.outputs.IS_CI_BUILD }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Need full history for git describe | |
| fetch-tags: true | |
| - name: Generate Version | |
| id: version | |
| run: | | |
| # Determine if this is a CI build (any branch other than main) | |
| if [[ "${{ github.ref_name }}" != "main" ]]; then | |
| echo "GENERATING CI VERSIONS" | |
| # For CI builds, use git describe but exclude any -ci- tags | |
| git fetch --tags | |
| # Use git describe with --match to only consider release tags (without -ci-) | |
| # This ensures we build from the last real release, not a CI tag | |
| CI_VERSION=$(git describe --tags --exclude="*-ci-*") | |
| # Remove the 'v' prefix if present | |
| CI_VERSION=${CI_VERSION#v} | |
| # Insert -ci- after the base version tag | |
| # This transforms v0.6.147-3-g7b637a7e to v0.6.147-ci-3-g7b637a7e | |
| # Pattern: base_version-commit_count-ghash becomes base_version-ci-commit_count-ghash | |
| if [[ "$CI_VERSION" =~ ^([0-9]+\.[0-9]+\.[0-9]+)-([0-9]+)-g([0-9a-f]+)(.*)$ ]]; then | |
| CI_VERSION="${BASH_REMATCH[1]}-ci-${BASH_REMATCH[2]}-g${BASH_REMATCH[3]}${BASH_REMATCH[4]}" | |
| # For Python, convert to PEP 440 compliant pre-release version | |
| # Use git hash (hex) converted to decimal for uniqueness | |
| # 0.6.147-ci-3-g7b637a7e -> 0.6.147.dev2066915966 (0x7b637a7e = 2066915966) | |
| GIT_HASH_HEX="${BASH_REMATCH[3]}" | |
| GIT_HASH_DEC=$((16#${GIT_HASH_HEX})) | |
| PYTHON_VERSION="${BASH_REMATCH[1]}.dev${GIT_HASH_DEC}" | |
| else | |
| echo "ERROR: Unexpected CI version format: ${CI_VERSION}" | |
| exit 1 | |
| fi | |
| VERSION="${CI_VERSION}" | |
| IS_CI_BUILD="true" | |
| else | |
| echo "GENERATING RELEASE VERSIONS" | |
| # For release builds (main branch), use the existing version.js script | |
| VERSION_OUTPUT=$(./scripts/version.js ${{ github.sha }}) | |
| VERSION=$(echo "$VERSION_OUTPUT" | grep "VERSION=" | cut -d'=' -f2) | |
| PYTHON_VERSION="${VERSION}" | |
| IS_CI_BUILD="false" | |
| fi | |
| echo "VERSION=${VERSION}" >> "$GITHUB_OUTPUT" | |
| echo "PYTHON_VERSION=${PYTHON_VERSION}" >> "$GITHUB_OUTPUT" | |
| echo "IS_CI_BUILD=${IS_CI_BUILD}" >> "$GITHUB_OUTPUT" | |
| echo "IS_CI_BUILD = ${IS_CI_BUILD}" | |
| echo "Generated Semantic Version: ${VERSION}" | |
| echo "Generated Python Version: ${PYTHON_VERSION}" | |
| - name: Create Release/Tag | |
| uses: ncipollo/release-action@v1 | |
| if: ${{ !inputs.dry-run }} | |
| with: | |
| tag: v${{ steps.version.outputs.VERSION }} | |
| generateReleaseNotes: ${{ steps.version.outputs.IS_CI_BUILD != 'true' }} | |
| prerelease: ${{ steps.version.outputs.IS_CI_BUILD == 'true' }} | |
| commit: ${{ github.sha }} | |
| - name: Auth | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| service_account: "mds-911@moose-hosting-node.iam.gserviceaccount.com" | |
| workload_identity_provider: "projects/724152421890/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-repos" | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v2" | |
| with: | |
| install_components: "gsutil" | |
| - name: Upload Version Info | |
| run: | | |
| if [[ "${{ steps.version.outputs.IS_CI_BUILD }}" == "true" ]]; then | |
| # CI builds go to the dev channel | |
| echo "${{ steps.version.outputs.VERSION }}" > moose-cli.version | |
| gsutil cp moose-cli.version gs://downloads.fiveonefour.com/dev/${{ steps.version.outputs.VERSION }}/ | |
| else | |
| # Release builds (main branch) update the latest stable version | |
| echo "${{ steps.version.outputs.VERSION }}" > moose-cli.version | |
| gsutil cp moose-cli.version gs://downloads.fiveonefour.com/stable/latest/ | |
| fi | |
| build-and-publish-py-moose-lib: | |
| runs-on: ubuntu-latest | |
| if: ${{ !inputs.dry-run && needs.version.outputs.is_ci_build == 'false' }} | |
| needs: | |
| - version | |
| # - cleanup-pypi # Commented out: job is disabled, skipped jobs cause dependents to skip | |
| # Add explicit permissions | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.x" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools wheel twine | |
| - uses: 1password/load-secrets-action@v1 | |
| id: op-load-secret | |
| with: | |
| export-env: false | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| PYPI_TOKEN: "op://drqe7p6legi6ug2ijq2fnrkmjq/PyPi Token/password" | |
| - name: Build and publish | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ steps.op-load-secret.outputs.PYPI_TOKEN }} | |
| run: | | |
| cd packages/py-moose-lib | |
| # Use PYTHON_VERSION for PEP 440 compliant versioning | |
| # CI builds: 0.6.147.dev3 (pre-release, won't become default) | |
| # Release builds: 0.6.147 (stable, becomes default) | |
| python setup.py --version ${{ needs.version.outputs.python_version }} sdist bdist_wheel | |
| twine upload dist/* | |
| package-and-publish-independant-ts-package: | |
| name: Package and Publish Independant TS Package | |
| runs-on: ubuntu-latest | |
| if: ${{ !inputs.dry-run }} | |
| needs: | |
| - version | |
| permissions: | |
| contents: "read" | |
| id-token: "write" | |
| env: | |
| TAG_LATEST: ${{ needs.version.outputs.is_ci_build == 'false' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - name: Install node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "24" | |
| cache: "pnpm" | |
| - name: Publish the NPM Moose CLI package | |
| shell: bash | |
| run: ./packages/ts-moose-lib/scripts/release-lib.sh ${{ needs.version.outputs.version }} | |
| - name: Wait for @514labs/ts-moose-lib to be available | |
| shell: bash | |
| run: ./scripts/wait-for-npm-package.sh @514labs/moose-lib ${{ needs.version.outputs.version }} | |
| - name: Publish the NPM Moose Connector API package | |
| shell: bash | |
| run: ./packages/ts-connector-api/scripts/release-lib.sh ${{ needs.version.outputs.version }} | |
| - name: Wait for @514labs/moose-connector-api to be available | |
| shell: bash | |
| run: ./scripts/wait-for-npm-package.sh @514labs/moose-connector-api ${{ needs.version.outputs.version }} | |
| - name: Publish the NPM Moose Connector S3 package | |
| shell: bash | |
| run: ./packages/ts-connector-s3/scripts/release-lib.sh ${{ needs.version.outputs.version }} | |
| - name: Wait for @514labs/moose-connector-s3 to be available | |
| shell: bash | |
| run: ./scripts/wait-for-npm-package.sh @514labs/moose-connector-s3 ${{ needs.version.outputs.version }} | |
| - name: Publish the NPM Moose design-system package | |
| shell: bash | |
| run: ./packages/design-system-base/scripts/release.sh ${{ needs.version.outputs.version }} | |
| - name: Wait for @514labs/design-system-base to be available | |
| shell: bash | |
| run: ./scripts/wait-for-npm-package.sh @514labs/design-system-base ${{ needs.version.outputs.version }} | |
| - name: Publish the NPM Moose event-capture package | |
| shell: bash | |
| run: ./packages/event-capture/scripts/release.sh ${{ needs.version.outputs.version }} | |
| - name: Wait for d@514labs/event-capture to be available | |
| shell: bash | |
| run: ./scripts/wait-for-npm-package.sh @514labs/event-capture ${{ needs.version.outputs.version }} | |
| - name: Publish the NPM Moose Protobuf package | |
| shell: bash | |
| run: ./packages/ts-moose-proto/scripts/release-lib.sh ${{ needs.version.outputs.version }} | |
| - name: Wait for @514labs/moose-proto to be available | |
| shell: bash | |
| run: ./scripts/wait-for-npm-package.sh @514labs/moose-proto ${{ needs.version.outputs.version }} | |
| package-and-publish-templates: | |
| name: Package and Publish Templates | |
| runs-on: ubuntu-latest | |
| needs: | |
| - version | |
| permissions: | |
| contents: "read" | |
| id-token: "write" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - name: Install node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| cache: "pnpm" | |
| - name: Install dependencies | |
| run: pnpm install --filter @repo/scripts | |
| - name: Package all templates | |
| run: ./scripts/package-templates.js | |
| - name: Authenticate with GCP | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| service_account: "github-actions@extended-ward-415018.iam.gserviceaccount.com" | |
| workload_identity_provider: "projects/167275965848/locations/global/workloadIdentityPools/github/providers/github-actions-repos" | |
| - name: Upload templates to GCP | |
| uses: "google-github-actions/upload-cloud-storage@v2" | |
| with: | |
| path: "template-packages" | |
| destination: "templates.514.dev/${{ needs.version.outputs.version }}/" | |
| parent: false | |
| - name: Upload Latest templates to GCP | |
| uses: "google-github-actions/upload-cloud-storage@v2" | |
| with: | |
| path: "template-packages" | |
| destination: "templates.514.dev/latest/" | |
| parent: false | |
| build-and-publish-binaries: | |
| name: Build CLI Binaries | |
| runs-on: ${{ matrix.build.os }} | |
| needs: version | |
| permissions: | |
| contents: "write" | |
| id-token: "write" | |
| env: | |
| POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} | |
| TAG_LATEST: ${{ needs.version.outputs.is_ci_build == 'false' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| build: | |
| - { | |
| NAME: linux-x64-glibc, | |
| OS: ubuntu-22-8-core, | |
| TARGET: x86_64-unknown-linux-gnu, | |
| } | |
| - { | |
| NAME: linux-arm64-glibc, | |
| OS: ubuntu-22-8-core, | |
| TARGET: aarch64-unknown-linux-gnu, | |
| } | |
| - { | |
| NAME: darwin-arm64, | |
| OS: macos-14-large, | |
| TARGET: aarch64-apple-darwin, | |
| } | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| if: ${{ matrix.build.OS != 'ubuntu-22-8-core' }} | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| target: ${{ matrix.build.TARGET }} | |
| - name: Run sccache-cache | |
| if: ${{ matrix.build.OS != 'ubuntu-22-8-core' }} | |
| uses: mozilla-actions/sccache-action@v0.0.9 | |
| with: | |
| # sccache-action points at the latest sccache version | |
| # sccache v0.13.0+ drops support for x86_64 darwin which will cause | |
| # the workflow to fail | |
| version: "v0.12.0" | |
| - name: Install node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "24" | |
| - uses: pnpm/action-setup@v4 | |
| - name: pnpm install for scripts # for package-templates.js | |
| run: pnpm install --filter=@repo/scripts --frozen-lockfile | |
| - name: Install Protoc (Needed for Temporal) | |
| uses: arduino/setup-protoc@v3 | |
| # For aarch64 we need to install protoc manually inside the before-script-Linux | |
| # that's ran inside the docker container | |
| if: ${{ matrix.build.OS != 'ubuntu-22-8-core' }} | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| version: "24.4" | |
| - name: Verify Protoc Installation | |
| # For aarch64 we need to install protoc manually inside the before-script-Linux | |
| # that's ran inside the docker container | |
| if: ${{ matrix.build.OS != 'ubuntu-22-8-core' }} | |
| run: | | |
| which protoc | |
| protoc --version | |
| echo "PROTOC=$(which protoc)" >> $GITHUB_ENV | |
| - name: Set Version | |
| run: | | |
| # Set Cargo.toml to base version only (strip -ci- part) | |
| # Maturin tries to convert this to Python format, so keep it simple | |
| BASE_VERSION=$(echo "${{ needs.version.outputs.version }}" | sed 's/-ci-.*//') | |
| if [[ "$OSTYPE" == "darwin"* ]]; then | |
| # macOS version of sed requires an empty string after -i | |
| sed -i '' "s/^version = \".*\"/version = \"${BASE_VERSION}\"/" Cargo.toml | |
| sed -i '' '/name = "moose-cli"/,/version = ".*"/ s/version = ".*"/version = "'"${BASE_VERSION}"'"/' ../../Cargo.lock | |
| else | |
| # Linux version of sed | |
| sed -i "s/^version = \".*\"/version = \"${BASE_VERSION}\"/" Cargo.toml | |
| sed -i '/name = "moose-cli"/,/version = ".*"/ s/version = ".*"/version = "'"${BASE_VERSION}"'"/' ../../Cargo.lock | |
| fi | |
| # Set Python package version in pyproject.toml (PEP 440 compliant) | |
| # Maturin will use this for the Python package metadata | |
| perl -i -pe 's/^(\[project\])$/$1\nversion = "${{ needs.version.outputs.python_version }}"/' pyproject.toml | |
| # Set MOOSE_CLI_VERSION environment variable for build.rs | |
| # This overrides the Cargo.toml version and gives the Rust binary the full version string | |
| echo "MOOSE_CLI_VERSION=${{ needs.version.outputs.version }}" >> $GITHUB_ENV | |
| # Debug: Show what we set | |
| echo "Cargo.toml version (base): ${BASE_VERSION}" | |
| echo "MOOSE_CLI_VERSION (full): ${{ needs.version.outputs.version }}" | |
| grep '^version = ' Cargo.toml || echo " (not found)" | |
| echo "pyproject.toml [project] section:" | |
| sed -n '/^\[project\]/,/^\[/p' pyproject.toml | head -10 | |
| working-directory: ./apps/framework-cli | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.build.TARGET }} | |
| args: --release --features rdkafka/cmake-build --locked --target ${{ matrix.build.TARGET }} --out dist | |
| sccache: ${{ matrix.build.OS != 'ubuntu-22-8-core' }} | |
| # https://github.com/briansmith/ring/issues/1728#issuecomment-1758180655 | |
| manylinux: ${{ (matrix.build.TARGET == 'aarch64-unknown-linux-gnu' || matrix.build.TARGET == 'x86_64-unknown-linux-gnu') && '2_28' || '' }} | |
| working-directory: ./apps/framework-cli | |
| # Otherwise it uses an centos machine that has a bunch of things missing. | |
| container: ${{ matrix.build.TARGET == 'x86_64-unknown-linux-gnu' && 'ghcr.io/rust-cross/manylinux_2_28-cross:x86_64' || '' }} | |
| # When running on manylinux we are inside a docker container and need to install protoc | |
| before-script-linux: | | |
| if [ -f /.dockerenv ]; then | |
| echo "Running in Docker container, installing required dependencies..." | |
| PROTOC_VERSION=24.4 | |
| PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip | |
| export POSTHOG_API_KEY=${{ secrets.POSTHOG_API_KEY }} | |
| export MOOSE_CLI_VERSION=${{ needs.version.outputs.version }} | |
| echo "MOOSE_CLI_VERSION set to: $MOOSE_CLI_VERSION" | |
| # Debian-based distribution | |
| apt install -y curl unzip | |
| mkdir -p /usr/local/bin | |
| mkdir -p /usr/local/include | |
| curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP | |
| unzip -o $PROTOC_ZIP -d /usr/local bin/protoc | |
| unzip -o $PROTOC_ZIP -d /usr/local 'include/*' | |
| rm -f $PROTOC_ZIP | |
| # Verify protoc installation | |
| protoc --version | |
| fi | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-linux-${{ matrix.build.TARGET }} | |
| path: ./apps/framework-cli/dist | |
| - name: Publish to NPM Moose | |
| if: ${{ !inputs.dry-run }} | |
| shell: bash | |
| working-directory: ./apps/moose-cli-npm | |
| run: ./scripts/release-bin.sh ${{ needs.version.outputs.version }} ${{ matrix.build.TARGET }} ${{ matrix.build.OS }} ${{ matrix.build.NAME }} | |
| - name: Upload binary | |
| uses: ncipollo/release-action@v1 | |
| if: ${{ !inputs.dry-run }} | |
| with: | |
| tag: v${{ needs.version.outputs.version }} | |
| allowUpdates: "true" | |
| replacesArtifacts: "false" | |
| artifacts: | | |
| ./target/${{ matrix.build.TARGET }}/release/moose-cli-${{ matrix.build.TARGET }} | |
| - name: Sign Binaries | |
| uses: 514-labs/secure-bin-signer-action@1.0.0 | |
| with: | |
| target: ${{ matrix.build.TARGET }} | |
| binary-path: ./target/${{ matrix.build.TARGET }}/release | |
| binary-name: moose-cli | |
| version: ${{ needs.version.outputs.version }} | |
| op-service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| # Upload to GCS | |
| - name: Auth | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| service_account: "mds-911@moose-hosting-node.iam.gserviceaccount.com" | |
| workload_identity_provider: "projects/724152421890/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-repos" | |
| - id: "upload-build" | |
| uses: "google-github-actions/upload-cloud-storage@v2" | |
| with: | |
| path: ./target/${{ matrix.build.TARGET }}/release | |
| glob: "moose-cli{,.sig,.sha256}" | |
| process_gcloudignore: false | |
| parent: false | |
| destination: "downloads.fiveonefour.com/${{ needs.version.outputs.is_ci_build == 'true' && 'dev' || 'stable' }}/${{ needs.version.outputs.version }}/${{ matrix.build.TARGET }}" | |
| headers: |- | |
| cache-control: ${{ needs.version.outputs.is_ci_build == 'true' && 'no-store, no-cache, must-revalidate' || 'public, max-age=3600' }} | |
| update-moose-version-index: | |
| needs: [build-and-publish-binaries, version] | |
| runs-on: ubuntu-latest | |
| concurrency: update-moose-version-index | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Auth | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| service_account: "mds-911@moose-hosting-node.iam.gserviceaccount.com" | |
| workload_identity_provider: "projects/724152421890/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-repos" | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v2" | |
| with: | |
| install_components: "gsutil,gcloud" | |
| - name: Determine channel | |
| id: channel | |
| run: | | |
| if [[ "${{ needs.version.outputs.is_ci_build }}" == "true" ]]; then | |
| echo "channel=dev" >> $GITHUB_OUTPUT | |
| else | |
| echo "channel=stable" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Update index file with CAS retry | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| OBJECT_URI="gs://downloads.fiveonefour.com/${{ steps.channel.outputs.channel }}/index.json" | |
| MAX_ATTEMPTS=8 | |
| for attempt in $(seq 1 "${MAX_ATTEMPTS}"); do | |
| echo "CAS update attempt ${attempt}/${MAX_ATTEMPTS} for ${OBJECT_URI}" | |
| describe_status=0 | |
| describe_output="" | |
| describe_err="" | |
| describe_err_file="$(mktemp)" | |
| describe_output="$(gcloud storage objects describe "${OBJECT_URI}" --format='value(generation)' 2>"${describe_err_file}")" || describe_status=$? | |
| describe_err="$(cat "${describe_err_file}")" | |
| rm -f "${describe_err_file}" | |
| if [ "${describe_status}" -eq 0 ]; then | |
| generation="$(echo "${describe_output}" | tr -d '[:space:]')" | |
| if ! [[ "${generation}" =~ ^[0-9]+$ ]]; then | |
| echo "Unexpected generation value from gcloud describe output: ${describe_output}" | |
| if [ -n "${describe_err}" ]; then | |
| echo "${describe_err}" | |
| fi | |
| exit 1 | |
| fi | |
| download_status=0 | |
| download_output="$(gcloud storage cp "${OBJECT_URI}" index.json 2>&1)" || download_status=$? | |
| if [ "${download_status}" -ne 0 ]; then | |
| echo "${download_output}" | |
| if [ "${attempt}" -lt "${MAX_ATTEMPTS}" ]; then | |
| sleep_seconds=$((attempt * 2)) | |
| echo "Failed to download current index file. Retrying in ${sleep_seconds}s..." | |
| sleep "${sleep_seconds}" | |
| continue | |
| fi | |
| echo "Failed to download current index file after ${MAX_ATTEMPTS} attempts." | |
| exit 1 | |
| fi | |
| elif echo "${describe_err}" | grep -Eq "No URLs matched|Not found|404"; then | |
| generation=0 | |
| echo '{"moose-cli":{"versions":[]}}' > index.json | |
| else | |
| if [ -n "${describe_err}" ]; then | |
| echo "${describe_err}" | |
| fi | |
| if [ -n "${describe_output}" ]; then | |
| echo "${describe_output}" | |
| fi | |
| exit 1 | |
| fi | |
| # This jq command: | |
| # 1. Ensures the moose-cli structure exists | |
| # 2. Adds the new version entry to the versions array | |
| # 3. Sorts all versions by date (newest first) | |
| # 4. Limits the list to 10000 entries to prevent unlimited growth | |
| jq --arg version "${{ needs.version.outputs.version }}" \ | |
| --arg date "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ | |
| 'if .["moose-cli"] == null then . + {"moose-cli": {"versions": []}} else . end | .["moose-cli"].versions = (.["moose-cli"].versions | if . == null then [] else . end) | .["moose-cli"].versions = (.["moose-cli"].versions | map(select(.version != $version))) | .["moose-cli"].versions = ([{"version": $version, "date": $date}] + .["moose-cli"].versions) | .["moose-cli"].versions = (.["moose-cli"].versions | sort_by(.date) | reverse | .[0:10000])' \ | |
| index.json > index.json.tmp | |
| if upload_output="$(gcloud storage cp index.json.tmp "${OBJECT_URI}" --if-generation-match="${generation}" 2>&1)"; then | |
| echo "Index updated successfully" | |
| gcloud compute url-maps invalidate-cdn-cache bins-distribution-url-map --path="/${{ steps.channel.outputs.channel }}/index.json" --async | |
| exit 0 | |
| fi | |
| echo "${upload_output}" | |
| if echo "${upload_output}" | grep -Eq "412|Precondition Failed|conditionNotMet"; then | |
| if [ "${attempt}" -lt "${MAX_ATTEMPTS}" ]; then | |
| sleep_seconds=$((attempt * 2)) | |
| echo "CAS conflict detected (412). Retrying in ${sleep_seconds}s..." | |
| sleep "${sleep_seconds}" | |
| continue | |
| fi | |
| echo "CAS conflict persisted after ${MAX_ATTEMPTS} attempts." | |
| exit 1 | |
| fi | |
| echo "Upload failed with non-retryable error." | |
| exit 1 | |
| done | |
| release-python: | |
| name: Release Python Wheels | |
| runs-on: ubuntu-latest | |
| # cleanup-pypi commented out: job is disabled, skipped jobs cause dependents to skip | |
| needs: [version, build-and-publish-binaries] # cleanup-pypi | |
| if: ${{ !inputs.dry-run && needs.version.outputs.is_ci_build == 'false' }} | |
| # Add explicit permissions | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| - uses: 1password/load-secrets-action@v1 | |
| id: op-load-secret | |
| with: | |
| export-env: false | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| PYPI_TOKEN: "op://drqe7p6legi6ug2ijq2fnrkmjq/PyPi Token/password" | |
| - name: Publish to PyPI | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| command: upload | |
| args: --non-interactive --skip-existing wheels-*/* | |
| env: | |
| MATURIN_PYPI_TOKEN: ${{ steps.op-load-secret.outputs.PYPI_TOKEN }} | |
| publish-npm-base: | |
| name: Publish the base NPM package | |
| needs: | |
| - version | |
| - build-and-publish-binaries | |
| - package-and-publish-independant-ts-package | |
| runs-on: ubuntu-latest | |
| if: ${{ !inputs.dry-run }} | |
| # Add explicit permissions | |
| permissions: | |
| contents: read | |
| id-token: write | |
| env: | |
| TAG_LATEST: ${{ needs.version.outputs.is_ci_build == 'false' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 100 | |
| - uses: 1password/load-secrets-action@v1 | |
| id: op-load-secret | |
| with: | |
| export-env: false | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| SENTRY_AUTH_TOKEN: "op://drqe7p6legi6ug2ijq2fnrkmjq/Sentry/credential" | |
| SENTRY_ORG: "op://drqe7p6legi6ug2ijq2fnrkmjq/Sentry/org" | |
| - uses: pnpm/action-setup@v4 | |
| - name: Install node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "24" | |
| cache: "pnpm" | |
| - name: Install dependencies | |
| shell: bash | |
| run: pnpm --filter ...create-moose-app --filter ...@514labs/moose-cli install | |
| - name: Wait for @514labs/moose-cli-darwin-arm64 to be available | |
| shell: bash | |
| run: ./scripts/wait-for-npm-package.sh @514labs/moose-cli-darwin-arm64 ${{ needs.version.outputs.version }} | |
| - name: Wait for @514labs/moose-cli-linux-arm64 to be available | |
| shell: bash | |
| run: ./scripts/wait-for-npm-package.sh @514labs/moose-cli-linux-arm64 ${{ needs.version.outputs.version }} | |
| - name: Wait for @514labs/moose-cli-linux-x64 to be available | |
| shell: bash | |
| run: ./scripts/wait-for-npm-package.sh @514labs/moose-cli-linux-x64 ${{ needs.version.outputs.version }} | |
| - name: Publish the NPM Moose CLI package | |
| shell: bash | |
| run: ./apps/moose-cli-npm/scripts/release-cli.sh ${{ needs.version.outputs.version }} | |
| - name: Wait for @514labs/moose-cli to be available | |
| shell: bash | |
| run: ./scripts/wait-for-npm-package.sh @514labs/moose-cli ${{ needs.version.outputs.version }} | |
| - name: Publish the NPM Moose create app package | |
| shell: bash | |
| run: ./apps/create-moose-app/scripts/release.sh ${{ needs.version.outputs.version }} | |
| build-and-publish-fullstack-image: | |
| name: Moose Production images | |
| runs-on: ubuntu-latest | |
| needs: | |
| - version | |
| - build-and-publish-binaries | |
| # Add explicit permissions | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - uses: 1password/load-secrets-action@v1 | |
| id: op-load-secret | |
| with: | |
| export-env: false | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| DOCKER_HUB_ACCESS_KEY: "op://drqe7p6legi6ug2ijq2fnrkmjq/Docker Hub - Bot/Access Token" | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: 514iceman | |
| password: ${{ steps.op-load-secret.outputs.DOCKER_HUB_ACCESS_KEY }} | |
| - name: Build and push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ./apps/framework-cli/deploy | |
| file: ./apps/framework-cli/deploy/Dockerfile.fullstack | |
| push: ${{ !inputs.dry-run }} | |
| tags: 514labs/moose-fullstack:latest, 514labs/moose-fullstack:0.0.0, 514labs/moose-fullstack:${{ needs.version.outputs.version }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| build-args: | | |
| FRAMEWORK_VERSION=${{ needs.version.outputs.version }} | |
| notify-slack-on-failure: | |
| needs: | |
| [ | |
| build-and-publish-py-moose-lib, | |
| package-and-publish-independant-ts-package, | |
| package-and-publish-templates, | |
| build-and-publish-binaries, | |
| release-python, | |
| publish-npm-base, | |
| build-and-publish-fullstack-image, | |
| ] | |
| runs-on: ubuntu-latest | |
| # Add explicit permissions | |
| permissions: | |
| contents: read | |
| if: failure() && github.ref == 'refs/heads/main' | |
| steps: | |
| - name: Notify Slack on failure | |
| uses: 514-labs/slack-notification-action@1.0.0 | |
| with: | |
| slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} | |
| slack-webhook-url: ${{ secrets.SLACK_GITHUB_ACTIONS_WEBHOOK_URL }} | |
| cleanup-pypi: | |
| name: Cleanup PyPI | |
| runs-on: ubuntu-latest | |
| # DISABLED: PyPI no longer supports automated deletion of releases. | |
| # The pypi-cleanup tool uses web scraping which breaks due to: | |
| # 1. PyPI sends device confirmation emails for each new IP (GitHub Actions runners change IPs) | |
| # 2. PyPI has no public API for deleting releases | |
| # Open feature requests to watch: | |
| # - https://github.com/pypi/warehouse/issues/12810 (API to delete old .dev wheels) | |
| # - https://github.com/pypi/warehouse/issues/11397 (Automatically delete old releases) | |
| # Re-enable this job if/when PyPI adds a proper deletion API. | |
| if: false | |
| continue-on-error: true | |
| needs: | |
| - version | |
| # Add explicit permissions | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.12" | |
| - name: Install pypi Cleanup | |
| run: | | |
| sudo apt install -y expect | |
| pip install pypi-cleanup | |
| - uses: 1password/load-secrets-action@v1 | |
| id: op-load-secret | |
| with: | |
| export-env: false | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| PYPI_TOKEN: "op://drqe7p6legi6ug2ijq2fnrkmjq/PyPi Token/password" | |
| PYPI_CLEANUP_PASSWORD: "op://drqe7p6legi6ug2ijq2fnrkmjq/PyPI/new_password" | |
| PYPI_OTP: "op://drqe7p6legi6ug2ijq2fnrkmjq/PyPI/Saved on pypi.org/one-time password?attribute=otp" | |
| - name: Clean up PyPI and only keep the latest 80 days of moose CLI and lib | |
| run: | | |
| # Disable set -e so we can continue even if one package cleanup fails | |
| set +e | |
| ERROR_CODE=0 | |
| # Function to run pypi-cleanup with error handling | |
| cleanup_package() { | |
| local package_name=$1 | |
| local days=$2 | |
| expect << EOF | |
| spawn pypi-cleanup -u 514 -p ${package_name} -d ${days} -r ".*" --do-it --yes | |
| expect { | |
| "Password:" { | |
| send "\$env(PYPI_CLEANUP_PASSWORD)\r" | |
| exp_continue | |
| } | |
| "Authentication code:" { | |
| send "\$env(PYPI_OTP)\r" | |
| exp_continue | |
| } | |
| "No releases were found" { | |
| # Nothing to do, this is fine | |
| } | |
| eof | |
| } | |
| # Check exit status | |
| catch wait result | |
| set exit_code [lindex \$result 3] | |
| if {\$exit_code != 0} { | |
| puts "ERROR: pypi-cleanup for ${package_name} failed with exit code \$exit_code" | |
| puts "" | |
| puts "This is most likely due to PyPI device authentication." | |
| puts "Please check your email for a message from noreply@pypi.org" | |
| puts "with the subject line 'Unrecognized login to your PyPI account'." | |
| puts "Click the confirmation link in that email to authorize this device." | |
| puts "" | |
| puts "For more details, see: https://pypi.org/help/#utfkey" | |
| exit \$exit_code | |
| } | |
| EOF | |
| local exit_code=$? | |
| if [ $exit_code -ne 0 ]; then | |
| echo "::error::pypi-cleanup for ${package_name} failed with exit code $exit_code" | |
| echo "" | |
| echo "This is most likely due to PyPI device authentication." | |
| echo "Please check your email for a message from noreply@pypi.org" | |
| echo "with the subject line 'Unrecognized login to your PyPI account'." | |
| echo "Click the confirmation link in that email to authorize this device." | |
| echo "For more details, see: https://pypi.org/help/#utfkey" | |
| ERROR_CODE=1 | |
| fi | |
| } | |
| # Clean up both packages (both will run even if first fails) | |
| cleanup_package "moose-cli" 80 | |
| cleanup_package "moose-lib" 80 | |
| # Fail the step if any error occurred | |
| if [ $ERROR_CODE -eq 1 ]; then | |
| exit 1 | |
| fi | |
| env: | |
| PYPI_CLEANUP_PASSWORD: ${{ steps.op-load-secret.outputs.PYPI_CLEANUP_PASSWORD }} | |
| PYPI_OTP: ${{ steps.op-load-secret.outputs.PYPI_OTP }} |