Bao-tree and logfile ingest factory; FilePhysicalSeries design #123
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rust CI | |
| on: | |
| push: | |
| branches: [ "main" ] | |
| tags: | |
| - 'v*' | |
| pull_request: | |
| branches: [ "main" ] | |
| workflow_dispatch: | |
| env: | |
| RUST_BACKTRACE: 1 | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }}/duckpond | |
| jobs: | |
| check: | |
| name: Check, Build & Test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Free up disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/share/boost | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| df -h | |
| - name: Setup Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Check formatting | |
| run: cargo fmt --all -- --check | |
| - name: Run tests | |
| run: cargo test --workspace | |
| - name: Run clippy | |
| run: cargo clippy --workspace --all-features -- -D warnings | |
| build-pond-cli: | |
| name: Build & Publish Container | |
| needs: check | |
| # Run on main/tags always, on PRs only with 'build-container' label | |
| if: | | |
| (success() || failure()) && ( | |
| github.event_name != 'pull_request' || | |
| contains(github.event.pull_request.labels.*.name, 'build-container') | |
| ) | |
| runs-on: ubuntu-latest | |
| # Note: PRs from forks won't have write access to packages, so the push step will be skipped for them | |
| # For PRs from the same repo, this builds and pushes :nightly, :pr-N, and :pr-N-SHA tags | |
| permissions: | |
| contents: read | |
| packages: write | |
| id-token: write # For Sigstore signing | |
| pull-requests: write # For commenting on PRs | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Podman | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install podman | |
| - name: Install Cosign | |
| uses: sigstore/[email protected] | |
| - name: Log in to Container Registry (Podman) | |
| if: github.event_name != 'pull_request' | |
| run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Log in to Container Registry (Cosign) | |
| if: github.event_name != 'pull_request' | |
| run: echo "${{ secrets.GITHUB_TOKEN }}" | cosign login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Extract metadata | |
| id: meta | |
| run: | | |
| IMAGE_ID=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| # For PRs: use nightly tag and also create a PR-specific tag | |
| VERSION=nightly | |
| PR_TAG=pr-${{ github.event.pull_request.number }} | |
| SHORT_SHA=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7) | |
| echo "pr_tag=$PR_TAG" >> $GITHUB_OUTPUT | |
| echo "sha_tag=${PR_TAG}-${SHORT_SHA}" >> $GITHUB_OUTPUT | |
| else | |
| VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') | |
| [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') | |
| [ "$VERSION" == "main" ] && VERSION=latest | |
| fi | |
| echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT | |
| echo "version=$VERSION" >> $GITHUB_OUTPUT | |
| echo "is_pr=${{ github.event_name == 'pull_request' }}" >> $GITHUB_OUTPUT | |
| - name: Log in to Container Registry (Podman) for PR | |
| if: github.event_name == 'pull_request' | |
| run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Log in to Container Registry (Cosign) for PR | |
| if: github.event_name == 'pull_request' | |
| run: echo "${{ secrets.GITHUB_TOKEN }}" | cosign login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Set up QEMU for multi-arch builds | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Free up more disk space before builds | |
| run: | | |
| df -h | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/local/.ghcup | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune -a -f || true | |
| df -h | |
| - name: Build multi-arch container image | |
| run: | | |
| # Create a manifest list | |
| podman manifest create ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.version }} | |
| # Build for AMD64 | |
| echo "Building AMD64..." | |
| df -h | |
| podman build \ | |
| --platform linux/amd64 \ | |
| --manifest ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.version }} \ | |
| -f Dockerfile . | |
| # Clean up only build cache, not images (--filter keeps images referenced by manifests) | |
| echo "Cleaning build cache after AMD64..." | |
| podman builder prune -f || true | |
| df -h | |
| # Build for ARM64 | |
| echo "Building ARM64..." | |
| podman build \ | |
| --platform linux/arm64 \ | |
| --manifest ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.version }} \ | |
| -f Dockerfile . | |
| # For PRs, copy the manifest for additional tags (don't rebuild) | |
| if [[ "${{ steps.meta.outputs.is_pr }}" == "true" ]]; then | |
| echo "Creating PR tags from existing manifest..." | |
| podman manifest create ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.pr_tag }} | |
| podman manifest create ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.sha_tag }} | |
| # Add existing images to PR manifests by digest | |
| for digest in $(podman manifest inspect ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.version }} | jq -r '.manifests[].digest'); do | |
| podman manifest add ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.pr_tag }} ${{ steps.meta.outputs.image_id }}@$digest | |
| podman manifest add ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.sha_tag }} ${{ steps.meta.outputs.image_id }}@$digest | |
| done | |
| fi | |
| - name: Push container image | |
| if: github.event_name != 'pull_request' | |
| id: push | |
| run: | | |
| podman manifest push ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.version }} docker://${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.version }} | |
| DIGEST=$(podman manifest inspect ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.version }} | jq -r '.manifests[0].digest') | |
| echo "digest=$DIGEST" >> $GITHUB_OUTPUT | |
| - name: Push PR container images (nightly) | |
| if: github.event_name == 'pull_request' | |
| id: push-pr | |
| run: | | |
| # Push nightly tag (overwrites previous) | |
| podman manifest push ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.version }} docker://${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.version }} | |
| # Push PR-specific tag | |
| podman manifest push ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.pr_tag }} docker://${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.pr_tag }} | |
| # Push SHA-specific tag for exact commit tracking | |
| podman manifest push ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.sha_tag }} docker://${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.sha_tag }} | |
| DIGEST=$(podman manifest inspect ${{ steps.meta.outputs.image_id }}:${{ steps.meta.outputs.version }} | jq -r '.manifests[0].digest') | |
| echo "digest=$DIGEST" >> $GITHUB_OUTPUT | |
| - name: Sign container image with Sigstore | |
| if: github.event_name != 'pull_request' | |
| run: | | |
| cosign sign --yes ${{ steps.meta.outputs.image_id }}@${{ steps.push.outputs.digest }} | |
| - name: Sign PR container image with Sigstore | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| cosign sign --yes ${{ steps.meta.outputs.image_id }}@${{ steps.push-pr.outputs.digest }} | |
| - name: Comment PR with image tags | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const imageId = '${{ steps.meta.outputs.image_id }}'; | |
| const nightlyTag = '${{ steps.meta.outputs.version }}'; | |
| const prTag = '${{ steps.meta.outputs.pr_tag }}'; | |
| const shaTag = '${{ steps.meta.outputs.sha_tag }}'; | |
| const body = `🐳 **Container images built and pushed!** | |
| You can test this PR using any of these tags: | |
| | Tag | Image | Description | | |
| |-----|-------|-------------| | |
| | \`nightly\` | \`${imageId}:${nightlyTag}\` | Latest PR build (overwrites on each PR commit) | | |
| | \`${prTag}\` | \`${imageId}:${prTag}\` | Latest build for this PR | | |
| | \`${shaTag}\` | \`${imageId}:${shaTag}\` | This exact commit | | |
| \`\`\`bash | |
| # Pull and run the nightly image | |
| podman pull ${imageId}:${nightlyTag} | |
| # or | |
| docker pull ${imageId}:${nightlyTag} | |
| \`\`\` | |
| `; | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: body | |
| }); |