Refactor actions to run pg build and test on PRs. #81
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PostgreSQL Extension Distribution | |
| permissions: | |
| contents: read | |
| id-token: write | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pg_version: | |
| description: "PostgreSQL version to build (16, 17, 18, or all)" | |
| required: false | |
| default: "18" | |
| type: choice | |
| options: | |
| - "16" | |
| - "17" | |
| - "18" | |
| - all | |
| pull_request: | |
| branches: | |
| - main | |
| types: | |
| - opened | |
| - synchronize | |
| push: | |
| tags: | |
| - v* | |
| concurrency: | |
| group: "${{ github.workflow }}-${{ github.ref_name }}" | |
| cancel-in-progress: true | |
| env: | |
| YELLOW: "\e[93m" | |
| GREEN: "\e[92m" | |
| RED: "\e[91m" | |
| DEFAULT: "\e[0m" | |
| PG_VERSION: "${{ inputs.pg_version || '18' }}" | |
| jobs: | |
| setup: | |
| name: Determine PostgreSQL Versions | |
| runs-on: ubuntu-latest | |
| outputs: | |
| versions: "${{ steps.set-versions.outputs.versions }}" | |
| versions-list: "${{ steps.set-versions.outputs.versions-list }}" | |
| steps: | |
| - name: set versions to build | |
| id: set-versions | |
| run: |- | |
| if [ "${PG_VERSION}" == "all" ]; then | |
| echo "versions=[\"16\",\"17\",\"18\"]" >> "${GITHUB_OUTPUT}" | |
| echo "versions-list=16,17,18" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "versions=[\"${PG_VERSION}\"]" >> "${GITHUB_OUTPUT}" | |
| echo "versions-list=${PG_VERSION}" >> "${GITHUB_OUTPUT}" | |
| fi | |
| extension-build: | |
| name: Build PostgreSQL Extension | |
| needs: [setup] | |
| outputs: | |
| version: ${{ steps.extract-version.outputs.version }} | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| arch: [x86_64, aarch64] | |
| runs-on: "deeplake-linux-${{ matrix.arch }}-runner" | |
| container: "quay.io/activeloop/gcc-pypa-manylinux:2_28" | |
| env: | |
| VCPKG_ROOT: "${{ github.workspace }}/vcpkg" | |
| VCPKG_FEATURE_FLAGS: dependencygraph | |
| VCPKG_BINARY_SOURCES: "clear;files,/vcpkg-cache,readwrite" | |
| VCPKG_DISABLE_METRICS: true | |
| BUILD_PRESET: prod | |
| steps: | |
| - name: checkout | |
| shell: bash | |
| env: | |
| ORG_GH_BOT_PAT: "${{ secrets.ORG_GH_BOT_PAT }}" | |
| run: |- | |
| branch="${{ github.head_ref || github.ref_name }}" | |
| git clone -b "${branch}" "https://activeloop-bot:${ORG_GH_BOT_PAT}@github.com/${{ github.repository }}" . | |
| - name: configure aws credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: "arn:aws:iam::067976305224:role/github" | |
| aws-region: us-east-1 | |
| role-duration-seconds: "7200" | |
| - name: install vcpkg | |
| shell: bash | |
| run: |- | |
| bash scripts/build_scripts/manage_cache.sh download postgres | |
| git clone https://github.com/microsoft/vcpkg.git && cd vcpkg && git checkout 6f29f12e82a8293156836ad81cc9bf5af41fe836 && ./bootstrap-vcpkg.sh | |
| - name: install python dependencies | |
| shell: bash | |
| run: |- | |
| python3 -m pip install -U pip | |
| python3 -m pip install requests | |
| - name: build pg_deeplake for "${{ matrix.arch }}" | |
| shell: bash | |
| run: |- | |
| echo -e "${YELLOW}Building the PostgreSQL extension...${DEFAULT}" | |
| # Determine which versions to build based on input | |
| PG_VERSION="${{ inputs.pg_version || '18' }}" | |
| echo "Building for PostgreSQL version(s): ${PG_VERSION}" | |
| # Run the build with the appropriate --pg-versions flag | |
| python3 scripts/build_pg_ext.py prod --deeplake-static --pg-versions "${PG_VERSION}" | |
| echo -e "${GREEN}Done.${DEFAULT}" | |
| - name: extract version | |
| id: extract-version | |
| shell: bash | |
| run: |- | |
| if [ -f "cpp/.ext/deeplake_api/lib/pkgconfig/deeplake_api.pc" ]; then | |
| VERSION=$(PKG_CONFIG_PATH="cpp/.ext/deeplake_api/lib/pkgconfig" pkg-config --modversion deeplake_api) | |
| echo "Extracted version from pkg-config: ${VERSION}" | |
| elif [[ "${GITHUB_REF}" == refs/tags/v* ]]; then | |
| VERSION="${GITHUB_REF#refs/tags/v}" | |
| echo "Extracted version from git tag: ${VERSION}" | |
| elif [[ -f "python/deeplake/__init__.py" ]]; then | |
| VERSION=$(grep -E '^__version__ *= *"' python/deeplake/__init__.py | cut -d'"' -f2) | |
| else | |
| echo "Error: Could not determine version" | |
| exit 1 | |
| fi | |
| echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" | |
| - name: test built packages | |
| shell: bash | |
| run: |- | |
| if [ "${PG_VERSION}" == "18" ] || [ "${PG_VERSION}" == "all" ]; then | |
| echo -e "${YELLOW}Running tests for PostgreSQL 18...${DEFAULT}" | |
| if [ ! -f "postgres/pg_deeplake_18.so" ]; then | |
| echo -e "${RED}Extension file pg_deeplake_18.so not found${DEFAULT}" | |
| echo "Built files in postgres/:" | |
| ls -la postgres/*.so || echo "No .so files found" | |
| exit 1 | |
| fi | |
| # Create postgres user for PostgreSQL to run as | |
| useradd -r -m -U -d /var/lib/postgresql -s /bin/bash postgres | |
| chown -R postgres:postgres /var/lib/postgresql | |
| chown -R postgres postgres/ | |
| # Install Python test dependencies (pip already available from earlier step) | |
| echo -e "${YELLOW}Installing Python test dependencies...${DEFAULT}" | |
| python3 -m pip install -r postgres/tests/py_tests/requirements.txt | |
| # Run Python tests for PostgreSQL 18 (run as root in container) | |
| echo -e "${YELLOW}Running Python tests...${DEFAULT}" | |
| export PG_MAJOR_VERSION=18 | |
| export USER=postgres | |
| cd postgres/tests/py_tests | |
| python3 -m pytest -v -m 'not slow and not tpch' --tb=short || { | |
| echo -e "${RED}Tests failed for PostgreSQL 18${DEFAULT}" | |
| exit 1 | |
| } | |
| cd ../.. | |
| echo -e "${GREEN}PostgreSQL 18 tests passed${DEFAULT}" | |
| else | |
| echo -e "${YELLOW}Skipping tests - only PostgreSQL 18 is tested (selected version: ${PG_VERSION})${DEFAULT}" | |
| fi | |
| - name: save cache and build artifacts | |
| shell: bash | |
| run: |- | |
| artifact_file="postgres-$(arch)" | |
| bash scripts/build_scripts/manage_cache.sh upload postgres | |
| mv ./postgres "${artifact_file}" && tar cf "${artifact_file}".tar "${artifact_file}" | |
| s5cmd cp "${artifact_file}".tar "s3://activeloop-platform-tests/indra/artifacts/postgres/${artifact_file}.tar" | |
| repo-build: | |
| name: Build PostgreSQL Extension Repo | |
| if: github.event_name == 'workflow_dispatch' | |
| needs: [setup, extension-build] | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" | |
| REPOSITORY: /tmp/activeloop-packages | |
| SUPPORTED_VERSIONS: "${{ needs.setup.outputs.versions-list }}" | |
| VERSION: "${{ needs.extension-build.outputs.version }}" | |
| runs-on: ubuntu-latest | |
| container: "quay.io/activeloopai/repo-builder:latest" | |
| steps: | |
| - name: checkout | |
| uses: actions/[email protected] | |
| - name: load 1pass | |
| uses: 1password/load-secrets-action@v3 | |
| with: | |
| export-env: true | |
| env: | |
| R2_ACCESS_KEY_ID: "op://GitHub Actions/registry-creds/cloudflare/CF_R2_ACCESS_KEY" | |
| R2_SECRET_ACCESS_KEY: "op://GitHub Actions/registry-creds/cloudflare/CF_R2_SECRET_KEY" | |
| R2_BUCKET_NAME: "op://GitHub Actions/common/REPOSITORY_BUCKET" | |
| R2_ENDPOINT_URL: "op://GitHub Actions/common/REPOSITORY_BUCKET_ENDPOINT" | |
| - name: configure aws credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: "arn:aws:iam::067976305224:role/github" | |
| aws-region: us-east-1 | |
| - name: download build artifacts | |
| shell: bash | |
| run: |- | |
| for arch in x86_64 aarch64; do | |
| s5cmd cp "s3://activeloop-platform-tests/indra/artifacts/postgres/postgres-${arch}.tar" postgres-"${arch}".tar | |
| tar xf postgres-"${arch}".tar | |
| done | |
| - name: build deb and rpm repositories | |
| shell: bash | |
| run: |- | |
| op read --force --out-file private.asc "op://GitHub Actions/files/packages_activeloop_io_private_key" | |
| op read --force --out-file public.asc "op://GitHub Actions/files/packages_activeloop_io_public_key" | |
| gpg --import public.asc | |
| gpg --import private.asc | |
| KEY_ID=$(gpg --with-colons \ | |
| --import-options show-only \ | |
| --import public.asc | grep '^pub' | cut -d: -f5) | |
| rm -f public.asc; rm -f private.asc | |
| mkdir -p "${REPOSITORY}"/scripts/pg-deeplake/ && | |
| cp -f postgres/scripts/install.sh "${REPOSITORY}"/scripts/pg-deeplake/ | |
| # Use version from extension-build job output | |
| if [ -n "${PACKAGE_VERSION}" ]; then | |
| VERSION="${PACKAGE_VERSION}-1" | |
| echo -e "${GREEN}Using version from build job: ${VERSION}${DEFAULT}" | |
| else | |
| echo -e "${RED}Error: PACKAGE_VERSION not set${DEFAULT}" | |
| exit 1 | |
| fi | |
| for type in deb rpm; do | |
| for arch in amd64 arm64; do | |
| echo -e "${YELLOW}Building the ${type} repository...${DEFAULT}" | |
| bash postgres/scripts/build_${type}.sh "${VERSION}" "${REPOSITORY}" "${arch}" "${KEY_ID}" "${SUPPORTED_VERSIONS}" | |
| done | |
| done | |
| - name: index and upload repository to r2 | |
| shell: bash | |
| run: |- | |
| echo -e "${YELLOW}Generating index HTMLs...${GREEN}" | |
| unset AWS_SESSION_TOKEN && bash postgres/scripts/clone.sh | |
| find "${REPOSITORY}" -name "index.html" -exec rm -f {} \; | |
| python3 postgres/scripts/indexer.py --recursive "${{ env.REPOSITORY }}" && | |
| echo -e "${GREEN}Done.${DEFAULT}" | |
| echo -e "${YELLOW}Uploading the repositories to R2 bucket...${DEFAULT}" | |
| python3 postgres/scripts/uploader.py "${{ env.REPOSITORY }}" && | |
| echo -e "${GREEN}Done.${DEFAULT}" | |
| build_container_images: | |
| name: Build Container Images | |
| if: github.event_name == 'workflow_dispatch' | |
| needs: [setup, extension-build, repo-build] | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" | |
| VERSION: "${{ needs.extension-build.outputs.version }}" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: "${{ fromJson(needs.setup.outputs.versions) }}" | |
| steps: | |
| - name: checkout | |
| uses: actions/[email protected] | |
| - name: load 1pass | |
| id: load-1pass | |
| uses: 1password/load-secrets-action@v3 | |
| env: | |
| QUAY_USERNAME: "op://GitHub Actions/registry-creds/quay/QUAY_USERNAME" | |
| QUAY_PASSWORD: "op://GitHub Actions/registry-creds/quay/QUAY_PASSWORD" | |
| - name: login to quay | |
| uses: docker/[email protected] | |
| with: | |
| username: "${{ steps.load-1pass.outputs.QUAY_USERNAME }}" | |
| password: "${{ steps.load-1pass.outputs.QUAY_PASSWORD }}" | |
| registry: quay.io | |
| - name: setup docker qemu | |
| uses: docker/[email protected] | |
| - name: setup docker buildx | |
| uses: docker/[email protected] | |
| - name: build and push container images | |
| shell: bash | |
| env: | |
| PG_VERSION: ${{ matrix.version }} | |
| PACKAGE_VERSION: ${{ needs.extension-build.outputs.version }} | |
| run: |- | |
| mkdir -p ./debs | |
| # Use version from extension-build job output | |
| if [ -z "${PACKAGE_VERSION}" ]; then | |
| echo "Error: PACKAGE_VERSION not set" | |
| exit 1 | |
| fi | |
| VERSION="${PACKAGE_VERSION}" | |
| echo "Using package version: ${VERSION}" | |
| wget https://packages.activeloop.io/deb/pg-deeplake/pool/main/pg-deeplake-${PG_VERSION}_${VERSION}-1_amd64.deb -O ./debs/pg-deeplake-${PG_VERSION}_${VERSION}-1_amd64.deb | |
| wget https://packages.activeloop.io/deb/pg-deeplake/pool/main/pg-deeplake-${PG_VERSION}_${VERSION}-1_arm64.deb -O ./debs/pg-deeplake-${PG_VERSION}_${VERSION}-1_arm64.deb | |
| sed s/BASE_IMAGE/postgres:${PG_VERSION}-bookworm/g postgres/Dockerfile > Dockerfile.build | |
| docker buildx build \ | |
| --push \ | |
| --tag quay.io/activeloopai/pg-deeplake:"${PG_VERSION}"_"${VERSION}" \ | |
| --tag quay.io/activeloopai/pg-deeplake:"${PG_VERSION}" \ | |
| --build-arg VERSION="${PG_VERSION}"_"${VERSION}"-1 \ | |
| --platform linux/amd64,linux/arm64 \ | |
| -f Dockerfile.build \ | |
| . |