Optimize commit path #108
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PostgreSQL Extension Distribution | |
| permissions: | |
| contents: read | |
| id-token: write | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pg_version: | |
| description: "PostgreSQL version to build (16, 17, 18, or all)" | |
| required: false | |
| default: "18" | |
| type: choice | |
| options: | |
| - "16" | |
| - "17" | |
| - "18" | |
| - all | |
| pull_request: | |
| branches: | |
| - main | |
| types: | |
| - opened | |
| - synchronize | |
| push: | |
| tags: | |
| - v* | |
| concurrency: | |
| group: "${{ github.workflow }}-${{ github.ref_name }}" | |
| cancel-in-progress: true | |
| env: | |
| YELLOW: "\e[93m" | |
| GREEN: "\e[92m" | |
| RED: "\e[91m" | |
| DEFAULT: "\e[0m" | |
| PG_VERSION: "${{ inputs.pg_version || '18' }}" | |
| jobs: | |
| setup: | |
| name: Determine PostgreSQL Versions | |
| runs-on: ubuntu-latest | |
| outputs: | |
| versions: "${{ steps.set-versions.outputs.versions }}" | |
| versions-list: "${{ steps.set-versions.outputs.versions-list }}" | |
| steps: | |
| - name: set versions to build | |
| id: set-versions | |
| run: |- | |
| if [ "${PG_VERSION}" == "all" ]; then | |
| echo "versions=[\"16\",\"17\",\"18\"]" >> "${GITHUB_OUTPUT}" | |
| echo "versions-list=16,17,18" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "versions=[\"${PG_VERSION}\"]" >> "${GITHUB_OUTPUT}" | |
| echo "versions-list=${PG_VERSION}" >> "${GITHUB_OUTPUT}" | |
| fi | |
| extension-build: | |
| name: Build PostgreSQL Extension | |
| needs: [setup] | |
| outputs: | |
| version: "${{ steps.extract-version.outputs.version }}" | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| arch: [x86_64, aarch64] | |
| runs-on: "deeplake-linux-${{ matrix.arch }}-runner" | |
| container: "quay.io/activeloop/gcc-pypa-manylinux:2_28" | |
| env: | |
| VCPKG_ROOT: "${{ github.workspace }}/vcpkg" | |
| VCPKG_FEATURE_FLAGS: dependencygraph | |
| VCPKG_BINARY_SOURCES: "clear;files,/vcpkg-cache,readwrite" | |
| VCPKG_DISABLE_METRICS: true | |
| BUILD_PRESET: prod | |
| steps: | |
| - name: checkout | |
| shell: bash | |
| env: | |
| ORG_GH_BOT_PAT: "${{ secrets.ORG_GH_BOT_PAT }}" | |
| run: |- | |
| branch="${{ github.head_ref || github.ref_name }}" | |
| git clone -b "${branch}" "https://activeloop-bot:${ORG_GH_BOT_PAT}@github.com/${{ github.repository }}" . | |
| - name: configure aws credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: "arn:aws:iam::067976305224:role/github" | |
| aws-region: us-east-1 | |
| role-duration-seconds: "7200" | |
| - name: install vcpkg | |
| shell: bash | |
| run: |- | |
| bash scripts/build_scripts/manage_cache.sh download postgres | |
| git clone https://github.com/microsoft/vcpkg.git && cd vcpkg && git checkout 6f29f12e82a8293156836ad81cc9bf5af41fe836 && ./bootstrap-vcpkg.sh | |
| - name: install python dependencies | |
| shell: bash | |
| run: |- | |
| python3 -m pip install -U pip | |
| python3 -m pip install requests | |
| - name: build pg_deeplake for "${{ matrix.arch }}" | |
| shell: bash | |
| run: |- | |
| echo -e "${YELLOW}Building the PostgreSQL extension...${DEFAULT}" | |
| echo "Building for PostgreSQL version(s): ${PG_VERSION}" | |
| python3 scripts/build_pg_ext.py prod --deeplake-static --pg-versions "${PG_VERSION}" | |
| echo -e "${GREEN}Done.${DEFAULT}" | |
| - name: extract version | |
| id: extract-version | |
| shell: bash | |
| run: |- | |
| if [ -f "cpp/.ext/deeplake_api/lib/pkgconfig/deeplake_api.pc" ]; then | |
| VERSION=$(PKG_CONFIG_PATH="cpp/.ext/deeplake_api/lib/pkgconfig" pkg-config --modversion deeplake_api) | |
| echo "Extracted version from pkg-config: ${VERSION}" | |
| elif [[ "${GITHUB_REF}" == refs/tags/v* ]]; then | |
| VERSION="${GITHUB_REF#refs/tags/v}" | |
| echo "Extracted version from git tag: ${VERSION}" | |
| elif [[ -f "python/deeplake/__init__.py" ]]; then | |
| VERSION=$(grep -E '^__version__ *= *"' python/deeplake/__init__.py | cut -d'"' -f2) | |
| else | |
| echo "Error: Could not determine version" | |
| exit 1 | |
| fi | |
| echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" | |
| - name: test built packages | |
| shell: bash | |
| run: |- | |
| if [ "${PG_VERSION}" == "18" ] || [ "${PG_VERSION}" == "all" ]; then | |
| echo -e "${YELLOW}Running tests for PostgreSQL 18...${DEFAULT}" | |
| if [ ! -f "postgres/pg_deeplake_18.so" ]; then | |
| echo -e "${RED}Extension file pg_deeplake_18.so not found${DEFAULT}" | |
| echo "Built files in postgres/:" | |
| ls -la postgres/*.so || echo "No .so files found" | |
| exit 1 | |
| fi | |
| useradd -r -m -U -d /var/lib/postgresql -s /bin/bash postgres | |
| chown -R postgres:postgres /var/lib/postgresql | |
| chown -R postgres postgres/ | |
| echo -e "${YELLOW}Installing Python test dependencies...${DEFAULT}" | |
| python3 -m pip install -r postgres/tests/py_tests/requirements.txt | |
| echo -e "${YELLOW}Running Python tests...${DEFAULT}" | |
| export PG_MAJOR_VERSION=18 | |
| export USER=postgres | |
| cd postgres/tests/py_tests || exit 1 | |
| python3 -m pytest -v -m 'not slow and not tpch' --tb=short || { | |
| echo -e "${RED}Tests failed for PostgreSQL 18${DEFAULT}" | |
| exit 1 | |
| } | |
| cd ../.. | |
| echo -e "${GREEN}PostgreSQL 18 tests passed${DEFAULT}" | |
| else | |
| echo -e "${YELLOW}Skipping tests - only PostgreSQL 18 is tested (selected version: ${PG_VERSION})${DEFAULT}" | |
| fi | |
| - name: save cache and build artifacts | |
| shell: bash | |
| run: |- | |
| artifact_file="postgres-$(arch)" | |
| bash scripts/build_scripts/manage_cache.sh upload postgres | |
| mv ./postgres "${artifact_file}" && tar cf "${artifact_file}".tar "${artifact_file}" | |
| s5cmd cp "${artifact_file}".tar "s3://activeloop-platform-tests/indra/artifacts/postgres/${artifact_file}.tar" | |
| repo-build: | |
| name: Build PostgreSQL Extension Repo | |
| if: github.event_name == 'workflow_dispatch' | |
| needs: [setup, extension-build] | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" | |
| REPOSITORY: /tmp/activeloop-packages | |
| SUPPORTED_VERSIONS: "${{ needs.setup.outputs.versions-list }}" | |
| VERSION: "${{ needs.extension-build.outputs.version }}" | |
| runs-on: ubuntu-latest | |
| container: "quay.io/activeloopai/repo-builder:latest" | |
| steps: | |
| - name: checkout | |
| uses: actions/[email protected] | |
| - name: load 1pass | |
| uses: 1password/[email protected] | |
| with: | |
| export-env: true | |
| env: | |
| R2_ACCESS_KEY_ID: "op://GitHub Actions/registry-creds/cloudflare/CF_R2_ACCESS_KEY" | |
| R2_SECRET_ACCESS_KEY: "op://GitHub Actions/registry-creds/cloudflare/CF_R2_SECRET_KEY" | |
| R2_BUCKET_NAME: "op://GitHub Actions/common/REPOSITORY_BUCKET" | |
| R2_ENDPOINT_URL: "op://GitHub Actions/common/REPOSITORY_BUCKET_ENDPOINT" | |
| - name: configure aws credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: "arn:aws:iam::067976305224:role/github" | |
| aws-region: us-east-1 | |
| - name: download build artifacts | |
| shell: bash | |
| run: |- | |
| for arch in x86_64 aarch64; do | |
| s5cmd cp "s3://activeloop-platform-tests/indra/artifacts/postgres/postgres-${arch}.tar" postgres-"${arch}".tar | |
| tar xf postgres-"${arch}".tar | |
| done | |
| - name: build deb and rpm repositories | |
| shell: bash | |
| run: |- | |
| op read --force --out-file private.asc "op://GitHub Actions/files/packages_activeloop_io_private_key" | |
| op read --force --out-file public.asc "op://GitHub Actions/files/packages_activeloop_io_public_key" | |
| gpg --import public.asc | |
| gpg --import private.asc | |
| KEY_ID=$(gpg --with-colons \ | |
| --import-options show-only \ | |
| --import public.asc | grep '^pub' | cut -d: -f5) | |
| rm -f public.asc; rm -f private.asc | |
| mkdir -p "${REPOSITORY}"/scripts/pg-deeplake/ && | |
| cp -f postgres/scripts/install.sh "${REPOSITORY}"/scripts/pg-deeplake/ | |
| for type in deb rpm; do | |
| for arch in amd64 arm64; do | |
| echo -e "${YELLOW}Building the ${type} repository...${DEFAULT}" | |
| bash postgres/scripts/build_${type}.sh "${VERSION}"-1 "${REPOSITORY}" "${arch}" "${KEY_ID}" "${SUPPORTED_VERSIONS}" | |
| done | |
| done | |
| - name: install aws cli | |
| shell: bash | |
| run: |- | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-$(arch).zip" -o "awscliv2.zip" | |
| unzip awscliv2.zip && ./aws/install | |
| - name: index and upload repository to r2 | |
| shell: bash | |
| run: |- | |
| echo -e "${YELLOW}Generating index HTMLs...${GREEN}" | |
| unset AWS_SESSION_TOKEN && bash postgres/scripts/clone.sh | |
| find "${REPOSITORY}" -name "index.html" -exec rm -f {} \; | |
| python3 postgres/scripts/indexer.py --recursive "${{ env.REPOSITORY }}" && | |
| echo -e "${GREEN}Done.${DEFAULT}" | |
| echo -e "${YELLOW}Uploading the repositories to R2 bucket...${DEFAULT}" | |
| python3 postgres/scripts/uploader.py "${{ env.REPOSITORY }}" && | |
| echo -e "${GREEN}Done.${DEFAULT}" | |
| build_container_images: | |
| name: Build Container Images | |
| if: github.event_name == 'workflow_dispatch' | |
| needs: [setup, extension-build, repo-build] | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" | |
| VERSION: "${{ needs.extension-build.outputs.version }}" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: "${{ fromJson(needs.setup.outputs.versions) }}" | |
| steps: | |
| - name: checkout | |
| uses: actions/[email protected] | |
| - name: load 1pass | |
| id: load-1pass | |
| uses: 1password/[email protected] | |
| env: | |
| QUAY_USERNAME: "op://GitHub Actions/registry-creds/quay/QUAY_USERNAME" | |
| QUAY_PASSWORD: "op://GitHub Actions/registry-creds/quay/QUAY_PASSWORD" | |
| - name: login to quay | |
| uses: docker/[email protected] | |
| with: | |
| username: "${{ steps.load-1pass.outputs.QUAY_USERNAME }}" | |
| password: "${{ steps.load-1pass.outputs.QUAY_PASSWORD }}" | |
| registry: quay.io | |
| - name: setup docker qemu | |
| uses: docker/[email protected] | |
| - name: setup docker buildx | |
| uses: docker/[email protected] | |
| - name: build and push container images | |
| shell: bash | |
| env: | |
| PG_VERSION: "${{ matrix.version }}" | |
| VERSION: "${{ needs.extension-build.outputs.version }}" | |
| run: |- | |
| mkdir -p ./debs | |
| echo "Using package version: ${VERSION}" | |
| wget https://packages.activeloop.io/deb/pg-deeplake/pool/main/pg-deeplake-"${PG_VERSION}"_"${VERSION}"-1_amd64.deb -O ./debs/pg-deeplake-"${PG_VERSION}"_"${VERSION}"-1_amd64.deb | |
| wget https://packages.activeloop.io/deb/pg-deeplake/pool/main/pg-deeplake-"${PG_VERSION}"_"${VERSION}"-1_arm64.deb -O ./debs/pg-deeplake-"${PG_VERSION}"_"${VERSION}"-1_arm64.deb | |
| sed s/BASE_IMAGE/postgres:"${PG_VERSION}"-bookworm/g postgres/Dockerfile > Dockerfile.build | |
| docker buildx build \ | |
| --push \ | |
| --tag quay.io/activeloopai/pg-deeplake:"${PG_VERSION}"_"${VERSION}" \ | |
| --tag quay.io/activeloopai/pg-deeplake:"${PG_VERSION}" \ | |
| --build-arg VERSION="${PG_VERSION}"_"${VERSION}"-1 \ | |
| --platform linux/amd64,linux/arm64 \ | |
| -f Dockerfile.build \ | |
| . |