Publish Artifacts #21
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish Artifacts | |
| # Workflow control - runs nightly or on manual trigger | |
| on: | |
| schedule: | |
| # Run nightly at 6.30 PM UTC or 12 AM IST | |
| - cron: '30 18 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| CONTAINER_TAG: | |
| description: 'Custom tag for containers (optional)' | |
| required: false | |
| default: '' | |
| ARTIFACTORY_VERSION: | |
| description: 'Artifactory version (optional, defaults to auto-generated from get_version.sh)' | |
| required: false | |
| default: '' | |
| env: | |
| RELEASE_TYPE: dev | |
| jobs: | |
| # ============================================================================ | |
| # PUBLISH WHEEL | |
| # ============================================================================ | |
| publish-wheel: | |
| name: Build and Publish Python Wheel | |
| runs-on: ubuntu-latest | |
| container: | |
| image: python:3.10 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set artifactory version | |
| run: | | |
| if [ -n "${{ github.event.inputs.ARTIFACTORY_VERSION }}" ]; then | |
| echo "Using custom Artifactory version: ${{ github.event.inputs.ARTIFACTORY_VERSION }}" | |
| echo "ARTIFACTORY_VERSION=${{ github.event.inputs.ARTIFACTORY_VERSION }}" >> $GITHUB_ENV | |
| else | |
| echo "Using version from get_version.sh..." | |
| chmod +x ./ci/get_version.sh | |
| DEFAULT_VERSION=$(./ci/get_version.sh) | |
| echo "Generated default version: $DEFAULT_VERSION" | |
| echo "ARTIFACTORY_VERSION=$DEFAULT_VERSION" >> $GITHUB_ENV | |
| fi | |
| - name: Install dependencies | |
| run: | | |
| pip install uv==0.8.12 | |
| apt-get update && apt-get install -y wget unzip | |
| - name: Build wheel with version | |
| run: | | |
| echo "Building wheel with version: $ARTIFACTORY_VERSION" | |
| sed -i "s#^version = \".*\"#version = \"$ARTIFACTORY_VERSION\"#" pyproject.toml | |
| uv build | |
| ls -la dist/ | |
| - name: Upload wheel artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheel-${{ env.ARTIFACTORY_VERSION }} | |
| path: dist/*.whl | |
| retention-days: 30 | |
| - name: Install NGC CLI | |
| env: | |
| NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| run: | | |
| echo "Installing NGC CLI..." | |
| wget --content-disposition https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/4.9.10/files/ngccli_linux.zip -O ngccli_linux.zip | |
| unzip -o ngccli_linux.zip | |
| chmod u+x ngc-cli/ngc | |
| # Add NGC CLI to PATH for subsequent steps | |
| echo "$(pwd)/ngc-cli" >> $GITHUB_PATH | |
| echo "NGC CLI installed successfully" | |
| - name: Publish wheel to NGC | |
| env: | |
| NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| run: | | |
| # Find the wheel file | |
| WHEEL_FILE=$(ls dist/*.whl | tail -n 1) | |
| echo "Found wheel file: $WHEEL_FILE" | |
| # Extract just the filename for display | |
| WHEEL_FILENAME=$(basename "$WHEEL_FILE") | |
| echo "Wheel filename: $WHEEL_FILENAME" | |
| # Publish to NGC | |
| echo "Publishing wheel to NGC: nvstaging/blueprint/nvidia_rag:$ARTIFACTORY_VERSION" | |
| ngc registry resource upload-version \ | |
| "nvstaging/blueprint/nvidia_rag:$ARTIFACTORY_VERSION" \ | |
| --source "$WHEEL_FILE" \ | |
| --org nvstaging | |
| echo "Wheel published to NGC successfully" | |
| echo "NGC Resource: nvstaging/blueprint/nvidia_rag:$ARTIFACTORY_VERSION" | |
| # ============================================================================ | |
| # PUBLISH RAG SERVER CONTAINER | |
| # ============================================================================ | |
| publish-rag-server: | |
| name: Build and Publish RAG Server Container | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Determine TAG | |
| id: tag | |
| run: | | |
| if [ -n "${{ github.event.inputs.CONTAINER_TAG }}" ]; then | |
| echo "Using custom TAG from input: ${{ github.event.inputs.CONTAINER_TAG }}" | |
| TAG="${{ github.event.inputs.CONTAINER_TAG }}" | |
| else | |
| echo "Using auto-generated version as TAG" | |
| VERSION=$(./ci/get_version.sh) | |
| TAG=$VERSION | |
| fi | |
| echo "TAG=$TAG" >> $GITHUB_ENV | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| echo "Final TAG value: $TAG" | |
| - name: Login to NGC Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: nvcr.io | |
| username: '$oauthtoken' | |
| password: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| - name: Build and push RAG server container | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| echo "Building rag-server container with tag ${TAG}..." | |
| export TAG=${TAG} | |
| export DOWNLOAD_LEGAL_COMPLIANCE=true | |
| docker compose -f deploy/compose/docker-compose-rag-server.yaml build rag-server | |
| # Tag and push to NGC Container Registry | |
| echo "Pushing rag-server to NGC Container Registry..." | |
| docker push nvcr.io/nvstaging/blueprint/rag-server:$TAG | |
| docker tag nvcr.io/nvstaging/blueprint/rag-server:$TAG nvcr.io/nvstaging/blueprint/rag-server:latest | |
| docker push nvcr.io/nvstaging/blueprint/rag-server:latest | |
| echo "RAG server container publishing completed successfully" | |
| - name: Cleanup Docker images | |
| if: always() | |
| run: | | |
| echo "Cleaning up rag-server Docker images..." | |
| docker images | grep "rag-server" | awk '{print $3}' | xargs -r docker rmi -f || echo "No rag-server images to delete" | |
| docker system prune -f || true | |
| # ============================================================================ | |
| # PUBLISH INGESTOR SERVER CONTAINER | |
| # ============================================================================ | |
| publish-ingestor-server: | |
| name: Build and Publish Ingestor Server Container | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Determine TAG | |
| id: tag | |
| run: | | |
| if [ -n "${{ github.event.inputs.CONTAINER_TAG }}" ]; then | |
| echo "Using custom TAG from input: ${{ github.event.inputs.CONTAINER_TAG }}" | |
| TAG="${{ github.event.inputs.CONTAINER_TAG }}" | |
| else | |
| echo "Using auto-generated version as TAG" | |
| VERSION=$(./ci/get_version.sh) | |
| TAG=$VERSION | |
| fi | |
| echo "TAG=$TAG" >> $GITHUB_ENV | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| echo "Final TAG value: $TAG" | |
| - name: Login to NGC Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: nvcr.io | |
| username: '$oauthtoken' | |
| password: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| - name: Build and push ingestor server container | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| echo "Building ingestor-server container with tag ${TAG}..." | |
| export TAG=${TAG} | |
| export DOWNLOAD_LEGAL_COMPLIANCE=true | |
| docker compose -f deploy/compose/docker-compose-ingestor-server.yaml build ingestor-server | |
| # Tag and push to NGC Container Registry | |
| echo "Pushing ingestor-server to NGC Container Registry..." | |
| docker push nvcr.io/nvstaging/blueprint/ingestor-server:$TAG | |
| docker tag nvcr.io/nvstaging/blueprint/ingestor-server:$TAG nvcr.io/nvstaging/blueprint/ingestor-server:latest | |
| docker push nvcr.io/nvstaging/blueprint/ingestor-server:latest | |
| echo "Ingestor server container publishing completed successfully" | |
| - name: Cleanup Docker images | |
| if: always() | |
| run: | | |
| echo "Cleaning up ingestor-server Docker images..." | |
| docker images | grep "ingestor-server" | awk '{print $3}' | xargs -r docker rmi -f || echo "No ingestor-server images to delete" | |
| docker system prune -f || true | |
| # ============================================================================ | |
| # PUBLISH RAG FRONTEND CONTAINER | |
| # ============================================================================ | |
| publish-rag-frontend: | |
| name: Build and Publish RAG Frontend Container | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Determine TAG | |
| id: tag | |
| run: | | |
| if [ -n "${{ github.event.inputs.CONTAINER_TAG }}" ]; then | |
| echo "Using custom TAG from input: ${{ github.event.inputs.CONTAINER_TAG }}" | |
| TAG="${{ github.event.inputs.CONTAINER_TAG }}" | |
| else | |
| echo "Using auto-generated version as TAG" | |
| VERSION=$(./ci/get_version.sh) | |
| TAG=$VERSION | |
| fi | |
| echo "TAG=$TAG" >> $GITHUB_ENV | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| echo "Final TAG value: $TAG" | |
| - name: Login to NGC Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: nvcr.io | |
| username: '$oauthtoken' | |
| password: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| - name: Build and push RAG frontend container | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| echo "Building rag-frontend container with tag ${TAG}..." | |
| export TAG=${TAG} | |
| export DOWNLOAD_LEGAL_COMPLIANCE=true | |
| docker compose -f deploy/compose/docker-compose-rag-server.yaml build rag-frontend | |
| # Tag and push to NGC Container Registry | |
| echo "Pushing rag-frontend to NGC Container Registry..." | |
| docker push nvcr.io/nvstaging/blueprint/rag-frontend:$TAG | |
| docker tag nvcr.io/nvstaging/blueprint/rag-frontend:$TAG nvcr.io/nvstaging/blueprint/rag-frontend:latest | |
| docker push nvcr.io/nvstaging/blueprint/rag-frontend:latest | |
| echo "RAG frontend container publishing completed successfully" | |
| - name: Cleanup Docker images | |
| if: always() | |
| run: | | |
| echo "Cleaning up rag-frontend Docker images..." | |
| docker images | grep "rag-frontend" | awk '{print $3}' | xargs -r docker rmi -f || echo "No rag-frontend images to delete" | |
| docker system prune -f || true | |