generate-homeval.yaml

name: generate-homeval

on:
  workflow_dispatch:
    inputs:
      year:
        type: string
        description: >
          Assessment year: Use values and comps from the final model for this year.
          Leave blank if you are supplying a run ID instead
        required: false
        default: ""

      run_id:
        type: string
        description: >
          Run ID: The model run whose comps should be used. Leave blank if you are supplying
          an assessment year instead
        required: false
        default: ""

      pins:
        type: string
        description: >
          PINs: One or more space‑separated Cook County PINs. Leave blank to
          generate reports for all PINs in the County, or if you are supplying
          township codes
        required: false
        default: ""

      townships:
        type: string
        description: >
          Township codes: One or more space-separated township codes. Leave blank to
          generate reports for all PINs in the County, or if you are supplying PINs
        required: false
        default: ""

      environment:
        type: choice
        description: >
          Environment: Choose where to upload the reports
        options:
          - Development bucket
          - Production bucket
        default: Development bucket
        required: true

      deploy-assets:
        type: boolean
        description: >
          Deploy static assets like favicons and error pages in addition
          to reports
        required: false
        default: false

      hugo-build-by-nbhd:
        type: boolean
        description: >
          Build reports by neighborhood: Slower, but uses less memory
        required: false
        default: false

      eligible-only:
        type: boolean
        description: >
          Only generate reports for PINs that are eligible for reports (i.e. no error pages)
        required: false
        default: false

env:
  PYTHONUNBUFFERED: "1"
  UV_SYSTEM_PYTHON: 1
  AWS_REGION: us-east-1
  AWS_ATHENA_S3_STAGING_DIR: s3://ccao-athena-results-us-east-1/
  HUGO_VERSION: "0.147.5"

defaults:
  run:
    # Always use builtin bash shell to take advantage of automatic -eo pipefail.
    # See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#defaultsrunshell
    shell: bash

jobs:
###############################################################################
# Setup
###############################################################################
  setup:
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    outputs:
      matrix: ${{ steps.resolve-metadata.outputs.matrix }}
      count: ${{ steps.resolve-metadata.outputs.count }}
      assessment-year: ${{ steps.resolve-metadata.outputs.assessment-year }}
      run-id: ${{ steps.resolve-metadata.outputs.run-id }}
      triad: ${{ steps.resolve-metadata.outputs.triad }}
      bucket: ${{ steps.resolve-aws.outputs.bucket }}
      distribution-id: ${{ steps.resolve-aws.outputs.distribution-id }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Mask sensitive information
        run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}"

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          enable-cache: true
          cache-dependency-glob: scripts/generate_homeval/uv.lock

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: scripts/generate_homeval/.python-version

      - name: Install Python requirements
        run: uv pip install .
        working-directory: scripts/generate_homeval

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }}
          aws-region: us-east-1

      - name: Resolve model metadata
        id: resolve-metadata
        run: |
          # Parse PIN and township lists to array so we can safely pass them
          # into this script, which accepts multiple space-delimited values for
          # both arguments
          read -ra PINS <<< "${{ inputs.pins }}"
          read -ra TOWNSHIPS <<< "${{ inputs.townships }}"
          ELIG_FLAG=""
          if [ "${{ inputs.eligible-only }}" = "true" ]; then
            ELIG_FLAG="--eligible-only"
          fi
          python3 scripts/generate_homeval/resolve_model_metadata.py \
            --year "${{ inputs.year }}" \
            --run-id "${{ inputs.run_id }}" \
            --pin "${PINS[@]}" \
            --township "${TOWNSHIPS[@]}" \
            --write-github-output \
            $ELIG_FLAG

      - name: Resolve AWS resources
        id: resolve-aws
        run: |
          if [[ "${{ inputs.environment }}" == 'Development bucket' ]]; then
            BUCKET="${{ vars.AWS_S3_BUCKET_PINVAL_STAGING }}"
            DISTRIBUTION_ID="${{ vars.AWS_CLOUDFRONT_DISTRIBUTION_ID_STAGING }}"
          else
            BUCKET="${{ vars.AWS_S3_BUCKET_PINVAL_PROD }}"
            DISTRIBUTION_ID="${{ vars.AWS_CLOUDFRONT_DISTRIBUTION_ID_PROD }}"
          fi

          BUCKET_STR="bucket=${BUCKET}"
          echo "$BUCKET_STR"
          echo "$BUCKET_STR" >> "$GITHUB_OUTPUT"

          DISTRIBUTION_ID_STR="distribution-id=${DISTRIBUTION_ID}"
          echo "$DISTRIBUTION_ID_STR"
          echo "$DISTRIBUTION_ID_STR" >> "$GITHUB_OUTPUT"

###############################################################################
# Generate HomeVal reports for a triad or explicit PINs
###############################################################################
  generate-homeval:
    needs: setup
    runs-on: ubuntu-latest
    strategy:
      matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
      max-parallel: ${{ fromJson(needs.setup.outputs.count) }}
    env:
      TOWNSHIP: ${{ matrix.township }}
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Mask sensitive information
        run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}"

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          enable-cache: true
          cache-dependency-glob: scripts/generate_homeval/uv.lock

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: scripts/generate_homeval/.python-version

      - name: Install Python requirements
        run: uv pip install .
        working-directory: scripts/generate_homeval

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }}
          aws-region: us-east-1
          role-duration-seconds: 7200

      - name: Cache Hugo package
        id: cache-hugo
        uses: actions/cache@v4
        with:
          path: ${{ runner.temp }}/hugo.deb
          key: hugo-deb-${{ env.HUGO_VERSION }}

      - name: Download Hugo
        if: steps.cache-hugo.outputs.cache-hit != 'true'
        run: |
          wget -O ${{ runner.temp }}/hugo.deb \
            https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb

      - name: Install Hugo
        run: sudo dpkg -i ${{ runner.temp }}/hugo.deb

      - name: Generate Hugo content files and build reports
        env:
          RUN_ID: ${{ needs.setup.outputs.run-id }}
          ENV_CODE: ${{ inputs.environment == 'Production bucket' && 'prod' || 'dev' }}
        run: |
          read -ra PINS <<< "${{ inputs.pins }}"
          NBHD_FLAG=""
          if [ "${{ inputs.hugo-build-by-nbhd }}" = "true" ]; then
            NBHD_FLAG="--build-by-neighborhood"
          fi
          ELIG_FLAG=""
          if [ "${{ inputs.eligible-only }}" = "true" ]; then
            ELIG_FLAG="--eligible-only"
          fi
          # Unquoted neighborhood flag is intentional in this call, since we
          # want this to expand to nothing (rather than an empty string) if
          # the variable is empty
          python3 scripts/generate_homeval/generate_homeval.py \
            --run-id "$RUN_ID" \
            --township "$TOWNSHIP" \
            --pin "${PINS[@]}" \
            --environment "$ENV_CODE" \
            $NBHD_FLAG \
            $ELIG_FLAG

      - name: Sync reports to S3
        env:
          TARGET_BUCKET: ${{ needs.setup.outputs.bucket }}
          YEAR: ${{ needs.setup.outputs.assessment-year }}
        run: |
          LOCAL="hugo/public"
          REMOTE="${TARGET_BUCKET%/}"
          if [[ "${{ inputs.deploy-assets }}" == 'true' ]]; then
            echo "Deploying static assets in addition to reports"
          else
            echo "Skipping static assets and deploying reports only"
            LOCAL="${LOCAL}/${YEAR}"
            REMOTE="${REMOTE}/${YEAR}"
          fi

          aws s3 sync "${LOCAL}/" "${REMOTE}/" --no-progress

###############################################################################
# Invalidate cache and notify
###############################################################################
  notify:
    needs:
      - setup
      - generate-homeval
    runs-on: ubuntu-latest
    permissions:
      id-token: write
    steps:
      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }}
          aws-region: us-east-1

      - name: Mask AWS identifiers
        run: |
          echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}"
          echo "::add-mask::${{ secrets.AWS_SNS_NOTIFICATION_TOPIC_ARN }}"

      - name: Invalidate cache
        env:
          YEAR: ${{ needs.setup.outputs.assessment-year }}
          TARGET_DISTRIBUTION_ID: ${{ needs.setup.outputs.distribution-id }}
        run: |
          # If the user passed in PINs, we want to only invalidate the cache
          # for those PINs, and preserve the cache for everything else to
          # improve performance for unrelated reports
          read -ra PINS <<< "${{ inputs.pins }}"
          INVALIDATION_PATHS=()
          if (( ${#PINS[@]} > 0 )); then
            for pin in "${PINS[@]:-}"; do
              INVALIDATION_PATHS+=("/${YEAR}/${pin}.html")
            done
          else
            # If the user didn't pass in PINs, we must have updated the whole
            # tri, so invalidate everything in this year
            INVALIDATION_PATHS+=("/${YEAR}/*.html")
          fi

          # If the user is deploying static assets, also invalidate those
          # assets
          if [[ "${{ inputs.deploy-assets }}" == 'true' ]]; then
            # This pattern will exclude robots.txt, since that file must live at
            # the root, but this is fine since we rarely if ever expect that file
            # to change
            INVALIDATION_PATHS+=("/static/*")
          fi

          aws cloudfront create-invalidation \
            --distribution-id "${TARGET_DISTRIBUTION_ID}" \
            --paths "${INVALIDATION_PATHS[@]}"

      - name: Publish SNS notification
        env:
          RUN_ID: ${{ needs.setup.outputs.run-id }}
          YEAR: ${{ needs.setup.outputs.assessment-year }}
          TRIAD: ${{ needs.setup.outputs.triad }}
          ENV_LABEL: ${{ inputs.environment }}
          BUCKET: ${{ needs.setup.outputs.bucket }}
        run: |
          read -ra PINS <<< "${{ inputs.pins }}"
          SUBJECT="HomeVal reports ready for run-id $RUN_ID"
          MESSAGE=$(cat <<EOF
          Your requested HomeVal reports have been generated.

          Assessment year: $YEAR
          Run-ID: $RUN_ID
          Triad: $TRIAD
          PIN list: ${PINS[@]}
          Environment: $ENV_LABEL

          You can browse or download the reports at the following S3 path:
          $BUCKET/$YEAR/
          EOF
          )
          aws sns publish \
            --topic-arn "${{ secrets.AWS_SNS_NOTIFICATION_TOPIC_ARN }}" \
            --subject "$SUBJECT" \
            --message "$MESSAGE"