build.yml

# Copyright 2022 Goldman Sachs
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Build CI

env:
  CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }}
  CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master

# Cancel running jobs from previous pipelines of the same workflow on PR to save resource when commits are pushed quickly
# NOTE: we don't want this behavior on default branch
# See https://stackoverflow.com/a/68422069
concurrency:
  group: ${{ github.ref == 'refs/heads/master' && format('ci-default-branch-{0}-{1}', github.sha, github.workflow) || format('ci-pr-{0}-{1}', github.ref, github.workflow) }}
  cancel-in-progress: true

jobs:
  build:
    name: Build
    # NOTE: Only run this when not in fork as those likely do not have access to larger runners and will fail on normal runner due to resource limitation
    # Also, skip this build for release commits
    if: "!contains(github.event.head_commit.message, '[maven-release-plugin]') && github.repository == 'finos/legend-engine'"
    runs-on: ubuntu-latest
    outputs:
      testMatrix: ${{ steps.testMatrix.outputs.testMatrix }}
      profiles: ${{ steps.masterBuild.outputs.PROFILES }}

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        env:
          cache-name: cache-mvn-deps
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: 11
          distribution: 'zulu'
          server-id: ossrh
          server-username: CI_DEPLOY_USERNAME
          server-password: CI_DEPLOY_PASSWORD

      - name: Check Java version
        run: java -version

      - name: Configure git
        run: |
          git config --global committer.email "infra@finos.org"
          git config --global committer.name "FINOS Admin"
          git config --global author.email "${GITHUB_ACTOR}@users.noreply.github.com"
          git config --global author.name "${GITHUB_ACTOR}"

      - name: Download deps and plugins
        run: mvn de.qaware.maven:go-offline-maven-plugin:resolve-dependencies

      - name: Collect Workflow Telemetry
        uses: runforesight/workflow-telemetry-action@v1
        with:
          theme: dark

      - name: Build (PR)
        if: github.ref != 'refs/heads/master'
        env:
          MAVEN_OPTS: "-Xmx8g"
        run: |
          mvn -o -B -e -T2 -DskipTests=true install

      - name: Build (with Maven Deploy + Docker Snapshot)
        if: github.ref == 'refs/heads/master'
        env:
          DOCKER_USERNAME: finos
          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
          MAVEN_OPTS: "-Xmx8g"
        # NOTE: here we publish snapshot to a staging Maven registry
        # some registry like Maven Central requires javadoc, but for now we
        # don't need to, if we do, call javadoc:jar goal instead of javadoc:javadoc
        # as the latter binds to generate-sources which runs before compile phase
        # and can cause problem with some code generators
        # See https://github.com/finos/legend-engine/pull/924
        run: mvn -B -e -T2 -DskipTests=true deploy -P docker-snapshot,pct-cloud-test

      - name: Set profiles for master branch
        id: masterBuild
        if: github.ref == 'refs/heads/master'
        run: echo PROFILES='-P pct-cloud-test' >> $GITHUB_OUTPUT

      - name: Store build output artifacts
        uses: actions/upload-artifact@v4
        with:
          name: build-output
          # upload the target directory as it is the output of the build
          # upload also the m2 repo to store the state of the install step
          path: |
            ~/**/target/
            ~/.m2/repository/org/finos/legend/engine/
          retention-days: 1
          overwrite: true
          include-hidden-files: true
          if-no-files-found: error

      - name: Set Test Matrix
        id: testMatrix
        run: echo "testMatrix=$(jq -c . < .github/workflows/resources/modulesToTest.json)" >> $GITHUB_OUTPUT

      - name: Upload CI Event
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: event-file
          path: ${{ github.event_path }}

  test:
    name: Run Unit Tests
    runs-on: ubuntu-latest
    needs:
      - build

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Before Test
        uses: ./.github/workflows/actions/before-test

      - name: Excluded Modules
        id: modulesToExclude
        run: echo EXCLUSION_MODULES=`jq -c '.include | map(.modules) | flatten | map("!org.finos.legend.engine:" + .) | join(",")' ./.github/workflows/resources/modulesToTest.json` >> $GITHUB_OUTPUT

      - name: Test
        env:
          MAVEN_OPTS: "-Xmx1g"
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          mvn -o -B -e surefire:test -pl ${{ steps.modulesToExclude.outputs.EXCLUSION_MODULES }} ${{ needs.build.outputs.profiles }} -T2 -DargLine="-Xmx6g" -Dsurefire.reports.directory=${GITHUB_WORKSPACE}/surefire-reports-aggregate

      - name: Upload Test Results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-results
          path: ${{ github.workspace }}/surefire-reports-aggregate/*.xml

  test-modules:
    name: Test Module - ${{ matrix.name }}
    runs-on: ubuntu-latest
    needs:
      - build

    strategy:
      matrix: ${{fromJson(needs.build.outputs.testMatrix)}}

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Before Test
        uses: ./.github/workflows/actions/before-test

      - name: Included Modules
        id: modulesToInclude
        env:
          MODULES: ${{ toJSON(matrix.modules) }}
        run: |
          echo INCLUSION_MODULES=`jq -c 'map("org.finos.legend.engine:" + .) | join(",")' <<< $MODULES` >> $GITHUB_OUTPUT

      - name: Test
        env:
          MAVEN_OPTS: "-Xmx1g"
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}          
        run: |
          mvn -o -B -e surefire:test -pl ${{ steps.modulesToInclude.outputs.INCLUSION_MODULES }} ${{ needs.build.outputs.profiles }} -T2 -DargLine="-Xmx6g" -Dsurefire.reports.directory=${GITHUB_WORKSPACE}/surefire-reports-aggregate

      - name: Upload Test Results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-results-${{ matrix.name }}
          path: ${{ github.workspace }}/surefire-reports-aggregate/*.xml

  pct-report:
    name: PCT Report
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/master'
    needs:
      - build
      - test
      - test-modules
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Setup
        uses: ./.github/workflows/actions/before-test

      - name: Generate PCT Report
        run: |
          mvn -o exec:java -pl org.finos.legend.engine:legend-engine-server-http-server -Dexec.mainClass="org.finos.legend.engine.server.core.pct.PCT_to_SimpleHTML"

      - name: Upload PCT HTML Report
        uses: actions/upload-artifact@v4
        with:
          name: pct-html-report
          path: target/ok.html

      - name: Checkout Legend Docs
        uses: actions/checkout@v4
        with:
          path: 'legend-docs'
          ref: 'master'
          repository: 'finos/legend'
          token: ${{ secrets.FINOS_GITHUB_TOKEN }}
          fetch-depth: 0
          persist-credentials: true

      - name: Copy Legend Docs
        shell: bash
        run: |
          mkdir -p legend-docs/website/static/pct/
          cp -rvT target/ok.html legend-docs/website/static/pct/PCT_Report_Compatibility.html

      - name: Check for new documentation files
        id: newFiles
        shell: bash
        working-directory: legend-docs
        run: |
          git add -A
          git status --porcelain | wc -l
          if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
            echo  "hasNewFiles=true" >> $GITHUB_OUTPUT
          else
            echo  "hasNewFiles=false" >> $GITHUB_OUTPUT
          fi

      - name: Commit new documentation files
        if: ${{ steps.newFiles.outputs.hasNewFiles == 'true' }}
        shell: bash
        working-directory: legend-docs
        run: |
          git commit -m "maint: update legend-engine docs from ref: ${{ github.sha }}"
          git push


#  cleanup:
#    name: Cleanup
#    runs-on: ubuntu-latest
#    if: always()
#    needs:
#      - build
#      - test
#      - test-modules
#      - pct-report
#
#    steps:
#      - uses: geekyeggo/delete-artifact@v5
#        with:
#          name: build-output