Openalex schema merge (#282) #2250
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Unit Tests | |
| on: [push] | |
| jobs: | |
| build: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| max-parallel: 4 | |
| matrix: | |
| python-version: ["3.10"] | |
| os: [ubuntu-latest] | |
| steps: | |
| - name: Checkout ${{ matrix.python-version }} | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| - name: Set up GCP Service Account Credentials | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.TEST_GCP_SERVICE_KEY }} # Store your GCP service account key as a GitHub secret | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Minikube | |
| uses: medyagh/setup-minikube@latest | |
| with: | |
| kubernetes-version: v1.33.1 | |
| driver: docker | |
| addons: gcp-auth | |
| - name: Run test environment setup | |
| env: | |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_application_credentials.json | |
| run: | | |
| echo "${{ secrets.TEST_GCP_SERVICE_KEY }}" | base64 --decode > /tmp/google_application_credentials.json | |
| bash bin/test-env-up.sh --remote | |
| # Manually add the minikube host alias because it sometimes doens't work and google won't fix it | |
| # https://github.com/kubernetes/minikube/issues/8439 | |
| eval $(minikube docker-env --unset) | |
| sudo chown $USER /etc/hosts | |
| echo $(minikube ssh grep host.minikube.internal /etc/hosts | cut -f1) host.minikube.internal >> /etc/hosts | |
| eval $(minikube docker-env) | |
| - name: Debug Minikube status and context | |
| run: | | |
| echo "\nMinikube Status:" | |
| minikube status | |
| echo "\nKubectl config:" | |
| kubectl config view --minify | |
| echo "\nKubectl cluster info:" | |
| kubectl cluster-info | |
| echo "\nKubectl context:" | |
| kubectl config current-context | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install pigz -y | |
| python -m pip install --upgrade pip | |
| cd .. | |
| # Add -b feature/my-branch to run tests for a feature branch | |
| git clone https://github.com/The-Academic-Observatory/observatory-platform.git | |
| pip install -e ./observatory-platform[tests] --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.11.2/constraints-${{ matrix.python-version }}.txt | |
| cd academic-observatory-workflows | |
| pip install -e ./academic-observatory-workflows[tests] --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.11.2/constraints-${{ matrix.python-version }}.txt | |
| cd .. | |
| curl -LO https://github.com/peak/s5cmd/releases/download/v2.1.0/s5cmd_2.1.0_linux_amd64.deb | |
| sudo dpkg -i s5cmd_2.1.0_linux_amd64.deb | |
| pip freeze # for debugging | |
| airflow db init | |
| - name: Check licenses | |
| run: | | |
| # stop the build if there are licensing issues | |
| python bin/extract_deps.py | |
| liccheck --sfile strategy.ini --rfile .liccheck_requirements.txt --level CAUTIOUS --reporting liccheck-output.txt --no-deps | |
| - name: Lint with flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Run unit tests with coverage | |
| env: | |
| TEST_GCP_DATA_LOCATION: ${{ secrets.TEST_GCP_DATA_LOCATION }} | |
| TEST_GCP_BUCKET_NAME: ${{ secrets.TEST_GCP_BUCKET_NAME }} | |
| TEST_GCP_PROJECT_ID: ${{ secrets.TEST_GCP_PROJECT_ID }} | |
| TEST_GCP_SERVICE_KEY: ${{ secrets.TEST_GCP_SERVICE_KEY }} | |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_application_credentials.json | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} | |
| AIRFLOW__CORE__LOAD_EXAMPLES: false | |
| run: | | |
| echo "${{ secrets.TEST_GCP_SERVICE_KEY }}" | base64 --decode > /tmp/google_application_credentials.json | |
| coverage run -m unittest discover ./academic-observatory-workflows -v | |
| coverage xml | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v1 | |
| with: | |
| file: ./coverage.xml | |
| fail_ci_if_error: false |