Bump pypdf from 3.9.1 to 6.6.0 in /utils #309
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # # Copyright 2022 Google LLC | |
| # # | |
| # # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # # you may not use this file except in compliance with the License. | |
| # # You may obtain a copy of the License at | |
| # # | |
| # # http://www.apache.org/licenses/LICENSE-2.0 | |
| # # | |
| # # Unless required by applicable law or agreed to in writing, software | |
| # # distributed under the License is distributed on an "AS IS" BASIS, | |
| # # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # # See the License for the specific language governing permissions and | |
| # # limitations under the License. | |
| # # This workflow will install Python dependencies, run tests and lint with a variety of Python versions | |
| # # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: E2E API Tests on GKE Microservices | |
| on: | |
| workflow_call: | |
| inputs: | |
| USE_GMAIL_ACCOUNT_STUDENT_ENROLLMENT: | |
| required: false | |
| type: boolean | |
| default: false | |
| secrets: | |
| GKE_POD_SA_KEY: | |
| required: true | |
| FIREBASE_API_KEY: | |
| required: true | |
| API_DOMAIN: | |
| required: true | |
| FIREBASE_APP_ID: | |
| required: true | |
| DEPLOYMENT_DEV_SA_KEY: | |
| required: true | |
| GH_PAT: | |
| required: true | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - 'microservices/**' | |
| - 'common/**' | |
| - 'e2e/**' | |
| - 'utils/**' | |
| - '.github/workflows/e2e_gke_api_test.yaml' | |
| - 'skaffold.yaml' | |
| workflow_dispatch: | |
| inputs: | |
| USE_GMAIL_ACCOUNT_STUDENT_ENROLLMENT: | |
| description: "set flag to use gmail account" | |
| required: false | |
| type: boolean | |
| default: false | |
| env: | |
| PROJECT_ID: gcp-classroom-dev | |
| SKAFFOLD_DEFAULT_REPO: gcr.io/gcp-classroom-dev | |
| GKE_CLUSTER: gcp-classroom-dev-us-central1 | |
| GKE_ZONE: us-central1 | |
| KUSTOMIZE_VERSION: 4.1.3 | |
| SKAFFOLD_VERSION: 1.39.2 | |
| SKAFFOLD_CACHE_GCS: gs://gcp-classroom-dev/skaffold/cache | |
| WORKSPACE_ADMIN_EMAIL: [email protected] | |
| GKE_POD_SA_KEY: ${{ secrets.GKE_CLASSROOM_POD_SA_KEY }} | |
| FIREBASE_API_KEY: ${{ secrets.ASU_CLASSROOM_DEV_FIREBASE_API_KEY }} | |
| USE_GKE_GCLOUD_AUTH_PLUGIN: True | |
| MEMORYSTORE_ENABLED: true | |
| CLOUD_LOGGING_ENABLED: False | |
| API_DOMAIN: ${{ secrets.ASU_CLASSROOM_DEV_API_DOMAIN }} | |
| FIREBASE_APP_ID: ${{ secrets.ASU_CLASSROOM_DEV_FIREBASE_APP_ID }} | |
| EXTERNAL_USER_PROPERTY_PREFIX: SNHU_ | |
| SKAFFOLD_BUILD_CONCURRENCY: 0 | |
| USE_GMAIL_ACCOUNT_STUDENT_ENROLLMENT: ${{ inputs.USE_GMAIL_ACCOUNT_STUDENT_ENROLLMENT }} | |
| E2E_PREFIX: ${{ inputs.USE_GMAIL_ACCOUNT_STUDENT_ENROLLMENT == true && 'e2e_approval_pr' || 'e2e_pr' }}${{ github.event.pull_request.number }}_ | |
| E2E_NAMESPACE_PREFIX: ${{ inputs.USE_GMAIL_ACCOUNT_STUDENT_ENROLLMENT == true && 'e2e-pr-approval-' || 'e2e-pr-' }}${{ github.event.pull_request.number }} | |
| CONTENT_SERVING_BUCKET: gcp-classroom-dev-content-serving-bucket | |
| # cookiecutter:raw | |
| jobs: | |
| admin-teacher: | |
| name: Set admin-teacher account as env | |
| runs-on: ubuntu-latest | |
| outputs: | |
| CLASSROOM_ADMIN_EMAIL: ${{steps.account-setup.outputs.account}} | |
| steps: | |
| - name: 'Set up Gcloud auth' | |
| id: 'gcloud_auth' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: '${{ secrets.CLP_DEV_SA_KEY }}' | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v1' | |
| - name: export admin-teacher account in github env | |
| id: account-setup | |
| run: | | |
| secret_id=e2e-admin-teacher-$(shuf -n1 -i 1-4)-username | |
| account=$(gcloud secrets versions access latest --secret="$secret_id") | |
| echo "account=$account" >> "$GITHUB_OUTPUT" | |
| deploy: | |
| name: Deploy all microservices in PR namespace | |
| runs-on: ubuntu-latest | |
| needs: admin-teacher | |
| env: | |
| CLASSROOM_ADMIN_EMAIL: ${{ needs.admin-teacher.outputs.CLASSROOM_ADMIN_EMAIL }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v3 | |
| - name: 'Set up Gcloud auth' | |
| id: 'gcloud_auth' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: '${{ secrets.CLP_DEV_SA_KEY }}' | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v1' | |
| - name: Install GKE Auth Plugin | |
| run: | | |
| gcloud components install gke-gcloud-auth-plugin | |
| - name: Connect to cluster | |
| run: | | |
| gcloud container clusters get-credentials "$GKE_CLUSTER" --zone "$GKE_ZONE" | |
| - name: Install Kustomize | |
| run: | | |
| wget -O kustomize.tar.gz https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v$KUSTOMIZE_VERSION/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz | |
| sudo tar -xvf kustomize.tar.gz -C /usr/local/bin/ | |
| - name: Install Skaffold release binary | |
| run: | | |
| curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v$SKAFFOLD_VERSION/skaffold-linux-amd64 | |
| sudo install skaffold /usr/local/bin/skaffold | |
| - name: Set default repo for Skaffold | |
| run: | | |
| skaffold config set default-repo "$SKAFFOLD_DEFAULT_REPO" | |
| - name: Download Skaffold cache if exists | |
| run: | | |
| gsutil cp $SKAFFOLD_CACHE_GCS ~/.skaffold/ || true | |
| - name: Set up Kubernetes namespace and Service Account | |
| run: | | |
| NAMESPACE=$E2E_NAMESPACE_PREFIX | |
| kubectl create namespace $NAMESPACE --dry-run=client -o yaml | kubectl apply -f - | |
| NAMESPACE=${NAMESPACE} bash setup/setup_ksa.sh | |
| - name: Add indexing to collections in firestore | |
| run: | | |
| export DATABASE_PREFIX=$E2E_PREFIX | |
| cd utils/ | |
| python -m pip install -r requirements.txt | |
| PYTHONPATH=../common/src python firestore_e2e_indexing_setup.py | |
| - name: Initialize Pub/Sub | |
| run: | | |
| cd e2e | |
| python -m pip install -r requirements.txt | |
| cd utils | |
| PYTHONPATH=../common/src DATABASE_PREFIX=$E2E_PREFIX python pub_sub_setup.py | |
| - name: Initialize BQ | |
| run: | | |
| cd utils/ | |
| python -m pip install -r requirements.txt | |
| BQ_DATASET=lms_analytics DATABASE_PREFIX=$E2E_PREFIX python bq_setup.py --create-dataset=true | |
| BQ_DATASET=lrs DATABASE_PREFIX=$E2E_PREFIX python bq_setup.py --create-dataset=true | |
| - name: Local setup | |
| run: | | |
| NAMESPACE=$E2E_NAMESPACE_PREFIX | |
| SKAFFOLD_NAMESPACE=${NAMESPACE} REGION=us-central1 bash setup/setup_local.sh | |
| - name: Build and deploy to cluster | |
| run: | | |
| NAMESPACE=$E2E_NAMESPACE_PREFIX | |
| export DATABASE_PREFIX=$E2E_PREFIX | |
| skaffold run \ | |
| --profile dev \ | |
| --namespace $NAMESPACE \ | |
| --module backends,local-redis \ | |
| -l commit=$GITHUB_SHA \ | |
| --tag PR${{ github.event.pull_request.number }} | |
| - name: Upload Skaffold cache | |
| if: always() | |
| run: | | |
| gsutil cp ~/.skaffold/cache $SKAFFOLD_CACHE_GCS | |
| e2e-api-test: | |
| runs-on: ubuntu-latest | |
| needs: [deploy,admin-teacher] | |
| env: | |
| CLASSROOM_ADMIN_EMAIL: ${{ needs.admin-teacher.outputs.CLASSROOM_ADMIN_EMAIL }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [3.9] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: 'Set up Gcloud auth' | |
| id: 'gcloud_auth' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: '${{ secrets.CLP_DEV_SA_KEY }}' | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v1' | |
| - name: Install GKE Auth Plugin | |
| run: | | |
| gcloud components install gke-gcloud-auth-plugin | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install pytest pytest-custom_exit_code pytest-cov pylint pytest-mock mock | |
| python -m pip install -r e2e/requirements.txt | |
| - name: Connect to cluster | |
| run: | | |
| gcloud container clusters get-credentials "$GKE_CLUSTER" --zone "$GKE_ZONE" | |
| - name: Set up port forwarding | |
| run: | | |
| NAMESPACE=$E2E_NAMESPACE_PREFIX | |
| python e2e/utils/port_forward.py --namespace $NAMESPACE | |
| - name: Setup redis | |
| uses: supercharge/[email protected] | |
| with: | |
| redis-version: 6 | |
| - name: Install redis cli | |
| run: sudo apt-get install -y redis-tools | |
| - name: Verify that redis is up | |
| run: redis-cli ping | |
| - name: Cleaning up database | |
| run: | | |
| cd e2e/utils | |
| PYTHONPATH=../common/src DATABASE_PREFIX=$E2E_PREFIX python database_cleanup.py | |
| - name: Cleaning up classroom | |
| run: | | |
| cd e2e/utils | |
| PYTHONPATH=../common/src DATABASE_PREFIX=$E2E_PREFIX python classroom_cleanup.py | |
| - name: Wait for 5s for database cleaning up | |
| run: sleep 10s | |
| shell: bash | |
| - name: Initialize database | |
| run: | | |
| cd e2e/utils | |
| PYTHONPATH=../common/src DATABASE_PREFIX=$E2E_PREFIX python database_setup.py | |
| - name: Run pytest | |
| run: | | |
| PYTHONPATH=common/src DATABASE_PREFIX=$E2E_PREFIX python -m pytest e2e/gke_api_tests/ | |
| - name: Run behave | |
| run: | | |
| PYTHONPATH=common/src DATABASE_PREFIX=$E2E_PREFIX python -m behave e2e/features/ --tags=~@test --tags=~@auth-api --tags=~@filter-api --tags=~@single-sign-on-auth-flow --tags=~@matching-engine --tags=~@publish-draft-mode --tags=~@inspace --tags=~@not-implemented --tags=~@assessment --tags=~@content-serving-api | |
| e2e-ui-test-lms-admin-ui: | |
| runs-on: ubuntu-latest | |
| needs: deploy | |
| if: ${{contains(github.event.pull_request.labels.*.name, 'LMS_ADMIN_UI') || inputs.USE_GMAIL_ACCOUNT_STUDENT_ENROLLMENT == true}} | |
| env: | |
| HEADLESS: enables | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node-version: [16.x] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: 'Set up Gcloud auth' | |
| id: 'gcloud_auth' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: '${{ secrets.CLP_DEV_SA_KE }}' | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v1' | |
| - name: Install GKE Auth Plugin | |
| run: | | |
| gcloud components install gke-gcloud-auth-plugin | |
| - id: 'secrets' | |
| uses: 'google-github-actions/get-secretmanager-secrets@v1' | |
| with: | |
| secrets: |- | |
| TOKEN:gcp-classroom-dev/e2e_test_email_password | |
| - name: Connect to cluster | |
| run: | | |
| gcloud container clusters get-credentials "$GKE_CLUSTER" --zone "$GKE_ZONE" | |
| - name: Set up port forwarding | |
| run: | | |
| NAMESPACE=$E2E_NAMESPACE_PREFIX | |
| python -m pip install -r e2e/requirements.txt | |
| python e2e/utils/port_forward.py --namespace $NAMESPACE | |
| - name: Install wait-for-it | |
| run: sudo apt install -y wait-for-it | |
| - name: Install dependencies | |
| run: | | |
| cd ./microservices/lms_admin_ui | |
| npm install --save-exact | |
| - name: Run frontend server on localhost | |
| run: | | |
| cd ./microservices/lms_admin_ui | |
| npm start & wait-for-it -h localhost -p 4200 -t 0 -- echo "Frontend Dev Server up" | |
| - name: Waiting 30 sec | |
| run: | | |
| sleep 30s | |
| - name: Create new user for e2e | |
| env: | |
| ACCOUNT : '${{ steps.secrets.outputs.TOKEN }}' | |
| run: | | |
| echo $ACCOUNT | |
| PR_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") | |
| export DATABASE_PREFIX=e2e_pr${PR_NUMBER}_ | |
| python -m pip install -r common/requirements.txt | |
| cd utils/ | |
| python -m pip install -r requirements.txt | |
| PYTHONPATH=../common/src python create_e2e_user.py | |
| - name: Run E2E UI Tests | |
| env: | |
| ACCOUNT : '${{ steps.secrets.outputs.TOKEN }}' | |
| run: | | |
| echo $ACCOUNT | |
| cd ./microservices/lms_admin_ui | |
| npm run e2e-ui | |
| dataset-cleanup: | |
| runs-on: ubuntu-latest | |
| needs: [e2e-api-test,admin-teacher] | |
| if: always() | |
| env: | |
| CLASSROOM_ADMIN_EMAIL: ${{ needs.admin-teacher.outputs.CLASSROOM_ADMIN_EMAIL }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [3.9] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: 'Set up Gcloud auth' | |
| id: 'gcloud_auth' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: '${{ secrets.CLP_DEV_SA_KEY }}' | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v1' | |
| - name: Install GKE Auth Plugin | |
| run: | | |
| gcloud components install gke-gcloud-auth-plugin | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install pytest pytest-custom_exit_code pytest-cov pylint pytest-mock mock | |
| python -m pip install -r e2e/requirements.txt | |
| - name: Removing firestore indexes | |
| run: | | |
| export DATABASE_PREFIX=$E2E_PREFIX | |
| cd utils/ | |
| python -m pip install -r requirements.txt | |
| PYTHONPATH=../common/src python firestore_indexing_cleanup.py | |
| - name: Cleaning up database | |
| run: | | |
| cd e2e/utils | |
| PYTHONPATH=../common/src DATABASE_PREFIX=$E2E_PREFIX python database_cleanup.py | |
| - name: Cleaning up classroom | |
| run: | | |
| cd e2e/utils | |
| PYTHONPATH=../common/src DATABASE_PREFIX=$E2E_PREFIX python classroom_cleanup.py | |
| deployment-cleanup: | |
| name: Clean up deployment in PR namespace | |
| needs: [e2e-api-test,admin-teacher] | |
| if: always() | |
| env: | |
| CLASSROOM_ADMIN_EMAIL: ${{ needs.admin-teacher.outputs.CLASSROOM_ADMIN_EMAIL }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v3 | |
| - name: 'Set up Gcloud auth' | |
| id: 'gcloud_auth' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: '${{ secrets.CLP_DEV_SA_KEY }}' | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v1' | |
| - name: Install GKE Auth Plugin | |
| run: | | |
| gcloud components install gke-gcloud-auth-plugin | |
| - name: Connect to cluster | |
| run: | | |
| gcloud container clusters get-credentials "$GKE_CLUSTER" --zone "$GKE_ZONE" | |
| - name: Install Kustomize | |
| run: | | |
| wget -O kustomize.tar.gz https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v4.0.4/kustomize_v4.0.4_linux_amd64.tar.gz | |
| sudo tar -xvf kustomize.tar.gz -C /usr/local/bin/ | |
| - name: Install Skaffold release binary | |
| run: | | |
| curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v$SKAFFOLD_VERSION/skaffold-linux-amd64 | |
| sudo install skaffold /usr/local/bin/skaffold | |
| - name: Set default repo for Skaffold | |
| run: | | |
| skaffold config set default-repo "$SKAFFOLD_DEFAULT_REPO" | |
| - name: Delete all deployments and namespace | |
| run: | | |
| NAMESPACE=$E2E_NAMESPACE_PREFIX | |
| skaffold delete --namespace $NAMESPACE | |
| kubectl delete namespace $NAMESPACE | |
| - name: Cleaning up Pub/Sub | |
| run: | | |
| cd e2e | |
| python -m pip install -r requirements.txt | |
| cd utils | |
| PYTHONPATH=../common/src DATABASE_PREFIX=$E2E_PREFIX python pub_sub_cleanup.py | |
| - name: Cleaning up BQ | |
| run: | | |
| cd utils/ | |
| python -m pip install -r requirements.txt | |
| BQ_DATASET=lms_analytics DATABASE_PREFIX=$E2E_PREFIX python bq_cleanup.py | |
| BQ_DATASET=lrs DATABASE_PREFIX=$E2E_PREFIX python bq_cleanup.py | |
| # cookiecutter:endraw |