Data Source Cleanup #8713
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Script(s) to clean up any loose data left behind from test runs. | |
| # These can primarily happen if CI is stopped while tests are running | |
| # for big query, etc. | |
| name: Data Source Cleanup | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 * * * *" # Run at the start of each hour | |
| jobs: | |
| cleanup-big-query: | |
| runs-on: ubuntu-latest | |
| env: | |
| GE_TEST_GCP_CREDENTIALS: ${{secrets.GE_TEST_GCP_CREDENTIALS}} | |
| GE_TEST_GCP_PROJECT: ${{secrets.GE_TEST_GCP_PROJECT}} | |
| GE_TEST_BIGQUERY_DATASET: ${{secrets.GE_TEST_BIGQUERY_DATASET}} | |
| GOOGLE_APPLICATION_CREDENTIALS: "gcp-credentials.json" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4.3.1 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| reqs/requirements-dev-test.txt | |
| setup.py | |
| - name: Create JSON file GCP | |
| run: | | |
| echo "$GE_TEST_GCP_CREDENTIALS" > gcp-credentials.json | |
| - name: Install dependencies | |
| run: | | |
| pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt) | |
| invoke deps --gx-install -m bigquery | |
| pip install -e . | |
| - name: Run BigQuery cleanup script | |
| run: | | |
| python ./scripts/cleanup/cleanup_big_query.py | |
| cleanup-redshift: | |
| runs-on: ubuntu-latest | |
| env: | |
| # aws-redshift | |
| REDSHIFT_USERNAME: ${{secrets.REDSHIFT_USERNAME}} | |
| REDSHIFT_PASSWORD: ${{secrets.REDSHIFT_PASSWORD}} | |
| REDSHIFT_HOST: ${{secrets.REDSHIFT_HOST}} | |
| REDSHIFT_PORT: ${{secrets.REDSHIFT_PORT}} | |
| REDSHIFT_DATABASE: ${{secrets.REDSHIFT_DATABASE}} | |
| REDSHIFT_SSLMODE: ${{secrets.REDSHIFT_SSLMODE}} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4.3.1 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| reqs/requirements-dev-test.txt | |
| setup.py | |
| - name: Install dependencies | |
| run: | | |
| pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt) | |
| invoke deps --gx-install -m gx-redshift | |
| pip install -e . | |
| - name: Run Redshift cleanup script | |
| run: | | |
| python ./scripts/cleanup/cleanup_redshift.py | |
| cleanup-databricks: | |
| runs-on: ubuntu-latest | |
| env: | |
| # databricks | |
| # DATABRICKS_TOKEN is minted at runtime from the service principal's OAuth | |
| # credentials in the "Mint Databricks OAuth access token" step below. | |
| DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}} | |
| DATABRICKS_HTTP_PATH: ${{secrets.DATABRICKS_HTTP_PATH}} | |
| DATABRICKS_CATALOG: ${{ vars.DATABRICKS_CATALOG }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4.3.1 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| reqs/requirements-dev-test.txt | |
| setup.py | |
| - name: Install dependencies | |
| run: | | |
| pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt) | |
| invoke deps --gx-install -m databricks | |
| pip install -e . | |
| # Authenticate the service principal via OAuth M2M and export the | |
| # masked bearer token as DATABRICKS_TOKEN for the cleanup script. | |
| - name: Mint Databricks OAuth access token | |
| env: | |
| DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }} | |
| DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }} | |
| run: | | |
| TOKEN="$(python ./scripts/mint_databricks_token.py)" | |
| echo "::add-mask::$TOKEN" | |
| echo "DATABRICKS_TOKEN=$TOKEN" >> "$GITHUB_ENV" | |
| - name: Run Databricks cleanup script | |
| run: | | |
| python ./scripts/cleanup/cleanup_databricks.py | |
| cleanup-snowflake: | |
| runs-on: ubuntu-latest | |
| env: | |
| # snowflake | |
| SNOWFLAKE_CI_ACCOUNT: ${{secrets.SNOWFLAKE_CI_ACCOUNT}} | |
| SNOWFLAKE_CI_PRIVATE_KEY: ${{secrets.SNOWFLAKE_CI_PRIVATE_KEY}} | |
| SNOWFLAKE_CI_USER: ${{secrets.SNOWFLAKE_CI_USER}} | |
| SNOWFLAKE_CI_DATABASE: ${{secrets.SNOWFLAKE_CI_DATABASE}} | |
| SNOWFLAKE_CI_WAREHOUSE: ${{secrets.SNOWFLAKE_CI_WAREHOUSE}} | |
| SNOWFLAKE_CI_ROLE: ${{secrets.SNOWFLAKE_CI_ROLE}} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4.3.1 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| reqs/requirements-dev-test.txt | |
| setup.py | |
| - name: Install dependencies | |
| run: | | |
| pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt) | |
| invoke deps --gx-install -m snowflake | |
| pip install -e . | |
| - name: Run Snowflake cleanup script | |
| run: | | |
| python ./scripts/cleanup/cleanup_snowflake.py |