data_source_cleanup.yml

# Script(s) to clean up any loose data left behind from test runs.
# These can primarily happen if CI is stopped while tests are running
# for big query, etc.

name: Data Source Cleanup

on:
  workflow_dispatch:
  schedule:
    - cron: "0 * * * *" # Run at the start of each hour

jobs:
  cleanup-big-query:
    runs-on: ubuntu-latest
    env:
      # google
      GE_TEST_GCP_CREDENTIALS: ${{secrets.GE_TEST_GCP_CREDENTIALS}}
      GE_TEST_GCP_PROJECT: ${{secrets.GE_TEST_GCP_PROJECT}}
      GE_TEST_BIGQUERY_DATASET: ${{secrets.GE_TEST_BIGQUERY_DATASET}}
      GOOGLE_APPLICATION_CREDENTIALS: "gcp-credentials.json"
    steps:
      - name: Checkout
        uses: actions/checkout@v4.3.1
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            reqs/requirements-dev-test.txt
            setup.py
      - name: Create JSON file GCP
        run: |
          echo "$GE_TEST_GCP_CREDENTIALS" > gcp-credentials.json
      - name: Install dependencies
        run: |
          pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
          invoke deps --gx-install -m bigquery
          pip install -e .
      - name: Run BigQuery cleanup script
        run: |
          python ./scripts/cleanup/cleanup_big_query.py

  cleanup-redshift:
    runs-on: ubuntu-latest
    env:
      # aws-redshift
      REDSHIFT_USERNAME: ${{secrets.REDSHIFT_USERNAME}}
      REDSHIFT_PASSWORD: ${{secrets.REDSHIFT_PASSWORD}}
      REDSHIFT_HOST: ${{secrets.REDSHIFT_HOST}}
      REDSHIFT_PORT: ${{secrets.REDSHIFT_PORT}}
      REDSHIFT_DATABASE: ${{secrets.REDSHIFT_DATABASE}}
      REDSHIFT_SSLMODE: ${{secrets.REDSHIFT_SSLMODE}}
    steps:
      - name: Checkout
        uses: actions/checkout@v4.3.1
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            reqs/requirements-dev-test.txt
            setup.py

      - name: Install dependencies
        run: |
          pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
          invoke deps --gx-install -m gx-redshift
          pip install -e .
      - name: Run Redshift cleanup script
        run: |
          python ./scripts/cleanup/cleanup_redshift.py

  cleanup-databricks:
    runs-on: ubuntu-latest
    env:
      # databricks
      # DATABRICKS_TOKEN is minted at runtime from the service principal's OAuth
      # credentials in the "Mint Databricks OAuth access token" step below.
      DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
      DATABRICKS_HTTP_PATH: ${{secrets.DATABRICKS_HTTP_PATH}}
      DATABRICKS_CATALOG: ${{ vars.DATABRICKS_CATALOG }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4.3.1
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            reqs/requirements-dev-test.txt
            setup.py
      - name: Install dependencies
        run: |
          pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
          invoke deps --gx-install -m databricks
          pip install -e .
      # Authenticate the service principal via OAuth M2M and export the
      # masked bearer token as DATABRICKS_TOKEN for the cleanup script.
      - name: Mint Databricks OAuth access token
        env:
          DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}
        run: |
          TOKEN="$(python ./scripts/mint_databricks_token.py)"
          echo "::add-mask::$TOKEN"
          echo "DATABRICKS_TOKEN=$TOKEN" >> "$GITHUB_ENV"
      - name: Run Databricks cleanup script
        run: |
          python ./scripts/cleanup/cleanup_databricks.py

  cleanup-snowflake:
    runs-on: ubuntu-latest
    env:
      # snowflake
      SNOWFLAKE_CI_ACCOUNT: ${{secrets.SNOWFLAKE_CI_ACCOUNT}}
      SNOWFLAKE_CI_PRIVATE_KEY: ${{secrets.SNOWFLAKE_CI_PRIVATE_KEY}}
      SNOWFLAKE_CI_USER: ${{secrets.SNOWFLAKE_CI_USER}}
      SNOWFLAKE_CI_DATABASE: ${{secrets.SNOWFLAKE_CI_DATABASE}}
      SNOWFLAKE_CI_WAREHOUSE: ${{secrets.SNOWFLAKE_CI_WAREHOUSE}}
      SNOWFLAKE_CI_ROLE: ${{secrets.SNOWFLAKE_CI_ROLE}}
    steps:
      - name: Checkout
        uses: actions/checkout@v4.3.1
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            reqs/requirements-dev-test.txt
            setup.py
      - name: Install dependencies
        run: |
          pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
          invoke deps --gx-install -m snowflake
          pip install -e .
      - name: Run Snowflake cleanup script
        run: |
          python ./scripts/cleanup/cleanup_snowflake.py