From 34665e9e646d819cae0d684378ae1ed7681ff518 Mon Sep 17 00:00:00 2001 From: Phil Snyder Date: Wed, 11 Dec 2024 14:33:24 -0800 Subject: [PATCH 1/4] run JSON to Parquet workflow every sunday 2am --- templates/glue-workflow.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/glue-workflow.j2 b/templates/glue-workflow.j2 index 5483368..37a7466 100644 --- a/templates/glue-workflow.j2 +++ b/templates/glue-workflow.j2 @@ -63,7 +63,7 @@ Parameters: When `IsMainNamespace`, the respective trigger is active from the moment of deployment. Otherwise, the trigger is disabled so that we don't waste resources running our development pipelines every day. - Default: cron(0 2 * * ? *) + Default: cron(0 2 * * 0 *) CompareParquetStagingNamespace: Type: String From 69b612f52d2ee873f3bbe70ef3583daa02e21183 Mon Sep 17 00:00:00 2001 From: Phil Snyder Date: Wed, 11 Dec 2024 16:13:43 -0800 Subject: [PATCH 2/4] use latest configure-aws-credentials action --- .github/workflows/upload-and-deploy.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index ea8b877..dd41abc 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -183,7 +183,7 @@ jobs: environment: develop steps: - name: Assume AWS role - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }} aws-region: "us-east-1" @@ -240,10 +240,11 @@ jobs: password: ${{ needs.pytest-docker.outputs.ecr-password }} env: DISABLE_SSL: true + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true options: "--user root" steps: - name: Assume AWS role - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }} aws-region: "us-east-1" From 7f3426f3c1cc95a1f61b63c74b0bdb3e3e6a17dc Mon Sep 17 00:00:00 2001 From: Phil Snyder Date: Wed, 11 Dec 2024 16:37:11 -0800 Subject: [PATCH 3/4] use legacy configure-aws-credentials action for glue docker --- .github/workflows/upload-and-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index dd41abc..28d8bc9 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -244,7 +244,7 @@ jobs: options: "--user root" steps: - name: Assume AWS role - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }} aws-region: "us-east-1" From 94e69f6f34f9e77d2ed76c1b843a684669a434b5 Mon Sep 17 00:00:00 2001 From: Phil Snyder Date: Fri, 13 Dec 2024 14:01:45 -0800 Subject: [PATCH 4/4] fix json to parquet trigger cron --- .github/workflows/upload-and-deploy.yaml | 1 - templates/glue-workflow.j2 | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index 28d8bc9..77857b2 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -240,7 +240,6 @@ jobs: password: ${{ needs.pytest-docker.outputs.ecr-password }} env: DISABLE_SSL: true - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true options: "--user root" steps: - name: Assume AWS role diff --git a/templates/glue-workflow.j2 b/templates/glue-workflow.j2 index 37a7466..08cca52 100644 --- a/templates/glue-workflow.j2 +++ b/templates/glue-workflow.j2 @@ -63,7 +63,7 @@ Parameters: When `IsMainNamespace`, the respective trigger is active from the moment of deployment. Otherwise, the trigger is disabled so that we don't waste resources running our development pipelines every day. - Default: cron(0 2 * * 0 *) + Default: cron(0 2 ? * 1 *) CompareParquetStagingNamespace: Type: String