Merge pull request #86 from Sage-Bionetworks/etl-567

[ETL-567] Use ECS logging in S3 to JSON job

Author: philerooski

.github/workflows/upload-and-deploy.yaml

@@ -78,6 +78,10 @@ jobs:
           role_session_name: ${{ github.event.repository.name }}-${{ github.run_id }}-nonglue-unit-tests
           python_version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install additional python dependency
+        run: |
+          pipenv install ecs_logging~=2.0
+
       - name: Test lambda scripts with pytest
         run: |
           pipenv run python -m pytest tests/test_s3_event_config_lambda.py -v

Pipfile

@@ -17,3 +17,4 @@ pandas = "<1.5"
 moto = "~=4.1"
 datacompy = "~=0.8"
 docker = "~=6.1"
+ecs_logging = "~=2.0"

config/develop/namespaced/glue-job-S3ToJsonS3.yaml

@@ -11,5 +11,6 @@ parameters:
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/s3_to_json.py'
   GluePythonShellVersion: "{{ stack_group_config.glue_python_shell_python_version }}"
   GlueVersion: "{{ stack_group_config.glue_python_shell_glue_version }}"
+  AdditionalPythonModules: ecs_logging~=2.0
 stack_tags:
   {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/glue-job-S3ToJsonS3.yaml

@@ -11,5 +11,6 @@ parameters:
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/s3_to_json.py'
   GluePythonShellVersion: "{{ stack_group_config.glue_python_shell_python_version }}"
   GlueVersion: "{{ stack_group_config.glue_python_shell_glue_version }}"
+  AdditionalPythonModules: ecs_logging~=2.0
 stack_tags:
   {{ stack_group_config.default_stack_tags }}