feat: cleanup up artifacts from containers during ingestion

.github/workflows/build-images.yml

@@ -29,7 +29,6 @@ jobs:
           - frontend # pushed both the frontend and backend images
           - upload_failures
           - upload_success
-          - dataset_submissions
           - processing
           - wmg_processing
           - cellguide_pipeline

.github/workflows/rdev-tests.yml

@@ -234,6 +234,6 @@ jobs:
         if: always()
         with:
           name: logged-in-test-results
-          path: frontend/playwright-report/
+          path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/frontend/playwright-report
           retention-days: 30
           if-no-files-found: error

.happy/config.json

@@ -11,7 +11,6 @@
     "backend-wmg",
     "cellguide_pipeline",
     "processing",
-    "dataset_submissions",
     "upload_failures",
     "upload_success",
     "wmg_processing"
@@ -38,9 +37,6 @@
     "wmg_processing": {
       "profile": "wmg_processing"
     },
-    "dataset_submissions": {
-      "profile": "dataset_submissions"
-    },
     "upload_failures": {
       "profile": "upload_failures"
     },

.happy/terraform/modules/batch/main.tf

@@ -11,7 +11,7 @@ resource aws_batch_job_definition batch_job_def {
   container_properties = jsonencode({
   "jobRoleArn": "${var.batch_role_arn}",
   "image": "${var.image}",
-  "memory": var.batch_container_memory_limit,
+  "memory": 8000,
   "environment": [
     {
       "name": "ARTIFACT_BUCKET",
@@ -42,11 +42,55 @@ resource aws_batch_job_definition batch_job_def {
       "value": "${var.frontend_url}"
     }
   ],
-  "vcpus": 8,
-  "linuxParameters": {
-     "maxSwap": 800000,
-     "swappiness": 60
-  },
+  "vcpus": 1,
+  "logConfiguration": {
+    "logDriver": "awslogs",
+    "options": {
+      "awslogs-group": "${aws_cloudwatch_log_group.cloud_watch_logs_group.id}",
+      "awslogs-region": "${data.aws_region.current.name}"
+    }
+  }
+})
+}
+
+resource aws_batch_job_definition cxg_job_def {
+  type = "container"
+  name = "dp-${var.deployment_stage}-${var.custom_stack_name}-convert"
+  container_properties = jsonencode({
+  "jobRoleArn": "${var.batch_role_arn}",
+  "image": "${var.image}",
+  "memory": 16000,
+  "environment": [
+    {
+      "name": "ARTIFACT_BUCKET",
+      "value": "${var.artifact_bucket}"
+    },
+    {
+      "name": "CELLXGENE_BUCKET",
+      "value": "${var.cellxgene_bucket}"
+    },
+    {
+      "name": "DATASETS_BUCKET",
+      "value": "${var.datasets_bucket}"
+    },
+    {
+      "name": "DEPLOYMENT_STAGE",
+      "value": "${var.deployment_stage}"
+    },
+    {
+      "name": "AWS_DEFAULT_REGION",
+      "value": "${data.aws_region.current.name}"
+    },
+    {
+      "name": "REMOTE_DEV_PREFIX",
+      "value": "${var.remote_dev_prefix}"
+    },
+    {
+      "name": "FRONTEND_URL",
+      "value": "${var.frontend_url}"
+    }
+  ],
+  "vcpus": 2,
   "logConfiguration": {
     "logDriver": "awslogs",
     "options": {

.happy/terraform/modules/batch/outputs.tf

@@ -8,6 +8,11 @@ output batch_job_definition_no_revision {
   description = "ARN for the batch job definition"
 }
 
+output cxg_job_definition_no_revision {
+  value       = "arn:aws:batch:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:job-definition/${aws_batch_job_definition.cxg_job_def.name}"
+  description = "ARN for the cxg batch job definition"
+}
+
 output batch_job_log_group {
   value       = aws_cloudwatch_log_group.cloud_watch_logs_group.id
   description = "Name of the CloudWatch log group for the batch job"

.happy/terraform/modules/ecs-stack/main.tf

@@ -386,6 +386,7 @@ module upload_error_lambda {
 module upload_sfn {
   source                 = "../sfn"
   job_definition_arn     = module.upload_batch.batch_job_definition_no_revision
+  cxg_definition_arn     = module.upload_batch.cxg_job_definition_no_revision
   job_queue_arn          = local.job_queue_arn
   role_arn               = local.sfn_role_arn
   custom_stack_name      = local.custom_stack_name
@@ -404,22 +405,6 @@ module upload_sfn {
   batch_job_log_group = module.upload_batch.batch_job_log_group
 }
 
-module dataset_submissions_lambda {
-  source                     = "../lambda"
-  image                      = "${local.lambda_dataset_submissions_repo}:${local.image_tag}"
-  name                       = "dataset-submissions"
-  custom_stack_name          = local.custom_stack_name
-  remote_dev_prefix          = local.remote_dev_prefix
-  deployment_stage           = local.deployment_stage
-  artifact_bucket            = local.artifact_bucket
-  cellxgene_bucket           = local.cellxgene_bucket
-  datasets_bucket            = local.datasets_bucket
-  lambda_execution_role      = aws_iam_role.dataset_submissions_lambda_service_role.arn
-  step_function_arn          = module.upload_sfn.step_function_arn
-  subnets                    = local.subnets
-  security_groups            = local.security_groups
-}
-
 module schema_migration {
   source                        = "../schema_migration"
   image                         = "${local.upload_image_repo}:${local.image_tag}"
@@ -527,22 +512,3 @@ resource "aws_iam_role_policy_attachment" "lambda_step_function_execution_policy
   role       = aws_iam_role.dataset_submissions_lambda_service_role.name
   policy_arn = aws_iam_policy.lambda_step_function_execution_policy.arn
 }
-
-resource "aws_lambda_permission" "allow_dataset_submissions_lambda_execution" {
-  statement_id  = "AllowExecutionFromS3Bucket"
-  action        = "lambda:InvokeFunction"
-  function_name = module.dataset_submissions_lambda.arn
-  principal     = "s3.amazonaws.com"
-  source_arn    = try(local.secret["s3_buckets"]["dataset_submissions"]["arn"], "")
-}
-
-resource "aws_s3_bucket_notification" "on_dataset_submissions_object_created" {
-  bucket = local.dataset_submissions_bucket
-
-  lambda_function {
-    lambda_function_arn = module.dataset_submissions_lambda.arn
-    events = ["s3:ObjectCreated:*"]
-  }
-
-  depends_on = [aws_lambda_permission.allow_dataset_submissions_lambda_execution]
-}

.happy/terraform/modules/schema_migration/main.tf

@@ -5,66 +5,13 @@ data aws_caller_identity current {}
 locals {
   name = "schema-migration"
   job_definition_arn = "arn:aws:batch:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:job-definition/dp-${var.deployment_stage}-${var.custom_stack_name}-schema-migration"
-  swap_job_definition_arn = "${local.job_definition_arn}-swap"
 }
 
 resource aws_cloudwatch_log_group batch_cloud_watch_logs_group {
   retention_in_days = 365
   name              = "/dp/${var.deployment_stage}/${var.custom_stack_name}/${local.name}-batch"
 }
 
-resource aws_batch_job_definition schema_migrations_swap {
-  type = "container"
-  name = "dp-${var.deployment_stage}-${var.custom_stack_name}-${local.name}-swap"
-  container_properties = jsonencode({
-    jobRoleArn= var.batch_role_arn,
-    image= var.image,
-    environment= [
-      {
-        name= "ARTIFACT_BUCKET",
-        value= var.artifact_bucket
-      },
-      {
-        name= "DEPLOYMENT_STAGE",
-        value= var.deployment_stage
-      },
-      {
-        name= "AWS_DEFAULT_REGION",
-        value= data.aws_region.current.name
-      },
-      {
-        name= "REMOTE_DEV_PREFIX",
-        value= var.remote_dev_prefix
-      },
-      {
-        name= "DATASETS_BUCKET",
-        value= var.datasets_bucket
-      },
-    ],
-    resourceRequirements = [
-      {
-        type= "VCPU",
-        Value="32"
-      },
-      {
-        Type="MEMORY",
-        Value = "256000"
-      }
-    ]
-    linuxParameters= {
-     maxSwap= 0,
-     swappiness= 60
-    },
-    logConfiguration= {
-      logDriver= "awslogs",
-      options= {
-        awslogs-group= aws_cloudwatch_log_group.batch_cloud_watch_logs_group.id,
-        awslogs-region= data.aws_region.current.name
-      }
-    }
-  })
-}
-
 resource aws_batch_job_definition schema_migrations {
   type = "container"
   name = "dp-${var.deployment_stage}-${var.custom_stack_name}-${local.name}"
@@ -94,14 +41,14 @@ resource aws_batch_job_definition schema_migrations {
       },
     ],
     resourceRequirements = [
-                  {
-              type= "VCPU",
-              Value="2"
-            },
-            {
-              Type="MEMORY",
-              Value = "2048"
-            }
+        {
+          type= "VCPU",
+          Value="1"
+        },
+        {
+          Type="MEMORY",
+          Value = "8000"
+        }
     ]
     logConfiguration= {
       logDriver= "awslogs",
@@ -385,7 +332,7 @@ resource aws_sfn_state_machine sfn_schema_migration {
                   "Type": "Task",
                   "Resource": "arn:aws:states:::batch:submitJob.sync",
                   "Parameters": {
-                    "JobDefinition": "${resource.aws_batch_job_definition.schema_migrations_swap.arn}",
+                    "JobDefinition": "${resource.aws_batch_job_definition.schema_migrations.arn}",
                     "JobName": "dataset_migration",
                     "JobQueue": "${var.job_queue_arn}",
                     "Timeout": {
@@ -458,7 +405,7 @@ resource aws_sfn_state_machine sfn_schema_migration {
                     "Name.$": "$.result.sfn_name",
                     "Input": {
                       "AWS_STEP_FUNCTIONS_STARTED_BY_EXECUTION_ID.$": "$$.Execution.Id",
-                      "url.$": "$.result.uri",
+                      "manifest.$": "$.result.manifest",
                       "dataset_version_id.$": "$.result.dataset_version_id",
                       "collection_version_id.$": "$.result.collection_version_id",
                       "job_queue": "${var.job_queue_arn}"
@@ -518,7 +465,7 @@ resource aws_sfn_state_machine sfn_schema_migration {
           "Key.$": "$.key_name"
         }
       },
-      "MaxConcurrency": 10,
+      "MaxConcurrency": 30,
       "Next": "report",
       "Catch": [
         {