Publish action update (#1886)

* remove branches kw that probably isn't needed

* drop all tables on the derived schema first

* add copy concepts to version specific schema

* fix to target physionet-data

Author: alistairewj

.github/workflows/main.yml

@@ -2,8 +2,10 @@ name: Generate tables on BigQuery
 
 on:
   release:
-    types: [published]
-    branches: ["main"]
+    types: [released]
+
+env:
+  MIMIC_IV_VERSION: 3_1
 
 jobs:
   create-tables:
@@ -25,4 +27,9 @@ jobs:
         run: |
             echo "Generating tables on BigQuery"
             cd mimic-iv/concepts
-            bash make_concepts.sh
+            bash make_concepts.sh
+      
+      - name: Copy to release specific schema
+        run: |
+            echo "Copying tables to release specific schema: mimiciv_${MIMIC_IV_VERSION}_derived"
+            bash mimic-iv/concepts/copy_concepts_to_versioned_schema.sh ${MIMIC_IV_VERSION}

mimic-iv/concepts/copy_concepts_to_versioned_schema.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# This script copies the concepts in the BigQuery table mimiciv_derived to mimiciv_${VERSION}_derived.
+if [ -z "$$1" ]; then
+  echo "Usage: $0 <version>"
+  exit 1
+fi
+export SOURCE_DATASET=mimiciv_derived
+export TARGET_DATASET=mimiciv_$1_derived
+export PROJECT_ID=physionet-data
+
+# check if the target dataset exists
+if bq ls --datasets --project_id ${PROJECT_ID} | grep -q ${TARGET_DATASET}; then
+    echo "Using existing dataset ${TARGET_DATASET}."
+    # drop the existing tables in the target dataset
+    # this includes ones which may not be in the source dataset
+    for TABLE in `bq ls ${PROJECT_ID}:${TARGET_DATASET} | cut -d' ' -f3`;
+    do
+        # skip the first line of dashes
+        if [[ "${TABLE:0:2}" == '--' ]]; then
+            continue
+        fi
+        bq rm -f -q ${PROJECT_ID}:${TARGET_DATASET}.${TABLE}
+    done
+else
+    echo "Creating dataset ${PROJECT_ID}:${TARGET_DATASET}"
+    bq mk --dataset ${PROJECT_ID}:${TARGET_DATASET}
+fi
+
+echo "Copying tables from ${SOURCE_DATASET} to ${TARGET_DATASET}."
+for TABLE in `bq ls ${PROJECT_ID}:${SOURCE_DATASET} | cut -d' ' -f3`;
+do
+    # skip the first line of dashes
+    if [[ "${TABLE:0:2}" == '--' ]]; then
+      continue
+    fi
+    bq cp -f -q ${PROJECT_ID}:${SOURCE_DATASET}.${TABLE} ${PROJECT_ID}:${TARGET_DATASET}.${TABLE}
+done

mimic-iv/concepts/make_concepts.sh

@@ -6,6 +6,17 @@ export TARGET_DATASET=mimiciv_derived
 # note: max_rows=1 *displays* only one row, but all rows are inserted into the destination table
 BQ_OPTIONS='--quiet --headless --max_rows=0 --use_legacy_sql=False --replace'
 
+# drop the existing tables in the target dataset
+for TABLE in `bq ls physionet-data:${TARGET_DATASET} | cut -d' ' -f3`;
+do
+    # skip the first line of dashes
+    if [[ "${TABLE:0:2}" == '--' ]]; then
+      continue
+    fi
+  echo "Dropping table ${TARGET_DATASET}.${TABLE}"
+  bq rm -f -q ${TARGET_DATASET}.${TABLE}
+done
+
 # generate a few tables first as the desired order isn't alphabetical
 for table_path in demographics/icustay_times;
 do