Update BigQuery dataset build to use latest gh actions (#1889)

* update gcp auth and setup packages to use workload identity federation

* specify minimum version for workload identity federation

* add version table

* add creation of _version table to mimiciv_derived

* shell script which adds _version table to mimiciv_* schemas

* rename _version to _metadata

Author: alistairewj

.github/workflows/main.yml

@@ -10,19 +10,24 @@ env:
 jobs:
   create-tables:
     runs-on: ubuntu-latest
+    permissions:
+        contents: 'read'
+        id-token: 'write'
     steps:
       - name: Check out repository code
         uses: actions/checkout@v3
 
       - id: 'auth'
-        uses: 'google-github-actions/auth@v0'
+        uses: 'google-github-actions/auth@v2'
         with:
-            project_id: ${{ secrets.GCP_PROJECT_ID }}
-            credentials_json: ${{ secrets.GCP_SA_KEY }}
+          service_account: '[email protected]'
+          workload_identity_provider: 'projects/569883598760/locations/global/workloadIdentityPools/github/providers/mimic-code'
 
       - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v0'
-
+        uses: 'google-github-actions/setup-gcloud@v2'
+        with:
+          version: '>= 363.0.0'
+          
       - name: Run make_concepts
         run: |
             echo "Generating tables on BigQuery"

mimic-iv/buildmimic/bigquery/make_version.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# This script makes the _version table for each schema in BigQuery,
+# Currently hard-coded to the status as of 2025-04-20.
+export METADATA_TABLE="_metadata"
+
+# create an array of target datasets and versions
+# loop through them at the same time
+datasets=(
+  "mimiciv_icu:3.1"
+  "mimiciv_hosp:3.1"
+  "mimiciv_note:2.2"
+  "mimiciv_ed:2.2"
+)
+
+for entry in "${datasets[@]}"; do
+  TARGET_DATASET="${entry%%:*}"
+  MIMIC_VERSION="${entry##*:}"
+  export TARGET_DATASET
+  export MIMIC_VERSION
+  
+  echo "Creating ${TARGET_DATASET}.${METADATA_TABLE} table"
+    bq query <<EOF
+CREATE TABLE IF NOT EXISTS \`physionet-data.${TARGET_DATASET}.${METADATA_TABLE}\` (
+  attribute STRING,
+  value STRING
+);
+
+TRUNCATE TABLE \`physionet-data.${TARGET_DATASET}.${METADATA_TABLE}\`;
+
+INSERT INTO \`physionet-data.${TARGET_DATASET}.${METADATA_TABLE}\` (attribute, value)
+VALUES
+  ('mimic_version', '${MIMIC_VERSION}');
+EOF
+done

mimic-iv/concepts/make_concepts.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 # This script generates the concepts in the BigQuery table mimiciv_derived.
 export TARGET_DATASET=mimiciv_derived
+export METADATA_TABLE="_metadata"
+export MIMIC_VERSION="3.1"
 
 # specify bigquery query command options
 # note: max_rows=1 *displays* only one row, but all rows are inserted into the destination table
@@ -17,6 +19,26 @@ do
   bq rm -f -q ${TARGET_DATASET}.${TABLE}
 done
 
+# create a _version table to store the mimic-iv version, git commit hash, and latest git tag
+GIT_COMMIT_HASH=$(git rev-parse HEAD)
+LATEST_GIT_TAG=$(git describe --tags --abbrev=0)
+
+echo "Creating ${TARGET_DATASET}.${METADATA_TABLE} table"
+bq query <<EOF
+CREATE TABLE IF NOT EXISTS \`physionet-data.${TARGET_DATASET}.${METADATA_TABLE}\` (
+  attribute STRING,
+  value STRING
+);
+
+TRUNCATE TABLE \`physionet-data.${TARGET_DATASET}.${METADATA_TABLE}\`;
+
+INSERT INTO \`physionet-data.${TARGET_DATASET}.${METADATA_TABLE}\` (attribute, value)
+VALUES
+  ('mimic_version', '${MIMIC_VERSION}'),
+  ('mimic_code_version', '${LATEST_GIT_TAG}'),
+  ('mimic_code_commit_hash', '${GIT_COMMIT_HASH}');
+EOF
+
 # generate a few tables first as the desired order isn't alphabetical
 for table_path in demographics/icustay_times;
 do