Merge pull request #1947 from MIT-LCP/alistair/fix_derived_build

Fix SQLFluff test and derived tables copying

Author: tompollard

.github/workflows/lint_sqlfluff.yml

@@ -8,42 +8,37 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Python
-        uses: "actions/setup-python@v2"
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.7"
+          python-version: "3.12"
       - name: install sqlfluff
-        run: "pip install sqlfluff==1.4.5"
+        run: "pip install sqlfluff==3.5.0"
       - name: Get changed files
         id: get_file_changes
         uses: trilom/[email protected]
         with:
           output: ' '
       - name: Get changed .sql files in mimic-iv concepts folder
         id: get_files_to_lint
-        shell: bash -l {0}
+        shell: bash
         run: |
-          # Set the command in the $() brackets as an output to use in later steps
-          echo "lintees=$(
-          echo \
-          $(echo ${{ steps.get_file_changes.outputs.files_modified }} |
-          tr -s ' ' '\n' |
-          grep -E '^mimic-iv/concepts/.*[.]sql$' |
-          tr -s '\n' ' ') \
-          $(echo ${{ steps.get_file_changes.outputs.files_added }} |
-          tr -s ' ' '\n' |
-          grep -E '^mimic-iv/concepts/.*[.]sql$' |
-          tr -s '\n' ' ')
-          ) >> $GITHUB_OUTPUT"
+          # Compose list of changed SQL files under mimic-iv/concepts and export to step output
+          raw="${{ steps.get_file_changes.outputs.files_modified }} ${{ steps.get_file_changes.outputs.files_added }}"
+          # Check for mimic-iv SQL files which may have changed
+          filtered="$(printf '%s' "$raw" | tr -s ' ' '\n' | grep -E '^mimic-iv/concepts/.*[.]sql$' || true)"
+          # Turn this into a space separated list for the next step
+          files="$(printf '%s' "$filtered" | tr -s '\n' ' ')"
+          echo "lintees=${files}" >> "$GITHUB_OUTPUT"
 
       - name: Lint SQL files
         id: sqlfluff_json
         if: steps.get_files_to_lint.outputs.lintees != ''
-        shell: bash -l {0}
+        shell: bash
         run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json
       - name: Annotate
-        uses: yuzutech/annotations-action@v0.3.0
+        uses: yuzutech/annotations-action@v0.5.0
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           title: "SQLFluff Lint"

.sqlfluff

@@ -1,20 +1,21 @@
-[sqlfluff:core]
-rules = core,L019
-large_file_skip_byte_limit = 40000
+[sqlfluff]
 dialect = bigquery
+large_file_skip_byte_limit = 40000
+exclude_rules = LT02
+
+# Enable core rules plus explicit layout.commas (leading commas)
+[sqlfluff:core]
+rules = core,layout.commas
 
 [sqlfluff:layout:type:comma]
 line_position = leading
 
-[sqlfluff:indentation]
-indented_joins = false
-indented_using_on = true
-template_blocks_indent = false
-
-[sqlfluff:rules:L010]
-# Keywords should be upper case
+# Capitalisation rules
+[sqlfluff:rules:capitalisation.keywords]
 capitalisation_policy = upper
 
-[sqlfluff:rules:L030]
-# Functions should be upper case
-extended_capitalisation_policy = upper
+[sqlfluff:rules:capitalisation.functions]
+extended_capitalisation_policy = upper
+
+[sqlfluff:rules:capitalisation.identifiers]
+extended_capitalisation_policy = lower

mimic-iv/concepts/copy_concepts_to_versioned_schema.sh

@@ -27,7 +27,7 @@ else
 fi
 
 echo "Copying tables from ${SOURCE_DATASET} to ${TARGET_DATASET}."
-for TABLE in `bq ls ${PROJECT_ID}:${SOURCE_DATASET} | cut -d' ' -f3`;
+for TABLE in `bq ls -n 500 ${PROJECT_ID}:${SOURCE_DATASET} | cut -d' ' -f3`;
 do
     # skip the first line of dashes
     if [[ "${TABLE:0:2}" == '--' ]]; then

mimic-iv/concepts/firstday/first_day_bg_art.sql

@@ -28,7 +28,8 @@ SELECT
     , MIN(sodium) AS sodium_min, MAX(sodium) AS sodium_max
 FROM `physionet-data.mimiciv_icu.icustays` ie
 LEFT JOIN `physionet-data.mimiciv_derived.bg` bg
-    ON ie.subject_id = bg.subject_id
+    ON
+        ie.subject_id = bg.subject_id
         AND bg.specimen = 'ART.'
         AND bg.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR)
         AND bg.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY)

mimic-iv/concepts/medication/arb.sql

@@ -1,16 +1,17 @@
-WITH arb_drug AS (  
+WITH arb_drug AS (
   SELECT DISTINCT
     drug
     , CASE
-        WHEN UPPER(drug) LIKE '%AZILSARTAN%' OR UPPER(drug) LIKE '%EDARBI%' THEN 1
-        WHEN UPPER(drug) LIKE '%CANDESARTAN%' OR UPPER(drug) LIKE '%ATACAND%' THEN 1
-        WHEN UPPER(drug) LIKE '%IRBESARTAN%' OR UPPER(drug) LIKE '%AVAPRO%' THEN 1
-        WHEN UPPER(drug) LIKE '%LOSARTAN%' OR UPPER(drug) LIKE '%COZAAR%' THEN 1
-        WHEN UPPER(drug) LIKE '%OLMESARTAN%' OR UPPER(drug) LIKE '%BENICAR%' THEN 1
-        WHEN UPPER(drug) LIKE '%TELMISARTAN%' OR UPPER(drug) LIKE '%MICARDIS%' THEN 1
-        WHEN UPPER(drug) LIKE '%VALSARTAN%' OR UPPER(drug) LIKE '%DIOVAN%' THEN 1
-        WHEN UPPER(drug) LIKE '%SACUBITRIL%' OR UPPER(drug) LIKE '%ENTRESTO%' THEN 1
-        ELSE 0
+      WHEN UPPER(drug) LIKE '%AZILSARTAN%' OR UPPER(drug) LIKE '%EDARBI%'
+      OR UPPER(drug) LIKE '%CANDESARTAN%' OR UPPER(drug) LIKE '%ATACAND%'
+      OR UPPER(drug) LIKE '%IRBESARTAN%' OR UPPER(drug) LIKE '%AVAPRO%'
+      OR UPPER(drug) LIKE '%LOSARTAN%' OR UPPER(drug) LIKE '%COZAAR%'
+      OR UPPER(drug) LIKE '%OLMESARTAN%' OR UPPER(drug) LIKE '%BENICAR%'
+      OR UPPER(drug) LIKE '%TELMISARTAN%' OR UPPER(drug) LIKE '%MICARDIS%'
+      OR UPPER(drug) LIKE '%VALSARTAN%' OR UPPER(drug) LIKE '%DIOVAN%'
+      OR UPPER(drug) LIKE '%SACUBITRIL%' OR UPPER(drug) LIKE '%ENTRESTO%'
+        THEN 1
+      ELSE 0
       END AS arb
   FROM `physionet-data.mimiciv_hosp.prescriptions`
 )
@@ -24,8 +25,7 @@ SELECT
 FROM
   `physionet-data.mimiciv_hosp.prescriptions` pr
   INNER JOIN arb_drug
-  ON 
-    pr.drug = arb_drug.drug
+  ON pr.drug = arb_drug.drug
 WHERE
   arb_drug.arb = 1
 ;