Merge pull request #1209 from MIT-LCP/bg_minor_fix

Fixes to bg, postgres vent queries, and tests

Author: alistairewj

mimic-iv/concepts/firstday/first_day_bg_art.sql

@@ -26,7 +26,7 @@ select
 FROM `physionet-data.mimic_icu.icustays` ie
 LEFT JOIN `physionet-data.mimic_derived.bg` bg
     ON ie.subject_id = bg.subject_id
-    AND bg.specimen_pred = 'ART.'
+    AND bg.specimen = 'ART.'
     AND bg.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR)
     AND bg.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY)
 GROUP BY ie.subject_id, ie.stay_id

mimic-iv/concepts/postgres/firstday/first_day_bg_art.sql

@@ -28,7 +28,7 @@ select
 FROM mimic_icu.icustays ie
 LEFT JOIN mimic_derived.bg bg
     ON ie.subject_id = bg.subject_id
-    AND bg.specimen_pred = 'ART.'
+    AND bg.specimen = 'ART.'
     AND bg.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR)
     AND bg.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY)
 GROUP BY ie.subject_id, ie.stay_id

mimic-iv/concepts/postgres/measurement/bg.sql

@@ -13,7 +13,7 @@ select
   -- specimen_id *may* have different storetimes, so this is taking the latest
   , MAX(storetime) AS storetime
   , le.specimen_id
-  , MAX(CASE WHEN itemid = 52028 THEN value ELSE NULL END) AS specimen
+  , MAX(CASE WHEN itemid = 52033 THEN value ELSE NULL END) AS specimen
   , MAX(CASE WHEN itemid = 50801 THEN valuenum ELSE NULL END) AS aado2
   , MAX(CASE WHEN itemid = 50802 THEN valuenum ELSE NULL END) AS baseexcess
   , MAX(CASE WHEN itemid = 50803 THEN valuenum ELSE NULL END) AS bicarbonate
@@ -50,7 +50,7 @@ FROM mimic_hosp.labevents le
 where le.ITEMID in
 -- blood gases
 (
-    52028 -- specimen
+    52033 -- specimen
   , 50801 -- aado2
   , 50802 -- base excess
   , 50803 -- bicarb
@@ -129,22 +129,6 @@ where bg.po2 is not null
 select bg.*
   , ROW_NUMBER() OVER (partition by bg.subject_id, bg.charttime order by s2.charttime DESC) as lastRowFiO2
   , s2.fio2_chartevents
-  -- create our specimen prediction
-  ,  1/(1+exp(-(-0.02544
-  +    0.04598 * po2
-  + coalesce(-0.15356 * spo2             , -0.15356 *   97.49420 +    0.13429)
-  + coalesce( 0.00621 * fio2_chartevents ,  0.00621 *   51.49550 +   -0.24958)
-  + coalesce( 0.10559 * hemoglobin       ,  0.10559 *   10.32307 +    0.05954)
-  + coalesce( 0.13251 * so2              ,  0.13251 *   93.66539 +   -0.23172)
-  + coalesce(-0.01511 * pco2             , -0.01511 *   42.08866 +   -0.01630)
-  + coalesce( 0.01480 * fio2             ,  0.01480 *   63.97836 +   -0.31142)
-  + coalesce(-0.00200 * aado2            , -0.00200 *  442.21186 +   -0.01328)
-  + coalesce(-0.03220 * bicarbonate      , -0.03220 *   22.96894 +   -0.06535)
-  + coalesce( 0.05384 * totalco2         ,  0.05384 *   24.72632 +   -0.01405)
-  + coalesce( 0.08202 * lactate          ,  0.08202 *    3.06436 +    0.06038)
-  + coalesce( 0.10956 * ph               ,  0.10956 *    7.36233 +   -0.00617)
-  + coalesce( 0.00848 * o2flow           ,  0.00848 *    7.59362 +   -0.35803)
-  ))) as specimen_prob
 from stg2 bg
 left join stg_fio2 s2
   -- same patient
@@ -158,14 +142,8 @@ select
     stg3.subject_id
   , stg3.hadm_id
   , stg3.charttime
-  -- raw data indicating sample type
+  -- drop down text indicating the specimen type
   , specimen
-  -- prediction of specimen for obs missing the actual specimen
-  , case
-        when specimen is not null then specimen
-        when specimen_prob > 0.75 then 'ART.'
-      else null end as specimen_pred
-  , specimen_prob
 
   -- oxygen related parameters
   , so2

mimic-iv/concepts/postgres/measurement/oxygen_delivery.sql

@@ -8,7 +8,7 @@ with ce_stg1 as
     , ce.charttime
     , CASE
         -- merge o2 flows into a single row
-        WHEN itemid IN (223834, 227582, 224691) THEN 223834
+        WHEN itemid IN (223834, 227582) THEN 223834
       ELSE itemid END AS itemid
     , value
     , valuenum
@@ -20,9 +20,10 @@ with ce_stg1 as
   (
       223834 -- o2 flow
     , 227582 -- bipap o2 flow
-    , 224691 -- Flow Rate (L)
     -- additional o2 flow is its own column
     , 227287 -- additional o2 flow
+    -- below flow rate is *not* o2 flow, and not included
+    -- , 224691 -- Flow Rate (L)
   )
 )
 , ce_stg2 AS

mimic-iv/concepts/postgres/measurement/ventilator_setting.sql

@@ -50,6 +50,7 @@ with ce as
     , 223849 -- vent mode
     , 229314 -- vent mode (Hamilton)
     , 223848 -- vent type
+    , 224691 -- Flow Rate (L)
   )
 )
 SELECT
@@ -66,6 +67,7 @@ SELECT
     , MAX(CASE WHEN itemid = 224696 THEN valuenum ELSE NULL END) AS plateau_pressure
     , MAX(CASE WHEN itemid in (220339, 224700) THEN valuenum ELSE NULL END) AS peep
     , MAX(CASE WHEN itemid = 223835 THEN valuenum ELSE NULL END) AS fio2
+    , MAX(CASE WHEN itemid = 224691 THEN valuenum ELSE NULL END) AS flow_rate
     , MAX(CASE WHEN itemid = 223849 THEN value ELSE NULL END) AS ventilator_mode
     , MAX(CASE WHEN itemid = 229314 THEN value ELSE NULL END) AS ventilator_mode_hamilton
     , MAX(CASE WHEN itemid = 223848 THEN value ELSE NULL END) AS ventilator_type

mimic-iv/concepts/postgres/postgres-make-concepts.sql

@@ -56,8 +56,8 @@
 \i treatment/ventilation.sql
 
 -- firstday
-\i firstday/first_day_bg_art.sql
 \i firstday/first_day_bg.sql
+\i firstday/first_day_bg_art.sql
 \i firstday/first_day_gcs.sql
 \i firstday/first_day_height.sql
 \i firstday/first_day_lab.sql

mimic-iv/concepts/postgres/treatment/ventilation.sql

@@ -1,10 +1,26 @@
 -- THIS SCRIPT IS AUTOMATICALLY GENERATED. DO NOT EDIT IT DIRECTLY.
 DROP TABLE IF EXISTS ventilation; CREATE TABLE ventilation AS 
--- Calculate duration of mechanical ventilation.
+-- Classify oxygen devices and ventilator modes into six clinical categories.
+
+-- Categories include..
+--  Invasive oxygen delivery types: 
+--      Tracheostomy (with or without positive pressure ventilation) 
+--      InvasiveVent (positive pressure ventilation via endotracheal tube, could be oro/nasotracheal or tracheostomy)
+--  Non invasive oxygen delivery types (divided similar to doi:10.1001/jama.2020.9524):
+--      NonInvasiveVent (non-invasive positive pressure ventilation)
+--      HFNC (high flow nasal oxygen / cannula)
+--      SupplementalOxygen (all other non-rebreather, facemask, face tent, nasal prongs...)
+--  No oxygen device:
+--      None
+
+-- When conflicting settings occur (rare), the priority is:
+--  trach > mech vent > NIV > high flow > o2
+
 -- Some useful cases for debugging:
 --  stay_id = 30019660 has a tracheostomy placed in the ICU
 --  stay_id = 30000117 has explicit documentation of extubation
--- classify vent settings into modes
+
+-- first we collect all times which have relevant documentation
 WITH tm AS
 (
   SELECT stay_id, charttime
@@ -25,11 +41,13 @@ WITH tm AS
     -- tracheostomy
     WHEN o2_delivery_device_1 IN
     (
-        'Tracheostomy tube'
-    -- 'Trach mask ' -- 16435 observations
+        'Tracheostomy tube',
+        'Trach mask ' -- 16435 observations
+        -- 'T-piece', -- 1135 observations (T-piece could be either InvasiveVent or Tracheostomy)
+
     )
-        THEN 'Trach'
-    -- mechanical ventilation
+        THEN 'Tracheostomy'
+    -- mechanical / invasive ventilation
     WHEN o2_delivery_device_1 IN
     (
         'Endotracheal tube'
@@ -100,30 +118,33 @@ WITH tm AS
         'NIV-ST'
     )
         THEN 'NonInvasiveVent'
-    -- high flow
+    -- high flow nasal cannula
     when o2_delivery_device_1 IN
     (
-        'High flow neb', -- 10785 observations
         'High flow nasal cannula' -- 925 observations
     )
-        THEN 'HighFlow'
-    -- normal oxygen delivery
+        THEN 'HFNC'
+    -- non rebreather
     WHEN o2_delivery_device_1 in
-    (
-        'Nasal cannula', -- 153714 observations
+    ( 
+        'Non-rebreather', -- 5182 observations
         'Face tent', -- 24601 observations
         'Aerosol-cool', -- 24560 observations
-        'Non-rebreather', -- 5182 observations
         'Venti mask ', -- 1947 observations
         'Medium conc mask ', -- 1888 observations
-        'T-piece', -- 1135 observations
         'Ultrasonic neb', -- 9 observations
         'Vapomist', -- 3 observations
-        'Oxymizer' -- 1301 observations
+        'Oxymizer', -- 1301 observations
+        'High flow neb', -- 10785 observations
+        'Nasal cannula'
     )
-        THEN 'Oxygen'
-    -- Not categorized:
-    -- 'Other', 'None'
+        THEN 'SupplementalOxygen'
+    WHEN o2_delivery_device_1 in 
+    (
+        'None'
+    )
+        THEN 'None'
+    -- not categorized: other
     ELSE NULL END AS ventilation_status
   FROM tm
   LEFT JOIN mimic_derived.ventilator_setting vs
@@ -138,8 +159,8 @@ WITH tm AS
     SELECT
       stay_id, charttime
       -- source data columns, here for debug
-      , o2_delivery_device_1
-      , vent_mode
+      -- , o2_delivery_device_1
+      -- , vent_mode
       -- carry over the previous charttime which had the same state
       , LAG(charttime, 1) OVER (PARTITION BY stay_id, ventilation_status ORDER BY charttime) AS charttime_lag
       -- bring back the next charttime, regardless of the state
@@ -155,34 +176,42 @@ WITH tm AS
 (
     SELECT
         stay_id
-        -- source data columns, here for debug
-        , o2_delivery_device_1
-        , vent_mode
-        , charttime_lag
         , charttime
+        , charttime_lag
         , charttime_lead
         , ventilation_status
 
+        -- source data columns, here for debug
+        -- , o2_delivery_device_1
+        -- , vent_mode
+
         -- calculate the time since the last event
         , DATETIME_DIFF(charttime,charttime_lag,'MINUTE')/60 as ventduration
 
         -- now we determine if the current ventilation status is "new", or continuing the previous
         , CASE
+            -- if lag is null, this is the first event for the patient
+            WHEN ventilation_status_lag IS NULL THEN 1
             -- a 14 hour gap always initiates a new event
             WHEN DATETIME_DIFF(charttime,charttime_lag,'HOUR') >= 14 THEN 1
-            WHEN ventilation_status_lag IS NULL THEN 1
             -- not a new event if identical to the last row
             WHEN ventilation_status_lag != ventilation_status THEN 1
           ELSE 0
-          END AS new_status
+          END AS new_ventilation_event
     FROM vd0
 )
 , vd2 as
 (
-    SELECT vd1.*
+    SELECT vd1.stay_id, vd1.charttime
+    , ventduration, new_ventilation_event
     -- create a cumulative sum of the instances of new ventilation
-    -- this results in a monotonic integer assigned to each instance of ventilation
-    , SUM(new_status) OVER (PARTITION BY stay_id ORDER BY charttime) AS vent_num
+    -- this results in a monotonically increasing integer assigned 
+    -- to each instance of ventilation
+    , SUM(new_ventilation_event) OVER
+    (
+        PARTITION BY stay_id
+        ORDER BY charttime
+    ) AS vent_seq
     FROM vd1
 )
 -- create the durations for each ventilation instance
@@ -200,9 +229,10 @@ SELECT stay_id
         END
    ) AS endtime
    -- all rows with the same vent_num will have the same ventilation_status
-   -- for efficiency, we use an aggregate here, but we could equally well group by this column
+   -- for efficiency, we use an aggregate here,
+   -- but we could equally well group by this column
   , MAX(ventilation_status) AS ventilation_status
 FROM vd2
-GROUP BY stay_id, vent_num
+GROUP BY stay_id, vent_seq
 HAVING min(charttime) != max(charttime)
 ;

mimic-iv/tests/README.md

@@ -53,9 +53,7 @@ With the account configured, we can create a key for the service, and download t
 **This is the only copy of the private key.** Don't lose it :)
 
 ```sh
-gcloud iam service-accounts keys create `key.json` --iam-account [email protected]
+gcloud iam service-accounts keys create "key.json" --iam-account [email protected]
 ```
 
 The tests will use the `key.json` file in the `tests/` folder by default, so you should now be able to run the tests!
-
-(NOTE: this service account will not have access to MIMIC-IV itself. This is a TODO!).

mimic-iv/tests/conftest.py

@@ -1,5 +1,6 @@
 import os
 from collections import namedtuple
+from pathlib import Path
 
 import pytest
 from google.cloud import bigquery
@@ -33,3 +34,25 @@ def project_id():
     Return the name of the BigQuery project used.
     """
     return 'physionet-data'
+
+@pytest.fixture(scope="session")
+def concept_folders():
+    """
+    Returns the folders containing concepts.
+    """
+    return ['comorbidity', 'demographics', 'measurement', 'medication', 'organfailure', 'treatment', 'score', 'sepsis', 'firstday']
+
+@pytest.fixture(scope="session")
+def concepts(concept_folders):
+    """
+    Returns all concepts which should be generated.
+    """
+    concepts = {}
+    current_dir = Path(__file__).parent
+    concept_dir = current_dir / '../concepts'
+    for folder in concept_folders:
+        files = os.listdir(concept_dir / folder)
+        # add list of the concepts in this folder to the dict
+        concepts[folder] = [f[:-4] for f in files if f.endswith('.sql')]
+
+    return concepts

mimic-iv/tests/test_all_tables.py

@@ -0,0 +1,15 @@
+import pandas as pd
+from pandas.io import gbq
+
+def test_tables_have_data(dataset, project_id, concepts):
+    """Verifies each table has data."""
+
+    for folder, concept_list in concepts.items():
+        for concept_name in concept_list:
+            query = f"""
+            SELECT *
+            FROM {dataset}.{concept_name}
+            LIMIT 5
+            """
+            df = gbq.read_gbq(query, project_id=project_id, dialect="standard")
+            assert df.shape[0] > 0, f'did not find table for {folder}.{concept_name}'