|
| 1 | +#!/bin/sh |
| 2 | + |
| 3 | +# Copyright (c) 2023 MIT Laboratory for Computational Physiology |
| 4 | +# Copyright (c) 2021 Thomas Ward <[email protected]> |
| 5 | +# |
| 6 | +# Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | +# of this software and associated documentation files (the "Software"), to deal |
| 8 | +# in the Software without restriction, including without limitation the rights |
| 9 | +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 10 | +# copies of the Software, and to permit persons to whom the Software is |
| 11 | +# furnished to do so, subject to the following conditions: |
| 12 | +# |
| 13 | +# The above copyright notice and this permission notice shall be included in all |
| 14 | +# copies or substantial portions of the Software. |
| 15 | +# |
| 16 | +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | +# SOFTWARE. |
| 23 | + |
| 24 | +yell () { echo "$0: $*" >&2; } |
| 25 | +die () { yell "$*"; exit 111; } |
| 26 | +try () { "$@" || die "Exiting. Failed to run: \"$*\""; } |
| 27 | + |
| 28 | +usage () { |
| 29 | + die " |
| 30 | +USAGE: ./import_duckdb.sh mimic_data_dir [output_db] |
| 31 | +WHERE: |
| 32 | + mimic_data_dir directory that contains csv.tar.gz or csv files |
| 33 | + output_db: optional filename for duckdb file (default: mimic4_note.db)\ |
| 34 | +" |
| 35 | +} |
| 36 | + |
| 37 | +# Print help if requested |
| 38 | +echo "$0 $* " | grep -Eq " -h | --help " && usage |
| 39 | + |
| 40 | +# rename CLI positional args to more friendly variable names |
| 41 | +MIMIC_DIR=$1 |
| 42 | +# allow optional specification of duckdb name, otherwise default to mimic4_note.db |
| 43 | +OUTFILE=mimic4_note.db |
| 44 | +if [ -n "$2" ]; then |
| 45 | + OUTFILE=$2 |
| 46 | +fi |
| 47 | + |
| 48 | + |
| 49 | +# basic error checking before running |
| 50 | +if [ -z "$MIMIC_DIR" ]; then |
| 51 | + yell "Please specify a mimic data directory" |
| 52 | + die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" |
| 53 | +elif [ ! -d "$MIMIC_DIR" ]; then |
| 54 | + yell "Specified directory \"$MIMIC_DIR\" does not exist." |
| 55 | + die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" |
| 56 | +elif [ -n "$3" ]; then |
| 57 | + yell "import.sh takes a maximum of two arguments." |
| 58 | + die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" |
| 59 | +elif [ -s "$OUTFILE" ]; then |
| 60 | + yell "File \"$OUTFILE\" already exists." |
| 61 | + read -p "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " yn |
| 62 | + case $yn in |
| 63 | + [Yy]* ) ;; # OK |
| 64 | + [Nn]* ) exit;; |
| 65 | + [Dd]* ) rm "$OUTFILE";; |
| 66 | + * ) die "Unrecognized input.";; |
| 67 | + esac |
| 68 | +fi |
| 69 | + |
| 70 | +# trim trailing slash from MIMIC_DIR, if present |
| 71 | +MIMIC_DIR=${MIMIC_DIR%/} |
| 72 | + |
| 73 | +# we will copy the postgresql create.sql file, and apply regex |
| 74 | +# to fix the following issues: |
| 75 | +# 1. Remove optional precision value from TIMESTAMP(NN) -> TIMESTAMP |
| 76 | +# duckdb does not support this. |
| 77 | +export REGEX_TIMESTAMP='s/TIMESTAMP\([0-9]+\)/TIMESTAMP/g' |
| 78 | + |
| 79 | +# use sed + above regex to create tables within db |
| 80 | +sed -r -e "${REGEX_TIMESTAMP}" ../postgres/create.sql | \ |
| 81 | + duckdb "$OUTFILE" |
| 82 | + |
| 83 | +# goal: get path from find, e.g., ./1.0/icu/d_items |
| 84 | +# and return database table name for it, e.g., mimiciv_icu.d_items |
| 85 | +make_table_name () { |
| 86 | + # strip leading directories (e.g., ./icu/hello.csv.gz -> hello.csv.gz) |
| 87 | + BASENAME=${1##*/} |
| 88 | + # strip suffix (e.g., hello.csv.gz -> hello; hello.csv -> hello) |
| 89 | + TABLE_NAME=${BASENAME%%.*} |
| 90 | + # strip basename (e.g., ./icu/hello.csv.gz -> ./icu) |
| 91 | + PATHNAME=${1%/*} |
| 92 | + # strip leading directories from PATHNAME (e.g. ./icu -> icu) |
| 93 | + DIRNAME=${PATHNAME##*/} |
| 94 | + TABLE_NAME="mimiciv_$DIRNAME.$TABLE_NAME" |
| 95 | +} |
| 96 | + |
| 97 | + |
| 98 | +# load data into database |
| 99 | +find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do |
| 100 | + make_table_name "$FILE" |
| 101 | + |
| 102 | + # skip directories which we do not expect in mimic-iv-note |
| 103 | + # avoids syntax errors if mimic-iv in the same dir |
| 104 | + case $DIRNAME in |
| 105 | + (note) ;; # OK |
| 106 | + (*) continue; |
| 107 | + esac |
| 108 | + echo "Loading $FILE .." |
| 109 | + OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL |
| 110 | + COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); |
| 111 | +EOSQL |
| 112 | + ) |
| 113 | + # If the table is missing in the DB, we emit a warning and continue. |
| 114 | + # Otherwise, the script repeats the error and exits. |
| 115 | + STATUS=$? |
| 116 | + if [ $STATUS -ne 0 ]; then |
| 117 | + echo "$OUTPUT" | grep -qiE 'table .* does not exist' && { |
| 118 | + echo "skipped (table $TABLE_NAME not found)"; |
| 119 | + continue; |
| 120 | + } |
| 121 | + yell "Failed loading $FILE into $TABLE_NAME" |
| 122 | + yell "$OUTPUT" |
| 123 | + die "Exiting due to load error." |
| 124 | + fi |
| 125 | + echo "done!" |
| 126 | +done && echo "Successfully finished loading data into $OUTFILE." |
0 commit comments