Add general purpose script to import a SWE dataset

dbeaver/data_import/005-aso_import.sh

@@ -1,5 +1,5 @@
-#/usr/bin/env bash
-# 
+#!/usr/bin/env bash
+#
 # Import ASO SWE tifs
 #
 # Arguments:
@@ -13,8 +13,8 @@
 #
 # ASO UC extent for all flights in EPSG:32613
 #   -te 231453.000 4530200.00 446853.203 4129449.623
-# Converted with: cs2cs --only-best -f "%.8f" EPSG:32613 EPSG:4269  
-#   -te -108.18706066 40.87885627 -105.59976043 37.31016729 
+# Converted with: cs2cs --only-best -f "%.8f" EPSG:32613 EPSG:4269
+#   -te -108.18706066 40.87885627 -105.59976043 37.31016729
 
 set -e
 

dbeaver/data_import/006-isnobal.sh

@@ -1,4 +1,4 @@
-#/usr/bin/env bash
+#!/usr/bin/env bash
 #
 # Import snow.nc file from iSnobal and store in the DB.
 #

dbeaver/data_import/008-import_dataset.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# Import snow.nc file from iSnobal and store in the DB.
+#
+# Arguments:
+#   -s: Path to file to import
+#   -t: Table name to import data in
+#   -d: Date of imported data
+#   -c: Create table and import records
+#   -a: Append records to table (Default)
+#
+# Example call for initial import:
+#   import_dataset.sh -s _path_to_file_ -t table_name -c -d 20240101_
+#
+# Requires:
+# * Configured ~/.pg_service.conf file with host, port, user, dbname
+
+# -f = Prevent glob expansion when importing multiple files
+set -ef
+
+SCRIPT_PATH="$(dirname "$(realpath $0)")"
+DB_CONNECT_OPTIONS='service=swe_db'
+
+# Parse script options
+source ${SCRIPT_PATH}/import_script_options.sh
+
+if [[ "$IMPORT_MODE" == "$APPEND_RECORDS" ]]; then
+    IMPORT_OPTIONS="-a"
+elif [[ "$IMPORT_MODE" == "$CREATE_TABLE" ]]; then
+    IMPORT_OPTIONS="-d -C -x -I -l 2,3"
+    # Add post create table steps
+    POST_STEP=$(sed "s/__tablename__/${TABLE}/g" "${SCRIPT_PATH}/008-update_db_table.sql")
+fi
+# Add post steps for new records
+POST_STEP+=$(sed "s/__tablename__/${TABLE}/g" "${SCRIPT_PATH}/008-update_db_records.sql")
+
+# Get the file date
+[[ $DB_FILE =~ ([0-9]{8}) ]]
+# Set date for post step
+POST_STEP=${POST_STEP/_filedate_/"'${BASH_REMATCH[1]}'"}
+
+# Import table into database
+raster2pgsql ${IMPORT_OPTIONS} -M -Y -t 32x32 \
+  ${SOURCE_FILE} ${TABLE} | \
+  psql ${DB_CONNECT_OPTIONS}
+
+# Update imported rows
+psql ${DB_CONNECT_OPTIONS} -c "${POST_STEP}"
+
+# Table maintenance after bigger import
+psql ${DB_CONNECT_OPTIONS} -c "VACUUM FULL ${TABLE}"

dbeaver/data_import/008-update_db_records.sql

@@ -0,0 +1,7 @@
+-- Update recently imported records based on filename
+UPDATE __tablename__
+  SET swe_date = to_date(
+    _filedate_,
+    'YYYYMMDD'
+  )
+  WHERE swe_date is NULL;

dbeaver/data_import/008-update_db_table.sql

@@ -0,0 +1,4 @@
+ALTER TABLE __tablename__
+    ADD swe_date date;
+
+CREATE INDEX ON __tablename__(swe_date);

dbeaver/data_import/import_script_options.sh

@@ -19,8 +19,8 @@ IMPORT_MODE=$APPEND_RECORDS
 DB_FILE_PATTERN="^([0-9]{8}_).*"
 
 # Script argument options
-LONG_OPTIONS=source:db-file:create,append
-SHORT_OPTIONS=s:d:ca
+LONG_OPTIONS=source:db-file:table:create,append
+SHORT_OPTIONS=s:d:t:ca
 
 # Parse arguments and set variables
 ARGUMENTS=$(getopt --options=${SHORT_OPTIONS} --longoptions=${LONG_OPTIONS} --name $0 -- "$@") || exit 1
@@ -44,6 +44,10 @@ while true; do
             DB_FILE="$2"
             shift 2
             ;;
+        -t|--table)
+            TABLE="$2"
+            shift 2
+            ;;
         --)
             shift
             break