From 2f1577c27134511916353598b5d731c01b58d217 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Wed, 26 Feb 2025 08:35:38 -0600 Subject: [PATCH 1/3] Scripts - DB helpers - Add command line option for table name Add the ability to parse a '--table' option for bash scripts. --- dbeaver/data_import/import_script_options.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dbeaver/data_import/import_script_options.sh b/dbeaver/data_import/import_script_options.sh index dc85007..9a577d3 100644 --- a/dbeaver/data_import/import_script_options.sh +++ b/dbeaver/data_import/import_script_options.sh @@ -19,8 +19,8 @@ IMPORT_MODE=$APPEND_RECORDS DB_FILE_PATTERN="^([0-9]{8}_).*" # Script argument options -LONG_OPTIONS=source:db-file:create,append -SHORT_OPTIONS=s:d:ca +LONG_OPTIONS=source:db-file:table:create,append +SHORT_OPTIONS=s:d:t:ca # Parse arguments and set variables ARGUMENTS=$(getopt --options=${SHORT_OPTIONS} --longoptions=${LONG_OPTIONS} --name $0 -- "$@") || exit 1 @@ -44,6 +44,10 @@ while true; do DB_FILE="$2" shift 2 ;; + -t|--table) + TABLE="$2" + shift 2 + ;; --) shift break From 2197c4f0fd94399d0dc9f5eab3cd3f586c2fed86 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Wed, 26 Feb 2025 08:36:48 -0600 Subject: [PATCH 2/3] DB import - Add general SWE data set import Add a script that accepts the table name as an argument and imports given datasets into the DB. Also includes steps for post table structure and imported records update. --- dbeaver/data_import/008-import_dataset.sh | 51 +++++++++++++++++++ dbeaver/data_import/008-update_db_records.sql | 7 +++ dbeaver/data_import/008-update_db_table.sql | 4 ++ 3 files changed, 62 insertions(+) create mode 100755 dbeaver/data_import/008-import_dataset.sh create mode 100644 dbeaver/data_import/008-update_db_records.sql create mode 100644 dbeaver/data_import/008-update_db_table.sql diff --git a/dbeaver/data_import/008-import_dataset.sh b/dbeaver/data_import/008-import_dataset.sh new file mode 100755 index 0000000..636c4d4 --- /dev/null +++ b/dbeaver/data_import/008-import_dataset.sh @@ -0,0 +1,51 @@ +#/usr/bin/env bash +# +# Import snow.nc file from iSnobal and store in the DB. +# +# Arguments: +# -s: Path to file to import +# -t: Table name to import data in +# -d: Date of imported data +# -c: Create table and import records +# -a: Append records to table (Default) +# +# Example call for initial import: +# import_dataset.sh -s _path_to_file_ -t table_name -c -d 20240101_ +# +# Requires: +# * Configured ~/.pg_service.conf file with host, port, user, dbname + +# -f = Prevent glob expansion when importing multiple files +set -ef + +SCRIPT_PATH="$(dirname "$(realpath $0)")" +DB_CONNECT_OPTIONS='service=swe_db' + +# Parse script options +source ${SCRIPT_PATH}/import_script_options.sh + +if [[ "$IMPORT_MODE" == "$APPEND_RECORDS" ]]; then + IMPORT_OPTIONS="-a" +elif [[ "$IMPORT_MODE" == "$CREATE_TABLE" ]]; then + IMPORT_OPTIONS="-d -C -x -I -l 2,3" + # Add post create table steps + POST_STEP=$(sed "s/__tablename__/${TABLE}/g" "${SCRIPT_PATH}/008-update_db_table.sql") +fi +# Add post steps for new records +POST_STEP+=$(sed "s/__tablename__/${TABLE}/g" "${SCRIPT_PATH}/008-update_db_records.sql") + +# Get the file date +[[ $DB_FILE =~ ([0-9]{8}) ]] +# Set date for post step +POST_STEP=${POST_STEP/_filedate_/"'${BASH_REMATCH[1]}'"} + +# Import table into database +raster2pgsql ${IMPORT_OPTIONS} -M -Y -t 32x32 \ + ${SOURCE_FILE} ${TABLE} | \ + psql ${DB_CONNECT_OPTIONS} + +# Update imported rows +psql ${DB_CONNECT_OPTIONS} -c "${POST_STEP}" + +# Table maintenance after bigger import +psql ${DB_CONNECT_OPTIONS} -c "VACUUM FULL ${TABLE}" diff --git a/dbeaver/data_import/008-update_db_records.sql b/dbeaver/data_import/008-update_db_records.sql new file mode 100644 index 0000000..df637af --- /dev/null +++ b/dbeaver/data_import/008-update_db_records.sql @@ -0,0 +1,7 @@ +-- Update recently imported records based on filename +UPDATE __tablename__ + SET swe_date = to_date( + _filedate_, + 'YYYYMMDD' + ) + WHERE swe_date is NULL; diff --git a/dbeaver/data_import/008-update_db_table.sql b/dbeaver/data_import/008-update_db_table.sql new file mode 100644 index 0000000..9efde8c --- /dev/null +++ b/dbeaver/data_import/008-update_db_table.sql @@ -0,0 +1,4 @@ +ALTER TABLE __tablename__ + ADD swe_date date; + +CREATE INDEX ON __tablename__(swe_date); From 83fb6c16ea0304e97b050025ee0274653e87d176 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Wed, 26 Feb 2025 08:43:59 -0600 Subject: [PATCH 3/3] DB scripts - Fix missing '!' with shebang line --- dbeaver/data_import/005-aso_import.sh | 8 ++++---- dbeaver/data_import/006-isnobal.sh | 2 +- dbeaver/data_import/008-import_dataset.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbeaver/data_import/005-aso_import.sh b/dbeaver/data_import/005-aso_import.sh index 7ef302d..1a97fbf 100755 --- a/dbeaver/data_import/005-aso_import.sh +++ b/dbeaver/data_import/005-aso_import.sh @@ -1,5 +1,5 @@ -#/usr/bin/env bash -# +#!/usr/bin/env bash +# # Import ASO SWE tifs # # Arguments: @@ -13,8 +13,8 @@ # # ASO UC extent for all flights in EPSG:32613 # -te 231453.000 4530200.00 446853.203 4129449.623 -# Converted with: cs2cs --only-best -f "%.8f" EPSG:32613 EPSG:4269 -# -te -108.18706066 40.87885627 -105.59976043 37.31016729 +# Converted with: cs2cs --only-best -f "%.8f" EPSG:32613 EPSG:4269 +# -te -108.18706066 40.87885627 -105.59976043 37.31016729 set -e diff --git a/dbeaver/data_import/006-isnobal.sh b/dbeaver/data_import/006-isnobal.sh index eebd081..12e5045 100755 --- a/dbeaver/data_import/006-isnobal.sh +++ b/dbeaver/data_import/006-isnobal.sh @@ -1,4 +1,4 @@ -#/usr/bin/env bash +#!/usr/bin/env bash # # Import snow.nc file from iSnobal and store in the DB. # diff --git a/dbeaver/data_import/008-import_dataset.sh b/dbeaver/data_import/008-import_dataset.sh index 636c4d4..99e7353 100755 --- a/dbeaver/data_import/008-import_dataset.sh +++ b/dbeaver/data_import/008-import_dataset.sh @@ -1,4 +1,4 @@ -#/usr/bin/env bash +#!/usr/bin/env bash # # Import snow.nc file from iSnobal and store in the DB. #