diff --git a/dbeaver/data_import/005-aso_import.sh b/dbeaver/data_import/005-aso_import.sh index 7ef302d..1a97fbf 100755 --- a/dbeaver/data_import/005-aso_import.sh +++ b/dbeaver/data_import/005-aso_import.sh @@ -1,5 +1,5 @@ -#/usr/bin/env bash -# +#!/usr/bin/env bash +# # Import ASO SWE tifs # # Arguments: @@ -13,8 +13,8 @@ # # ASO UC extent for all flights in EPSG:32613 # -te 231453.000 4530200.00 446853.203 4129449.623 -# Converted with: cs2cs --only-best -f "%.8f" EPSG:32613 EPSG:4269 -# -te -108.18706066 40.87885627 -105.59976043 37.31016729 +# Converted with: cs2cs --only-best -f "%.8f" EPSG:32613 EPSG:4269 +# -te -108.18706066 40.87885627 -105.59976043 37.31016729 set -e diff --git a/dbeaver/data_import/006-isnobal.sh b/dbeaver/data_import/006-isnobal.sh index eebd081..12e5045 100755 --- a/dbeaver/data_import/006-isnobal.sh +++ b/dbeaver/data_import/006-isnobal.sh @@ -1,4 +1,4 @@ -#/usr/bin/env bash +#!/usr/bin/env bash # # Import snow.nc file from iSnobal and store in the DB. # diff --git a/dbeaver/data_import/008-import_dataset.sh b/dbeaver/data_import/008-import_dataset.sh new file mode 100755 index 0000000..99e7353 --- /dev/null +++ b/dbeaver/data_import/008-import_dataset.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Import snow.nc file from iSnobal and store in the DB. +# +# Arguments: +# -s: Path to file to import +# -t: Table name to import data in +# -d: Date of imported data +# -c: Create table and import records +# -a: Append records to table (Default) +# +# Example call for initial import: +# import_dataset.sh -s _path_to_file_ -t table_name -c -d 20240101_ +# +# Requires: +# * Configured ~/.pg_service.conf file with host, port, user, dbname + +# -f = Prevent glob expansion when importing multiple files +set -ef + +SCRIPT_PATH="$(dirname "$(realpath $0)")" +DB_CONNECT_OPTIONS='service=swe_db' + +# Parse script options +source ${SCRIPT_PATH}/import_script_options.sh + +if [[ "$IMPORT_MODE" == "$APPEND_RECORDS" ]]; then + IMPORT_OPTIONS="-a" +elif [[ "$IMPORT_MODE" == "$CREATE_TABLE" ]]; then + IMPORT_OPTIONS="-d -C -x -I -l 2,3" + # Add post create table steps + POST_STEP=$(sed "s/__tablename__/${TABLE}/g" "${SCRIPT_PATH}/008-update_db_table.sql") +fi +# Add post steps for new records +POST_STEP+=$(sed "s/__tablename__/${TABLE}/g" "${SCRIPT_PATH}/008-update_db_records.sql") + +# Get the file date +[[ $DB_FILE =~ ([0-9]{8}) ]] +# Set date for post step +POST_STEP=${POST_STEP/_filedate_/"'${BASH_REMATCH[1]}'"} + +# Import table into database +raster2pgsql ${IMPORT_OPTIONS} -M -Y -t 32x32 \ + ${SOURCE_FILE} ${TABLE} | \ + psql ${DB_CONNECT_OPTIONS} + +# Update imported rows +psql ${DB_CONNECT_OPTIONS} -c "${POST_STEP}" + +# Table maintenance after bigger import +psql ${DB_CONNECT_OPTIONS} -c "VACUUM FULL ${TABLE}" diff --git a/dbeaver/data_import/008-update_db_records.sql b/dbeaver/data_import/008-update_db_records.sql new file mode 100644 index 0000000..df637af --- /dev/null +++ b/dbeaver/data_import/008-update_db_records.sql @@ -0,0 +1,7 @@ +-- Update recently imported records based on filename +UPDATE __tablename__ + SET swe_date = to_date( + _filedate_, + 'YYYYMMDD' + ) + WHERE swe_date is NULL; diff --git a/dbeaver/data_import/008-update_db_table.sql b/dbeaver/data_import/008-update_db_table.sql new file mode 100644 index 0000000..9efde8c --- /dev/null +++ b/dbeaver/data_import/008-update_db_table.sql @@ -0,0 +1,4 @@ +ALTER TABLE __tablename__ + ADD swe_date date; + +CREATE INDEX ON __tablename__(swe_date); diff --git a/dbeaver/data_import/import_script_options.sh b/dbeaver/data_import/import_script_options.sh index dc85007..9a577d3 100644 --- a/dbeaver/data_import/import_script_options.sh +++ b/dbeaver/data_import/import_script_options.sh @@ -19,8 +19,8 @@ IMPORT_MODE=$APPEND_RECORDS DB_FILE_PATTERN="^([0-9]{8}_).*" # Script argument options -LONG_OPTIONS=source:db-file:create,append -SHORT_OPTIONS=s:d:ca +LONG_OPTIONS=source:db-file:table:create,append +SHORT_OPTIONS=s:d:t:ca # Parse arguments and set variables ARGUMENTS=$(getopt --options=${SHORT_OPTIONS} --longoptions=${LONG_OPTIONS} --name $0 -- "$@") || exit 1 @@ -44,6 +44,10 @@ while true; do DB_FILE="$2" shift 2 ;; + -t|--table) + TABLE="$2" + shift 2 + ;; --) shift break