devops: Fix psql \copy paths for Docker and add DOCKER_PSQL flag

- Use relative paths in sample scripts so \copy works both with local
  psql and Docker psql (which runs with -w /statbus)
- Add DOCKER_PSQL=1 flag to force Docker psql for testing
- Accept both DEBUG=1 and DEBUG=true for consistency

Author: jhf

devops/manage-statbus.sh

@@ -2,9 +2,9 @@
 # devops/manage-statbus.sh
 set -euo pipefail # Exit on error, unbound variable, or any failure in a pipeline
 
-# Check for DEBUG environment variable
-if [ "${DEBUG:-}" = "true" ]; then
-  set -x # Print all commands before running them if DEBUG is true
+# Check for DEBUG environment variable (accepts "true" or "1")
+if [ "${DEBUG:-}" = "true" ] || [ "${DEBUG:-}" = "1" ]; then
+  set -x # Print all commands before running them if DEBUG is enabled
 fi
 
 # Ensure Homebrew environment is set up for tools like 'shards'
@@ -328,7 +328,7 @@ case "$action" in
         done
 
         debug_arg=""
-        if [ "${DEBUG:-}" = "true" ]; then
+        if [ "${DEBUG:-}" = "true" ] || [ "${DEBUG:-}" = "1" ]; then
           debug_arg="--debug"
         fi
 
@@ -1069,7 +1069,8 @@ EOS
         eval $(./devops/manage-statbus.sh postgres-variables)
         # The local psql is always tried first, as it has access to files
         # used for copying in data.
-        if $(which psql > /dev/null); then
+        # Set DOCKER_PSQL=1 to force using Docker psql (useful for testing).
+        if [ -z "${DOCKER_PSQL:-}" ] && $(which psql > /dev/null); then
           psql "$@"
         else
           if test -t 0 && test -t 1 && test ! -p /dev/stdin && test ! -f /dev/stdin; then

samples/demo/hierarchical/import-hierarchical-demo-data.sh

@@ -38,16 +38,18 @@ echo "Creating import jobs for hierarchical demo data"
 $WORKSPACE/devops/manage-statbus.sh psql -v USER_EMAIL="${USER_EMAIL}" -f samples/demo/hierarchical/import-hierarchical-demo-data.sql
 
 echo "Loading data into import tables"
+# Note: Use relative paths for \copy to work both locally and in Docker
+# (Docker psql runs with -w /statbus, so relative paths resolve correctly)
 
 # Load legal units with hierarchical identifiers
 # - Uganda units: census_ident (4-level)
 # - Morocco units: judicial_ident (2-level)
 echo "Loading legal units with hierarchical identifiers"
-$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_hierarchical_demo_lu_current_upload(tax_ident,stat_ident,name,birth_date,physical_region_code,physical_country_iso_2,primary_activity_category_code,legal_form_code,sector_code,employees,turnover,data_source_code,census_ident_census,census_ident_region,census_ident_surveyor,census_ident_unit_no,judicial_ident_court,judicial_ident_unit_no) FROM '$WORKSPACE/samples/demo/hierarchical/legal_units_hierarchical_demo.csv' WITH CSV HEADER;"
+$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_hierarchical_demo_lu_current_upload(tax_ident,stat_ident,name,birth_date,physical_region_code,physical_country_iso_2,primary_activity_category_code,legal_form_code,sector_code,employees,turnover,data_source_code,census_ident_census,census_ident_region,census_ident_surveyor,census_ident_unit_no,judicial_ident_court,judicial_ident_unit_no) FROM 'samples/demo/hierarchical/legal_units_hierarchical_demo.csv' WITH CSV HEADER;"
 
 # Load formal establishments with hierarchical identifiers
 echo "Loading formal establishments with hierarchical identifiers"
-$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_hierarchical_demo_es_for_lu_current_upload(tax_ident,stat_ident,name,physical_region_code,physical_country_iso_2,primary_activity_category_code,employees,turnover,legal_unit_tax_ident,data_source_code,census_ident_census,census_ident_region,census_ident_surveyor,census_ident_unit_no,judicial_ident_court,judicial_ident_unit_no) FROM '$WORKSPACE/samples/demo/hierarchical/formal_establishments_hierarchical_demo.csv' WITH CSV HEADER;"
+$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_hierarchical_demo_es_for_lu_current_upload(tax_ident,stat_ident,name,physical_region_code,physical_country_iso_2,primary_activity_category_code,employees,turnover,legal_unit_tax_ident,data_source_code,census_ident_census,census_ident_region,census_ident_surveyor,census_ident_unit_no,judicial_ident_court,judicial_ident_unit_no) FROM 'samples/demo/hierarchical/formal_establishments_hierarchical_demo.csv' WITH CSV HEADER;"
 
 echo "Checking import job states"
 $WORKSPACE/devops/manage-statbus.sh psql -c "SELECT slug, state FROM public.import_job WHERE slug LIKE 'import_hierarchical_demo_%' ORDER BY slug;"

samples/demo/import-demo-data-with-source-dates.sh

@@ -34,16 +34,18 @@ $WORKSPACE/devops/manage-statbus.sh psql -v USER_EMAIL="${USER_EMAIL}" -f sample
 
 echo "Loading data into import tables"
 # Load legal units data
+# Note: Use relative paths for \copy to work both locally and in Docker
+# (Docker psql runs with -w /statbus, so relative paths resolve correctly)
 echo "Loading legal units with source dates demo data"
-$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_lu_wsd_upload(tax_ident,stat_ident,name,valid_from,physical_address_part1,valid_to,postal_address_part1,postal_address_part2,physical_address_part2,physical_postcode,postal_postcode,physical_address_part3,physical_postplace,postal_address_part3,postal_postplace,phone_number,landline,mobile_number,fax_number,web_address,email_address,secondary_activity_category_code,physical_latitude,physical_longitude,physical_altitude,birth_date,physical_region_code,postal_country_iso_2,physical_country_iso_2,primary_activity_category_code,legal_form_code,sector_code,employees,turnover,data_source_code,status_code,unit_size_code) FROM '$WORKSPACE/app/public/demo/legal_units_with_source_dates_demo.csv' WITH CSV HEADER;"
+$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_lu_wsd_upload(tax_ident,stat_ident,name,valid_from,physical_address_part1,valid_to,postal_address_part1,postal_address_part2,physical_address_part2,physical_postcode,postal_postcode,physical_address_part3,physical_postplace,postal_address_part3,postal_postplace,phone_number,landline,mobile_number,fax_number,web_address,email_address,secondary_activity_category_code,physical_latitude,physical_longitude,physical_altitude,birth_date,physical_region_code,postal_country_iso_2,physical_country_iso_2,primary_activity_category_code,legal_form_code,sector_code,employees,turnover,data_source_code,status_code,unit_size_code) FROM 'app/public/demo/legal_units_with_source_dates_demo.csv' WITH CSV HEADER;"
 
 # Load formal establishments data
 echo "Loading formal establishments with source dates demo data"
-$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_es_for_lu_wsd_upload(tax_ident,stat_ident,name,physical_region_code,valid_from,valid_to,postal_country_iso_2,physical_country_iso_2,primary_activity_category_code,secondary_activity_category_code,employees,turnover,legal_unit_tax_ident,data_source_code,physical_address_part1,physical_address_part2,physical_address_part3,postal_address_part1,postal_address_part2,postal_address_part3,phone_number,mobile_number,landline,fax_number,web_address,email_address,physical_latitude,physical_longitude,physical_altitude,birth_date,unit_size_code,status_code) FROM '$WORKSPACE/app/public/demo/formal_establishments_units_with_source_dates_demo.csv' WITH CSV HEADER;"
+$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_es_for_lu_wsd_upload(tax_ident,stat_ident,name,physical_region_code,valid_from,valid_to,postal_country_iso_2,physical_country_iso_2,primary_activity_category_code,secondary_activity_category_code,employees,turnover,legal_unit_tax_ident,data_source_code,physical_address_part1,physical_address_part2,physical_address_part3,postal_address_part1,postal_address_part2,postal_address_part3,phone_number,mobile_number,landline,fax_number,web_address,email_address,physical_latitude,physical_longitude,physical_altitude,birth_date,unit_size_code,status_code) FROM 'app/public/demo/formal_establishments_units_with_source_dates_demo.csv' WITH CSV HEADER;"
 
 # Load informal establishments data
 echo "Loading informal establishments with source dates demo data"
-$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_es_without_lu_wsd_upload(tax_ident,stat_ident,name,physical_region_code,valid_from,valid_to,physical_country_iso_2,postal_country_iso_2,primary_activity_category_code,secondary_activity_category_code,employees,turnover,data_source_code,physical_address_part1,physical_address_part2,physical_address_part3,postal_address_part1,postal_address_part2,postal_address_part3,phone_number,mobile_number,landline,fax_number,web_address,email_address,unit_size_code,status_code,physical_latitude,physical_longitude,physical_altitude) FROM '$WORKSPACE/app/public/demo/informal_establishments_units_with_source_dates_demo.csv' WITH CSV HEADER;"
+$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_es_without_lu_wsd_upload(tax_ident,stat_ident,name,physical_region_code,valid_from,valid_to,physical_country_iso_2,postal_country_iso_2,primary_activity_category_code,secondary_activity_category_code,employees,turnover,data_source_code,physical_address_part1,physical_address_part2,physical_address_part3,postal_address_part1,postal_address_part2,postal_address_part3,phone_number,mobile_number,landline,fax_number,web_address,email_address,unit_size_code,status_code,physical_latitude,physical_longitude,physical_altitude) FROM 'app/public/demo/informal_establishments_units_with_source_dates_demo.csv' WITH CSV HEADER;"
 
 echo "Checking import job states"
 $WORKSPACE/devops/manage-statbus.sh psql -c "SELECT slug, state FROM public.import_job WHERE slug LIKE 'import_demo_%_wsd' ORDER BY slug;"

samples/demo/import-demo-data.sh

@@ -33,17 +33,19 @@ echo "Creating import jobs for demo data"
 $WORKSPACE/devops/manage-statbus.sh psql -v USER_EMAIL="${USER_EMAIL}" -f samples/demo/import-demo-data.sql
 
 echo "Loading data into import tables"
+# Note: Use relative paths for \copy to work both locally and in Docker
+# (Docker psql runs with -w /statbus, so relative paths resolve correctly)
 # Load legal units data
 echo "Loading legal units demo data"
-$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_lu_current_upload(tax_ident,stat_ident,name,birth_date,physical_region_code,physical_country_iso_2,primary_activity_category_code,legal_form_code,sector_code,employees,turnover,data_source_code) FROM '$WORKSPACE/app/public/demo/legal_units_demo.csv' WITH CSV HEADER;"
+$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_lu_current_upload(tax_ident,stat_ident,name,birth_date,physical_region_code,physical_country_iso_2,primary_activity_category_code,legal_form_code,sector_code,employees,turnover,data_source_code) FROM 'app/public/demo/legal_units_demo.csv' WITH CSV HEADER;"
 
 # Load formal establishments data
 echo "Loading formal establishments demo data"
-$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_es_for_lu_current_upload(tax_ident,stat_ident,name,physical_region_code,physical_country_iso_2,primary_activity_category_code,employees,turnover,legal_unit_tax_ident,data_source_code) FROM '$WORKSPACE/app/public/demo/formal_establishments_units_demo.csv' WITH CSV HEADER;"
+$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_es_for_lu_current_upload(tax_ident,stat_ident,name,physical_region_code,physical_country_iso_2,primary_activity_category_code,employees,turnover,legal_unit_tax_ident,data_source_code) FROM 'app/public/demo/formal_establishments_units_demo.csv' WITH CSV HEADER;"
 
 # Load informal establishments data
 echo "Loading informal establishments demo data"
-$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_es_without_lu_current_upload(tax_ident,stat_ident,name,physical_region_code,physical_country_iso_2,primary_activity_category_code,employees,turnover,data_source_code) FROM '$WORKSPACE/app/public/demo/informal_establishments_units_demo.csv' WITH CSV HEADER;"
+$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_demo_es_without_lu_current_upload(tax_ident,stat_ident,name,physical_region_code,physical_country_iso_2,primary_activity_category_code,employees,turnover,data_source_code) FROM 'app/public/demo/informal_establishments_units_demo.csv' WITH CSV HEADER;"
 
 echo "Checking import job states"
 $WORKSPACE/devops/manage-statbus.sh psql -c "SELECT slug, state FROM public.import_job WHERE slug LIKE 'import_demo_%_current' ORDER BY slug;"

samples/norway/brreg/brreg-import-selection.sh

@@ -71,13 +71,15 @@ FROM def
 ON CONFLICT (slug) DO NOTHING;"
 
 echo "Loading data into import tables"
+# Note: Use relative paths for \copy to work both locally and in Docker
+# (Docker psql runs with -w /statbus, so relative paths resolve correctly)
 # Load hovedenhet (legal units) data
 echo "Loading hovedenhet selection data"
-$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_hovedenhet_${YEAR}_selection_upload FROM '$WORKSPACE/samples/norway/legal_unit/enheter-selection.csv' WITH CSV HEADER;"
+$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_hovedenhet_${YEAR}_selection_upload FROM 'samples/norway/legal_unit/enheter-selection.csv' WITH CSV HEADER;"
 
 # Load underenhet (establishments) data
 echo "Loading underenhet selection data"
-$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_underenhet_${YEAR}_selection_upload FROM '$WORKSPACE/samples/norway/establishment/underenheter-selection.csv' WITH CSV HEADER;"
+$WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_underenhet_${YEAR}_selection_upload FROM 'samples/norway/establishment/underenheter-selection.csv' WITH CSV HEADER;"
 
 echo "Checking import job states"
 $WORKSPACE/devops/manage-statbus.sh psql -c "SELECT slug, state FROM public.import_job WHERE slug LIKE '%selection' ORDER BY slug;"

samples/norway/history/load.sh

@@ -35,7 +35,9 @@ echo "Adding import definitions for BRREG units"
 $WORKSPACE/devops/manage-statbus.sh psql < samples/norway/brreg/create-import-definition-hovedenhet-2024.sql
 $WORKSPACE/devops/manage-statbus.sh psql < samples/norway/brreg/create-import-definition-underenhet-2024.sql
 
-YEARS=$(ls $WORKSPACE/samples/norway/history/*-enheter.csv | sed -E 's/.*\/([0-9]{4})-enheter\.csv/\1/' | sort -u)
+# Note: Use relative paths for \copy to work both locally and in Docker
+# (Docker psql runs with -w /statbus, so relative paths resolve correctly)
+YEARS=$(ls samples/norway/history/*-enheter.csv | sed -E 's/.*\/([0-9]{4})-enheter\.csv/\1/' | sort -u)
 
 echo "Creating import jobs for each year"
 for YEAR in $YEARS; do
@@ -76,11 +78,11 @@ for YEAR in $YEARS; do
 
     # Load hovedenhet (legal units) data
     echo "Loading hovedenhet data for $YEAR"
-    $WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_hovedenhet_${YEAR}_history_upload FROM '$WORKSPACE/samples/norway/history/${YEAR}-enheter.csv' WITH CSV HEADER;"
+    $WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_hovedenhet_${YEAR}_history_upload FROM 'samples/norway/history/${YEAR}-enheter.csv' WITH CSV HEADER;"
 
     # Load underenhet (establishments) data
     echo "Loading underenhet data for $YEAR"
-    $WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_underenhet_${YEAR}_history_upload FROM '$WORKSPACE/samples/norway/history/${YEAR}-underenheter.csv' WITH CSV HEADER;"
+    $WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_underenhet_${YEAR}_history_upload FROM 'samples/norway/history/${YEAR}-underenheter.csv' WITH CSV HEADER;"
 done
 
 echo "Checking import job states"

samples/norway/small-history/load.sh

@@ -35,7 +35,9 @@ echo "Adding import definitions for BRREG units"
 $WORKSPACE/devops/manage-statbus.sh psql < samples/norway/brreg/create-import-definition-hovedenhet-2024.sql
 $WORKSPACE/devops/manage-statbus.sh psql < samples/norway/brreg/create-import-definition-underenhet-2024.sql
 
-YEARS=$(ls $WORKSPACE/samples/norway/small-history/*-enheter.csv | sed -E 's/.*\/([0-9]{4})-enheter\.csv/\1/' | sort -u)
+# Note: Use relative paths for \copy to work both locally and in Docker
+# (Docker psql runs with -w /statbus, so relative paths resolve correctly)
+YEARS=$(ls samples/norway/small-history/*-enheter.csv | sed -E 's/.*\/([0-9]{4})-enheter\.csv/\1/' | sort -u)
 
 echo "Creating import jobs for each year"
 for YEAR in $YEARS; do
@@ -76,11 +78,11 @@ for YEAR in $YEARS; do
 
     # Load hovedenhet (legal units) data
     echo "Loading hovedenhet data for $YEAR"
-    $WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_hovedenhet_${YEAR}_small_history_upload FROM '$WORKSPACE/samples/norway/small-history/${YEAR}-enheter.csv' WITH CSV HEADER;"
+    $WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_hovedenhet_${YEAR}_small_history_upload FROM 'samples/norway/small-history/${YEAR}-enheter.csv' WITH CSV HEADER;"
 
     # Load underenhet (establishments) data
     echo "Loading underenhet data for $YEAR"
-    $WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_underenhet_${YEAR}_small_history_upload FROM '$WORKSPACE/samples/norway/small-history/${YEAR}-underenheter.csv' WITH CSV HEADER;"
+    $WORKSPACE/devops/manage-statbus.sh psql -c "\copy public.import_underenhet_${YEAR}_small_history_upload FROM 'samples/norway/small-history/${YEAR}-underenheter.csv' WITH CSV HEADER;"
 done
 
 #echo "Running worker processing to process import jobs"