Create output directory for data files in multiple scripts, fix other slurm issues

Author: JohnPalmer

code/get_forecast_data.R

@@ -179,6 +179,9 @@ if(length(all_forecasts) > 0) {
   cat("Sample data:\n")
   print(head(final_data, 3))
 
+  # Ensure output directory exists
+  dir.create("data/output", recursive = TRUE, showWarnings = FALSE)
+  
   # Save the data
   output_file = paste0("data/output/municipal_forecasts_", Sys.Date(), ".csv")
   write.csv(final_data, output_file, row.names = FALSE)

priority_municipal_data.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-# filepath: /Users/palmer/research/weather-data-collector-spain/priority_municipal_data.sh
 #SBATCH --job-name=municipal-priority
 #SBATCH --partition=ceab
 #SBATCH --nodes=1
@@ -10,9 +9,7 @@
 #SBATCH --mail-type=BEGIN,END,FAIL
 #SBATCH [email protected]
 #SBATCH --output=logs/municipal_priority_%j.out
-#SBATCH --error=logs/municipal_priority_%j.err filepath: /Users/palmer/research/weather-data-collector-spain/priority_municipal_data.sh
-#SBATCH --job-name=municipal-priority
-#SBATCH --partition=standard
+#SBATCH --error=logs/municipal_priority_%j.err
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
 #SBATCH --cpus-per-task=1
@@ -48,6 +45,9 @@ cd ~/research/weather-data-collector-spain
 # Create logs directory
 mkdir -p logs
 
+# Create output directory
+mkdir -p data/output
+
 # Activate renv
 R --slave --no-restore --file=- <<EOF
 renv::activate()

priority_municipal_forecast.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-# filepath: /Users/palmer/research/weather-data-collector-spain/priority_municipal_forecast.sh
 #SBATCH --job-name=municipal-forecast-priority
 #SBATCH --partition=ceab
 #SBATCH --nodes=1
@@ -37,6 +36,9 @@ cd ~/research/weather-data-collector-spain
 # Create logs directory
 mkdir -p logs
 
+# Create output directory
+mkdir -p data/output
+
 # Initialize status reporting
 JOB_NAME="municipal-forecast-priority"
 STATUS_SCRIPT="./scripts/update_weather_status.sh"

update_historical_weather.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-# filepath: /Users/palmer/research/weather-data-collector-spain/update_historical_weather.sh
 #SBATCH --job-name=historical-weather
 #SBATCH --partition=ceab
 #SBATCH --nodes=1
@@ -37,6 +36,9 @@ cd ~/research/weather-data-collector-spain
 # Create logs directory
 mkdir -p logs
 
+# Create output directory
+mkdir -p data/output
+
 # Activate renv
 R --slave --no-restore --file=- <<EOF
 renv::activate()

update_weather.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-# filepath: /Users/palmer/research/weather-data-collector-spain/update_weather.sh
 #SBATCH --job-name=weather-collect
 #SBATCH --partition=ceab
 #SBATCH --nodes=1
@@ -38,6 +37,9 @@ cd ~/research/weather-data-collector-spain
 # Create logs directory if it doesn't exist
 mkdir -p logs
 
+# Create output directory if it doesn't exist
+mkdir -p data/output
+
 # Initialize status reporting
 JOB_NAME="weather-data-collector"
 STATUS_SCRIPT="./scripts/update_weather_status.sh"