Skip to content

Commit a6422db

Browse files
JohnPalmerJohnPalmer
committed
Merge remote-tracking branch 'origin/main'
2 parents 61dbfb4 + 165d98c commit a6422db

File tree

3 files changed

+211
-0
lines changed

3 files changed

+211
-0
lines changed

‎update_barcelona_local.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Barcelona-only pipeline for standalone hosts.
5+
6+
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
7+
cd "${REPO_ROOT}"
8+
9+
mkdir -p logs
10+
11+
echo "=== Collecting Barcelona Datasets (Local) ==="
12+
echo "Starting: $(date --utc) UTC"
13+
14+
SENTINEL_FILE="data/output/barcelona_last_success.txt"
15+
JOB_FAILED=0
16+
17+
run_step() {
18+
local description=$1
19+
shift
20+
echo "${description}..."
21+
set +e
22+
"$@"
23+
local status=$?
24+
set -e
25+
if [ ${status} -eq 0 ]; then
26+
echo "✅ ${description} completed"
27+
else
28+
echo "❌ ${description} failed"
29+
JOB_FAILED=1
30+
fi
31+
return ${status}
32+
}
33+
34+
if ! run_step "Barcelona Dataset 1: Historical daily stations" Rscript scripts/r/get_historical_data_barcelona.R --key-pool=a,b,c,d,e; then
35+
echo "Stopping Barcelona job after failure in Dataset 1"
36+
exit 1
37+
fi
38+
39+
if ! run_step "Barcelona Dataset 2: Current daily stations" Rscript scripts/r/aggregate_daily_stations_current_barcelona.R; then
40+
echo "Stopping Barcelona job after failure in Dataset 2"
41+
exit 1
42+
fi
43+
44+
if ! run_step "Barcelona Dataset 3: Hourly station ongoing" Rscript scripts/r/get_latest_data_barcelona.R --key-pool=a,b,c,d,e; then
45+
echo "Stopping Barcelona job after failure in Dataset 3"
46+
exit 1
47+
fi
48+
49+
if ! run_step "Barcelona Dataset 4: Municipal forecasts" Rscript scripts/r/get_forecast_barcelona.R --key-pool=a,b,c,d,e --elaborado-max-attempts=4 --elaborado-wait-seconds=900; then
50+
echo "Stopping Barcelona job after failure in Dataset 4"
51+
exit 1
52+
fi
53+
54+
if [ ${JOB_FAILED} -eq 0 ]; then
55+
date --utc > "${SENTINEL_FILE}"
56+
echo "Barcelona sentinel updated at $(cat "${SENTINEL_FILE}")"
57+
else
58+
echo "Barcelona job experienced failures; sentinel not updated"
59+
fi
60+
61+
echo "=== Barcelona Collection Summary ==="
62+
echo "Completed: $(date --utc) UTC"
63+
ls -la data/output/*barcelona*.csv.gz 2>/dev/null || true

‎update_municipal_forecasts_only_local.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Run only the municipal forecast collection without Slurm.
5+
6+
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
7+
cd "${REPO_ROOT}"
8+
9+
mkdir -p logs
10+
11+
KEY_POOLS=(a b c d e)
12+
SHARD_COUNT=${#KEY_POOLS[@]}
13+
14+
launch_forecast_shards() {
15+
local status=0
16+
local shard=0
17+
declare -a pids
18+
19+
for key_pool in "${KEY_POOLS[@]}"; do
20+
shard=$((shard + 1))
21+
echo "Launching shard ${shard}/${SHARD_COUNT} with key pool '${key_pool}'"
22+
(
23+
set -euo pipefail
24+
Rscript scripts/r/get_forecast_data_hybrid.R \
25+
--shard-index=${shard} \
26+
--shard-count=${SHARD_COUNT} \
27+
--key-pool=${key_pool}
28+
) &
29+
pids[${shard}]=$!
30+
done
31+
32+
for shard in "${!pids[@]}"; do
33+
if wait "${pids[${shard}]}"; then
34+
echo "✅ Forecast shard ${shard}/${SHARD_COUNT} completed"
35+
else
36+
echo "❌ Forecast shard ${shard}/${SHARD_COUNT} failed"
37+
status=1
38+
fi
39+
done
40+
41+
return ${status}
42+
}
43+
44+
if launch_forecast_shards; then
45+
echo "✅ Municipal forecasts completed"
46+
else
47+
echo "❌ Municipal forecasts encountered failures"
48+
exit 1
49+
fi
50+
51+
echo "Completed: $(date --utc) UTC"
52+
ls -la data/output/municipal_forecasts*.csv.gz 2>/dev/null || true

‎update_weather_local.sh‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Run the full weather data pipeline without Slurm. Designed for standalone hosts
5+
# such as cloud VMs or local machines where R and required system libraries are
6+
# installed beforehand.
7+
8+
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
9+
cd "${REPO_ROOT}"
10+
11+
mkdir -p logs
12+
13+
BARCELONA_SENTINEL="data/output/barcelona_last_success.txt"
14+
if [ -f "${BARCELONA_SENTINEL}" ]; then
15+
echo "Barcelona sentinel timestamp: $(cat "${BARCELONA_SENTINEL}")"
16+
else
17+
echo "Barcelona sentinel not found; ensure update_barcelona_only.sh has run recently."
18+
fi
19+
20+
echo "=== Collecting Three Datasets (Local) ==="
21+
echo "Starting: $(date --utc) UTC"
22+
23+
run_step() {
24+
local description=$1
25+
shift
26+
echo "${description}..."
27+
set +e
28+
"$@"
29+
local status=$?
30+
set -e
31+
if [ ${status} -eq 0 ]; then
32+
echo "✅ ${description} completed"
33+
else
34+
echo "❌ ${description} failed"
35+
fi
36+
return ${status}
37+
}
38+
39+
if ! run_step "Dataset 1: Historical daily stations" Rscript scripts/r/get_historical_data.R; then
40+
echo "Stopping after failure in Dataset 1"
41+
exit 1
42+
fi
43+
44+
if ! run_step "Dataset 2: Current daily stations" Rscript scripts/r/aggregate_daily_stations_current.R; then
45+
echo "Stopping after failure in Dataset 2"
46+
exit 1
47+
fi
48+
49+
if ! run_step "Dataset 3: Hourly station ongoing" Rscript scripts/r/get_latest_data.R; then
50+
echo "Stopping after failure in Dataset 3"
51+
exit 1
52+
fi
53+
54+
KEY_POOLS=(a b c d e)
55+
SHARD_COUNT=${#KEY_POOLS[@]}
56+
57+
echo "Dataset 4: Municipal forecasts (launching ${SHARD_COUNT} shards in parallel)..."
58+
59+
launch_forecast_shards() {
60+
local status=0
61+
local shard=0
62+
declare -a pids
63+
64+
for key_pool in "${KEY_POOLS[@]}"; do
65+
shard=$((shard + 1))
66+
echo " • Starting shard ${shard}/${SHARD_COUNT} with key pool '${key_pool}'"
67+
(
68+
set -euo pipefail
69+
Rscript scripts/r/get_forecast_data_hybrid.R \
70+
--shard-index=${shard} \
71+
--shard-count=${SHARD_COUNT} \
72+
--key-pool=${key_pool}
73+
) &
74+
pids[${shard}]=$!
75+
done
76+
77+
for shard in "${!pids[@]}"; do
78+
if wait "${pids[${shard}]}"; then
79+
echo "✅ Forecast shard ${shard}/${SHARD_COUNT} completed"
80+
else
81+
echo "❌ Forecast shard ${shard}/${SHARD_COUNT} failed"
82+
status=1
83+
fi
84+
done
85+
86+
return ${status}
87+
}
88+
89+
if ! launch_forecast_shards; then
90+
echo "Municipal forecast job experienced failures"
91+
exit 1
92+
fi
93+
94+
echo "=== Collection Summary ==="
95+
echo "Completed: $(date --utc) UTC"
96+
ls -la data/output/*.csv.gz 2>/dev/null || true

0 commit comments

Comments
 (0)