Skip to content

Commit 8aa6bf4

Browse files
jpn--jpn--
committed
mp scripts
1 parent 5e88520 commit 8aa6bf4

3 files changed

Lines changed: 157 additions & 0 deletions

File tree

‎configs/resident/settings_mp_sharrow.yaml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
inherit_settings: settings.yaml
2+
3+
sharrow: require
4+
recode_pipeline_columns: True
5+
6+
multiprocess: True
7+
households_sample_size: 0
8+
num_processes: 40
9+
trace_hh_id:
10+
chunk_training_mode: disabled
11+
# chunk_size: 240_000_000_000
12+
13+
use_shadow_pricing: True
14+
15+
memory_profile: False
16+
17+
# raise error if any sub-process fails without waiting for others to complete
18+
# (Shadow pricing requires fail_fast setting in multiprocessing mode)
19+
fail_fast: True
20+
21+
resume_after:
22+
23+
models:
24+
### mp_init_proto_pop (single process)
25+
- initialize_proto_population # Separate step so proto tables can be split for multiprocess.
26+
### mp_disaggregate_accessibility
27+
- compute_disaggregate_accessibility
28+
### mp_initialize_hhs (single process)
29+
- initialize_landuse
30+
- initialize_households
31+
### mp_accessibility
32+
- compute_accessibility
33+
### mp_households
34+
- av_ownership
35+
- auto_ownership_simulate
36+
- work_from_home
37+
- external_worker_identification
38+
- external_workplace_location
39+
- school_location
40+
- workplace_location
41+
- transit_pass_subsidy
42+
- transit_pass_ownership
43+
- vehicle_type_choice
44+
- adjust_auto_operating_cost
45+
- transponder_ownership
46+
- free_parking
47+
- telecommute_frequency
48+
- telecommute_status
49+
- cdap_simulate
50+
- mandatory_tour_frequency
51+
- mandatory_tour_scheduling
52+
- school_escorting
53+
- joint_tour_frequency_composition
54+
- external_joint_tour_identification
55+
- joint_tour_participation
56+
- joint_tour_destination
57+
- external_joint_tour_destination
58+
- joint_tour_scheduling
59+
- non_mandatory_tour_frequency
60+
- external_non_mandatory_identification
61+
- non_mandatory_tour_destination
62+
- external_non_mandatory_destination
63+
- non_mandatory_tour_scheduling
64+
- vehicle_allocation
65+
- tour_mode_choice_simulate
66+
- atwork_subtour_frequency
67+
- atwork_subtour_destination
68+
- atwork_subtour_scheduling
69+
- atwork_subtour_mode_choice
70+
- stop_frequency
71+
- trip_purpose
72+
- trip_destination
73+
- trip_purpose_and_destination
74+
- trip_scheduling
75+
- trip_mode_choice
76+
- parking_location
77+
### mp_summarize (single process)
78+
- write_data_dictionary
79+
- track_skim_usage
80+
- write_trip_matrices
81+
- write_tables
82+
83+
84+
multiprocess_steps:
85+
- name: mp_init_proto_pop
86+
begin: initialize_proto_population
87+
- name: mp_disaggregate_accessibility
88+
num_processes: 20
89+
begin: compute_disaggregate_accessibility
90+
slice:
91+
tables:
92+
- proto_households
93+
- proto_persons
94+
- proto_tours
95+
- name: mp_initialize_hhs
96+
begin: initialize_landuse
97+
- name: mp_accessibility
98+
begin: compute_accessibility
99+
num_processes: 10
100+
slice:
101+
tables:
102+
- accessibility
103+
exclude: True # this is needed so landuse (i.e. destinations) doesn't get split
104+
- name: mp_households
105+
begin: av_ownership
106+
slice:
107+
tables:
108+
- households
109+
- persons
110+
- name: mp_summarize
111+
begin: write_data_dictionary

‎scripts/run-large-legacy-mp.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env bash
2+
3+
# exit on error, undefined variable, and fail on pipe errors
4+
set -euxo pipefail
5+
6+
# set the directory to the location of this script
7+
cd "$(dirname "$0")/.."
8+
9+
# set output directory to name of script without .sh extension
10+
OUTPUT_DIR="output-$(basename "$0" .sh)"
11+
12+
# run activitysim with multiprocessing settings
13+
uv run --project .. activitysim run \
14+
-c configs/common \
15+
-c configs/resident \
16+
-d data-full \
17+
-o scripts/${OUTPUT_DIR} \
18+
-s settings_mp.yaml \
19+
--ext extensions \
20+
--multiprocess 8 \
21+
--persist-sharrow-cache \
22+
--households_sample_size 500000
23+

‎scripts/run-large-sharrow-mp.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env bash
2+
3+
# exit on error, undefined variable, and fail on pipe errors
4+
set -euxo pipefail
5+
6+
# set the directory to the location of this script
7+
cd "$(dirname "$0")/.."
8+
9+
# set output directory to name of script without .sh extension
10+
OUTPUT_DIR="output-$(basename "$0" .sh)"
11+
12+
# run activitysim with multiprocessing settings
13+
uv run --project .. activitysim run \
14+
-c configs/common \
15+
-c configs/resident \
16+
-d data-full \
17+
-o scripts/${OUTPUT_DIR} \
18+
-s settings_mp_sharrow.yaml \
19+
--ext extensions \
20+
--multiprocess 8 \
21+
--persist-sharrow-cache \
22+
--households_sample_size 500000
23+

0 commit comments

Comments
 (0)