Currently, the output directory creation step occuers near the beginning of the process. importantly, prior to the pre-flight checks and data scraping, meaning that if any of those fail, the process will create an empty directory and then error out, leaving it empty.
Suggest moving output directory creation to later in the process, just prior to when we start writing files.
|
if (dir.exists(config[["data_prep_outputs_path"]])) { |
|
logger::log_warn("POTENTIAL DATA LOSS: Output directory already exists, and files may be overwritten ({config[[\"data_prep_outputs_path\"]]}).") |
|
warning("Output directory exists. Files may be overwritten.") |
|
} else { |
|
logger::log_trace("Creating output directory: \"{config[[\"data_prep_outputs_path\"]]}\"") |
|
dir.create(config[["data_prep_outputs_path"]], recursive = TRUE) |
|
} |
|
logger::log_info("Fetching pre-flight data.") |
|
|
|
if (config[["update_currencies"]]) { |
|
logger::log_info("Fetching currency data.") |
|
input_filepaths <- c( |
|
input_filepaths, |
|
currencies_preflight_data_path = currencies_preflight_data_path |
|
) |
|
currencies <- pacta.data.scraping::get_currency_exchange_rates( |
|
quarter = config[["imf_quarter_timestamp"]] |
|
) |
|
saveRDS(currencies, currencies_preflight_data_path) |
|
} else { |
|
logger::log_info("Using pre-existing currency data.") |
|
# This requires the preflight path to be defined in the config |
|
currencies <- readRDS(currencies_preflight_data_path) |
|
} |
|
|
|
logger::log_info("Scraping index regions.") |
|
input_filepaths <- c( |
|
input_filepaths, |
|
index_regions_preflight_data_path = index_regions_preflight_data_path |
|
) |
|
index_regions <- pacta.data.scraping::get_index_regions() |
|
saveRDS(index_regions, index_regions_preflight_data_path) |
|
|
|
logger::log_info("Fetching pre-flight data done.") |
AB#10858
