filing

kathsherratt · kathsherratt · commit bfcafba9b4eb · 2025-12-20T00:58:14.000Z
diff --git a/R/analysis-descriptive.R b/R/analysis-descriptive.R
@@ -1,6 +1,6 @@
 # Aim: describe interval score in terms of model structure and country target type
 # Load data:
-# source(here("R", "prep-data.R"))
+# source(here("R", "process-data.R"))
 # scores <- prep_data(scoring_scale = "log")
 library(here)
 library(dplyr)
diff --git a/R/analysis-model.R b/R/analysis-model.R
@@ -7,8 +7,8 @@ library(purrr)
 library(mgcv)
 library(gammit)
 library(gratia)
-source(here("R", "prep-data.R"))
-source(here("R", "descriptive.R"))
+source(here("R", "process-data.R"))
+source(here("R", "analysis-descriptive.R"))
 
 # --- Get data ---
 data <- prep_data(scoring_scale = "log")
diff --git a/R/plot-model-flow.R b/R/plot-model-flow.R
diff --git a/R/plot-model-results.R b/R/plot-model-results.R
@@ -3,8 +3,8 @@ library("dplyr")
 library("ggplot2")
 library("patchwork")
 library("gammit")
-source(here("R", "prep-data.R"))
-source(here("R", "descriptive.R"))
+source(here("R", "process-data.R"))
+source(here("R", "analysis-descriptive.R"))
 
 plot_models <- function(random_effects, scores, x_labels = TRUE,
                         anonymise = TRUE) {
diff --git a/R/process-data.R b/R/process-data.R
@@ -36,12 +36,12 @@ classify_models <- function(file = here("data", "model-classification.csv")) {
 # Get scores for all forecasts and add explanatory variables used:
 #   number of country targets, method classification, trend of observed incidence
 prep_data <- function(scoring_scale = "log") {
-  scores_files <- list.files(here("output"), pattern = "scores-raw-.*\\.csv")
+  scores_files <- list.files(here("data"), pattern = "scores-raw-.*\\.csv")
   names(scores_files) <- sub("scores-raw-(.*)\\..*$", "\\1", scores_files)
   # Get raw interval score
   scores_raw <- scores_files |>
     map(\(file) {
-      read_csv(here("output", file))
+      read_csv(here("data", file))
     }) |>
     bind_rows(.id = "outcome_target") |>
     filter(scale == scoring_scale)
diff --git a/R/process-score.R b/R/process-score.R
@@ -2,7 +2,8 @@ library(here)
 library(dplyr)
 library(purrr)
 library(scoringutils)
-source(here("R", "import-data.R"))
+source(here("R", "utils-data.R"))
+source(here("R", "process-data.R"))
 
 walk(c("case", "death"), \(target) {
   # Get forecasts & observations -----
diff --git a/R/utils-data.R b/R/utils-data.R
diff --git a/R/utils-metadata.R b/R/utils-metadata.R
diff --git a/README.Rmd b/README.Rmd
@@ -63,31 +63,23 @@ renv::restore()
 #### Data
 
 All the data used in the analysis is stored in the `data/` directory. It
-has been obtained from public sources. In order to re-download the data
-in the `data/` directory, use
-
-``` {r}
-## Get metadata from googlesheet; save to data/
-source(here("R", "get-metadata.R"))
-
-## Get observed data and all Hub forecasts; exclude forecasts; save to data/
-source(here("R", "import-data.R"))
-```
+has been obtained from public sources. To re-download the data
+in the `data/` directory, see the [data/README](data/README.md).
 
 #### Analyses
 
 In order to re-generate the forecast scores, use
 
 ``` {r}
 ## Score forecasts & ensembles on the log and natural scales; save to data/
-source(here("R", "score.R"))
+source(here("R", "process-score.R"))
 ```
 
 In order to run the GAM on the scores, use
 
 ``` {r}
 ## Model the weighted interval score; save to data/
-source(here("R", "model-wis.R"))
+source(here("R", "analysis-model.R"))
 ```
 
 #### Results
diff --git a/README.md b/README.md
@@ -77,31 +77,23 @@ renv::restore()
 #### Data
 
 All the data used in the analysis is stored in the `data/` directory. It
-has been obtained from public sources. In order to re-download the data
-in the `data/` directory, use
-
-``` r
-## Get metadata from googlesheet; save to data/
-source(here("R", "get-metadata.R"))
-
-## Get observed data and all Hub forecasts; exclude forecasts; save to data/
-source(here("R", "import-data.R"))
-```
+has been obtained from public sources. To re-download the data in the
+`data/` directory, see the [data/README](data/README.md).
 
 #### Analyses
 
 In order to re-generate the forecast scores, use
 
 ``` r
 ## Score forecasts & ensembles on the log and natural scales; save to data/
-source(here("R", "score.R"))
+source(here("R", "process-score.R"))
 ```
 
 In order to run the GAM on the scores, use
 
 ``` r
 ## Model the weighted interval score; save to data/
-source(here("R", "model-wis.R"))
+source(here("R", "analysis-model.R"))
 ```
 
 #### Results
diff --git a/data/README.Rmd b/data/README.Rmd
@@ -0,0 +1,77 @@
+---
+title: "README"
+output: github_document
+---
+
+### Data processing
+
+The following R scripts were used to process the data stored here:
+
+- [`R/utils-data.R`](R/utils-data.R) : Code to access forecasts, observed data, and population size for each location
+   - `populations.csv`
+   - `observed-case.csv`, `observed-death.csv`
+   - `covid19-forecast-hub-europe.parquet`
+- [`R/utils-metadata.R`](R/utils-metadata.R) : Functions to access model names, model submissions, and raw metadata from Hub GitHub repos
+  - `model-classification.csv`
+- [`R/process-score.R`](R/process-score.R) : Code to score forecasts using scoringutils
+  - `scores-raw-case.csv`, `scores-raw-death.csv`
+
+#### Metadata
+
+Code used to access and store metadata:
+
+```{r, eval=FALSE}
+library(here)
+library(purrr)
+library(lubridate)
+source(here("R", "utils-metadata.R"))
+
+# set repo paths
+repos <- c(
+  "euro" = "covid19-forecast-hub-europe/covid19-forecast-hub-europe",
+  "us" = "reichlab/covid19-forecast-hub"
+)
+
+# date range for submissions
+min_date <- "2021-03-07"
+max_date <- as.Date("2023-03-10") - weeks(4)
+
+# get euro metadata
+eu_submissions <- get_model_submissions(
+  repo = repos[["euro"]],
+  min_date = min_date,
+  max_date = max_date
+)
+eu_metadata_raw <- get_metadata_raw(
+  repo = repos[["euro"]],
+  model_abbr = unique(eu_submissions$model)
+)
+write_metadata(
+  metadata_raw = eu_metadata_raw,
+  sheet_name = "euro-metadata-raw", write_local = FALSE
+)
+
+```
+
+Code can also be used to access US Hub metadata:
+
+```{r, eval=FALSE}
+# get US metadata
+us_submissions <- get_model_submissions(
+  repo = repos[["us"]],
+  min_date = min_date,
+  max_date = max_date
+)
+us_metadata_raw <- get_metadata_raw(
+  repo = repos[["us"]],
+  model_abbr = unique(us_submissions$model)
+)
+
+us_metadata_raw <- us_metadata_raw |>
+  mutate(euro_overlap = model_abbr %in% unique(eu_submissions$model))
+
+write_metadata(
+  metadata_raw = us_metadata_raw,
+  sheet_name = "us-metadata-raw", write_local = FALSE
+)
+```
diff --git a/data/README.md b/data/README.md
@@ -1,7 +1,31 @@
+README
+================
+
+### Data processing
+
+The following R scripts were used to process the data stored here:
+
+- [`R/utils-data.R`](R/utils-data.R) : Code to access forecasts,
+  observed data, and population size for each location
+  - `populations.csv`
+  - `observed-case.csv`, `observed-death.csv`
+  - `covid19-forecast-hub-europe.parquet`
+- [`R/utils-metadata.R`](R/utils-metadata.R) : Functions to access model
+  names, model submissions, and raw metadata from Hub GitHub repos
+  - `model-classification.csv`
+- [`R/process-score.R`](R/process-score.R) : Code to score forecasts
+  using scoringutils
+  - `scores-raw-case.csv`, `scores-raw-death.csv`
+
+#### Metadata
+
+Code used to access and store metadata:
+
+``` r
 library(here)
 library(purrr)
 library(lubridate)
-source(here("R", "metadata-utils.R"))
+source(here("R", "utils-metadata.R"))
 
 # set repo paths
 repos <- c(
@@ -27,7 +51,11 @@ write_metadata(
   metadata_raw = eu_metadata_raw,
   sheet_name = "euro-metadata-raw", write_local = FALSE
 )
+```
+
+Code can also be used to access US Hub metadata:
 
+``` r
 # get US metadata
 us_submissions <- get_model_submissions(
   repo = repos[["us"]],
@@ -46,3 +74,4 @@ write_metadata(
   metadata_raw = us_metadata_raw,
   sheet_name = "us-metadata-raw", write_local = FALSE
 )
+```
diff --git a/data/scores-raw-case.csv b/data/scores-raw-case.csv
diff --git a/data/scores-raw-death.csv b/data/scores-raw-death.csv
diff --git a/report/results.Rmd b/report/results.Rmd
@@ -24,9 +24,9 @@ knitr::opts_chunk$set(
 ```
 
 ```{r, data-pipeline}
-source(here("R", "prep-data.R"))
-source(here("R", "descriptive.R"))
-source(here("R", "model-plots.R"))
+source(here("R", "process-data.R"))
+source(here("R", "analysis-descriptive.R"))
+source(here("R", "plot-model-results.R"))
 scores <- prep_data(scoring_scale = "log")
 ensemble <- scores |>
   filter(grepl("EuroCOVIDhub-ensemble", Model))
diff --git a/report/supplement/Supplement.Rmd b/report/supplement/Supplement.Rmd
@@ -52,7 +52,7 @@ Data for this work were downloaded on 30th May 2023. These data are available in
 ## Eligibility criteria
 
 ```{r model-flow, fig.cap="Eligibility criteria for models contributing case (left) and death (right) forecasts to the European COVID-19 Forecast Hub, March 2021 - March 2023"}
-# source(here("R", "model-flow.R"))
+# source(here("R", "plot-model-flow.R"))
 # flow_chart <- create_model_flow()
 knitr::include_graphics(here("plots", "flowchart.png"))
 ```
@@ -61,8 +61,8 @@ knitr::include_graphics(here("plots", "flowchart.png"))
 
 ```{r load-data}
 # Load data
-source(here("R", "prep-data.R"))
-source(here("R", "descriptive.R"))
+source(here("R", "process-data.R"))
+source(here("R", "analysis-descriptive.R"))
 scores <- prep_data(scoring_scale = "log")
 ensemble <- scores |>
   filter(grepl("EuroCOVIDhub-ensemble", Model))
@@ -126,3 +126,4 @@ knitr::include_graphics(here("plots", "check_Cases.pdf"))
 # QQ plot, residuals
 knitr::include_graphics(here("plots", "check_Deaths.pdf"))
 ```
+
