Merge branch 'main' into azure-backfill-api-v2

Author: natemcintosh

DESCRIPTION

@@ -20,7 +20,7 @@ License: Apache License (>= 2)
 Encoding: UTF-8
 Remotes:
     github::epiforecasts/EpiNow2@9b8cd4fcceca41ac34545a38989f6f295ddeeaf7,
-    github::stan-dev/cmdstanr
+    github::stan-dev/cmdstanr@da99e2ba954658bdad63bffb738c4444c33a4e0e
 Roxygen: list(markdown = TRUE)
 RoxygenNote: 7.3.2
 Suggests:
@@ -51,7 +51,9 @@ Imports:
     tidyr,
     tidybayes,
     optparse,
-    Microsoft365R
+    Microsoft365R,
+    stringr,
+    glue
 Additional_repositories:
 	https://stan-dev.r-universe.dev
 URL: https://cdcgov.github.io/cfa-epinow2-pipeline/

Makefile

@@ -3,12 +3,21 @@ IMAGE_NAME=cfa-epinow2-pipeline
 BRANCH=$(shell git branch --show-current)
 CONFIG_CONTAINER=rt-epinow2-config
 CNTR_MGR=docker
+DATA_API=v1
 ifeq ($(BRANCH), main)
 TAG=latest
 else
 TAG=$(BRANCH)
 endif
 
+ifeq ($(DATA_API),v1)
+API_CONTAINER := nssp-etl
+else ifeq ($(DATA_API),v2)
+API_CONTAINER := nssp-etl-api-v2
+else
+$(error Unknown DATA_API '$(DATA_API)'. Expected v1 or v2)
+endif
+
 CONFIG=test.json
 POOL="cfa-epinow2-$(TAG)"
 TIMESTAMP:=$(shell  date -u +"%Y%m%d_%H%M%S")
@@ -38,12 +47,14 @@ config: ## Generates a configuration file for running the model
 		--disease="COVID-19,Influenza,RSV" \
 		--state=all \
 		--output-container=nssp-rt-v2 \
+		--input-container=$(API_CONTAINER) \
 		--job-id=$(JOB) \
 		--report-date-str=$(REPORT_DATE)
 
 rerun-config: ## Generate a configuration file to rerun a previous model
 	uv run azure/generate_rerun_configs.py \
 		--output-container=nssp-rt-v2 \
+		--input-container=$(API_CONTAINER) \
 		--job-id=$(JOB) \
 		--report-date-str=$(REPORT_DATE)
 
@@ -84,6 +95,7 @@ test-batch: ## Run GitHub Actions workflow and then job.py for testing on Azure
 		--disease="COVID-19,Influenza,RSV" \
 		--state=NY \
 		--output-container=nssp-rt-testing \
+		--input-container=$(API_CONTAINER) \
 		--job-id=$(JOB) \
 		--report-date-str=$(REPORT_DATE)
 	uv run --env-file .env \

NEWS.md

@@ -1,7 +1,12 @@
 # CFAEpiNow2Pipeline v0.2.0
 
 ## Features
+<<<<<<< azure-backfill-api-v2
 * Added a PEP723 script for running backfills more easily.
+=======
+
+* Add the ability to read data from API v2 as well as v1
+>>>>>>> main
 * dependabot updating package docker/login-action from version 3 to version 4 new
 * Adding image tag validation when dependabot PRs are opened and automatic update to NEWs md
 * Add a utility script for reading, preparing, and uploading Rt review decisions

R/config.R

@@ -158,6 +158,12 @@ Data <- S7::new_class(
 #' "YYYY-MM-DD".
 #' @param output_container An optional string specifying the output blob storage
 #' container.
+#' @param facility_active_proportion A numeric value between 0 and 1 specifying
+#' the proportion of days during the modeling period that facilities must have
+#' reported at least one informative discharge diagnosis (DDI) to be included in
+#' the analysis. Default is 0.94 (require
+#' active reporting for >=53 of 56 days in the training period).
+#' Lower values allow inclusion of facilities with fewer active days.
 #' @family config
 #' @export
 Config <- S7::new_class(
@@ -190,7 +196,26 @@ Config <- S7::new_class(
     # Would add default values, but Roxygen isn't happy about them yet.
     sampler_opts = S7::class_list,
     exclusions = S7::S7_class(Exclusions()),
-    output_container = character_or_null
+    output_container = character_or_null,
+    facility_active_proportion = S7::new_property(
+      S7::class_double,
+      default = quote(0.94),
+      validator = \(value) {
+        if (
+          rlang::is_bare_numeric(value) &&
+            length(value) == 1 &&
+            value >= 0 &&
+            value <= 1
+        ) {
+          NULL
+        } else {
+          paste0(
+            "Invalid value for facility_active_proportion. ",
+            "It must be a single numeric value between 0 and 1."
+          )
+        }
+      }
+    )
   )
 )
 

R/pipeline.R

@@ -85,7 +85,7 @@ orchestrate_pipeline <- function(
       )
       read_json_into_config(
         config_path,
-        c("exclusions", "output_container")
+        c("exclusions", "output_container", "facility_active_proportion")
       )
     },
     error = function(con) {
@@ -94,6 +94,7 @@ orchestrate_pipeline <- function(
     }
   )
   if (typeof(config) == "logical") {
+    cli::cli_warn("Failed to read config", class = "Bad_config")
     return(invisible(FALSE))
   }
 
@@ -113,15 +114,19 @@ orchestrate_pipeline <- function(
   logfile_connection <- file(file.path(logfile_path, "logs.txt"), open = "wt")
   sink(
     logfile_connection,
+    # "output" means general (non-error) output
     type = "output",
     append = TRUE,
     # Send output to logs and to console
     split = TRUE
   )
   sink(
     logfile_connection,
+    # "message" means error messages
     type = "message",
     append = TRUE
+    # Unfortunately, we can't split messages to both console and log file.
+    # It will error out if we try
   )
   on.exit(sink(file = NULL))
   cli::cli_alert_info("Starting run at {Sys.time()}")
@@ -189,7 +194,8 @@ execute_model_logic <- function(config, input_dir, output_dir) {
     geo_value = config@geo_value,
     report_date = config@report_date,
     max_reference_date = config@max_reference_date,
-    min_reference_date = config@min_reference_date
+    min_reference_date = config@min_reference_date,
+    facility_active_proportion = config@facility_active_proportion
   )
 
   # rlang::is_empty() checks for empty and NULL values
@@ -289,7 +295,8 @@ execute_model_logic <- function(config, input_dir, output_dir) {
       config@exclusions@blob_storage_container
     ),
     # Add the config container here when refactoring out to outer func
-    run_at = format(Sys.time(), "%Y-%m-%dT%H:%M:%S%z")
+    run_at = format(Sys.time(), "%Y-%m-%dT%H:%M:%S%z"),
+    facility_active_proportion = config@facility_active_proportion
   )
 
   write_model_outputs(