add commercial vignette

Author: Stephanie Owen

R/get_commercial_data.R

@@ -34,11 +34,11 @@
 #' @export
 #' 
 
-library(ROracle)
-library(DBI)
-library(fredr)
-library(tidyverse)
-library(DescTools) # For Winsorize
+# library(ROracle)
+# library(DBI)
+# library(fredr)
+# library(tidyverse)
+# library(DescTools) # For Winsorize
 
 get_commercial_data <- function(
     ora_id,
@@ -62,6 +62,7 @@ get_commercial_data <- function(
   #source("//nefscdata/SOE_ESP_Data/ESPs/connect_socioeco_oracle.R")
   source(here::here('data-raw/scripts/connect_socioeco_oracle.R'))
 
+  
   # Using the name consistent with your loop
   nefscusers.connect.string <- paste0(
     "(DESCRIPTION=",
@@ -70,19 +71,19 @@ get_commercial_data <- function(
   )
 
   # 3. Connect ONCE outside the loop
-  drv <- dbDriver("Oracle")
-  conn <- dbConnect(drv, ora_id, password = oraprod_pw, dbname = nefscusers.connect.string)
+  drv <- DBI::dbDriver("Oracle")
+  conn <- DBI::dbConnect(drv, ora_id, password = oraprod_pw, dbname = nefscusers.connect.string)
 
   ##### 4. Price deflator 
 
-  gdp_deflator <- fredr(
+  gdp_deflator <- fredr::fredr(
     series_id = "GDPDEF",
     observation_start = as.Date(paste0(START.YEAR, "-01-01")),
     observation_end = as.Date(paste0(END.YEAR, "-12-31")),
     frequency = "a" # Annual
-  ) %>%
-    mutate(YEAR = as.numeric(format(date, "%Y"))) %>%
-    select(YEAR, GDPDEF = value)
+  ) |>
+    dplyr::mutate(YEAR = as.numeric(format(date, "%Y"))) |>
+    dplyr::select(YEAR, GDPDEF = value)
 
 
   # 1. Build the query string using your R variables
@@ -97,22 +98,22 @@ get_commercial_data <- function(
   )
 
   # 2. Execute the query
-  landings_data <- dbGetQuery(conn, query_landings)
+  landings_data <- DBI::dbGetQuery(conn, query_landings)
 
   # 3. View the result
   print(head(landings_data))
 
 
   # Standardize the data frame to match the required indicator format
-  landings_final <- landings_data %>%
-    mutate(
+  landings_final <- landings_data |>
+    dplyr::mutate(
       CATEGORY = "Commercial",
       INDICATOR_NAME = paste0("Commercial_", spp_name, "_Landings_LBS"),
       INDICATOR_TYPE = "Socioeconomic"
-    ) %>%
+    ) |>
     # Rename the sum column to DATA_VALUE (Equivalent to Stata rename)
-    rename(DATA_VALUE = !!paste0("TOTAL_", spp_name)) %>%
-    select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
+    dplyr::rename(DATA_VALUE = !!paste0("TOTAL_", spp_name)) |>
+    dplyr::select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
 
   #
 
@@ -130,20 +131,20 @@ get_commercial_data <- function(
   )
 
   # 2. Execute the query
-  Nvessels_data <- dbGetQuery(conn, query_Nvessels)
+  Nvessels_data <- DBI::dbGetQuery(conn, query_Nvessels)
 
   # 3. Format to match Stata indicators
-  Nvessels_final <- Nvessels_data %>%
-    mutate(
+  Nvessels_final <- Nvessels_data |>
+    dplyr::mutate(
       CATEGORY = "Commercial",
       # This creates the long name you want in the final table
       INDICATOR_NAME = paste0("N_Commercial_Vessels_Landing_", spp_name),
       INDICATOR_TYPE = "Socioeconomic"
-    ) %>%
+    ) |>
     # Now this rename will work because the SQL alias matches 'N_VESSELS'
     # Note: Oracle often returns names in UPPERCASE, so we check for both.
-    rename(DATA_VALUE = any_of(c("N_VESSELS", "N_vessels"))) %>%
-    select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
+    dplyr::rename(DATA_VALUE = any_of(c("N_VESSELS", "N_vessels"))) |>
+    dplyr::select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
 
   # 4. View the result
   print(head(Nvessels_final))
@@ -158,42 +159,42 @@ get_commercial_data <- function(
     " AND YEAR BETWEEN ", START.YEAR, " AND ", END.YEAR
   )
 
-  price_raw <- dbGetQuery(conn, query_price)
+  price_raw <- DBI::dbGetQuery(conn, query_price)
 
   #--- 2. Calculate Average Annual Prices (Manual Winsorize) ---
-  price_annual <- price_raw %>%
-    mutate(price_lb = SPPVALUE / SPPLNDLB) %>%
+  price_annual <- price_raw |>
+    dplyr::mutate(price_lb = SPPVALUE / SPPLNDLB) |>
     # Remove Infinity or NA if pounds were 0
-    filter(is.finite(price_lb)) %>% 
-    group_by(YEAR) %>%
-    mutate(
+    dplyr::filter(is.finite(price_lb)) |> 
+    dplyr::group_by(YEAR) |>
+    dplyr::mutate(
       # Calculate the 1st and 99th percentiles for this year
-      p01 = quantile(price_lb, 0.01, na.rm = TRUE),
-      p99 = quantile(price_lb, 0.99, na.rm = TRUE),
+      p01 = stats::quantile(price_lb, 0.01, na.rm = TRUE),
+      p99 = stats::quantile(price_lb, 0.99, na.rm = TRUE),
       # "Squish" values outside that range (This is Winsorizing!)
-      price_lb_w = case_when(
+      price_lb_w = dplyr::case_when(
         price_lb < p01 ~ p01,
         price_lb > p99 ~ p99,
         TRUE ~ price_lb
       )
-    ) %>%
-    summarise(AVG_NOMINAL_PRICE = mean(price_lb_w, na.rm = TRUE)) %>%
-    ungroup()
+    ) |>
+    dplyr::summarise(AVG_NOMINAL_PRICE = mean(price_lb_w, na.rm = TRUE)) |>
+    dplyr::ungroup()
 
   # --- 3. Adjust for Inflation (Deflate) ---
   # (Keep this the same as before)
 
-  base_index_val <- gdp_deflator$GDPDEF[gdp_deflator$YEAR == deflate_yr]
+  base_index_val <- DBI::gdp_deflator$GDPDEF[gdp_deflator$YEAR == deflate_yr]
 
-  price_final <- price_annual %>%
-    left_join(gdp_deflator, by = "YEAR") %>%
-    mutate(
+  price_final <- price_annual |>
+    dplyr::left_join(gdp_deflator, by = "YEAR") |>
+    dplyr::mutate(
       DATA_VALUE = (AVG_NOMINAL_PRICE / GDPDEF) * base_index_val,
       CATEGORY = "Commercial",
       INDICATOR_NAME = paste0("AVGPRICE_", spp_name, "_", deflate_yr, "_DOLlb"),
       INDICATOR_TYPE = "Socioeconomic"
-    ) %>%
-    select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
+    ) |>
+    dplyr::select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
 
   # --- 4. Cleanup ---
 
@@ -216,43 +217,43 @@ get_commercial_data <- function(
     " AND YEAR BETWEEN ", START.YEAR, " AND ", END.YEAR,
     " GROUP BY YEAR ORDER BY YEAR"
   )
-  revs_raw <- dbGetQuery(conn, query_revs)
+  revs_raw <- DBI::dbGetQuery(conn, query_revs)
 
   # 2. Deflate and Format
-  revs_final <- revs_raw %>%
-    left_join(gdp_deflator, by = "YEAR") %>%
-    mutate(
+  revs_final <- revs_raw |>
+    dplyr::left_join(gdp_deflator, by = "YEAR") |>
+    dplyr::mutate(
       DATA_VALUE = (TOTAL_REV / GDPDEF) * base_index_val,
       CATEGORY = "Commercial",
       INDICATOR_NAME = paste0("TOTALANNUALREV_", spp_name, "_", deflate_yr, "Dols"),
       INDICATOR_TYPE = "Socioeconomic"
-    ) %>%
-    select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
+    ) |>
+    dplyr::select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
 
   ################## Fuel Prices ##################
 
   # 1. Pull Diesel Price from FRED with Year Range
-  fuel_raw <- fredr(
+  fuel_raw <- fredr::fredr(
     series_id = "DDFUELNYH",
     observation_start = as.Date(paste0(START.YEAR, "-01-01")),
     observation_end = as.Date(paste0(END.YEAR, "-12-31")),
     frequency = "a" 
-  ) %>%
-    mutate(YEAR = as.numeric(format(date, "%Y"))) %>%
+  ) |>
+    dplyr::mutate(YEAR = as.numeric(format(date, "%Y"))) |>
     # Stata 'drop if missing(DDFUELNYH)' equivalent:
-    filter(!is.na(value)) %>%
-    select(YEAR, DDFUELNYH = value)
+    dplyr::filter(!is.na(value)) |>
+    dplyr::select(YEAR, DDFUELNYH = value)
 
   # 2. Deflate and Format
-  fuel_final <- fuel_raw %>%
-    left_join(gdp_deflator, by = "YEAR") %>%
-    mutate(
+  fuel_final <- fuel_raw |>
+    dplyr::left_join(gdp_deflator, by = "YEAR") |>
+    dplyr::mutate(
       DATA_VALUE = (DDFUELNYH / GDPDEF) * base_index_val,
       CATEGORY = "Commercial",
       INDICATOR_NAME = paste0("AVGANNUAL_DIESEL_PRICE", deflate_yr, "dols"),
       INDICATOR_TYPE = "Socioeconomic"
-    ) %>%
-    select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
+    ) |>
+    dplyr::select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
 
   ############ Average Revenue Per Vessel ##################
 
@@ -264,23 +265,23 @@ get_commercial_data <- function(
     " AND YEAR BETWEEN ", START.YEAR, " AND ", END.YEAR,
     " GROUP BY YEAR, PERMIT"
   )
-  ves_rev_raw <- dbGetQuery(conn, query_ves_rev)
+  ves_rev_raw <- DBI::dbGetQuery(conn, query_ves_rev)
 
   # 2. Calculate Mean per Year and Deflate
-  av_ves_rev_final <- ves_rev_raw %>%
-    group_by(YEAR) %>%
-    summarise(AVG_VESSEL_REV = mean(VESSEL_TOTAL_REV, na.rm = TRUE)) %>%
-    left_join(gdp_deflator, by = "YEAR") %>%
-    mutate(
+  av_ves_rev_final <- ves_rev_raw |>
+    dplyr::group_by(YEAR) |>
+    dplyr:: summarise(AVG_VESSEL_REV = mean(VESSEL_TOTAL_REV, na.rm = TRUE)) |>
+    dplyr::left_join(gdp_deflator, by = "YEAR") |>
+    dplyr::mutate(
       DATA_VALUE = (AVG_VESSEL_REV / GDPDEF) * base_index_val,
       CATEGORY = "Commercial",
       INDICATOR_NAME = paste0("AVGVESREVperYr_", spp_name, "_", deflate_yr, "_DOLlb"),
       INDICATOR_TYPE = "Socioeconomic"
-    ) %>%
-    select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
+    ) |>
+    dplyr::select(YEAR, DATA_VALUE, CATEGORY, INDICATOR_NAME, INDICATOR_TYPE)
 
   ##disconnect from oracle 
-  dbDisconnect(conn)
+  DBI::dbDisconnect(conn)
   ################## MASTER APPEND ##################
 
   # 1. Create a list of all your final data frames
@@ -296,12 +297,12 @@ get_commercial_data <- function(
 
   # 2. Use bind_rows to stack them into one long file
   # This is identical to running 'append' multiple times in Stata
-  final_master_file <- bind_rows(indicator_list)
+  final_master_file <- dplyr::bind_rows(indicator_list)
 
   # 3. Final Quality Check (Filtering by your start/end years)
-  final_master_file <- final_master_file %>%
-    filter(YEAR >= START.YEAR & YEAR <= END.YEAR) %>%
-    arrange(INDICATOR_NAME, YEAR)
+  final_master_file <- final_master_file |>
+    dplyr::filter(YEAR >= START.YEAR & YEAR <= END.YEAR) |>
+    dplyr::arrange(INDICATOR_NAME, YEAR)
 
   # 4. Save the file (Equivalent to Stata's 'save ..., replace')
   # Use file.path to make sure the folder and filename are joined correctly

_pkgdown.yml

@@ -37,6 +37,8 @@ navbar:
          href: articles/using_mrip_data.html
       -  text: "Using {survdat} for Bottom Trawl Indicators"
          href: articles/using_survdat_data.html
+      -  text: "Pull Commercial Fisheries Data from Oracle"
+         href: articles/create-commercial-indicators.html   
     - text: "Functions"
       href: reference/index.html
     - text: "News"