Merge pull request #135 from mountainMath/v0.4.3

V0.4.3

Author: mountainMath

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: cansim
 Type: Package
 Title: Accessing Statistics Canada Data Table and Vectors
-Version: 0.4.2
+Version: 0.4.3
 Authors@R: c(
     person("Jens", "von Bergmann", email = "jens@mountainmath.ca", role = c("aut","cre")),
     person("Dmitry", "Shkolnik", email = "shkolnikd@gmail.com", role = c("aut")))

NEWS.md

@@ -1,3 +1,9 @@
+# cansim 0.4.3
+## Minor changes
+* better handling of duplicated levels in metadata, ignore duplication for geography names of census tables but emit warning
+* fix issue with accessing tables without footnotes
+
+
 # cansim 0.4.2
 ## Minor changes
 * ensure proper ordering of levels even if StatCan metadata is not ordered

R/cansim.R

@@ -152,7 +152,8 @@ normalize_cansim_values <- function(data, replacement_value="val_norm", normaliz
     for (field in fields) {
       if (!is.null(getOption("cansim.debug"))) message(paste0('Converting ',field,' to factors'))
       tryCatch({
-        level_table <- get_deduped_column_level_data(cansimTableNumber = cansimTableNumber,language=language,column=field) %>%
+        level_table <- get_deduped_column_level_data(cansimTableNumber = cansimTableNumber,
+                                                     language=language,column=field) %>%
           arrange(as.integer(.data$`...dim`),as.integer(.data$`...id`))
         if (!(field %in% names(data))) {
           geography_column <- ifelse(cleaned_language=="eng","Geography|Geographic name",paste0("G",intToUtf8(0x00E9),"ographie|Nom g",intToUtf8(0x00E9),"ographique"))
@@ -171,7 +172,9 @@ normalize_cansim_values <- function(data, replacement_value="val_norm", normaliz
           column_position <- which(names(data)==field)
           column_before <- names(data)[column_position-1]
 
-          data$`...id` <- stringr::str_split(data[[coordinate_column]],"\\.") %>% lapply(\(x)x[dimension_id]) %>% unlist()
+          data$`...id` <- stringr::str_split(data[[coordinate_column]],"\\.") %>%
+            lapply(\(x)x[dimension_id]) %>%
+            unlist()
 
           data <- data %>%
             select(-all_of(field)) %>%

R/cansim_helpers.R

@@ -460,31 +460,43 @@ get_deduped_column_level_data <- function(cansimTableNumber,language,column) {
     mutate(...n=as.integer(.data$...id)) %>%
     arrange("...n") %>%
     select(-"...n") %>%
-    mutate(...count=n(),.by="...name") %>%
+    mutate(...count=n(),.by=c("...dim","...name")) %>%
     mutate(...duplicated=.data$...count>1) %>%
     mutate(...original=!.data$...duplicated) %>%
     mutate(...original_name=.data$...name) %>%
-    mutate(...name=ifelse(.data$...duplicated & is.na(.data$...parent_id),
-                          paste0(.data$...name," [",.data$...id,"]"), # deals with 36-10-0108
-                          .data$...name)) %>%
-  mutate(...count=n(),.by="...name") %>%
-    mutate(...duplicated=.data$...count>1)
+    mutate(...last_parent_id=.data$...parent_id)
+
+  fixed_level_table <- NULL
+  # don't try to dedup census geographies, too messy
+  if (substr(naked_ndm_table_number(cansimTableNumber),1,4)=="9810" && sum(filter(level_table,.data$...dim=="1")$...duplicated)>0) {
+    warning(paste0("Table ",cansimTableNumber," is a census data table that has duplicate geography names, not converting to factors. Treat with caution when accessng geographies by name and check geographic identifiers."))
+    fixed_level_table <- level_table %>%
+      filter(.data$...dim=="1")
+    level_table <- level_table %>%
+      filter(.data$...dim!="1")
+  }
 
+  # try to dedup
   max_run <- 30
   while (sum(level_table$...duplicated)>0 && max_run>0) { # deals with 36-10-0580
     max_run <- max_run - 1
     level_table <- level_table %>%
-      left_join(level_table %>% select("...id",...parent_name="...name"),
-                by=c("...parent_id"="...id")) %>%
-      mutate(...name=ifelse(.data$...duplicated,
-                            paste0(.data$...name," ==> ",.data$...parent_name),
-                            .data$...name)) %>%
-      select(-"...parent_name") %>%
-      mutate(...count=n(),.by="...name") %>%
-      mutate(...duplicated=.data$...count>1)
+      left_join(level_table %>% select("...id","...dim",...parent_name="...original_name",...new_parent_id="...last_parent_id"),
+                by=c("...last_parent_id"="...id","...dim"="...dim")) %>%
+      mutate(...name=case_when(.data$...duplicated & is.na(.data$...parent_name) ~ paste0(.data$...name," [",.data$...id,"]"),
+                               .data$...duplicated & !is.na(.data$...parent_name) ~  paste0(.data$...name," ==> ",.data$...parent_name),
+                              TRUE ~ .data$...name)) %>%
+      mutate(...last_parent_id=ifelse(.data$...duplicated,
+                                      .data$...new_parent_id,
+                                      .data$...last_parent_id)) %>%
+      mutate(...count=n(),.by=c("...dim","...name")) %>%
+      mutate(...duplicated=.data$...count>1) %>%
+      select(-any_of(c("...parent_name","...new_parent_id")))
+
   }
 
-  level_table %>%
+  bind_rows(fixed_level_table,level_table) %>%
+    arrange("...dim") %>%
     select("...dim","...id","...name","...original","...original_name")
 }
 

R/cansim_metadata.R

@@ -233,8 +233,11 @@ get_cansim_cube_metadata <- function(cansimTableNumber, type="overview",refresh=
           mutate(across(where(is.integer),as.character)) %>%
           arrange(as.integer(.data$footnoteId))
       }) %>%
-      unique() %>%
+      unique()
+    if (nrow(m3)>0) {
+      m3 <- m3 %>%
       arrange(as.integer(.data$footnoteId),as.integer(.data$dimensionPositionId),as.integer(.data$memberId))
+    }
     saveRDS(m3, meta3_path)
   } else {
     m3 <- readRDS(meta3_path)
@@ -465,7 +468,8 @@ get_cansim_series_info_cube_coord <- function(cansimTableNumber,coordinates, tim
 #' @examples
 #' \dontrun{
 #' template <- get_cansim_table_template("34-10-0013")
-#' template |> filter(Geography=="Canada") |>
+#' template |>
+#'   filter(Geography=="Canada") |>
 #'   add_cansim_vectors_to_template()
 #' }
 #' @export

R/cansim_vectors.R

@@ -466,7 +466,7 @@ get_cansim_data_for_table_coord_periods<-function(tableCoordinates, periods=NULL
       new_failed_coordinates <- NULL
       if (length(data2)>0) {
         # message(paste0("Failed to load for ",length(data2)," coordinates "))
-        new_failed_coordinates <- data2 %>% purrr::map(function(x){x$object$coordinate}) |> unlist()
+        new_failed_coordinates <- data2 %>% purrr::map(function(x){x$object$coordinate}) %>% unlist()
         new_failed_coordinates <- tibble::tibble(cansimTableNumber=unique(working_data$cansimTableNumber),
                                                  COORDINATE=new_failed_coordinates)
 

README.md

@@ -1,9 +1,10 @@
 # cansim
 
 <!-- badges: start -->
-[![R-CMD-check](https://github.com/mountainMath/cansim/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/mountainMath/cansim/actions/workflows/R-CMD-check.yaml)
 [![CRAN status](https://www.r-pkg.org/badges/version/cansim)](https://CRAN.R-project.org/package=cansim)
 [![CRAN_Downloads_Badge](https://cranlogs.r-pkg.org/badges/cansim)](https://cranlogs.r-pkg.org/badges/cansim)
+[![R-CMD-check](https://github.com/mountainMath/cansim/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/mountainMath/cansim/actions/workflows/R-CMD-check.yaml)
+[![DOI](https://img.shields.io/badge/doi-10.32614/CRAN.package.cansim-#d2b24a.svg)](https://doi.org/10.32614/CRAN.package.cansim)
 <!-- badges: end -->
 
 <a href="https://mountainmath.github.io/cansim/index.html"><img src="https://raw.githubusercontent.com/mountainMath/cansim/master/images/cansim-sticker.png" alt="cansim logo" align="right" width = "25%" height = "25%"/></a>
@@ -219,19 +220,17 @@ If you want to get in touch, we are pretty good at responding via email or via t
 
 ### Related packages
 
-* The [CANSIM2R package](https://CRAN.R-project.org/package=CANSIM2R) provides functionality to download entire StatCan tables.
-
-* The [statcanR package](https://CRAN.R-project.org/package=statcanR) provides functionality to download entire StatCan tables and basic data discovery functionality.
-
-* [CANSIM-dataviewer](https://github.com/bcgov/CANSIM-dataviewer) is another tool that depends on the existing *CANSIM2R* package with a focus on uses for the Province of British Columbia.
+* The [statcanR package](https://CRAN.R-project.org/package=statcanR) is an alternative package providing basic access to StatCan NDM tables and data discovery.
 
 * [cancensus](https://github.com/mountainMath/cancensus) is a package designed to access, retrieve, and work with Canadian Census data and geography. The *cansim* package is designed to work in conjunction with *cancensus* and data can easily be joined on standard geographic identifiers exposed and harmonized by both packages.
 
+* [cmhc](https://github.com/mountainMath/cmhc) is a package designed to access, retrieve, and work with CMHC data.
+
 ### Cite cansim
 
 If you wish to cite the `cansim` package in your work:
 
-  von Bergmann, J., Dmitry Shkolnik (2024). cansim: functions and convenience tools for accessing Statistics Canada data tables. v0.4.2. DOI: 10.32614/CRAN.package.cansim
+  von Bergmann, J., Dmitry Shkolnik (2024). cansim: functions and convenience tools for accessing Statistics Canada data tables. v0.4.3. DOI: 10.32614/CRAN.package.cansim
 
 A BibTeX entry for LaTeX users is
 
@@ -241,7 +240,7 @@ A BibTeX entry for LaTeX users is
     title = {cansim: functions and convenience tools for accessing Statistics Canada data tables},
     year = {2025},
     doi = {10.32614/CRAN.package.cansim},
-    note = {R package version 0.4.2},
+    note = {R package version 0.4.3},
     url = {https://mountainmath.github.io/cansim/}
   }
 ```

cran-comments.md

@@ -141,3 +141,8 @@ There were no ERRORs or WARNINGs or NOTEs.
 * generate table template and facilitate adding vector info to aid pinpointed data download
 * enable downloading of data by vector and multiple coordinates in get_cansim_data_for_table_coord_periods (breaking changes with change to parameter)
 
+# cansim 0.4.3
+## Minor changes
+* better handling of duplicated levels in metadata, ignore duplication for geography names of census tables but emit warning
+* fix issue with accessing tables without footnotes
+