Merge pull request #134 from mountainMath/v0.4.2

V0.4.2 CRAN version

Author: mountainMath

.github/workflows/R-CMD-check.yaml

@@ -31,6 +31,7 @@ jobs:
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
       R_KEEP_PKG_SOURCE: yes
+      COMPILE_VIG: ${{ secrets.COMPILE_VIG }}
 
     steps:
       - uses: actions/checkout@v4

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: cansim
 Type: Package
 Title: Accessing Statistics Canada Data Table and Vectors
-Version: 0.4.1
+Version: 0.4.2
 Authors@R: c(
     person("Jens", "von Bergmann", email = "jens@mountainmath.ca", role = c("aut","cre")),
     person("Dmitry", "Shkolnik", email = "shkolnikd@gmail.com", role = c("aut")))
@@ -33,7 +33,7 @@ Imports: digest (>= 0.6),
     RSQLite (>= 2.3),
     utils (>= 4.3),
     dbplyr (>= 2.5)
-RoxygenNote: 7.3.1
+RoxygenNote: 7.3.2
 Suggests: 
     knitr,
     rmarkdown,

NAMESPACE

@@ -1,5 +1,6 @@
 # Generated by roxygen2: do not edit by hand
 
+export(add_cansim_vectors_to_template)
 export(add_provincial_abbreviations)
 export(cansim_old_to_new)
 export(cansim_repartition_cached_table)
@@ -15,6 +16,7 @@ export(get_cansim_connection)
 export(get_cansim_cube_metadata)
 export(get_cansim_data_for_table_coord_periods)
 export(get_cansim_key_release_schedule)
+export(get_cansim_series_info_cube_coord)
 export(get_cansim_sqlite)
 export(get_cansim_table_info)
 export(get_cansim_table_last_release_date)
@@ -23,6 +25,7 @@ export(get_cansim_table_overview)
 export(get_cansim_table_short_notes)
 export(get_cansim_table_subject)
 export(get_cansim_table_survey)
+export(get_cansim_table_template)
 export(get_cansim_table_url)
 export(get_cansim_vector)
 export(get_cansim_vector_for_latest_periods)

NEWS.md

@@ -1,3 +1,13 @@
+# cansim 0.4.2
+## Minor changes
+* ensure proper ordering of levels even if StatCan metadata is not ordered
+* better error messages and information on how to disable peer checking when StatCan SSL certificates have problems
+* automatically batch vector or coordinate data retrieval in case users request more than 300 series at a time
+## Major changes
+* enable series information by table and coordinate
+* generate table template and facilitate adding vector info to aid pinpointed data download
+* enable downloading of data by vector and multiple coordinates in get_cansim_data_for_table_coord_periods (breaking changes with change to parameter)
+
 # cansim 0.4.1
 ## Minor changes
 * fix problem with parsing census data tables
@@ -17,7 +27,7 @@
 ## Minor changes
 * fix problem with reading French tables released by the census division
 * restore original column order after converting to factors
-* convery geography column to factor if available
+* convert geography column to factor if available
 * fix problem with `add_provincial_abbreviations` that could lead to mislabelling of provinces in some cases
 * improve handling of metadata, enable downloading only metadata instead of only via full table download
 * fold metadata into data when accessing via vector or coordinates
@@ -56,7 +66,7 @@
 # cansim 0.3.10
 ## Minor changes
 * Better error handling when StatCan returns empty tables
-* Add Hierachy for Geography in sqlite tables
+* Add Hierarchy for Geography in sqlite tables
 * Better fallback and warning messages when StatCan table categories are internally inconsistent
 * Performance improvements
 
@@ -83,7 +93,7 @@
 # cansim 0.3.6
 ## Major changes
 * Fold part of `normalize_cansim_values` into the default table and vector output, in particular always add a scaled variable column called `val_norm` and an imputed `Date` column and covert categories to factors by default.
-* New `get_cansim_sqlite` function that stores tables in an SQLite database and facilitates access and managemet of data.
+* New `get_cansim_sqlite` function that stores tables in an SQLite database and facilitates access and management of data.
 
 ## Minor changes
 * Adapt to changes in dplyr, tidyr, and tibble

R/cansim.R

@@ -117,11 +117,7 @@ normalize_cansim_values <- function(data, replacement_value="val_norm", normaliz
   cansimTableNumber <- cleaned_ndm_table_number(cansimTableNumber)
   cleaned_number <- cleaned_ndm_table_number(cansimTableNumber)
   cleaned_language <- cleaned_ndm_language(language)
-  geography_columns <- case_when(cleaned_language=="eng" ~
-                              c("Geography","Geographic name","Geography of origin"),
-                              TRUE ~ c(paste0("G",intToUtf8(0x00E9),"ographie"),
-                                paste0("Nom g",intToUtf8(0x00E9),"ographique"),
-                                paste0("G",intToUtf8(0x00E9),"ographie d'origine")))
+  geography_columns <- geography_colum_names(cleaned_language)
 
   base_table <- naked_ndm_table_number(cansimTableNumber)
   path <- paste0(base_path_for_table_language(cansimTableNumber,language),".zip")
@@ -156,7 +152,8 @@ normalize_cansim_values <- function(data, replacement_value="val_norm", normaliz
     for (field in fields) {
       if (!is.null(getOption("cansim.debug"))) message(paste0('Converting ',field,' to factors'))
       tryCatch({
-        level_table <- get_deduped_column_level_data(cansimTableNumber = cansimTableNumber,language=language,column=field)
+        level_table <- get_deduped_column_level_data(cansimTableNumber = cansimTableNumber,language=language,column=field) %>%
+          arrange(as.integer(.data$`...dim`),as.integer(.data$`...id`))
         if (!(field %in% names(data))) {
           geography_column <- ifelse(cleaned_language=="eng","Geography|Geographic name",paste0("G",intToUtf8(0x00E9),"ographie|Nom g",intToUtf8(0x00E9),"ographique"))
           data_geography_column <- ifelse(language=="eng","GEO",paste0("G",intToUtf8(0x00C9),"O"))
@@ -1030,7 +1027,7 @@ get_cansim_table_notes <- function(cansimTableNumber,language="en",refresh=FALSE
       tidyr::unnest_longer(!!note_id_column) %>%
       mutate(!!note_id_column:=as.character(!!as.name(note_id_column))) %>%
       full_join(notes %>% mutate(!!note_id_column:=as.character(!!as.name(note_id_column))),by=note_id_column) %>%
-      arrange(!!as.integer(as.name(note_id_column)))
+      arrange(as.integer(!!as.name(note_id_column)))
   } else {
     full_notes <- get_cansim_cube_metadata(cansimTableNumber,type="notes",refresh=refresh)
     members <- get_cansim_cube_metadata(cansimTableNumber,type="members",refresh = refresh)
@@ -1064,7 +1061,7 @@ get_cansim_table_notes <- function(cansimTableNumber,language="en",refresh=FALSE
 #' This can be used to check when a table has last been updated.
 #'
 #' @param cansimTableNumber the NDM table number
-#' @return A datatime object if a release data is available, NULL otherwise.
+#' @return A datetime object if a release data is available, NULL otherwise.
 #'
 #' @examples
 #' \dontrun{

R/cansim_helpers.R

@@ -85,6 +85,14 @@ get_with_timeout_retry <- function(url,timeout=200,retry=3,path=NA,warn_only=FAL
                                          httr::timeout(timeout))
   }
   if (!is.null(response$error)){
+    if ("curl_error_peer_failed_verification" %in% class(response$error)) {
+      stop(stringr::str_wrap(gsub(".+\\): ","",as.character(response$error),80)),"\n",
+           "This means that the authenticity of the StatCan API server can't be verified.\n",
+           "Statistics Canada has a history of failty SSL certificats on their API,\n",
+           "if you are reasonably sure that your connection is not getting hijacked you\n",
+           "can disable peer checking for the duration of the R session by typing\n\n",
+           "httr::set_config(httr::config(ssl_verifypeer=0,ssl_verifystatus=0))","\n\n","into the console.")
+    }
     if (retry>0) {
       message("Got timeout from StatCan, trying again")
       response <- get_with_timeout_retry(url,timeout=timeout,retry=retry-1,path=path)
@@ -127,6 +135,14 @@ post_with_timeout_retry <- function(url,body,timeout=200,retry=3,warn_only=FALSE
                                         httr::add_headers("Content-Type"="application/json"),
                                         httr::timeout(timeout))
   if (!is.null(response$error)){
+    if ("curl_error_peer_failed_verification" %in% class(response$error)) {
+      stop(stringr::str_wrap(gsub(".+\\): ","",as.character(response$error),80)),"\n",
+           "This means that the authenticity of the StatCan API server can't be verified.\n",
+           "Statistics Canada has a history of failty SSL certificats on their API,\n",
+           "if you are reasonably sure that your connection is not getting hijacked you\n",
+           "can disable peer checking for the duration of the R session by typing\n\n",
+           "httr::set_config(httr::config(ssl_verifypeer=0,ssl_verifystatus=0))","\n\n","into the console.")
+    }
     if (retry>0) {
       message("Got timeout from StatCan, trying again")
       response <- post_with_timeout_retry(url,body=body,timeout=timeout,retry=retry-1)
@@ -530,3 +546,26 @@ rename_columns_for_language <- function(data,from_language,to_language) {
     rename(!!!renames)
 
 }
+
+geography_colum_names <- function(language) {
+  geography_columns <- case_when(language=="eng" ~
+                                   c("Geography","Geographic name","Geography of origin"),
+                                 TRUE ~ c(paste0("G",intToUtf8(0x00E9),"ographie"),
+                                          paste0("Nom g",intToUtf8(0x00E9),"ographique"),
+                                          paste0("G",intToUtf8(0x00E9),"ographie d'origine")))
+}
+
+
+normalize_coordinates <- function(coordinates){
+  coordinates <- lapply(coordinates,\(coordinate)
+                        coordinate %>%
+                          strsplit("\\.") %>%
+                          unlist() %>%
+                          c(., rep(0, pmax(0,10-length(.)))) %>%
+                          paste(collapse = ".")
+  ) %>% unlist()
+
+}
+
+
+