Merge pull request #123 from mountainMath/list-region-fix

cran 0.2 update

Ready for merge and cran submission

Updated version passed checks on 
* local
* travis
* rhub

Author: dshkol

.Rbuildignore

@@ -8,3 +8,6 @@
 R/hex_sticker.R
 ^doc$
 ^Meta$
+images
+images/*
+card_fix.sh

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: cancensus
 Type: Package
-Title: Canadian Census Data and Geography from the 'CensusMapper' API
-Version: 0.1.8
+Title: Access, Retrieve, and Work with Canadian Census Data and Geography
+Version: 0.2.0
 Authors@R: c(
     person("Jens", "von Bergmann", email = "jens@mountainmath.ca", role = c("aut"), comment = "API creator and maintainer"),
     person("Dmitry", "Shkolnik", email = "shkolnikd@gmail.com", role = c("aut", "cre"), comment = "Package maintainer, responsible for correspondence"),
@@ -24,7 +24,7 @@ Imports: digest (>= 0.1),
     httr (>= 1.0.0),
     jsonlite (>= 1.0),
     rlang
-RoxygenNote: 6.1.0
+RoxygenNote: 6.1.1
 Suggests: knitr,
     ggplot2,
     leaflet,

NEWS.md

@@ -1,3 +1,16 @@
+## cancensus 0.2.0
+
+### Major changes
+- Added a `max_leaves_option` for the `search_census_vectors` function. Adds functionality to set maximum depth for child census vectors, i.e. `max_level=NA` as an additional parameter. Then e.g. `max_level=1` would only get direct children and no grandchildren.
+- Resolved issues with mislabelled CD UID for CSD level data
+- Improve reliability of character/numeric alignments for geographic UIDS
+- `get_census_geography` is now soft deprecated and rolled into the standard `get_census` with parameters
+
+### Minor changes
+- Minor documentation improvements
+- Soften dependency on `readr` package
+- Allow for search of internal CensusMapper variables 
+
 ## cancensus 0.1.8
 
 ### Major changes

R/cancensus.R

@@ -3,9 +3,6 @@
 #' This function allows convenient access to Canadian census data and boundary
 #' files through the CensusMapper API. An API key is required to retrieve data.
 #'
-#' \code{get_census_geometry} is a convenience function
-#' that retrieves only Census geography boundaries.
-#'
 #' For help selecting regions and vectors, see \code{\link{list_census_regions}}
 #' and \code{\link{list_census_vectors}}, or check out the interactive selection
 #' tool at \url{https://censusmapper.ca/api}.
@@ -71,7 +68,7 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
   } else if (is.null(names(regions)) || !all(names(regions) %in% VALID_LEVELS)) {
     stop("regions must be composed of valid census aggregation levels.")
   } else {
-    regions <- jsonlite::toJSON(regions)
+    regions <- jsonlite::toJSON(lapply(regions,as.character)) # cast to character in case regions are supplied as numeric/interger
   }
 
   # Remind to set cache directory
@@ -105,7 +102,7 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
     stop("the `geo_format` parameter must be one of 'sf', 'sp', or NA")
   }
 
-  base_url="https://CensusMapper.ca/api/v1/"
+  base_url=paste0(cancensus_base_url(),"/api/v1/")
   # load data variables
   if (length(vectors)>0 || is.na(geo_format)) {
     param_string <- paste0("regions=", regions,
@@ -128,25 +125,38 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
         httr::GET(url)
       }
       handle_cm_status_code(response, NULL)
-      na_strings <- c("x", "F", "...", "..")
+      na_strings <- c("x", "F", "...", "..", "-","N")
+
+      as.num = function(x, na.strings = "NA") {
+        stopifnot(is.character(x))
+        na = x %in% na.strings
+        x[na] = 0
+        x = as.numeric(x)
+        x[na] = NA_real_
+        x
+      }
+
       # Read the data file and transform to proper data types
       result <- if (requireNamespace("readr", quietly = TRUE)) {
         # Use readr::read_csv if it's available.
         httr::content(response, type = "text", encoding = "UTF-8") %>%
           readr::read_csv(na = na_strings,
-                          col_types = list(.default = "d", GeoUID = "c",
-                                           Type = "c", "Region Name" = "c")) %>%
+                          col_types = list(.default = "c")) %>%
+          dplyr::mutate_at(c(dplyr::intersect(names(.),c("Population","Households","Dwellings","Area (sq km)")),
+                             names(.)[grepl("v_",names(.))]),
+                           as.num,na.strings=na_strings) %>%
           dplyr::mutate(Type = as.factor(.data$Type),
                         `Region Name` = as.factor(.data$`Region Name`))
       } else {
         httr::content(response, type = "text", encoding = "UTF-8") %>%
           textConnection %>%
-          utils::read.csv(na = na_strings,
-                          colClasses = c("GeoUID" = "character",
-                                         "Type" = "factor",
-                                         "Region Name" = "factor"),
-                          stringsAsFactors = FALSE, check.names = FALSE) %>%
-          dplyr::as_tibble()
+          utils::read.csv(colClasses = "character", stringsAsFactors = FALSE, check.names = FALSE) %>%
+          dplyr::as_tibble() %>%
+          dplyr::mutate_at(c(dplyr::intersect(names(.),c("Population","Households","Dwellings","Area (sq km)")),
+                             names(.)[grepl("v_",names(.))]),
+                           as.num,na.strings=na_strings) %>%
+          dplyr::mutate(Type = as.factor(.data$Type),
+                        `Region Name` = as.factor(.data$`Region Name`))
       }
       if (is.na(geo_format)) result <- result %>% transform_geo(level)
       attr(result, "last_updated") <- Sys.time()
@@ -231,15 +241,10 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
 
 #' @rdname get_census
 #' @export
-#'
-#' @examples
-#' \dontrun{
-#' # Query the API for census subdivision boundary geometry within Vancouver.
-#' vc_csds <- get_census_geometry(dataset='CA16', regions=list(CMA="59933"),
-#'                                level='CSD', geo_format = "sf")
-#'}
-get_census_geometry <- function (dataset, level, regions, geo_format = "sf", ...) {
-  return(get_census(dataset, level, regions, vectors=c(), geo_format=geo_format, ...))
+#' @keywords internal
+get_census_geometry <- function (dataset, regions, level=NA, geo_format = "sf", ...) {
+  .Deprecated("get_census")
+  return(get_census(dataset=dataset, regions=regions, level=level, vectors=c(), geo_format=geo_format, ...))
 }
 
 # This is the set of valid census aggregation levels, also used in the named
@@ -407,6 +412,8 @@ parent_census_vectors <- function(vector_list){
 #' @param vector_list The list of vectors to be used
 #' @param leaves_only Boolean flag to indicate if only leaf vectors should be returned,
 #' i.e. vectors that don't have children
+#' @param max_level optional, maximum depth to look for child vectors. Default is NA will return all
+#' child census vectors..
 #'
 #' @export
 #'
@@ -416,16 +423,18 @@ parent_census_vectors <- function(vector_list){
 #' list_census_vectors("CA16") %>%
 #'   filter(vector == "v_CA16_4092") %>%
 #'   child_census_vectors(TRUE)
-child_census_vectors <- function(vector_list, leaves_only=FALSE){
+child_census_vectors <- function(vector_list, leaves_only=FALSE,max_level=NA){
   base_list <- vector_list
   dataset <- attr(base_list,'dataset')
-  n=0
+  n <- 0
+  child_level <- 1
   if (!is.null(dataset)) {
     vector_list <-
       list_census_vectors(dataset, use_cache = TRUE, quiet = TRUE) %>%
       dplyr::filter(.data$parent_vector %in% base_list$vector) %>%
       dplyr::distinct(vector, .keep_all = TRUE)
-    while (n!=nrow(vector_list)) {
+    while (n!=nrow(vector_list) && (is.na(max_level) || child_level<max_level)) {
+      child_level <- child_level+1
       n=nrow(vector_list)
       new_list <- list_census_vectors(dataset, use_cache = TRUE, quiet = TRUE) %>%
         dplyr::filter(.data$parent_vector %in% vector_list$vector)
@@ -538,14 +547,15 @@ list_census_regions <- function(dataset, use_cache = FALSE, quiet = FALSE) {
     handle_cm_status_code(response, NULL)
     content <- httr::content(response, type = "text", encoding = "UTF-8")
     result <- if (!requireNamespace("readr", quietly = TRUE)) {
-      dplyr::as_data_frame(utils::read.csv(textConnection(content), stringsAsFactors = FALSE))
+      dplyr::as_data_frame(utils::read.csv(textConnection(content), colClasses = 'character',stringsAsFactors = FALSE))
     } else {
-      readr::read_csv(content)
+      readr::read_csv(content,col_types = readr::cols(.default='c'))
     }
     result <- dplyr::select(result, region = .data$geo_uid, .data$name,
                             level = .data$type, pop = .data$population,
                             municipal_status = .data$flag, .data$CMA_UID,
-                            .data$CD_UID, .data$PR_UID)
+                            .data$CD_UID, .data$PR_UID)  %>%
+      dplyr::mutate(pop=as.integer(.data$pop))
     attr(result, "last_updated") <- Sys.time()
     save(result, file = cache_file)
     result
@@ -763,6 +773,10 @@ transform_geo <- function(g, level) {
     new=c("GeoUID","Shape Area" ,"Type" ,"Dwellings","Households","Population","Adjusted Population (previous Census)","NHS Non-Return Rate","Quality Flags","Population 2011","Population 2016","Households 2011","Households 2016","Dwellings 2011","Dwellings 2016")
   )
   #geo uid name changes
+  if (level=='Regions') {
+    l=g$t %>% unique()
+    if (length(l)==1) level=l
+  }
   if (level=='DB') {
     name_change <- name_change %>% rbind(
       c('rpid','DA_UID'),
@@ -786,7 +800,7 @@ transform_geo <- function(g, level) {
   }
   if (level=='CSD') {
     name_change <- name_change %>% rbind(
-      c('rpid','CSD_UID'),
+      c('rpid','CD_UID'),
       c('rgid','PR_UID'),
       c('ruid','CMA_UID'))
   }

R/helpers.R

@@ -0,0 +1,5 @@
+cancensus_base_url <- function(){
+  url <- getOption("cancensus.base_url")
+  if (is.null(url)) url <- "https://censusmapper.ca"
+  url
+}

README.md

@@ -7,12 +7,13 @@
 
 <a href="https://mountainmath.github.io/cancensus/index.html"><img src="https://raw.githubusercontent.com/mountainMath/cancensus/master/images/cancensus-sticker.png" alt="cancensus logo" align="right" width = "25%" height = "25%"/></a>
 
+
 Access, retrieve, and work with Canadian Census data and geography. 
 
 * Download data and Census geography in tidy and analysis-ready format
 * Convenience tools for searching for and working with Census regions and variable hierarchies
 * Provides Census geography in multiple R spatial formats
-* Provides data and geography at multiple Census geographic levels including province, Census Metropolitan Area, Census Division, Census Subdividision, Census Tract, and Dissemination Areas
+* Provides data and geography at multiple Census geographic levels including province, Census Metropolitan Area, Census Division, Census Subdivision, Census Tract, and Dissemination Areas
 * Provides up-to-date data for the 2016, 2011, and 2006 Censuses
 
 ### Reference
@@ -40,7 +41,7 @@ To check your API key, just go to "Edit Profile" (in the top-right of the Census
 
 ### Local Cache
 
-For performance reasons, and to avoid unneccessarily drawing down API quotas, **cancensus** caches data queries under the hood. By default, **cancensus** caches in R's temporary directory, but this cache is not persistent across sessions. In order to speed up performance, reduce quota usage, and reduce the need for unnecessary network calls, we recommend assigning a persistent local cache using `options(cancensus.cache_path = 'XXX')`, this enables better control over the data. This option can be stored stored in your .Rprofile alongside your API key. Users will be prompted with a suggestion to change their default cache location when making API calls if one has not been set yet. 
+For performance reasons, and to avoid unnecessarily drawing down API quotas, **cancensus** caches data queries under the hood. By default, **cancensus** caches in R's temporary directory, but this cache is not persistent across sessions. In order to speed up performance, reduce quota usage, and reduce the need for unnecessary network calls, we recommend assigning a persistent local cache using `options(cancensus.cache_path = 'XXX')`, this enables better control over the data. This option can be stored stored in your .Rprofile alongside your API key. Users will be prompted with a suggestion to change their default cache location when making API calls if one has not been set yet. 
 
 ### Currently available datasets
 
@@ -98,24 +99,25 @@ We'd love to feature examples of work or projects that use cancensus.
 
 If you wish to cite cancensus:
 
-  von Bergmann, J., Aaron Jacobs, Dmitry Shkolnik (2019). cancensus: an R package to
-  access, retrieve, and work with Canadian Census data and geography. v0.1.8.
+  von Bergmann, J., Aaron Jacobs, Dmitry Shkolnik (2019). cancensus: R package to
+  access, retrieve, and work with Canadian Census data and geography. v0.2.0.
+
 
 A BibTeX entry for LaTeX users is
 ```
   @Manual{,
     author = {Jens {von Bergmann} and Dmitry Shkolnik and Aaron Jacobs},
-    title = {cancensus: An R Package to Access, Retrieve, and Work With Canadian Census Data and Geography},
+    title = {cancensus: R package to access, retrieve, and work With Canadian Census data and geography},
     year = {2019},
-    note = {R package version 0.1.8},
+    note = {R package version 0.2.0},
     url = {https://mountainmath.github.io/cancensus/},
   }
 ```
 ### Related packages
 
 The cancensus package is designed for working with Canadian Census data. In addition to Census data, Statistics Canada provides access to a vast [socio-economic data repository](https://www150.statcan.gc.ca/n1/en/type/data) with thousands of data tables available for public access. 
 
-The [cansim package](https://mountainmath.github.io/cansim/index.html) is designed to retrieve and work with public Statistics Canada data tables. The cansim prepares retrieved data tables as analysis-ready tidy dataframes and provides a number of convenience tools and functions to make it easier to work with Statistics Canada data. 
+The [cansim package](https://mountainmath.github.io/cansim/index.html) is designed to retrieve and work with public Statistics Canada data tables. The cansim prepares retrieved data tables as analysis-ready tidy dataframes and provides a number of convenience tools and functions to make it easier to work with Statistics Canada data. It is available on CRAN and on [Github](https://github.com/mountainMath/cancensus). 
 
 Data downloaded through the cansim package that comes with standard geographic attributes will typically share a common geographic ID that can be matched to Census data.
 

card_fix.sh

@@ -0,0 +1,24 @@
+og_img1='<meta property="og:image" content="/logo.png"'
+og_img2='<meta property="og:image" content="https://mountainmath.github.io/cancensus/logo.png"'
+
+replace_meta () {
+  file=$(find . -name '*.html')
+  for i in $file; do
+    cat "${i}" | sed "s,${og_img1},${og_img2},g" > "temp"
+    cat "temp" > "${i}"
+  done
+  rm temp
+}
+
+cd ./docs
+  replace_meta
+  cd articles
+  replace_meta
+  cd ..
+  cd news
+  replace_meta
+  cd ..
+  cd reference
+  replace_meta
+  cd ..
+cd ..

cran-comments.md

@@ -1,3 +1,12 @@
+## Update - v.0.2.0
+
+Maintenance update of package with some quality of life improvements. Changes include:
+* Improved variable tree searching
+* Better handling of numeric/string/factor alignments
+* Backward compatibility for some soft deprecated functions
+
+Release checked locally, on r-hub, and winbuilder. 
+
 ## Update - v.0.1.8
 
 Minor update of package. Changes include: