Merge pull request #151 from mountainMath/1996

Merging dev into master. Will update to CRAN later this week. I'm thinking we roll this out now with the changes to make 1996  work seamlessly and worry about the rest later. Not sure yet how we want to surface the geometry intersection capabilities - let's leave that for 0.3.3.

Author: dshkol

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: cancensus
 Type: Package
 Title: Access, Retrieve, and Work with Canadian Census Data and Geography
-Version: 0.3.1
+Version: 0.3.2
 Authors@R: c(
     person("Jens", "von Bergmann", email = "jens@mountainmath.ca", role = c("aut"), comment = "API creator and maintainer"),
     person("Dmitry", "Shkolnik", email = "shkolnikd@gmail.com", role = c("aut", "cre"), comment = "Package maintainer, responsible for correspondence"),

NEWS.md

@@ -1,3 +1,12 @@
+# cancensus 0.3.2
+
+## Major changes
+- Support for 1996 census
+- Public availability of dissemination block area level data
+
+## Minor changes
+- Fixes [bug](https://github.com/mountainMath/cancensus/issues/150) in `find_census_vectors()`
+
 # cancensus 0.3.1
 
 ## Minor changes

R/cancensus.R

@@ -9,7 +9,7 @@
 #'
 #' @param dataset A CensusMapper dataset identifier.
 #' @param regions A named list of census regions to retrieve. Names must be valid census aggregation levels.
-#' @param level The census aggregation level to retrieve, defaults to \code{"Regions"}. One of \code{"Regions"}, \code{"PR"}, \code{"CMA"}, \code{"CD"}, \code{"CSD"}, \code{"CT"} or \code{"DA"}.
+#' @param level The census aggregation level to retrieve, defaults to \code{"Regions"}. One of \code{"Regions"}, \code{"PR"}, \code{"CMA"}, \code{"CD"}, \code{"CSD"}, \code{"CT"}, \code{"DA"}, \code{"EA"} (for 1996), or \code{"DB"} (for 2001-2016).
 #' @param vectors An R vector containing the CensusMapper variable names of the census variables to download. If no vectors are specified only geographic data will get downloaded.
 #' @param geo_format By default is set to \code{NA} and appends no geographic information. To include geographic information with census data, specify one of either \code{"sf"} to return an \code{\link[sf]{sf}} object (requires the \code{sf} package) or \code{"sp"} to return a \code{\link[sp]{SpatialPolygonsDataFrame-class}} object (requires the \code{rgdal} package).
 #' @param labels Set to "detailed" by default, but truncated Census variable names can be selected by setting labels = "short". Use \code{label_vectors(...)} to return variable label information in detail.
@@ -54,11 +54,11 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
 
   # Turn the region list into a valid JSON dictionary.
   if (is.character(regions)) {
-    if (!quiet) warning(paste("passing `regions` as a character vector is",
+    if (!quiet) warning(paste("Passing `regions` as a character vector is",
                               "depreciated, and will be removed in future",
                               "versions"))
   } else if (is.null(names(regions)) || !all(names(regions) %in% VALID_LEVELS)) {
-    stop("regions must be composed of valid census aggregation levels.")
+    stop("Regions must be composed of valid census aggregation levels.")
   } else {
     regions <- jsonlite::toJSON(lapply(regions,as.character)) # cast to character in case regions are supplied as numeric/interger
   }
@@ -81,7 +81,7 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
 
   # Check if the aggregation level is valid.
   if (!level %in% VALID_LEVELS) {
-    stop("the `level` parameter must be one of 'Regions', 'PR', 'CMA', 'CD', 'CSD', 'CT', or 'DA'")
+    stop("the `level` parameter must be one of 'Regions', 'PR', 'CMA', 'CD', 'CSD', 'CT', 'DA', 'EA' or 'DB'")
   }
 
   # Check that we can read the requested geo format.
@@ -115,26 +115,16 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
         httr::GET(url)
       }
       handle_cm_status_code(response, NULL)
-      na_strings <- c("x", "F", "...", "..", "-","N","*","**")
-
-      as.num = function(x, na.strings = "NA") {
-        stopifnot(is.character(x))
-        na = x %in% na.strings
-        x[na] = 0
-        x = as.numeric(x)
-        x[na] = NA_real_
-        x
-      }
+
 
       # Read the data file and transform to proper data types
       result <- if (requireNamespace("readr", quietly = TRUE)) {
         # Use readr::read_csv if it's available.
         httr::content(response, type = "text", encoding = "UTF-8") %>%
-          readr::read_csv(na = na_strings,
+          readr::read_csv(na = cancensus_na_strings,
                           col_types = list(.default = "c")) %>%
           dplyr::mutate_at(c(dplyr::intersect(names(.),c("Population","Households","Dwellings","Area (sq km)")),
-                             names(.)[grepl("v_",names(.))]),
-                           as.num,na.strings=na_strings) %>%
+                             names(.)[grepl("v_",names(.))]), as.num) %>%
           dplyr::mutate(Type = as.factor(.data$Type),
                         `Region Name` = as.factor(.data$`Region Name`))
       } else {
@@ -143,8 +133,7 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
           utils::read.csv(colClasses = "character", stringsAsFactors = FALSE, check.names = FALSE) %>%
           dplyr::as_tibble(.name_repair = "minimal") %>%
           dplyr::mutate_at(c(dplyr::intersect(names(.),c("Population","Households","Dwellings","Area (sq km)")),
-                             names(.)[grepl("v_",names(.))]),
-                           as.num,na.strings=na_strings) %>%
+                             names(.)[grepl("v_",names(.))]), as.num) %>%
           dplyr::mutate(Type = as.factor(.data$Type),
                         `Region Name` = as.factor(.data$`Region Name`))
       }
@@ -188,8 +177,8 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
       geos
     } else if (!is.na(geo_format)) {
       # the sf object needs to be first in join to retain all spatial information
-      dplyr::select(result, -.data$Population, -.data$Dwellings,
-                    -.data$Households, -.data$Type) %>%
+      to_remove <- setdiff(dplyr::intersect(names(geos),names(result)),"GeoUID")
+      dplyr::select(result, -dplyr::one_of(to_remove)) %>%
         dplyr::inner_join(geos, ., by = "GeoUID")
     }
   }
@@ -255,7 +244,7 @@ get_census_geometry <- function (dataset, regions, level=NA, geo_format = "sf",
 
 # This is the set of valid census aggregation levels, also used in the named
 # elements of the `regions` parameter.
-VALID_LEVELS <- c("Regions","C","PR", "CMA", "CD", "CSD", "CT", "DA", "DB")
+VALID_LEVELS <- c("Regions","C","PR", "CMA", "CD", "CSD", "CT", "DA", 'EA', "DB")
 
 #' Query the CensusMapper API for available datasets.
 #'
@@ -330,9 +319,6 @@ dataset_attribution <- function(datasets){
 
   commons %>% lapply(function(c){
     matches <- attribution[grepl(paste0("^",c,"$"),attribution)]
-
-    #years <- stringr::str_extract(matches, "\\d{4}") %>% sort()
-    # avoid stringr dependency
     parts <- strsplit(c, split = "\\\\d\\{4\\}") %>%
       unlist()
     years <- matches
@@ -347,8 +333,6 @@ dataset_attribution <- function(datasets){
     paste0(collapse="; ")
 }
 
-
-
 #' Return Census variable names and labels as a tidy data frame
 #'
 #' @param x A data frame, \code{sp} or \code{sf} object returned from
@@ -443,7 +427,7 @@ transform_geo <- function(g, level) {
   g <- g %>%
     dplyr::mutate_at(dplyr::intersect(names(g), as_character), as.character) %>%
     dplyr::mutate_at(dplyr::intersect(names(g), as_numeric), as.numeric)  %>%
-    dplyr::mutate_at(dplyr::intersect(names(g), as_integer), as.integer)  %>%
+    dplyr::mutate_at(dplyr::intersect(names(g), as_integer), as.int)  %>%
     dplyr::mutate_at(dplyr::intersect(names(g), as_factor), as.factor)
 
   # Change names
@@ -464,7 +448,7 @@ transform_geo <- function(g, level) {
       c('ruid','CT_UID'),
       c('rguid','CMA_UID'))
   }
-  if (level=='DA') {
+  if (level=='DA'|level=='EA') {
     name_change <- name_change %>% rbind(
       c('rpid','CSD_UID'),
       c('rgid','CD_UID'),

R/census_regions.R

@@ -36,9 +36,8 @@
 #' @examples
 #' list_census_regions('CA16')
 list_census_regions <- function(dataset, use_cache = TRUE, quiet = FALSE) {
-  dataset = toupper(dataset)
   cache_file <- file.path(tempdir(),paste0(dataset, "_regions.rda"))
-  #cache_file <- cache_path(dataset, "_regions.rda")
+
   if (!use_cache || !file.exists(cache_file)) {
     if (!quiet) message("Querying CensusMapper API for regions data...")
     response <- httr::GET(paste0("https://censusmapper.ca/data_sets/", dataset,

R/census_vectors.R

@@ -25,7 +25,6 @@
 #' # List all vectors for a given Census dataset in CensusMapper
 #' list_census_vectors('CA16')
 list_census_vectors <- function(dataset, use_cache = TRUE, quiet = TRUE) {
-  #cache_file <- cache_path(dataset, "_vectors.rda")
   cache_file <- file.path(tempdir(),paste0(dataset, "_vectors.rda"))
   if (!use_cache || !file.exists(cache_file)) {
     url <- paste0(cancensus_base_url(),"/api/v1/vector_info/", dataset, ".csv")
@@ -56,6 +55,8 @@ list_census_vectors <- function(dataset, use_cache = TRUE, quiet = TRUE) {
         grepl("^3.", add) ~ gsub(".", ", ", gsub("^3\\.", "Median of ", add),
                                  fixed = TRUE),
         grepl("^4.", add) ~ gsub(".", ", ", gsub("^4\\.", "Average to ", add),
+                                 fixed = TRUE),
+        grepl("^9.", add) ~ gsub(".", ", ", gsub("^9\\.", "Standard error based on ", add),
                                  fixed = TRUE)
       )) %>%
       dplyr::select(.data$vector, .data$type, .data$label, .data$units,

R/helpers.R

@@ -32,8 +32,28 @@ dataset_from_vector_list <- function(vector_list){
   dataset
 }
 
-# To activate later
-# valid_datasets <- c("CA01","CA06","CA11","CA16",
+cancensus_na_strings <- c("x", "F", "...", "..", "-","N","*","**")
+
+as.num = function(x, na.strings = cancensus_na_strings) {
+  stopifnot(is.character(x))
+  na = x %in% na.strings
+  x[na] = 0
+  x = as.numeric(x)
+  x[na] = NA_real_
+  x
+}
+
+as.int = function(x, na.strings = cancensus_na_strings) {
+  stopifnot(is.character(x))
+  na = x %in% na.strings
+  x[na] = 0
+  x = as.integer(x)
+  x[na] = NA_integer_
+  x
+}
+
+# List of eligible datasets
+# VALID_DATASETS <- c("CA1996","CA01","CA06","CA11","CA16",
 #                     "CA01xSD", "CA06xSD", "CA11xSD", "CA16xSD",
 #                     "TX2000", "TX2001", "TX2002", "TX2003", "TX2004",
 #                     "TX2005", "TX2006", "TX2007", "TX2008", "TX2009",

README.md

@@ -12,8 +12,8 @@ Access, retrieve, and work with Canadian Census data and geography.
 * Download data and Census geography in tidy and analysis-ready format
 * Convenience tools for searching for and working with Census regions and variable hierarchies
 * Provides Census geography in multiple R spatial formats
-* Provides data and geography at multiple Census geographic levels including province, Census Metropolitan Area, Census Division, Census Subdivision, Census Tract, and Dissemination Areas
-* Provides data for the 2016, 2011, 2006, and 2001 Census releases
+* Provides data and geography at multiple Census geographic levels including province, Census Metropolitan Area, Census Division, Census Subdivision, Census Tract, Dissemination Areas, Enumeration Areas (for 1996), and Dissemination Blocks (for 2001-2016)
+* Provides data for the 2016, 2011, 2006, 2001, and 1996 Census releases
 * Access to taxfiler data at the Census Tract level for tax years 2000 through 2017
 
 ### Reference
@@ -35,7 +35,7 @@ library(cancensus)
 
 ### API key
 
-This package relies on queries to the CensusMapper API, which requires a Censusmapper API key. You can obtain a free API key by [signing up](https://censusmapper.ca/users/sign_up) for a CensusMapper account. CensusMapper API keys are free; however, API requests are limited in volume. For larger quotas, please get in touch with Jens [directly](mailto:jens@censusmapper.ca).  
+**cancensus** requires a valid CensusMapper API key to use. You can obtain a free API key by [signing up](https://censusmapper.ca/users/sign_up) for a CensusMapper account. CensusMapper API keys are free and public API quotas are generous; however, due to incremental costs of serving large quantities of data, there limits to API usage in place. For most use cases, these API limits should not be an issue. Production uses with large extracts of fine grained geographies may run into API quota limits. For larger quotas, please get in touch with Jens [directly](mailto:jens@censusmapper.ca).
 
 To check your API key, just go to "Edit Profile" (in the top-right of the CensusMapper menu bar). Once you have your key, you can store it in your system environment so it is automatically used in API calls. To do so just enter `options(cancensus.api_key = "your_api_key")`.
 
@@ -45,7 +45,7 @@ For performance reasons, and to avoid unnecessarily drawing down API quotas, **c
 
 ### Currently available datasets
 
-**cancensus** can access Statistics Canada Census data for the 2001 Census, the 2006 Census, the 2011 Census and National Household Survey, as well as the 2016 Census. You can run `list_census_datasets` to check what datasets are currently available for access through the CensusMapper API. Additional data for the 2016 Census will be included in Censusmapper within a day or two after public release by Statistics Canada. Statistics Canada maintains a release schedule for the Census 2016 Program which can be viewed on their [website](http://www12.statcan.gc.ca/census-recensement/2016/ref/release-dates-diffusion-eng.cfm).
+**cancensus** can access Statistics Canada Census data for Census years 1996, 2001, 2006, 2011, and 2016. You can run `list_census_datasets` to check what datasets are currently available for access through the CensusMapper API. Additional data for the 2016 Census will be included in Censusmapper within a day or two after public release by Statistics Canada. Statistics Canada maintains a release schedule for the Census 2016 Program which can be viewed on their [website](http://www12.statcan.gc.ca/census-recensement/2016/ref/release-dates-diffusion-eng.cfm).
 
 Thanks to contributions by the Canada Mortgage and Housing Corporation (CMHC), **cancensus** now includes additional Census-linked datasets as open-data releases. These include annual taxfiler data at the census tract level for tax years 2000 through 2017, which includes data on incomes and demographics, as well as specialized crosstabs for Structural type of dwelling by Document type, which details occupancy status for residences. These crosstabs are available for the 2001, 2006, 2011, and 2016 Census years at all levels starting with census tract.
 
@@ -107,7 +107,7 @@ We'd love to feature examples of work or projects that use cancensus.
 If you wish to cite cancensus:
 
   von Bergmann, J., Aaron Jacobs, Dmitry Shkolnik (2020). cancensus: R package to
-  access, retrieve, and work with Canadian Census data and geography. v0.3.1.
+  access, retrieve, and work with Canadian Census data and geography. v0.3.2.
 
 
 A BibTeX entry for LaTeX users is
@@ -116,7 +116,7 @@ A BibTeX entry for LaTeX users is
     author = {Jens {von Bergmann} and Dmitry Shkolnik and Aaron Jacobs},
     title = {cancensus: R package to access, retrieve, and work With Canadian Census data and geography},
     year = {2020},
-    note = {R package version 0.3.1},
+    note = {R package version 0.3.2},
     url = {https://mountainmath.github.io/cancensus/},
   }
 ```

cran-comments.md

@@ -1,3 +1,9 @@
+## Update - v.0.3.2
+
+- Add functionality for 1996 census and more refined geographies
+- Expanded vignettes
+- Fix minor bugs flagged by users
+
 ## Update - v.0.3.1
 
 Addressing warning and note in CRAN checks from upload v.0.3.0