Merge pull request #242 from microsoft/copilot/fix-35

Add detection of text missing values in validation_report()

Author: martinctc

.github/analyst_guide.md

@@ -38,7 +38,7 @@ To get the most out of **wpa**, make sure to leverage these additional resources
 
 1. Our official [**wpa** cheat sheet](https://github.com/microsoft/wpa/blob/main/man/figures/wpa%20cheatsheet.pdf).
 2. A growing list of [articles](https://microsoft.github.io/wpa/articles/) with detailed walkthroughs, written by multiple contributors. 
-3. Our [Microsoft Learn module](https://docs.microsoft.com/en-us/learn/modules/workplace-analytics-r-package/) _Analyze Microsoft Workplace Analytics data using the wpa R package_, which takes you step-by-step through the R package and its key features. 
+3. Our [Microsoft Learn module](https://learn.microsoft.com/en-us/training/modules/workplace-analytics-r-package/) _Analyze Microsoft Workplace Analytics data using the wpa R package_, which takes you step-by-step through the R package and its key features. 
 
 ## Ready to go?
 

CRAN-SUBMISSION

@@ -1,3 +1,3 @@
-Version: 1.9.1
-Date: 2024-06-06 12:00:40 UTC
-SHA: 0d960e8011577d9963a3be121dc52ef78cde61fb
+Version: 1.9.2
+Date: 2025-05-28 14:01:14 UTC
+SHA: d2bbd2a998182433f85e93dc51d36460b43e2ef7

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: wpa
 Type: Package
 Title: Tools for Analysing and Visualising Viva Insights Data
-Version: 1.9.1.9000
+Version: 1.9.2
 Authors@R: c(
     person(given = "Martin", family = "Chan", role = c("aut", "cre"), email = "[email protected]"),
     person(given = "Carlos", family = "Morales", role = "aut", email = "[email protected]"),

NEWS.md

@@ -1,4 +1,7 @@
-# wpa (development version)
+# wpa 1.9.2
+
+- Improved missing value detection for `validation_report()`, `hrvar_count_all()` (#35)
+- Added support for logical outcome variables for `create_IV()` (#240)
 
 # wpa 1.9.1
 

R/hrvar_count_all.R

@@ -23,6 +23,8 @@
 #'   Default is 100.
 #' @param maxna The max percentage of NAs allowable for any column. Default is
 #'   20.
+#' @param na_values Character vector of values to be treated as missing. Default is
+#'   c("NA", "N/A", "#N/A", " ").
 #'
 #' @import dplyr
 #'
@@ -40,12 +42,25 @@
 #'   - `'message'`: outputs a message indicating which values are
 #' beyond the specified thresholds.
 #'
+#' @note
+#' As of v1.6.3, the function can detect and report text values like "NA",
+#' "N/A", "#N/A", and spaces that represent missing values, by treating them as
+#' NA values. You can customize which values are treated as missing with the
+#' `na_values` parameter.
+#' This can be validated as per:
+#' ```
+#' dv_data %>%
+#'   mutate(TempOrg = sample(c("NA", "#N/A", " "), size = nrow(.), replace = TRUE)) %>%
+#'   hrvar_count_all(return = "table")
+#' ```
+#'
 #' @export
 hrvar_count_all <- function(data,
                             n_var = 50,
                             return = "message",
                             threshold = 100,
-                            maxna = 20
+                            maxna = 20,
+                            na_values = c("NA", "N/A", "#N/A", " ")
                             ){
 
   ## Character vector of HR attributes
@@ -56,6 +71,11 @@ hrvar_count_all <- function(data,
     exclude_constants = FALSE
     )
 
+  # Ensure na_values is not NULL
+  if(is.null(na_values)){
+    na_values <- character(0)
+  }
+
   summary_table_n <-
     data %>%
     select(PersonId, extracted_chr) %>%
@@ -67,23 +87,57 @@ hrvar_count_all <- function(data,
     select(PersonId, extracted_chr) %>%
     summarise_at(vars(extracted_chr),
                  list(`WPAn_unique` = ~n_distinct(., na.rm = TRUE), # Excludes NAs from unique count
-                      `WPAper_na` = ~(sum(is.na(.))/ nrow(data) * 100),
-                      `WPAsum_na` = ~sum(is.na(.)) # Number of missing values
-                      )) %>% # % of missing values
+                      `WPAper_na` = ~(sum(is.na(.) | . %in% na_values, na.rm = TRUE)/ nrow(data) * 100), # % of missing values including na_values
+                      `WPAsum_na` = ~sum(is.na(.) | . %in% na_values, na.rm = TRUE), # Number of missing values including na_values
+                      `WPAtext_na` = ~sum(!is.na(.) & . %in% na_values, na.rm = TRUE) # Number of text values considered as NA
+                      )) %>%
     tidyr::gather(attribute, values) %>%
     tidyr::separate(col = attribute, into = c("attribute", "calculation"), sep = "_WPA") %>%
     tidyr::spread(calculation, values)
 
+    # Initialize printMessage
+    printMessage <- ""
+
     ## Single print message
     if(sum(results$n_unique >= threshold)==0){
-
       printMessage <- paste("No attributes have greater than", threshold, "unique values.")
     }
 
     if(sum(results$per_na >= maxna)==0){
       newMessage <- paste("No attributes have more than", maxna, "percent NA values.")
       printMessage <- paste(printMessage, newMessage, collapse = "\n")
+    }
 
+    # Check for text NA values
+    if(length(na_values) > 0 && any(colnames(results) == "text_na")) {
+      total_text_na <- sum(results$text_na, na.rm = TRUE)
+
+      if(total_text_na > 0) {
+        # Find which NA values were actually found in the data
+        found_na_values <- c()
+        for(na_val in na_values) {
+          for(col in extracted_chr) {
+            if(col %in% names(data)) {
+              if(any(data[[col]] == na_val, na.rm = TRUE)) {
+                found_na_values <- c(found_na_values, na_val)
+                break
+              }
+            }
+          }
+        }
+
+        found_na_values <- unique(found_na_values)
+
+        if(length(found_na_values) > 0) {
+          newMessage <- paste0(
+            "There are ", total_text_na,
+            " values which may potentially represent missing values: ",
+            paste(found_na_values, collapse = ", "),
+            "."
+          )
+          printMessage <- paste(printMessage, newMessage, collapse = "\n")
+        }
+      }
     }
 
     for (i in 1:nrow(results)) {

R/validation_report.R

@@ -55,6 +55,8 @@
 #'   to the `threshold` argument within `hrvar_count_all()`.
 #' @param timestamp Logical vector specifying whether to include a timestamp in
 #'   the file name. Defaults to `TRUE`.
+#' @param na_values Character vector of values to be treated as missing in addition
+#'   to NA values. Defaults to c("NA", "N/A", "#N/A", " ").
 #'
 #' @section Creating a report:
 #'
@@ -81,7 +83,8 @@ validation_report <- function(data,
                               hrvar = "Organization",
                               path = "validation report",
                               hrvar_threshold = 150,
-                              timestamp = TRUE){
+                              timestamp = TRUE,
+                              na_values = c("NA", "N/A", "#N/A", " ")){
 
   ## Create timestamped path (if applicable)
   if(timestamp == TRUE){
@@ -195,7 +198,7 @@ validation_report <- function(data,
 
          read_preamble("organizational_data_quality.md"), #13, Header - 2. Organizational Data Quality
          read_preamble("attributes_available.md"),#14
-         data %>% hrvar_count_all(return = "table", threshold = hrvar_threshold),
+         data %>% hrvar_count_all(return = "table", threshold = hrvar_threshold, na_values = na_values),
 
          read_preamble("groups_under_privacy_threshold_1.md"), #16, Header - 2.2 Groups under Privacy Threshold
          paste(">", data %>% identify_privacythreshold(return="text")),

README.md

@@ -8,7 +8,7 @@
 
 ## Analyze and Visualize Viva Leader Insights data
 
-This is an R package for analyzing and visualizing data from [Microsoft Workplace Analytics](https://docs.microsoft.com/en-us/workplace-analytics/). For analyzing data from [Microsoft Viva Insights](https://analysis.insights.viva.office.com/), please see our other package [**vivainsights**](https://microsoft.github.io/vivainsights/). 
+This is an R package for analyzing and visualizing data from Microsoft Workplace Analytics. For analyzing data from [Microsoft Viva Insights](https://analysis.insights.viva.office.com/), please see our other package [**vivainsights**](https://microsoft.github.io/vivainsights/). 
 
 ## With the **wpa** package, you can...
 
@@ -34,7 +34,7 @@ To get started with the package,  please see the following links:
 - [Full function list](https://microsoft.github.io/wpa/reference/index.html)
 - [Analyst Guide](https://microsoft.github.io/wpa/analyst_guide.html)
 - [FAQ](https://microsoft.github.io/wpa/faq.html)
-- [Microsoft Learn module](https://docs.microsoft.com/en-us/learn/modules/workplace-analytics-r-package/)
+- [Microsoft Learn module](https://learn.microsoft.com/en-us/training/modules/workplace-analytics-r-package/)
 
 Also check out our package cheat sheet for a quick glimpse of what **wpa** offers:
 

cran-comments.md

@@ -6,8 +6,3 @@
 ## R CMD check results
 
 0 errors | 0 warnings | 0 note
-
-## Submission 1.9.1
-
-Minor bug fixes
-