99# matching col_vals_equal(..., na_pass = FALSE).
1010tol_col_counts <- function (tbl , col ) {
1111 ok <- tol_col_bool(tbl , col = col )
12- list (n = length(ok ), n_failed = sum(is.na(ok ) | ! ok ))
12+ # length - sum(TRUE) counts FALSE and NA in one pass without allocating
13+ # the two intermediate logical vectors of is.na(ok) | !ok
14+ list (n = length(ok ), n_failed = length(ok ) - sum(ok , na.rm = TRUE ))
1315}
1416
1517# n / n_failed for an equality column (resolved boolean, no NA).
@@ -18,54 +20,109 @@ eq_col_counts <- function(tbl, col, ref_suffix, na_equal) {
1820 list (n = length(b ), n_failed = sum(! b ))
1921}
2022
23+ # (n, n_failed) for every boolean validation column of a LAZY table, computed
24+ # in ONE SQL aggregate scan: SUM(CASE WHEN col THEN 0 ELSE 1 END) counts the
25+ # not-TRUE values (FALSE and NULL alike), matching the local reducers
26+ # (tolerance: is.na | !ok; equality: !b, NA-free by construction). This is
27+ # what lets the lazy verdict avoid collecting N x columns booleans into R.
28+ # Returns a named list keyed by the UNDERLYING column name:
29+ # list(<col> = list(n, n_failed), ...).
30+ lazy_boolean_counts <- function (tbl_lazy , tol_cols , eq_cols ) {
31+ con <- dbplyr :: remote_con(tbl_lazy )
32+ sub <- dbplyr :: sql_render(tbl_lazy )
33+ q <- function (x ) as.character(DBI :: dbQuoteIdentifier(con , x = x ))
34+
35+ bool_cols <- c(datadiff_ok_col(tol_cols ), datadiff_eq_col(eq_cols ))
36+ underlying <- c(tol_cols , eq_cols )
37+ nf_exprs <- vapply(X = seq_along(bool_cols ), FUN = function (i ) {
38+ sprintf(" SUM(CASE WHEN %s THEN 0 ELSE 1 END) AS %s"
39+ q(bool_cols [i ]), q(sprintf(" ..datadiff_nf_%d" i )))
40+ }, FUN.VALUE = character (1 ))
41+
42+ sql <- sprintf(
43+ " SELECT COUNT(*) AS %s%s FROM (%s) AS %s"
44+ q(" ..datadiff_total"
45+ if (length(nf_exprs ) > 0 ) {
46+ paste0(" , " nf_exprs , collapse = " , "
47+ } else {
48+ " "
49+ },
50+ sub , q(" datadiff_counts"
51+ )
52+ row <- DBI :: dbGetQuery(con , statement = sql )
53+
54+ total <- as.numeric(row [[" ..datadiff_total"
55+ counts <- vector(" list" underlying ))
56+ names(counts ) <- underlying
57+ for (i in seq_along(underlying )) {
58+ nf <- as.numeric(row [[sprintf(" ..datadiff_nf_%d" i )]])
59+ # SUM over an empty table is NULL: zero rows means zero failures
60+ counts [[i ]] <- list (n = total , n_failed = if (is.na(nf )) {
61+ 0
62+ } else {
63+ nf
64+ })
65+ }
66+ counts
67+ }
68+
2169# ' Build a faithful coverage summary of the checks performed
2270# '
2371# ' @param tbl Local data.frame holding the boolean validation columns (and, on
2472# ' the local path, the raw + reference columns for equality recomputation).
73+ # ' Ignored when `counts` is supplied.
2574# ' @param tol_cols,eq_cols Character vectors of tolerance / equality columns.
2675# ' @param missing_in_candidate,type_mismatch_cols Structural failures.
2776# ' @param row_validation_info List with `check_count`.
2877# ' @param row_count_ok Logical row-count outcome.
2978# ' @param ref_suffix Suffix identifying reference columns.
3079# ' @param na_equal Logical; NA equality semantics for equality columns.
80+ # ' @param counts Optional precomputed per-column counts (named list
81+ # ' `col -> list(n, n_failed)` covering every tolerance and equality column),
82+ # ' as produced by `lazy_boolean_counts()`: the lazy path aggregates in SQL
83+ # ' instead of collecting the booleans.
3184# ' @return A `datadiff_coverage` data.frame with columns `column`, `check`,
3285# ' `n`, `n_failed`, `status` (one row per check performed).
3386# ' @noRd
3487build_coverage <- function (tbl , tol_cols , eq_cols ,
3588 missing_in_candidate , type_mismatch_cols ,
3689 row_validation_info , row_count_ok ,
37- ref_suffix , na_equal ) {
90+ ref_suffix , na_equal , counts = NULL ) {
3891 columns <- character (0 )
3992 checks <- character (0 )
40- ns <- integer (0 )
41- n_failed <- integer (0 )
93+ ns <- numeric (0 )
94+ n_failed <- numeric (0 )
4295
4396 add <- function (column , check , n , nf ) {
4497 columns <<- c(columns , column )
4598 checks <<- c(checks , check )
46- ns <<- c(ns , as.integer(n ))
47- n_failed <<- c(n_failed , as.integer(nf ))
99+ # numeric, not integer: the lazy counts come from SQL aggregates and can
100+ # exceed 2^31 - 1, where as.integer() would overflow to NA and silently
101+ # corrupt the coverage and the verdict
102+ ns <<- c(ns , as.numeric(n ))
103+ n_failed <<- c(n_failed , as.numeric(nf ))
48104 }
49105
50106 # Existence checks: every common (non-type-mismatch) column gets a col_exists
51107 # check, distinct from its value check. These pass (the column is present in
52108 # both datasets by construction), mirroring a full per-column pointblank run.
53- for (c in c(tol_cols , eq_cols )) {
54- add(c , " col_exists" 1L , 0L )
109+ for (col_nm in c(tol_cols , eq_cols )) {
110+ add(col_nm , " col_exists" 1L , 0L )
55111 }
56- for (c in tol_cols ) {
57- cnt <- tol_col_counts(tbl , col = c )
58- add(c , " tolerance" cnt $ n , cnt $ n_failed )
112+ for (col_nm in tol_cols ) {
113+ cnt <- counts [[ col_nm ]] % || % tol_col_counts(tbl , col = col_nm )
114+ add(col_nm , " tolerance" cnt $ n , cnt $ n_failed )
59115 }
60- for (c in eq_cols ) {
61- cnt <- eq_col_counts(tbl , col = c , ref_suffix = ref_suffix , na_equal = na_equal )
62- add(c , " equality" cnt $ n , cnt $ n_failed )
116+ for (col_nm in eq_cols ) {
117+ cnt <- counts [[col_nm ]] %|| %
118+ eq_col_counts(tbl , col = col_nm , ref_suffix = ref_suffix , na_equal = na_equal )
119+ add(col_nm , " equality" cnt $ n , cnt $ n_failed )
63120 }
64- for (c in missing_in_candidate ) {
65- add(c , " missing_column" 1L , 1L )
121+ for (col_nm in missing_in_candidate ) {
122+ add(col_nm , " missing_column" 1L , 1L )
66123 }
67- for (c in type_mismatch_cols ) {
68- add(c , " type_mismatch" 1L , 1L )
124+ for (col_nm in type_mismatch_cols ) {
125+ add(col_nm , " type_mismatch" 1L , 1L )
69126 }
70127 if (isTRUE(row_validation_info $ check_count )) {
71128 add(" <row_count>" " row_count" 1L , if (isTRUE(row_count_ok )) 0L else 1L )
0 commit comments