Perf

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: datadiff
 Title: Data Validation Based on YAML Rules
-Version: 0.4.4
+Version: 0.4.5
 Authors@R: c(
     person("Vincent", "Guyader", , "vincent@thinkr.fr", role = c("cre", "aut"),
            comment = c(ORCID = "0000-0003-0671-9270")),

NAMESPACE

@@ -1,5 +1,6 @@
 # Generated by roxygen2: do not edit by hand
 
+export("%||%")
 export(add_tolerance_columns)
 export(analyze_columns)
 export(compare_datasets_from_yaml)

R/compare_datasets_from_yaml.R

@@ -24,7 +24,7 @@
 #' @param date_equal_mode Default comparison mode for date columns
 #' @param datetime_equal_mode Default comparison mode for datetime columns
 #' @param logical_equal_mode Default comparison mode for logical columns
-#' @return The path to the created YAML file, invisibly.
+#' @return The \code{path} to the written YAML file, returned invisibly.
 #' @importFrom yaml write_yaml
 #' @importFrom stats setNames
 #' @importFrom dplyr collect
@@ -43,8 +43,9 @@ write_rules_template <- function(data_reference,
                                  datetime_equal_mode = "exact",
                                  logical_equal_mode = "exact") {
 
-  # 0-row collect to retrieve column names and types without loading data:
-  # works for both local data.frames and lazy tables (tbl_lazy).
+  # Validate key if provided
+  # Use a 0-row collect to retrieve column names and types: works for both local
+  # data.frames and lazy tables (tbl_lazy) without loading all rows.
   .ref_schema    <- dplyr::collect(utils::head(data_reference, 0L))
   .ref_col_names <- names(.ref_schema)
 
@@ -61,7 +62,7 @@ write_rules_template <- function(data_reference,
       )
     }
   }
-  if (is.null(label) || label == "") {label <- paste("comparison", deparse1(substitute(data_reference)))
+  if (is.null(label) || label == "") {label <- paste("comparaison", deparse1(substitute(data_reference)))
 
   }
   types <- detect_column_types(.ref_schema)
@@ -127,15 +128,15 @@ read_rules <- function(path) {
 #' @param label Descriptive label for the validation report
 #' @param error_msg_no_key Error message when datasets have different row counts without keys
 #' @param lang Language code for pointblank reports. Defaults to the
-#'   \code{datadiff.lang} option if set, otherwise \code{"en"}. Set globally
-#'   with \code{options(datadiff.lang = "fr")}. Supported values include
+#'   \code{datadiff.lang} option if set, otherwise \code{"fr"}. Override globally
+#'   with \code{options(datadiff.lang = "en")}. Supported values include
 #'   "en" (English), "fr" (French), "de" (German), "it" (Italian), "es" (Spanish),
 #'   "pt" (Portuguese), "zh" (Chinese), "ja" (Japanese), "ru" (Russian), etc.
 #'   See pointblank documentation for full list.
 #' @param locale Locale code for number and date formatting. Defaults to the
-#'   \code{datadiff.locale} option if set, otherwise \code{"en_US"}. Set globally
-#'   with \code{options(datadiff.locale = "fr_FR")}. Examples: "en_GB", "fr_FR",
-#'   "de_DE", "es_ES", "pt_BR", "zh_CN", "ja_JP".
+#'   \code{datadiff.locale} option if set, otherwise \code{"fr_FR"}. Override
+#'   globally with \code{options(datadiff.locale = "en_US")}. Examples: "en_US",
+#'   "en_GB", "de_DE", "es_ES", "pt_BR", "zh_CN", "ja_JP".
 #' @param extract_failed Logical indicating whether to collect rows that failed validation
 #'   (default: TRUE). Set to FALSE to reduce memory usage for large datasets with many errors.
 #' @param get_first_n Integer specifying the maximum number of failed rows to extract per
@@ -187,8 +188,8 @@ compare_datasets_from_yaml <- function(data_reference,
                                        ref_suffix = "__reference",
                                        label = NULL,
                                        error_msg_no_key = "Without keys, both tables must have the same number of rows.",
-                                       lang = getOption("datadiff.lang", "en"),
-                                       locale = getOption("datadiff.locale", "en_US"),
+                                       lang = getOption("datadiff.lang", "fr"),
+                                       locale = getOption("datadiff.locale", "fr_FR"),
                                        extract_failed = TRUE,
                                        get_first_n = NULL,
                                        sample_n = NULL,
@@ -319,6 +320,7 @@ compare_datasets_from_yaml <- function(data_reference,
     cand_has_dups <- nrow(cand_dups) > 0
 
     if (ref_has_dups || cand_has_dups) {
+      # Build detailed warning message
       warning_parts <- c()
 
       if (ref_has_dups) {
@@ -327,9 +329,9 @@ compare_datasets_from_yaml <- function(data_reference,
         key_cols_ref <- ref_dups[, key, drop = FALSE]
 
         examples_ref <- if (n_dup_keys_ref <= 3) {
-          apply(key_cols_ref, 1, function(r) paste(key, "=", r, collapse = ", "))
+          apply(key_cols_ref, 1, function(r) { paste(key, "=", r, collapse = ", ") })
         } else {
-          c(apply(key_cols_ref[1:3, , drop = FALSE], 1, function(r) paste(key, "=", r, collapse = ", ")), "...")
+          c(apply(key_cols_ref[1:3, , drop = FALSE], 1, function(r) { paste(key, "=", r, collapse = ", ") }), "...")
         }
 
         warning_parts <- c(warning_parts, sprintf(
@@ -344,9 +346,9 @@ compare_datasets_from_yaml <- function(data_reference,
         key_cols_cand <- cand_dups[, key, drop = FALSE]
 
         examples_cand <- if (n_dup_keys_cand <= 3) {
-          apply(key_cols_cand, 1, function(r) paste(key, "=", r, collapse = ", "))
+          apply(key_cols_cand, 1, function(r) { paste(key, "=", r, collapse = ", ") })
         } else {
-          c(apply(key_cols_cand[1:3, , drop = FALSE], 1, function(r) paste(key, "=", r, collapse = ", ")), "...")
+          c(apply(key_cols_cand[1:3, , drop = FALSE], 1, function(r) { paste(key, "=", r, collapse = ", ") }), "...")
         }
 
         warning_parts <- c(warning_parts, sprintf(
@@ -369,6 +371,7 @@ compare_datasets_from_yaml <- function(data_reference,
     }
   }
 
+  # Analyze columns
   col_analysis <- analyze_columns(data_reference, data_candidate, ignore_columns = ignore_columns)
   cols_reference <- col_analysis$cols_reference
   cols_candidate <- col_analysis$cols_candidate
@@ -412,6 +415,7 @@ compare_datasets_from_yaml <- function(data_reference,
   data_reference_p <- preprocess_dataframe(data_reference, col_rules, schema = schema_ref)
   data_candidate_p <- preprocess_dataframe(data_candidate, col_rules, schema = schema_cand)
 
+  # Get row validation information
   row_validation_info <- validate_row_counts(data_reference_p, data_candidate_p, rules)
 
   if (!is.null(key)) {
@@ -460,6 +464,7 @@ compare_datasets_from_yaml <- function(data_reference,
   }, FUN.VALUE = logical(1))]
   tol_cols <- setdiff(tol_cols, type_mismatch_cols)
 
+  # Add tolerance columns
   cmp <- add_tolerance_columns(cmp, tol_cols, col_rules, ref_suffix, na_equal)
 
   # For lazy tables, pre-compute equality columns for non-tolerance columns.
@@ -495,7 +500,9 @@ compare_datasets_from_yaml <- function(data_reference,
   }
 
   # Add row count validation column if needed
+  row_count_ok <- TRUE
   if (row_validation_info$check_count) {
+    # Calculate if row count validation passes
     expected <- row_validation_info$expected_count
     if (!is.null(expected)) {
       row_count_ok <- abs(row_validation_info$cand_count - expected) <= row_validation_info$tolerance
@@ -533,27 +540,58 @@ compare_datasets_from_yaml <- function(data_reference,
     cmp_for_agent     <- dplyr::collect(cmp_slim_computed)
   }
 
-  # Configure pointblank agent.
-  # Type-mismatched columns are excluded from common_cols (they must not go
-  # through equality comparison, which could silently pass due to R coercion)
-  # and handled via the dedicated type_mismatch_cols failing steps.
-  agent <- setup_pointblank_agent(
-    cmp_for_agent,
-    cols_reference,
-    setdiff(common_cols, type_mismatch_cols),
-    tol_cols,
-    row_validation_info,
-    ref_suffix,
-    warn_at,
-    stop_at,
-    label,
-    na_equal,
-    lang,
-    locale,
-    missing_in_candidate = missing_in_candidate,
-    type_mismatch_cols = type_mismatch_cols,
-    add_col_exists_steps = !is_lazy
-  )
+  # Fast all-pass short-circuit.
+  # The verdict is fully determined by the boolean validation columns already
+  # computed above (plus the structural checks). When everything passes there
+  # are no cells to extract, so the expensive per-column pointblank agent (one
+  # step per column, ~quadratic on wide tables) can be replaced by a constant
+  # cost trivially-passing agent. all_passed stays identical and
+  # get_data_extracts() is empty either way. Any failure falls through to the
+  # full per-column agent so failing cells remain extractable byte-for-byte.
+  eq_cols_for_check <- setdiff(setdiff(common_cols, type_mismatch_cols), tol_cols)
+  all_passed_fast <-
+    length(missing_in_candidate) == 0 &&
+    length(type_mismatch_cols) == 0 &&
+    isTRUE(row_count_ok) &&
+    all_validations_pass(
+      tbl = cmp_for_agent, tol_cols = tol_cols, eq_cols = eq_cols_for_check,
+      ref_suffix = ref_suffix, na_equal = na_equal
+    )
+
+  if (all_passed_fast) {
+    agent <- build_pass_agent(
+      tbl = cmp_for_agent, label = label,
+      warn_at = warn_at, stop_at = stop_at, lang = lang, locale = locale
+    )
+  } else {
+    # Failure path: build pointblank steps only for the columns that actually
+    # fail. Columns that pass produce empty data extracts, so omitting their
+    # steps leaves get_data_extracts() byte-for-byte unchanged while avoiding
+    # the per-column agent overhead on the passing majority. col_exists steps
+    # (which only ever pass) are dropped for the same reason. Structural
+    # failures (missing columns, type mismatches, row count) are always kept.
+    fail <- failing_columns(
+      tbl = cmp_for_agent, tol_cols = tol_cols, eq_cols = eq_cols_for_check,
+      ref_suffix = ref_suffix, na_equal = na_equal
+    )
+    agent <- setup_pointblank_agent(
+      cmp_for_agent,
+      cols_reference,
+      fail$eq,
+      fail$tol,
+      row_validation_info,
+      ref_suffix,
+      warn_at,
+      stop_at,
+      label,
+      na_equal,
+      lang,
+      locale,
+      missing_in_candidate = missing_in_candidate,
+      type_mismatch_cols = type_mismatch_cols,
+      add_col_exists_steps = FALSE
+    )
+  }
 
   reponse <- interrogate(
     agent,

R/fast_path.R

@@ -0,0 +1,107 @@
+# Per-column pass predicates, shared by the all-pass short-circuit and the
+# failing-column selection. They reproduce pointblank's verdict exactly:
+#   - a tolerance column passes iff every <col>__ok value is TRUE (NA counts as
+#     a failure, matching col_vals_equal(..., na_pass = FALSE));
+#   - an equality column passes iff its pre-computed <col>__eq column is all TRUE
+#     (lazy path) or, when no __eq exists (local path), the raw comparison holds
+#     under the same na_pass = na_equal semantics pointblank uses (any NA on
+#     either side passes iff na_equal).
+
+tol_col_passes <- function(tbl, col) {
+  isTRUE(all(tbl[[paste0(col, "__ok")]]))
+}
+
+eq_col_passes <- function(tbl, col, ref_suffix, na_equal) {
+  eq_precomputed <- tbl[[paste0(col, "__eq")]]
+  if (!is.null(eq_precomputed)) {
+    return(isTRUE(all(eq_precomputed)))
+  }
+  cand_vals <- tbl[[col]]
+  ref_vals  <- tbl[[paste0(col, ref_suffix)]]
+  cmp_res   <- cand_vals == ref_vals
+  row_pass  <- ifelse(is.na(cmp_res), na_equal, cmp_res)
+  isTRUE(all(row_pass))
+}
+
+#' Fast global pass/fail without building the per-column pointblank agent
+#'
+#' The verdict produced by \code{compare_datasets_from_yaml} is entirely
+#' determined by boolean columns already computed in a single vectorised pass
+#' (\code{<col>__ok} for tolerance columns, \code{<col>__eq} for equality
+#' columns on the lazy path). pointblank then re-checks those booleans through
+#' one validation step per column, which dominates runtime on wide tables. When
+#' every check passes there is nothing to extract, so the agent can be skipped.
+#'
+#' @param tbl Local data.frame holding the boolean validation columns (and, for
+#'   the local path, the raw and reference equality columns).
+#' @param tol_cols Character vector of tolerance column names.
+#' @param eq_cols Character vector of equality column names (already stripped of
+#'   key, type-mismatch and tolerance columns).
+#' @param ref_suffix Suffix identifying reference columns.
+#' @param na_equal Logical; whether NA equals NA for equality columns.
+#' @return \code{TRUE} iff every tolerance and equality check passes.
+#' @keywords internal
+all_validations_pass <- function(tbl, tol_cols, eq_cols, ref_suffix, na_equal) {
+  for (c in tol_cols) {
+    if (!tol_col_passes(tbl, col = c)) {
+      return(FALSE)
+    }
+  }
+  for (c in eq_cols) {
+    if (!eq_col_passes(tbl, col = c, ref_suffix = ref_suffix, na_equal = na_equal)) {
+      return(FALSE)
+    }
+  }
+  TRUE
+}
+
+#' Subset of columns whose validation fails
+#'
+#' Used on the failure path to build pointblank steps only for columns that
+#' actually fail. Columns that pass produce empty data extracts, so omitting
+#' their steps leaves \code{get_data_extracts()} unchanged while avoiding the
+#' per-column agent overhead on the passing majority.
+#'
+#' @inheritParams all_validations_pass
+#' @return A list with \code{tol} and \code{eq} character vectors of failing
+#'   column names.
+#' @keywords internal
+failing_columns <- function(tbl, tol_cols, eq_cols, ref_suffix, na_equal) {
+  fail_tol <- tol_cols[vapply(tol_cols, function(c) {
+    !tol_col_passes(tbl, col = c)
+  }, logical(1))]
+  fail_eq <- eq_cols[vapply(eq_cols, function(c) {
+    !eq_col_passes(tbl, col = c, ref_suffix = ref_suffix, na_equal = na_equal)
+  }, logical(1))]
+  list(tol = fail_tol, eq = fail_eq)
+}
+
+#' Build a minimal pointblank agent whose interrogation passes
+#'
+#' Used for the all-pass short-circuit: produces a valid interrogated
+#' \code{ptblank_agent} for which \code{pointblank::all_passed()} is \code{TRUE}
+#' and \code{pointblank::get_data_extracts()} is empty, at constant cost,
+#' instead of one validation step per column.
+#'
+#' @param tbl Table the agent reports on (only its first column is used, via a
+#'   \code{col_exists} step that always passes).
+#' @param label,warn_at,stop_at,lang,locale Passed through to
+#'   \code{pointblank::create_agent} / \code{action_levels}.
+#' @return A configured (not yet interrogated) \code{ptblank_agent}.
+#' @keywords internal
+build_pass_agent <- function(tbl, label, warn_at, stop_at, lang, locale) {
+  # The trivially-passing col_exists step needs at least one column to target.
+  # Degenerate inputs (no tolerance, equality or row-count columns to validate)
+  # produce a 0-column table, on which tbl[, 1] would error; synthesise a dummy
+  # column so create_agent()/col_exists() stay well-defined and still pass.
+  if (ncol(tbl) == 0) {
+    tbl <- data.frame(.datadiff_pass = TRUE)
+  }
+  slim <- tbl[, 1, drop = FALSE]
+  agent <- pointblank::create_agent(
+    tbl = slim, label = label,
+    actions = pointblank::action_levels(warn_at = warn_at, stop_at = stop_at),
+    lang = lang, locale = locale
+  )
+  agent %>% pointblank::col_exists(columns = tidyselect::all_of(names(slim)[1]))
+}

R/pointblank_setup.R

@@ -13,16 +13,11 @@
 #' @param stop_at Stop threshold (fraction of failing tests)
 #' @param label Descriptive label for the validation
 #' @param na_equal Logical indicating if NA values are considered equal
-#' @param lang Language code for pointblank reports. Defaults to the
-#'   \code{datadiff.lang} option if set, otherwise \code{"en"}. Set globally
-#'   with \code{options(datadiff.lang = "fr")}. Supported values include
-#'   "en" (English), "fr" (French), "de" (German), "it" (Italian), "es" (Spanish),
-#'   "pt" (Portuguese), "zh" (Chinese), "ja" (Japanese), "ru" (Russian), etc.
-#'   See pointblank documentation for full list.
-#' @param locale Locale code for number and date formatting. Defaults to the
-#'   \code{datadiff.locale} option if set, otherwise \code{"en_US"}. Set globally
-#'   with \code{options(datadiff.locale = "fr_FR")}. Examples: "en_GB", "fr_FR",
-#'   "de_DE", "es_ES", "pt_BR", "zh_CN", "ja_JP".
+#' @param lang Language code for pointblank reports (default: "fr"). Supported values include
+#'   "en" (English), "fr" (French), "de" (German), "it" (Italian), "es" (Spanish), "pt" (Portuguese),
+#'   "zh" (Chinese), "ja" (Japanese), "ru" (Russian), etc. See pointblank documentation for full list.
+#' @param locale Locale code for number and date formatting (default: "fr_FR"). Examples: "en_US",
+#'   "en_GB", "de_DE", "es_ES", "pt_BR", "zh_CN", "ja_JP".
 #' @param missing_in_candidate Character vector of columns missing in candidate dataset
 #' @param type_mismatch_cols Character vector of columns whose type differs between
 #'   reference and candidate (e.g. numeric in reference, character in candidate).
@@ -45,7 +40,7 @@
 #' @export
 setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols,
                                    row_validation_info = NULL, ref_suffix, warn_at, stop_at, label,
-                                   na_equal, lang = getOption("datadiff.lang", "en"), locale = getOption("datadiff.locale", "en_US"),
+                                   na_equal, lang = "fr", locale = "fr_FR",
                                    missing_in_candidate = character(0),
                                    type_mismatch_cols = character(0),
                                    add_col_exists_steps = TRUE) {
@@ -133,6 +128,8 @@ setup_pointblank_agent <- function(cmp, cols_reference, common_cols, tol_cols,
     agent <- agent %>% col_vals_equal(columns = all_of(ok_col), value = TRUE, na_pass = FALSE)
   }
 
+  # Add row count validation if needed
+  # Use colnames() instead of names() to work correctly with lazy tables
   if (!is.null(row_validation_info) && isTRUE(row_validation_info$check_count) && "row_count_ok" %in% get_col_names(cmp)) {
     agent <- agent %>% col_vals_equal(columns = all_of("row_count_ok"), value = TRUE, na_pass = FALSE)
   }