postprocessing: split summary output into behavioral & ERP tables

- generate separate tables for behavioral and ERP (post-processing) analyses
- track data attribution between collection and final ERP-ready trials
- fix percentage calculations to use correct denominators
- add sum columns for error trials (soc-vis-err, nonsoc-vis-err)
- match MATLAB column order for compatibility

Author: Buch

code/postprocessing/postprocessing-behavior/batch_behavior_postprocessing.R renamed to code/postprocessing-behavior/batch_behavior_postprocessing.R

@@ -8,25 +8,29 @@ cat("session started:", as.character(Sys.time()), "\n\n")
 # clear environment
 rm(list = ls())
 
+# load required libraries
+library(tidyverse)
+library(jsonlite)
+
 # load configuration & functions
 source("config/paths.R")
 
 # get repo root from paths
-repo_root <- file.path(here::here(), "..", "..", "..")
+repo_root <- file.path(here::here(), "..", "..")
 
 source("config/settings.R")
 source("functions/load_behavioral_data.R")
 source("functions/load_eeg_trial_info.R")
 source("functions/apply_eeg_exclusions.R")
 source("functions/apply_rt_trimming.R")
 source("functions/check_inclusion_criteria.R")
-source("functions/validate_against_eeg.R")
 source("functions/generate_reports.R")
 
 # === user input: define subject list ===
 # slash-separated string of subject IDs to be processed in this run
 # leave empty ("") to process all available subjects
-subjects_to_process = "390002/390003/390004/390005/390006/390007/390008/390009/390010/390011/390012/390013/390014/390015/390020/390021/390022/390023/390024/390025/390026/390027/390028/390030/390031/390032/390033/390034/390036/390037/390038/390039/390041/390042";
+# subjects_to_process = "390001/390002/390003/390004/390005/390006/390007/390008/390009/390010/390011/390012/390013/390014/390015/390020/390021/390022/390023/390024/390025/390026/390027/390028/390030/390031/390032/390033/390034/390036/390037/390038/390039/390041/390042/390043/390044";
+subjects_to_process = "390035"
 
 # parse subject list
 if (subjects_to_process == "") {
@@ -100,11 +104,94 @@ inclusion_file <- file.path(output_dir, "inclusion_summary.rds")
 saveRDS(inclusion_summary, inclusion_file)
 cat("saved inclusion summary to:", inclusion_file, "\n")
 
+processing_params <- list(
+  rt_lower_bound = RT_LOWER_BOUND,
+  rt_outlier_threshold = RT_OUTLIER_THRESHOLD,
+  min_accuracy = MIN_ACCURACY,
+  min_trials_per_code = MIN_EPOCHS_PER_CODE
+)
+
+# === SAVE CSV OUTPUTS FOR MATLAB ===
+cat("\n=== SAVING CSV OUTPUTS FOR MATLAB ===\n")
+
+# 1. save behavioral summary as CSV for MATLAB
+behavioral_summary_csv <- inclusion_summary %>%
+  select(
+    subject,
+    included,
+    exclusion_reason,
+    accuracy,
+    total_trials,
+    n_code_102, n_code_104, n_code_111, n_code_112, n_code_113,
+    n_code_202, n_code_204, n_code_211, n_code_212, n_code_213
+  ) %>%
+  mutate(
+    included = as.numeric(included),  # convert TRUE/FALSE to 1/0
+    exclusion_reason = replace_na(exclusion_reason, "none")
+  )
+
+behavioral_summary_file <- file.path(output_dir, "behavioral_summary.csv")
+write_csv(behavioral_summary_csv, behavioral_summary_file)
+cat("saved behavioral summary csv to:", behavioral_summary_file, "\n")
+
+# 2. save trial-level data as CSV for MATLAB (if needed for future analyses)
+behavioral_trials_csv <- behavioral_with_rt %>%
+  select(
+    subject,
+    trial_idx,
+    code,
+    rt = flankerResponse.rt, 
+    accuracy = responseType,
+    in_eeg,
+    eeg_analysis_ready,
+    eeg_exclusion_reason,
+    rt_exclusion_reason
+  ) %>%
+  mutate(
+    in_eeg = as.numeric(in_eeg),
+    eeg_analysis_ready = as.numeric(eeg_analysis_ready),
+    accuracy = as.numeric(accuracy),
+    eeg_exclusion_reason = replace_na(eeg_exclusion_reason, "none"),
+    rt_exclusion_reason = replace_na(rt_exclusion_reason, "none")
+  )
+
+behavioral_trials_file <- file.path(output_dir, "behavioral_trials.csv")
+write_csv(behavioral_trials_csv, behavioral_trials_file)
+cat("saved trial-level data csv to:", behavioral_trials_file, "\n")
+
+# 3. save processing metadata as JSON for documentation
+metadata <- list(
+  processing_date = Sys.time(),
+  parameters = processing_params,
+  n_subjects_processed = n_distinct(behavioral_data$subject),
+  n_subjects_included = sum(inclusion_summary$included),
+  n_subjects_excluded = sum(!inclusion_summary$included),
+  output_files = list(
+    behavioral_summary = "behavioral_summary.csv",
+    behavioral_trials = "behavioral_trials.csv",
+    subject_summary_table = paste0("subject_summary_table_", format(Sys.time(), "%Y-%m-%d_%H-%M-%S"), ".txt")
+  )
+)
+
+metadata_file <- file.path(output_dir, "processing_metadata.json")
+jsonlite::write_json(metadata, metadata_file, pretty = TRUE)
+cat("saved processing metadata to:", metadata_file, "\n")
+
 # generate reports
 cat("\n=== GENERATING REPORTS ===\n")
 
-# subject summary table
-subject_table <- generate_subject_summary_table(
+# generate reports
+cat("\n=== GENERATING REPORTS ===\n")
+
+# generate both summary tables
+behavioral_table <- generate_behavioral_summary_table(
+  behavioral_with_rt,
+  inclusion_summary,
+  output_dir,
+  verbose = TRUE
+)
+
+erp_table <- generate_erp_summary_table(
   behavioral_with_rt,
   inclusion_summary,
   output_dir,
@@ -119,36 +206,9 @@ processing_params <- list(
   min_trials_per_code = MIN_EPOCHS_PER_CODE
 )
 
-cat("\n=== VALIDATION ===\n")
-
-# validate against most recent eeg postprocessing
-# find most recent eeg postprocessing date
-eeg_dates <- list.dirs(derivatives_dir, 
-                       full.names = FALSE, recursive = FALSE)
-eeg_dates <- eeg_dates[grepl("_erp-postprocessing$", eeg_dates)]
-
-if (length(eeg_dates) > 0) {
-  most_recent_eeg <- sort(eeg_dates, decreasing = TRUE)[1]
-  eeg_date <- gsub("_erp-postprocessing", "", most_recent_eeg)
-  
-  cat("\nvalidating against eeg postprocessing from:", eeg_date, "\n")
-  
-  eeg_summary <- load_eeg_summary_table(eeg_date, derivatives_dir)
-  validation_results <- validate_trial_counts(inclusion_summary, eeg_summary, verbose = TRUE)
-  
-  # save validation results
-  validation_file <- file.path(output_dir, "validation_results.rds")
-  saveRDS(validation_results, validation_file)
-  cat("\nsaved validation results to:", validation_file, "\n")
-} else {
-  cat("\nwarning: no eeg postprocessing folders found, skipping validation\n")
-}
-
-# generate final summary report (needs validation results)
 summary_report <- generate_postprocessing_summary(
   behavioral_with_rt,
   inclusion_summary,
-  validation_results,
   output_dir,
   processing_params,
   verbose = TRUE

code/postprocessing/postprocessing-behavior/config/paths.R renamed to code/postprocessing-behavior/config/paths.R

@@ -6,7 +6,7 @@ library(stringr)
 
 # read link to preprocessed data (same file used by matlab)
 # here() gives us the r project root, need to go up to repo root
-repo_root <- file.path(here(), "..", "..", "..")
+repo_root <- file.path(here(), "..", "..")
 preprocessed_link <- file.path(repo_root, "input", "preprocessed")
 preprocessed_path <- readLines(preprocessed_link, warn = FALSE) %>% str_trim()