add config-driven scripts for htcondor

FAIR_universe_Higgs_tautau/config.pipeline.yaml

@@ -31,3 +31,181 @@ data_loader:
   output:
     dir: "./saved_datasets"
 
+# Section for Data Preprocessing
+data_preprocessing:
+
+  config_path: "./config.yaml"
+  features:
+
+    no_jets:
+      - PRI_lep_pt
+      - PRI_lep_eta
+      - PRI_lep_phi
+      - PRI_had_pt
+      - PRI_had_eta
+      - PRI_had_phi
+      - PRI_met
+      - PRI_met_phi
+      - DER_mass_transverse_met_lep
+      - DER_mass_vis
+      - DER_pt_h
+      - DER_deltar_had_lep
+      - DER_pt_tot
+      - DER_sum_pt
+      - DER_pt_ratio_lep_had
+      - DER_met_phi_centrality
+
+    one_jet:
+      - PRI_jet_leading_pt
+      - PRI_jet_leading_eta
+      - PRI_jet_leading_phi
+      - PRI_jet_all_pt
+
+    two_jets:
+      - PRI_jet_subleading_pt
+      - PRI_jet_subleading_eta
+      - PRI_jet_subleading_phi
+      - DER_deltaeta_jet_jet
+      - DER_mass_jet_jet
+      - DER_prodeta_jet_jet
+      - DER_lep_eta_centrality
+
+
+    n_jets:
+      - PRI_n_jets
+
+parameter_fitting:
+  configs:
+    hist: "./config_hist.yml"   #Configuration file path for histogram settings
+    nsbi: "./config.yml" 
+  measurement: "higgs_tautau_signal_strength" #Measurement to fit in the analysis.
+
+  output:
+    plots_dir: "./output/plots/parameter_fitting"
+    logs_dir: "./output/logs/parameter_fitting"
+  scan:
+    parameter: "mu_htautau"
+    range: [0.0, 3.0]
+    steps: 50
+
+
+preselection_network:
+  # Path to the original config required by nsbi_common_utils
+  nsbi_config: "./config.yml"
+
+  model_path: "./saved_datasets/preselection_model/"
+  force_train: false
+
+  training:
+    test_size: 0.2
+    random_state: 42
+    batch_size: 1024
+    epochs: 50
+    learning_rate: 0.1
+
+    labels:
+      htautau: 0
+      ttbar: 1
+      ztautau: 2
+
+  preselection_observable:
+    signal_processes: ["htautau"]
+    background_processes: ["ttbar", "ztautau"]
+    pre_factor_dict:
+      htautau: 1.0
+      ttbar: 1.0
+      ztautau: 1.0
+
+  output:
+    plots_dir: "./output/plots/preselection_network"
+
+
+
+
+# Section for Density Ratio Estimation
+neural_likelihood_ratio_estimation:
+  nsbi_config: "./config.yml"
+  saved_data_path: "./saved_datasets/"
+  output_training_dir: "output_training_nominal"
+
+  use_log_loss: false
+  delete_existing_models: false
+  force_train: false
+
+  # ---------------------------------------------------------
+  # DEFINITION OF COMMON HYPERPARAMETERS
+  # ---------------------------------------------------------
+  common_training_params: &common_params
+    hidden_layers: 4
+    neurons: 1000
+    number_of_epochs: 100
+    batch_size: 512
+    learning_rate: 0.1
+    scalerType: "MinMax"
+
+    calibration: false
+    recalibrate_output: false
+    type_of_calibration: "histogram"
+    num_bins_cal: null
+
+    callback: true
+    callback_patience: 30
+    callback_factor: 0.01
+
+    validation_split: 0.1
+    holdout_split: 0.25
+
+    num_ensemble_members: 10
+    load_trained_models: true  # 'force_train' flag in script overrides this
+
+    verbose: 1
+    plot_scaled_features: false
+    summarize_model: true
+
+  # ---------------------------------------------------------
+  # PROCESS SPECIFIC SETTINGS
+  # ---------------------------------------------------------
+  training_settings:
+    htautau:
+      <<: *common_params
+
+    ttbar:
+      <<: *common_params
+
+    ztautau:
+      <<: *common_params
+
+
+
+# Section for Systematic Uncertainty Estimation
+systematic_uncertainty:
+  nsbi_config: "./config.yml"
+  saved_data_path: "./saved_datasets/"
+  filter_region: "SR"
+  force_train: false
+
+  training_settings:
+    hidden_layers: 3
+    neurons: 100
+    number_of_epochs: 50
+    batch_size: 10000
+    learning_rate: 0.001
+    scalerType: "MinMax"
+
+    calibration: false
+    recalibrate_output: false
+    type_of_calibration: "histogram"
+    num_bins_cal: null
+
+    callback: true
+    callback_patience: 10
+    callback_factor: 0.5
+
+    validation_split: 0.2
+    holdout_split: 0.2
+
+    verbose: 0
+    plot_scaled_features: false
+    load_trained_models: true
+    num_ensemble_members: 1
+    summarize_model: false

FAIR_universe_Higgs_tautau/htcondor/job.sub

@@ -19,8 +19,8 @@ should_transfer_files = YES
 when_to_transfer_output = ON_EXIT
 
 transfer_input_files = pyproject.toml, src, FAIR_universe_Higgs_tautau, README.md
-transfer_output_files = FAIR_universe_Higgs_tautau/saved_datasets
-transfer_output_remaps = "saved_datasets = ./FAIR_universe_Higgs_tautau/saved_datasets"
+transfer_output_files = FAIR_universe_Higgs_tautau/saved_datasets, FAIR_universe_Higgs_tautau/output
+transfer_output_remaps = "saved_datasets = ./FAIR_universe_Higgs_tautau/saved_datasets; output = ./FAIR_universe_Higgs_tautau/output"
 
 +SingularityImage = "/staging/jsandesara/nsbi-env-gpu.sif"
 

FAIR_universe_Higgs_tautau/htcondor/workflow.dag

@@ -1,2 +1,37 @@
-JOB LOAD FAIR_universe_Higgs_tautau/htcondor/job.sub
-VARS LOAD CONFIG="config.pipeline.yaml" STEP="data_loader" CPUS="12" MEM="128GB" GPUS="0" DISK="64GB"
+# Global Config Variable
+CONFIG = config.pipeline.yaml
+
+# --- Step 1: Data Loading ---
+
+JOB data_loader FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS data_loader STEP="data_loader" CONFIG="$(CONFIG)" CPUS="1" MEM="8GB" GPUS="0" DISK="32GB"
+
+# --- Step 2: Preprocessing ---
+
+JOB data_preprocessing FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS data_preprocessing STEP="data_preprocessing" CONFIG="$(CONFIG)" CPUS="4" MEM="32GB" GPUS="0" DISK="32GB"
+PARENT data_loader CHILD data_preprocessing
+
+# --- Step 3: Preselection Network ---
+
+JOB preselection_network FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS preselection_network STEP="preselection_network" CONFIG="$(CONFIG)" CPUS="4" MEM="32GB" GPUS="1" DISK="32GB"
+PARENT data_preprocessing CHILD preselection_network
+
+# --- Step 4: Density Ratio Estimation ---
+
+JOB likelihood_ratio_estimation FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS likelihood_ratio_estimation STEP="neural_likelihood_ratio_estimation" CONFIG="$(CONFIG)" CPUS="4" MEM="32GB" GPUS="1" DISK="32GB"
+PARENT preselection_network CHILD likelihood_ratio_estimation
+
+# --- Step 5: Systematic Uncertainty ---
+
+JOB systematic_uncertainty FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS systematic_uncertainty STEP="systematic_uncertainty_training" CONFIG="$(CONFIG)" CPUS="4" MEM="32GB" GPUS="1" DISK="32GB"
+PARENT likelihood_ratio_estimation CHILD systematic_uncertainty
+
+# --- Step 6: Inference ---
+
+JOB parameter_fitting FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS parameter_fitting STEP="parameter_fitting" CONFIG="$(CONFIG)" CPUS="8" MEM="16GB" GPUS="0" DISK="16GB"
+PARENT systematic_uncertainty CHILD parameter_fitting