steps defined in htcondor dag files. Also output dir into the job.sh

Author: gsharma99

FAIR_universe_Higgs_tautau/htcondor/job.sub

@@ -19,8 +19,8 @@ should_transfer_files = YES
 when_to_transfer_output = ON_EXIT
 
 transfer_input_files = pyproject.toml, src, FAIR_universe_Higgs_tautau, README.md
-transfer_output_files = FAIR_universe_Higgs_tautau/saved_datasets
-transfer_output_remaps = "saved_datasets = ./FAIR_universe_Higgs_tautau/saved_datasets"
+transfer_output_files = FAIR_universe_Higgs_tautau/saved_datasets, FAIR_universe_Higgs_tautau/output
+transfer_output_remaps = "saved_datasets = ./FAIR_universe_Higgs_tautau/saved_datasets; output = ./FAIR_universe_Higgs_tautau/output"
 
 +SingularityImage = "/staging/jsandesara/nsbi-env-gpu.sif"
 

FAIR_universe_Higgs_tautau/htcondor/workflow.dag

@@ -1,2 +1,37 @@
-JOB LOAD FAIR_universe_Higgs_tautau/htcondor/job.sub
-VARS LOAD CONFIG="config.pipeline.yaml" STEP="data_loader" CPUS="12" MEM="128GB" GPUS="0" DISK="64GB"
+# Global Config Variable
+CONFIG = config.pipeline.yaml
+
+# --- Step 1: Data Loading ---
+
+JOB data_loader FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS data_loader STEP="data_loader" CONFIG="$(CONFIG)" CPUS="1" MEM="8GB" GPUS="0" DISK="32GB"
+
+# --- Step 2: Preprocessing ---
+
+JOB data_preprocessing FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS data_preprocessing STEP="data_preprocessing" CONFIG="$(CONFIG)" CPUS="4" MEM="32GB" GPUS="0" DISK="32GB"
+PARENT data_loader CHILD data_preprocessing
+
+# --- Step 3: Preselection Network ---
+
+JOB preselection_network FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS preselection_network STEP="preselection_network" CONFIG="$(CONFIG)" CPUS="4" MEM="32GB" GPUS="1" DISK="32GB"
+PARENT data_preprocessing CHILD preselection_network
+
+# --- Step 4: Density Ratio Estimation ---
+
+JOB likelihood_ratio_estimation FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS likelihood_ratio_estimation STEP="neural_likelihood_ratio_estimation" CONFIG="$(CONFIG)" CPUS="4" MEM="32GB" GPUS="1" DISK="32GB"
+PARENT preselection_network CHILD likelihood_ratio_estimation
+
+# --- Step 5: Systematic Uncertainty ---
+
+JOB systematic_uncertainty FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS systematic_uncertainty STEP="systematic_uncertainty_training" CONFIG="$(CONFIG)" CPUS="4" MEM="32GB" GPUS="1" DISK="32GB"
+PARENT likelihood_ratio_estimation CHILD systematic_uncertainty
+
+# --- Step 6: Inference ---
+
+JOB parameter_fitting FAIR_universe_Higgs_tautau/htcondor/job.sub
+VARS parameter_fitting STEP="parameter_fitting" CONFIG="$(CONFIG)" CPUS="8" MEM="16GB" GPUS="0" DISK="16GB"
+PARENT systematic_uncertainty CHILD parameter_fitting