Merge pull request #339 from cta-observatory/use-lstcam-env

Allow to decide in the cfg file if the lstcam-env should be used or not

Author: marialainez

extra/history_files/sequence_LST1_04183.history

@@ -1,2 +1,2 @@
-04179 lstcam_calib_onsite_create_drs4_pedestal_file v0.7.0 2021-12-19 22:11 None None 0
-04183 lstcam_calib_onsite_create_calibration_file v0.7.0 2021-12-19 22:11 None None 0
+04179 onsite_create_drs4_pedestal_file v0.7.0 2021-12-19 22:11 None None 0
+04183 onsite_create_calibration_file v0.7.0 2021-12-19 22:11 None None 0

src/osa/configs/sequencer.cfg

@@ -50,9 +50,9 @@ DL2_PROD_ID: model2
 # Calibration steps in calibration pipeline script
 #drs4_baseline: onsite_create_drs4_pedestal_file
 #charge_calibration: onsite_create_calibration_file
-drs4_baseline: lstcam_calib_onsite_create_drs4_pedestal_file
-charge_calibration: lstcam_calib_onsite_create_calibration_file
-catB_calibration: lstcam_calib_onsite_create_cat_B_calibration_file
+drs4_baseline: onsite_create_drs4_pedestal_file
+charge_calibration: onsite_create_calibration_file
+catB_calibration: onsite_create_cat_B_calibration_file
 use_ff_heuristic_id: False
 
 # Data processing steps in datasequence script
@@ -75,6 +75,8 @@ dl2_config: /software/lstchain/data/lstchain_standard_config.json
 mc_prod: 20240918_v0.10.12_allsky_
 dl3_config: /software/lstchain/data/dl3_std_config.json
 max_tries: 3
+use_lstcam_env_for_CatA_calib: False
+use_lstcam_env_for_CatB_calib: False
 
 [MC]
 IRF_file: /path/to/irf.fits

src/osa/job.py

@@ -515,7 +515,10 @@ def calibration_sequence_job_template(sequence):
     # Get the job header template.
     job_header = job_header_template(sequence)
 
-    commandargs = ["conda", "run", "-n", "lstcam-env", "calibration_pipeline"]
+    if cfg.getboolean("lstchain", "use_lstcam_env_for_CatA_calib"):
+        commandargs = ["conda", "run", "-n", "lstcam-env", "calibration_pipeline"]
+    else:
+        commandargs = ["calibration_pipeline"]
 
     if options.verbose:
         commandargs.append("-v")

src/osa/scripts/sequencer_catB_tailcuts.py

@@ -127,7 +127,10 @@ def launch_catB_calibration(run_id: int):
             return 
 
         command = cfg.get("lstchain", "catB_calibration")
-        env_command = f"conda run -n lstcam-env {command}"
+        if cfg.getboolean("lstchain", "use_lstcam_env_for_CatB_calib"):
+            env_command = f"conda run -n lstcam-env {command}"
+        else:
+            env_command = command
         options.filters = get_calib_filters(run_id) 
         base_dir = Path(cfg.get(options.tel_id, "BASE")).resolve()
         r0_dir = Path(cfg.get(options.tel_id, "R0_DIR")).resolve()

src/osa/scripts/tests/test_osa_scripts.py

@@ -107,17 +107,17 @@ def test_simulate_processing(
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 37
-    assert len(dl1["activity"]) == 3
-    assert len(dl1["used"]) == 12
-    assert len(dl1["wasGeneratedBy"]) == 6
+    assert len(dl1["entity"]) == 44
+    assert len(dl1["activity"]) == 5
+    assert len(dl1["used"]) == 15
+    assert len(dl1["wasGeneratedBy"]) == 10
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)
-    assert len(dl2["entity"]) == 37
-    assert len(dl2["activity"]) == 3
-    assert len(dl2["used"]) == 12
-    assert len(dl2["wasGeneratedBy"]) == 6
+    assert len(dl2["entity"]) == 44
+    assert len(dl2["activity"]) == 5
+    assert len(dl2["used"]) == 15
+    assert len(dl2["wasGeneratedBy"]) == 10
 
     rc = run_program("simulate_processing", "-p", "-d", "2020-01-17", "LST1")
     assert rc.returncode == 0

src/osa/tests/test_jobs.py

@@ -381,10 +381,6 @@ def test_create_job_scheduler_calibration(sequence_list):
     with tempfile.TemporaryDirectory() as tmpdirname:
         os.environ['NUMBA_CACHE_DIR'] = tmpdirname
         proc = subprocess.run([
-            'conda',
-            'run',
-            '-n',
-            'lstcam-env',
             'calibration_pipeline',
             '--config',
             '{DEFAULT_CFG}',

src/osa/workflow/tests/test_stages.py

@@ -79,7 +79,6 @@ def test_analysis_stage(
     assert lines[-1].split(" ")[1] == cmd[0]
     assert lines[-1].split(" ")[-1] == "255\n"
 
-@pytest.mark.skip(reason="Currently not working with osa environment (needs lstcam-env environment)")
 def test_calibration_steps(running_analysis_dir):
     from osa.workflow.stages import DRS4PedestalStage, ChargeCalibrationStage
     from osa.scripts.calibration_pipeline import drs4_pedestal_command, calibration_file_command