Merge pull request #337 from cta-observatory/calib-prod-id

Produce and look for calibration files with lstcam_calib version

Author: marialainez

src/osa/configs/sequencer.cfg

@@ -36,6 +36,7 @@ SOURCE_CATALOG: %(OSA_DIR)s/Catalog
 SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb
 GAIN_SELECTION_FLAG_DIR: %(OSA_DIR)s/GainSel
 GAIN_SELECTION_WEB_DIR: %(OSA_DIR)s/GainSelWeb
+CALIB_ENV: /fefs/aswg/software/conda/envs/lstcam-env
 
 # To be set by the user. Using PROD-ID will overcome the automatic
 # fetching of lstchain version. Otherwise leave it empty (and without the colon symbol).

src/osa/conftest.py

@@ -26,7 +26,6 @@
 from osa.scripts.tests.test_osa_scripts import run_program
 from osa.utils.utils import date_to_dir
 from datetime import datetime
-import lstchain
 
 date = datetime.fromisoformat("2020-01-17")
 nightdir = date_to_dir(date)
@@ -87,14 +86,14 @@ def drive_log(monitoring_dir):
 
 @pytest.fixture(scope="session")
 def calibration_dir(calibration_base_dir):
-    directory = calibration_base_dir / "calibration" / nightdir / f"v{lstchain.__version__}"
+    directory = calibration_base_dir / "calibration" / nightdir / "v0.1.1"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 
 
 @pytest.fixture(scope="session")
 def drs4_baseline_dir(calibration_base_dir):
-    directory = calibration_base_dir / "drs4_baseline" / nightdir / f"v{lstchain.__version__}"
+    directory = calibration_base_dir / "drs4_baseline" / nightdir / "v0.1.1"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 

src/osa/high_level/significance.py

@@ -206,6 +206,8 @@ def main(
     log.debug(f"Config: {config.resolve()}")
 
     # Initial setup of global parameters
+    if simulate:
+        options.simulate = True
     options.date = date
     flat_date = date_to_dir(date)
     options.tel_id = telescope

src/osa/nightsummary/tests/test_extract.py

@@ -1,5 +1,7 @@
 from datetime import datetime
+from osa.configs import options
 
+options.test = True
 obs_date = datetime.fromisoformat("2020-01-17")
 
 

src/osa/paths.py

@@ -9,7 +9,6 @@
 import subprocess
 import time
 import json
-import lstchain
 from astropy.table import Table
 from lstchain.onsite import (find_systematics_correction_file,
                              find_time_calibration_file)
@@ -103,10 +102,12 @@ def get_drs4_pedestal_filename(run_id: int, prod_id: str) -> Path:
         return files[-1]  # Get the latest production among the major lstchain version
 
     date = utils.date_to_dir(get_run_date(run_id))
+    lstcam_env = Path(cfg.get("LST1", "CALIB_ENV"))
+    lstcam_calib_version = utils.get_lstcam_calib_version(lstcam_env)
     return (
         DRS4_PEDESTAL_BASEDIR
         / date
-        / f"v{lstchain.__version__}/drs4_pedestal.Run{run_id:05d}.0000.h5"
+        / f"v{lstcam_calib_version}/drs4_pedestal.Run{run_id:05d}.0000.h5"
     ).resolve()
 
 
@@ -138,10 +139,12 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
     date = utils.date_to_dir(get_run_date(run_id))
     options.filters = utils.get_calib_filters(run_id)
 
+    lstcam_env = Path(cfg.get("LST1", "CALIB_ENV"))
+    lstcam_calib_version = utils.get_lstcam_calib_version(lstcam_env)
     return (
         CALIB_BASEDIR
         / date
-        / f"v{lstchain.__version__}/calibration_filters_{options.filters}.Run{run_id:05d}.0000.h5"
+        / f"v{lstcam_calib_version}/calibration_filters_{options.filters}.Run{run_id:05d}.0000.h5"
     ).resolve()
 
 

src/osa/scripts/calibration_pipeline.py

@@ -20,7 +20,6 @@
 from osa.utils.cliopts import calibration_pipeline_cliparsing
 from osa.utils.logging import myLogger
 from osa.workflow.stages import DRS4PedestalStage, ChargeCalibrationStage
-from osa.utils.utils import get_lstchain_version
 
 __all__ = [
     "calibration_sequence",
@@ -47,30 +46,25 @@ def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list:
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
     r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     command = cfg.get("lstchain", "drs4_baseline")
-    prod_id = get_lstchain_version()
     return [
         command,
         "-r", str(drs4_pedestal_run_id),
         "-b", base_dir,
         f"--r0-dir={r0_dir}",
         "--no-progress",
-        "-v",prod_id,
     ]
 
 def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> list:
     """Build the create_calibration_file command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
     r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     command = cfg.get("lstchain", "charge_calibration")
-    prod_id = get_lstchain_version()
     cmd = [
         command,
         "-p", str(drs4_pedestal_run_id),
         "-r", str(pedcal_run_id),
         "-b", base_dir,
         f"--r0-dir={r0_dir}",
-        "-v",prod_id,
-
     ]
     # In case of problems with trigger tagging:
     if cfg.getboolean("lstchain", "use_ff_heuristic_id"):

src/osa/scripts/tests/test_osa_scripts.py

@@ -350,8 +350,6 @@ def test_drs4_pedestal_cmd(base_test_dir):
         base_test_dir,
         f"--r0-dir={r0_dir}",
         "--no-progress",
-        "-v","v0.11.0.post1"
-
     ]
     assert cmd == expected_command
 
@@ -370,7 +368,6 @@ def test_calibration_file_cmd(base_test_dir):
         "-b",
         base_test_dir,
         f"--r0-dir={r0_dir}",
-        "-v","v0.11.0.post1"
     ]
     assert cmd == expected_command
 

src/osa/utils/utils.py

@@ -11,6 +11,7 @@
 from datetime import datetime, timedelta
 from pathlib import Path
 from socket import gethostname
+import subprocess as sp
 from gammapy.data import observatory_locations
 from astropy import units as u
 from astropy.table import Table
@@ -402,3 +403,15 @@ def get_RF_model(run_id: int) -> Path:
     rf_model_path = rf_models_dir / declination_str
 
     return rf_model_path.resolve()
+
+
+def get_lstcam_calib_version(env_path: Path) -> str:
+    """Get the version of the lstcam_calib package installed in the given environment."""
+    if options.test or options.simulate:
+        return "0.1.1"
+    python_exe = f"{str(env_path)}/bin/python"
+    cmd = [python_exe, "-m", "pip", "show", "lstcam_calib"]
+    result = sp.run(cmd, capture_output=True, text=True, check=True)
+    for line in result.stdout.split('\n'):
+        if line.startswith('Version:'):
+            return line.split(':', 1)[1].strip()