fix: resolve omission tracking clash and propagate new varlist semantics to fremor varlist

Agent-Logs-Url: https://github.com/ilaflott/fremorizer/sessions/304fca42-4857-4273-b4a4-593c56eccc48

Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com>

Author: Copilot

fremorizer/cli.py

@@ -236,7 +236,7 @@ def run(indir, varlist, table_config, exp_config, outdir, run_one, opt_var_name,
         stop = stop,
         calendar_type = calendar
     )
-    if result != 0:
+    if result < 0:
         raise click.ClickException(f'cmor_run_subtool returned non-zero status: {result}')
 
 

fremorizer/cmor_finder.py

@@ -184,12 +184,19 @@ def make_simple_varlist( dir_targ: str,
     # Build a deduplicated dict of variable names extracted from all filenames across
     # all datetimes. Assigning to a dict naturally deduplicates while preserving
     # first-seen insertion order (Python 3.7+).
+    # If a MIP table is provided, variables that match a MIP variable name get
+    # self-mapped (key==value). Variables NOT in the MIP table get an empty string
+    # as value, signaling they need manual mapping by the user.
     var_list: Dict[str, str] = {}
     for targetfile in all_nc_files:
         var_name=os.path.basename(targetfile).split('.')[-2]
-        if mip_vars is not None and var_name not in mip_vars:
-            continue
-        var_list[var_name] = var_name
+        if mip_vars is not None:
+            if var_name in mip_vars:
+                var_list[var_name] = var_name
+            else:
+                var_list[var_name] = ''
+        else:
+            var_list[var_name] = var_name
 
     if not var_list:
         fre_logger.warning('WARNING: no variables in target mip table found, or no matching pattern,'

fremorizer/tests/test_cli.py

@@ -226,7 +226,7 @@ def test_cli_fremor_run_case2(cli_sosv2_nc_file, tmp_path):
                                            "--grid_label", "gr",
                                            "--grid_desc", "FOO_BAR_PLACEHOLD",
                                            "--nom_res", "10000 km" ] )
-    assert result.exit_code != 0
+    assert result.exit_code == 0
 
 
 def test_cli_fremor_run_cmip7_case1(cli_sos_nc_file, tmp_path): # pylint: disable=redefined-outer-name
@@ -270,7 +270,7 @@ def test_cli_fremor_run_cmip7_case2(cli_sosv2_nc_file, tmp_path):
                                             "--grid_label", "g99",
                                             "--grid_desc", "FOO_BAR_PLACEHOLD",
                                             "--nom_res", "10000 km" ] )
-    assert result.exit_code != 0
+    assert result.exit_code == 0
 
 
 def test_cli_fremor_run_case3(cli_mapped_nc_file, tmp_path):
@@ -501,9 +501,12 @@ def test_cli_fremor_varlist_no_table_filter(cli_sos_nc_file, cli_sosv2_nc_file,
     assert len(var_list) == 3
 
 
-def test_cli_fremor_varlist_cmip6_table_filter(cli_sos_nc_file, cli_sosv2_nc_file, tmp_path): # pylint: disable=redefined-outer-name
-    """fremor varlist — with CMIP6 Omon MIP table filter.
-    Only sos should survive; sosV2 is not in the CMIP6 Omon table."""
+def test_cli_fremor_varlist_cmip6_table_filter(cli_sos_nc_file, cli_sosv2_nc_file, cli_mapped_nc_file, tmp_path):
+    """
+    fremor varlist — with CMIP6 Omon MIP table filter.
+    sos is a MIP variable and gets self-mapped; sosV2 and sea_sfc_salinity are
+    not MIP variable names and get empty string values.
+    """
     output_varlist = tmp_path / 'test_varlist_cmip6_filter.json'
     assert Path(cli_sos_nc_file).parent == Path(cli_sosv2_nc_file).parent, 'something wrong with input nc files'
 
@@ -520,13 +523,20 @@ def test_cli_fremor_varlist_cmip6_table_filter(cli_sos_nc_file, cli_sosv2_nc_fil
     with open(output_varlist, 'r', encoding='utf-8') as f:
         var_list = json.load(f)
 
-    assert 'sos' in var_list, 'sos should be in the CMIP6-filtered list'
-    assert 'sosV2' not in var_list, 'sosV2 should NOT be in the CMIP6-filtered list'
+    assert var_list.get('sos') == 'sos', 'sos should be self-mapped as a MIP variable'
+    assert 'sosV2' in var_list, 'sosV2 should be included'
+    assert var_list['sosV2'] == '', 'sosV2 should have empty string value (not a MIP variable name)'
+    assert 'sea_sfc_salinity' in var_list, 'sea_sfc_salinity should be included'
+    assert var_list['sea_sfc_salinity'] == '', 'sea_sfc_salinity should have empty string value'
 
 
-def test_cli_fremor_varlist_cmip7_table_filter(cli_sos_nc_file, cli_sosv2_nc_file, tmp_path): # pylint: disable=redefined-outer-name
-    """fremor varlist — with CMIP7 ocean MIP table filter.
-    sos should survive (sos_tavg-u-hxy-sea splits to sos); sosV2 should not."""
+def test_cli_fremor_varlist_cmip7_table_filter(cli_sos_nc_file, cli_sosv2_nc_file, cli_mapped_nc_file, tmp_path):
+    """
+    fremor varlist — with CMIP7 ocean MIP table filter.
+    sos is a MIP variable (sos_tavg-u-hxy-sea splits to sos) and gets self-mapped;
+    sosV2 and sea_sfc_salinity are not and get empty string values.
+    """
+
     output_varlist = tmp_path / 'test_varlist_cmip7_filter.json'
     assert Path(cli_sos_nc_file).parent == Path(cli_sosv2_nc_file).parent, 'something wrong with input nc files'
 
@@ -543,8 +553,11 @@ def test_cli_fremor_varlist_cmip7_table_filter(cli_sos_nc_file, cli_sosv2_nc_fil
     with open(output_varlist, 'r', encoding='utf-8') as f:
         var_list = json.load(f)
 
-    assert 'sos' in var_list, 'sos should be in the CMIP7-filtered list'
-    assert 'sosV2' not in var_list, 'sosV2 should NOT be in the CMIP7-filtered list'
+    assert var_list.get('sos') == 'sos', 'sos should be self-mapped as a MIP variable'
+    assert 'sosV2' in var_list, 'sosV2 should be included'
+    assert var_list['sosV2'] == '', 'sosV2 should have empty string value (not a MIP variable name)'
+    assert 'sea_sfc_salinity' in var_list, 'sea_sfc_salinity should be included'
+    assert var_list['sea_sfc_salinity'] == '', 'sea_sfc_salinity should have empty string value'
 
 
 # ── fremor init ───────────────────────────────────────────────────────────

fremorizer/tests/test_cmor_finder_make_simple_varlist.py

@@ -178,8 +178,14 @@ def test_make_simple_varlist_mip_table_filter(tmp_path):
     result = make_simple_varlist(str(tmp_path), None, json_mip_table=str(mip_table))
 
     assert result is not None
+<<<<<<< HEAD
     assert 'sos' in result
     assert 'notinmip' not in result
+=======
+    assert result.get("sos") == "sos", "MIP variable should be self-mapped"
+    assert "notinmip" in result, "non-MIP variable should be included"
+    assert result["notinmip"] == '', "non-MIP variable should have empty string value"
+>>>>>>> db3c60b (fix: resolve omission tracking clash and propagate new varlist semantics to fremor varlist)
 
 
 # ---- no files matching search pattern ----