Constructed warning about removed data

Signed-off-by: David Turner <djturner@umbc.edu>

Author: David Turner

xga/sources/base.py

@@ -1,5 +1,5 @@
 #  This code is part of X-ray: Generate and Analyse (XGA), a module designed for the XMM Cluster Survey (XCS).
-#  Last modified by David J Turner (djturner@umbc.edu) 5/21/26, 3:19 PM. Copyright (c) The Contributors.
+#  Last modified by David J Turner (djturner@umbc.edu) 5/21/26, 3:53 PM. Copyright (c) The Contributors.
 
 import contextlib
 import gc
@@ -310,36 +310,6 @@ def __init__(self, ra: float, dec: float, redshift: float = None, name: str = No
         #  actual event list/image/expmap/region files - those initial products are loaded into XGA products
         self._products, region_dict = self._initial_products(obs, load_regions, null_load_products)
 
-        # # We need to make sure that we only keep instruments (and ObsIDs) that actually have an
-        # #  event list associated with them - it is possible for initial_products to have not been
-        # #  able to load them
-        # new_prods = {t: {o: {i: self._products[t][o][i] for i in self._products[t][o]
-        #                      if 'events' in self._products[t][o][i]}
-        #                  for o in self._products[t]} for t in self._products}
-        # # Then we remove any ObsIDs that no longer have any instruments
-        # new_prods = {t: {o: new_prods[t][o] for o in new_prods[t]
-        #                  if len(new_prods[t][o]) != 0}
-        #              for t in new_prods}
-        # # Replicate the new_prods structure for the 'obs' and 'region_dict' variables
-        # new_obs = {t: {o: [i for i in obs[t][o] if i in new_prods[t][o]] for o in new_prods[t]} for t in new_prods}
-        # new_regs = {t: {o: region_dict[t][o] for o in new_prods[t]} for t in new_prods}
-        #
-        # # Warn the user about removals
-        # # Aggregate all removed combinations for a single warning
-        # all_removed = ["{t}: {o}({i})".format(t=t, o=o, i=", ".join([i for i in obs[t][o] if o not in new_obs[t]
-        #                                                              or i not in new_obs[t][o]]))
-        #                for t in obs for o in obs[t] if o not in new_obs[t]
-        #                or any(i not in new_obs[t][o] for i in obs[t][o])]
-        #
-        # if all_removed:
-        #     warn(f"Some observations/instruments were removed as their event lists could not "
-        #          f"be read: {"; ".join(all_removed)}", stacklevel=2)
-
-        # # Assign the altered dictionaries
-        # self._products = new_prods
-        # obs = new_obs
-        # region_dict = new_regs
-
         # Now we do ANOTHER check just like the one above, but on the product attribute, as it is possible that
         #  all those files cannot be found
         cur_obs_nums = {tel: len(self._products[tel]) for tel in self._products}
@@ -352,10 +322,36 @@ def __init__(self, ra: float, dec: float, redshift: float = None, name: str = No
             new_obs = {tel: obs[tel] for tel, num in cur_obs_nums.items() if num != 0}
             new_prods = {tel: self._products[tel] for tel, num in cur_obs_nums.items() if num != 0}
             new_regs = {tel: region_dict[tel] for tel, num in cur_obs_nums.items() if num != 0}
-            # Then assign the new cut down dictionaries to their original names
-            obs = new_obs
-            self._products = new_prods
-            region_dict = new_regs
+
+        else:
+            new_obs = obs
+            new_prods = self._products
+            new_regs = region_dict
+
+        # Identify any observations or instruments that were removed because their event lists
+        #  could not be read in successfully
+        removed_obs = {t: {o: [i for i in obs[t][o] if o not in new_prods[t] or i not in new_prods[t][o]]
+                           for o in obs[t]} for t in obs}
+        # Clean up the dictionary so it only contains entries that actually have removed instruments
+        removed_obs = {t: {o: i for o, i in removed_obs[t].items() if len(i) != 0} for t in removed_obs}
+        removed_obs = {t: o for t, o in removed_obs.items() if len(o) != 0}
+
+        # If there are any removed observations/instruments, we raise a warning
+        if len(removed_obs) != 0:
+            # We construct a message to warn the user
+            warn_msg = "The following observations/instruments could not be read in and have been removed: "
+            # This nested join formats the dictionary into Telescope: ObsID(inst1, inst2); ...
+            warn_msg += "; ".join(["{t}: {o}".format(t=t, o=", ".join(["{o}({i})".format(o=o, i=", ".join(i))
+                                                                       for o, i in removed_obs[t].items()]))
+                                   for t in removed_obs])
+            warn(warn_msg, stacklevel=2)
+
+        # Assign the new, potentially cut down, dictionaries to their original names - this is separated from the
+        #  checking logic above so that we can raise a warning by comparing obs and new_prods before obs
+        #  gets overwritten
+        obs = new_obs
+        self._products = new_prods
+        region_dict = new_regs
 
         # This is somewhat inelegant, but oh well; it ensures that there are individual instrument entries for each
         #  ObsID in the products dictionary, for cases where the data are shipped with all instruments combined
@@ -1133,16 +1129,13 @@ def read_default_products(en_lims: tuple) -> Tuple[str, dict]:
                 #  the copy so that any modifications don't harm the original file.
                 reg_file = rel_sec["region_file"].format(obs_id=obs_id)
 
-
-
-
+                # Load the event list - if this doesn't work then there isn't any point continuing to the
+                #  rest of the initial product loading process for the current telescope-ObsID-instrument
                 evt_list = EventList(evt_file, obs_id=obs_id, instrument=inst, stdout_str="", stderr_str="",
                                      gen_cmd="", telescope=tel)
                 if not evt_list.usable:
                     continue
 
-
-
                 # Attitude file is a special type of data product, we shouldn't ever deal with it directly so it
                 #  doesn't have a product object. It also isn't guaranteed to be a separate thing for all
                 #  telescopes, so we do check that the configuration file actually has an entry for it.
@@ -1184,14 +1177,10 @@ def read_default_products(en_lims: tuple) -> Tuple[str, dict]:
                 else:
                     mask_prod = None
 
+                # If we've gotten to this part of the method we know that the EventList exists and is usable, so
+                #  to proceed to populating the obs_dict we check that the attitude file (if one is meant to
+                #  exist for this telescope) actually exists.
                 if (att_prod is not None and att_prod.usable) or att_prod is None:
-                    # An instrument subsection of an observation will ONLY be populated if the events file exists
-                    # Otherwise nothing can be done with it.
-                    # evt_list = EventList(evt_file, obs_id=obs_id, instrument=inst, stdout_str="", stderr_str="",
-                    #                      gen_cmd="",  telescope=tel)
-                    # if not evt_list.usable:
-                    #     continue
-
                     # Start off with the events list going in - make sure there is an inst entry first
                     obs_dict[tel][obs_id].setdefault(inst, {})
                     obs_dict[tel][obs_id][inst]["events"] = evt_list