EUCLIDMNGT-1430 include dataSetReleases

Author: Jorge Fernandez Hernandez

astroquery/esa/euclid/core.py

@@ -1144,45 +1144,52 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
         if product_type in conf.OBSERVATION_STACK_PRODUCTS:
             table = 'sedm.observation_stack'
 
-            dsr_condition = self.get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'observation_stack')
-            extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
+            dsr_condition = self.__get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'observation_stack')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
 
             query = (f"SELECT observation_stack.file_name, observation_stack.observation_stack_oid, "
                      f"observation_stack.observation_id, observation_stack.ra, observation_stack.dec, "
                      f"observation_stack.instrument_name, observation_stack.filter_name, "
                      "observation_stack.release_name, observation_stack.category, observation_stack.second_type, "
                      f"observation_stack.technique, observation_stack.product_type, observation_stack.start_time, "
-                     f"observation_stack.duration FROM {table} WHERE "
+                     f"observation_stack.duration, observation_stack.data_set_release_part1, "
+                     f"observation_stack.data_set_release_part2, observation_stack.data_set_release_part3 FROM "
+                     f"{table} WHERE "
                      f" observation_stack.observation_id = '{observation_id}' AND observation_stack.product_type = '"
                      f"{product_type}' {extra_condition};")
 
         if product_type in conf.BASIC_DOWNLOAD_DATA_PRODUCTS:
             table = 'sedm.basic_download_data'
 
-            dsr_condition = self.get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'basic_download_data')
-            extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
+            dsr_condition = self.__get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3,
+                                                                  'basic_download_data')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
 
             query = (
                 f"SELECT basic_download_data.basic_download_data_oid, basic_download_data.product_type, "
                 f"basic_download_data.product_id, CAST(basic_download_data.observation_id_list as text) AS "
                 f"observation_id_list, CAST(basic_download_data.tile_index_list as text) AS tile_index_list, "
                 f"CAST(basic_download_data.patch_id_list as text) AS patch_id_list, "
-                f"CAST(basic_download_data.filter_name as text) AS filter_name, basic_download_data.release_name FROM "
+                f"CAST(basic_download_data.filter_name as text) AS filter_name, basic_download_data.release_name, "
+                f"basic_download_data.data_set_release_part2, basic_download_data.data_set_release_part3 FROM "
                 f"{table} WHERE '{observation_id}'=ANY(observation_id_list) AND product_type = '"
                 f"{product_type}' {extra_condition}"
                 f"ORDER BY observation_id_list ASC;")
 
         if product_type in conf.MER_SEGMENTATION_MAP_PRODUCTS:
             table = 'sedm.mer_segmentation_map'
 
-            dsr_condition = self.get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'mer_segmentation_map')
-            extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
+            dsr_condition = self.__get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3,
+                                                                  'mer_segmentation_map')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
 
             query = (
                 f"SELECT mer_segmentation_map.file_name, mer_segmentation_map.segmentation_map_oid, "
                 f"mer_segmentation_map.ra, mer_segmentation_map.dec, mer_segmentation_map.stc_s, "
                 f"mer_segmentation_map.tile_index, "
-                f"mer_segmentation_map.product_type, mer_segmentation_map.product_id FROM {table} "
+                f"mer_segmentation_map.product_type, mer_segmentation_map.product_id, "
+                f"mer_segmentation_map.release_name, mer_segmentation_map.data_set_release_part2, "
+                f"mer_segmentation_map.data_set_release_part3 FROM {table} "
                 f"WHERE ( observation_id_list = '{observation_id}' OR observation_id_list like '{observation_id},"
                 f"%' OR observation_id_list "
                 f"like '%,{observation_id}' OR CAST(observation_id_list as TEXT) like '%,{observation_id},%' ) AND "
@@ -1191,8 +1198,8 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
         if product_type in conf.RAW_FRAME_PRODUCTS:
             table = 'sedm.raw_frame'
 
-            dsr_condition = self.get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'raw_frame')
-            extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
+            dsr_condition = self.__get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'raw_frame')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
 
             if product_type == "dpdNispRawFrame":
                 instrument_name = "NISP"
@@ -1204,47 +1211,52 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                 f"raw_frame.instrument_name, raw_frame.data_set_release, raw_frame.filter_name, "
                 f"raw_frame.observation_mode, raw_frame.grism_wheel_pos, raw_frame.cal_block_id, "
                 f"raw_frame.cal_block_variant, raw_frame.ra, raw_frame.dec, raw_frame.obs_time_utc, "
-                f"raw_frame.exposure_time FROM {table} WHERE raw_frame.observation_id = '{observation_id}' "
+                f"raw_frame.exposure_time, raw_frame.release_name, raw_frame.data_set_release_part1, "
+                f"raw_frame.data_set_release_part2, raw_frame.data_set_release_part3 FROM {table} WHERE "
+                f"raw_frame.observation_id = '{observation_id}' "
                 f"AND raw_frame.instrument_name = '{instrument_name}' {extra_condition};")
 
         if product_type in conf.CALIBRATED_FRAME_PRODUCTS:
             table = 'sedm.calibrated_frame'
 
-            dsr_condition = self.get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'calibrated_frame')
-            extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
+            dsr_condition = self.__get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'calibrated_frame')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
 
             query = (
                 f"SELECT calibrated_frame.file_name, calibrated_frame.calibrated_frame_oid, "
                 f"calibrated_frame.observation_id, calibrated_frame.instrument_name, calibrated_frame.filter_name, "
                 f"calibrated_frame.ra, calibrated_frame.dec, calibrated_frame.stc_s, calibrated_frame.start_time, "
-                f"calibrated_frame.end_time, calibrated_frame.duration "
+                f"calibrated_frame.end_time, calibrated_frame.duration, calibrated_frame.data_set_release_part1, "
+                f"calibrated_frame.data_set_release_part2, calibrated_frame.data_set_release_part3 "
                 f"FROM {table} WHERE calibrated_frame.observation_id = '{observation_id}' AND "
                 f"calibrated_frame.product_type = '{product_type}' {extra_condition};")
 
         if product_type in conf.FRAME_CATALOG_PRODUCTS:
             table = 'sedm.frame_catalog'
 
-            dsr_condition = self.get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'frame_catalog')
-            extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
+            dsr_condition = self.__get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'frame_catalog')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
 
             query = (
                 f"SELECT frame_catalog.file_name, frame_catalog.catalog_oid, frame_catalog.observation_id, "
                 f"frame_catalog.instrument_name, frame_catalog.filter_name, frame_catalog.ra, frame_catalog.dec, "
                 f"frame_catalog.datarange_start_time, frame_catalog.datarange_end_time, "
-                f"frame_catalog.product_type, frame_catalog.product_id FROM {table} "
+                f"frame_catalog.product_type, frame_catalog.product_id, frame_catalog.data_set_release_part1, "
+                f"frame_catalog.data_set_release_part2, frame_catalog.data_set_release_part3 FROM {table} "
                 f"WHERE frame_catalog.observation_id = '{observation_id}' AND frame_catalog.product_type = '"
                 f"{product_type}' {extra_condition};")
 
         if product_type in conf.COMBINED_SPECTRA_PRODUCTS:
             table = 'sedm.combined_spectra'
 
-            dsr_condition = self.get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'combined_spectra')
-            extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
+            dsr_condition = self.__get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'combined_spectra')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
 
             query = (
                 f"SELECT combined_spectra.combined_spectra_oid, combined_spectra.lambda_range, "
                 f"combined_spectra.tile_index, combined_spectra.stc_s, combined_spectra.product_type, "
-                f"combined_spectra.product_id FROM {table} "
+                f"combined_spectra.product_id, combined_spectra.data_set_release_part1, "
+                f"combined_spectra.data_set_release_part2, combined_spectra.data_set_release_part3 FROM {table} "
                 f"WHERE ( observation_id_list = '{observation_id}' OR observation_id_list like '{observation_id} %' "
                 f"OR observation_id_list "
                 f"like '% {observation_id}' OR observation_id_list like '% {observation_id} %' ) AND "
@@ -1253,15 +1265,16 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
         if product_type in conf.SIR_SCIENCE_FRAME_PRODUCTS:
             table = 'sedm.sir_science_frame'
 
-            dsr_condition = self.get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'sir_science_frame')
-            extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
+            dsr_condition = self.__get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3, 'sir_science_frame')
+            extra_condition = '' if dsr_condition is None else f' AND {dsr_condition}'
 
             instrument_name = "NISP"
 
             query = (
                 f"SELECT sir_science_frame.file_name, sir_science_frame.science_frame_oid, "
                 f"sir_science_frame.observation_id, sir_science_frame.instrument_name, sir_science_frame.stc_s, "
-                f"sir_science_frame.prod_sdc FROM {table} "
+                f"sir_science_frame.prod_sdc, sir_science_frame.data_set_release_part1, "
+                f"sir_science_frame.data_set_release_part2, sir_science_frame.data_set_release_part3 FROM {table} "
                 f"WHERE sir_science_frame.observation_id = '{observation_id}' AND sir_science_frame.instrument_name = '"
                 f"{instrument_name}' {extra_condition};")
 
@@ -1272,7 +1285,31 @@ def get_product_list(self, *, observation_id=None, tile_index=None, product_type
                                  format_with_results_compressed=('votable_gzip',))
         return job.get_results()
 
-    def get_data_set_release_condition(self, dsr_1_value=None, dsr_2_value=None, dsr_3_value=None, alias=None):
+    def __get_data_set_release_condition(self, dsr_1_value=None, dsr_2_value=None, dsr_3_value=None, alias=None):
+
+        query = None
+        if dsr_1_value is not None:
+            dsr_1_final = '.'.join(filter(None, [alias, 'data_set_release_part1']))
+            query = f"{dsr_1_final} = '{dsr_1_value}'"
+
+        if dsr_2_value is not None:
+            dsr_2_final = '.'.join(filter(None, [alias, 'data_set_release_part2']))
+            if query is not None:
+                query = f"{query} AND {dsr_2_final} = '{dsr_2_value}'"
+            else:
+                query = f"{dsr_2_final} = '{dsr_2_value}'"
+
+        if dsr_3_value is not None:
+            dsr_3_final = '.'.join(filter(None, [alias, 'data_set_release_part3']))
+            if query is not None:
+                query = f"{query} AND {dsr_3_final} = {dsr_3_value}"
+            else:
+                query = f"{dsr_3_final} = {dsr_3_value}"
+
+        return query
+
+    def __get_data_set_release_condition_from_environment(self, dsr_1_value=None, dsr_2_value=None, dsr_3_value=None,
+                                                          alias=None):
 
         query = None
         if dsr_1_value is not None:
@@ -1604,7 +1641,7 @@ def get_scientific_product_list(self, *, observation_id=None, tile_index=None, c
         if tile_index is not None:
             query_extra_condition = f" AND '{tile_index}' = ANY(tile_index_list) "
 
-        dsr_condition = self.get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3)
+        dsr_condition = self.__get_data_set_release_condition(dsr_part1, dsr_part2, dsr_part3)
         dsr_extra_condition = '' if dsr_condition is None else f'AND {dsr_condition}'
 
         if observation_id is not None:
@@ -1710,7 +1747,9 @@ def get_scientific_product_list(self, *, observation_id=None, tile_index=None, c
             f"basic_download_data.product_id, CAST(basic_download_data.observation_id_list as text) AS "
             f"observation_id_list, CAST(basic_download_data.tile_index_list as text) AS tile_index_list, "
             f"CAST(basic_download_data.patch_id_list as text) AS patch_id_list, "
-            f"CAST(basic_download_data.filter_name as text) AS filter_name FROM {table} WHERE "
+            f"CAST(basic_download_data.filter_name as text) AS filter_name, "
+            f"basic_download_data.data_set_release_part1, basic_download_data.data_set_release_part2, "
+            f"basic_download_data.data_set_release_part3 FROM {table} WHERE "
             f"release_name='{dataset_release}' {query_extra_condition} {dsr_extra_condition} ORDER BY "
             f"observation_id_list ASC")