Merge pull request #3500 from esdc-esac-esa-int/ESA_gaia_C9GACS-1079_datalink_ecsv_format

GAIA: the method load_data must parse ecsv files

Author: bsipocz

CHANGES.rst

@@ -34,6 +34,7 @@ gaia
 ^^^^
 
 - New datalink DR4 retrieval type RESIDUAL_IMAGE. [#3489]
+- The method ``load_data`` parses ecsv files [#3500].
 
 esa.hubble
 ^^^^^^^^^^

astroquery/gaia/core.py

@@ -20,6 +20,7 @@
 from astropy.coordinates import Angle
 from astropy.io import fits
 from astropy.io import votable
+from astropy.io.fits import TableHDU, BinTableHDU
 from astropy.table import Table
 from astropy.units import Quantity
 from astropy.utils.decorators import deprecated_renamed_argument
@@ -194,9 +195,9 @@ def load_data(self, ids, *, data_release=None, data_structure='INDIVIDUAL', retr
             'EPOCH_PHOTOMETRY_CROWDED_FIELD', 'EPOCH_IMAGE', 'EPOCH_PHOTOMETRY_CCD', 'EPOCH_SPECTRUM_XP_SSO',
             'EPOCH_SPECTRUM_XP_CROWDING', 'MEAN_SPECTRUM_XP', 'EPOCH_SPECTRUM_XP', 'CROWDED_FIELD_IMAGE',
             'EPOCH_ASTROMETRY_BRIGHT', 'MEAN_SPECTRUM_XP_GRAVLENS', 'EPOCH_FLAGS_NSS', 'EPOCH_PARAMETERS_RVS_SINGLE',
-            'EPOCH_PARAMETERS_RVS_DOUBLE', 'EPOCH_FLAGS_VARI']. Note that for 'CROWDED_FIELD_IMAGE', only the format
-            'fits' can be used, and its image, in the principal header, will not be available in the returned
-            dictionary. Set 'output_file' to retrieve all data: image + tables. Note that for 'RESIDUAL_IMAGE',
+            'EPOCH_PARAMETERS_RVS_DOUBLE', 'EPOCH_FLAGS_VARI', 'RESIDUAL_IMAGE']. Note that for 'CROWDED_FIELD_IMAGE',
+            only the format 'fits' can be used, and its image, in the principal header, will not be available in the
+            returned dictionary. Set 'output_file' to retrieve all data: image + tables. Note that for 'RESIDUAL_IMAGE',
             only the format 'fits' can be used. Since the fits files only contain images, the returned table will be
             empty. Therefore, set 'output_file' to save the files to get access to their content.
         linking_parameter : str, optional, default SOURCE_ID, valid values: SOURCE_ID, TRANSIT_ID, IMAGE_ID
@@ -308,7 +309,9 @@ def load_data(self, ids, *, data_release=None, data_structure='INDIVIDUAL', retr
                 shutil.rmtree(path)
             else:
                 for file in files.keys():
-                    os.remove(os.path.join(os.getcwd(), path, file))
+                    final_file = os.path.join(os.getcwd(), path, file)
+                    if os.path.isfile(final_file):
+                        os.remove(final_file)
 
         if verbose:
             if output_file_specified:
@@ -340,13 +343,13 @@ def __get_data_files(output_file, path):
 
             if key.endswith('.fits'):
                 tables = []
-                with fits.open(value) as hduList:
-                    num_hdus = len(hduList)
-                    for i in range(1, num_hdus):
-                        table = Table.read(hduList[i], format='fits')
-                        Gaia.correct_table_units(table)
-                        tables.append(table)
-                    files[key] = tables
+                with fits.open(value, memmap=False) as hduList:
+                    for hdu in hduList:
+                        if isinstance(hdu, (TableHDU, BinTableHDU)):
+                            table = Table.read(hdu, format='fits')
+                            Gaia.correct_table_units(table)
+                            tables.append(table)
+                files[key] = tables
 
             elif key.endswith('.xml'):
                 tables = []
@@ -360,6 +363,12 @@ def __get_data_files(output_file, path):
                 tables.append(table)
                 files[key] = tables
 
+            elif key.endswith('.ecsv'):
+                tables = []
+                table = Table.read(value, format='ascii.ecsv', fast_reader=False)
+                tables.append(table)
+                files[key] = tables
+
             elif key.endswith('.json'):
                 tables = []
                 with open(value) as f:

astroquery/gaia/tests/test_gaiatap.py

@@ -942,11 +942,14 @@ def test_datalink_querier_load_data_ecsv(mock_datalink_querier_ecsv):
 
     assert len(result_dict) == 3
 
-    files = list(result_dict.keys())
-    files.sort()
-    assert files[0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.ecsv'
-    assert files[1] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.ecsv'
-    assert files[2] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.ecsv'
+    files = list(sorted(result_dict.items()))
+    assert files[0][0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.ecsv'
+    assert files[1][0] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.ecsv'
+    assert files[2][0] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.ecsv'
+
+    assert isinstance(files[0][1][0], Table)
+    assert isinstance(files[1][1][0], Table)
+    assert isinstance(files[2][1][0], Table)
 
     os.remove(os.path.join(os.getcwd(), datalink_output))
 
@@ -980,18 +983,21 @@ def test_datalink_querier_load_data_csv(mock_datalink_querier_csv):
 
     assert len(result_dict) == 3
 
-    files = list(result_dict.keys())
-    files.sort()
-    assert files[0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.csv'
-    assert files[1] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.csv'
-    assert files[2] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.csv'
+    files = list(sorted(result_dict.items()))
+    assert files[0][0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.csv'
+    assert files[1][0] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.csv'
+    assert files[2][0] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.csv'
+
+    assert isinstance(files[0][1][0], Table)
+    assert isinstance(files[1][1][0], Table)
+    assert isinstance(files[2][1][0], Table)
 
     os.remove(os.path.join(os.getcwd(), datalink_output))
 
     assert not os.path.exists(datalink_output)
 
 
-@pytest.mark.skip(reason="Thes fits files generate an error relatate to the unit 'log(cm.s**-2)")
+@pytest.mark.filterwarnings("ignore:")
 def test_datalink_querier_load_data_fits(mock_datalink_querier_fits):
     result_dict = mock_datalink_querier_fits.load_data(ids=[5937083312263887616], data_release='Gaia DR3',
                                                        data_structure='INDIVIDUAL',
@@ -1019,11 +1025,14 @@ def test_datalink_querier_load_data_fits(mock_datalink_querier_fits):
 
     assert len(result_dict) == 3
 
-    files = list(result_dict.keys())
-    files.sort()
-    assert files[0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.fits'
-    assert files[1] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.fits'
-    assert files[2] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.fits'
+    files = list(sorted(result_dict.items()))
+    assert files[0][0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.fits'
+    assert files[1][0] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.fits'
+    assert files[2][0] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.fits'
+
+    assert isinstance(files[0][1][0], Table)
+    assert isinstance(files[1][1][0], Table)
+    assert isinstance(files[2][1][0], Table)
 
     os.remove(os.path.join(os.getcwd(), datalink_output))
 
@@ -1072,10 +1081,12 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
     path.unlink()
 
 
-@pytest.mark.skip(reason="Thes fits files generate an error relatate to the unit 'log(cm.s**-2)")
-def test_load_data_fits(monkeypatch, tmp_path, tmp_path_factory):
+@pytest.mark.filterwarnings("ignore:")
+def test_load_data_fits(monkeypatch, tmp_path, tmp_path_factory, patch_datetime_now):
+    assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME
+
     now = datetime.datetime.now(datetime.timezone.utc)
-    output_file = 'datalink_output_' + now.strftime("%Y%m%dT%H%M%S") + '.zip'
+    output_file = 'datalink_output_' + now.strftime("%Y%m%dT%H%M%S.%f") + '.zip'
 
     path = Path(os.getcwd(), output_file)
 
@@ -1092,7 +1103,7 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
             "RETRIEVAL_TYPE": "epoch_photometry",
             "DATA_STRUCTURE": "INDIVIDUAL",
             "USE_ZIP_ALWAYS": "true"}
-        assert output_file == Path(os.getcwd(), 'datalink_output.zip')
+        assert str(path) == output_file
         assert verbose is True
 
     monkeypatch.setattr(TapPlus, "load_data", load_data_monkeypatched)