Skip to content

GAIA: the method load_data must parse ecsv files #3500

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bsipocz merged 2 commits into from
Jan 16, 2026
+52 −31
Merged

GAIA: the method load_data must parse ecsv files #3500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bb505a5
C9GACS-1079 the method load_data properly parses ecsv files
Jan 10, 2026
7a1dc3d
C9GACS-1079 add PR reference number
Jan 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ gaia
^^^^

- New datalink DR4 retrieval type RESIDUAL_IMAGE. [#3489]
- The method ``load_data`` parses ecsv files [#3500].

esa.hubble
^^^^^^^^^^
Expand Down
31 changes: 20 additions & 11 deletions astroquery/gaia/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from astropy.coordinates import Angle
from astropy.io import fits
from astropy.io import votable
from astropy.io.fits import TableHDU, BinTableHDU
from astropy.table import Table
from astropy.units import Quantity
from astropy.utils.decorators import deprecated_renamed_argument
Expand Down Expand Up @@ -194,9 +195,9 @@ def load_data(self, ids, *, data_release=None, data_structure='INDIVIDUAL', retr
'EPOCH_PHOTOMETRY_CROWDED_FIELD', 'EPOCH_IMAGE', 'EPOCH_PHOTOMETRY_CCD', 'EPOCH_SPECTRUM_XP_SSO',
'EPOCH_SPECTRUM_XP_CROWDING', 'MEAN_SPECTRUM_XP', 'EPOCH_SPECTRUM_XP', 'CROWDED_FIELD_IMAGE',
'EPOCH_ASTROMETRY_BRIGHT', 'MEAN_SPECTRUM_XP_GRAVLENS', 'EPOCH_FLAGS_NSS', 'EPOCH_PARAMETERS_RVS_SINGLE',
'EPOCH_PARAMETERS_RVS_DOUBLE', 'EPOCH_FLAGS_VARI']. Note that for 'CROWDED_FIELD_IMAGE', only the format
'fits' can be used, and its image, in the principal header, will not be available in the returned
dictionary. Set 'output_file' to retrieve all data: image + tables. Note that for 'RESIDUAL_IMAGE',
'EPOCH_PARAMETERS_RVS_DOUBLE', 'EPOCH_FLAGS_VARI', 'RESIDUAL_IMAGE']. Note that for 'CROWDED_FIELD_IMAGE',
only the format 'fits' can be used, and its image, in the principal header, will not be available in the
returned dictionary. Set 'output_file' to retrieve all data: image + tables. Note that for 'RESIDUAL_IMAGE',
only the format 'fits' can be used. Since the fits files only contain images, the returned table will be
empty. Therefore, set 'output_file' to save the files to get access to their content.
linking_parameter : str, optional, default SOURCE_ID, valid values: SOURCE_ID, TRANSIT_ID, IMAGE_ID
Expand Down Expand Up @@ -308,7 +309,9 @@ def load_data(self, ids, *, data_release=None, data_structure='INDIVIDUAL', retr
shutil.rmtree(path)
else:
for file in files.keys():
os.remove(os.path.join(os.getcwd(), path, file))
final_file = os.path.join(os.getcwd(), path, file)
if os.path.isfile(final_file):
os.remove(final_file)

if verbose:
if output_file_specified:
Expand Down Expand Up @@ -340,13 +343,13 @@ def __get_data_files(output_file, path):

if key.endswith('.fits'):
tables = []
with fits.open(value) as hduList:
num_hdus = len(hduList)
for i in range(1, num_hdus):
table = Table.read(hduList[i], format='fits')
Gaia.correct_table_units(table)
tables.append(table)
files[key] = tables
with fits.open(value, memmap=False) as hduList:
for hdu in hduList:
if isinstance(hdu, (TableHDU, BinTableHDU)):
table = Table.read(hdu, format='fits')
Gaia.correct_table_units(table)
tables.append(table)
files[key] = tables

elif key.endswith('.xml'):
tables = []
Expand All @@ -360,6 +363,12 @@ def __get_data_files(output_file, path):
tables.append(table)
files[key] = tables

elif key.endswith('.ecsv'):
tables = []
table = Table.read(value, format='ascii.ecsv', fast_reader=False)
tables.append(table)
files[key] = tables

elif key.endswith('.json'):
tables = []
with open(value) as f:
Expand Down
51 changes: 31 additions & 20 deletions astroquery/gaia/tests/test_gaiatap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -942,11 +942,14 @@ def test_datalink_querier_load_data_ecsv(mock_datalink_querier_ecsv):

assert len(result_dict) == 3

files = list(result_dict.keys())
files.sort()
assert files[0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.ecsv'
assert files[1] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.ecsv'
assert files[2] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.ecsv'
files = list(sorted(result_dict.items()))
assert files[0][0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.ecsv'
assert files[1][0] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.ecsv'
assert files[2][0] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.ecsv'

assert isinstance(files[0][1][0], Table)
assert isinstance(files[1][1][0], Table)
assert isinstance(files[2][1][0], Table)

os.remove(os.path.join(os.getcwd(), datalink_output))

Expand Down Expand Up @@ -980,18 +983,21 @@ def test_datalink_querier_load_data_csv(mock_datalink_querier_csv):

assert len(result_dict) == 3

files = list(result_dict.keys())
files.sort()
assert files[0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.csv'
assert files[1] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.csv'
assert files[2] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.csv'
files = list(sorted(result_dict.items()))
assert files[0][0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.csv'
assert files[1][0] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.csv'
assert files[2][0] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.csv'

assert isinstance(files[0][1][0], Table)
assert isinstance(files[1][1][0], Table)
assert isinstance(files[2][1][0], Table)

os.remove(os.path.join(os.getcwd(), datalink_output))

assert not os.path.exists(datalink_output)


@pytest.mark.skip(reason="Thes fits files generate an error relatate to the unit 'log(cm.s**-2)")
@pytest.mark.filterwarnings("ignore:")
def test_datalink_querier_load_data_fits(mock_datalink_querier_fits):
result_dict = mock_datalink_querier_fits.load_data(ids=[5937083312263887616], data_release='Gaia DR3',
data_structure='INDIVIDUAL',
Expand Down Expand Up @@ -1019,11 +1025,14 @@ def test_datalink_querier_load_data_fits(mock_datalink_querier_fits):

assert len(result_dict) == 3

files = list(result_dict.keys())
files.sort()
assert files[0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.fits'
assert files[1] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.fits'
assert files[2] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.fits'
files = list(sorted(result_dict.items()))
assert files[0][0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.fits'
assert files[1][0] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.fits'
assert files[2][0] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.fits'

assert isinstance(files[0][1][0], Table)
assert isinstance(files[1][1][0], Table)
assert isinstance(files[2][1][0], Table)

os.remove(os.path.join(os.getcwd(), datalink_output))

Expand Down Expand Up @@ -1072,10 +1081,12 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
path.unlink()


@pytest.mark.skip(reason="Thes fits files generate an error relatate to the unit 'log(cm.s**-2)")
def test_load_data_fits(monkeypatch, tmp_path, tmp_path_factory):
@pytest.mark.filterwarnings("ignore:")
def test_load_data_fits(monkeypatch, tmp_path, tmp_path_factory, patch_datetime_now):
assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME

now = datetime.datetime.now(datetime.timezone.utc)
output_file = 'datalink_output_' + now.strftime("%Y%m%dT%H%M%S") + '.zip'
output_file = 'datalink_output_' + now.strftime("%Y%m%dT%H%M%S.%f") + '.zip'

path = Path(os.getcwd(), output_file)

Expand All @@ -1092,7 +1103,7 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
"RETRIEVAL_TYPE": "epoch_photometry",
"DATA_STRUCTURE": "INDIVIDUAL",
"USE_ZIP_ALWAYS": "true"}
assert output_file == Path(os.getcwd(), 'datalink_output.zip')
assert str(path) == output_file
assert verbose is True

monkeypatch.setattr(TapPlus, "load_data", load_data_monkeypatched)
Expand Down
Loading