diff --git a/CHANGES.rst b/CHANGES.rst index 35c77dd068..74f3e12603 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,12 +26,14 @@ heasarc - Heasarc.locate_data returns empty rows with an error in the error_message column if there are no data associated with that row rather than filtering it out. [#3275] - utils.tap ^^^^^^^^^ - Get the cookie associated to the keys JSESSIONID or SESSION due to the tap library release at ESAC. [#3289] +- The method ``upload_table`` accepts file formats accepted by astropy's + ``Table.read()``. [#3295] + Infrastructure, Utility and Other Changes and Additions ------------------------------------------------------- diff --git a/astroquery/esa/euclid/__init__.py b/astroquery/esa/euclid/__init__.py index 1a8eaddf8e..f7c870854a 100644 --- a/astroquery/esa/euclid/__init__.py +++ b/astroquery/esa/euclid/__init__.py @@ -15,12 +15,6 @@ class Conf(_config.ConfigNamespace): Configuration parameters for `astroquery.esa.euclid`. """ - URL_BASE = _config.ConfigItem('https://eas.esac.esa.int/', 'Euclid base URL') - - EUCLID_TAP_SERVER = _config.ConfigItem('https://easidr.esac.esa.int/tap-server/tap', 'Euclid TAP Server') - EUCLID_DATALINK_SERVER = _config.ConfigItem("https://easidr.esac.esa.int/sas-dd/data?", "Euclid DataLink Server") - EUCLID_CUTOUT_SERVER = _config.ConfigItem("https://easidr.esac.esa.int/sas-cutout/cutout?", "Euclid Cutout Server") - ROW_LIMIT = _config.ConfigItem(50, "Number of rows to return from database query (set to -1 for unlimited).") diff --git a/astroquery/utils/tap/conn/tests/DummyConnHandler.py b/astroquery/utils/tap/conn/tests/DummyConnHandler.py index 02544c8765..8862b7f9d4 100644 --- a/astroquery/utils/tap/conn/tests/DummyConnHandler.py +++ b/astroquery/utils/tap/conn/tests/DummyConnHandler.py @@ -14,10 +14,11 @@ """ -from astroquery.utils.tap import taputils - import requests +from astroquery.utils.tap import taputils +from astroquery.utils.tap.conn.tapconn import TapConn + class DummyConnHandler: @@ -158,3 +159,12 @@ def execute_secure(self, subcontext=None, data=None, verbose=False): def get_host_url(self): return "my fake object" + + def encode_multipart(self, fields, files): + tap = TapConn(ishttps=False, host='host') + return tap.encode_multipart(fields, files) + + def execute_upload(self, data, + content_type="application/x-www-form-urlencoded", *, + verbose=False): + return self.defaultResponse diff --git a/astroquery/utils/tap/core.py b/astroquery/utils/tap/core.py index f13d515e6d..7475e290d8 100755 --- a/astroquery/utils/tap/core.py +++ b/astroquery/utils/tap/core.py @@ -15,11 +15,12 @@ """ import getpass import os -import requests import tempfile -from astropy.table.table import Table from urllib.parse import urlencode +import requests +from astropy.table.table import Table + from astroquery import log from astroquery.utils.tap import taputils from astroquery.utils.tap.conn.tapconn import TapConn @@ -1341,8 +1342,9 @@ def upload_table(self, *, upload_resource=None, table_name=None, table_descripti resource temporary table name associated to the uploaded resource table_description : str, optional, default None table description - format : str, optional, default 'VOTable' - resource format + format : str, optional, default 'votable' + resource format. Only formats described in + https://docs.astropy.org/en/stable/io/unified.html#built-in-table-readers-writers are accepted. verbose : bool, optional, default 'False' flag to display information about the process """ @@ -1381,9 +1383,7 @@ def upload_table(self, *, upload_resource=None, table_name=None, table_descripti log.info(f"Uploaded table '{table_name}'.") return None - def __uploadTableMultipart(self, resource, *, table_name=None, - table_description=None, - resource_format="VOTable", + def __uploadTableMultipart(self, resource, *, table_name=None, table_description=None, resource_format="votable", verbose=False): connHandler = self.__getconnhandler() if isinstance(resource, Table): @@ -1397,24 +1397,38 @@ def __uploadTableMultipart(self, resource, *, table_name=None, fh = tempfile.NamedTemporaryFile(delete=False) resource.write(fh, format='votable') fh.close() - f = open(fh.name, "r") - chunk = f.read() - f.close() + + with open(fh.name, "r") as f: + chunk = f.read() + os.unlink(fh.name) files = [['FILE', 'pytable', chunk]] - contentType, body = connHandler.encode_multipart(args, files) + content_type, body = connHandler.encode_multipart(args, files) else: if not (str(resource).startswith("http")): # upload from file args = { "TASKID": str(-1), "TABLE_NAME": str(table_name), "TABLE_DESC": str(table_description), - "FORMAT": str(resource_format)} + "FORMAT": 'votable'} log.info(f"Sending file: {resource}") - with open(resource, "r") as f: - chunk = f.read() - files = [['FILE', os.path.basename(resource), chunk]] - contentType, body = connHandler.encode_multipart(args, files) + if resource_format.lower() == 'votable': + with open(resource, "r") as f: + chunk = f.read() + files = [['FILE', os.path.basename(resource), chunk]] + else: + table = Table.read(str(resource), format=resource_format) + fh = tempfile.NamedTemporaryFile(delete=False) + table.write(fh, format='votable') + fh.close() + + with open(fh.name, "r") as f: + chunk = f.read() + + os.unlink(fh.name) + files = [['FILE', 'pytable', chunk]] + + content_type, body = connHandler.encode_multipart(args, files) else: # upload from URL args = { "TASKID": str(-1), @@ -1423,8 +1437,8 @@ def __uploadTableMultipart(self, resource, *, table_name=None, "FORMAT": str(resource_format), "URL": str(resource)} files = [['FILE', "", ""]] - contentType, body = connHandler.encode_multipart(args, files) - response = connHandler.execute_upload(body, contentType) + content_type, body = connHandler.encode_multipart(args, files) + response = connHandler.execute_upload(body, content_type) if verbose: print(response.status, response.reason) print(response.getheaders()) diff --git a/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.csv b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.csv new file mode 100644 index 0000000000..7a1d905640 --- /dev/null +++ b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.csv @@ -0,0 +1,3 @@ +source_id,ra,dec +3834447128563320320,149.8678677871318,1.12773018116361 +3834447162923057280,149.8953864417848,1.1345712682426434 diff --git a/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.ecsv b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.ecsv new file mode 100644 index 0000000000..562b5408df --- /dev/null +++ b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.ecsv @@ -0,0 +1,41 @@ +# %ECSV 1.0 +# --- +# delimiter: ',' +# datatype: +# - +# name: source_id +# datatype: int64 +# description: Unique source identifier (unique within a particular Data Release) +# meta: +# ucd: meta.id +# - +# name: ra +# datatype: float64 +# unit: deg +# description: Right ascension +# meta: +# ucd: pos.eq.ra;meta.main +# utype: stc:AstroCoords.Position3D.Value3.C1 +# CoosysSystem: ICRS +# CoosysEpoch: J2016.0 +# - +# name: dec +# datatype: float64 +# unit: deg +# description: Declination +# meta: +# ucd: pos.eq.dec;meta.main +# utype: stc:AstroCoords.Position3D.Value3.C2 +# CoosysSystem: ICRS +# CoosysEpoch: J2016.0 +# meta: +# name: votable +# QUERY_STATUS: OK +# QUERY: 'SELECT TOP 2 source_id, ra, dec FROM gaiadr3.gaia_source ' +# CAPTION: 'How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html' +# CITATION: 'How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html' +# JOBID: 1744351221317O +# RELEASE: Gaia DR3 +source_id,ra,dec +3834447128563320320,149.8678677871318,1.12773018116361 +3834447162923057280,149.8953864417848,1.1345712682426434 diff --git a/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.fits b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.fits new file mode 100644 index 0000000000..473480db99 Binary files /dev/null and b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.fits differ diff --git a/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.json b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.json new file mode 100644 index 0000000000..c4607947ec --- /dev/null +++ b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.json @@ -0,0 +1,95 @@ +{ + "VOTABLE": { + "RESOURCE": { + "INFO": [ + { + "_name": "QUERY_STATUS", + "_value": "OK" + }, + { + "_name": "QUERY", + "_value": "SELECT TOP 20 * FROM user_jferna01.my_votable_fits ", + "__cdata": "SELECT TOP 20 *\nFROM user_jferna01.my_votable_fits " + }, + { + "_name": "CAPTION", + "_value": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html", + "__cdata": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html" + }, + { + "_name": "CITATION", + "_value": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html", + "_ucd": "meta.bib", + "__cdata": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html" + }, + { + "_name": "PAGE", + "_value": "" + }, + { + "_name": "PAGE_SIZE", + "_value": "" + }, + { + "_name": "JOBID", + "_value": "1744360074103O", + "__cdata": "1744360074103O" + }, + { + "_name": "JOBNAME", + "_value": "" + } + ], + "TABLE": { + "FIELD": [ + { + "DESCRIPTION": "Object Identifier", + "_datatype": "int", + "_name": "my_votable_fits_oid" + }, + { + "_datatype": "long", + "_name": "source_id" + }, + { + "_datatype": "double", + "_name": "ra", + "_unit": "deg" + }, + { + "_datatype": "double", + "_name": "dec", + "_unit": "deg" + } + ], + "DATA": { + "TABLEDATA": { + "TR": [ + { + "TD": [ + "1", + "3834447128563320320", + "149.8678677871318", + "1.12773018116361" + ] + }, + { + "TD": [ + "2", + "3834447162923057280", + "149.8953864417848", + "1.1345712682426434" + ] + } + ] + } + } + }, + "_type": "results" + }, + "_version": "1.4", + "_xmlns": "http://www.ivoa.net/xml/VOTable/v1.3", + "_xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "_xsi:schemaLocation": "http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/votable-1.4.xsd" + } +} \ No newline at end of file diff --git a/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.vot b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.vot new file mode 100644 index 0000000000..1c25899ee9 --- /dev/null +++ b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.vot @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + +Object Identifier + + + + + + + +AAAAAAE1NrFZAAiGAEBiu8WSql90P/ILLs1s9TAAAAAAAjU2sWEACICAQGK8pwF3 +l+w/8ic0M8FVmA== + + + +
+ + diff --git a/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result_plain.vot b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result_plain.vot new file mode 100644 index 0000000000..00f9971752 --- /dev/null +++ b/astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result_plain.vot @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + +Object Identifier + + + + + + + + + + + +
13834447128563320320149.86786778713181.12773018116361
23834447162923057280149.89538644178481.1345712682426434
+ + diff --git a/astroquery/utils/tap/tests/setup_package.py b/astroquery/utils/tap/tests/setup_package.py index b141aa4098..43a3fddf1a 100644 --- a/astroquery/utils/tap/tests/setup_package.py +++ b/astroquery/utils/tap/tests/setup_package.py @@ -15,7 +15,6 @@ """ - import os @@ -24,7 +23,17 @@ def get_package_data(): paths = [os.path.join('data', '*.vot'), os.path.join('data', '*.xml'), + os.path.join('data', '*.csv'), + os.path.join('data', '*.ecsv'), + os.path.join('data', '*.json'), + os.path.join('data', '*.fits'), os.path.join('data', '*.fits.gz'), + os.path.join('data/test_upload_file', '*.vot'), + os.path.join('data/test_upload_file', '*.xml'), + os.path.join('data/test_upload_file', '*.csv'), + os.path.join('data/test_upload_file', '*.ecsv'), + os.path.join('data/test_upload_file', '*.json'), + os.path.join('data/test_upload_file', '*.fits'), ] # etc, add other extensions # you can also enlist files individually by names # finally construct and return a dict for the sub module diff --git a/astroquery/utils/tap/tests/test_tap.py b/astroquery/utils/tap/tests/test_tap.py index d2cdaaff4d..9d31e42979 100644 --- a/astroquery/utils/tap/tests/test_tap.py +++ b/astroquery/utils/tap/tests/test_tap.py @@ -4,22 +4,22 @@ TAP plus ============= -@author: Juan Carlos Segovia -@contact: juan.carlos.segovia@sciops.esa.int - European Space Astronomy Centre (ESAC) European Space Agency (ESA) - -Created on 30 jun. 2016 """ import gzip +import os + from pathlib import Path from unittest.mock import patch from urllib.parse import quote_plus, urlencode import numpy as np import pytest +from astropy.io.registry import IORegistryError from astropy.table import Table +from astropy.utils.data import get_pkg_data_filename + from requests import HTTPError from astroquery.utils.tap import taputils @@ -37,16 +37,16 @@ def read_file(filename): return filename.read_text() -TEST_DATA = {f.name: read_file(f) for f in Path(__file__).with_name("data").iterdir()} +TEST_DATA = {f.name: read_file(f) for f in Path(__file__).with_name("data").iterdir() if os.path.isfile(f)} def test_load_tables(): - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) responseLoadTable = DummyResponse(500) responseLoadTable.set_data(method='GET', body=TEST_DATA["test_tables.xml"]) tableRequest = "tables" - connHandler.set_response(tableRequest, responseLoadTable) + conn_handler.set_response(tableRequest, responseLoadTable) with pytest.raises(Exception): tap.load_tables() @@ -78,39 +78,39 @@ def test_load_tables(): def test_load_tables_parameters(): - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) responseLoadTable = DummyResponse(200) responseLoadTable.set_data(method='GET', body=TEST_DATA["test_tables.xml"]) tableRequest = "tables" - connHandler.set_response(tableRequest, responseLoadTable) + conn_handler.set_response(tableRequest, responseLoadTable) # empty request tap.load_tables() - assert connHandler.request == tableRequest + assert conn_handler.request == tableRequest # flag only_names=false & share_accessible=false: equals to # empty request tap.load_tables(only_names=False, include_shared_tables=False) - assert connHandler.request == tableRequest + assert conn_handler.request == tableRequest # flag only_names tableRequest = "tables?only_tables=true" - connHandler.set_response(tableRequest, responseLoadTable) + conn_handler.set_response(tableRequest, responseLoadTable) tap.load_tables(only_names=True) - assert connHandler.request == tableRequest + assert conn_handler.request == tableRequest # flag share_accessible=true tableRequest = "tables?share_accessible=true" - connHandler.set_response(tableRequest, responseLoadTable) + conn_handler.set_response(tableRequest, responseLoadTable) tap.load_tables(include_shared_tables=True) - assert connHandler.request == tableRequest + assert conn_handler.request == tableRequest # flag only_names=true & share_accessible=true tableRequest = "tables?only_tables=true&share_accessible=true" - connHandler.set_response(tableRequest, responseLoadTable) + conn_handler.set_response(tableRequest, responseLoadTable) tap.load_tables(only_names=True, include_shared_tables=True) - assert connHandler.request == tableRequest + assert conn_handler.request == tableRequest def test_load_table(): @@ -145,8 +145,8 @@ def test_load_table(): def test_launch_sync_job(): - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) responseLaunchJob = DummyResponse(500) responseLaunchJob.set_data(method='POST', body=TEST_DATA["job_1.vot"]) query = 'select top 5 * from table' @@ -159,7 +159,7 @@ def test_launch_sync_job(): "QUERY": quote_plus(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) jobRequest = f"sync?{sortedKey}" - connHandler.set_response(jobRequest, responseLaunchJob) + conn_handler.set_response(jobRequest, responseLaunchJob) with pytest.raises(Exception): tap.launch_job(query, maxrec=10) @@ -199,8 +199,8 @@ def test_launch_sync_job(): def test_launch_sync_job_secure(): - connHandler = DummyConnHandler() - tap = TapPlus(url="https://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="https://test:1111/tap", connhandler=conn_handler) responseLaunchJob = DummyResponse(500) responseLaunchJob.set_data(method='POST', body=TEST_DATA["job_1.vot"]) query = 'select top 5 * from table' @@ -213,7 +213,7 @@ def test_launch_sync_job_secure(): "QUERY": quote_plus(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) jobRequest = f"sync?{sortedKey}" - connHandler.set_response(jobRequest, responseLaunchJob) + conn_handler.set_response(jobRequest, responseLaunchJob) with pytest.raises(Exception): tap.launch_job(query, maxrec=10) @@ -253,8 +253,8 @@ def test_launch_sync_job_secure(): def test_launch_sync_job_redirect(): - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) responseLaunchJob = DummyResponse(500) jobid = '12345' resultsReq = f'sync/{jobid}' @@ -273,11 +273,11 @@ def test_launch_sync_job_redirect(): "QUERY": quote_plus(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) jobRequest = f"sync?{sortedKey}" - connHandler.set_response(jobRequest, responseLaunchJob) + conn_handler.set_response(jobRequest, responseLaunchJob) # Results response responseResultsJob = DummyResponse(500) responseResultsJob.set_data(method='GET', body=TEST_DATA["job_1.vot"]) - connHandler.set_response(resultsReq, responseResultsJob) + conn_handler.set_response(resultsReq, responseResultsJob) with pytest.raises(Exception): tap.launch_job(query) @@ -333,8 +333,8 @@ def test_launch_sync_job_redirect(): def test_launch_async_job(): - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) jobid = '12345' # Launch response responseLaunchJob = DummyResponse(500) @@ -353,17 +353,17 @@ def test_launch_async_job(): "QUERY": str(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) req = f"async?{sortedKey}" - connHandler.set_response(req, responseLaunchJob) + conn_handler.set_response(req, responseLaunchJob) # Phase response responsePhase = DummyResponse(500) responsePhase.set_data(method='GET', body="COMPLETED") req = f"async/{jobid}/phase" - connHandler.set_response(req, responsePhase) + conn_handler.set_response(req, responsePhase) # Results response responseResultsJob = DummyResponse(500) responseResultsJob.set_data(method='GET', body=TEST_DATA["job_1.vot"]) req = f"async/{jobid}/results/result" - connHandler.set_response(req, responseResultsJob) + conn_handler.set_response(req, responseResultsJob) with pytest.raises(Exception): tap.launch_job_async(query) @@ -409,14 +409,14 @@ def test_launch_async_job(): def test_start_job(): - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) jobid = '12345' # Phase POST response responsePhase = DummyResponse(200) responsePhase.set_data(method='POST') req = f"async/{jobid}/phase?PHASE=RUN" - connHandler.set_response(req, responsePhase) + conn_handler.set_response(req, responsePhase) # Launch response responseLaunchJob = DummyResponse(303) # list of list (httplib implementation for headers in response) @@ -433,17 +433,17 @@ def test_start_job(): "QUERY": str(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) req = f"async?{sortedKey}" - connHandler.set_response(req, responseLaunchJob) + conn_handler.set_response(req, responseLaunchJob) # Phase response responsePhase = DummyResponse(200) responsePhase.set_data(method='GET', body="COMPLETED") req = f"async/{jobid}/phase" - connHandler.set_response(req, responsePhase) + conn_handler.set_response(req, responsePhase) # Results response responseResultsJob = DummyResponse(200) responseResultsJob.set_data(method='GET', body=TEST_DATA["job_1.vot"]) req = f"async/{jobid}/results/result" - connHandler.set_response(req, responseResultsJob) + conn_handler.set_response(req, responseResultsJob) responseResultsJob.set_status_code(200) job = tap.launch_job_async(query, autorun=False) @@ -464,14 +464,14 @@ def test_start_job(): def test_abort_job(): - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) jobid = '12345' # Phase POST response responsePhase = DummyResponse(200) responsePhase.set_data(method='POST') req = f"async/{jobid}/phase?PHASE=ABORT" - connHandler.set_response(req, responsePhase) + conn_handler.set_response(req, responsePhase) # Launch response responseLaunchJob = DummyResponse(303) # list of list (httplib implementation for headers in response) @@ -489,7 +489,7 @@ def test_abort_job(): "QUERY": str(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) req = f"async?{sortedKey}" - connHandler.set_response(req, responseLaunchJob) + conn_handler.set_response(req, responseLaunchJob) job = tap.launch_job_async(query, autorun=False, maxrec=10) assert job is not None @@ -503,8 +503,8 @@ def test_abort_job(): def test_job_parameters(): - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) jobid = '12345' # Launch response responseLaunchJob = DummyResponse(303) @@ -523,17 +523,17 @@ def test_job_parameters(): "QUERY": str(query)} sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) req = f"async?{sortedKey}" - connHandler.set_response(req, responseLaunchJob) + conn_handler.set_response(req, responseLaunchJob) # Phase response responsePhase = DummyResponse(200) responsePhase.set_data(method='GET', body="COMPLETED") req = f"async/{jobid}/phase" - connHandler.set_response(req, responsePhase) + conn_handler.set_response(req, responsePhase) # Results response responseResultsJob = DummyResponse(200) responseResultsJob.set_data(method='GET', body=TEST_DATA["job_1.vot"]) req = f"async/{jobid}/results/result" - connHandler.set_response(req, responseResultsJob) + conn_handler.set_response(req, responseResultsJob) responseResultsJob.set_status_code(200) job = tap.launch_job_async(query, maxrec=10, autorun=False) @@ -544,12 +544,12 @@ def test_job_parameters(): responseParameters = DummyResponse(200) responseParameters.set_data(method='GET') req = f"async/{jobid}?param1=value1" - connHandler.set_response(req, responseParameters) + conn_handler.set_response(req, responseParameters) # Phase POST response responsePhase = DummyResponse(200) responsePhase.set_data(method='POST') req = f"async/{jobid}/phase?PHASE=RUN" - connHandler.set_response(req, responsePhase) + conn_handler.set_response(req, responsePhase) # send parameter OK job.send_parameter(name="param1", value="value1") @@ -562,12 +562,12 @@ def test_job_parameters(): def test_list_async_jobs(): - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) response = DummyResponse(500) response.set_data(method='GET', body=TEST_DATA["jobs_list.xml"]) req = "async" - connHandler.set_response(req, response) + conn_handler.set_response(req, response) with pytest.raises(Exception): tap.list_async_jobs() @@ -581,16 +581,16 @@ def test_list_async_jobs(): def test_data(): - connHandler = DummyConnHandler() + conn_handler = DummyConnHandler() tap = TapPlus(url="http://test:1111/tap", data_context="data", - connhandler=connHandler) + connhandler=conn_handler) responseResultsJob = DummyResponse(200) responseResultsJob.set_data(method='GET', body=TEST_DATA["job_1.vot"]) req = "?ID=1%2C2&format=votable" - connHandler.set_response(req, responseResultsJob) + conn_handler.set_response(req, responseResultsJob) req = "?ID=1%2C2" - connHandler.set_response(req, responseResultsJob) + conn_handler.set_response(req, responseResultsJob) # error responseResultsJob.set_status_code(500) @@ -614,14 +614,14 @@ def test_data(): def test_datalink(): - connHandler = DummyConnHandler() + conn_handler = DummyConnHandler() tap = TapPlus(url="http://test:1111/tap", datalink_context="datalink", - connhandler=connHandler) + connhandler=conn_handler) responseResultsJob = DummyResponse(200) responseResultsJob.set_data(method='GET', body=TEST_DATA["job_1.vot"]) req = "links?ID=1,2" - connHandler.set_response(req, responseResultsJob) + conn_handler.set_response(req, responseResultsJob) # error responseResultsJob.set_status_code(500) @@ -779,12 +779,12 @@ def test_get_current_column_values_for_update(): def test_update_user_table(): tableName = 'table' - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) dummyResponse = DummyResponse(200) dummyResponse.set_data(method='GET', body=TEST_DATA["test_table_update.xml"]) tableRequest = f"tables?tables={tableName}" - connHandler.set_response(tableRequest, dummyResponse) + conn_handler.set_response(tableRequest, dummyResponse) with pytest.raises(Exception): tap.update_user_table() @@ -835,7 +835,7 @@ def test_update_user_table(): } sortedKey = taputils.taputil_create_sorted_dict_key(dictTmp) req = f"tableEdit?{sortedKey}" - connHandler.set_response(req, responseEditTable) + conn_handler.set_response(req, responseEditTable) list_of_changes = [['alpha', 'flags', 'Ra'], ['delta', 'flags', 'Dec']] tap.update_user_table(table_name=tableName, list_of_changes=list_of_changes) @@ -845,8 +845,8 @@ def test_rename_table(): tableName = 'user_test.table_test_rename' newTableName = 'user_test.table_test_rename_new' newColumnNames = {'ra': 'alpha', 'dec': 'delta'} - connHandler = DummyConnHandler() - tap = TapPlus(url="http://test:1111/tap", connhandler=connHandler) + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) dummyResponse = DummyResponse(200) dummyResponse.set_data(method='GET', body=TEST_DATA["test_table_rename.xml"]) @@ -865,7 +865,7 @@ def test_rename_table(): "new_table_name": newTableName, "table_name": tableName, } - connHandler.set_response(f"TableTool?{urlencode(dictArgs)}", responseRenameTable) + conn_handler.set_response(f"TableTool?{urlencode(dictArgs)}", responseRenameTable) tap.rename_table(table_name=tableName, new_table_name=newTableName, new_column_names_dict=newColumnNames) @@ -938,7 +938,7 @@ def test_logout(mock_logout): assert (mock_logout.call_count == 2) -def test_upload_table(): +def test_upload_table_exception(): conn_handler = DummyConnHandler() tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) a = [1, 2, 3] @@ -969,6 +969,10 @@ def test___findCookieInHeader(): assert (result == "SESSION=ZjQ3MjIzMDAtNjNiYy00Mj") + result = tap._Tap__findCookieInHeader(headers, verbose=True) + + assert (result == "SESSION=ZjQ3MjIzMDAtNjNiYy00Mj") + headers = [('Date', 'Sat, 12 Apr 2025 05:10:47 GMT'), ('Server', 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips mod_jk/1.2.43'), ('Set-Cookie', 'JSESSIONID=E677B51BA5C4837347D1E17D4E36647E; Path=/data-server; Secure; HttpOnly'), @@ -980,3 +984,123 @@ def test___findCookieInHeader(): result = tap._Tap__findCookieInHeader(headers) assert (result == "JSESSIONID=E677B51BA5C4837347D1E17D4E36647E") + + headers = [('Date', 'Sat, 12 Apr 2025 05:10:47 GMT'), + ('Server', 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips mod_jk/1.2.43'), + ('X-Content-Type-Options', 'nosniff'), ('X-XSS-Protection', '0'), + ('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'), ('Pragma', 'no-cache'), + ('Expires', '0'), ('X-Frame-Options', 'SAMEORIGIN'), + ('Transfer-Encoding', 'chunked'), ('Content-Type', 'text/plain; charset=UTF-8')] + + result = tap._Tap__findCookieInHeader(headers) + + assert (result is None) + + headers = [('Date', 'Sat, 12 Apr 2025 05:10:47 GMT'), + ('Server', 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips mod_jk/1.2.43'), + ('Set-Cookie', 'HOLA=E677B51BA5C4837347D1E17D4E36647E; Path=/data-server; Secure; HttpOnly'), + ('X-Content-Type-Options', 'nosniff'), ('X-XSS-Protection', '0'), + ('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'), ('Pragma', 'no-cache'), + ('Expires', '0'), ('X-Frame-Options', 'SAMEORIGIN'), + ('Transfer-Encoding', 'chunked'), ('Content-Type', 'text/plain; charset=UTF-8')] + + result = tap._Tap__findCookieInHeader(headers) + + assert (result is None) + + +def test_upload_table(): + conn_handler = DummyConnHandler() + tap = TapPlus(url="http://test:1111/tap", connhandler=conn_handler) + + jobid = '12345' + dummyResponse = DummyResponse(303) + conn_handler.set_default_response(dummyResponse) + launchResponseHeaders = [ + ['location', f'http://test:1111/tap/async/{jobid}'] + ] + dummyResponse.set_data(method='POST', headers=launchResponseHeaders) + + package = "astroquery.utils.tap.tests" + + table_name = 'my_table' + file_csv = get_pkg_data_filename(os.path.join("data", 'test_upload_file', '1744351221317O-result.csv'), + package=package) + job = tap.upload_table(upload_resource=file_csv, table_name=table_name, format='csv') + + assert (job.jobid == jobid) + + file_ecsv = get_pkg_data_filename(os.path.join("data", 'test_upload_file', '1744351221317O-result.ecsv'), + package=package) + job = tap.upload_table(upload_resource=file_ecsv, table_name=table_name, format='ascii.ecsv') + + assert (job.jobid == jobid) + + file_fits = get_pkg_data_filename(os.path.join("data", 'test_upload_file', '1744351221317O-result.fits'), + package=package) + job = tap.upload_table(upload_resource=file_fits, table_name=table_name, format='fits') + + assert (job.jobid == jobid) + + file_vot = get_pkg_data_filename(os.path.join("data", 'test_upload_file', '1744351221317O-result.vot'), + package=package) + job = tap.upload_table(upload_resource=file_vot, table_name=table_name) + + assert (job.jobid == jobid) + + file_plain_vot = get_pkg_data_filename(os.path.join("data", 'test_upload_file', '1744351221317O-result_plain.vot'), + package=package) + job = tap.upload_table(upload_resource=file_plain_vot, table_name=table_name, table_description="my description") + + assert (job.jobid == jobid) + + # check invalid file + file_json = get_pkg_data_filename(os.path.join("data", 'test_upload_file', '1744351221317O-result.json'), + package=package) + + with pytest.raises(IORegistryError) as exc_info: + job = tap.upload_table(upload_resource=file_json, table_name=table_name, format='json') + + argument_ = "No reader defined for format 'json' and class 'Table'." + assert (argument_ in str(exc_info.value)) + + # Make use of an astropy table + table = Table.read(str(file_ecsv)) + job = tap.upload_table(upload_resource=table, table_name=table_name, table_description="my description", + format='ecsv') + + assert (job.jobid == jobid) + + # check missing parameters + with pytest.raises(ValueError) as exc_info: + job = tap.upload_table(upload_resource=file_json, table_name=None) + + argument_ = "Missing mandatory argument 'table_name'" + assert (argument_ in str(exc_info.value)) + + with pytest.raises(ValueError) as exc_info: + job = tap.upload_table(upload_resource=None, table_name="my_table") + + argument_ = "Missing mandatory argument 'upload_resource'" + assert (argument_ in str(exc_info.value)) + + job = tap.upload_table(upload_resource="https://gea.esa.esac.int", table_name=table_name, + table_description="my description", + format='ecsv') + + assert (job.jobid == jobid) + + # check exception + dummyResponse = DummyResponse(500) + conn_handler.set_default_response(dummyResponse) + launchResponseHeaders = [ + ['location', f'http://test:1111/tap/async/{jobid}'], + ['multipart/form-data', 'boundary={aaaaaaaaaaaaaa}'] + ] + dummyResponse.set_data(method='POST', headers=launchResponseHeaders) + + with pytest.raises(AttributeError) as exc_info: + job = tap.upload_table(upload_resource=file_csv, table_name=table_name, format='csv') + + argument_ = "'NoneType' object has no attribute 'decode'" + assert (argument_ in str(exc_info.value)) diff --git a/docs/esa/euclid/euclid.rst b/docs/esa/euclid/euclid.rst index daaee8c7dc..44ac6d4f49 100644 --- a/docs/esa/euclid/euclid.rst +++ b/docs/esa/euclid/euclid.rst @@ -37,7 +37,7 @@ Agency EUCLID Archive using a TAP+ REST service. TAP+ is an extension of Table A specified by the International Virtual Observatory Alliance (IVOA: http://www.ivoa.net). -The TAP query language is Astronomical Data Query Language +The TAP query language is Astronomical Data Query Language (ADQL: https://www.ivoa.net/documents/ADQL/20231215/index.html ), which is similar to Structured Query Language (SQL), widely used to query databases. @@ -863,7 +863,11 @@ surrounded by quotation marks, i.e.: *user_.""*): 2.5.2. Uploading table from file ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -A file containing a table (votable, fits or csv) can be uploaded to the user's private area. +A file containing a table can be uploaded to the user private area. Only a file associated to any of the formats described in +https://docs.astropy.org/en/stable/io/unified.html#built-in-table-readers-writers, and automatically identified by its suffix +or content can be used. Note that for a multi-extension fits file with multiple tables, the first table found will be used. +For any other format, the file can be transformed into an astropy Table (https://docs.astropy.org/en/stable/io/unified.html#getting-started-with-table-i-o) +and passed to the method. The parameter 'format' must be provided when the input file is not a votable file. @@ -874,7 +878,7 @@ Your schema name will be automatically added to the provided table name. >>> from astroquery.esa.euclid import Euclid >>> Euclid.login() - >>> job = Euclid.upload_table(upload_resource="1535553556177O-result.vot", table_name="table_test_from_file", format="VOTable") + >>> job = Euclid.upload_table(upload_resource="1535553556177O-result.vot", table_name="table_test_from_file", format="votable") Sending file: 1535553556177O-result.vot Uploaded table 'table_test_from_file'. diff --git a/docs/gaia/gaia.rst b/docs/gaia/gaia.rst index 4fc1565059..ee8cc161ec 100644 --- a/docs/gaia/gaia.rst +++ b/docs/gaia/gaia.rst @@ -581,7 +581,11 @@ surrounded by quotation marks, i.e.: *user_.""*):: 2.3.2. Uploading table from file ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -A file containing a table (votable, fits or csv) can be uploaded to the user private area. +A file containing a table can be uploaded to the user private area. Only a file associated to any of the formats described in +https://docs.astropy.org/en/stable/io/unified.html#built-in-table-readers-writers, and automatically identified by its suffix +or content can be used. Note that for a multi-extension fits file with multiple tables, the first table found will be used. +For any other format, the file can be transformed into an astropy Table (https://docs.astropy.org/en/stable/io/unified.html#getting-started-with-table-i-o) +and passed to the method. The parameter 'format' must be provided when the input file is not a votable file. @@ -665,7 +669,7 @@ A table from the user private area can be deleted as follows:: >>> from astroquery.gaia import Gaia >>> Gaia.login_gui() - >>> job = Gaia.delete_user_table("table_test_from_file") + >>> job = Gaia.delete_user_table(table_name="table_test_from_file") Table 'table_test_from_file' deleted. 2.5. Updating table metadata