Garbled output when importing Gaia during maintenance

[astrofrog]

I'm currently seeing:

In [1]: from astroquery.gaia import Gaia
"ltr" dir="ltr" lang="en-GB">

Archive down for maintenance
"X-UA-Compatible" content="IE=edge" />

"UTF-8" />

Archive down for maintenance
"initial-scale=1.0, width=device-width" name="viewport" />

"Cache-Control" content="no-cache, no-store, must-revalidate">

"Pragma" content="no-cache">

"Expires" content="0">

"refresh" content="60" /> -->

"icon"

"https://www.cosmos.esa.int/o/CosmosTheme-theme/images/favicon.ico" />

Archive down for maintenance
"apple-touch-icon-precomposed"

"https://www.cosmos.esa.int/o/CosmosTheme-theme/images/apple-touch-icon-precomposed.png" />

Archive down for maintenance
Archive down for maintenance
"apple-touch-icon-precomposed" sizes="72x72"

"https://www.cosmos.esa.int/o/CosmosTheme-theme/images/apple-touch-icon-72x72-precomposed.png" />

Archive down for maintenance
"apple-touch-icon-precomposed" sizes="76x76"

"https://www.cosmos.esa.int/o/CosmosTheme-theme/images/apple-touch-icon-76x76-precomposed.png" />

Archive down for maintenance
"apple-touch-icon-precomposed" sizes="114x114"

"https://www.cosmos.esa.int/o/CosmosTheme-theme/images/apple-touch-icon-114x114-precomposed.png" />

Archive down for maintenance
"apple-touch-icon-precomposed" sizes="120x120"

"https://www.cosmos.esa.int/o/CosmosTheme-theme/images/apple-touch-icon-120x120-precomposed.png" />

Archive down for maintenance
"apple-touch-icon-precomposed" sizes="144x144"

"https://www.cosmos.esa.int/o/CosmosTheme-theme/images/apple-touch-icon-144x144-precomposed.png" />

Archive down for maintenance
"apple-touch-icon-precomposed" sizes="152x152"

"https://www.cosmos.esa.int/o/CosmosTheme-theme/images/apple-touch-icon-152x152-precomposed.png" />

Archive down for maintenance
"apple-touch-icon-precomposed" sizes="180x180"

"https://www.cosmos.esa.int/o/CosmosTheme-theme/images/apple-touch-icon-180x180-precomposed.png" />

Archive down for maintenance
"icon" sizes="180x180"

"https://www.cosmos.esa.int/o/CosmosTheme-theme/images/apple-touch-icon-180x180-precomposed.png" />

"text/css" rel="stylesheet" href="/gaia/html_page_resources/css/esdc.css">

"text/css" rel="stylesheet" href="/gaia/html_page_resources/css/gacs.css">

"text/css" rel="stylesheet" href="/gaia/html_page_resources/css/style.css">

Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
r;i[r]=i[r]||function(){

i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),

s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)

Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
"position: absolute; z-index: -32767; top: -20cm; width: 10cm; height: 10cm; visibility: hidden;"aria-hidden="true"></div>

"position: absolute; left: 0px; top: 0px; right: 0px; bottom: 0px; display: block;">

"position: absolute; left: 0px; top: 0px; right: 0px; bottom: 0px;">

"position: absolute; z-index: -32767; top: -20ex; width: 10em; height: 10ex; visibility: hidden;"aria-hidden="true">

Archive down for maintenance
"position: absolute; overflow: hidden; left: 0px; top: 0px; right: 0px; height: 120px;">

"position: absolute; left: 0px; top: 0px; right: 0px; bottom: 0px;">

"height: 14px; width: 100%; border-spacing: 0; border-collapse: collapse;" >

Archive down for maintenance
Archive down for maintenance
"left" style="vertical-align: top; padding: 0px">

"esdc_tb">

Archive down for maintenance
"esdc_top_sep"><img class="gwt-Image"

"1" height="20" src="/gaia/html_page_resources/images/top_sep.gif"

"height: 14px; width: 2px;"></img></li>

"esdc_external">

"esdch4">

"_blank" href="http://www.esa.int"> <span>

Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
"esdc_external">

"esdch4">

"http://www.esa.int/About_Us/ESAC" target="_blank">About

Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
"left" style="vertical-align: top; padding: 0px;"></td>

Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
"esdc_top-title" style="height: 100px">

"title" style="position: absolute; z-index: 2;">gaia archive (OPER)</div>

"http://www.esa.int">

"height: 100px; width: 851px; overflow: hidden; background: url('/gaia/html_page_resources/images/header-cosmos-gaia.png') no-repeat scroll 0px 0px transparent; position: absolute; top: 20px; right: 0px; z-index: 1;" />

Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
"esdc_tb"

"position: absolute; overflow: hidden; left: 0px; right: 0px; bottom: 0px; height: 20px;">

"float: right; margin: 0 0px 0 0;">

Archive down for maintenance
"esdch4">Copyright 2000 - 2020 © European Space

Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
"position: absolute; overflow: visible; top: 120px; bottom: 30px; left: 0px; right: 0px;">

Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
"content.html" width="100%" height="100%" seamless style="border: none"></iframe-->

"display: inline-block; position: fixed; top: 0; bottom: 0; left: 0; right: 0; width: 1024px; height: 600px; margin: auto; overflow: auto"-->

Archive down for maintenance
"top: 0; bottom: 0; left: 0; right: 0; width: 100%; height: 100%; margin:auto; overflow auto">-->

Archive down for maintenance
"top: 0; bottom: 0; left: 0; right: 0; width: 100%; height: 100%; margin: auto; overflow: auto">

"portlet-column portlet-column-only span12" id="column-1" style="margin: 20px 100px 20px 100px">

"portlet-dropzone portlet-column-content portlet-column-content-only" id="layout-column_column-1">

"portlet-boundary portlet-boundary_56_ portlet-static portlet-static-end portlet-borderless portlet-journal-content " id="p_p_id_56_INSTANCE_zChxaDZZCLJR_">

"p_56_INSTANCE_zChxaDZZCLJR"></span>

"portlet-borderless-container">

"portlet-body">

"journal-content-article">

"DR2background">

"padding:3%">

"https://www.cosmos.esa.int/web/gaia" target="_self">

"" src="https://www.cosmos.esa.int/documents/29201/29227/gaia_RGB_transparent_gif.gif/3bf5526a-3e81-4ce4-a648-0ac1f5bbd8ef?t=1378282536449" style="width: 100px; height: 59px; float: left;">

Archive down for maintenance
"http://gea.esac.esa.int/archive/" target="_blank">

"" src="https://www.cosmos.esa.int/documents/29201/29222/Gaia+Archive+logo.png/167275a5-9577-4d20-b0e0-a97625a73e0b?t=1469109633000" style="float: right;" width="60" height="59">

Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
"DataReleaseCountDown" style="text-align: center;color:#FFFFFF;">&nbsp;&nbsp;</h2> <p>&nbsp;</p>

"text-align: center;">

"color: rgb(137 8 8);font-size:30px;">Maintenance ongoing!</span>

Archive down for maintenance
"color: rgb(114,4,4);font-size:18px;">Our system is down for maintenance activities. Meanwhile, the Gaia data is accessible via our bulk-download repository (<a href="http://cdn.gea.esac.esa.int/Gaia/" target="_blank">http://cdn.gea.esac.esa.int/Gaia/</a>) - see our tutorial (<a href="https://www.cosmos.esa.int/web/gaia-users/archive/extract-data#bulk_download" target="_blank">https://www.cosmos.esa.int/web/gaia-users/archive/extract-data#bulk_download</a>) for more information - and via our partner and affiliated data centres (<a href="https://www.cosmos.esa.int/web/gaia/data-access#PartnerDataCentres" target="_blank">https://www.cosmos.esa.int/web/gaia/data-access#PartnerDataCentres</a>).</span><br/><br/><br/><br/>

"color: rgb(114,4,4);font-size:16px;">

"https://www.cosmos.esa.int/web/gaia-users/archive">Gaia's tutorials and help information</a>,

"https://www.cosmos.esa.int/web/gaia"> Gaia's resources for the scientific community</a>

"https://www.cosmos.esa.int/web/gaia/data-access#PartnerDataCentres">Gaia data from partner data centres</a>

Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance
Archive down for maintenance

[bsipocz]

This is unexpected.

1. We knew that they will be off for maintenance between 5-10 Dec, but it's not there yet
2. the prints are way too verbose and also includes non Gaia

cc @jespinosaar

[jespinosaar]

Dear all,
It seems that the message is a landing page provided during downtimes. In any case, Gaia is currently under maintenance, the team is currently applying some upgrades. This is independent from the downtime that will be held between 5-10 Dec.
I will ping @cosmoJFH , so he can let us know when Gaia is available again.

In any case, maybe we could agree on a way to inform about our downtimes. Maybe we can create an issue warning our users about this...

[cosmoJFH]

Hi,
it looks like that when the archive is under maintenance, the downtime page is returned.

Note that all the ESAC module would return a similar information, when their corresponding archive is down.

[bsipocz]

In any case, maybe we could agree on a way to inform about our downtimes. Maybe we can create an issue warning our users about this...

I'm afraid users very likely won't notice the issues.

As I recall these message warnings were much shorter for the other modules during downtime, so maybe we just need to keep it short and concise rather than returning the whole page?

[jespinosaar]

Yes, but if the services are down as well, no message will be provided. Maybe I can simply create an issue here informing all users?

[bsipocz]

Yes, but if the services are down as well, no message will be provided. Maybe I can simply create an issue here informing all users?

Users don't tend to read issues, and even the few who open issues usually try to resolve the problem locally first, before they end up reporting it.

Maybe we should add some basic logic to parse this landing page and return some warnings/exceptions based on that rather than trying to return the whole page as in the OP above?

[jespinosaar]

Just FYI, Gaia should be up and running again

[jespinosaar]

Yes, but if the services are down as well, no message will be provided. Maybe I can simply create an issue here informing all users?

Users don't tend to read issues, and even the few who open issues usually try to resolve the problem locally first, before they end up reporting it.

Maybe we should add some basic logic to parse this landing page and return some warnings/exceptions based on that rather than trying to return the whole page as in the OP above?

Mmm... this is something that only Gaia has, all the frontend URLs redirected to this page. For the rest of the Archives, an error 500 will be provided (I think). Then, maybe we can simply check during initialization if the connection to the TAP has really happened (e.g looking into the metadata) and, if not, provide a common message across the missions. This is something we can add to the code.

[bsipocz]

Oh, I see. The 500 may get handled better as we have a few raise_for_status in the code

[kslong]

Yes, I had this problem as well. For what is worth, this helped a bit to tame the large number of warnings

# --------------------------------------------------------------------------------
# Gaia loader (astroquery) — distinguishes dependency vs asset/service problems
# --------------------------------------------------------------------------------
def load_Gaia(probe_service: bool = True):
    """
    Return the astroquery.gaia.Gaia class.

    Raises RuntimeError with a message distinguishing:
      - astroquery not installed
      - external Gaia assets/services unavailable

    Parameters
    ----------
    probe_service : bool, default True
        If True, perform a tiny network-dependent check to detect
        external service unavailability immediately. If False, only
        import Gaia and let service failures occur later at first use.

    Returns
    -------
    Gaia : type
        The Gaia class from astroquery.gaia.

    Raises
    ------
    RuntimeError
        With a distinguishing message for the two failure cases.
    """
    # 1) Import-time: distinguish "not installed"
    try:
        from astroquery.gaia import Gaia
    except ImportError as e:
        raise RuntimeError(
            "astroquery (or 'astroquery.gaia') is not installed in this environment."
        ) from e

    # Optional: reduce astropy logger noise during later operations
    try:
        from astropy.logger import log
        log.setLevel('ERROR')
        from astropy.utils import iers
        iers.conf.auto_download = False
        iers.conf.auto_max_age = None
    except Exception:
        # If astropy settings change or aren't present, just continue.
        pass

    # 2) Service probe: distinguish "assets unavailable"
    if probe_service:
        try:
            # Minimal, fast probe (hits TAP briefly):
            Gaia.launch_job("SELECT TOP 1 source_id FROM gaiadr3.gaia_source", dump_to_file=False)
            # Alternatively, if you want absolutely minimal probing:
            # _ = Gaia.tap
        except Exception as e:
            raise RuntimeError(
                "Gaia external assets/services appear unavailable or unreachable."
            ) from e

    return Gaia

for my code