If a HEAD request raises an exception fallback to a GET (#185)

* If a HEAD request raises an exception fallback to a GET

* Fix it better

Author: Cadair

parfive/tests/test_downloader.py

@@ -869,3 +869,16 @@ def test_invalid_server_checksum(httpserver, tmpdir, caplog, checksum):
 
     assert len(f.errors) == 0
     assert "Got invalid checksum:" in caplog.messages[0]
+
+
+@pytest.mark.parametrize(
+    "url", ["https://gong2.nso.edu/oQR/zqs/201912/mrzqs191231/mrzqs191231t2304c2225_011.fits.gz"]
+)
+@pytest.mark.allow_hosts(True)
+def test_problematic_http_urls(url, tmpdir):
+    """
+    This test checks that certain URLs which have caused trouble continue to work.
+    """
+    res = Downloader.simple_download([url], path=tmpdir)
+    assert len(res) == 1, res.errors
+    assert not res.errors

parfive/utils.py

@@ -333,10 +333,14 @@ async def session_head_or_get(session: aiohttp.ClientSession, url: str, **kwargs
     Try and make a HEAD request to the resource and fallback to a get
     request if that fails.
     """
-    async with session.head(url, **kwargs) as resp:
-        if resp.status == 200:
-            yield resp
-            return
+    try:
+        async with session.head(url, **kwargs) as resp:
+            if resp.status == 200:
+                yield resp
+                return
+    # Catch the situation where the server just ignores the HEAD request
+    except aiohttp.client_exceptions.ServerDisconnectedError:
+        pass
 
     async with session.get(url, **kwargs) as resp:
         yield resp