APPS-3978 (Authenticated)HttpFileSource: Fallback to GET if HEAD is rejected (#149)

* fallback to GET if HEAD is rejected

* fix error

* pump version

---------

Co-authored-by: Blue Hoang <bhoang@dnanexus.com>

Author: wormsik

common/RELEASE_NOTES.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+* `HttpFileSource` and `AuthenticatedHttpFileSource` now transparently retry with `GET` when the server rejects `HEAD` with `405 Method Not Allowed`, so `exists` / `size` work against servers that selectively register methods.
+* `AuthenticatedHttpFileAccessProtocol` no longer attaches Bearer credentials to plain HTTP requests; credentials are only sent over HTTPS.
+* `AuthenticatedHttpFileSource.size` returns `-1L` when the server responds `405` to both HEAD and GET probes.
+
+## 0.12.0 (2026-06-11)
+
 * Adds `AuthenticatedHttpFileSource` and `AuthenticatedHttpFileAccessProtocol` for HTTP/HTTPS access with Bearer credentials.
 * Fixes `HttpFileSource.getParent` to return `None` at the root URI instead of a self-referential parent, and fixes `localize` to write cached bytes directly (preserving binary content) and to create missing parent directories.
 

common/src/main/resources/application.conf

@@ -1,5 +1,5 @@
 dxCommon {
-    version = "0.11.6-SNAPSHOT"
+    version = "0.12.1-SNAPSHOT"
 }
 
 #

common/src/main/scala/dx/util/AuthenticatedHttpFileSource.scala

@@ -53,14 +53,25 @@ case class AuthenticatedHttpFileSource(
 
   private var hasBytes: Boolean = false
 
+  private def openConnection(method: String): HttpURLConnection = {
+    val conn = uri.toURL.openConnection().asInstanceOf[HttpURLConnection]
+    conn.setRequestMethod(method)
+    credentials.foreach { c =>
+      conn.setRequestProperty("Authorization", s"${c.scheme.value} ${c.credentials}")
+    }
+    conn
+  }
+
   private def withConnection[T](method: String = "HEAD")(fn: HttpURLConnection => T): T = {
-    val url = uri.toURL
     var conn: HttpURLConnection = null
     try {
-      conn = url.openConnection().asInstanceOf[HttpURLConnection]
-      conn.setRequestMethod(method)
-      credentials.foreach { c =>
-        conn.setRequestProperty("Authorization", s"${c.scheme.value} ${c.credentials}")
+      conn = openConnection(method)
+      // Many servers reject HEAD on a resource that GET would serve (RFC 7231
+      // §6.5.5). Transparently retry such requests with GET so callers can rely
+      // on HEAD-style "exists / size" probes regardless of server quirks.
+      if (method == "HEAD" && conn.getResponseCode == HttpURLConnection.HTTP_BAD_METHOD) {
+        conn.disconnect()
+        conn = openConnection("GET")
       }
       fn(conn)
     } finally {
@@ -156,11 +167,20 @@ case class AuthenticatedHttpFileSource(
   override lazy val size: Long = {
     withConnection() { conn =>
       val responseCode = conn.getResponseCode
-      if (responseCode != HttpURLConnection.HTTP_OK) {
-        throwOnWrongAuth(responseCode)
-        throw new Exception(s"Error getting size of URL ${uri}: HTTP ${responseCode}")
+      responseCode match {
+        case HttpURLConnection.HTTP_OK =>
+          conn.getContentLengthLong
+        case HttpURLConnection.HTTP_BAD_METHOD =>
+          // Some endpoints reject both HEAD and GET for metadata probes.
+          // Returning unknown size allows downstream reads to proceed.
+          logger.trace(
+              s"Server at ${uri.getHost} returned 405 for size probe; skipping file size check"
+          )
+          -1L
+        case _ =>
+          throwOnWrongAuth(responseCode)
+          throw new Exception(s"Error getting size of URL ${uri}: HTTP ${responseCode}")
       }
-      conn.getContentLengthLong
     }
   }
 
@@ -268,9 +288,17 @@ case class AuthenticatedHttpFileAccessProtocol(
   }
 
   private def credentialsForUri(uri: URI): Option[HttpCredentials] = {
-    domainBearerTokenForUri(uri).map { token =>
-      logger.trace(s"Using Bearer token authenticated HTTP for import from: ${uri.getHost}")
-      HttpCredentials(HttpAuthenticationScheme.Bearer, token)
+    domainBearerTokenForUri(uri).flatMap { token =>
+      Option(uri.getScheme) match {
+        case Some(scheme) if scheme.equalsIgnoreCase(FileUtils.HttpsScheme) =>
+          logger.trace(s"Using Bearer token authenticated HTTP for import from: ${uri.getHost}")
+          Some(HttpCredentials(HttpAuthenticationScheme.Bearer, token))
+        case _ =>
+          logger.warning(
+              s"Skipping Bearer token for ${uri.getHost}; credentials are only attached to HTTPS requests"
+          )
+          None
+      }
     }
   }
 

common/src/main/scala/dx/util/FileSource.scala

@@ -565,12 +565,23 @@ case class HttpFileSource(
 
   private var hasBytes: Boolean = false
 
+  private def openConnection(method: String): HttpURLConnection = {
+    val conn = uri.toURL.openConnection().asInstanceOf[HttpURLConnection]
+    conn.setRequestMethod(method)
+    conn
+  }
+
   private def withConnection[T](fn: HttpURLConnection => T): T = {
-    val url = uri.toURL
     var conn: HttpURLConnection = null
     try {
-      conn = url.openConnection().asInstanceOf[HttpURLConnection]
-      conn.setRequestMethod("HEAD")
+      conn = openConnection("HEAD")
+      // Many servers reject HEAD on a resource that GET would serve (RFC 7231
+      // §6.5.5). Transparently retry such requests with GET so HEAD-style
+      // "exists / size" probes work regardless of server quirks.
+      if (conn.getResponseCode == HttpURLConnection.HTTP_BAD_METHOD) {
+        conn.disconnect()
+        conn = openConnection("GET")
+      }
       fn(conn)
     } finally {
       if (conn != null) {

common/src/test/scala/dx/util/AuthenticatedHttpFileSourceTest.scala

@@ -45,6 +45,26 @@ class AuthenticatedHttpFileSourceTest extends AnyFlatSpec with Matchers with Bef
     if (header.contains(s"Bearer $expectedToken")) respond(exchange, 200, okBody)
     else respond(exchange, 401, Array.emptyByteArray)
   }
+  // Refuses HEAD with 405 but serves GET normally — simulates servers that
+  // selectively register methods (Allow: GET, POST), common with some
+  // application frameworks, CDNs, and dynamic endpoints.
+  private val handlerHeadNotAllowed: HttpHandler = (exchange: HttpExchange) => {
+    if (exchange.getRequestMethod == "HEAD") {
+      exchange.getResponseHeaders.set("Allow", "GET")
+      exchange.sendResponseHeaders(405, -1)
+      exchange.close()
+    } else {
+      respond(exchange, 200, okBody)
+    }
+  }
+
+  // Refuses both HEAD and GET with 405.
+  private val handlerAlways405: HttpHandler = (exchange: HttpExchange) => {
+    exchange.getResponseHeaders.set("Allow", "POST")
+    exchange.sendResponseHeaders(405, -1)
+    exchange.close()
+  }
+
   // Responds 200 OK without a Content-Length header — simulates servers using
   // chunked transfer encoding (e.g. GitHub raw URLs).
   private val handlerNoLength: HttpHandler = (exchange: HttpExchange) => {
@@ -71,6 +91,8 @@ class AuthenticatedHttpFileSourceTest extends AnyFlatSpec with Matchers with Bef
     server.createContext("/protected", handlerBearerProtected)
     server.createContext("/binary", (exchange: HttpExchange) => respond(exchange, 200, binaryBody))
     server.createContext("/chunked", handlerNoLength)
+    server.createContext("/head405", handlerHeadNotAllowed)
+    server.createContext("/always405", handlerAlways405)
     server.setExecutor(null)
     server.start()
     val port = server.getAddress.getPort
@@ -263,6 +285,33 @@ class AuthenticatedHttpFileSourceTest extends AnyFlatSpec with Matchers with Bef
     fs("/chunked/file.txt").size shouldBe -1L
   }
 
+  it should "transparently retry with GET when the server rejects HEAD with 405" in {
+    fs("/head405/file.txt").exists shouldBe true
+    fs("/head405/file.txt").size shouldBe okBody.length.toLong
+  }
+
+  it should "return -1 from size when both HEAD and GET are rejected with 405" in {
+    fs("/always405/file.txt").size shouldBe -1L
+  }
+
+  // --- protocol credential attachment ----------------------------------
+
+  it should "not attach bearer credentials to plain HTTP URLs" in {
+    val protocol = AuthenticatedHttpFileAccessProtocol(
+        domainBearerTokens = Map("example.com" -> "secret")
+    )
+    val resolved = protocol.resolve("http://example.com/workflow.wdl")
+    resolved.credentials shouldBe None
+  }
+
+  it should "attach bearer credentials to HTTPS URLs" in {
+    val protocol = AuthenticatedHttpFileAccessProtocol(
+        domainBearerTokens = Map("example.com" -> "secret")
+    )
+    val resolved = protocol.resolve("https://example.com/workflow.wdl")
+    resolved.credentials shouldBe Some(bearer("secret"))
+  }
+
   // --- getParent edge cases --------------------------------------------
 
   it should "return None from getParent at the root URI" in {

common/src/test/scala/dx/util/HttpFileSourceTest.scala

@@ -31,6 +31,17 @@ class HttpFileSourceTest extends AnyFlatSpec with Matchers with BeforeAndAfterAl
   override def beforeAll(): Unit = {
     server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0)
     server.createContext("/binary", (exchange: HttpExchange) => respond(exchange, 200, binaryBody))
+    // Refuses HEAD with 405 but serves GET normally — mirrors servers that
+    // selectively register methods (RFC 7231 §6.5.5).
+    server.createContext(
+        "/head405",
+        (exchange: HttpExchange) =>
+          if (exchange.getRequestMethod == "HEAD") {
+            exchange.getResponseHeaders.set("Allow", "GET")
+            exchange.sendResponseHeaders(405, -1)
+            exchange.close()
+          } else respond(exchange, 200, binaryBody)
+    )
     server.setExecutor(null)
     server.start()
     baseUri = URI.create(s"http://127.0.0.1:${server.getAddress.getPort}")
@@ -74,4 +85,9 @@ class HttpFileSourceTest extends AnyFlatSpec with Matchers with BeforeAndAfterAl
       FileUtils.deleteRecursive(tempRoot)
     }
   }
+
+  it should "transparently retry with GET when the server rejects HEAD with 405" in {
+    fs("/head405/file.bin").exists shouldBe true
+    fs("/head405/file.bin").size shouldBe binaryBody.length.toLong
+  }
 }