/vsis3/: support changing region and endpoint in a single operation, for virtual hosting as well

Previously this optimization was limited only to non virtual hosting scenarios.
An for virtual hosting, we did first the endpoint change and then the
region change.

Author: rouault

autotest/gcore/vsis3.py

@@ -988,6 +988,43 @@ def method_req_2(self, request):
 ###############################################################################
 
 
+@gdaltest.enable_exceptions()
+def test_vsis3_permanent_redirect_and_region_change(aws_test_config, webserver_port):
+
+    handler = webserver.SequentialHandler()
+    handler.add(
+        "GET",
+        "/test_vsis3_permanent_redirect_and_region_change/?delimiter=%2F&list-type=2",
+        301,
+        {"Content-type": "application/xml"},
+        f"""<?xml version="1.0" encoding="UTF-8"?>
+            <Error><Code>PermanentRedirect</Code><Message>The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.</Message><Endpoint>localhost:{webserver_port}</Endpoint><Bucket>test_vsis3_permanent_redirect_and_region_change</Bucket></Error>""",
+    )
+    handler.add(
+        "GET",
+        "/test_vsis3_permanent_redirect_and_region_change/?delimiter=%2F&list-type=2",
+        200,
+        {"Content-type": "application/xml"},
+        """<?xml version="1.0" encoding="UTF-8"?>
+            <ListBucketResult>
+                <Prefix></Prefix>
+                <Contents>
+                    <Key>test.bin</Key>
+                    <LastModified>1970-01-01T00:00:01.000Z</LastModified>
+                    <Size>123456</Size>
+                </Contents>
+            </ListBucketResult>""",
+    )
+    with webserver.install_http_handler(handler):
+        with gdal.VSIFile(
+            "/vsis3/test_vsis3_permanent_redirect_and_region_change/test.bin", "rb"
+        ):
+            pass
+
+
+###############################################################################
+
+
 @pytest.mark.parametrize("with_error_message", [True, False])
 def test_vsis3_request_timeout(aws_test_config, webserver_port, with_error_message):
     gdal.VSICurlClearCache()

port/cpl_aws.cpp

@@ -2809,48 +2809,54 @@ bool VSIS3HandleHelper::CanRestartOnError(const char *pszErrorMsg,
             }
             return false;
         }
-        if (!m_bUseVirtualHosting &&
-            strncmp(pszEndpoint, m_osBucket.c_str(), m_osBucket.size()) == 0 &&
-            pszEndpoint[m_osBucket.size()] == '.')
+
+        /* If we have a body with
+        <Error><Code>PermanentRedirect</Code><Message>The bucket you are
+        attempting to access must be addressed using the specified endpoint.
+        Please send all future requests to this
+        endpoint.</Message><Bucket>bucket</Bucket><Endpoint>bucket.region.s3.amazonaws.com</Endpoint></Error>
+        and headers like
+        x-amz-bucket-region: eu-west-1
+        then we must use s3-$(x-amz-bucket-region).amazon.com as endpoint. */
+        const char *pszRegionPtr =
+            (pszHeaders != nullptr)
+                ? strstr(pszHeaders, "x-amz-bucket-region: ")
+                : nullptr;
+        if (pszRegionPtr != nullptr)
         {
-            /* If we have a body with
-            <Error><Code>PermanentRedirect</Code><Message>The bucket you are
-            attempting to access must be addressed using the specified endpoint.
-            Please send all future requests to this
-            endpoint.</Message><Bucket>bucket.with.dot</Bucket><Endpoint>bucket.with.dot.s3.amazonaws.com</Endpoint></Error>
-            and headers like
-            x-amz-bucket-region: eu-west-1
-            and the bucket name has dot in it,
-            then we must use s3.$(x-amz-bucket-region).amazon.com as endpoint.
-            See #7154 */
-            const char *pszRegionPtr =
-                (pszHeaders != nullptr)
-                    ? strstr(pszHeaders, "x-amz-bucket-region: ")
-                    : nullptr;
-            if (strchr(m_osBucket.c_str(), '.') != nullptr &&
-                pszRegionPtr != nullptr)
+            std::string osRegion(pszRegionPtr +
+                                 strlen("x-amz-bucket-region: "));
+            size_t nPos = osRegion.find('\r');
+            if (nPos != std::string::npos)
+                osRegion.resize(nPos);
+            if (strncmp(pszEndpoint, m_osBucket.c_str(), m_osBucket.size()) ==
+                    0 &&
+                pszEndpoint[m_osBucket.size()] == '.')
             {
-                std::string osRegion(pszRegionPtr +
-                                     strlen("x-amz-bucket-region: "));
-                size_t nPos = osRegion.find('\r');
-                if (nPos != std::string::npos)
-                    osRegion.resize(nPos);
-                SetEndpoint(
-                    CPLSPrintf("s3.%s.amazonaws.com", osRegion.c_str()));
-                SetRegion(osRegion.c_str());
-                CPLDebug(AWS_DEBUG_KEY, "Switching to endpoint %s",
-                         m_osEndpoint.c_str());
-                CPLDebug(AWS_DEBUG_KEY, "Switching to region %s",
-                         m_osRegion.c_str());
-                CPLDestroyXMLNode(psTree);
-                if (!bIsTemporaryRedirect)
-                    VSIS3UpdateParams::UpdateMapFromHandle(this);
-                return true;
+                pszEndpoint += m_osBucket.size() + 1;
             }
 
+            SetEndpoint(pszEndpoint);
+            SetRegion(osRegion.c_str());
+            CPLDebug(AWS_DEBUG_KEY, "Switching to endpoint %s",
+                     m_osEndpoint.c_str());
+            CPLDebug(AWS_DEBUG_KEY, "Switching to region %s",
+                     m_osRegion.c_str());
+            CPLDestroyXMLNode(psTree);
+            if (!bIsTemporaryRedirect)
+                VSIS3UpdateParams::UpdateMapFromHandle(this);
+            return true;
+        }
+
+        if (!m_bUseVirtualHosting &&
+            m_osBucket.find('.') == std::string::npos &&
+            strncmp(pszEndpoint, m_osBucket.c_str(), m_osBucket.size()) == 0 &&
+            pszEndpoint[m_osBucket.size()] == '.')
+        {
             m_bUseVirtualHosting = true;
             CPLDebug(AWS_DEBUG_KEY, "Switching to virtual hosting");
         }
+
         SetEndpoint(m_bUseVirtualHosting ? pszEndpoint + m_osBucket.size() + 1
                                          : pszEndpoint);
         CPLDebug(AWS_DEBUG_KEY, "Switching to endpoint %s",