From 4c2d1bbde58a242c9ffef714ad5c3a5aaa663ffe Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 23 May 2026 16:38:21 +0200 Subject: [PATCH] /vsis3/: support changing region and endpoint in a single operation, for virtual hosting as well Previously this optimization was limited only to non virtual hosting scenarios. An for virtual hosting, we did first the endpoint change and then the region change. --- autotest/gcore/vsis3.py | 37 ++++++++++++++++++++ port/cpl_aws.cpp | 76 ++++++++++++++++++++++------------------- 2 files changed, 78 insertions(+), 35 deletions(-) diff --git a/autotest/gcore/vsis3.py b/autotest/gcore/vsis3.py index 8cec895db836..d4a94b6cb027 100755 --- a/autotest/gcore/vsis3.py +++ b/autotest/gcore/vsis3.py @@ -988,6 +988,43 @@ def method_req_2(self, request): ############################################################################### +@gdaltest.enable_exceptions() +def test_vsis3_permanent_redirect_and_region_change(aws_test_config, webserver_port): + + handler = webserver.SequentialHandler() + handler.add( + "GET", + "/test_vsis3_permanent_redirect_and_region_change/?delimiter=%2F&list-type=2", + 301, + {"Content-type": "application/xml"}, + f""" + PermanentRedirectThe bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.localhost:{webserver_port}test_vsis3_permanent_redirect_and_region_change""", + ) + handler.add( + "GET", + "/test_vsis3_permanent_redirect_and_region_change/?delimiter=%2F&list-type=2", + 200, + {"Content-type": "application/xml"}, + """ + + + + test.bin + 1970-01-01T00:00:01.000Z + 123456 + + """, + ) + with webserver.install_http_handler(handler): + with gdal.VSIFile( + "/vsis3/test_vsis3_permanent_redirect_and_region_change/test.bin", "rb" + ): + pass + + +############################################################################### + + @pytest.mark.parametrize("with_error_message", [True, False]) def test_vsis3_request_timeout(aws_test_config, webserver_port, with_error_message): gdal.VSICurlClearCache() diff --git a/port/cpl_aws.cpp b/port/cpl_aws.cpp index ac4aabe69443..63017884f216 100644 --- a/port/cpl_aws.cpp +++ b/port/cpl_aws.cpp @@ -2809,48 +2809,54 @@ bool VSIS3HandleHelper::CanRestartOnError(const char *pszErrorMsg, } return false; } - if (!m_bUseVirtualHosting && - strncmp(pszEndpoint, m_osBucket.c_str(), m_osBucket.size()) == 0 && - pszEndpoint[m_osBucket.size()] == '.') + + /* If we have a body with + PermanentRedirectThe bucket you are + attempting to access must be addressed using the specified endpoint. + Please send all future requests to this + endpoint.bucketbucket.region.s3.amazonaws.com + and headers like + x-amz-bucket-region: eu-west-1 + then we must use s3-$(x-amz-bucket-region).amazon.com as endpoint. */ + const char *pszRegionPtr = + (pszHeaders != nullptr) + ? strstr(pszHeaders, "x-amz-bucket-region: ") + : nullptr; + if (pszRegionPtr != nullptr) { - /* If we have a body with - PermanentRedirectThe bucket you are - attempting to access must be addressed using the specified endpoint. - Please send all future requests to this - endpoint.bucket.with.dotbucket.with.dot.s3.amazonaws.com - and headers like - x-amz-bucket-region: eu-west-1 - and the bucket name has dot in it, - then we must use s3.$(x-amz-bucket-region).amazon.com as endpoint. - See #7154 */ - const char *pszRegionPtr = - (pszHeaders != nullptr) - ? strstr(pszHeaders, "x-amz-bucket-region: ") - : nullptr; - if (strchr(m_osBucket.c_str(), '.') != nullptr && - pszRegionPtr != nullptr) + std::string osRegion(pszRegionPtr + + strlen("x-amz-bucket-region: ")); + size_t nPos = osRegion.find('\r'); + if (nPos != std::string::npos) + osRegion.resize(nPos); + if (strncmp(pszEndpoint, m_osBucket.c_str(), m_osBucket.size()) == + 0 && + pszEndpoint[m_osBucket.size()] == '.') { - std::string osRegion(pszRegionPtr + - strlen("x-amz-bucket-region: ")); - size_t nPos = osRegion.find('\r'); - if (nPos != std::string::npos) - osRegion.resize(nPos); - SetEndpoint( - CPLSPrintf("s3.%s.amazonaws.com", osRegion.c_str())); - SetRegion(osRegion.c_str()); - CPLDebug(AWS_DEBUG_KEY, "Switching to endpoint %s", - m_osEndpoint.c_str()); - CPLDebug(AWS_DEBUG_KEY, "Switching to region %s", - m_osRegion.c_str()); - CPLDestroyXMLNode(psTree); - if (!bIsTemporaryRedirect) - VSIS3UpdateParams::UpdateMapFromHandle(this); - return true; + pszEndpoint += m_osBucket.size() + 1; } + SetEndpoint(pszEndpoint); + SetRegion(osRegion.c_str()); + CPLDebug(AWS_DEBUG_KEY, "Switching to endpoint %s", + m_osEndpoint.c_str()); + CPLDebug(AWS_DEBUG_KEY, "Switching to region %s", + m_osRegion.c_str()); + CPLDestroyXMLNode(psTree); + if (!bIsTemporaryRedirect) + VSIS3UpdateParams::UpdateMapFromHandle(this); + return true; + } + + if (!m_bUseVirtualHosting && + m_osBucket.find('.') == std::string::npos && + strncmp(pszEndpoint, m_osBucket.c_str(), m_osBucket.size()) == 0 && + pszEndpoint[m_osBucket.size()] == '.') + { m_bUseVirtualHosting = true; CPLDebug(AWS_DEBUG_KEY, "Switching to virtual hosting"); } + SetEndpoint(m_bUseVirtualHosting ? pszEndpoint + m_osBucket.size() + 1 : pszEndpoint); CPLDebug(AWS_DEBUG_KEY, "Switching to endpoint %s",