From c70f2ceaf3b914b96084c223321c864a8d326ce0 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 31 Jan 2024 17:07:40 +0100 Subject: [PATCH 01/90] feat(S3): initial commit for AWS S3 support --- CMakeLists.txt | 8 ++ src/eckit/CMakeLists.txt | 18 +++- src/eckit/io/s3/S3Exception.cc | 51 +++++++++++ src/eckit/io/s3/S3Exception.h | 39 +++++++++ src/eckit/io/s3/S3Handle.cc | 155 +++++++++++++++++++++++++++++++++ src/eckit/io/s3/S3Handle.h | 79 +++++++++++++++++ src/eckit/io/s3/S3Macros.h | 47 ++++++++++ src/eckit/io/s3/S3Name.cc | 45 ++++++++++ src/eckit/io/s3/S3Name.h | 55 ++++++++++++ src/eckit/io/s3/S3Object.cc | 77 ++++++++++++++++ src/eckit/io/s3/S3Object.h | 98 +++++++++++++++++++++ tests/io/CMakeLists.txt | 6 ++ tests/io/test_s3handle.cc | 51 +++++++++++ 13 files changed, 728 insertions(+), 1 deletion(-) create mode 100644 src/eckit/io/s3/S3Exception.cc create mode 100644 src/eckit/io/s3/S3Exception.h create mode 100644 src/eckit/io/s3/S3Handle.cc create mode 100644 src/eckit/io/s3/S3Handle.h create mode 100644 src/eckit/io/s3/S3Macros.h create mode 100644 src/eckit/io/s3/S3Name.cc create mode 100644 src/eckit/io/s3/S3Name.h create mode 100644 src/eckit/io/s3/S3Object.cc create mode 100644 src/eckit/io/s3/S3Object.h create mode 100644 tests/io/test_s3handle.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index f375e67f2..a4014c9c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,6 +103,14 @@ ecbuild_add_option( FEATURE RADOS DESCRIPTION "Ceph/Rados storage support" REQUIRED_PACKAGES Rados ) +### S3 Support + +ecbuild_add_option( FEATURE AWS_S3 + DEFAULT OFF + ADVANCED + REQUIRED_PACKAGES "AWSSDK REQUIRED COMPONENTS s3" + DESCRIPTION "Enables AWS S3 support" ) + ### Armadillo ecbuild_add_option( FEATURE ARMADILLO diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 2ed224638..2f47530f4 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -286,6 +286,20 @@ io/rados/RadosAttributes.cc ) endif() +if(HAVE_AWS_S3) +list( APPEND eckit_io_srcs +io/s3/S3Exception.cc +io/s3/S3Exception.h +io/s3/S3Handle.cc +io/s3/S3Handle.h +io/s3/S3Name.cc +io/s3/S3Name.h +io/s3/S3Object.cc +io/s3/S3Object.h +io/s3/S3Macros.h +) +endif(HAVE_AWS_S3) + list( APPEND eckit_filesystem_srcs filesystem/BasePathName.cc filesystem/BasePathName.h @@ -934,12 +948,14 @@ ecbuild_add_library( $ PRIVATE_INCLUDES + $ "${CURL_INCLUDE_DIRS}" "${SNAPPY_INCLUDE_DIRS}" "${LZ4_INCLUDE_DIRS}" "${BZIP2_INCLUDE_DIRS}" "${AEC_INCLUDE_DIRS}" "${RADOS_INCLUDE_DIRS}" + "${AWSSDK_INCLUDE_DIRS}" "${OPENSSL_INCLUDE_DIR}" "${AIO_INCLUDE_DIRS}" @@ -953,6 +969,7 @@ ecbuild_add_library( "${CURL_LIBRARIES}" "${AIO_LIBRARIES}" "${RADOS_LIBRARIES}" + "${AWSSDK_LINK_LIBRARIES}" PUBLIC_LIBS ${CMATH_LIBRARIES} @@ -984,4 +1001,3 @@ add_subdirectory( web ) if( HAVE_ECKIT_CODEC ) add_subdirectory( codec ) endif() - diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc new file mode 100644 index 000000000..9c196c514 --- /dev/null +++ b/src/eckit/io/s3/S3Exception.cc @@ -0,0 +1,51 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Exception.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/s3/S3Exception.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +namespace helper { + +inline auto addCode(const std::string& msg, int code) -> std::string { + std::ostringstream oss; + oss << "Code(" << strerror(-code) << ") " << msg; + return oss.str(); +}; + +} // namespace helper + +//---------------------------------------------------------------------------------------------------------------------- + +S3SeriousBug::S3SeriousBug(const std::string& msg, const CodeLocation& loc): + SeriousBug(std::string("[S3 Error] ") + msg, loc) { } + +S3SeriousBug::S3SeriousBug(const std::string& msg, const int code, const CodeLocation& loc): + S3SeriousBug(helper::addCode(msg, code), loc) { } + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Exception.h b/src/eckit/io/s3/S3Exception.h new file mode 100644 index 000000000..644976ffc --- /dev/null +++ b/src/eckit/io/s3/S3Exception.h @@ -0,0 +1,39 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Exception.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/exception/Exceptions.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3SeriousBug: public SeriousBug { +public: + S3SeriousBug(const std::string& msg, const CodeLocation& loc); + S3SeriousBug(const std::string& msg, int code, const CodeLocation& loc); +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc new file mode 100644 index 000000000..3fe5427c7 --- /dev/null +++ b/src/eckit/io/s3/S3Handle.cc @@ -0,0 +1,155 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eckit/io/s3/S3Handle.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Name.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Handle::S3HandleInternal { +public: + S3HandleInternal(const PathName& name, const eckit::Offset& offset): name_(name), pos_(offset) { } + + friend std::ostream& operator<<(std::ostream& os, const S3HandleInternal& handle) { + os << "name: " << handle.name_ << ", pos: " << handle.pos_ << " can write: " << handle.canWrite_ << std::endl; + return os; + } + +private: + // const PathName name_; + S3Name name_; + + // S3Object object_; + + Offset pos_ {0}; + + bool canWrite_ {false}; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +S3Handle::S3Handle(const PathName& name): S3Handle(name, 0) { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; +} + +S3Handle::S3Handle(const PathName& name, const eckit::Offset& offset): + impl_(std::make_unique(name, offset)) { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; +} + +S3Handle::~S3Handle() { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; +} + +void S3Handle::print(std::ostream& out) const { + out << "S3Handle[" << impl_ << "]"; +} + +//---------------------------------------------------------------------------------------------------------------------- + +eckit::Length S3Handle::openForRead() { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; + + // canWrite_ = false; + // object_ = name_.object(); + // if (object_->open()) { return object_->size(); } + + return {}; +} + +void S3Handle::openForWrite(const eckit::Length& length) { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; + // canWrite_ = true; + // object_ = name_.object(); + // if (!object_->create(name_.pool())) { throw NotFound("Cannot create object!", -1, Here()); } +} + +void S3Handle::openForAppend(const eckit::Length& length) { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; + // NOTIMP; +} + +//---------------------------------------------------------------------------------------------------------------------- + +long S3Handle::read(void* buffer, const long length) { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; + return 0; + // return object_ ? object_->read(buffer, length, pos_) : 0; +} + +long S3Handle::write(const void* buffer, const long length) { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; + // if (object_ && canWrite_) { return object_->write(buffer, length, pos_); } + return 0; +} + +void S3Handle::flush() { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; +} + +void S3Handle::close() { + // canWrite_ = false; + // object_.reset(); + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; +} + +eckit::Length S3Handle::size() { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; + return {}; +} + +eckit::Offset S3Handle::position() { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; + // return pos_; + return {}; +} + +eckit::Offset S3Handle::seek(const eckit::Offset& offset) { + // pos_ = pos_ + offset; + // return pos_; + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; + return {}; +} + +std::string S3Handle::title() const { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; + // NOTIMP; + return {}; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h new file mode 100644 index 000000000..6a8cdbcaa --- /dev/null +++ b/src/eckit/io/s3/S3Handle.h @@ -0,0 +1,79 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Handle.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/DataHandle.h" +#include "eckit/io/s3/S3Macros.h" + +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Handle: public DataHandle { +public: + NO_COPY_NO_MOVE(S3Handle) + + // NOTE: @Nicolau we will try S3 stream later which is not here yet + + explicit S3Handle(const PathName& name); + + explicit S3Handle(const PathName& name, const eckit::Offset& offset); + + ~S3Handle() override; + + void print(std::ostream& out) const override; + +public: + eckit::Length openForRead() override; + + void openForWrite(const eckit::Length& length) override; + + void openForAppend(const eckit::Length& length) override; + + long read(void* buffer, long length) override; + + long write(const void* buffer, long length) override; + + void flush() override; + + void close() override; + + eckit::Length size() override; + + eckit::Offset position() override; + + eckit::Offset seek(const eckit::Offset& offset) override; + + std::string title() const override; + + bool canSeek() const override { return true; } + +private: // members + class S3HandleInternal; + UPtr impl_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Macros.h b/src/eckit/io/s3/S3Macros.h new file mode 100644 index 000000000..341d921c8 --- /dev/null +++ b/src/eckit/io/s3/S3Macros.h @@ -0,0 +1,47 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Macros.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#define NODISCARD [[nodiscard]] + +#define NO_COPY(TypeName) \ + TypeName(const TypeName&) = delete; \ + TypeName& operator=(const TypeName&) = delete; + +#define NO_MOVE(TypeName) \ + TypeName(TypeName&&) = delete; \ + TypeName& operator=(TypeName&&) = delete; + +#define NO_COPY_NO_MOVE(TypeName) \ + NO_COPY(TypeName) \ + NO_MOVE(TypeName) + +//---------------------------------------------------------------------------------------------------------------------- + +#include + +template +using UPtr = std::unique_ptr; + +template +using SPtr = std::shared_ptr; + +//---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc new file mode 100644 index 000000000..bd584e457 --- /dev/null +++ b/src/eckit/io/s3/S3Name.cc @@ -0,0 +1,45 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eckit/io/s3/S3Name.h" + +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Object.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3Name::S3Name(const std::string& name): uri_("S3", name, 9000) { } + +S3Name::~S3Name() = default; + +void S3Name::print(std::ostream& out) const { + out << "S3Name[uri=" << uri_ << "]"; +} + +auto S3Name::uri() const -> URI { + return uri_; +} + +auto S3Name::createObject() -> UPtr { + NOTIMP; + return UPtr(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h new file mode 100644 index 000000000..d58d3fa4a --- /dev/null +++ b/src/eckit/io/s3/S3Name.h @@ -0,0 +1,55 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Name.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3Macros.h" + +namespace eckit { + +class S3Object; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Name { +public: // methods + S3Name(const std::string& name); + + ~S3Name(); + + auto uri() const -> URI; + + NODISCARD + auto createObject() -> UPtr; + + friend std::ostream& operator<<(std::ostream& out, const S3Name& name) { + name.print(out); + return out; + } + +private: // methods + void print(std::ostream& out) const; + +private: // members + URI uri_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Object.cc b/src/eckit/io/s3/S3Object.cc new file mode 100644 index 000000000..43743cfbc --- /dev/null +++ b/src/eckit/io/s3/S3Object.cc @@ -0,0 +1,77 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/io/s3/S3Object.h" + +#include "eckit/io/s3/S3Exception.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Object::S3ObjectInternal { +public: + // S3ObjectInternal(const std::string& name): name_(name) { } + + S3ObjectInternal() = default; + ~S3ObjectInternal() = default; + + // private: + // const std::string& name_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +S3Object::S3Object(const std::string& name): S3ObjectBase(name), impl_(std::make_unique()) { } + +S3Object::~S3Object() { } + +//---------------------------------------------------------------------------------------------------------------------- + +bool S3Object::create(const std::string& name) { + NOTIMP; + return false; +} + +bool S3Object::remove() { + NOTIMP; + return false; +} + +bool S3Object::open() { + NOTIMP; + return false; +} + +long S3Object::read(void* buffer, long length, off_t offset) const { + NOTIMP; + return 0; +} + +long S3Object::write(const void* buffer, long length, off_t offset) const { + NOTIMP; + return 0; +} + +bool S3Object::close() { + NOTIMP; + return false; +} + +long S3Object::size() { + NOTIMP; + return 0; +} + +void S3Object::print(std::ostream& out) const { } + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Object.h b/src/eckit/io/s3/S3Object.h new file mode 100644 index 000000000..d87aa0f1b --- /dev/null +++ b/src/eckit/io/s3/S3Object.h @@ -0,0 +1,98 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Object.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#include "eckit/io/s3/S3Macros.h" + +// #include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3ObjectBase { +public: // methods + NO_COPY_NO_MOVE(S3ObjectBase) + + explicit S3ObjectBase(const std::string& name): name_(name) { } + + virtual ~S3ObjectBase() = default; + + friend std::ostream& operator<<(std::ostream& out, const S3ObjectBase& obj) { + obj.print(out); + return out; + } + + virtual bool create(const std::string& name) = 0; + + virtual bool remove() = 0; + + virtual bool open() = 0; + + virtual long read(void* buffer, long length, off_t offset) const = 0; + + virtual long write(const void* buffer, long length, off_t offset) const = 0; + + virtual bool close() = 0; + +private: // methods + virtual void print(std::ostream& out) const = 0; + +private: // members + std::string name_; + + bool isOpen_ {false}; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Object: public S3ObjectBase { +public: // methods + NO_COPY_NO_MOVE(S3Object) + + S3Object(const std::string& name); + + ~S3Object() override; + + bool create(const std::string& name) override; + + bool remove() override; + + bool open() override; + + long read(void* buffer, long length, off_t offset) const override; + + long write(const void* buffer, long length, off_t offset) const override; + + bool close() override; + + long size(); + +private: // methods + void print(std::ostream& out) const override; + +private: // members + class S3ObjectInternal; + std::unique_ptr impl_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/tests/io/CMakeLists.txt b/tests/io/CMakeLists.txt index b2a35896e..a4e936fcb 100644 --- a/tests/io/CMakeLists.txt +++ b/tests/io/CMakeLists.txt @@ -74,6 +74,12 @@ ecbuild_add_test( TARGET eckit_test_radoshandle INCLUDES ${RADOS_INCLUDE_DIRS} LIBS eckit ) +ecbuild_add_test( TARGET eckit_test_s3handle + SOURCES test_s3handle.cc + CONDITION HAVE_AWS_S3 + INCLUDES ${AWSSDK_INCLUDE_DIRS} + LIBS eckit ) + ecbuild_add_test( TARGET eckit_rados-performance SOURCES rados-performance.cc CONDITION HAVE_EXTRA_TESTS AND HAVE_RADOS diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc new file mode 100644 index 000000000..d3f1b9b52 --- /dev/null +++ b/tests/io/test_s3handle.cc @@ -0,0 +1,51 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file test_s3handle.cc +/// @author Metin Cakircali +/// @date Jan 2024 + +#include "eckit/io/Buffer.h" +#include "eckit/io/s3/S3Handle.h" +#include "eckit/testing/Test.h" + +#include + +using namespace std; +using namespace eckit; +using namespace eckit::testing; + +namespace eckit::test { + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3Handle") { + const char buf[] = "abcdefghijklmnopqrstuvwxyz"; + + S3Handle h("blablaregion/bucketname/objectname"); + + h.openForWrite(sizeof(buf)); + h.write(buf, sizeof(buf)); + h.close(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test + +int main(int argc, char** argv) { + return run_tests(argc, argv); +} From c1825746992a3c9c8c7b4fc9af4a6c306319f722 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 1 Feb 2024 10:54:13 +0100 Subject: [PATCH 02/90] refactor(S3): removed UPtr and SPtr --- src/eckit/io/s3/S3Handle.h | 2 +- src/eckit/io/s3/S3Macros.h | 12 ------------ src/eckit/io/s3/S3Name.cc | 4 ++-- src/eckit/io/s3/S3Name.h | 2 +- src/eckit/io/s3/S3Object.h | 4 +++- 5 files changed, 7 insertions(+), 17 deletions(-) diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index 6a8cdbcaa..4ecd8773c 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -71,7 +71,7 @@ class S3Handle: public DataHandle { private: // members class S3HandleInternal; - UPtr impl_; + std::unique_ptr impl_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Macros.h b/src/eckit/io/s3/S3Macros.h index 341d921c8..053fe1ed2 100644 --- a/src/eckit/io/s3/S3Macros.h +++ b/src/eckit/io/s3/S3Macros.h @@ -33,15 +33,3 @@ #define NO_COPY_NO_MOVE(TypeName) \ NO_COPY(TypeName) \ NO_MOVE(TypeName) - -//---------------------------------------------------------------------------------------------------------------------- - -#include - -template -using UPtr = std::unique_ptr; - -template -using SPtr = std::shared_ptr; - -//---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index bd584e457..1964360d7 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -35,9 +35,9 @@ auto S3Name::uri() const -> URI { return uri_; } -auto S3Name::createObject() -> UPtr { +auto S3Name::createObject() -> std::unique_ptr { NOTIMP; - return UPtr(); + return std::unique_ptr(); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index d58d3fa4a..7d214a6a8 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -36,7 +36,7 @@ class S3Name { auto uri() const -> URI; NODISCARD - auto createObject() -> UPtr; + auto createObject() -> std::unique_ptr; friend std::ostream& operator<<(std::ostream& out, const S3Name& name) { name.print(out); diff --git a/src/eckit/io/s3/S3Object.h b/src/eckit/io/s3/S3Object.h index d87aa0f1b..ce20ad639 100644 --- a/src/eckit/io/s3/S3Object.h +++ b/src/eckit/io/s3/S3Object.h @@ -18,9 +18,11 @@ /// @author Metin Cakircali /// @date Jan 2024 +#pragma once + #include "eckit/io/s3/S3Macros.h" -// #include +#include #include namespace eckit { From e92e14e6e02ad04d3e2a1590c5e0a58f9d159e65 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 6 Feb 2024 12:53:33 +0100 Subject: [PATCH 03/90] feat(S3)!: adds Config, Session, Client, Context --- src/eckit/CMakeLists.txt | 11 ++++ src/eckit/io/s3/S3Client.cc | 52 ++++++++++++++++ src/eckit/io/s3/S3Client.h | 53 ++++++++++++++++ src/eckit/io/s3/S3Config.h | 39 ++++++++++++ src/eckit/io/s3/S3Context.cc | 42 +++++++++++++ src/eckit/io/s3/S3Context.h | 54 ++++++++++++++++ src/eckit/io/s3/S3Session.cc | 93 ++++++++++++++++++++++++++++ src/eckit/io/s3/S3Session.h | 61 ++++++++++++++++++ src/eckit/io/s3/aws/S3ClientAWS.cc | 95 +++++++++++++++++++++++++++++ src/eckit/io/s3/aws/S3ClientAWS.h | 49 +++++++++++++++ src/eckit/io/s3/aws/S3ContextAWS.cc | 65 ++++++++++++++++++++ src/eckit/io/s3/aws/S3ContextAWS.h | 54 ++++++++++++++++ 12 files changed, 668 insertions(+) create mode 100644 src/eckit/io/s3/S3Client.cc create mode 100644 src/eckit/io/s3/S3Client.h create mode 100644 src/eckit/io/s3/S3Config.h create mode 100644 src/eckit/io/s3/S3Context.cc create mode 100644 src/eckit/io/s3/S3Context.h create mode 100644 src/eckit/io/s3/S3Session.cc create mode 100644 src/eckit/io/s3/S3Session.h create mode 100644 src/eckit/io/s3/aws/S3ClientAWS.cc create mode 100644 src/eckit/io/s3/aws/S3ClientAWS.h create mode 100644 src/eckit/io/s3/aws/S3ContextAWS.cc create mode 100644 src/eckit/io/s3/aws/S3ContextAWS.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 2f47530f4..056a351d5 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -288,6 +288,15 @@ endif() if(HAVE_AWS_S3) list( APPEND eckit_io_srcs +io/s3/aws/S3ClientAWS.cc +io/s3/aws/S3ClientAWS.h +io/s3/aws/S3ContextAWS.cc +io/s3/aws/S3ContextAWS.h +io/s3/S3Client.cc +io/s3/S3Client.h +io/s3/S3Config.h +io/s3/S3Context.cc +io/s3/S3Context.h io/s3/S3Exception.cc io/s3/S3Exception.h io/s3/S3Handle.cc @@ -296,6 +305,8 @@ io/s3/S3Name.cc io/s3/S3Name.h io/s3/S3Object.cc io/s3/S3Object.h +io/s3/S3Session.cc +io/s3/S3Session.h io/s3/S3Macros.h ) endif(HAVE_AWS_S3) diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc new file mode 100644 index 000000000..fdc6de920 --- /dev/null +++ b/src/eckit/io/s3/S3Client.cc @@ -0,0 +1,52 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eckit/io/s3/S3Client.h" + +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/io/s3/aws/S3ClientAWS.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3Client::S3Client(S3ContextSPtr context): context_(context) { } + +S3Client::~S3Client() = default; + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { + if (config.type == S3Types::AWS) { return std::make_unique(config); } + Log::error() << "Empty client!" << std::endl; + return {}; +} + +//---------------------------------------------------------------------------------------------------------------------- + +// auto S3Client::createBucket(const std::string& name) const -> bool { +// NOTIMP; +// return false; +// } + +void S3Client::listBuckets() const { + NOTIMP; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h new file mode 100644 index 000000000..f8212827d --- /dev/null +++ b/src/eckit/io/s3/S3Client.h @@ -0,0 +1,53 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Client.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/s3/S3Context.h" + +#include + +namespace eckit { + +using S3ContextSPtr = std::shared_ptr; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Client { +public: // methods + NO_COPY_NO_MOVE(S3Client) + + explicit S3Client(S3ContextSPtr context); + + virtual ~S3Client(); + + static auto makeUnique(const S3Config& config) -> std::unique_ptr; + + virtual auto createBucket(const std::string& name) const -> bool = 0; + + virtual void listBuckets() const; + +private: // members + S3ContextSPtr context_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h new file mode 100644 index 000000000..5c8e921a5 --- /dev/null +++ b/src/eckit/io/s3/S3Config.h @@ -0,0 +1,39 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Config.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include + +namespace eckit { + +enum class S3Types { NONE, AWS, REST }; + +//---------------------------------------------------------------------------------------------------------------------- + +struct S3Config { + S3Types type; + std::string name; + std::string region; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Context.cc b/src/eckit/io/s3/S3Context.cc new file mode 100644 index 000000000..0d0c025ab --- /dev/null +++ b/src/eckit/io/s3/S3Context.cc @@ -0,0 +1,42 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eckit/io/s3/S3Context.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/io/s3/aws/S3ContextAWS.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3Context::S3Context(const S3Types type): type_(type) { } + +S3Context::~S3Context() = default; + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Context::makeShared(S3Types type) -> std::shared_ptr { + // AWS SDK API + if (type == S3Types::AWS) { return S3ContextAWS::makeShared(); } + // REST API + if (type == S3Types::REST) { NOTIMP; } + return {}; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Context.h b/src/eckit/io/s3/S3Context.h new file mode 100644 index 000000000..916fc5016 --- /dev/null +++ b/src/eckit/io/s3/S3Context.h @@ -0,0 +1,54 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Context.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3Macros.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Context { +public: // methods + NO_COPY_NO_MOVE(S3Context) + + explicit S3Context(S3Types type); + + virtual ~S3Context(); + + NODISCARD + auto getType() const -> S3Types { return type_; } + +private: // methods + friend class S3Session; + + static auto makeShared(S3Types type) -> std::shared_ptr; + +private: // members + const S3Types type_ {S3Types::NONE}; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc new file mode 100644 index 000000000..53f734c5a --- /dev/null +++ b/src/eckit/io/s3/S3Session.cc @@ -0,0 +1,93 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eckit/io/s3/S3Session.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/aws/S3ClientAWS.h" + +namespace eckit { + +namespace utils { + +/// @brief Functor for S3Context type +struct IsContextType { + S3Types type_; + + bool operator()(const std::shared_ptr& ctx) const { return ctx->getType() == type_; } +}; + +} // namespace utils + +//---------------------------------------------------------------------------------------------------------------------- + +S3Session& S3Session::instance() { + thread_local S3Session session; // TODO: thread local vs static + return session; +} + +// auto S3Session::createClient(const S3Config& config) -> std::unique_ptr { +// LOG_DEBUG_LIB(LibEcKit) << "Creating S3Client..." << std::endl; +// auto ctx = getContext(config.type); +// if (config.type == S3Types::AWS) { +// auto client = std::make_unique(ctx); +// client->configure(config); +// return client; +// } +// return {}; +// } + +//---------------------------------------------------------------------------------------------------------------------- + +S3Session::S3Session() = default; + +S3Session::~S3Session() = default; + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Session::getContext(const S3Types type) -> std::shared_ptr { + // return if found + if (auto context = findContext(type)) { return context; } + + // not found + auto context = S3Context::makeShared(type); + registry_.push_back(context); + + return context; +} + +auto S3Session::findContext(const S3Types type) -> std::shared_ptr { + LOG_DEBUG_LIB(LibEcKit) << "Find context!" << std::endl; + // search by type + auto context = std::find_if(registry_.begin(), registry_.end(), utils::IsContextType({type})); + // found + if (context != registry_.end()) { return *context; } + // not found + return {}; +} + +void S3Session::removeContext(const S3Types type) { + registry_.remove_if(utils::IsContextType({type})); +} + +void S3Session::clear() { + registry_.clear(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h new file mode 100644 index 000000000..7ae5355a6 --- /dev/null +++ b/src/eckit/io/s3/S3Session.h @@ -0,0 +1,61 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Session.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/s3/S3Client.h" + +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Session { +public: // methods + NO_COPY_NO_MOVE(S3Session) + + static S3Session& instance(); + + // auto createClient(const S3Config& config) -> std::unique_ptr; + + NODISCARD + auto getContext(S3Types type) -> std::shared_ptr; + + NODISCARD + auto findContext(S3Types type) -> std::shared_ptr; + + void removeContext(S3Types type); + + void clear(); + +private: // methods + S3Session(); + + ~S3Session(); + +private: // members + std::list> registry_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc new file mode 100644 index 000000000..e66e03d94 --- /dev/null +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -0,0 +1,95 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eckit/io/s3/aws/S3ClientAWS.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/io/s3/aws/S3ContextAWS.h" + +#include + +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3ClientAWS::S3ClientAWS(const S3Config& config): S3Client(S3Session::instance().getContext(config.type)) { + LOG_DEBUG_LIB(LibEcKit) << "S3ClientAWS::setConfiguration" << std::endl; + + config_.region = config.region; + + // S3Config holds handles credentials + + // config_.proxyHost = "127.0.0.1"; + // config_.proxyPort = 9000; + // config_.proxyScheme = Aws::Http::Scheme::HTTPS; + // config_.verifySSL = false; + config_.endpointOverride = "http://127.0.0.1:9000"; +} + +S3ClientAWS::~S3ClientAWS() = default; + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ClientAWS::createBucket(const std::string& name) const -> bool { + Aws::S3::S3Client client(config_); + + Aws::S3::Model::CreateBucketRequest request; + request.SetBucket(name); + + // Change the bucket location constraint enum to target Region. + if (config_.region != "eu-central-1") { + Aws::S3::Model::CreateBucketConfiguration bucketConfig; + bucketConfig.SetLocationConstraint( + Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(config_.region)); + request.SetCreateBucketConfiguration(bucketConfig); + } + + Aws::S3::Model::CreateBucketOutcome outcome = client.CreateBucket(request); + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + Log::error() << "Failed to create bucket [" << name << "]! " << err.GetExceptionName() << ": " + << err.GetMessage() << std::endl; + } else { + LOG_DEBUG_LIB(LibEcKit) << "Created bucket " << name << std::endl; + } + + return outcome.IsSuccess(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +void S3ClientAWS::listBuckets() const { + Aws::S3::S3Client client(config_); + + auto outcome = client.ListBuckets(); + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Found " << outcome.GetResult().GetBuckets().size() << " buckets:\n"; + for (auto& bucket : outcome.GetResult().GetBuckets()) { + LOG_DEBUG_LIB(LibEcKit) << bucket.GetName() << std::endl; + } + } else { + Log::error() << "List failed! Error: " << outcome.GetError(); + } +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h new file mode 100644 index 000000000..58ea7ae5f --- /dev/null +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -0,0 +1,49 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3ClientAWS.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/s3/S3Client.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3ClientAWS: public S3Client { +public: // methods + NO_COPY_NO_MOVE(S3ClientAWS) + + explicit S3ClientAWS(const S3Config& config); + + ~S3ClientAWS(); + + auto createBucket(const std::string& name) const -> bool override; + + void listBuckets() const override; + +private: // members + Aws::Client::ClientConfiguration config_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ContextAWS.cc b/src/eckit/io/s3/aws/S3ContextAWS.cc new file mode 100644 index 000000000..bc64e4b90 --- /dev/null +++ b/src/eckit/io/s3/aws/S3ContextAWS.cc @@ -0,0 +1,65 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eckit/io/s3/aws/S3ContextAWS.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/aws/S3ClientAWS.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3ContextAWS::S3ContextAWS(const Aws::SDKOptions& options): S3Context(S3Types::AWS), options_(options) { + LOG_DEBUG_LIB(LibEcKit) << "Initialize AWS API!" << std::endl; + Aws::InitAPI(options_); +} + +S3ContextAWS::~S3ContextAWS() { + LOG_DEBUG_LIB(LibEcKit) << "Shutdown AWS API!" << std::endl; + Aws::ShutdownAPI(options_); +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ContextAWS::makeShared() -> std::shared_ptr { + Aws::SDKOptions options; + // TODO: remove debugging logs + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + return std::make_shared(options); +} + +auto S3ContextAWS::checkCredentials(const char* chain) -> bool { + // AWS S3 permits unauthenticated requests; client returns "success" but 0 buckets + auto provider = Aws::MakeShared(chain); + + auto creds = provider->GetAWSCredentials(); + if (creds.IsExpiredOrEmpty()) { + Log::warning() << "Authentication failed!" << std::endl; + return false; + } + + Log::info() << "AWS authentication was successful!" << std::endl; + + return true; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ContextAWS.h b/src/eckit/io/s3/aws/S3ContextAWS.h new file mode 100644 index 000000000..9dc4ffbc4 --- /dev/null +++ b/src/eckit/io/s3/aws/S3ContextAWS.h @@ -0,0 +1,54 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3ContextAWS.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/s3/S3Context.h" + +#include + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3ContextAWS: public S3Context { +public: // methods + NO_MOVE(S3ContextAWS) + + explicit S3ContextAWS(const Aws::SDKOptions& options); + + ~S3ContextAWS(); + + static auto checkCredentials(const char* chain) -> bool; + +private: // methods + friend S3Context; + + static auto makeShared() -> std::shared_ptr; + +private: // members + Aws::SDKOptions options_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit From 5fbf9320227bb0354bdf801a75b11f7c0337d478 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 7 Feb 2024 15:01:44 +0100 Subject: [PATCH 04/90] feat(S3): add create,delete,list for bucket and put,list object --- src/eckit/io/s3/S3Client.cc | 5 -- src/eckit/io/s3/S3Client.h | 10 ++- src/eckit/io/s3/S3Config.h | 4 +- src/eckit/io/s3/aws/S3ClientAWS.cc | 115 ++++++++++++++++++++++++----- src/eckit/io/s3/aws/S3ClientAWS.h | 8 +- tests/io/CMakeLists.txt | 6 ++ tests/io/test_s3client.cc | 61 +++++++++++++++ 7 files changed, 182 insertions(+), 27 deletions(-) create mode 100644 tests/io/test_s3client.cc diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index fdc6de920..a6f248490 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -38,11 +38,6 @@ auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { //---------------------------------------------------------------------------------------------------------------------- -// auto S3Client::createBucket(const std::string& name) const -> bool { -// NOTIMP; -// return false; -// } - void S3Client::listBuckets() const { NOTIMP; } diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index f8212827d..d14b76307 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -40,9 +40,15 @@ class S3Client { static auto makeUnique(const S3Config& config) -> std::unique_ptr; - virtual auto createBucket(const std::string& name) const -> bool = 0; + virtual auto createBucket(const std::string& bucketName) const -> bool = 0; - virtual void listBuckets() const; + virtual auto deleteBucket(const std::string& bucketName) const -> bool = 0; + + virtual auto putObject(const std::string& bucketName, const std::string& objectName) const -> bool = 0; + + virtual auto listObjects(const std::string& bucketName) const -> bool = 0; + + virtual void listBuckets() const = 0; private: // members S3ContextSPtr context_; diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 5c8e921a5..e1efd3aa7 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -29,9 +29,11 @@ enum class S3Types { NONE, AWS, REST }; //---------------------------------------------------------------------------------------------------------------------- struct S3Config { - S3Types type; + S3Types type {S3Types::AWS}; std::string name; std::string region; + std::string endpoint {"127.0.0.1"}; + int port {0}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index e66e03d94..e55c9da7c 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -22,6 +22,9 @@ #include "eckit/io/s3/aws/S3ContextAWS.h" #include +#include +#include +#include #include #include @@ -33,26 +36,31 @@ namespace eckit { S3ClientAWS::S3ClientAWS(const S3Config& config): S3Client(S3Session::instance().getContext(config.type)) { LOG_DEBUG_LIB(LibEcKit) << "S3ClientAWS::setConfiguration" << std::endl; - config_.region = config.region; + // we are not an ec2 instance + config_.disableIMDS = true; - // S3Config holds handles credentials + // setup region + config_.region = config.region; + if (!config.region.empty()) { config_.region = config.region; } - // config_.proxyHost = "127.0.0.1"; - // config_.proxyPort = 9000; // config_.proxyScheme = Aws::Http::Scheme::HTTPS; // config_.verifySSL = false; - config_.endpointOverride = "http://127.0.0.1:9000"; + + // setup endpoint + if (!config.endpoint.empty()) { config_.endpointOverride = "http://" + config.endpoint; } + if (config.port > 0) { config_.endpointOverride += ":" + std::to_string(config.port); } } S3ClientAWS::~S3ClientAWS() = default; //---------------------------------------------------------------------------------------------------------------------- +// BUCKET -auto S3ClientAWS::createBucket(const std::string& name) const -> bool { +auto S3ClientAWS::createBucket(const std::string& bucketName) const -> bool { Aws::S3::S3Client client(config_); Aws::S3::Model::CreateBucketRequest request; - request.SetBucket(name); + request.SetBucket(bucketName); // Change the bucket location constraint enum to target Region. if (config_.region != "eu-central-1") { @@ -63,33 +71,104 @@ auto S3ClientAWS::createBucket(const std::string& name) const -> bool { } Aws::S3::Model::CreateBucketOutcome outcome = client.CreateBucket(request); - if (!outcome.IsSuccess()) { - auto err = outcome.GetError(); - Log::error() << "Failed to create bucket [" << name << "]! " << err.GetExceptionName() << ": " - << err.GetMessage() << std::endl; + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Created bucket: " << bucketName << std::endl; } else { - LOG_DEBUG_LIB(LibEcKit) << "Created bucket " << name << std::endl; + const auto err = outcome.GetError(); + Log::error() << "Failed to create bucket: " << bucketName << " Error: " << err.GetExceptionName() << ": " + << err.GetMessage() << std::endl; } return outcome.IsSuccess(); } -//---------------------------------------------------------------------------------------------------------------------- +auto S3ClientAWS::deleteBucket(const std::string& bucketName) const -> bool { + Aws::S3::S3Client client(config_); + + Aws::S3::Model::DeleteBucketRequest request; + request.SetBucket(bucketName); + + Aws::S3::Model::DeleteBucketOutcome outcome = client.DeleteBucket(request); + + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket: " << bucketName << std::endl; + } else { + const Aws::S3::S3Error& err = outcome.GetError(); + Log::error() << "Failed to delete bucket: " << bucketName << " Error: " << err.GetExceptionName() << ": " + << err.GetMessage() << std::endl; + } + + return outcome.IsSuccess(); +} void S3ClientAWS::listBuckets() const { Aws::S3::S3Client client(config_); auto outcome = client.ListBuckets(); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Found " << outcome.GetResult().GetBuckets().size() << " buckets:\n"; - for (auto& bucket : outcome.GetResult().GetBuckets()) { - LOG_DEBUG_LIB(LibEcKit) << bucket.GetName() << std::endl; - } + Log::info() << "Found " << outcome.GetResult().GetBuckets().size() << " buckets:\n"; + for (auto& bucket : outcome.GetResult().GetBuckets()) { Log::info() << bucket.GetName() << std::endl; } } else { - Log::error() << "List failed! Error: " << outcome.GetError(); + Log::error() << "List failed! " << outcome.GetError().GetMessage() << std::endl; } } +//---------------------------------------------------------------------------------------------------------------------- +// OBJECT + +auto S3ClientAWS::putObject(const std::string& bucketName, const std::string& objectName) const -> bool { + Aws::S3::S3Client s3_client(config_); + + Aws::S3::Model::PutObjectRequest request; + request.SetBucket(bucketName); + // We are using the name of the file as the key for the object in the bucket. + // However, this is just a string and can be set according to your retrieval needs. + request.SetKey(objectName); + + // std::shared_ptr inputData = Aws::MakeShared("SampleAllocationTag", + // objectName.c_str(), + // std::ios_base::in | + // std::ios_base::binary); + // if (!*inputData) { + // std::cerr << "Error unable to read file " << objectName << std::endl; + // return false; + // } + + const std::shared_ptr inputData = Aws::MakeShared(""); + + // *inputData << objectContent.c_str(); + + request.SetBody(inputData); + + Aws::S3::Model::PutObjectOutcome outcome = s3_client.PutObject(request); + + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Added object '" << objectName << "' to bucket '" << bucketName << "'."; + } else { + Log::error() << "Error: putObject: " << outcome.GetError().GetMessage() << std::endl; + } + + return outcome.IsSuccess(); +} + +auto S3ClientAWS::listObjects(const std::string& bucketName) const -> bool { + Aws::S3::S3Client s3_client(config_); + + Aws::S3::Model::ListObjectsRequest request; + request.WithBucket(bucketName); + + auto outcome = s3_client.ListObjects(request); + + if (!outcome.IsSuccess()) { + Log::error() << "Error: ListObjects: " << outcome.GetError().GetMessage() << std::endl; + } else { + Aws::Vector objects = outcome.GetResult().GetContents(); + for (Aws::S3::Model::Object& object : objects) { Log::info() << object.GetKey() << std::endl; } + } + + return outcome.IsSuccess(); +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 58ea7ae5f..9947b10de 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -36,10 +36,16 @@ class S3ClientAWS: public S3Client { ~S3ClientAWS(); - auto createBucket(const std::string& name) const -> bool override; + auto createBucket(const std::string& bucketName) const -> bool override; + + auto deleteBucket(const std::string& bucketName) const -> bool override; void listBuckets() const override; + auto putObject(const std::string& bucketName, const std::string& objectName) const -> bool override; + + auto listObjects(const std::string& bucketName) const -> bool override; + private: // members Aws::Client::ClientConfiguration config_; }; diff --git a/tests/io/CMakeLists.txt b/tests/io/CMakeLists.txt index a4e936fcb..fb9459f12 100644 --- a/tests/io/CMakeLists.txt +++ b/tests/io/CMakeLists.txt @@ -74,6 +74,12 @@ ecbuild_add_test( TARGET eckit_test_radoshandle INCLUDES ${RADOS_INCLUDE_DIRS} LIBS eckit ) +ecbuild_add_test( TARGET eckit_test_s3client + SOURCES test_s3client.cc + CONDITION HAVE_AWS_S3 + INCLUDES ${AWSSDK_INCLUDE_DIRS} + LIBS eckit ) + ecbuild_add_test( TARGET eckit_test_s3handle SOURCES test_s3handle.cc CONDITION HAVE_AWS_S3 diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc new file mode 100644 index 000000000..d2a375fc0 --- /dev/null +++ b/tests/io/test_s3client.cc @@ -0,0 +1,61 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file test_s3client.cc +/// @author Metin Cakircali +/// @date Jan 2024 + +#include "eckit/config/LibEcKit.h" +#include "eckit/io/Buffer.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/testing/Test.h" + +#include + +using namespace std; +using namespace eckit; +using namespace eckit::testing; + +namespace eckit::test { + +S3Config cfg {S3Types::AWS, "name", "eu-central-1", "127.0.0.1", 9000}; + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("create bucket") { + auto client = S3Client::makeUnique(cfg); + client->createBucket("test-bucket-1"); +} + +CASE("list bucket") { + auto client = S3Client::makeUnique(cfg); + client->listBuckets(); +} + +CASE("delete bucket") { + auto client = S3Client::makeUnique(cfg); + + client->deleteBucket("test-bucket-1"); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test + +int main(int argc, char** argv) { + return run_tests(argc, argv); +} From 4aaec564ec69510b8c028a2e16357c7709cc0ef9 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 7 Feb 2024 17:14:36 +0100 Subject: [PATCH 05/90] feat(S3): adds delete,list object --- src/eckit/io/s3/S3Client.cc | 6 --- src/eckit/io/s3/S3Client.h | 7 ++- src/eckit/io/s3/aws/S3ClientAWS.cc | 83 +++++++++++++++++++++++------- src/eckit/io/s3/aws/S3ClientAWS.h | 8 ++- tests/io/test_s3client.cc | 21 +++++--- 5 files changed, 89 insertions(+), 36 deletions(-) diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index a6f248490..e0a22d019 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -38,10 +38,4 @@ auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { //---------------------------------------------------------------------------------------------------------------------- -void S3Client::listBuckets() const { - NOTIMP; -} - -//---------------------------------------------------------------------------------------------------------------------- - } // namespace eckit diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index d14b76307..a3d3988f2 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -23,6 +23,7 @@ #include "eckit/io/s3/S3Context.h" #include +#include namespace eckit { @@ -44,11 +45,13 @@ class S3Client { virtual auto deleteBucket(const std::string& bucketName) const -> bool = 0; + virtual auto listBuckets() const -> std::vector = 0; + virtual auto putObject(const std::string& bucketName, const std::string& objectName) const -> bool = 0; - virtual auto listObjects(const std::string& bucketName) const -> bool = 0; + virtual auto deleteObject(const std::string& bucketName, const std::string& objectKey) const -> bool = 0; - virtual void listBuckets() const = 0; + virtual auto listObjects(const std::string& bucketName) const -> std::vector = 0; private: // members S3ContextSPtr context_; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index e55c9da7c..d33d5c741 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -22,7 +22,10 @@ #include "eckit/io/s3/aws/S3ContextAWS.h" #include +#include #include +#include +#include #include #include @@ -101,16 +104,19 @@ auto S3ClientAWS::deleteBucket(const std::string& bucketName) const -> bool { return outcome.IsSuccess(); } -void S3ClientAWS::listBuckets() const { +auto S3ClientAWS::listBuckets() const -> std::vector { Aws::S3::S3Client client(config_); + std::vector result; + auto outcome = client.ListBuckets(); if (outcome.IsSuccess()) { - Log::info() << "Found " << outcome.GetResult().GetBuckets().size() << " buckets:\n"; - for (auto& bucket : outcome.GetResult().GetBuckets()) { Log::info() << bucket.GetName() << std::endl; } + for (auto& bucket : outcome.GetResult().GetBuckets()) { result.emplace_back(bucket.GetName()); } } else { - Log::error() << "List failed! " << outcome.GetError().GetMessage() << std::endl; + Log::error() << "Cannot list buckets! " << outcome.GetError().GetMessage() << std::endl; } + + return result; } //---------------------------------------------------------------------------------------------------------------------- @@ -125,15 +131,6 @@ auto S3ClientAWS::putObject(const std::string& bucketName, const std::string& ob // However, this is just a string and can be set according to your retrieval needs. request.SetKey(objectName); - // std::shared_ptr inputData = Aws::MakeShared("SampleAllocationTag", - // objectName.c_str(), - // std::ios_base::in | - // std::ios_base::binary); - // if (!*inputData) { - // std::cerr << "Error unable to read file " << objectName << std::endl; - // return false; - // } - const std::shared_ptr inputData = Aws::MakeShared(""); // *inputData << objectContent.c_str(); @@ -151,22 +148,70 @@ auto S3ClientAWS::putObject(const std::string& bucketName, const std::string& ob return outcome.IsSuccess(); } -auto S3ClientAWS::listObjects(const std::string& bucketName) const -> bool { +auto S3ClientAWS::deleteObject(const std::string& bucketName, const std::string& objectKey) const -> bool { + Aws::S3::S3Client client(config_); + + Aws::S3::Model::DeleteObjectRequest request; + request.WithKey(objectKey).WithBucket(bucketName); + + Aws::S3::Model::DeleteObjectOutcome outcome = client.DeleteObject(request); + + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Successfully deleted the object." << std::endl; + } else { + auto err = outcome.GetError(); + Log::error() << "Error: DeleteObject: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl; + } + + return outcome.IsSuccess(); +} + +auto S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const -> bool { + Aws::S3::S3Client client(config_); + + Aws::S3::Model::DeleteObjectsRequest request; + + Aws::S3::Model::Delete deleteObject; + for (const auto& objectKey : objectKeys) { + deleteObject.AddObjects(Aws::S3::Model::ObjectIdentifier().WithKey(objectKey)); + } + + request.SetDelete(deleteObject); + request.SetBucket(bucketName); + + Aws::S3::Model::DeleteObjectsOutcome outcome = client.DeleteObjects(request); + + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Deleted " << objectKeys.size() << " objects from bucket " << bucketName << std::endl; + for (auto i = 0; i < objectKeys.size(); ++i) { + LOG_DEBUG_LIB(LibEcKit) << "Deleted object: " << objectKeys[i] << std::endl; + } + } else { + const auto err = outcome.GetError(); + Log::error() << "Error deleting objects. " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl; + } + + return outcome.IsSuccess(); +} + +auto S3ClientAWS::listObjects(const std::string& bucketName) const -> std::vector { Aws::S3::S3Client s3_client(config_); + std::vector result; + Aws::S3::Model::ListObjectsRequest request; request.WithBucket(bucketName); auto outcome = s3_client.ListObjects(request); - if (!outcome.IsSuccess()) { - Log::error() << "Error: ListObjects: " << outcome.GetError().GetMessage() << std::endl; - } else { + if (outcome.IsSuccess()) { Aws::Vector objects = outcome.GetResult().GetContents(); - for (Aws::S3::Model::Object& object : objects) { Log::info() << object.GetKey() << std::endl; } + for (Aws::S3::Model::Object& object : objects) { result.emplace_back(object.GetKey()); } + } else { + Log::error() << "Cannot list objects! " << outcome.GetError().GetMessage() << std::endl; } - return outcome.IsSuccess(); + return result; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 9947b10de..ad14fa066 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -40,11 +40,15 @@ class S3ClientAWS: public S3Client { auto deleteBucket(const std::string& bucketName) const -> bool override; - void listBuckets() const override; + auto listBuckets() const -> std::vector override; auto putObject(const std::string& bucketName, const std::string& objectName) const -> bool override; - auto listObjects(const std::string& bucketName) const -> bool override; + auto deleteObject(const std::string& bucketName, const std::string& objectKey) const -> bool; + + auto deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const -> bool; + + auto listObjects(const std::string& bucketName) const -> std::vector override; private: // members Aws::Client::ClientConfiguration config_; diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index d2a375fc0..7e5dea056 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -38,18 +38,25 @@ S3Config cfg {S3Types::AWS, "name", "eu-central-1", "127.0.0.1", 9000}; CASE("create bucket") { auto client = S3Client::makeUnique(cfg); - client->createBucket("test-bucket-1"); -} -CASE("list bucket") { - auto client = S3Client::makeUnique(cfg); - client->listBuckets(); + EXPECT(client->createBucket("test-bucket-1")); + + EXPECT_NOT(client->createBucket("test-bucket-1")); + + EXPECT(client->createBucket("test-bucket-2")); } -CASE("delete bucket") { +CASE("list buckets") { auto client = S3Client::makeUnique(cfg); - client->deleteBucket("test-bucket-1"); + const auto buckets = client->listBuckets(); + + EXPECT_EQUAL(buckets[0], "test-bucket-1"); + EXPECT_EQUAL(buckets[1], "test-bucket-2"); + + for (auto&& bucket : buckets) { EXPECT(client->deleteBucket(bucket)); } + + EXPECT_NOT(client->deleteBucket("test-bucket-1")); } //---------------------------------------------------------------------------------------------------------------------- From eda4bfd795686f7cc62a0c5fcf9ff06473ce217f Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 7 Feb 2024 17:25:31 +0100 Subject: [PATCH 06/90] fix(S3): throw on unkown S3 client type --- src/eckit/io/s3/S3Client.cc | 3 +-- src/eckit/io/s3/S3Config.h | 3 +-- tests/io/test_s3client.cc | 6 +++++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index e0a22d019..948691bd2 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -32,8 +32,7 @@ S3Client::~S3Client() = default; auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { if (config.type == S3Types::AWS) { return std::make_unique(config); } - Log::error() << "Empty client!" << std::endl; - return {}; + throw S3SeriousBug("Unkown S3 client type!", Here()); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index e1efd3aa7..311149191 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -30,8 +30,7 @@ enum class S3Types { NONE, AWS, REST }; struct S3Config { S3Types type {S3Types::AWS}; - std::string name; - std::string region; + std::string region {"eu-central-1"}; std::string endpoint {"127.0.0.1"}; int port {0}; }; diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index 7e5dea056..9f9e679e1 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -32,10 +32,14 @@ using namespace eckit::testing; namespace eckit::test { -S3Config cfg {S3Types::AWS, "name", "eu-central-1", "127.0.0.1", 9000}; +S3Config cfg {S3Types::AWS, "eu-central-1", "127.0.0.1", 9000}; //---------------------------------------------------------------------------------------------------------------------- +CASE("unkown S3 type") { + EXPECT_THROWS(S3Client::makeUnique({S3Types::NONE})); +} + CASE("create bucket") { auto client = S3Client::makeUnique(cfg); From 3fb4e4de29d20549985a05f0ace276b2a8c24454 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 7 Feb 2024 22:34:21 +0100 Subject: [PATCH 07/90] feat(S3): adds credentials to S3Config --- src/eckit/io/s3/S3Client.cc | 10 ++- src/eckit/io/s3/S3Client.h | 4 + src/eckit/io/s3/S3Config.h | 5 +- src/eckit/io/s3/aws/S3ClientAWS.cc | 118 +++++++++++++--------------- src/eckit/io/s3/aws/S3ClientAWS.h | 8 +- src/eckit/io/s3/aws/S3ContextAWS.cc | 3 +- tests/io/test_s3client.cc | 20 ++++- 7 files changed, 95 insertions(+), 73 deletions(-) diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index 948691bd2..e0a2d2b31 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -30,11 +30,17 @@ S3Client::~S3Client() = default; //---------------------------------------------------------------------------------------------------------------------- -auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { - if (config.type == S3Types::AWS) { return std::make_unique(config); } +auto S3Client::makeUnique(const S3Types type) -> std::unique_ptr { + if (type == S3Types::AWS) { return std::make_unique(); } throw S3SeriousBug("Unkown S3 client type!", Here()); } +auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { + auto client = makeUnique(config.type); + client->configure(config); + return client; +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index a3d3988f2..ebb1be2d1 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -39,8 +39,12 @@ class S3Client { virtual ~S3Client(); + static auto makeUnique(S3Types type) -> std::unique_ptr; + static auto makeUnique(const S3Config& config) -> std::unique_ptr; + virtual void configure(const S3Config& config) = 0; + virtual auto createBucket(const std::string& bucketName) const -> bool = 0; virtual auto deleteBucket(const std::string& bucketName) const -> bool = 0; diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 311149191..6b1beaeda 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -30,9 +30,12 @@ enum class S3Types { NONE, AWS, REST }; struct S3Config { S3Types type {S3Types::AWS}; + std::string keyID; + std::string secret; + std::string tag {"ALLOCATION_TAG"}; std::string region {"eu-central-1"}; std::string endpoint {"127.0.0.1"}; - int port {0}; + int port {8888}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index d33d5c741..a3d446efb 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -21,6 +21,8 @@ #include "eckit/io/s3/S3Session.h" #include "eckit/io/s3/aws/S3ContextAWS.h" +#include +#include #include #include #include @@ -36,84 +38,91 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3ClientAWS::S3ClientAWS(const S3Config& config): S3Client(S3Session::instance().getContext(config.type)) { - LOG_DEBUG_LIB(LibEcKit) << "S3ClientAWS::setConfiguration" << std::endl; +namespace helper { + +void printError(const Aws::S3::S3Error& error, const std::string& msg) { + Log::error() << msg << ", error: " << error.GetExceptionName() << ", msg: " << error.GetMessage() + << " Remote IP: " << error.GetRemoteHostIpAddress() << std::endl; +} + +} // namespace helper + +//---------------------------------------------------------------------------------------------------------------------- + +S3ClientAWS::S3ClientAWS(): + S3Client(S3Session::instance().getContext(S3Types::AWS)), client_(std::make_unique()) { } + +S3ClientAWS::~S3ClientAWS() = default; + +//---------------------------------------------------------------------------------------------------------------------- + +void S3ClientAWS::configure(const S3Config& config) { + LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client." << std::endl; + + Aws::Client::ClientConfiguration configuration; // we are not an ec2 instance - config_.disableIMDS = true; + configuration.disableIMDS = true; // setup region - config_.region = config.region; - if (!config.region.empty()) { config_.region = config.region; } + configuration.region = config.region; + if (!config.region.empty()) { configuration.region = config.region; } - // config_.proxyScheme = Aws::Http::Scheme::HTTPS; - // config_.verifySSL = false; + // configuration.proxyScheme = Aws::Http::Scheme::HTTPS; + // configuration.verifySSL = false; // setup endpoint - if (!config.endpoint.empty()) { config_.endpointOverride = "http://" + config.endpoint; } - if (config.port > 0) { config_.endpointOverride += ":" + std::to_string(config.port); } -} + if (!config.endpoint.empty()) { configuration.endpointOverride = "http://" + config.endpoint; } + if (config.port > 0) { configuration.endpointOverride += ":" + std::to_string(config.port); } -S3ClientAWS::~S3ClientAWS() = default; + // setup credentials + Aws::Auth::AWSCredentials credentials(config.keyID, config.secret); + + // endpoint provider + auto provider = Aws::MakeShared(config.tag.c_str()); + + client_ = std::make_unique(credentials, provider, configuration); +} //---------------------------------------------------------------------------------------------------------------------- // BUCKET auto S3ClientAWS::createBucket(const std::string& bucketName) const -> bool { - Aws::S3::S3Client client(config_); - Aws::S3::Model::CreateBucketRequest request; request.SetBucket(bucketName); - // Change the bucket location constraint enum to target Region. - if (config_.region != "eu-central-1") { - Aws::S3::Model::CreateBucketConfiguration bucketConfig; - bucketConfig.SetLocationConstraint( - Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(config_.region)); - request.SetCreateBucketConfiguration(bucketConfig); - } - - Aws::S3::Model::CreateBucketOutcome outcome = client.CreateBucket(request); + auto outcome = client_->CreateBucket(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Created bucket: " << bucketName << std::endl; } else { - const auto err = outcome.GetError(); - Log::error() << "Failed to create bucket: " << bucketName << " Error: " << err.GetExceptionName() << ": " - << err.GetMessage() << std::endl; + helper::printError(outcome.GetError(), "Failed to create bucket: " + bucketName); } return outcome.IsSuccess(); } auto S3ClientAWS::deleteBucket(const std::string& bucketName) const -> bool { - Aws::S3::S3Client client(config_); - Aws::S3::Model::DeleteBucketRequest request; request.SetBucket(bucketName); - Aws::S3::Model::DeleteBucketOutcome outcome = client.DeleteBucket(request); - + auto outcome = client_->DeleteBucket(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket: " << bucketName << std::endl; } else { - const Aws::S3::S3Error& err = outcome.GetError(); - Log::error() << "Failed to delete bucket: " << bucketName << " Error: " << err.GetExceptionName() << ": " - << err.GetMessage() << std::endl; + helper::printError(outcome.GetError(), "Failed to delete bucket: " + bucketName); } return outcome.IsSuccess(); } auto S3ClientAWS::listBuckets() const -> std::vector { - Aws::S3::S3Client client(config_); - std::vector result; - auto outcome = client.ListBuckets(); + auto outcome = client_->ListBuckets(); if (outcome.IsSuccess()) { for (auto& bucket : outcome.GetResult().GetBuckets()) { result.emplace_back(bucket.GetName()); } } else { - Log::error() << "Cannot list buckets! " << outcome.GetError().GetMessage() << std::endl; + helper::printError(outcome.GetError(), "Failed to list buckets!"); } return result; @@ -123,52 +132,41 @@ auto S3ClientAWS::listBuckets() const -> std::vector { // OBJECT auto S3ClientAWS::putObject(const std::string& bucketName, const std::string& objectName) const -> bool { - Aws::S3::S3Client s3_client(config_); - Aws::S3::Model::PutObjectRequest request; + request.SetBucket(bucketName); - // We are using the name of the file as the key for the object in the bucket. - // However, this is just a string and can be set according to your retrieval needs. request.SetKey(objectName); const std::shared_ptr inputData = Aws::MakeShared(""); - // *inputData << objectContent.c_str(); - request.SetBody(inputData); - Aws::S3::Model::PutObjectOutcome outcome = s3_client.PutObject(request); - + auto outcome = client_->PutObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Added object '" << objectName << "' to bucket '" << bucketName << "'."; } else { - Log::error() << "Error: putObject: " << outcome.GetError().GetMessage() << std::endl; + helper::printError(outcome.GetError(), "Failed to put object: " + objectName + " to bucket: " + bucketName); } return outcome.IsSuccess(); } auto S3ClientAWS::deleteObject(const std::string& bucketName, const std::string& objectKey) const -> bool { - Aws::S3::S3Client client(config_); - Aws::S3::Model::DeleteObjectRequest request; - request.WithKey(objectKey).WithBucket(bucketName); - Aws::S3::Model::DeleteObjectOutcome outcome = client.DeleteObject(request); + request.WithKey(objectKey).WithBucket(bucketName); + auto outcome = client_->DeleteObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Successfully deleted the object." << std::endl; } else { - auto err = outcome.GetError(); - Log::error() << "Error: DeleteObject: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl; + helper::printError(outcome.GetError(), "Failed to delete object: " + objectKey + " in bucket: " + bucketName); } return outcome.IsSuccess(); } auto S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const -> bool { - Aws::S3::S3Client client(config_); - Aws::S3::Model::DeleteObjectsRequest request; Aws::S3::Model::Delete deleteObject; @@ -179,36 +177,32 @@ auto S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector request.SetDelete(deleteObject); request.SetBucket(bucketName); - Aws::S3::Model::DeleteObjectsOutcome outcome = client.DeleteObjects(request); - + auto outcome = client_->DeleteObjects(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Deleted " << objectKeys.size() << " objects from bucket " << bucketName << std::endl; for (auto i = 0; i < objectKeys.size(); ++i) { LOG_DEBUG_LIB(LibEcKit) << "Deleted object: " << objectKeys[i] << std::endl; } } else { - const auto err = outcome.GetError(); - Log::error() << "Error deleting objects. " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl; + helper::printError(outcome.GetError(), "Failed to delete objects in bucket: " + bucketName); } return outcome.IsSuccess(); } auto S3ClientAWS::listObjects(const std::string& bucketName) const -> std::vector { - Aws::S3::S3Client s3_client(config_); - std::vector result; Aws::S3::Model::ListObjectsRequest request; - request.WithBucket(bucketName); - auto outcome = s3_client.ListObjects(request); + request.WithBucket(bucketName); + auto outcome = client_->ListObjects(request); if (outcome.IsSuccess()) { Aws::Vector objects = outcome.GetResult().GetContents(); for (Aws::S3::Model::Object& object : objects) { result.emplace_back(object.GetKey()); } } else { - Log::error() << "Cannot list objects! " << outcome.GetError().GetMessage() << std::endl; + helper::printError(outcome.GetError(), "Failed to list objects in bucket: " + bucketName); } return result; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index ad14fa066..34a332f5d 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -32,10 +32,12 @@ class S3ClientAWS: public S3Client { public: // methods NO_COPY_NO_MOVE(S3ClientAWS) - explicit S3ClientAWS(const S3Config& config); + explicit S3ClientAWS(); ~S3ClientAWS(); + void configure(const S3Config& config) override; + auto createBucket(const std::string& bucketName) const -> bool override; auto deleteBucket(const std::string& bucketName) const -> bool override; @@ -44,14 +46,14 @@ class S3ClientAWS: public S3Client { auto putObject(const std::string& bucketName, const std::string& objectName) const -> bool override; - auto deleteObject(const std::string& bucketName, const std::string& objectKey) const -> bool; + auto deleteObject(const std::string& bucketName, const std::string& objectKey) const -> bool override; auto deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const -> bool; auto listObjects(const std::string& bucketName) const -> std::vector override; private: // members - Aws::Client::ClientConfiguration config_; + std::unique_ptr client_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ContextAWS.cc b/src/eckit/io/s3/aws/S3ContextAWS.cc index bc64e4b90..abbc4fe96 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.cc +++ b/src/eckit/io/s3/aws/S3ContextAWS.cc @@ -49,8 +49,7 @@ auto S3ContextAWS::checkCredentials(const char* chain) -> bool { // AWS S3 permits unauthenticated requests; client returns "success" but 0 buckets auto provider = Aws::MakeShared(chain); - auto creds = provider->GetAWSCredentials(); - if (creds.IsExpiredOrEmpty()) { + if (provider->GetAWSCredentials().IsExpiredOrEmpty()) { Log::warning() << "Authentication failed!" << std::endl; return false; } diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index 9f9e679e1..434f6e768 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -32,12 +32,26 @@ using namespace eckit::testing; namespace eckit::test { -S3Config cfg {S3Types::AWS, "eu-central-1", "127.0.0.1", 9000}; +S3Config cfg {S3Types::AWS, "minio", "minio1234", "test-tag", "eu-central-1", "127.0.0.1", 9000}; //---------------------------------------------------------------------------------------------------------------------- -CASE("unkown S3 type") { - EXPECT_THROWS(S3Client::makeUnique({S3Types::NONE})); +CASE("different types") { + EXPECT_THROWS(S3Client::makeUnique(S3Types::NONE)); + EXPECT_NO_THROW(S3Client::makeUnique(S3Types::AWS)); +} + +CASE("wrong credentials") { + EXPECT_NOT(S3Client::makeUnique(S3Types::AWS)->createBucket("failed-bucket")); +} + +CASE("create bucket in missing region") { + S3Config config(cfg); + config.region = "eu-central-2"; + + auto client = S3Client::makeUnique(config); + + EXPECT_NOT(client->createBucket("test-bucket-1")); } CASE("create bucket") { From 408d9c27b9491a2c1882541880713e0cffe8f1d8 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 8 Feb 2024 11:42:28 +0100 Subject: [PATCH 08/90] refactor(S3): throw instead of return boolean --- src/eckit/io/s3/S3Client.h | 8 ++-- src/eckit/io/s3/S3Exception.h | 3 ++ src/eckit/io/s3/aws/S3ClientAWS.cc | 66 ++++++++++++++++-------------- src/eckit/io/s3/aws/S3ClientAWS.h | 10 ++--- tests/io/test_s3client.cc | 15 +++---- 5 files changed, 56 insertions(+), 46 deletions(-) diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index ebb1be2d1..a61096d16 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -45,15 +45,15 @@ class S3Client { virtual void configure(const S3Config& config) = 0; - virtual auto createBucket(const std::string& bucketName) const -> bool = 0; + virtual void createBucket(const std::string& bucketName) const = 0; - virtual auto deleteBucket(const std::string& bucketName) const -> bool = 0; + virtual void deleteBucket(const std::string& bucketName) const = 0; virtual auto listBuckets() const -> std::vector = 0; - virtual auto putObject(const std::string& bucketName, const std::string& objectName) const -> bool = 0; + virtual void putObject(const std::string& bucketName, const std::string& objectName) const = 0; - virtual auto deleteObject(const std::string& bucketName, const std::string& objectKey) const -> bool = 0; + virtual void deleteObject(const std::string& bucketName, const std::string& objectKey) const = 0; virtual auto listObjects(const std::string& bucketName) const -> std::vector = 0; diff --git a/src/eckit/io/s3/S3Exception.h b/src/eckit/io/s3/S3Exception.h index 644976ffc..6ccd7cde1 100644 --- a/src/eckit/io/s3/S3Exception.h +++ b/src/eckit/io/s3/S3Exception.h @@ -32,6 +32,9 @@ class S3SeriousBug: public SeriousBug { public: S3SeriousBug(const std::string& msg, const CodeLocation& loc); S3SeriousBug(const std::string& msg, int code, const CodeLocation& loc); + S3SeriousBug(const std::ostringstream& msg, const CodeLocation& loc): S3SeriousBug(msg.str(), loc) { } + S3SeriousBug(const std::ostringstream& msg, int code, const CodeLocation& loc): + S3SeriousBug(msg.str(), code, loc) { } }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index a3d446efb..c0c23ff4e 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -34,18 +34,19 @@ #include #include -namespace eckit { - //---------------------------------------------------------------------------------------------------------------------- -namespace helper { +namespace { -void printError(const Aws::S3::S3Error& error, const std::string& msg) { - Log::error() << msg << ", error: " << error.GetExceptionName() << ", msg: " << error.GetMessage() - << " Remote IP: " << error.GetRemoteHostIpAddress() << std::endl; +std::ostream& operator<<(std::ostream& os, const Aws::S3::S3Error& error) { + os << "Error: " << error.GetExceptionName() << ", Message: " << error.GetMessage() + << " Remote IP: " << error.GetRemoteHostIpAddress() << std::endl; + return os; } -} // namespace helper +} // namespace + +namespace eckit { //---------------------------------------------------------------------------------------------------------------------- @@ -87,32 +88,34 @@ void S3ClientAWS::configure(const S3Config& config) { //---------------------------------------------------------------------------------------------------------------------- // BUCKET -auto S3ClientAWS::createBucket(const std::string& bucketName) const -> bool { +void S3ClientAWS::createBucket(const std::string& bucketName) const { Aws::S3::Model::CreateBucketRequest request; request.SetBucket(bucketName); auto outcome = client_->CreateBucket(request); + if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Created bucket: " << bucketName << std::endl; } else { - helper::printError(outcome.GetError(), "Failed to create bucket: " + bucketName); + std::ostringstream msg; + msg << "Failed to create bucket: " << bucketName << outcome.GetError(); + throw S3SeriousBug(msg, Here()); } - - return outcome.IsSuccess(); } -auto S3ClientAWS::deleteBucket(const std::string& bucketName) const -> bool { +void S3ClientAWS::deleteBucket(const std::string& bucketName) const { Aws::S3::Model::DeleteBucketRequest request; request.SetBucket(bucketName); auto outcome = client_->DeleteBucket(request); + if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket: " << bucketName << std::endl; } else { - helper::printError(outcome.GetError(), "Failed to delete bucket: " + bucketName); + std::ostringstream msg; + msg << "Failed to delete bucket: " << bucketName << outcome.GetError(); + throw S3SeriousBug(msg, Here()); } - - return outcome.IsSuccess(); } auto S3ClientAWS::listBuckets() const -> std::vector { @@ -122,7 +125,7 @@ auto S3ClientAWS::listBuckets() const -> std::vector { if (outcome.IsSuccess()) { for (auto& bucket : outcome.GetResult().GetBuckets()) { result.emplace_back(bucket.GetName()); } } else { - helper::printError(outcome.GetError(), "Failed to list buckets!"); + Log::warning() << "Failed to list buckets!" << outcome.GetError(); } return result; @@ -131,7 +134,7 @@ auto S3ClientAWS::listBuckets() const -> std::vector { //---------------------------------------------------------------------------------------------------------------------- // OBJECT -auto S3ClientAWS::putObject(const std::string& bucketName, const std::string& objectName) const -> bool { +void S3ClientAWS::putObject(const std::string& bucketName, const std::string& objectName) const { Aws::S3::Model::PutObjectRequest request; request.SetBucket(bucketName); @@ -145,13 +148,13 @@ auto S3ClientAWS::putObject(const std::string& bucketName, const std::string& ob if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Added object '" << objectName << "' to bucket '" << bucketName << "'."; } else { - helper::printError(outcome.GetError(), "Failed to put object: " + objectName + " to bucket: " + bucketName); + std::ostringstream msg; + msg << "Failed to put object: " << objectName << " to bucket: " << bucketName << outcome.GetError(); + throw S3SeriousBug(msg, Here()); } - - return outcome.IsSuccess(); } -auto S3ClientAWS::deleteObject(const std::string& bucketName, const std::string& objectKey) const -> bool { +void S3ClientAWS::deleteObject(const std::string& bucketName, const std::string& objectKey) const { Aws::S3::Model::DeleteObjectRequest request; request.WithKey(objectKey).WithBucket(bucketName); @@ -160,13 +163,13 @@ auto S3ClientAWS::deleteObject(const std::string& bucketName, const std::string& if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Successfully deleted the object." << std::endl; } else { - helper::printError(outcome.GetError(), "Failed to delete object: " + objectKey + " in bucket: " + bucketName); + std::ostringstream msg; + msg << "Failed to delete object: " << objectKey << " in bucket: " << bucketName << outcome.GetError(); + throw S3SeriousBug(msg, Here()); } - - return outcome.IsSuccess(); } -auto S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const -> bool { +void S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const { Aws::S3::Model::DeleteObjectsRequest request; Aws::S3::Model::Delete deleteObject; @@ -184,10 +187,10 @@ auto S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector LOG_DEBUG_LIB(LibEcKit) << "Deleted object: " << objectKeys[i] << std::endl; } } else { - helper::printError(outcome.GetError(), "Failed to delete objects in bucket: " + bucketName); + std::ostringstream msg; + msg << "Failed to delete objects in bucket: " << bucketName << outcome.GetError(); + throw S3SeriousBug(msg, Here()); } - - return outcome.IsSuccess(); } auto S3ClientAWS::listObjects(const std::string& bucketName) const -> std::vector { @@ -200,9 +203,12 @@ auto S3ClientAWS::listObjects(const std::string& bucketName) const -> std::vecto auto outcome = client_->ListObjects(request); if (outcome.IsSuccess()) { Aws::Vector objects = outcome.GetResult().GetContents(); - for (Aws::S3::Model::Object& object : objects) { result.emplace_back(object.GetKey()); } + for (Aws::S3::Model::Object& object : objects) { + // object. + result.emplace_back(object.GetKey()); + } } else { - helper::printError(outcome.GetError(), "Failed to list objects in bucket: " + bucketName); + Log::warning() << "Failed to list objects in bucket: " << bucketName << outcome.GetError(); } return result; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 34a332f5d..a544bb434 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -38,17 +38,17 @@ class S3ClientAWS: public S3Client { void configure(const S3Config& config) override; - auto createBucket(const std::string& bucketName) const -> bool override; + void createBucket(const std::string& bucketName) const override; - auto deleteBucket(const std::string& bucketName) const -> bool override; + void deleteBucket(const std::string& bucketName) const override; auto listBuckets() const -> std::vector override; - auto putObject(const std::string& bucketName, const std::string& objectName) const -> bool override; + void putObject(const std::string& bucketName, const std::string& objectName) const override; - auto deleteObject(const std::string& bucketName, const std::string& objectKey) const -> bool override; + void deleteObject(const std::string& bucketName, const std::string& objectKey) const override; - auto deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const -> bool; + void deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const; auto listObjects(const std::string& bucketName) const -> std::vector override; diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index 434f6e768..a1fa585d6 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -42,7 +42,7 @@ CASE("different types") { } CASE("wrong credentials") { - EXPECT_NOT(S3Client::makeUnique(S3Types::AWS)->createBucket("failed-bucket")); + EXPECT_THROWS(S3Client::makeUnique(S3Types::AWS)->createBucket("failed-bucket")); } CASE("create bucket in missing region") { @@ -51,17 +51,17 @@ CASE("create bucket in missing region") { auto client = S3Client::makeUnique(config); - EXPECT_NOT(client->createBucket("test-bucket-1")); + EXPECT_THROWS(client->createBucket("test-bucket-1")); } CASE("create bucket") { auto client = S3Client::makeUnique(cfg); - EXPECT(client->createBucket("test-bucket-1")); + EXPECT_NO_THROW(client->createBucket("test-bucket-1")); - EXPECT_NOT(client->createBucket("test-bucket-1")); + EXPECT_THROWS(client->createBucket("test-bucket-1")); - EXPECT(client->createBucket("test-bucket-2")); + EXPECT_NO_THROW(client->createBucket("test-bucket-2")); } CASE("list buckets") { @@ -72,9 +72,10 @@ CASE("list buckets") { EXPECT_EQUAL(buckets[0], "test-bucket-1"); EXPECT_EQUAL(buckets[1], "test-bucket-2"); - for (auto&& bucket : buckets) { EXPECT(client->deleteBucket(bucket)); } + for (auto&& bucket : buckets) { EXPECT_NO_THROW(client->deleteBucket(bucket)); } - EXPECT_NOT(client->deleteBucket("test-bucket-1")); + EXPECT_THROWS(client->deleteBucket("test-bucket-1")); + EXPECT_THROWS(client->deleteBucket("test-bucket-2")); } //---------------------------------------------------------------------------------------------------------------------- From 028a4eeafce3e68fd3e72f0fe57649ba2e352dca Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 8 Feb 2024 11:43:43 +0100 Subject: [PATCH 09/90] chore(S3): remove advanced attribute from AWS_S3 cmake option --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a4014c9c6..b026032aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,7 +107,6 @@ ecbuild_add_option( FEATURE RADOS ecbuild_add_option( FEATURE AWS_S3 DEFAULT OFF - ADVANCED REQUIRED_PACKAGES "AWSSDK REQUIRED COMPONENTS s3" DESCRIPTION "Enables AWS S3 support" ) From 7423b5cb3e833675799686f34f81e1842f69ed50 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 8 Feb 2024 11:46:42 +0100 Subject: [PATCH 10/90] style(S3): fix todo style comment --- src/eckit/io/s3/aws/S3ContextAWS.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eckit/io/s3/aws/S3ContextAWS.cc b/src/eckit/io/s3/aws/S3ContextAWS.cc index abbc4fe96..99f17c3b2 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.cc +++ b/src/eckit/io/s3/aws/S3ContextAWS.cc @@ -40,7 +40,7 @@ S3ContextAWS::~S3ContextAWS() { auto S3ContextAWS::makeShared() -> std::shared_ptr { Aws::SDKOptions options; - // TODO: remove debugging logs + /// @todo: remove debugging logs options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; return std::make_shared(options); } From c3e93a2e6539e4e99c4af74984323490f60ae870 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 8 Feb 2024 12:01:52 +0100 Subject: [PATCH 11/90] refactor(S3): namedspace -> unnamed namespace --- src/eckit/io/s3/S3Exception.cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc index 9c196c514..e571a48ef 100644 --- a/src/eckit/io/s3/S3Exception.cc +++ b/src/eckit/io/s3/S3Exception.cc @@ -24,11 +24,9 @@ #include -namespace eckit { - //---------------------------------------------------------------------------------------------------------------------- -namespace helper { +namespace { inline auto addCode(const std::string& msg, int code) -> std::string { std::ostringstream oss; @@ -36,15 +34,16 @@ inline auto addCode(const std::string& msg, int code) -> std::string { return oss.str(); }; -} // namespace helper +} // namespace + +namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3SeriousBug::S3SeriousBug(const std::string& msg, const CodeLocation& loc): - SeriousBug(std::string("[S3 Error] ") + msg, loc) { } +S3SeriousBug::S3SeriousBug(const std::string& msg, const CodeLocation& loc): SeriousBug("[S3 Error] " + msg, loc) { } S3SeriousBug::S3SeriousBug(const std::string& msg, const int code, const CodeLocation& loc): - S3SeriousBug(helper::addCode(msg, code), loc) { } + S3SeriousBug(addCode(msg, code), loc) { } //---------------------------------------------------------------------------------------------------------------------- From 1aab48fff5e36d46acd1da0df41a1a729cb1d1a2 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 8 Feb 2024 12:19:49 +0100 Subject: [PATCH 12/90] refactor(S3): cleanup S3Session --- src/eckit/io/s3/S3Session.cc | 11 ----------- src/eckit/io/s3/S3Session.h | 2 -- 2 files changed, 13 deletions(-) diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index 53f734c5a..596875d12 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -40,17 +40,6 @@ S3Session& S3Session::instance() { return session; } -// auto S3Session::createClient(const S3Config& config) -> std::unique_ptr { -// LOG_DEBUG_LIB(LibEcKit) << "Creating S3Client..." << std::endl; -// auto ctx = getContext(config.type); -// if (config.type == S3Types::AWS) { -// auto client = std::make_unique(ctx); -// client->configure(config); -// return client; -// } -// return {}; -// } - //---------------------------------------------------------------------------------------------------------------------- S3Session::S3Session() = default; diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h index 7ae5355a6..f86be2594 100644 --- a/src/eckit/io/s3/S3Session.h +++ b/src/eckit/io/s3/S3Session.h @@ -35,8 +35,6 @@ class S3Session { static S3Session& instance(); - // auto createClient(const S3Config& config) -> std::unique_ptr; - NODISCARD auto getContext(S3Types type) -> std::shared_ptr; From d775bff936801f99d8291f37cd80d40e8e4e8066 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 8 Feb 2024 12:20:09 +0100 Subject: [PATCH 13/90] fix(S3): add missing pragma once --- src/eckit/io/s3/S3Name.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 7d214a6a8..4e83a8858 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -18,6 +18,8 @@ /// @author Metin Cakircali /// @date Jan 2024 +#pragma once + #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Macros.h" From 620fd1824c1b7a84a278c14239ebafdb6af1b5af Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 8 Feb 2024 12:20:55 +0100 Subject: [PATCH 14/90] refactor(S3): cleanup S3Handle --- src/eckit/io/s3/S3Handle.cc | 39 ++++++++----------------------------- src/eckit/io/s3/S3Handle.h | 26 ++++++++++++------------- 2 files changed, 21 insertions(+), 44 deletions(-) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 3fe5427c7..06fc95b2a 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -25,35 +25,12 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -class S3Handle::S3HandleInternal { -public: - S3HandleInternal(const PathName& name, const eckit::Offset& offset): name_(name), pos_(offset) { } - - friend std::ostream& operator<<(std::ostream& os, const S3HandleInternal& handle) { - os << "name: " << handle.name_ << ", pos: " << handle.pos_ << " can write: " << handle.canWrite_ << std::endl; - return os; - } - -private: - // const PathName name_; - S3Name name_; - - // S3Object object_; - - Offset pos_ {0}; - - bool canWrite_ {false}; -}; - -//---------------------------------------------------------------------------------------------------------------------- - S3Handle::S3Handle(const PathName& name): S3Handle(name, 0) { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; } -S3Handle::S3Handle(const PathName& name, const eckit::Offset& offset): - impl_(std::make_unique(name, offset)) { +S3Handle::S3Handle(const PathName& name, const Offset& offset): name_(name), pos_(offset) { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; } @@ -64,12 +41,12 @@ S3Handle::~S3Handle() { } void S3Handle::print(std::ostream& out) const { - out << "S3Handle[" << impl_ << "]"; + // out << "S3Handle[" << impl_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- -eckit::Length S3Handle::openForRead() { +Length S3Handle::openForRead() { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; @@ -80,7 +57,7 @@ eckit::Length S3Handle::openForRead() { return {}; } -void S3Handle::openForWrite(const eckit::Length& length) { +void S3Handle::openForWrite(const Length& length) { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; // canWrite_ = true; @@ -88,7 +65,7 @@ void S3Handle::openForWrite(const eckit::Length& length) { // if (!object_->create(name_.pool())) { throw NotFound("Cannot create object!", -1, Here()); } } -void S3Handle::openForAppend(const eckit::Length& length) { +void S3Handle::openForAppend(const Length& length) { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; // NOTIMP; @@ -122,20 +99,20 @@ void S3Handle::close() { LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; } -eckit::Length S3Handle::size() { +Length S3Handle::size() { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; return {}; } -eckit::Offset S3Handle::position() { +Offset S3Handle::position() { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; // return pos_; return {}; } -eckit::Offset S3Handle::seek(const eckit::Offset& offset) { +Offset S3Handle::seek(const Offset& offset) { // pos_ = pos_ + offset; // return pos_; /// @todo remove diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index 4ecd8773c..bdbf8782b 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -22,6 +22,7 @@ #include "eckit/io/DataHandle.h" #include "eckit/io/s3/S3Macros.h" +#include "eckit/io/s3/S3Name.h" #include #include @@ -31,25 +32,23 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class S3Handle: public DataHandle { -public: +public: // methods NO_COPY_NO_MOVE(S3Handle) - // NOTE: @Nicolau we will try S3 stream later which is not here yet - explicit S3Handle(const PathName& name); - explicit S3Handle(const PathName& name, const eckit::Offset& offset); + explicit S3Handle(const PathName& name, const Offset& offset); ~S3Handle() override; void print(std::ostream& out) const override; -public: - eckit::Length openForRead() override; +public: // methods + Length openForRead() override; - void openForWrite(const eckit::Length& length) override; + void openForWrite(const Length& length) override; - void openForAppend(const eckit::Length& length) override; + void openForAppend(const Length& length) override; long read(void* buffer, long length) override; @@ -59,19 +58,20 @@ class S3Handle: public DataHandle { void close() override; - eckit::Length size() override; + Length size() override; - eckit::Offset position() override; + Offset position() override; - eckit::Offset seek(const eckit::Offset& offset) override; + Offset seek(const Offset& offset) override; std::string title() const override; bool canSeek() const override { return true; } private: // members - class S3HandleInternal; - std::unique_ptr impl_; + S3Name name_; + Offset pos_ {0}; + // bool canWrite_ {false}; }; //---------------------------------------------------------------------------------------------------------------------- From de556ff085f75dfdabf9364519a25ac9f48d8ed1 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 8 Feb 2024 15:34:20 +0100 Subject: [PATCH 15/90] refactor(S3): more cleanup for S3Handle, S3Name --- src/eckit/io/s3/S3Handle.cc | 5 ++--- src/eckit/io/s3/S3Handle.h | 7 ++----- src/eckit/io/s3/S3Name.cc | 16 ++++++---------- src/eckit/io/s3/S3Name.h | 15 ++++++++------- tests/io/test_s3client.cc | 2 +- tests/io/test_s3handle.cc | 20 ++++++++++++++++---- 6 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 06fc95b2a..8e0092c49 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -17,7 +17,6 @@ #include "eckit/io/s3/S3Handle.h" #include "eckit/config/LibEcKit.h" -#include "eckit/filesystem/PathName.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Name.h" @@ -25,12 +24,12 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Handle::S3Handle(const PathName& name): S3Handle(name, 0) { +S3Handle::S3Handle(const S3Name& name): S3Handle(name, 0) { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; } -S3Handle::S3Handle(const PathName& name, const Offset& offset): name_(name), pos_(offset) { +S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; } diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index bdbf8782b..4aee755af 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -24,9 +24,6 @@ #include "eckit/io/s3/S3Macros.h" #include "eckit/io/s3/S3Name.h" -#include -#include - namespace eckit { //---------------------------------------------------------------------------------------------------------------------- @@ -35,9 +32,9 @@ class S3Handle: public DataHandle { public: // methods NO_COPY_NO_MOVE(S3Handle) - explicit S3Handle(const PathName& name); + explicit S3Handle(const S3Name& name); - explicit S3Handle(const PathName& name, const Offset& offset); + explicit S3Handle(const S3Name& name, const Offset& offset); ~S3Handle() override; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 1964360d7..bc17cb61d 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -17,27 +17,23 @@ #include "eckit/io/s3/S3Name.h" #include "eckit/io/s3/S3Exception.h" -#include "eckit/io/s3/S3Object.h" +#include "eckit/io/s3/S3Handle.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Name::S3Name(const std::string& name): uri_("S3", name, 9000) { } +S3Name::S3Name(const std::string& regionName, const std::string& bucketName, const std::string& objectName): + region_(regionName), bucket_(bucketName), object_(objectName) { } S3Name::~S3Name() = default; void S3Name::print(std::ostream& out) const { - out << "S3Name[uri=" << uri_ << "]"; + out << "S3Name[region=" << region_ << ",bucket=" << bucket_ << ",object=" << object_ << "]"; } -auto S3Name::uri() const -> URI { - return uri_; -} - -auto S3Name::createObject() -> std::unique_ptr { - NOTIMP; - return std::unique_ptr(); +auto S3Name::dataHandle() -> std::unique_ptr { + return std::make_unique(*this); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 4e83a8858..e392876cf 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -20,25 +20,24 @@ #pragma once -#include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Macros.h" +#include + namespace eckit { -class S3Object; +class DataHandle; //---------------------------------------------------------------------------------------------------------------------- class S3Name { public: // methods - S3Name(const std::string& name); + S3Name(const std::string& regionName, const std::string& bucketName, const std::string& objectName); ~S3Name(); - auto uri() const -> URI; - NODISCARD - auto createObject() -> std::unique_ptr; + auto dataHandle() -> std::unique_ptr; friend std::ostream& operator<<(std::ostream& out, const S3Name& name) { name.print(out); @@ -49,7 +48,9 @@ class S3Name { void print(std::ostream& out) const; private: // members - URI uri_; + std::string region_; + std::string bucket_; + std::string object_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index a1fa585d6..eb7a45e5b 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -46,7 +46,7 @@ CASE("wrong credentials") { } CASE("create bucket in missing region") { - S3Config config(cfg); + auto config = cfg; config.region = "eu-central-2"; auto client = S3Client::makeUnique(config); diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index d3f1b9b52..258a5bd2d 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -18,6 +18,7 @@ /// @author Metin Cakircali /// @date Jan 2024 +#include "eckit/filesystem/URI.h" #include "eckit/io/Buffer.h" #include "eckit/io/s3/S3Handle.h" #include "eckit/testing/Test.h" @@ -35,11 +36,22 @@ namespace eckit::test { CASE("S3Handle") { const char buf[] = "abcdefghijklmnopqrstuvwxyz"; - S3Handle h("blablaregion/bucketname/objectname"); + S3Name name("myS3region", "myS3bucket", "myS3object"); - h.openForWrite(sizeof(buf)); - h.write(buf, sizeof(buf)); - h.close(); + { + auto h = name.dataHandle(); + h->openForWrite(sizeof(buf)); + h->write(buf, sizeof(buf)); + } + // h->close(); + + URI uri("s3://hostname:port/region/bucket/object"); + // credentials + + // std::unique_ptr dh = uri.dataHandle(); + // dh->openForRead(); + // AutoClose closer(dh); + // dh->read(buf, length); } //---------------------------------------------------------------------------------------------------------------------- From ed4ed4bba3a6b252ec914ea6c018ddfc6aa45729 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 8 Feb 2024 15:34:44 +0100 Subject: [PATCH 16/90] fix(S3): aws error parsing --- src/eckit/io/s3/aws/S3ClientAWS.cc | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index c0c23ff4e..588b850bd 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -38,10 +38,10 @@ namespace { -std::ostream& operator<<(std::ostream& os, const Aws::S3::S3Error& error) { - os << "Error: " << error.GetExceptionName() << ", Message: " << error.GetMessage() - << " Remote IP: " << error.GetRemoteHostIpAddress() << std::endl; - return os; +inline std::string awsErrorMessage(const std::string& msg, const Aws::S3::S3Error& error) { + std::ostringstream oss; + oss << " AWS: " << error.GetMessage() << " Remote IP: " << error.GetRemoteHostIpAddress(); + return oss.str(); } } // namespace @@ -97,8 +97,7 @@ void S3ClientAWS::createBucket(const std::string& bucketName) const { if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Created bucket: " << bucketName << std::endl; } else { - std::ostringstream msg; - msg << "Failed to create bucket: " << bucketName << outcome.GetError(); + auto msg = awsErrorMessage("Failed to create bucket: " + bucketName, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } @@ -112,8 +111,7 @@ void S3ClientAWS::deleteBucket(const std::string& bucketName) const { if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket: " << bucketName << std::endl; } else { - std::ostringstream msg; - msg << "Failed to delete bucket: " << bucketName << outcome.GetError(); + auto msg = awsErrorMessage("Failed to delete bucket: " + bucketName, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } @@ -148,8 +146,8 @@ void S3ClientAWS::putObject(const std::string& bucketName, const std::string& ob if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Added object '" << objectName << "' to bucket '" << bucketName << "'."; } else { - std::ostringstream msg; - msg << "Failed to put object: " << objectName << " to bucket: " << bucketName << outcome.GetError(); + auto msg = + awsErrorMessage("Failed to put object: " + objectName + " to bucket: " + bucketName, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } @@ -163,8 +161,8 @@ void S3ClientAWS::deleteObject(const std::string& bucketName, const std::string& if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Successfully deleted the object." << std::endl; } else { - std::ostringstream msg; - msg << "Failed to delete object: " << objectKey << " in bucket: " << bucketName << outcome.GetError(); + auto msg = + awsErrorMessage("Failed to delete object: " + objectKey + " in bucket: " + bucketName, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } @@ -187,8 +185,7 @@ void S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector LOG_DEBUG_LIB(LibEcKit) << "Deleted object: " << objectKeys[i] << std::endl; } } else { - std::ostringstream msg; - msg << "Failed to delete objects in bucket: " << bucketName << outcome.GetError(); + auto msg = awsErrorMessage("Failed to delete objects in bucket: " + bucketName, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } @@ -203,10 +200,7 @@ auto S3ClientAWS::listObjects(const std::string& bucketName) const -> std::vecto auto outcome = client_->ListObjects(request); if (outcome.IsSuccess()) { Aws::Vector objects = outcome.GetResult().GetContents(); - for (Aws::S3::Model::Object& object : objects) { - // object. - result.emplace_back(object.GetKey()); - } + for (Aws::S3::Model::Object& object : objects) { result.emplace_back(object.GetKey()); } } else { Log::warning() << "Failed to list objects in bucket: " << bucketName << outcome.GetError(); } From 7eaa9be5e1fb05cb6314601dfc03e9291b9c6eb5 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 8 Feb 2024 17:31:24 +0100 Subject: [PATCH 17/90] feat(S3): rework credentials --- src/eckit/CMakeLists.txt | 1 + src/eckit/io/s3/S3Config.h | 2 -- src/eckit/io/s3/S3Context.cc | 2 ++ src/eckit/io/s3/S3Credential.h | 37 +++++++++++++++++++ src/eckit/io/s3/S3Macros.h | 9 +++++ src/eckit/io/s3/S3Session.cc | 56 ++++++++++++++++++++++++----- src/eckit/io/s3/S3Session.h | 15 ++++++-- src/eckit/io/s3/aws/S3ClientAWS.cc | 8 ++++- src/eckit/io/s3/aws/S3ContextAWS.cc | 14 -------- src/eckit/io/s3/aws/S3ContextAWS.h | 2 -- tests/io/test_s3client.cc | 5 ++- tests/io/test_s3handle.cc | 3 ++ 12 files changed, 123 insertions(+), 31 deletions(-) create mode 100644 src/eckit/io/s3/S3Credential.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 056a351d5..ffed56c30 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -297,6 +297,7 @@ io/s3/S3Client.h io/s3/S3Config.h io/s3/S3Context.cc io/s3/S3Context.h +io/s3/S3Credential.h io/s3/S3Exception.cc io/s3/S3Exception.h io/s3/S3Handle.cc diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 6b1beaeda..9ed15d4fe 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -30,8 +30,6 @@ enum class S3Types { NONE, AWS, REST }; struct S3Config { S3Types type {S3Types::AWS}; - std::string keyID; - std::string secret; std::string tag {"ALLOCATION_TAG"}; std::string region {"eu-central-1"}; std::string endpoint {"127.0.0.1"}; diff --git a/src/eckit/io/s3/S3Context.cc b/src/eckit/io/s3/S3Context.cc index 0d0c025ab..441eabf84 100644 --- a/src/eckit/io/s3/S3Context.cc +++ b/src/eckit/io/s3/S3Context.cc @@ -32,8 +32,10 @@ S3Context::~S3Context() = default; auto S3Context::makeShared(S3Types type) -> std::shared_ptr { // AWS SDK API if (type == S3Types::AWS) { return S3ContextAWS::makeShared(); } + // REST API if (type == S3Types::REST) { NOTIMP; } + return {}; } diff --git a/src/eckit/io/s3/S3Credential.h b/src/eckit/io/s3/S3Credential.h new file mode 100644 index 000000000..3b7cab3a7 --- /dev/null +++ b/src/eckit/io/s3/S3Credential.h @@ -0,0 +1,37 @@ +/* + * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file S3Credential.h +/// @author Metin Cakircali +/// @date Feb 2024 + +#pragma once + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +struct S3Credential { + std::string keyID; + std::string secret; + std::string endpoint {"127.0.0.1"}; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Macros.h b/src/eckit/io/s3/S3Macros.h index 053fe1ed2..e6dedf33c 100644 --- a/src/eckit/io/s3/S3Macros.h +++ b/src/eckit/io/s3/S3Macros.h @@ -20,7 +20,16 @@ #pragma once +// nodiscard is C++17 or higher +#if defined(__has_cpp_attribute) +#if __has_cpp_attribute(nodiscard) #define NODISCARD [[nodiscard]] +#endif +#elif !defined(_MSC_VER) +#define NODISCARD __attribute__((warn_unused_result)) +#else +#define NODISCARD +#endif #define NO_COPY(TypeName) \ TypeName(const TypeName&) = delete; \ diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index 596875d12..7a769d6b3 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -17,21 +17,32 @@ #include "eckit/io/s3/S3Session.h" #include "eckit/config/LibEcKit.h" +#include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/aws/S3ClientAWS.h" namespace eckit { -namespace utils { +namespace { /// @brief Functor for S3Context type struct IsContextType { - S3Types type_; + const S3Types type_; bool operator()(const std::shared_ptr& ctx) const { return ctx->getType() == type_; } }; -} // namespace utils +/// @brief Functor for S3Credential endpoint +struct IsCredentialEndpoint { + IsCredentialEndpoint(const std::string& endpoint): endpoint_(endpoint) { } + + bool operator()(const std::shared_ptr& cred) const { return cred->endpoint == endpoint_; } + +private: + const std::string& endpoint_; +}; + +} // namespace //---------------------------------------------------------------------------------------------------------------------- @@ -47,6 +58,34 @@ S3Session::S3Session() = default; S3Session::~S3Session() = default; //---------------------------------------------------------------------------------------------------------------------- +// CREDENTIALS + +auto S3Session::getCredentials(const std::string& endpoint) const -> std::shared_ptr { + LOG_DEBUG_LIB(LibEcKit) << "Searching credential for endpoint:" << endpoint << std::endl; + + /// @todo check if all keyid and secret different + // search by endpoint + auto cred = std::find_if(credentials_.begin(), credentials_.end(), IsCredentialEndpoint(endpoint)); + // found + if (cred != credentials_.end()) { return *cred; } + // not found + return {}; +} + +void S3Session::addCredentials(const S3Credential& credential) { + // check if already exists + if (getCredentials(credential.endpoint)) { return; } + // add new item + auto cred = std::make_shared(credential); + credentials_.emplace_back(cred); +} + +void S3Session::removeCredentials(const std::string& endpoint) { + credentials_.remove_if(IsCredentialEndpoint(endpoint)); +} + +//---------------------------------------------------------------------------------------------------------------------- +// CONTEXT auto S3Session::getContext(const S3Types type) -> std::shared_ptr { // return if found @@ -54,7 +93,7 @@ auto S3Session::getContext(const S3Types type) -> std::shared_ptr { // not found auto context = S3Context::makeShared(type); - registry_.push_back(context); + contexts_.push_back(context); return context; } @@ -62,19 +101,20 @@ auto S3Session::getContext(const S3Types type) -> std::shared_ptr { auto S3Session::findContext(const S3Types type) -> std::shared_ptr { LOG_DEBUG_LIB(LibEcKit) << "Find context!" << std::endl; // search by type - auto context = std::find_if(registry_.begin(), registry_.end(), utils::IsContextType({type})); + auto context = std::find_if(contexts_.begin(), contexts_.end(), IsContextType({type})); // found - if (context != registry_.end()) { return *context; } + if (context != contexts_.end()) { return *context; } // not found return {}; } void S3Session::removeContext(const S3Types type) { - registry_.remove_if(utils::IsContextType({type})); + contexts_.remove_if(IsContextType({type})); } void S3Session::clear() { - registry_.clear(); + contexts_.clear(); + credentials_.clear(); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h index f86be2594..45048d53f 100644 --- a/src/eckit/io/s3/S3Session.h +++ b/src/eckit/io/s3/S3Session.h @@ -21,6 +21,7 @@ #pragma once #include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Credential.h" #include #include @@ -36,10 +37,14 @@ class S3Session { static S3Session& instance(); NODISCARD - auto getContext(S3Types type) -> std::shared_ptr; + auto getCredentials(const std::string& endpoint) const -> std::shared_ptr; + + void addCredentials(const S3Credential& credential); + + void removeCredentials(const std::string& endpoint); NODISCARD - auto findContext(S3Types type) -> std::shared_ptr; + auto getContext(S3Types type) -> std::shared_ptr; void removeContext(S3Types type); @@ -50,8 +55,12 @@ class S3Session { ~S3Session(); + NODISCARD + auto findContext(S3Types type) -> std::shared_ptr; + private: // members - std::list> registry_; + std::list> contexts_; + std::list> credentials_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 588b850bd..4724e51f1 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -17,6 +17,7 @@ #include "eckit/io/s3/aws/S3ClientAWS.h" #include "eckit/config/LibEcKit.h" +#include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Session.h" #include "eckit/io/s3/aws/S3ContextAWS.h" @@ -77,11 +78,16 @@ void S3ClientAWS::configure(const S3Config& config) { if (config.port > 0) { configuration.endpointOverride += ":" + std::to_string(config.port); } // setup credentials - Aws::Auth::AWSCredentials credentials(config.keyID, config.secret); + Aws::Auth::AWSCredentials credentials; + if (auto cred = S3Session::instance().getCredentials(config.endpoint)) { + credentials.SetAWSAccessKeyId(cred->keyID); + credentials.SetAWSSecretKey(cred->secret); + } // endpoint provider auto provider = Aws::MakeShared(config.tag.c_str()); + // finally client client_ = std::make_unique(credentials, provider, configuration); } diff --git a/src/eckit/io/s3/aws/S3ContextAWS.cc b/src/eckit/io/s3/aws/S3ContextAWS.cc index 99f17c3b2..a4318fe82 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.cc +++ b/src/eckit/io/s3/aws/S3ContextAWS.cc @@ -45,20 +45,6 @@ auto S3ContextAWS::makeShared() -> std::shared_ptr { return std::make_shared(options); } -auto S3ContextAWS::checkCredentials(const char* chain) -> bool { - // AWS S3 permits unauthenticated requests; client returns "success" but 0 buckets - auto provider = Aws::MakeShared(chain); - - if (provider->GetAWSCredentials().IsExpiredOrEmpty()) { - Log::warning() << "Authentication failed!" << std::endl; - return false; - } - - Log::info() << "AWS authentication was successful!" << std::endl; - - return true; -} - //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ContextAWS.h b/src/eckit/io/s3/aws/S3ContextAWS.h index 9dc4ffbc4..c6ccc3f5e 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.h +++ b/src/eckit/io/s3/aws/S3ContextAWS.h @@ -38,8 +38,6 @@ class S3ContextAWS: public S3Context { ~S3ContextAWS(); - static auto checkCredentials(const char* chain) -> bool; - private: // methods friend S3Context; diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index eb7a45e5b..1cf1e9fb7 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -32,7 +32,7 @@ using namespace eckit::testing; namespace eckit::test { -S3Config cfg {S3Types::AWS, "minio", "minio1234", "test-tag", "eu-central-1", "127.0.0.1", 9000}; +S3Config cfg {S3Types::AWS, "test-tag", "eu-central-1", "127.0.0.1", 9000}; //---------------------------------------------------------------------------------------------------------------------- @@ -83,5 +83,8 @@ CASE("list buckets") { } // namespace eckit::test int main(int argc, char** argv) { + S3Credential cred {"minio", "minio1234", "127.0.0.1"}; + S3Session::instance().addCredentials(cred); + return run_tests(argc, argv); } diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 258a5bd2d..fbb2003af 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -21,6 +21,7 @@ #include "eckit/filesystem/URI.h" #include "eckit/io/Buffer.h" #include "eckit/io/s3/S3Handle.h" +#include "eckit/io/s3/S3Session.h" #include "eckit/testing/Test.h" #include @@ -59,5 +60,7 @@ CASE("S3Handle") { } // namespace eckit::test int main(int argc, char** argv) { + S3Credential cred {"minio", "minio1234", "127.0.0.1"}; + S3Session::instance().addCredentials(cred); return run_tests(argc, argv); } From 545f8c63135f481826bfc536514df928531324a1 Mon Sep 17 00:00:00 2001 From: Simon Smart Date: Thu, 8 Feb 2024 22:10:43 +0000 Subject: [PATCH 18/90] Add some URI infrastructure, and tweak some tests --- src/eckit/CMakeLists.txt | 2 ++ src/eckit/filesystem/URIManager.h | 2 +- src/eckit/io/s3/S3Name.cc | 23 ++++++++++++-- src/eckit/io/s3/S3Name.h | 9 ++++-- src/eckit/io/s3/S3URIManager.cc | 40 +++++++++++++++++++++++ src/eckit/io/s3/S3URIManager.h | 40 +++++++++++++++++++++++ tests/io/test_s3client.cc | 49 +++++++++++++++++++++++++--- tests/io/test_s3handle.cc | 53 +++++++++++++++++++++++++------ 8 files changed, 199 insertions(+), 19 deletions(-) create mode 100644 src/eckit/io/s3/S3URIManager.cc create mode 100644 src/eckit/io/s3/S3URIManager.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index ffed56c30..8b3c22916 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -309,6 +309,8 @@ io/s3/S3Object.h io/s3/S3Session.cc io/s3/S3Session.h io/s3/S3Macros.h +io/s3/S3URIManager.cc +io/s3/S3URIManager.h ) endif(HAVE_AWS_S3) diff --git a/src/eckit/filesystem/URIManager.h b/src/eckit/filesystem/URIManager.h index 79ed9d617..051f040ff 100644 --- a/src/eckit/filesystem/URIManager.h +++ b/src/eckit/filesystem/URIManager.h @@ -49,7 +49,7 @@ class URIManager { static URIManager& lookUp(const std::string&); protected: - URIManager(const std::string&); + explicit URIManager(const std::string&); virtual ~URIManager(); diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index bc17cb61d..2ff6746d3 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -18,6 +18,9 @@ #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Handle.h" +#include "eckit/filesystem/URI.h" +#include "eckit/utils/Tokenizer.h" +#include "eckit/utils/StringTools.h" namespace eckit { @@ -26,14 +29,30 @@ namespace eckit { S3Name::S3Name(const std::string& regionName, const std::string& bucketName, const std::string& objectName): region_(regionName), bucket_(bucketName), object_(objectName) { } +S3Name::S3Name(const URI& uri) { + ASSERT(uri.scheme() == "s3"); + region_ = uri.host(); + + const std::string& name = uri.name(); + size_t pos = name.find('/'); + ASSERT(pos != std::string::npos); + + bucket_ = name.substr(0, pos); + object_ = name.substr(pos + 1); +} + S3Name::~S3Name() = default; void S3Name::print(std::ostream& out) const { out << "S3Name[region=" << region_ << ",bucket=" << bucket_ << ",object=" << object_ << "]"; } -auto S3Name::dataHandle() -> std::unique_ptr { - return std::make_unique(*this); +bool S3Name::exists() const { + NOTIMP; +} + +DataHandle* S3Name::dataHandle() { + return new S3Handle(*this); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index e392876cf..7ea77ef84 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -27,17 +27,22 @@ namespace eckit { class DataHandle; +class URI; //---------------------------------------------------------------------------------------------------------------------- class S3Name { public: // methods + explicit S3Name(const URI& uri); + S3Name(const std::string& regionName, const std::string& bucketName, const std::string& objectName); ~S3Name(); - NODISCARD - auto dataHandle() -> std::unique_ptr; + bool exists() const; + + [[ nodiscard ]] + DataHandle* dataHandle(); friend std::ostream& operator<<(std::ostream& out, const S3Name& name) { name.print(out); diff --git a/src/eckit/io/s3/S3URIManager.cc b/src/eckit/io/s3/S3URIManager.cc new file mode 100644 index 000000000..7ffa78555 --- /dev/null +++ b/src/eckit/io/s3/S3URIManager.cc @@ -0,0 +1,40 @@ + +#include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3URIManager.h" +#include "eckit/io/s3/S3Name.h" + +namespace eckit { + +static S3URIManager manager_s3("s3"); + +//---------------------------------------------------------------------------------------------------------------------- + +S3URIManager::S3URIManager(const std::string& name) : + URIManager(name) {} + +S3URIManager::~S3URIManager() = default; + +bool S3URIManager::exists(const URI& uri) { + return S3Name(uri).exists(); +} + +DataHandle* S3URIManager::newWriteHandle(const URI& uri) { + return S3Name(uri).dataHandle(); +} + +DataHandle* S3URIManager::newReadHandle(const URI& uri) { + return S3Name(uri).dataHandle(); +} + +DataHandle* S3URIManager::newReadHandle(const URI& uri, const OffsetList&, const LengthList&) { + return S3Name(uri).dataHandle(); +} + +std::string S3URIManager::asString(const URI& uri) const { + return std::string("s3://") + uri.name(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} + diff --git a/src/eckit/io/s3/S3URIManager.h b/src/eckit/io/s3/S3URIManager.h new file mode 100644 index 000000000..8a5558c97 --- /dev/null +++ b/src/eckit/io/s3/S3URIManager.h @@ -0,0 +1,40 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#pragma once + +#include "eckit/filesystem/URIManager.h" + + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3URIManager : public URIManager { + +public: // methods + + S3URIManager(const std::string& name); + ~S3URIManager() override; + +private: // methods + + bool exists(const URI&) override; + + DataHandle* newWriteHandle(const URI&) override; + DataHandle* newReadHandle(const URI&) override; + DataHandle* newReadHandle(const URI&, const OffsetList&, const LengthList&) override; + + std::string asString(const URI& uri) const override; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index 1cf1e9fb7..521072d95 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -34,6 +34,23 @@ namespace eckit::test { S3Config cfg {S3Types::AWS, "test-tag", "eu-central-1", "127.0.0.1", 9000}; +//---------------------------------------------------------------------------------------------------------------------- + +void ensureClean() { + + auto client = S3Client::makeUnique(cfg); + auto&& tmp = client->listBuckets(); + std::set buckets(tmp.begin(), tmp.end()); + + for (const std::string& name : {"failed-bucket", "test-bucket-1", "test-bucket-2"}) { + if (buckets.find(name) != buckets.end()) { + client->deleteBucket(name); + } + } + +} + + //---------------------------------------------------------------------------------------------------------------------- CASE("different types") { @@ -42,10 +59,13 @@ CASE("different types") { } CASE("wrong credentials") { + ensureClean(); EXPECT_THROWS(S3Client::makeUnique(S3Types::AWS)->createBucket("failed-bucket")); } CASE("create bucket in missing region") { + ensureClean(); + auto config = cfg; config.region = "eu-central-2"; @@ -55,6 +75,7 @@ CASE("create bucket in missing region") { } CASE("create bucket") { + ensureClean(); auto client = S3Client::makeUnique(cfg); EXPECT_NO_THROW(client->createBucket("test-bucket-1")); @@ -65,14 +86,28 @@ CASE("create bucket") { } CASE("list buckets") { + + ensureClean(); + auto client = S3Client::makeUnique(cfg); + EXPECT_NO_THROW(client->createBucket("test-bucket-1")); + EXPECT_NO_THROW(client->createBucket("test-bucket-2")); + + { + const auto buckets = client->listBuckets(); - const auto buckets = client->listBuckets(); + EXPECT_EQUAL(buckets[0], "test-bucket-1"); + EXPECT_EQUAL(buckets[1], "test-bucket-2"); - EXPECT_EQUAL(buckets[0], "test-bucket-1"); - EXPECT_EQUAL(buckets[1], "test-bucket-2"); + for (auto&& bucket : buckets) { + client->deleteBucket(bucket); + } + } - for (auto&& bucket : buckets) { EXPECT_NO_THROW(client->deleteBucket(bucket)); } + { + const auto buckets = client->listBuckets(); + EXPECT(buckets.empty()); + } EXPECT_THROWS(client->deleteBucket("test-bucket-1")); EXPECT_THROWS(client->deleteBucket("test-bucket-2")); @@ -86,5 +121,9 @@ int main(int argc, char** argv) { S3Credential cred {"minio", "minio1234", "127.0.0.1"}; S3Session::instance().addCredentials(cred); - return run_tests(argc, argv); + auto ret = run_tests(argc, argv); + try { + eckit::test::ensureClean(); + } catch (...) {} + return ret; } diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index fbb2003af..752229565 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -19,7 +19,7 @@ /// @date Jan 2024 #include "eckit/filesystem/URI.h" -#include "eckit/io/Buffer.h" +#include "eckit/io/MemoryHandle.h" #include "eckit/io/s3/S3Handle.h" #include "eckit/io/s3/S3Session.h" #include "eckit/testing/Test.h" @@ -32,29 +32,57 @@ using namespace eckit::testing; namespace eckit::test { +S3Config cfg {S3Types::AWS, "test-tag", "eu-central-1", "127.0.0.1", 9000}; + +static const std::string TEST_BUCKET("test-bucket"); + //---------------------------------------------------------------------------------------------------------------------- +void ensureClean() { + auto client = S3Client::makeUnique(cfg); + auto&& tmp = client->listBuckets(); + std::set buckets(tmp.begin(), tmp.end()); + + for (const std::string& name : {TEST_BUCKET}) { + if (buckets.find(name) != buckets.end()) { + client->deleteBucket(name); + } + } +} + +void bucketSetup() { + ensureClean(); + auto client = S3Client::makeUnique(cfg); + client->createBucket(TEST_BUCKET); +} + CASE("S3Handle") { const char buf[] = "abcdefghijklmnopqrstuvwxyz"; S3Name name("myS3region", "myS3bucket", "myS3object"); { - auto h = name.dataHandle(); + std::unique_ptr h{name.dataHandle()}; h->openForWrite(sizeof(buf)); - h->write(buf, sizeof(buf)); + AutoClose closer(*h); + EXPECT(h->write(buf, sizeof(buf)) == sizeof(buf)); } - // h->close(); URI uri("s3://hostname:port/region/bucket/object"); // credentials - // std::unique_ptr dh = uri.dataHandle(); - // dh->openForRead(); - // AutoClose closer(dh); - // dh->read(buf, length); + { + std::unique_ptr dh(uri.newReadHandle()); + MemoryHandle mh; + dh->saveInto(mh); + + EXPECT(mh.size() == Length(sizeof(buf))); + EXPECT(::memcmp(mh.data(), buf, sizeof(buf)) == 0); + } } +// TODO: Also check that it doesn't work if the bucket doesn't exist, etc. + //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test @@ -62,5 +90,12 @@ CASE("S3Handle") { int main(int argc, char** argv) { S3Credential cred {"minio", "minio1234", "127.0.0.1"}; S3Session::instance().addCredentials(cred); - return run_tests(argc, argv); + + int ret = -1; + try { + eckit::test::bucketSetup(); + ret = run_tests(argc, argv); + eckit::test::ensureClean(); + } catch (...) {} + return ret; } From e8bcf923e1eec2ce602481657eb934e96b895c29 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 09:16:43 +0100 Subject: [PATCH 19/90] refactor(S3): rework S3Config --- src/eckit/io/s3/S3Client.cc | 2 +- src/eckit/io/s3/S3Client.h | 1 + src/eckit/io/s3/S3Config.h | 52 +++++++++++++++++++++++++++--- src/eckit/io/s3/aws/S3ClientAWS.cc | 12 +++---- tests/io/test_s3client.cc | 4 +-- tests/io/test_s3handle.cc | 16 ++++++--- 6 files changed, 69 insertions(+), 18 deletions(-) diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index e0a2d2b31..cd98210c4 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -36,7 +36,7 @@ auto S3Client::makeUnique(const S3Types type) -> std::unique_ptr { } auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { - auto client = makeUnique(config.type); + auto client = makeUnique(config.type()); client->configure(config); return client; } diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index a61096d16..6165bb962 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -20,6 +20,7 @@ #pragma once +#include "eckit/io/s3/S3Config.h" #include "eckit/io/s3/S3Context.h" #include diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 9ed15d4fe..ea066099d 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -20,6 +20,9 @@ #pragma once +#include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3Macros.h" + #include namespace eckit { @@ -29,11 +32,50 @@ enum class S3Types { NONE, AWS, REST }; //---------------------------------------------------------------------------------------------------------------------- struct S3Config { - S3Types type {S3Types::AWS}; - std::string tag {"ALLOCATION_TAG"}; - std::string region {"eu-central-1"}; - std::string endpoint {"127.0.0.1"}; - int port {8888}; + S3Config(const std::string& region, const std::string& hostname, const int port): + region_(region), host_(hostname), port_(port) { } + + S3Config(const URI& uri): S3Config(uri.name(), uri.host(), uri.port()) { + // host_ = uri.host(); + // port_ = uri.port(); + /// @todo need to parse region from hostname + /// example: access-point.s3-accesspoint.[AWS Region code].amazonaws.com + // region_ = uri.name(); + } + + NODISCARD + auto type() const -> S3Types { return type_; } + + NODISCARD + auto tag() const -> const std::string& { return tag_; } + + NODISCARD + auto region() const -> const std::string& { return region_; } + + void setRegion(const std::string& region) { region_ = region; } + + NODISCARD + auto host() const -> const std::string& { return host_; } + + NODISCARD + auto port() const -> int { return port_; } + + NODISCARD + auto getURI() const -> URI { return {"s3", host_, port_}; } + + friend std::ostream& operator<<(std::ostream& out, const S3Config& config) { + out << "S3Config[tag=" << config.tag_ << ",region=" << config.region_ << ",host=" << config.host_ + << ",port=" << config.port_ << "]"; + return out; + } + +private: + S3Types type_ {S3Types::AWS}; + std::string tag_ {"ALLOCATION_TAG"}; + std::string region_ {"ecmwf-central-1"}; + + std::string host_ {"127.0.0.1"}; + int port_ {-1}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 4724e51f1..cbb44eb0f 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -67,25 +67,25 @@ void S3ClientAWS::configure(const S3Config& config) { configuration.disableIMDS = true; // setup region - configuration.region = config.region; - if (!config.region.empty()) { configuration.region = config.region; } + configuration.region = config.region(); + if (!config.region().empty()) { configuration.region = config.region(); } // configuration.proxyScheme = Aws::Http::Scheme::HTTPS; // configuration.verifySSL = false; // setup endpoint - if (!config.endpoint.empty()) { configuration.endpointOverride = "http://" + config.endpoint; } - if (config.port > 0) { configuration.endpointOverride += ":" + std::to_string(config.port); } + if (!config.host().empty()) { configuration.endpointOverride = "http://" + config.host(); } + if (config.port() > 0) { configuration.endpointOverride += ":" + std::to_string(config.port()); } // setup credentials Aws::Auth::AWSCredentials credentials; - if (auto cred = S3Session::instance().getCredentials(config.endpoint)) { + if (auto cred = S3Session::instance().getCredentials(config.host())) { credentials.SetAWSAccessKeyId(cred->keyID); credentials.SetAWSSecretKey(cred->secret); } // endpoint provider - auto provider = Aws::MakeShared(config.tag.c_str()); + auto provider = Aws::MakeShared(config.tag().c_str()); // finally client client_ = std::make_unique(credentials, provider, configuration); diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index 1cf1e9fb7..9e9f86dc6 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -32,7 +32,7 @@ using namespace eckit::testing; namespace eckit::test { -S3Config cfg {S3Types::AWS, "test-tag", "eu-central-1", "127.0.0.1", 9000}; +const S3Config cfg("eu-central-1", "127.0.0.1", 9000); //---------------------------------------------------------------------------------------------------------------------- @@ -47,7 +47,7 @@ CASE("wrong credentials") { CASE("create bucket in missing region") { auto config = cfg; - config.region = "eu-central-2"; + config.setRegion("eu-central-2"); auto client = S3Client::makeUnique(config); diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index fbb2003af..2adcd03cb 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -24,7 +24,7 @@ #include "eckit/io/s3/S3Session.h" #include "eckit/testing/Test.h" -#include +#include using namespace std; using namespace eckit; @@ -37,7 +37,9 @@ namespace eckit::test { CASE("S3Handle") { const char buf[] = "abcdefghijklmnopqrstuvwxyz"; - S3Name name("myS3region", "myS3bucket", "myS3object"); + S3Config config("eu-central-1", "127.0.0.1", 9000); + + S3Name name(config, "myS3bucket", "myS3object"); { auto h = name.dataHandle(); @@ -46,8 +48,14 @@ CASE("S3Handle") { } // h->close(); - URI uri("s3://hostname:port/region/bucket/object"); - // credentials + URI uri("s3://127.0.0.1:9000/bucket/object"); + { + auto h = uri.newReadHandle(); + h->openForRead(); + std::string rbuf; + h->read(rbuf.data(), sizeof(buf)); + std::cout << rbuf << std::endl; + } // std::unique_ptr dh = uri.dataHandle(); // dh->openForRead(); From c84f0468da3fa99a3db716e19f959e899090a663 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 09:23:56 +0100 Subject: [PATCH 20/90] feat(S3): start S3Name and S3URIManager --- src/eckit/io/s3/S3Name.cc | 59 +++++++++++++++++++++++++++++++++++++-- src/eckit/io/s3/S3Name.h | 18 ++++++++++-- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index bc17cb61d..13647d36d 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -16,6 +16,10 @@ #include "eckit/io/s3/S3Name.h" +#include "eckit/config/LibEcKit.h" +#include "eckit/filesystem/URI.h" +#include "eckit/filesystem/URIManager.h" +#include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Handle.h" @@ -23,19 +27,68 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Name::S3Name(const std::string& regionName, const std::string& bucketName, const std::string& objectName): - region_(regionName), bucket_(bucketName), object_(objectName) { } +S3Name::S3Name(const URI& uri): config_(uri) { + LOG_DEBUG_LIB(LibEcKit) << "URI -----> " << uri << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "URI -----> host=" << uri.host() << std::endl; + LOG_DEBUG_LIB(LibEcKit) << " -----> " << *this << std::endl; +} + +S3Name::S3Name(const S3Config& config, const std::string& bucketName, const std::string& objectName): + config_(config), bucket_(bucketName), object_(objectName) { } S3Name::~S3Name() = default; +//---------------------------------------------------------------------------------------------------------------------- + void S3Name::print(std::ostream& out) const { - out << "S3Name[region=" << region_ << ",bucket=" << bucket_ << ",object=" << object_ << "]"; + out << "S3Name[config=" << config_ << ",bucket=" << bucket_ << ",object=" << object_ << "]"; +} + +auto S3Name::exists() -> bool { + NOTIMP; + return false; } auto S3Name::dataHandle() -> std::unique_ptr { + LOG_DEBUG_LIB(LibEcKit) << "dataHandle " << *this << std::endl; return std::make_unique(*this); } +//---------------------------------------------------------------------------------------------------------------------- +// S3 URI MANAGER + +class S3URIManager: public URIManager { + bool query() override { return false; } + bool fragment() override { return false; } + + bool exists(const URI& uri) override { return S3Name(uri).exists(); } + + DataHandle* newWriteHandle(const URI& uri) override { return S3Name(uri).dataHandle().release(); } + + DataHandle* newReadHandle(const URI& uri) override { return S3Name(uri).dataHandle().release(); } + + DataHandle* newReadHandle(const URI& uri, const OffsetList& ol, const LengthList& ll) override { + NOTIMP; + return S3Name(uri).dataHandle().release(); + } + + std::string asString(const URI& uri) const override { + std::string q = uri.query(); + if (!q.empty()) { q = "?" + q; } + std::string f = uri.fragment(); + if (!f.empty()) { f = "#" + f; } + + return uri.name() + q + f; + } + +public: + S3URIManager(const std::string& name): URIManager(name) { } +}; + +//---------------------------------------------------------------------------------------------------------------------- + +static S3URIManager manager_s3("s3"); + //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index e392876cf..fb03e95fb 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -20,22 +20,34 @@ #pragma once +#include "eckit/io/s3/S3Config.h" #include "eckit/io/s3/S3Macros.h" +// #include #include namespace eckit { +class URI; class DataHandle; +// class S3Client; //---------------------------------------------------------------------------------------------------------------------- class S3Name { public: // methods - S3Name(const std::string& regionName, const std::string& bucketName, const std::string& objectName); + explicit S3Name(const URI& uri); + + S3Name(const S3Config& config, const std::string& bucketName, const std::string& objectName); ~S3Name(); + NODISCARD + auto getConfig() const -> const S3Config& { return config_; } + + NODISCARD + auto exists() -> bool; + NODISCARD auto dataHandle() -> std::unique_ptr; @@ -48,9 +60,11 @@ class S3Name { void print(std::ostream& out) const; private: // members - std::string region_; + S3Config config_; std::string bucket_; std::string object_; + + // std::unique_ptr client_; }; //---------------------------------------------------------------------------------------------------------------------- From 563679e15b9a1af01dd111e310c36a9835f960f2 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 14:19:08 +0100 Subject: [PATCH 21/90] fix(S3): authority must be true --- src/eckit/io/s3/S3URIManager.cc | 13 ++++++++++--- src/eckit/io/s3/S3URIManager.h | 21 +++++++++++++++------ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/eckit/io/s3/S3URIManager.cc b/src/eckit/io/s3/S3URIManager.cc index 7ffa78555..0558078dc 100644 --- a/src/eckit/io/s3/S3URIManager.cc +++ b/src/eckit/io/s3/S3URIManager.cc @@ -1,3 +1,12 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3URIManager.h" @@ -9,8 +18,7 @@ static S3URIManager manager_s3("s3"); //---------------------------------------------------------------------------------------------------------------------- -S3URIManager::S3URIManager(const std::string& name) : - URIManager(name) {} +S3URIManager::S3URIManager(const std::string& name): URIManager(name) { } S3URIManager::~S3URIManager() = default; @@ -37,4 +45,3 @@ std::string S3URIManager::asString(const URI& uri) const { //---------------------------------------------------------------------------------------------------------------------- } - diff --git a/src/eckit/io/s3/S3URIManager.h b/src/eckit/io/s3/S3URIManager.h index 8a5558c97..7a5262e4e 100644 --- a/src/eckit/io/s3/S3URIManager.h +++ b/src/eckit/io/s3/S3URIManager.h @@ -8,24 +8,33 @@ * does it submit to any jurisdiction. */ +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3URIManager.h +/// @author Simon Smart +/// @author Metin Cakircali +/// @date Feb 2024 + #pragma once #include "eckit/filesystem/URIManager.h" - namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -class S3URIManager : public URIManager { - -public: // methods - +class S3URIManager: public URIManager { +public: // methods S3URIManager(const std::string& name); + ~S3URIManager() override; -private: // methods + bool authority() override { return true; } +private: // methods bool exists(const URI&) override; DataHandle* newWriteHandle(const URI&) override; From 2003a5c1607fe6575529df246b38584ee796e933 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 14:20:07 +0100 Subject: [PATCH 22/90] feat(Endpoint): construct from a URI --- src/eckit/net/Endpoint.cc | 9 +++++++-- src/eckit/net/Endpoint.h | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/eckit/net/Endpoint.cc b/src/eckit/net/Endpoint.cc index 8be1ff83d..ad58f976e 100644 --- a/src/eckit/net/Endpoint.cc +++ b/src/eckit/net/Endpoint.cc @@ -10,17 +10,22 @@ #include "eckit/net/Endpoint.h" -#include - #include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" #include "eckit/serialisation/Stream.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" +#include + namespace eckit::net { //---------------------------------------------------------------------------------------------------------------------- +Endpoint::Endpoint(const URI& uri): host_(uri.host()), port_(uri.port()) { + validate(); +} + Endpoint::Endpoint(const std::string& s) { Tokenizer tokenize(":"); std::vector tokens; diff --git a/src/eckit/net/Endpoint.h b/src/eckit/net/Endpoint.h index fa53731c7..6cb300238 100644 --- a/src/eckit/net/Endpoint.h +++ b/src/eckit/net/Endpoint.h @@ -21,6 +21,7 @@ namespace eckit { class Stream; +class URI; namespace net { @@ -29,6 +30,7 @@ namespace net { class Endpoint { public: // methods + Endpoint(const URI& uri); // gets hostname:port from uri Endpoint(const std::string&); // parses the std::string formated as hostname:port Endpoint(const std::string& host, int port); Endpoint(Stream& s); From ff342d143ffdd0778185a3401ae63dc61f1af5ea Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 20:23:47 +0100 Subject: [PATCH 23/90] fix(S3): client missing error msg --- src/eckit/io/s3/aws/S3ClientAWS.cc | 11 +++++------ src/eckit/io/s3/aws/S3ClientAWS.h | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index cbb44eb0f..3cba4a7c7 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -41,7 +41,7 @@ namespace { inline std::string awsErrorMessage(const std::string& msg, const Aws::S3::S3Error& error) { std::ostringstream oss; - oss << " AWS: " << error.GetMessage() << " Remote IP: " << error.GetRemoteHostIpAddress(); + oss << msg << " AWS: " << error.GetMessage() << " Remote IP: " << error.GetRemoteHostIpAddress(); return oss.str(); } @@ -112,7 +112,7 @@ void S3ClientAWS::deleteBucket(const std::string& bucketName) const { Aws::S3::Model::DeleteBucketRequest request; request.SetBucket(bucketName); - auto outcome = client_->DeleteBucket(request); + const auto outcome = client_->DeleteBucket(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket: " << bucketName << std::endl; @@ -127,7 +127,7 @@ auto S3ClientAWS::listBuckets() const -> std::vector { auto outcome = client_->ListBuckets(); if (outcome.IsSuccess()) { - for (auto& bucket : outcome.GetResult().GetBuckets()) { result.emplace_back(bucket.GetName()); } + for (const auto& bucket : outcome.GetResult().GetBuckets()) { result.emplace_back(bucket.GetName()); } } else { Log::warning() << "Failed to list buckets!" << outcome.GetError(); } @@ -145,7 +145,6 @@ void S3ClientAWS::putObject(const std::string& bucketName, const std::string& ob request.SetKey(objectName); const std::shared_ptr inputData = Aws::MakeShared(""); - // *inputData << objectContent.c_str(); request.SetBody(inputData); auto outcome = client_->PutObject(request); @@ -205,8 +204,8 @@ auto S3ClientAWS::listObjects(const std::string& bucketName) const -> std::vecto auto outcome = client_->ListObjects(request); if (outcome.IsSuccess()) { - Aws::Vector objects = outcome.GetResult().GetContents(); - for (Aws::S3::Model::Object& object : objects) { result.emplace_back(object.GetKey()); } + const auto& objects = outcome.GetResult().GetContents(); + for (const auto& object : objects) { result.emplace_back(object.GetKey()); } } else { Log::warning() << "Failed to list objects in bucket: " << bucketName << outcome.GetError(); } diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index a544bb434..780e64486 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -32,7 +32,7 @@ class S3ClientAWS: public S3Client { public: // methods NO_COPY_NO_MOVE(S3ClientAWS) - explicit S3ClientAWS(); + S3ClientAWS(); ~S3ClientAWS(); From 4fbcea13b5074199cc0b95dafdfb6634ac127675 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 20:52:36 +0100 Subject: [PATCH 24/90] feat(S3): S3Name: parse bucket/object, exists, size --- src/eckit/io/s3/S3Name.cc | 92 +++++++++++++++------------------------ src/eckit/io/s3/S3Name.h | 43 +++++++++++------- 2 files changed, 61 insertions(+), 74 deletions(-) diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 13647d36d..1faa02cd7 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -1,93 +1,69 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ #include "eckit/io/s3/S3Name.h" #include "eckit/config/LibEcKit.h" #include "eckit/filesystem/URI.h" -#include "eckit/filesystem/URIManager.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Handle.h" +#include "eckit/utils/Tokenizer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Name::S3Name(const URI& uri): config_(uri) { - LOG_DEBUG_LIB(LibEcKit) << "URI -----> " << uri << std::endl; - LOG_DEBUG_LIB(LibEcKit) << "URI -----> host=" << uri.host() << std::endl; - LOG_DEBUG_LIB(LibEcKit) << " -----> " << *this << std::endl; +S3Name::S3Name(const URI& uri): client_(S3Client::makeUnique({uri})) { + parse(uri.name()); } -S3Name::S3Name(const S3Config& config, const std::string& bucketName, const std::string& objectName): - config_(config), bucket_(bucketName), object_(objectName) { } - S3Name::~S3Name() = default; //---------------------------------------------------------------------------------------------------------------------- -void S3Name::print(std::ostream& out) const { - out << "S3Name[config=" << config_ << ",bucket=" << bucket_ << ",object=" << object_ << "]"; -} +void S3Name::parse(const std::string& path) { + const auto pairs = Tokenizer("/").tokenize(path); + + ASSERT(pairs.size() == 2); -auto S3Name::exists() -> bool { - NOTIMP; - return false; + bucket_ = pairs[0]; + object_ = pairs[1]; } -auto S3Name::dataHandle() -> std::unique_ptr { - LOG_DEBUG_LIB(LibEcKit) << "dataHandle " << *this << std::endl; - return std::make_unique(*this); +void S3Name::print(std::ostream& out) const { + out << "S3Name[bucket=" << bucket_ << ",object=" << object_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- -// S3 URI MANAGER - -class S3URIManager: public URIManager { - bool query() override { return false; } - bool fragment() override { return false; } - bool exists(const URI& uri) override { return S3Name(uri).exists(); } - - DataHandle* newWriteHandle(const URI& uri) override { return S3Name(uri).dataHandle().release(); } - - DataHandle* newReadHandle(const URI& uri) override { return S3Name(uri).dataHandle().release(); } - - DataHandle* newReadHandle(const URI& uri, const OffsetList& ol, const LengthList& ll) override { - NOTIMP; - return S3Name(uri).dataHandle().release(); - } - - std::string asString(const URI& uri) const override { - std::string q = uri.query(); - if (!q.empty()) { q = "?" + q; } - std::string f = uri.fragment(); - if (!f.empty()) { f = "#" + f; } - - return uri.name() + q + f; - } +auto S3Name::bucketExists() const -> bool { + return client_->bucketExists(bucket_); +} -public: - S3URIManager(const std::string& name): URIManager(name) { } -}; +auto S3Name::exists() const -> bool { + return client_->objectExists(bucket_, object_); +} -//---------------------------------------------------------------------------------------------------------------------- +auto S3Name::size() const -> Length { + return client_->objectSize(bucket_, object_); +} -static S3URIManager manager_s3("s3"); +auto S3Name::dataHandle() -> DataHandle* { + return new S3Handle(*this); +} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 46e03a3bd..429fa29a3 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -1,11 +1,16 @@ /* - * (C) Copyright 1996- ECMWF. + * (C) Copyright 1996- ECMWF. * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3Name.h @@ -15,15 +20,16 @@ #pragma once -#include "eckit/io/s3/S3Config.h" -#include "eckit/io/s3/S3Macros.h" +#include "eckit/io/Length.h" +#include #include namespace eckit { class URI; class DataHandle; +class S3Client; //---------------------------------------------------------------------------------------------------------------------- @@ -31,16 +37,20 @@ class S3Name { public: // methods explicit S3Name(const URI& uri); - S3Name(const S3Config& config, const std::string& bucketName, const std::string& objectName); - ~S3Name(); - auto getConfig() const -> const S3Config& { return config_; } + auto bucket() const -> const std::string& { return bucket_; } - auto exists() -> bool; + auto object() const -> const std::string& { return object_; } - NODISCARD - auto dataHandle() const -> DataHandle*; + auto bucketExists() const -> bool; + + auto exists() const -> bool; + + auto size() const -> Length; + + [[nodiscard]] + auto dataHandle() -> DataHandle*; friend std::ostream& operator<<(std::ostream& out, const S3Name& name) { name.print(out); @@ -48,14 +58,15 @@ class S3Name { } private: // methods + void parse(const std::string& path); + void print(std::ostream& out) const; private: // members - S3Config config_; std::string bucket_; std::string object_; - // std::unique_ptr client_; + std::unique_ptr client_; }; //---------------------------------------------------------------------------------------------------------------------- From 929c0f254ba13e22c2fcad97042ea26085ecf728 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 21:12:18 +0100 Subject: [PATCH 25/90] feat(S3): add bucket/object exists, object size --- src/eckit/io/s3/S3Client.cc | 25 ++++++------ src/eckit/io/s3/S3Client.h | 41 +++++++++++--------- src/eckit/io/s3/S3Config.h | 45 ++++++++-------------- src/eckit/io/s3/aws/S3ClientAWS.cc | 62 ++++++++++++++++++++++-------- src/eckit/io/s3/aws/S3ClientAWS.h | 31 ++++++++------- 5 files changed, 113 insertions(+), 91 deletions(-) diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index cd98210c4..ceb0ced68 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ #include "eckit/io/s3/S3Client.h" @@ -24,7 +23,7 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Client::S3Client(S3ContextSPtr context): context_(context) { } +S3Client::S3Client(std::shared_ptr context): context_(context) { } S3Client::~S3Client() = default; diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 6165bb962..17c52929f 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3Client.h @@ -21,22 +20,20 @@ #pragma once #include "eckit/io/s3/S3Config.h" -#include "eckit/io/s3/S3Context.h" +#include "eckit/memory/NonCopyable.h" #include #include namespace eckit { -using S3ContextSPtr = std::shared_ptr; +class S3Context; //---------------------------------------------------------------------------------------------------------------------- -class S3Client { +class S3Client: private NonCopyable { public: // methods - NO_COPY_NO_MOVE(S3Client) - - explicit S3Client(S3ContextSPtr context); + explicit S3Client(std::shared_ptr context); virtual ~S3Client(); @@ -50,6 +47,8 @@ class S3Client { virtual void deleteBucket(const std::string& bucketName) const = 0; + virtual auto bucketExists(const std::string& bucketName) const -> bool = 0; + virtual auto listBuckets() const -> std::vector = 0; virtual void putObject(const std::string& bucketName, const std::string& objectName) const = 0; @@ -58,8 +57,12 @@ class S3Client { virtual auto listObjects(const std::string& bucketName) const -> std::vector = 0; + virtual auto objectExists(const std::string& bucketName, const std::string& objectKey) const -> bool = 0; + + virtual auto objectSize(const std::string& bucketName, const std::string& objectKey) const -> Length = 0; + private: // members - S3ContextSPtr context_; + std::shared_ptr context_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index ea066099d..7ac42cfa6 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -21,7 +21,6 @@ #pragma once #include "eckit/filesystem/URI.h" -#include "eckit/io/s3/S3Macros.h" #include @@ -31,51 +30,37 @@ enum class S3Types { NONE, AWS, REST }; //---------------------------------------------------------------------------------------------------------------------- -struct S3Config { +class S3Config { +public: // methods S3Config(const std::string& region, const std::string& hostname, const int port): - region_(region), host_(hostname), port_(port) { } - - S3Config(const URI& uri): S3Config(uri.name(), uri.host(), uri.port()) { - // host_ = uri.host(); - // port_ = uri.port(); - /// @todo need to parse region from hostname - /// example: access-point.s3-accesspoint.[AWS Region code].amazonaws.com - // region_ = uri.name(); - } + region_(region), endpoint_(hostname, port) { } + + S3Config(const std::string& region, const URI& uri): region_(region), endpoint_(uri) { } + + S3Config(const URI& uri): endpoint_(uri) { } - NODISCARD auto type() const -> S3Types { return type_; } - NODISCARD auto tag() const -> const std::string& { return tag_; } - NODISCARD auto region() const -> const std::string& { return region_; } void setRegion(const std::string& region) { region_ = region; } - NODISCARD - auto host() const -> const std::string& { return host_; } + auto endpoint() const -> const net::Endpoint& { return endpoint_; } - NODISCARD - auto port() const -> int { return port_; } - - NODISCARD - auto getURI() const -> URI { return {"s3", host_, port_}; } + auto getURI() const -> URI { return {"s3", endpoint_.host(), endpoint_.port()}; } friend std::ostream& operator<<(std::ostream& out, const S3Config& config) { - out << "S3Config[tag=" << config.tag_ << ",region=" << config.region_ << ",host=" << config.host_ - << ",port=" << config.port_ << "]"; + out << "S3Config[tag=" << config.tag_ << ",region=" << config.region_ << ",endpoint=" << config.endpoint_ << "]"; return out; } -private: - S3Types type_ {S3Types::AWS}; - std::string tag_ {"ALLOCATION_TAG"}; - std::string region_ {"ecmwf-central-1"}; - - std::string host_ {"127.0.0.1"}; - int port_ {-1}; +private: // members + S3Types type_ {S3Types::AWS}; + std::string tag_ {"ALLOCATION_TAG"}; + std::string region_ {"ecmwf-central-1"}; + net::Endpoint endpoint_ {"127.0.0.1", 8888}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 3cba4a7c7..a957c92d4 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ #include "eckit/io/s3/aws/S3ClientAWS.h" @@ -29,6 +28,8 @@ #include #include #include +#include +#include #include #include @@ -74,12 +75,14 @@ void S3ClientAWS::configure(const S3Config& config) { // configuration.verifySSL = false; // setup endpoint - if (!config.host().empty()) { configuration.endpointOverride = "http://" + config.host(); } - if (config.port() > 0) { configuration.endpointOverride += ":" + std::to_string(config.port()); } + /// @todo handle http/https possibly via scheme flag in S3Config + const auto& endpoint = config.endpoint(); + if (!endpoint.host().empty()) { configuration.endpointOverride = "http://" + endpoint.host(); } + if (endpoint.port() > 0) { configuration.endpointOverride += ":" + std::to_string(endpoint.port()); } // setup credentials Aws::Auth::AWSCredentials credentials; - if (auto cred = S3Session::instance().getCredentials(config.host())) { + if (auto cred = S3Session::instance().getCredentials(endpoint.host())) { credentials.SetAWSAccessKeyId(cred->keyID); credentials.SetAWSSecretKey(cred->secret); } @@ -122,6 +125,12 @@ void S3ClientAWS::deleteBucket(const std::string& bucketName) const { } } +auto S3ClientAWS::bucketExists(const std::string& bucketName) const -> bool { + Aws::S3::Model::HeadBucketRequest request; + request.SetBucket(bucketName); + return client_->HeadBucket(request).IsSuccess(); +} + auto S3ClientAWS::listBuckets() const -> std::vector { std::vector result; @@ -213,6 +222,29 @@ auto S3ClientAWS::listObjects(const std::string& bucketName) const -> std::vecto return result; } +auto S3ClientAWS::objectExists(const std::string& bucketName, const std::string& objectKey) const -> bool { + Aws::S3::Model::HeadObjectRequest request; + request.WithKey(objectKey).WithBucket(bucketName); + return client_->HeadObject(request).IsSuccess(); +} + +auto S3ClientAWS::objectSize(const std::string& bucketName, const std::string& objectKey) const -> Length { + Length result; + + Aws::S3::Model::HeadObjectRequest request; + request.WithKey(objectKey).WithBucket(bucketName); + + const auto outcome = client_->HeadObject(request); + if (outcome.IsSuccess()) { + result = outcome.GetResult().GetContentLength(); + } else { + const auto msg = awsErrorMessage("Object '" + objectKey + "' doesn't exist or no access!", outcome.GetError()); + throw S3SeriousBug(msg, Here()); + } + + return result; +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 780e64486..dd265af15 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3ClientAWS.h @@ -30,8 +29,6 @@ namespace eckit { class S3ClientAWS: public S3Client { public: // methods - NO_COPY_NO_MOVE(S3ClientAWS) - S3ClientAWS(); ~S3ClientAWS(); @@ -42,6 +39,8 @@ class S3ClientAWS: public S3Client { void deleteBucket(const std::string& bucketName) const override; + auto bucketExists(const std::string& bucketName) const -> bool override; + auto listBuckets() const -> std::vector override; void putObject(const std::string& bucketName, const std::string& objectName) const override; @@ -52,6 +51,10 @@ class S3ClientAWS: public S3Client { auto listObjects(const std::string& bucketName) const -> std::vector override; + auto objectExists(const std::string& bucketName, const std::string& objectKey) const -> bool override; + + auto objectSize(const std::string& bucketName, const std::string& objectKey) const -> Length override; + private: // members std::unique_ptr client_; }; From 611742a7fcff45152e5c45430b4f5f380878c93d Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 21:15:25 +0100 Subject: [PATCH 26/90] refactor(S3): session cleanup --- src/eckit/io/s3/S3Session.cc | 7 ++----- src/eckit/io/s3/S3Session.h | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index 7a769d6b3..d4bbb6dbb 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -61,11 +61,9 @@ S3Session::~S3Session() = default; // CREDENTIALS auto S3Session::getCredentials(const std::string& endpoint) const -> std::shared_ptr { - LOG_DEBUG_LIB(LibEcKit) << "Searching credential for endpoint:" << endpoint << std::endl; - /// @todo check if all keyid and secret different // search by endpoint - auto cred = std::find_if(credentials_.begin(), credentials_.end(), IsCredentialEndpoint(endpoint)); + const auto cred = std::find_if(credentials_.begin(), credentials_.end(), IsCredentialEndpoint(endpoint)); // found if (cred != credentials_.end()) { return *cred; } // not found @@ -99,9 +97,8 @@ auto S3Session::getContext(const S3Types type) -> std::shared_ptr { } auto S3Session::findContext(const S3Types type) -> std::shared_ptr { - LOG_DEBUG_LIB(LibEcKit) << "Find context!" << std::endl; // search by type - auto context = std::find_if(contexts_.begin(), contexts_.end(), IsContextType({type})); + const auto context = std::find_if(contexts_.begin(), contexts_.end(), IsContextType({type})); // found if (context != contexts_.end()) { return *context; } // not found diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h index 45048d53f..2d52e5527 100644 --- a/src/eckit/io/s3/S3Session.h +++ b/src/eckit/io/s3/S3Session.h @@ -20,7 +20,7 @@ #pragma once -#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Context.h" #include "eckit/io/s3/S3Credential.h" #include From 894bbae49530eb4180fe2d5b9e7a42348feb2982 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 21:17:03 +0100 Subject: [PATCH 27/90] feat(S3): adds open and size to S3Handle --- src/eckit/io/s3/S3Handle.cc | 57 ++++++++++++++----------------------- src/eckit/io/s3/S3Handle.h | 29 ++++++++++--------- 2 files changed, 36 insertions(+), 50 deletions(-) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 8e0092c49..8bbb68c1a 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ #include "eckit/io/s3/S3Handle.h" @@ -40,34 +39,25 @@ S3Handle::~S3Handle() { } void S3Handle::print(std::ostream& out) const { - // out << "S3Handle[" << impl_ << "]"; + out << "S3Handle[name=" << name_ << ",position=" << pos_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- Length S3Handle::openForRead() { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; - - // canWrite_ = false; - // object_ = name_.object(); - // if (object_->open()) { return object_->size(); } - - return {}; + LOG_DEBUG_LIB(LibEcKit) << "S3Handle::openForRead" << std::endl; + canWrite_ = false; + return name_.size(); // asserts } void S3Handle::openForWrite(const Length& length) { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; - // canWrite_ = true; - // object_ = name_.object(); - // if (!object_->create(name_.pool())) { throw NotFound("Cannot create object!", -1, Here()); } + LOG_DEBUG_LIB(LibEcKit) << "S3Handle::openForWrite" << std::endl; + canWrite_ = true; } void S3Handle::openForAppend(const Length& length) { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; - // NOTIMP; + LOG_DEBUG_LIB(LibEcKit) << "S3Handle::openForAppend" << std::endl; + canWrite_ = true; } //---------------------------------------------------------------------------------------------------------------------- @@ -99,16 +89,11 @@ void S3Handle::close() { } Length S3Handle::size() { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; - return {}; + return name_.size(); // asserts } Offset S3Handle::position() { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; - // return pos_; - return {}; + return pos_; } Offset S3Handle::seek(const Offset& offset) { diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index 4aee755af..2110dce8f 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3Handle.h @@ -66,9 +65,11 @@ class S3Handle: public DataHandle { bool canSeek() const override { return true; } private: // members - S3Name name_; + const S3Name& name_; + Offset pos_ {0}; - // bool canWrite_ {false}; + + bool canWrite_ {false}; }; //---------------------------------------------------------------------------------------------------------------------- From 6a95ecbd6d36b010b51ce8c07db926d1e667b760 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 21:19:53 +0100 Subject: [PATCH 28/90] fix(S3): adapt s3handle test --- tests/io/test_s3client.cc | 23 ++++++++--------------- tests/io/test_s3handle.cc | 35 +++++++++++++++-------------------- 2 files changed, 23 insertions(+), 35 deletions(-) diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index 30dbab372..fd50e8637 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -1,21 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. */ /// @file test_s3client.cc /// @author Metin Cakircali +/// @author Simon Smart /// @date Jan 2024 #include "eckit/config/LibEcKit.h" @@ -24,8 +19,6 @@ #include "eckit/io/s3/S3Session.h" #include "eckit/testing/Test.h" -#include - using namespace std; using namespace eckit; using namespace eckit::testing; @@ -118,7 +111,7 @@ CASE("list buckets") { } // namespace eckit::test int main(int argc, char** argv) { - S3Credential cred {"minio", "minio1234", "127.0.0.1"}; + const S3Credential cred {"minio", "minio1234", "127.0.0.1"}; S3Session::instance().addCredentials(cred); auto ret = run_tests(argc, argv); diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 52dc67d1b..54b86a7d9 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -1,25 +1,21 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. */ /// @file test_s3handle.cc /// @author Metin Cakircali +/// @author Simon Smart /// @date Jan 2024 #include "eckit/filesystem/URI.h" #include "eckit/io/MemoryHandle.h" +#include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Handle.h" #include "eckit/io/s3/S3Session.h" #include "eckit/testing/Test.h" @@ -32,7 +28,7 @@ using namespace eckit::testing; namespace eckit::test { -S3Config cfg {S3Types::AWS, "test-tag", "eu-central-1", "127.0.0.1", 9000}; +S3Config cfg("eu-central-1", "127.0.0.1", 9000); static const std::string TEST_BUCKET("test-bucket"); @@ -59,18 +55,17 @@ void bucketSetup() { CASE("S3Handle") { const char buf[] = "abcdefghijklmnopqrstuvwxyz"; - S3Config config("eu-central-1", "127.0.0.1", 9000); - - S3Name name(config, "myS3bucket", "myS3object"); + URI uri("s3://127.0.0.1:9000/test-bucket/test-object"); { - std::unique_ptr h{name.dataHandle()}; + S3Name name(uri); + + std::unique_ptr h {name.dataHandle()}; h->openForWrite(sizeof(buf)); AutoClose closer(*h); EXPECT(h->write(buf, sizeof(buf)) == sizeof(buf)); } - URI uri("s3://127.0.0.1:9000/bucket/object"); { auto h = uri.newReadHandle(); h->openForRead(); @@ -89,14 +84,14 @@ CASE("S3Handle") { } } -// TODO: Also check that it doesn't work if the bucket doesn't exist, etc. +/// @todo Also check that it doesn't work if the bucket doesn't exist, etc. //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { - S3Credential cred {"minio", "minio1234", "127.0.0.1"}; + const S3Credential cred {"minio", "minio1234", "127.0.0.1"}; S3Session::instance().addCredentials(cred); int ret = -1; From 140d75d722329e770c3307c0f5d0677f853706cb Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 9 Feb 2024 21:36:01 +0100 Subject: [PATCH 29/90] refactor(S3): cleanup and copyright fix --- src/eckit/CMakeLists.txt | 2 - src/eckit/io/s3/S3Config.h | 23 +++---- src/eckit/io/s3/S3Context.cc | 23 +++---- src/eckit/io/s3/S3Context.h | 23 +++---- src/eckit/io/s3/S3Credential.h | 23 +++---- src/eckit/io/s3/S3Exception.cc | 25 +++---- src/eckit/io/s3/S3Exception.h | 23 +++---- src/eckit/io/s3/S3Object.cc | 77 --------------------- src/eckit/io/s3/S3Object.h | 100 ---------------------------- src/eckit/io/s3/S3Session.cc | 23 +++---- src/eckit/io/s3/S3Session.h | 23 +++---- src/eckit/io/s3/S3URIManager.cc | 5 ++ src/eckit/io/s3/aws/S3ContextAWS.cc | 23 +++---- src/eckit/io/s3/aws/S3ContextAWS.h | 23 +++---- 14 files changed, 115 insertions(+), 301 deletions(-) delete mode 100644 src/eckit/io/s3/S3Object.cc delete mode 100644 src/eckit/io/s3/S3Object.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 8b3c22916..c689de0ff 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -304,8 +304,6 @@ io/s3/S3Handle.cc io/s3/S3Handle.h io/s3/S3Name.cc io/s3/S3Name.h -io/s3/S3Object.cc -io/s3/S3Object.h io/s3/S3Session.cc io/s3/S3Session.h io/s3/S3Macros.h diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 7ac42cfa6..27cd38f9e 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3Config.h diff --git a/src/eckit/io/s3/S3Context.cc b/src/eckit/io/s3/S3Context.cc index 441eabf84..69b7fd30e 100644 --- a/src/eckit/io/s3/S3Context.cc +++ b/src/eckit/io/s3/S3Context.cc @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ #include "eckit/io/s3/S3Context.h" diff --git a/src/eckit/io/s3/S3Context.h b/src/eckit/io/s3/S3Context.h index 916fc5016..a16788f12 100644 --- a/src/eckit/io/s3/S3Context.h +++ b/src/eckit/io/s3/S3Context.h @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3Context.h diff --git a/src/eckit/io/s3/S3Credential.h b/src/eckit/io/s3/S3Credential.h index 3b7cab3a7..06fea910a 100644 --- a/src/eckit/io/s3/S3Credential.h +++ b/src/eckit/io/s3/S3Credential.h @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3Credential.h diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc index e571a48ef..a09bb8728 100644 --- a/src/eckit/io/s3/S3Exception.cc +++ b/src/eckit/io/s3/S3Exception.cc @@ -1,25 +1,22 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3Exception.h /// @author Metin Cakircali /// @date Jan 2024 -#pragma once - #include "eckit/io/s3/S3Exception.h" #include diff --git a/src/eckit/io/s3/S3Exception.h b/src/eckit/io/s3/S3Exception.h index 6ccd7cde1..d53b6272b 100644 --- a/src/eckit/io/s3/S3Exception.h +++ b/src/eckit/io/s3/S3Exception.h @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3Exception.h diff --git a/src/eckit/io/s3/S3Object.cc b/src/eckit/io/s3/S3Object.cc deleted file mode 100644 index 43743cfbc..000000000 --- a/src/eckit/io/s3/S3Object.cc +++ /dev/null @@ -1,77 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#include "eckit/io/s3/S3Object.h" - -#include "eckit/io/s3/S3Exception.h" - -namespace eckit { - -//---------------------------------------------------------------------------------------------------------------------- - -class S3Object::S3ObjectInternal { -public: - // S3ObjectInternal(const std::string& name): name_(name) { } - - S3ObjectInternal() = default; - ~S3ObjectInternal() = default; - - // private: - // const std::string& name_; -}; - -//---------------------------------------------------------------------------------------------------------------------- - -S3Object::S3Object(const std::string& name): S3ObjectBase(name), impl_(std::make_unique()) { } - -S3Object::~S3Object() { } - -//---------------------------------------------------------------------------------------------------------------------- - -bool S3Object::create(const std::string& name) { - NOTIMP; - return false; -} - -bool S3Object::remove() { - NOTIMP; - return false; -} - -bool S3Object::open() { - NOTIMP; - return false; -} - -long S3Object::read(void* buffer, long length, off_t offset) const { - NOTIMP; - return 0; -} - -long S3Object::write(const void* buffer, long length, off_t offset) const { - NOTIMP; - return 0; -} - -bool S3Object::close() { - NOTIMP; - return false; -} - -long S3Object::size() { - NOTIMP; - return 0; -} - -void S3Object::print(std::ostream& out) const { } - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace eckit diff --git a/src/eckit/io/s3/S3Object.h b/src/eckit/io/s3/S3Object.h deleted file mode 100644 index ce20ad639..000000000 --- a/src/eckit/io/s3/S3Object.h +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/// @file S3Object.h -/// @author Metin Cakircali -/// @date Jan 2024 - -#pragma once - -#include "eckit/io/s3/S3Macros.h" - -#include -#include - -namespace eckit { - -//---------------------------------------------------------------------------------------------------------------------- - -class S3ObjectBase { -public: // methods - NO_COPY_NO_MOVE(S3ObjectBase) - - explicit S3ObjectBase(const std::string& name): name_(name) { } - - virtual ~S3ObjectBase() = default; - - friend std::ostream& operator<<(std::ostream& out, const S3ObjectBase& obj) { - obj.print(out); - return out; - } - - virtual bool create(const std::string& name) = 0; - - virtual bool remove() = 0; - - virtual bool open() = 0; - - virtual long read(void* buffer, long length, off_t offset) const = 0; - - virtual long write(const void* buffer, long length, off_t offset) const = 0; - - virtual bool close() = 0; - -private: // methods - virtual void print(std::ostream& out) const = 0; - -private: // members - std::string name_; - - bool isOpen_ {false}; -}; - -//---------------------------------------------------------------------------------------------------------------------- - -class S3Object: public S3ObjectBase { -public: // methods - NO_COPY_NO_MOVE(S3Object) - - S3Object(const std::string& name); - - ~S3Object() override; - - bool create(const std::string& name) override; - - bool remove() override; - - bool open() override; - - long read(void* buffer, long length, off_t offset) const override; - - long write(const void* buffer, long length, off_t offset) const override; - - bool close() override; - - long size(); - -private: // methods - void print(std::ostream& out) const override; - -private: // members - class S3ObjectInternal; - std::unique_ptr impl_; -}; - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace eckit diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index d4bbb6dbb..e743546c6 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ #include "eckit/io/s3/S3Session.h" diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h index 2d52e5527..31232e8ca 100644 --- a/src/eckit/io/s3/S3Session.h +++ b/src/eckit/io/s3/S3Session.h @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3Session.h diff --git a/src/eckit/io/s3/S3URIManager.cc b/src/eckit/io/s3/S3URIManager.cc index 0558078dc..ac9c67207 100644 --- a/src/eckit/io/s3/S3URIManager.cc +++ b/src/eckit/io/s3/S3URIManager.cc @@ -8,6 +8,11 @@ * does it submit to any jurisdiction. */ +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3URIManager.h" #include "eckit/io/s3/S3Name.h" diff --git a/src/eckit/io/s3/aws/S3ContextAWS.cc b/src/eckit/io/s3/aws/S3ContextAWS.cc index a4318fe82..b49303fac 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.cc +++ b/src/eckit/io/s3/aws/S3ContextAWS.cc @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ #include "eckit/io/s3/aws/S3ContextAWS.h" diff --git a/src/eckit/io/s3/aws/S3ContextAWS.h b/src/eckit/io/s3/aws/S3ContextAWS.h index c6ccc3f5e..7f760c936 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.h +++ b/src/eckit/io/s3/aws/S3ContextAWS.h @@ -1,17 +1,16 @@ /* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). + * (C) Copyright 1996- ECMWF. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu */ /// @file S3ContextAWS.h From e63bd18032dbb87f1092416b044422d907ea77f4 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sat, 10 Feb 2024 20:56:23 +0100 Subject: [PATCH 30/90] refactor(S3): client and config cleanup --- src/eckit/io/s3/S3Config.h | 3 ++- src/eckit/io/s3/S3Exception.cc | 4 ---- src/eckit/io/s3/aws/S3ClientAWS.cc | 35 +++++++++++++++++------------- tests/io/test_s3handle.cc | 17 +++++++++------ 4 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 27cd38f9e..7d624e787 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -31,6 +31,7 @@ enum class S3Types { NONE, AWS, REST }; class S3Config { public: // methods + /// @todo region is part of hostname (s3express-control.region_code.amazonaws.com/bucket-name) S3Config(const std::string& region, const std::string& hostname, const int port): region_(region), endpoint_(hostname, port) { } @@ -58,7 +59,7 @@ class S3Config { private: // members S3Types type_ {S3Types::AWS}; std::string tag_ {"ALLOCATION_TAG"}; - std::string region_ {"ecmwf-central-1"}; + std::string region_ {"eu-central-1"}; net::Endpoint endpoint_ {"127.0.0.1", 8888}; }; diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc index a09bb8728..0645ab2ac 100644 --- a/src/eckit/io/s3/S3Exception.cc +++ b/src/eckit/io/s3/S3Exception.cc @@ -13,10 +13,6 @@ * (Project ID: 955811) iosea-project.eu */ -/// @file S3Exception.h -/// @author Metin Cakircali -/// @date Jan 2024 - #include "eckit/io/s3/S3Exception.h" #include diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index a957c92d4..dfb524bb4 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -50,6 +50,8 @@ inline std::string awsErrorMessage(const std::string& msg, const Aws::S3::S3Erro namespace eckit { +const char* ALLOC_TAG = "S3ClientAWS"; + //---------------------------------------------------------------------------------------------------------------------- S3ClientAWS::S3ClientAWS(): @@ -104,9 +106,9 @@ void S3ClientAWS::createBucket(const std::string& bucketName) const { auto outcome = client_->CreateBucket(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Created bucket: " << bucketName << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucketName << std::endl; } else { - auto msg = awsErrorMessage("Failed to create bucket: " + bucketName, outcome.GetError()); + auto msg = awsErrorMessage("Failed to create bucket=" + bucketName, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } @@ -118,9 +120,9 @@ void S3ClientAWS::deleteBucket(const std::string& bucketName) const { const auto outcome = client_->DeleteBucket(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket: " << bucketName << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucketName << std::endl; } else { - auto msg = awsErrorMessage("Failed to delete bucket: " + bucketName, outcome.GetError()); + auto msg = awsErrorMessage("Failed to delete bucket=" + bucketName, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } @@ -135,6 +137,7 @@ auto S3ClientAWS::listBuckets() const -> std::vector { std::vector result; auto outcome = client_->ListBuckets(); + if (outcome.IsSuccess()) { for (const auto& bucket : outcome.GetResult().GetBuckets()) { result.emplace_back(bucket.GetName()); } } else { @@ -153,15 +156,15 @@ void S3ClientAWS::putObject(const std::string& bucketName, const std::string& ob request.SetBucket(bucketName); request.SetKey(objectName); - const std::shared_ptr inputData = Aws::MakeShared(""); - request.SetBody(inputData); + // empty object + request.SetBody(Aws::MakeShared(ALLOC_TAG)); auto outcome = client_->PutObject(request); + if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Added object '" << objectName << "' to bucket '" << bucketName << "'."; + LOG_DEBUG_LIB(LibEcKit) << "Added object=" << objectName << " to bucket=" << bucketName << std::endl; } else { - auto msg = - awsErrorMessage("Failed to put object: " + objectName + " to bucket: " + bucketName, outcome.GetError()); + auto msg = awsErrorMessage("Failed to put object=" + objectName + " to bucket=" + bucketName, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } @@ -172,11 +175,12 @@ void S3ClientAWS::deleteObject(const std::string& bucketName, const std::string& request.WithKey(objectKey).WithBucket(bucketName); auto outcome = client_->DeleteObject(request); + if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Successfully deleted the object." << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << objectKey << " in bucket=" << bucketName << std::endl; } else { auto msg = - awsErrorMessage("Failed to delete object: " + objectKey + " in bucket: " + bucketName, outcome.GetError()); + awsErrorMessage("Failed to delete object=" + objectKey + " in bucket=" + bucketName, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } @@ -193,13 +197,12 @@ void S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector request.SetBucket(bucketName); auto outcome = client_->DeleteObjects(request); + if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Deleted " << objectKeys.size() << " objects from bucket " << bucketName << std::endl; - for (auto i = 0; i < objectKeys.size(); ++i) { - LOG_DEBUG_LIB(LibEcKit) << "Deleted object: " << objectKeys[i] << std::endl; - } + for (const auto& object : objectKeys) { LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << std::endl; } } else { - auto msg = awsErrorMessage("Failed to delete objects in bucket: " + bucketName, outcome.GetError()); + auto msg = awsErrorMessage("Failed to delete objects in bucket=" + bucketName, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } @@ -212,6 +215,7 @@ auto S3ClientAWS::listObjects(const std::string& bucketName) const -> std::vecto request.WithBucket(bucketName); auto outcome = client_->ListObjects(request); + if (outcome.IsSuccess()) { const auto& objects = outcome.GetResult().GetContents(); for (const auto& object : objects) { result.emplace_back(object.GetKey()); } @@ -235,6 +239,7 @@ auto S3ClientAWS::objectSize(const std::string& bucketName, const std::string& o request.WithKey(objectKey).WithBucket(bucketName); const auto outcome = client_->HeadObject(request); + if (outcome.IsSuccess()) { result = outcome.GetResult().GetContentLength(); } else { diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 54b86a7d9..b8aeb39c3 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -30,7 +30,7 @@ namespace eckit::test { S3Config cfg("eu-central-1", "127.0.0.1", 9000); -static const std::string TEST_BUCKET("test-bucket"); +static const std::string TEST_BUCKET("eckit-s3handle-test-bucket"); //---------------------------------------------------------------------------------------------------------------------- @@ -53,7 +53,10 @@ void bucketSetup() { } CASE("S3Handle") { - const char buf[] = "abcdefghijklmnopqrstuvwxyz"; + const std::string_view testData = "abcdefghijklmnopqrstuvwxyz"; + + const void* buffer = testData.data(); + const auto length = testData.size(); URI uri("s3://127.0.0.1:9000/test-bucket/test-object"); @@ -61,16 +64,16 @@ CASE("S3Handle") { S3Name name(uri); std::unique_ptr h {name.dataHandle()}; - h->openForWrite(sizeof(buf)); + h->openForWrite(length); AutoClose closer(*h); - EXPECT(h->write(buf, sizeof(buf)) == sizeof(buf)); + EXPECT(h->write(buffer, length) == length); } { auto h = uri.newReadHandle(); h->openForRead(); std::string rbuf; - h->read(rbuf.data(), sizeof(buf)); + h->read(rbuf.data(), length); std::cout << rbuf << std::endl; } @@ -79,8 +82,8 @@ CASE("S3Handle") { MemoryHandle mh; dh->saveInto(mh); - EXPECT(mh.size() == Length(sizeof(buf))); - EXPECT(::memcmp(mh.data(), buf, sizeof(buf)) == 0); + EXPECT(mh.size() == Length(length)); + EXPECT(::memcmp(mh.data(), buffer, length) == 0); } } From 75bb909615acd6fa14cd6991d4d145d0418bee80 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sat, 10 Feb 2024 20:57:27 +0100 Subject: [PATCH 31/90] feat(S3): adds putObject --- src/eckit/io/s3/S3Client.h | 3 +++ src/eckit/io/s3/S3Handle.cc | 9 +++++---- src/eckit/io/s3/S3Name.cc | 4 ++++ src/eckit/io/s3/S3Name.h | 2 ++ src/eckit/io/s3/aws/S3ClientAWS.cc | 23 +++++++++++++++++++++++ src/eckit/io/s3/aws/S3ClientAWS.h | 3 +++ 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 17c52929f..1a725ab78 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -51,6 +51,9 @@ class S3Client: private NonCopyable { virtual auto listBuckets() const -> std::vector = 0; + virtual void putObject(const std::string& bucketName, const std::string& objectName, const void* buffer, + const uint64_t length) const = 0; + virtual void putObject(const std::string& bucketName, const std::string& objectName) const = 0; virtual void deleteObject(const std::string& bucketName, const std::string& objectKey) const = 0; diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 8bbb68c1a..5510f5704 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -50,7 +50,7 @@ Length S3Handle::openForRead() { return name_.size(); // asserts } -void S3Handle::openForWrite(const Length& length) { +void S3Handle::openForWrite(const Length&) { LOG_DEBUG_LIB(LibEcKit) << "S3Handle::openForWrite" << std::endl; canWrite_ = true; } @@ -72,8 +72,9 @@ long S3Handle::read(void* buffer, const long length) { long S3Handle::write(const void* buffer, const long length) { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; - // if (object_ && canWrite_) { return object_->write(buffer, length, pos_); } - return 0; + name_.put(buffer, length); + pos_ += length; + return length; } void S3Handle::flush() { @@ -82,7 +83,7 @@ void S3Handle::flush() { } void S3Handle::close() { - // canWrite_ = false; + canWrite_ = false; // object_.reset(); /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 1faa02cd7..a1d24f041 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -49,6 +49,10 @@ void S3Name::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- +void S3Name::put(const void* buffer, const uint64_t length) const { + client_->putObject(bucket_, object_, buffer, length); +} + auto S3Name::bucketExists() const -> bool { return client_->bucketExists(bucket_); } diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 429fa29a3..58aa4c5eb 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -39,6 +39,8 @@ class S3Name { ~S3Name(); + void put(const void* buffer, uint64_t length) const; + auto bucket() const -> const std::string& { return bucket_; } auto object() const -> const std::string& { return object_; } diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index dfb524bb4..023f302d6 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -22,6 +22,7 @@ #include "eckit/io/s3/aws/S3ContextAWS.h" #include +#include #include #include #include @@ -150,6 +151,28 @@ auto S3ClientAWS::listBuckets() const -> std::vector { //---------------------------------------------------------------------------------------------------------------------- // OBJECT +void S3ClientAWS::putObject(const std::string& bucketName, const std::string& objectName, const void* buffer, + const uint64_t length) const { + Aws::S3::Model::PutObjectRequest request; + + request.SetBucket(bucketName); + request.SetKey(objectName); + request.SetContentLength(length); + + auto sBuffer = Aws::New(ALLOC_TAG, (unsigned char*)buffer, length); + auto sReader = Aws::MakeShared(ALLOC_TAG, sBuffer); + request.SetBody(sReader); + + auto outcome = client_->PutObject(request); + + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Added object=" << objectName << " to bucket=" << bucketName << std::endl; + } else { + auto msg = awsErrorMessage("Failed to put object=" + objectName + " to bucket=" + bucketName, outcome.GetError()); + throw S3SeriousBug(msg, Here()); + } +} + void S3ClientAWS::putObject(const std::string& bucketName, const std::string& objectName) const { Aws::S3::Model::PutObjectRequest request; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index dd265af15..3040c2c40 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -45,6 +45,9 @@ class S3ClientAWS: public S3Client { void putObject(const std::string& bucketName, const std::string& objectName) const override; + void putObject(const std::string& bucketName, const std::string& objectName, const void* buffer, + const uint64_t length) const override; + void deleteObject(const std::string& bucketName, const std::string& objectKey) const override; void deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const; From fc359d9adef776443a5bff480c6ea45b80dd5849 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sat, 10 Feb 2024 21:27:57 +0100 Subject: [PATCH 32/90] feat(S3): adds emptyBucket --- src/eckit/io/s3/S3Client.h | 2 ++ src/eckit/io/s3/aws/S3ClientAWS.cc | 13 ++++++++++++- src/eckit/io/s3/aws/S3ClientAWS.h | 2 ++ tests/io/test_s3handle.cc | 3 ++- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 1a725ab78..d97232536 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -45,6 +45,8 @@ class S3Client: private NonCopyable { virtual void createBucket(const std::string& bucketName) const = 0; + virtual void emptyBucket(const std::string& bucketName) const = 0; + virtual void deleteBucket(const std::string& bucketName) const = 0; virtual auto bucketExists(const std::string& bucketName) const -> bool = 0; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 023f302d6..1f94f5630 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -114,6 +114,10 @@ void S3ClientAWS::createBucket(const std::string& bucketName) const { } } +void S3ClientAWS::emptyBucket(const std::string& bucketName) const { + deleteObjects(bucketName, listObjects(bucketName)); +} + void S3ClientAWS::deleteBucket(const std::string& bucketName) const { Aws::S3::Model::DeleteBucketRequest request; request.SetBucket(bucketName); @@ -209,15 +213,22 @@ void S3ClientAWS::deleteObject(const std::string& bucketName, const std::string& } void S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const { + if (objectKeys.empty()) { + LOG_DEBUG_LIB(LibEcKit) << "No objects to delete in bucket=" << bucketName << std::endl; + return; + } + Aws::S3::Model::DeleteObjectsRequest request; + request.SetBucket(bucketName); + Aws::S3::Model::Delete deleteObject; for (const auto& objectKey : objectKeys) { deleteObject.AddObjects(Aws::S3::Model::ObjectIdentifier().WithKey(objectKey)); } + // deleteObject.SetQuiet(true); request.SetDelete(deleteObject); - request.SetBucket(bucketName); auto outcome = client_->DeleteObjects(request); diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 3040c2c40..b2e322cf8 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -37,6 +37,8 @@ class S3ClientAWS: public S3Client { void createBucket(const std::string& bucketName) const override; + void emptyBucket(const std::string& bucketName) const override; + void deleteBucket(const std::string& bucketName) const override; auto bucketExists(const std::string& bucketName) const -> bool override; diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index b8aeb39c3..53c5a5e84 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -41,6 +41,7 @@ void ensureClean() { for (const std::string& name : {TEST_BUCKET}) { if (buckets.find(name) != buckets.end()) { + client->emptyBucket(name); client->deleteBucket(name); } } @@ -58,7 +59,7 @@ CASE("S3Handle") { const void* buffer = testData.data(); const auto length = testData.size(); - URI uri("s3://127.0.0.1:9000/test-bucket/test-object"); + URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/test-object"); { S3Name name(uri); From eb09b131d57ab5848e04c2bd03ac4be8088a3469 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sun, 11 Feb 2024 09:09:32 +0100 Subject: [PATCH 33/90] refactor(S3): cleanup S3Client and S3Handle --- src/eckit/io/s3/S3Client.h | 23 ++++--- src/eckit/io/s3/S3Handle.cc | 42 +++--------- src/eckit/io/s3/S3Handle.h | 5 +- src/eckit/io/s3/aws/S3ClientAWS.cc | 105 ++++++++++++++--------------- src/eckit/io/s3/aws/S3ClientAWS.h | 22 +++--- 5 files changed, 87 insertions(+), 110 deletions(-) diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index d97232536..93b41e3f6 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -43,28 +43,31 @@ class S3Client: private NonCopyable { virtual void configure(const S3Config& config) = 0; - virtual void createBucket(const std::string& bucketName) const = 0; + virtual void createBucket(const std::string& bucket) const = 0; - virtual void emptyBucket(const std::string& bucketName) const = 0; + virtual void emptyBucket(const std::string& bucket) const = 0; - virtual void deleteBucket(const std::string& bucketName) const = 0; + virtual void deleteBucket(const std::string& bucket) const = 0; - virtual auto bucketExists(const std::string& bucketName) const -> bool = 0; + virtual auto bucketExists(const std::string& bucket) const -> bool = 0; virtual auto listBuckets() const -> std::vector = 0; - virtual void putObject(const std::string& bucketName, const std::string& objectName, const void* buffer, + virtual void putObject(const std::string& bucket, const std::string& object, const void* buffer, const uint64_t length) const = 0; - virtual void putObject(const std::string& bucketName, const std::string& objectName) const = 0; + virtual void putObject(const std::string& bucket, const std::string& object) const = 0; - virtual void deleteObject(const std::string& bucketName, const std::string& objectKey) const = 0; - virtual auto listObjects(const std::string& bucketName) const -> std::vector = 0; + virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0; - virtual auto objectExists(const std::string& bucketName, const std::string& objectKey) const -> bool = 0; + virtual void deleteObjects(const std::string& bucket, const std::vector& objects) const = 0; - virtual auto objectSize(const std::string& bucketName, const std::string& objectKey) const -> Length = 0; + virtual auto listObjects(const std::string& bucket) const -> std::vector = 0; + + virtual auto objectExists(const std::string& bucket, const std::string& object) const -> bool = 0; + + virtual auto objectSize(const std::string& bucket, const std::string& object) const -> Length = 0; private: // members std::shared_ptr context_; diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 5510f5704..6f61abe19 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -17,26 +17,16 @@ #include "eckit/config/LibEcKit.h" #include "eckit/io/s3/S3Exception.h" -#include "eckit/io/s3/S3Name.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Handle::S3Handle(const S3Name& name): S3Handle(name, 0) { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; -} +S3Handle::S3Handle(const S3Name& name): name_(name) { } -S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; -} +S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { } -S3Handle::~S3Handle() { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; -} +S3Handle::~S3Handle() = default; void S3Handle::print(std::ostream& out) const { out << "S3Handle[name=" << name_ << ",position=" << pos_ << "]"; @@ -45,18 +35,16 @@ void S3Handle::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- Length S3Handle::openForRead() { - LOG_DEBUG_LIB(LibEcKit) << "S3Handle::openForRead" << std::endl; canWrite_ = false; return name_.size(); // asserts } void S3Handle::openForWrite(const Length&) { - LOG_DEBUG_LIB(LibEcKit) << "S3Handle::openForWrite" << std::endl; canWrite_ = true; } -void S3Handle::openForAppend(const Length& length) { - LOG_DEBUG_LIB(LibEcKit) << "S3Handle::openForAppend" << std::endl; +void S3Handle::openForAppend(const Length&) { + NOTIMP; canWrite_ = true; } @@ -66,27 +54,23 @@ long S3Handle::read(void* buffer, const long length) { /// @todo remove LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; return 0; - // return object_ ? object_->read(buffer, length, pos_) : 0; } long S3Handle::write(const void* buffer, const long length) { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; name_.put(buffer, length); pos_ += length; return length; } void S3Handle::flush() { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; + NOTIMP; } void S3Handle::close() { + /// @todo remove + LOG_DEBUG_LIB(LibEcKit) << "close!" << std::endl; canWrite_ = false; // object_.reset(); - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; } Length S3Handle::size() { @@ -98,17 +82,11 @@ Offset S3Handle::position() { } Offset S3Handle::seek(const Offset& offset) { - // pos_ = pos_ + offset; - // return pos_; - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; - return {}; + pos_ = pos_ + offset; + return pos_; } std::string S3Handle::title() const { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; - // NOTIMP; return {}; } diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index 2110dce8f..d8dcacfd1 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -20,7 +20,6 @@ #pragma once #include "eckit/io/DataHandle.h" -#include "eckit/io/s3/S3Macros.h" #include "eckit/io/s3/S3Name.h" namespace eckit { @@ -29,11 +28,9 @@ namespace eckit { class S3Handle: public DataHandle { public: // methods - NO_COPY_NO_MOVE(S3Handle) - explicit S3Handle(const S3Name& name); - explicit S3Handle(const S3Name& name, const Offset& offset); + S3Handle(const S3Name& name, const Offset& offset); ~S3Handle() override; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 1f94f5630..2902918ec 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -51,7 +51,7 @@ inline std::string awsErrorMessage(const std::string& msg, const Aws::S3::S3Erro namespace eckit { -const char* ALLOC_TAG = "S3ClientAWS"; +const auto ALLOC_TAG = "S3ClientAWS"; //---------------------------------------------------------------------------------------------------------------------- @@ -100,88 +100,88 @@ void S3ClientAWS::configure(const S3Config& config) { //---------------------------------------------------------------------------------------------------------------------- // BUCKET -void S3ClientAWS::createBucket(const std::string& bucketName) const { +void S3ClientAWS::createBucket(const std::string& bucket) const { Aws::S3::Model::CreateBucketRequest request; - request.SetBucket(bucketName); + request.SetBucket(bucket); auto outcome = client_->CreateBucket(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucketName << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; } else { - auto msg = awsErrorMessage("Failed to create bucket=" + bucketName, outcome.GetError()); + auto msg = awsErrorMessage("Failed to create bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } -void S3ClientAWS::emptyBucket(const std::string& bucketName) const { - deleteObjects(bucketName, listObjects(bucketName)); +void S3ClientAWS::emptyBucket(const std::string& bucket) const { + deleteObjects(bucket, listObjects(bucket)); } -void S3ClientAWS::deleteBucket(const std::string& bucketName) const { +void S3ClientAWS::deleteBucket(const std::string& bucket) const { Aws::S3::Model::DeleteBucketRequest request; - request.SetBucket(bucketName); + request.SetBucket(bucket); - const auto outcome = client_->DeleteBucket(request); + auto outcome = client_->DeleteBucket(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucketName << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucket << std::endl; } else { - auto msg = awsErrorMessage("Failed to delete bucket=" + bucketName, outcome.GetError()); + auto msg = awsErrorMessage("Failed to delete bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } -auto S3ClientAWS::bucketExists(const std::string& bucketName) const -> bool { +auto S3ClientAWS::bucketExists(const std::string& bucket) const -> bool { Aws::S3::Model::HeadBucketRequest request; - request.SetBucket(bucketName); + request.SetBucket(bucket); return client_->HeadBucket(request).IsSuccess(); } auto S3ClientAWS::listBuckets() const -> std::vector { - std::vector result; + std::vector buckets; auto outcome = client_->ListBuckets(); if (outcome.IsSuccess()) { - for (const auto& bucket : outcome.GetResult().GetBuckets()) { result.emplace_back(bucket.GetName()); } + for (const auto& bucket : outcome.GetResult().GetBuckets()) { buckets.emplace_back(bucket.GetName()); } } else { Log::warning() << "Failed to list buckets!" << outcome.GetError(); } - return result; + return buckets; } //---------------------------------------------------------------------------------------------------------------------- // OBJECT -void S3ClientAWS::putObject(const std::string& bucketName, const std::string& objectName, const void* buffer, +void S3ClientAWS::putObject(const std::string& bucket, const std::string& object, const void* buffer, const uint64_t length) const { Aws::S3::Model::PutObjectRequest request; - request.SetBucket(bucketName); - request.SetKey(objectName); - request.SetContentLength(length); + request.SetBucket(bucket); + request.SetKey(object); + // request.SetContentLength(length); - auto sBuffer = Aws::New(ALLOC_TAG, (unsigned char*)buffer, length); - auto sReader = Aws::MakeShared(ALLOC_TAG, sBuffer); + auto* sBuffer = Aws::New(ALLOC_TAG, (unsigned char*)buffer, length); + auto sReader = Aws::MakeShared(ALLOC_TAG, sBuffer); request.SetBody(sReader); auto outcome = client_->PutObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Added object=" << objectName << " to bucket=" << bucketName << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Added object=" << object << " to bucket=" << bucket << std::endl; } else { - auto msg = awsErrorMessage("Failed to put object=" + objectName + " to bucket=" + bucketName, outcome.GetError()); + auto msg = awsErrorMessage("Failed to put object=" + object + " to bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } -void S3ClientAWS::putObject(const std::string& bucketName, const std::string& objectName) const { +void S3ClientAWS::putObject(const std::string& bucket, const std::string& object) const { Aws::S3::Model::PutObjectRequest request; - request.SetBucket(bucketName); - request.SetKey(objectName); + request.SetBucket(bucket); + request.SetKey(object); // empty object request.SetBody(Aws::MakeShared(ALLOC_TAG)); @@ -189,43 +189,42 @@ void S3ClientAWS::putObject(const std::string& bucketName, const std::string& ob auto outcome = client_->PutObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Added object=" << objectName << " to bucket=" << bucketName << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Added object=" << object << " to bucket=" << bucket << std::endl; } else { - auto msg = awsErrorMessage("Failed to put object=" + objectName + " to bucket=" + bucketName, outcome.GetError()); + auto msg = awsErrorMessage("Failed to put object=" + object + " to bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } void S3ClientAWS::deleteObject(const std::string& bucketName, const std::string& objectKey) const { + +void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& object) const { Aws::S3::Model::DeleteObjectRequest request; - request.WithKey(objectKey).WithBucket(bucketName); + request.WithKey(object).WithBucket(bucket); auto outcome = client_->DeleteObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << objectKey << " in bucket=" << bucketName << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << " in bucket=" << bucket << std::endl; } else { - auto msg = - awsErrorMessage("Failed to delete object=" + objectKey + " in bucket=" + bucketName, outcome.GetError()); + auto msg = awsErrorMessage("Failed to delete object=" + object + " in bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } -void S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const { - if (objectKeys.empty()) { - LOG_DEBUG_LIB(LibEcKit) << "No objects to delete in bucket=" << bucketName << std::endl; +void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vector& objects) const { + if (objects.empty()) { + LOG_DEBUG_LIB(LibEcKit) << "No objects to delete in bucket=" << bucket << std::endl; return; } Aws::S3::Model::DeleteObjectsRequest request; - request.SetBucket(bucketName); + request.SetBucket(bucket); Aws::S3::Model::Delete deleteObject; - for (const auto& objectKey : objectKeys) { - deleteObject.AddObjects(Aws::S3::Model::ObjectIdentifier().WithKey(objectKey)); - } + for (const auto& object : objects) { deleteObject.AddObjects(Aws::S3::Model::ObjectIdentifier().WithKey(object)); } // deleteObject.SetQuiet(true); request.SetDelete(deleteObject); @@ -233,20 +232,20 @@ void S3ClientAWS::deleteObjects(const std::string& bucketName, const std::vector auto outcome = client_->DeleteObjects(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Deleted " << objectKeys.size() << " objects from bucket " << bucketName << std::endl; - for (const auto& object : objectKeys) { LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << std::endl; } + LOG_DEBUG_LIB(LibEcKit) << "Deleted " << objects.size() << " objects from bucket " << bucket << std::endl; + for (const auto& object : objects) { LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << std::endl; } } else { - auto msg = awsErrorMessage("Failed to delete objects in bucket=" + bucketName, outcome.GetError()); + auto msg = awsErrorMessage("Failed to delete objects in bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } -auto S3ClientAWS::listObjects(const std::string& bucketName) const -> std::vector { +auto S3ClientAWS::listObjects(const std::string& bucket) const -> std::vector { std::vector result; Aws::S3::Model::ListObjectsRequest request; - request.WithBucket(bucketName); + request.WithBucket(bucket); auto outcome = client_->ListObjects(request); @@ -254,30 +253,30 @@ auto S3ClientAWS::listObjects(const std::string& bucketName) const -> std::vecto const auto& objects = outcome.GetResult().GetContents(); for (const auto& object : objects) { result.emplace_back(object.GetKey()); } } else { - Log::warning() << "Failed to list objects in bucket: " << bucketName << outcome.GetError(); + Log::warning() << "Failed to list objects in bucket: " << bucket << outcome.GetError(); } return result; } -auto S3ClientAWS::objectExists(const std::string& bucketName, const std::string& objectKey) const -> bool { +auto S3ClientAWS::objectExists(const std::string& bucket, const std::string& object) const -> bool { Aws::S3::Model::HeadObjectRequest request; - request.WithKey(objectKey).WithBucket(bucketName); + request.WithKey(object).WithBucket(bucket); return client_->HeadObject(request).IsSuccess(); } -auto S3ClientAWS::objectSize(const std::string& bucketName, const std::string& objectKey) const -> Length { +auto S3ClientAWS::objectSize(const std::string& bucket, const std::string& object) const -> Length { Length result; Aws::S3::Model::HeadObjectRequest request; - request.WithKey(objectKey).WithBucket(bucketName); + request.WithKey(object).WithBucket(bucket); - const auto outcome = client_->HeadObject(request); + auto outcome = client_->HeadObject(request); if (outcome.IsSuccess()) { result = outcome.GetResult().GetContentLength(); } else { - const auto msg = awsErrorMessage("Object '" + objectKey + "' doesn't exist or no access!", outcome.GetError()); + const auto msg = awsErrorMessage("Object '" + object + "' doesn't exist or no access!", outcome.GetError()); throw S3SeriousBug(msg, Here()); } diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index b2e322cf8..6ae34fcc1 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -35,30 +35,30 @@ class S3ClientAWS: public S3Client { void configure(const S3Config& config) override; - void createBucket(const std::string& bucketName) const override; + void createBucket(const std::string& bucket) const override; - void emptyBucket(const std::string& bucketName) const override; + void emptyBucket(const std::string& bucket) const override; - void deleteBucket(const std::string& bucketName) const override; + void deleteBucket(const std::string& bucket) const override; - auto bucketExists(const std::string& bucketName) const -> bool override; + auto bucketExists(const std::string& bucket) const -> bool override; auto listBuckets() const -> std::vector override; - void putObject(const std::string& bucketName, const std::string& objectName) const override; + void putObject(const std::string& bucket, const std::string& object) const override; - void putObject(const std::string& bucketName, const std::string& objectName, const void* buffer, + void putObject(const std::string& bucket, const std::string& object, const void* buffer, const uint64_t length) const override; - void deleteObject(const std::string& bucketName, const std::string& objectKey) const override; + void deleteObject(const std::string& bucket, const std::string& object) const override; - void deleteObjects(const std::string& bucketName, const std::vector& objectKeys) const; + void deleteObjects(const std::string& bucket, const std::vector& objects) const override; - auto listObjects(const std::string& bucketName) const -> std::vector override; + auto listObjects(const std::string& bucket) const -> std::vector override; - auto objectExists(const std::string& bucketName, const std::string& objectKey) const -> bool override; + auto objectExists(const std::string& bucket, const std::string& object) const -> bool override; - auto objectSize(const std::string& bucketName, const std::string& objectKey) const -> Length override; + auto objectSize(const std::string& bucket, const std::string& object) const -> Length override; private: // members std::unique_ptr client_; From a4a6835500ecc9d66e2dfd4a3d01441abfc9f868 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sun, 11 Feb 2024 09:10:40 +0100 Subject: [PATCH 34/90] feat(S3): structure for get object (WIP) --- src/eckit/io/s3/S3Client.h | 1 + src/eckit/io/s3/aws/S3ClientAWS.cc | 16 +++++++++++++++- src/eckit/io/s3/aws/S3ClientAWS.h | 2 ++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 93b41e3f6..93948b29a 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -58,6 +58,7 @@ class S3Client: private NonCopyable { virtual void putObject(const std::string& bucket, const std::string& object) const = 0; + virtual void getObject(const std::string& bucket, const std::string& object) const = 0; virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 2902918ec..6a3a16430 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -196,7 +197,20 @@ void S3ClientAWS::putObject(const std::string& bucket, const std::string& object } } -void S3ClientAWS::deleteObject(const std::string& bucketName, const std::string& objectKey) const { +void S3ClientAWS::getObject(const std::string& bucket, const std::string& object) const { + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(bucket); + request.SetKey(object); + + auto outcome = client_->GetObject(request); + + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Retrieved object=" << object << " from bucket=" << bucket << std::endl; + } else { + auto msg = awsErrorMessage("Failed to retrieve object=" + object + " to bucket=" + bucket, outcome.GetError()); + throw S3SeriousBug(msg, Here()); + } +} void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& object) const { Aws::S3::Model::DeleteObjectRequest request; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 6ae34fcc1..cc3da0c2e 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -47,6 +47,8 @@ class S3ClientAWS: public S3Client { void putObject(const std::string& bucket, const std::string& object) const override; + void getObject(const std::string& bucket, const std::string& object) const override; + void putObject(const std::string& bucket, const std::string& object, const void* buffer, const uint64_t length) const override; From 3650136b5d8bc61df0e91506a7cf6c89a8813ef4 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sun, 11 Feb 2024 09:11:10 +0100 Subject: [PATCH 35/90] feat(S3): add S3Name asString --- src/eckit/io/s3/S3Handle.cc | 2 +- src/eckit/io/s3/S3Name.cc | 4 ++++ src/eckit/io/s3/S3Name.h | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 6f61abe19..501dfb8b8 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -87,7 +87,7 @@ Offset S3Handle::seek(const Offset& offset) { } std::string S3Handle::title() const { - return {}; + return name_.asString(); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index a1d24f041..cf642a17f 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -69,6 +69,10 @@ auto S3Name::dataHandle() -> DataHandle* { return new S3Handle(*this); } +auto S3Name::asString() const -> std::string { + return bucket_ + "/" + object_; +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 58aa4c5eb..56cf3ab10 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -54,6 +54,8 @@ class S3Name { [[nodiscard]] auto dataHandle() -> DataHandle*; + auto asString() const -> std::string; + friend std::ostream& operator<<(std::ostream& out, const S3Name& name) { name.print(out); return out; From f08c0b2e5da26221ab590d50827f737d48b34263 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sun, 11 Feb 2024 19:31:37 +0100 Subject: [PATCH 36/90] fix(S3): S3Name ref member in S3Handle --- src/eckit/io/s3/S3Client.cc | 31 ++++++++++++++++++++++++++---- src/eckit/io/s3/S3Client.h | 9 +++++---- src/eckit/io/s3/S3Config.h | 5 +---- src/eckit/io/s3/S3Handle.cc | 4 +--- src/eckit/io/s3/S3Handle.h | 10 ++++------ src/eckit/io/s3/S3Name.cc | 4 +--- src/eckit/io/s3/S3Name.h | 4 +--- src/eckit/io/s3/aws/S3ClientAWS.cc | 9 +++++---- src/eckit/io/s3/aws/S3ClientAWS.h | 2 +- 9 files changed, 46 insertions(+), 32 deletions(-) diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index ceb0ced68..d7c133e82 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -21,6 +21,20 @@ namespace eckit { +namespace { + +inline auto makeSharedS3Client(const S3Types type) -> std::shared_ptr { + if (type == S3Types::AWS) { return std::make_shared(); } + throw S3SeriousBug("Unkown S3 client type!", Here()); +} + +inline auto makeUniqueS3Client(const S3Types type) -> std::unique_ptr { + if (type == S3Types::AWS) { return std::make_unique(); } + throw S3SeriousBug("Unkown S3 client type!", Here()); +} + +} // namespace + //---------------------------------------------------------------------------------------------------------------------- S3Client::S3Client(std::shared_ptr context): context_(context) { } @@ -29,15 +43,24 @@ S3Client::~S3Client() = default; //---------------------------------------------------------------------------------------------------------------------- -auto S3Client::makeUnique(const S3Types type) -> std::unique_ptr { - if (type == S3Types::AWS) { return std::make_unique(); } - throw S3SeriousBug("Unkown S3 client type!", Here()); +auto S3Client::makeShared(const S3Config& config) -> std::shared_ptr { + // if (auto client = makeSharedS3Client(config.type())) { + auto client = makeSharedS3Client(config.type()); + client->configure(config); + return client; + // } + // throw S3SeriousBug("Unkown S3 client type!", Here()); } +//---------------------------------------------------------------------------------------------------------------------- + auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { - auto client = makeUnique(config.type()); + // if (auto client = makeUniqueS3Client(config.type())) { + auto client = makeUniqueS3Client(config.type()); client->configure(config); return client; + // } + // throw S3SeriousBug("Unkown S3 client type!", Here()); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 93948b29a..25387e648 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -33,11 +33,9 @@ class S3Context; class S3Client: private NonCopyable { public: // methods - explicit S3Client(std::shared_ptr context); - virtual ~S3Client(); - static auto makeUnique(S3Types type) -> std::unique_ptr; + static auto makeShared(const S3Config& config) -> std::shared_ptr; static auto makeUnique(const S3Config& config) -> std::unique_ptr; @@ -58,7 +56,7 @@ class S3Client: private NonCopyable { virtual void putObject(const std::string& bucket, const std::string& object) const = 0; - virtual void getObject(const std::string& bucket, const std::string& object) const = 0; + virtual void getObject(const std::string& bucket, const std::string& object, void* buffer, long length) const = 0; virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0; @@ -70,6 +68,9 @@ class S3Client: private NonCopyable { virtual auto objectSize(const std::string& bucket, const std::string& object) const -> Length = 0; +protected: // methods + explicit S3Client(std::shared_ptr context); + private: // members std::shared_ptr context_; }; diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 7d624e787..232149fd1 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -41,8 +41,6 @@ class S3Config { auto type() const -> S3Types { return type_; } - auto tag() const -> const std::string& { return tag_; } - auto region() const -> const std::string& { return region_; } void setRegion(const std::string& region) { region_ = region; } @@ -52,13 +50,12 @@ class S3Config { auto getURI() const -> URI { return {"s3", endpoint_.host(), endpoint_.port()}; } friend std::ostream& operator<<(std::ostream& out, const S3Config& config) { - out << "S3Config[tag=" << config.tag_ << ",region=" << config.region_ << ",endpoint=" << config.endpoint_ << "]"; + out << "S3Config[region=" << config.region_ << ",endpoint=" << config.endpoint_ << "]"; return out; } private: // members S3Types type_ {S3Types::AWS}; - std::string tag_ {"ALLOCATION_TAG"}; std::string region_ {"eu-central-1"}; net::Endpoint endpoint_ {"127.0.0.1", 8888}; }; diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 501dfb8b8..47ef8a93f 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -22,12 +22,10 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Handle::S3Handle(const S3Name& name): name_(name) { } +S3Handle::S3Handle(const S3Name& name): S3Handle(name, 0) { } S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { } -S3Handle::~S3Handle() = default; - void S3Handle::print(std::ostream& out) const { out << "S3Handle[name=" << name_ << ",position=" << pos_ << "]"; } diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index d8dcacfd1..a88cc77c1 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -32,11 +32,6 @@ class S3Handle: public DataHandle { S3Handle(const S3Name& name, const Offset& offset); - ~S3Handle() override; - - void print(std::ostream& out) const override; - -public: // methods Length openForRead() override; void openForWrite(const Length& length) override; @@ -61,8 +56,11 @@ class S3Handle: public DataHandle { bool canSeek() const override { return true; } +private: // methods + void print(std::ostream& out) const override; + private: // members - const S3Name& name_; + const S3Name name_; Offset pos_ {0}; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index cf642a17f..67b43491f 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -26,12 +26,10 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Name::S3Name(const URI& uri): client_(S3Client::makeUnique({uri})) { +S3Name::S3Name(const URI& uri): client_(S3Client::makeShared({uri})) { parse(uri.name()); } -S3Name::~S3Name() = default; - //---------------------------------------------------------------------------------------------------------------------- void S3Name::parse(const std::string& path) { diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 56cf3ab10..807e29e59 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -37,8 +37,6 @@ class S3Name { public: // methods explicit S3Name(const URI& uri); - ~S3Name(); - void put(const void* buffer, uint64_t length) const; auto bucket() const -> const std::string& { return bucket_; } @@ -70,7 +68,7 @@ class S3Name { std::string bucket_; std::string object_; - std::unique_ptr client_; + std::shared_ptr client_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 6a3a16430..b3cc3af19 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -92,7 +92,7 @@ void S3ClientAWS::configure(const S3Config& config) { } // endpoint provider - auto provider = Aws::MakeShared(config.tag().c_str()); + auto provider = Aws::MakeShared(ALLOC_TAG); // finally client client_ = std::make_unique(credentials, provider, configuration); @@ -197,8 +197,9 @@ void S3ClientAWS::putObject(const std::string& bucket, const std::string& object } } -void S3ClientAWS::getObject(const std::string& bucket, const std::string& object) const { +void S3ClientAWS::getObject(const std::string& bucket, const std::string& object, void* buffer, const long length) const { Aws::S3::Model::GetObjectRequest request; + request.SetBucket(bucket); request.SetKey(object); @@ -246,7 +247,7 @@ void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vectorDeleteObjects(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Deleted " << objects.size() << " objects from bucket " << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Deleted " << objects.size() << " objects in bucket=" << bucket << std::endl; for (const auto& object : objects) { LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << std::endl; } } else { auto msg = awsErrorMessage("Failed to delete objects in bucket=" + bucket, outcome.GetError()); @@ -267,7 +268,7 @@ auto S3ClientAWS::listObjects(const std::string& bucket) const -> std::vector(buffer), size)) { } -namespace eckit { + ~BufferIOStream() override { delete rdbuf(); } +}; -const auto ALLOC_TAG = "S3ClientAWS"; +} // namespace //---------------------------------------------------------------------------------------------------------------------- @@ -164,9 +173,8 @@ void S3ClientAWS::putObject(const std::string& bucket, const std::string& object request.SetKey(object); // request.SetContentLength(length); - auto* sBuffer = Aws::New(ALLOC_TAG, (unsigned char*)buffer, length); - auto sReader = Aws::MakeShared(ALLOC_TAG, sBuffer); - request.SetBody(sReader); + auto streamBuffer = Aws::MakeShared(ALLOC_TAG, const_cast(buffer), length); + request.SetBody(streamBuffer); auto outcome = client_->PutObject(request); @@ -202,11 +210,17 @@ void S3ClientAWS::getObject(const std::string& bucket, const std::string& object request.SetBucket(bucket); request.SetKey(object); + request.SetResponseStreamFactory([buffer, length]() { return Aws::New(ALLOC_TAG, buffer, length); }); auto outcome = client_->GetObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Retrieved object=" << object << " from bucket=" << bucket << std::endl; + // const auto& body = outcome.GetResult().GetBody(); + + // buffer = body.rdbuf(); + + // objectStream << body.rdbuf(); } else { auto msg = awsErrorMessage("Failed to retrieve object=" + object + " to bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 53c5a5e84..9fa5502ff 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -30,7 +30,10 @@ namespace eckit::test { S3Config cfg("eu-central-1", "127.0.0.1", 9000); +constexpr std::string_view TEST_DATA = "abcdefghijklmnopqrstuvwxyz"; + static const std::string TEST_BUCKET("eckit-s3handle-test-bucket"); +static const std::string TEST_OBJECT("eckit-s3handle-test-object"); //---------------------------------------------------------------------------------------------------------------------- @@ -54,28 +57,29 @@ void bucketSetup() { } CASE("S3Handle") { - const std::string_view testData = "abcdefghijklmnopqrstuvwxyz"; + const void* buffer = TEST_DATA.data(); + const auto length = TEST_DATA.size(); - const void* buffer = testData.data(); - const auto length = testData.size(); + std::cout << "write buffer: " << TEST_DATA << std::endl; - URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/test-object"); + URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); { S3Name name(uri); - std::unique_ptr h {name.dataHandle()}; + std::unique_ptr h(name.dataHandle()); h->openForWrite(length); AutoClose closer(*h); EXPECT(h->write(buffer, length) == length); } { - auto h = uri.newReadHandle(); + std::unique_ptr h(uri.newReadHandle()); h->openForRead(); std::string rbuf; + std::cout << "read buffer: " << rbuf << std::endl; h->read(rbuf.data(), length); - std::cout << rbuf << std::endl; + std::cout << "read buffer: " << rbuf << std::endl; } { From ef4859208ddfc7273e082f975884c7cfe69f1204 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sun, 11 Feb 2024 21:39:46 +0100 Subject: [PATCH 38/90] feat(S3): fix tests3client and add print to S3Client --- src/eckit/io/s3/S3Client.cc | 25 +++++++++---- src/eckit/io/s3/S3Client.h | 7 ++++ src/eckit/io/s3/aws/S3ClientAWS.cc | 5 +++ tests/io/test_s3client.cc | 59 ++++++++++++++++++------------ 4 files changed, 65 insertions(+), 31 deletions(-) diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index d7c133e82..7e3cfcc34 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -43,24 +43,33 @@ S3Client::~S3Client() = default; //---------------------------------------------------------------------------------------------------------------------- +void S3Client::print(std::ostream& out) const { + out << "S3Client[type="; + const auto type = context_->getType(); + if (type == S3Types::AWS) { + out << "AWS"; + } else if (type == S3Types::REST) { + out << "REST"; + } else { + out << "REST"; + } + out << "]"; +} + +//---------------------------------------------------------------------------------------------------------------------- + auto S3Client::makeShared(const S3Config& config) -> std::shared_ptr { - // if (auto client = makeSharedS3Client(config.type())) { - auto client = makeSharedS3Client(config.type()); + auto client = makeSharedS3Client(config.type); client->configure(config); return client; - // } - // throw S3SeriousBug("Unkown S3 client type!", Here()); } //---------------------------------------------------------------------------------------------------------------------- auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { - // if (auto client = makeUniqueS3Client(config.type())) { - auto client = makeUniqueS3Client(config.type()); + auto client = makeUniqueS3Client(config.type); client->configure(config); return client; - // } - // throw S3SeriousBug("Unkown S3 client type!", Here()); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 25387e648..0362d050f 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -68,9 +68,16 @@ class S3Client: private NonCopyable { virtual auto objectSize(const std::string& bucket, const std::string& object) const -> Length = 0; + friend std::ostream& operator<<(std::ostream& out, const S3Client& client) { + client.print(out); + return out; + } + protected: // methods explicit S3Client(std::shared_ptr context); + virtual void print(std::ostream& out) const; + private: // members std::shared_ptr context_; }; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index de56683bd..479a934db 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -107,6 +107,11 @@ void S3ClientAWS::configure(const S3Config& config) { client_ = std::make_unique(credentials, provider, configuration); } +void S3ClientAWS::print(std::ostream& out) const { + S3Client::print(out); + out << "S3ClientAWS[]"; +} + //---------------------------------------------------------------------------------------------------------------------- // BUCKET diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index fd50e8637..191ae4e26 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -29,46 +29,60 @@ const S3Config cfg("eu-central-1", "127.0.0.1", 9000); //---------------------------------------------------------------------------------------------------------------------- +bool findString(const std::vector& list, const std::string& item) { + return (std::find(list.begin(), list.end(), item) != list.end()); +} + void ensureClean() { auto client = S3Client::makeUnique(cfg); auto&& tmp = client->listBuckets(); std::set buckets(tmp.begin(), tmp.end()); - for (const std::string& name : {"failed-bucket", "test-bucket-1", "test-bucket-2"}) { + for (const std::string& name : {"test-bucket-1", "test-bucket-2"}) { if (buckets.find(name) != buckets.end()) { + client->emptyBucket(name); client->deleteBucket(name); } } - } - //---------------------------------------------------------------------------------------------------------------------- CASE("different types") { - EXPECT_THROWS(S3Client::makeUnique(S3Types::NONE)); - EXPECT_NO_THROW(S3Client::makeUnique(S3Types::AWS)); -} + S3Config cfgTmp(cfg); -CASE("wrong credentials") { - ensureClean(); - EXPECT_THROWS(S3Client::makeUnique(S3Types::AWS)->createBucket("failed-bucket")); + cfgTmp.type = S3Types::NONE; + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); + + cfgTmp.type = S3Types::AWS; + EXPECT_NO_THROW(S3Client::makeUnique(cfgTmp)); + + cfgTmp.type = S3Types::REST; + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); } -CASE("create bucket in missing region") { - ensureClean(); +// CASE("wrong credentials") { +// ensureClean(); +// +// S3Config cfgTmp(cfg); +// cfgTmp.region = "no-region-random"; +// +// EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("failed-bucket")); +// } - auto config = cfg; - config.setRegion("eu-central-2"); +CASE("create bucket in non-existing region") { + ensureClean(); - auto client = S3Client::makeUnique(config); + S3Config cfgTmp(cfg); + cfgTmp.region = "non-existing-region-random"; - EXPECT_THROWS(client->createBucket("test-bucket-1")); + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("test-bucket-1")); } CASE("create bucket") { ensureClean(); + auto client = S3Client::makeUnique(cfg); EXPECT_NO_THROW(client->createBucket("test-bucket-1")); @@ -79,7 +93,6 @@ CASE("create bucket") { } CASE("list buckets") { - ensureClean(); auto client = S3Client::makeUnique(cfg); @@ -89,17 +102,17 @@ CASE("list buckets") { { const auto buckets = client->listBuckets(); - EXPECT_EQUAL(buckets[0], "test-bucket-1"); - EXPECT_EQUAL(buckets[1], "test-bucket-2"); + EXPECT(findString(buckets, "test-bucket-1")); + EXPECT(findString(buckets, "test-bucket-2")); - for (auto&& bucket : buckets) { - client->deleteBucket(bucket); - } + EXPECT_NO_THROW(client->deleteBucket("test-bucket-1")); + EXPECT_NO_THROW(client->deleteBucket("test-bucket-2")); } { const auto buckets = client->listBuckets(); - EXPECT(buckets.empty()); + EXPECT_NOT(findString(buckets, "test-bucket-1")); + EXPECT_NOT(findString(buckets, "test-bucket-2")); } EXPECT_THROWS(client->deleteBucket("test-bucket-1")); @@ -117,6 +130,6 @@ int main(int argc, char** argv) { auto ret = run_tests(argc, argv); try { eckit::test::ensureClean(); - } catch (...) {} + } catch (...) { } return ret; } From 709e51b560f583ce3e6ffbad66d40166fd2626c6 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sun, 11 Feb 2024 21:46:04 +0100 Subject: [PATCH 39/90] feat(S3): rework s3 config --- src/eckit/CMakeLists.txt | 1 + src/eckit/io/s3/S3Config.cc | 37 ++++++++++++++++++++++++++++++ src/eckit/io/s3/S3Config.h | 33 ++++++++------------------ src/eckit/io/s3/aws/S3ClientAWS.cc | 12 +++++----- src/eckit/io/s3/aws/S3ClientAWS.h | 3 +++ 5 files changed, 57 insertions(+), 29 deletions(-) create mode 100644 src/eckit/io/s3/S3Config.cc diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index c689de0ff..046b0a2cf 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -294,6 +294,7 @@ io/s3/aws/S3ContextAWS.cc io/s3/aws/S3ContextAWS.h io/s3/S3Client.cc io/s3/S3Client.h +io/s3/S3Config.cc io/s3/S3Config.h io/s3/S3Context.cc io/s3/S3Context.h diff --git a/src/eckit/io/s3/S3Config.cc b/src/eckit/io/s3/S3Config.cc new file mode 100644 index 000000000..d09604880 --- /dev/null +++ b/src/eckit/io/s3/S3Config.cc @@ -0,0 +1,37 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Config.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3Config::S3Config(std::string region, const URI& uri): region(std::move(region)), endpoint(uri) { } + +S3Config::S3Config(std::string region, const std::string& hostname, const int port): + region(std::move(region)), endpoint(hostname, port) { } + +S3Config::S3Config(const URI& uri): endpoint(uri) { } + +//---------------------------------------------------------------------------------------------------------------------- + +void S3Config::print(std::ostream& out) const { + out << "S3Config[region=" << region << ",endpoint=" << endpoint << "]"; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 232149fd1..8c3e6a005 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -29,35 +29,22 @@ enum class S3Types { NONE, AWS, REST }; //---------------------------------------------------------------------------------------------------------------------- -class S3Config { -public: // methods +struct S3Config { /// @todo region is part of hostname (s3express-control.region_code.amazonaws.com/bucket-name) - S3Config(const std::string& region, const std::string& hostname, const int port): - region_(region), endpoint_(hostname, port) { } - - S3Config(const std::string& region, const URI& uri): region_(region), endpoint_(uri) { } - - S3Config(const URI& uri): endpoint_(uri) { } - - auto type() const -> S3Types { return type_; } - - auto region() const -> const std::string& { return region_; } - - void setRegion(const std::string& region) { region_ = region; } - - auto endpoint() const -> const net::Endpoint& { return endpoint_; } - - auto getURI() const -> URI { return {"s3", endpoint_.host(), endpoint_.port()}; } + S3Config(std::string region, const URI& uri); + S3Config(std::string region, const std::string& hostname, int port); + S3Config(const URI& uri); friend std::ostream& operator<<(std::ostream& out, const S3Config& config) { - out << "S3Config[region=" << config.region_ << ",endpoint=" << config.endpoint_ << "]"; + config.print(out); return out; } -private: // members - S3Types type_ {S3Types::AWS}; - std::string region_ {"eu-central-1"}; - net::Endpoint endpoint_ {"127.0.0.1", 8888}; + void print(std::ostream& out) const; + + S3Types type {S3Types::AWS}; + std::string region {"eu-central-1"}; + net::Endpoint endpoint {"127.0.0.1", -1}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 479a934db..1b0183543 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -81,21 +81,21 @@ void S3ClientAWS::configure(const S3Config& config) { configuration.disableIMDS = true; // setup region - configuration.region = config.region(); - if (!config.region().empty()) { configuration.region = config.region(); } + // configuration.region = config.region; + if (!config.region.empty()) { configuration.region = config.region; } // configuration.proxyScheme = Aws::Http::Scheme::HTTPS; // configuration.verifySSL = false; // setup endpoint /// @todo handle http/https possibly via scheme flag in S3Config - const auto& endpoint = config.endpoint(); - if (!endpoint.host().empty()) { configuration.endpointOverride = "http://" + endpoint.host(); } - if (endpoint.port() > 0) { configuration.endpointOverride += ":" + std::to_string(endpoint.port()); } + // const auto& endpoint = config.endpoint; + if (!config.endpoint.host().empty()) { configuration.endpointOverride = "http://" + config.endpoint.host(); } + if (config.endpoint.port() > 0) { configuration.endpointOverride += ":" + std::to_string(config.endpoint.port()); } // setup credentials Aws::Auth::AWSCredentials credentials; - if (auto cred = S3Session::instance().getCredentials(endpoint.host())) { + if (auto cred = S3Session::instance().getCredentials(config.endpoint.host())) { credentials.SetAWSAccessKeyId(cred->keyID); credentials.SetAWSSecretKey(cred->secret); } diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 897b6188b..37114d941 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -62,6 +62,9 @@ class S3ClientAWS: public S3Client { auto objectSize(const std::string& bucket, const std::string& object) const -> Length override; +private: // methods + void print(std::ostream& out) const override; + private: // members std::unique_ptr client_; }; From 6b9f2a9f7096124a747b0f627b1f41dacc1c809b Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 13 Feb 2024 13:41:25 +0100 Subject: [PATCH 40/90] feat(S3): adds range get object and S3Handle fixes (WIP) --- src/eckit/io/s3/S3Client.h | 12 +++-- src/eckit/io/s3/S3Handle.cc | 67 ++++++++++++++++------- src/eckit/io/s3/S3Handle.h | 7 ++- src/eckit/io/s3/S3Name.cc | 19 ++++--- src/eckit/io/s3/S3Name.h | 6 ++- src/eckit/io/s3/aws/S3ClientAWS.cc | 86 +++++++++++++++++++++++------- src/eckit/io/s3/aws/S3ClientAWS.h | 10 ++-- 7 files changed, 153 insertions(+), 54 deletions(-) diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 0362d050f..65c78ee3d 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -51,12 +51,16 @@ class S3Client: private NonCopyable { virtual auto listBuckets() const -> std::vector = 0; - virtual void putObject(const std::string& bucket, const std::string& object, const void* buffer, - const uint64_t length) const = 0; - virtual void putObject(const std::string& bucket, const std::string& object) const = 0; - virtual void getObject(const std::string& bucket, const std::string& object, void* buffer, long length) const = 0; + virtual auto putObject(const std::string& bucket, const std::string& object, const void* buffer, + uint64_t length) const -> long long = 0; + + virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t length) const + -> long long = 0; + + virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t offset, + uint64_t length) const -> long long = 0; virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0; diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 7a8998c4d..383e013b7 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -27,37 +27,70 @@ S3Handle::S3Handle(const S3Name& name): S3Handle(name, 0) { } S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { } void S3Handle::print(std::ostream& out) const { - out << "S3Handle[name=" << name_ << ",position=" << pos_ << "]"; + out << "S3Handle[name=" << name_ << ",position=" << pos_ << ", open=" << open_ << ", writable=" << canWrite_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- +void S3Handle::open(const Offset& position) { + ASSERT(!open_); + pos_ = position; + open_ = true; +} + Length S3Handle::openForRead() { + ASSERT(name_.exists()); + + open(0); + canWrite_ = false; - return name_.size(); // asserts + + return size(); } -void S3Handle::openForWrite(const Length&) { +void S3Handle::openForWrite(const Length& length) { + open(Offset(length)); + + pos_ += length; + canWrite_ = true; } -void S3Handle::openForAppend(const Length&) { - NOTIMP; +void S3Handle::openForAppend(const Length& length) { + open(Offset(length)); + + pos_ += length; + canWrite_ = true; } //---------------------------------------------------------------------------------------------------------------------- long S3Handle::read(void* buffer, const long length) { - name_.get(buffer, length); - pos_ += length; - return length; + if (!open_) { throw S3SeriousBug("S3 object is not writable!", Here()); } + + long len = 0; + if (pos_ == Offset(0)) { + len = name_.get(buffer, length); + } else if (pos_ > Offset(0)) { + len = name_.get(buffer, pos_, length); + } + + pos_ += len; + + return len; } long S3Handle::write(const void* buffer, const long length) { - name_.put(buffer, length); - pos_ += length; - return length; + ASSERT(!name_.exists()); + + if (!open_ || !canWrite_) { throw S3SeriousBug("S3 object is not writable!", Here()); } + + const auto len = name_.put(buffer, length); + + pos_ += len; + + return len; } void S3Handle::flush() { @@ -65,16 +98,18 @@ void S3Handle::flush() { } void S3Handle::close() { - /// @todo remove - LOG_DEBUG_LIB(LibEcKit) << "close!" << std::endl; + open_ = false; canWrite_ = false; - // object_.reset(); } Length S3Handle::size() { return name_.size(); // asserts } +Length S3Handle::estimate() { + return size(); +} + Offset S3Handle::position() { return pos_; } @@ -84,10 +119,6 @@ Offset S3Handle::seek(const Offset& offset) { return pos_; } -std::string S3Handle::title() const { - return name_.asString(); -} - //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index a88cc77c1..ec3d79729 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -48,22 +48,25 @@ class S3Handle: public DataHandle { Length size() override; + Length estimate() override; + Offset position() override; Offset seek(const Offset& offset) override; - std::string title() const override; - bool canSeek() const override { return true; } private: // methods void print(std::ostream& out) const override; + void open(const Offset& position); + private: // members const S3Name name_; Offset pos_ {0}; + bool open_ {false}; bool canWrite_ {false}; }; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 45d24dc75..366507694 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -34,11 +34,12 @@ S3Name::S3Name(const URI& uri): client_(S3Client::makeShared({uri})) { void S3Name::parse(const std::string& path) { const auto pairs = Tokenizer("/").tokenize(path); + const auto pSize = pairs.size(); - ASSERT(pairs.size() == 2); + ASSERT(pSize == 1 || pSize == 2); - bucket_ = pairs[0]; - object_ = pairs[1]; + if (pSize > 0) { bucket_ = pairs[0]; } + if (pSize > 1) { object_ = pairs[1]; } } void S3Name::print(std::ostream& out) const { @@ -47,12 +48,16 @@ void S3Name::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- -void S3Name::put(const void* buffer, const uint64_t length) const { - client_->putObject(bucket_, object_, buffer, length); +auto S3Name::put(const void* buffer, const long length) const -> long { + return client_->putObject(bucket_, object_, buffer, length); } -void S3Name::get(void* buffer, const uint64_t length) const { - client_->getObject(bucket_, object_, buffer, length); +auto S3Name::get(void* buffer, const long offset, const long length) const -> long { + return client_->getObject(bucket_, object_, buffer, offset, length); +} + +auto S3Name::get(void* buffer, const long length) const -> long { + return client_->getObject(bucket_, object_, buffer, length); } auto S3Name::bucketExists() const -> bool { diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index af48f79fc..3ed26e192 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -37,9 +37,11 @@ class S3Name { public: // methods explicit S3Name(const URI& uri); - void put(const void* buffer, uint64_t length) const; + auto put(const void* buffer, long length) const -> long; - void get(void* buffer, const uint64_t length) const; + auto get(void* buffer, long length) const -> long; + + auto get(void* buffer, const long offset, const long length) const -> long; auto bucket() const -> const std::string& { return bucket_; } diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 1b0183543..c5cd63e80 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -55,7 +55,7 @@ inline std::string awsErrorMessage(const std::string& msg, const Aws::S3::S3Erro class BufferIOStream: public Aws::IOStream { public: /// @todo reinterpret_cast - BufferIOStream(void* buffer, uint64_t size): + BufferIOStream(void* buffer, const uint64_t size): Aws::IOStream(new Aws::Utils::Stream::PreallocatedStreamBuf(reinterpret_cast(buffer), size)) { } ~BufferIOStream() override { delete rdbuf(); } @@ -168,18 +168,16 @@ auto S3ClientAWS::listBuckets() const -> std::vector { } //---------------------------------------------------------------------------------------------------------------------- -// OBJECT +// PUT OBJECT -void S3ClientAWS::putObject(const std::string& bucket, const std::string& object, const void* buffer, - const uint64_t length) const { +void S3ClientAWS::putObject(const std::string& bucket, const std::string& object) const { Aws::S3::Model::PutObjectRequest request; request.SetBucket(bucket); request.SetKey(object); - // request.SetContentLength(length); - auto streamBuffer = Aws::MakeShared(ALLOC_TAG, const_cast(buffer), length); - request.SetBody(streamBuffer); + // empty object + request.SetBody(Aws::MakeShared(ALLOC_TAG)); auto outcome = client_->PutObject(request); @@ -191,47 +189,99 @@ void S3ClientAWS::putObject(const std::string& bucket, const std::string& object } } -void S3ClientAWS::putObject(const std::string& bucket, const std::string& object) const { +auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object, const void* buffer, + const uint64_t length) const -> long long { Aws::S3::Model::PutObjectRequest request; request.SetBucket(bucket); request.SetKey(object); + // request.SetContentLength(length); - // empty object - request.SetBody(Aws::MakeShared(ALLOC_TAG)); + auto streamBuffer = Aws::MakeShared(ALLOC_TAG, const_cast(buffer), length); + request.SetBody(streamBuffer); auto outcome = client_->PutObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Added object=" << object << " to bucket=" << bucket << std::endl; + /// @todo actual size of written bytes + return length; + // return request.GetContentLength(); } else { auto msg = awsErrorMessage("Failed to put object=" + object + " to bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } } -void S3ClientAWS::getObject(const std::string& bucket, const std::string& object, void* buffer, const long length) const { +// auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object, const void* buffer, +// const uint64_t start, const uint64_t end) const -> long long { +// Aws::S3::Model::PutObjectRequest request; +// +// request.SetBucket(bucket); +// request.SetKey(object); +// // request.SetContentLength(length); +// +// auto streamBuffer = Aws::MakeShared(ALLOC_TAG, const_cast(buffer), length); +// request.SetBody(streamBuffer); +// +// auto outcome = client_->PutObject(request); +// +// if (outcome.IsSuccess()) { +// LOG_DEBUG_LIB(LibEcKit) << "Added object=" << object << " to bucket=" << bucket << std::endl; +// /// @todo actual size of written bytes +// return end - start; +// // return request.GetContentLength(); +// } +// +// auto msg = awsErrorMessage("Failed to put object=" + object + " to bucket=" + bucket, outcome.GetError()); +// throw S3SeriousBug(msg, Here()); +// } + +//---------------------------------------------------------------------------------------------------------------------- +// GET OBJECT + +auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object, void* buffer, + const uint64_t length) const -> long long { Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket); request.SetKey(object); - request.SetResponseStreamFactory([buffer, length]() { return Aws::New(ALLOC_TAG, buffer, length); }); + request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(ALLOC_TAG, buffer, length); }); auto outcome = client_->GetObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Retrieved object=" << object << " from bucket=" << bucket << std::endl; - // const auto& body = outcome.GetResult().GetBody(); + return outcome.GetResult().GetContentLength(); + } - // buffer = body.rdbuf(); + auto msg = awsErrorMessage("Failed to retrieve object=" + object + " to bucket=" + bucket, outcome.GetError()); + throw S3SeriousBug(msg, Here()); +} - // objectStream << body.rdbuf(); - } else { - auto msg = awsErrorMessage("Failed to retrieve object=" + object + " to bucket=" + bucket, outcome.GetError()); - throw S3SeriousBug(msg, Here()); +auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object, void* buffer, const uint64_t offset, + const uint64_t length) const -> long long { + Aws::S3::Model::GetObjectRequest request; + + request.SetBucket(bucket); + request.SetKey(object); + request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(ALLOC_TAG, buffer, length); }); + request.SetRange(std::to_string(offset) + "-" + std::to_string(offset + length)); + + auto outcome = client_->GetObject(request); + + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Retrieved object=" << object << " from bucket=" << bucket << std::endl; + return outcome.GetResult().GetContentLength(); } + + auto msg = awsErrorMessage("Failed to retrieve object=" + object + " to bucket=" + bucket, outcome.GetError()); + throw S3SeriousBug(msg, Here()); } +//---------------------------------------------------------------------------------------------------------------------- +// OBJECT + void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& object) const { Aws::S3::Model::DeleteObjectRequest request; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 37114d941..c1e934b88 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -47,10 +47,14 @@ class S3ClientAWS: public S3Client { void putObject(const std::string& bucket, const std::string& object) const override; - void getObject(const std::string& bucket, const std::string& object, void* buffer, long length) const override; + auto putObject(const std::string& bucket, const std::string& object, const void* buffer, uint64_t length) const + -> long long override; - void putObject(const std::string& bucket, const std::string& object, const void* buffer, - const uint64_t length) const override; + auto getObject(const std::string& bucket, const std::string& object, void* buffer, const uint64_t offset, + const uint64_t length) const -> long long override; + + auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t length) const + -> long long override; void deleteObject(const std::string& bucket, const std::string& object) const override; From e19d6715f97daa9061676f939e4f25ada4d9cadf Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 13 Feb 2024 13:43:09 +0100 Subject: [PATCH 41/90] chore(S3): issue with S3Handle saveInto a MemoryHandle --- tests/io/test_s3handle.cc | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 9fa5502ff..1d0ad2ac1 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -14,6 +14,7 @@ /// @date Jan 2024 #include "eckit/filesystem/URI.h" +#include "eckit/io/Buffer.h" #include "eckit/io/MemoryHandle.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Handle.h" @@ -58,7 +59,7 @@ void bucketSetup() { CASE("S3Handle") { const void* buffer = TEST_DATA.data(); - const auto length = TEST_DATA.size(); + const long length = TEST_DATA.size(); std::cout << "write buffer: " << TEST_DATA << std::endl; @@ -76,15 +77,21 @@ CASE("S3Handle") { { std::unique_ptr h(uri.newReadHandle()); h->openForRead(); + std::string rbuf; - std::cout << "read buffer: " << rbuf << std::endl; - h->read(rbuf.data(), length); - std::cout << "read buffer: " << rbuf << std::endl; + rbuf.resize(length); + + const auto len = h->read(rbuf.data(), length); + + EXPECT(len == length); + EXPECT(rbuf == TEST_DATA); } { std::unique_ptr dh(uri.newReadHandle()); - MemoryHandle mh; + + MemoryHandle mh(length); + /// @todo this creates issues because internal buffer size is set to 64*1024*1024 dh->saveInto(mh); EXPECT(mh.size() == Length(length)); From a0cc783bab39c64a4b284efb753a15460716079b Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 13 Feb 2024 13:45:57 +0100 Subject: [PATCH 42/90] refactor(S3): correction throw messages --- src/eckit/io/s3/S3Handle.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 383e013b7..e62a0cb04 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -67,7 +67,7 @@ void S3Handle::openForAppend(const Length& length) { //---------------------------------------------------------------------------------------------------------------------- long S3Handle::read(void* buffer, const long length) { - if (!open_) { throw S3SeriousBug("S3 object is not writable!", Here()); } + if (!open_) { throw S3SeriousBug("S3 handle is not open!", Here()); } long len = 0; if (pos_ == Offset(0)) { @@ -84,7 +84,7 @@ long S3Handle::read(void* buffer, const long length) { long S3Handle::write(const void* buffer, const long length) { ASSERT(!name_.exists()); - if (!open_ || !canWrite_) { throw S3SeriousBug("S3 object is not writable!", Here()); } + if (!open_ || !canWrite_) { throw S3SeriousBug("S3 handle is not writable!", Here()); } const auto len = name_.put(buffer, length); From 61c0f8903a2615085b03d6b44c26dc43dc8bf215 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Feb 2024 19:07:09 +0100 Subject: [PATCH 43/90] refactor(S3): small cleanup S3Handle and S3Client --- src/eckit/io/s3/S3Client.h | 3 -- src/eckit/io/s3/S3Handle.cc | 11 +------ src/eckit/io/s3/S3Name.cc | 4 --- src/eckit/io/s3/S3Name.h | 4 +-- src/eckit/io/s3/aws/S3ClientAWS.cc | 51 ++++-------------------------- src/eckit/io/s3/aws/S3ClientAWS.h | 3 -- tests/io/test_s3handle.cc | 3 +- 7 files changed, 10 insertions(+), 69 deletions(-) diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 65c78ee3d..7f9055486 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -56,9 +56,6 @@ class S3Client: private NonCopyable { virtual auto putObject(const std::string& bucket, const std::string& object, const void* buffer, uint64_t length) const -> long long = 0; - virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t length) const - -> long long = 0; - virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t offset, uint64_t length) const -> long long = 0; diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index e62a0cb04..372721658 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -51,16 +51,12 @@ Length S3Handle::openForRead() { void S3Handle::openForWrite(const Length& length) { open(Offset(length)); - pos_ += length; - canWrite_ = true; } void S3Handle::openForAppend(const Length& length) { open(Offset(length)); - pos_ += length; - canWrite_ = true; } @@ -69,12 +65,7 @@ void S3Handle::openForAppend(const Length& length) { long S3Handle::read(void* buffer, const long length) { if (!open_) { throw S3SeriousBug("S3 handle is not open!", Here()); } - long len = 0; - if (pos_ == Offset(0)) { - len = name_.get(buffer, length); - } else if (pos_ > Offset(0)) { - len = name_.get(buffer, pos_, length); - } + const auto len = name_.get(buffer, pos_, length); pos_ += len; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 366507694..4b6559f06 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -56,10 +56,6 @@ auto S3Name::get(void* buffer, const long offset, const long length) const -> lo return client_->getObject(bucket_, object_, buffer, offset, length); } -auto S3Name::get(void* buffer, const long length) const -> long { - return client_->getObject(bucket_, object_, buffer, length); -} - auto S3Name::bucketExists() const -> bool { return client_->bucketExists(bucket_); } diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 3ed26e192..104c0b598 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -39,9 +39,7 @@ class S3Name { auto put(const void* buffer, long length) const -> long; - auto get(void* buffer, long length) const -> long; - - auto get(void* buffer, const long offset, const long length) const -> long; + auto get(void* buffer, long offset, long length) const -> long; auto bucket() const -> const std::string& { return bucket_; } diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index c5cd63e80..572579981 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -89,7 +89,6 @@ void S3ClientAWS::configure(const S3Config& config) { // setup endpoint /// @todo handle http/https possibly via scheme flag in S3Config - // const auto& endpoint = config.endpoint; if (!config.endpoint.host().empty()) { configuration.endpointOverride = "http://" + config.endpoint.host(); } if (config.endpoint.port() > 0) { configuration.endpointOverride += ":" + std::to_string(config.endpoint.port()); } @@ -213,65 +212,27 @@ auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object } } -// auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object, const void* buffer, -// const uint64_t start, const uint64_t end) const -> long long { -// Aws::S3::Model::PutObjectRequest request; -// -// request.SetBucket(bucket); -// request.SetKey(object); -// // request.SetContentLength(length); -// -// auto streamBuffer = Aws::MakeShared(ALLOC_TAG, const_cast(buffer), length); -// request.SetBody(streamBuffer); -// -// auto outcome = client_->PutObject(request); -// -// if (outcome.IsSuccess()) { -// LOG_DEBUG_LIB(LibEcKit) << "Added object=" << object << " to bucket=" << bucket << std::endl; -// /// @todo actual size of written bytes -// return end - start; -// // return request.GetContentLength(); -// } -// -// auto msg = awsErrorMessage("Failed to put object=" + object + " to bucket=" + bucket, outcome.GetError()); -// throw S3SeriousBug(msg, Here()); -// } - //---------------------------------------------------------------------------------------------------------------------- // GET OBJECT -auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object, void* buffer, - const uint64_t length) const -> long long { - Aws::S3::Model::GetObjectRequest request; - - request.SetBucket(bucket); - request.SetKey(object); - request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(ALLOC_TAG, buffer, length); }); - - auto outcome = client_->GetObject(request); - - if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Retrieved object=" << object << " from bucket=" << bucket << std::endl; - return outcome.GetResult().GetContentLength(); - } - - auto msg = awsErrorMessage("Failed to retrieve object=" + object + " to bucket=" + bucket, outcome.GetError()); - throw S3SeriousBug(msg, Here()); -} - auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object, void* buffer, const uint64_t offset, const uint64_t length) const -> long long { + // no throw; POSIX + if (length == 0) { return 0; } + Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket); request.SetKey(object); request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(ALLOC_TAG, buffer, length); }); - request.SetRange(std::to_string(offset) + "-" + std::to_string(offset + length)); + if (offset > 0) { request.SetRange(std::to_string(offset) + "-" + std::to_string(offset + length)); } auto outcome = client_->GetObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Retrieved object=" << object << " from bucket=" << bucket << std::endl; + + /// @todo (try to check) if EOF and return 0; according to POSIX return outcome.GetResult().GetContentLength(); } diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index c1e934b88..0aaf0f3e9 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -53,9 +53,6 @@ class S3ClientAWS: public S3Client { auto getObject(const std::string& bucket, const std::string& object, void* buffer, const uint64_t offset, const uint64_t length) const -> long long override; - auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t length) const - -> long long override; - void deleteObject(const std::string& bucket, const std::string& object) const override; void deleteObjects(const std::string& bucket, const std::vector& objects) const override; diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 1d0ad2ac1..4479700c6 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -83,7 +83,8 @@ CASE("S3Handle") { const auto len = h->read(rbuf.data(), length); - EXPECT(len == length); + // EXPECT(len == 0); // POSIX + EXPECT(rbuf == TEST_DATA); } From d3d3ebe9429925566824c2555723413837f39faa Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Wed, 14 Feb 2024 22:38:41 +0100 Subject: [PATCH 44/90] feat(S3): adds S3Bucket. --- src/eckit/CMakeLists.txt | 2 + src/eckit/io/s3/S3Bucket.cc | 74 +++++++++++++++++++++++++++++ src/eckit/io/s3/S3Bucket.h | 92 +++++++++++++++++++++++++++++++++++++ src/eckit/io/s3/S3Config.cc | 2 + src/eckit/io/s3/S3Config.h | 1 + src/eckit/io/s3/S3Name.cc | 4 ++ src/eckit/io/s3/S3Name.h | 3 ++ 7 files changed, 178 insertions(+) create mode 100644 src/eckit/io/s3/S3Bucket.cc create mode 100644 src/eckit/io/s3/S3Bucket.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 046b0a2cf..e6bd3aa7f 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -310,6 +310,8 @@ io/s3/S3Session.h io/s3/S3Macros.h io/s3/S3URIManager.cc io/s3/S3URIManager.h +io/s3/S3Bucket.cc +io/s3/S3Bucket.h ) endif(HAVE_AWS_S3) diff --git a/src/eckit/io/s3/S3Bucket.cc b/src/eckit/io/s3/S3Bucket.cc new file mode 100644 index 000000000..5e86e456e --- /dev/null +++ b/src/eckit/io/s3/S3Bucket.cc @@ -0,0 +1,74 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Bucket.h" + +// #include "eckit/config/LibEcKit.h" +// #include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3Client.h" +// #include "eckit/io/s3/S3Exception.h" +// #include "eckit/io/s3/S3Handle.h" +// #include "eckit/utils/Tokenizer.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3Bucket::S3Bucket(const URI& uri): client_(eckit::S3Client::makeShared({uri})) { + parse(uri.name()); +} + +S3Bucket::S3Bucket(const eckit::net::Endpoint& endpoint, const std::string& name): + client_(S3Client::makeShared({endpoint})), name_(name) { } + +//---------------------------------------------------------------------------------------------------------------------- + +void S3Bucket::parse(const std::string& path) { + + ASSERT(path.find("/") == std::string::npos); + + name_ = path; + +} + +void S3Bucket::print(std::ostream& out) const { + out << "S3Bucket[bucket=" << name_ << "]"; +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Bucket::exists() const -> bool { + return client_->bucketExists(name_); +} + +void S3Bucket::create() { + return client_->createBucket(name_); +} + +void S3Bucket::destroy() { + return client_->deleteBucket(name_); +} + +std::vector S3Bucket::listObjects() const { + std::vector objects; + for (auto obj : client_->listObjects(name_)) { + objects.push_back(eckit::S3Name{*this, obj}); + } + return objects; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Bucket.h b/src/eckit/io/s3/S3Bucket.h new file mode 100644 index 000000000..d0c8bd883 --- /dev/null +++ b/src/eckit/io/s3/S3Bucket.h @@ -0,0 +1,92 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3Bucket.h +/// @author Metin Cakircali +/// @author Simon Smart +/// @date Jan 2024 + +#pragma once + +#include +#include + +#include "eckit/io/Length.h" +#include "eckit/net/Endpoint.h" + +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Name.h" + +namespace eckit { + +class URI; +class DataHandle; +class S3Client; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Bucket { +public: // methods + + explicit S3Bucket(const URI& uri); + explicit S3Bucket(const eckit::net::Endpoint& endpoint, const std::string& name); + + auto name() const -> const std::string& { return name_; } + + auto exists() const -> bool; + + // Create the bucket on the + // n.b. throws if bucket already exists + void create(); + + // Destroy the bucket. + // n.b. throws if bucket does not exist + void destroy(); + + // Ensure that the bucket is created. Already existing is not a problem + // (this is very useful in a multi-client environment) + void ensureCreated() { NOTIMP; }; + + // Similarly for destroying bucket + void ensureDestroyed() { NOTIMP; }; + + // List the objects contained inside the bucket + // This function can have variants according to the various S3 functionality available + // e.g. selecting by prefix + // + // TO DISCUSS: How do we handle there being lots of keys. Really we want to return an + // iterator that internally handles the relevant pagination. + std::vector listObjects() const; + + // friend std::ostream& operator<<(std::ostream& out, const S3Bucket& name) { + // name.print(out); + // return out; + // } + + auto asString() const -> const std::string& { return name_; } + +private: // methods + void parse(const std::string& path); + + void print(std::ostream& out) const; + +private: // members + std::shared_ptr client_; + std::string name_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Config.cc b/src/eckit/io/s3/S3Config.cc index d09604880..b78aef069 100644 --- a/src/eckit/io/s3/S3Config.cc +++ b/src/eckit/io/s3/S3Config.cc @@ -26,6 +26,8 @@ S3Config::S3Config(std::string region, const std::string& hostname, const int po S3Config::S3Config(const URI& uri): endpoint(uri) { } +S3Config::S3Config(const net::Endpoint& e): endpoint(e) { } + //---------------------------------------------------------------------------------------------------------------------- void S3Config::print(std::ostream& out) const { diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 8c3e6a005..4ccf2eff8 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -34,6 +34,7 @@ struct S3Config { S3Config(std::string region, const URI& uri); S3Config(std::string region, const std::string& hostname, int port); S3Config(const URI& uri); + S3Config(const net::Endpoint&); friend std::ostream& operator<<(std::ostream& out, const S3Config& config) { config.print(out); diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 366507694..874b86de5 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -20,6 +20,7 @@ #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Handle.h" +#include "eckit/io/s3/S3Bucket.h" #include "eckit/utils/Tokenizer.h" namespace eckit { @@ -30,6 +31,9 @@ S3Name::S3Name(const URI& uri): client_(S3Client::makeShared({uri})) { parse(uri.name()); } +S3Name::S3Name(const eckit::S3Bucket& bucket, const std::string& key) : + bucket_(bucket.name()), object_(key) {} + //---------------------------------------------------------------------------------------------------------------------- void S3Name::parse(const std::string& path) { diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 3ed26e192..b019813bd 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -30,6 +30,7 @@ namespace eckit { class URI; class DataHandle; class S3Client; +class S3Bucket; //---------------------------------------------------------------------------------------------------------------------- @@ -37,6 +38,8 @@ class S3Name { public: // methods explicit S3Name(const URI& uri); + S3Name(const eckit::S3Bucket&, const std::string& key); + auto put(const void* buffer, long length) const -> long; auto get(void* buffer, long length) const -> long; From 4cfdb360bba92a117db1fc7318864d8e9c032b13 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Feb 2024 23:27:10 +0100 Subject: [PATCH 45/90] fix(S3): bug fixes and test for S3Handle, S3Client, S3Name --- src/eckit/io/s3/S3Client.h | 7 +- src/eckit/io/s3/S3Handle.cc | 57 +++++++------ src/eckit/io/s3/S3Handle.h | 2 +- src/eckit/io/s3/S3Name.cc | 9 ++- src/eckit/io/s3/S3Name.h | 6 +- src/eckit/io/s3/aws/S3ClientAWS.cc | 124 +++++++++++++---------------- src/eckit/io/s3/aws/S3ClientAWS.h | 6 +- tests/io/test_s3handle.cc | 24 ++++-- 8 files changed, 122 insertions(+), 113 deletions(-) diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 7f9055486..b6b3fc651 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -19,6 +19,7 @@ #pragma once +#include "eckit/io/Length.h" #include "eckit/io/s3/S3Config.h" #include "eckit/memory/NonCopyable.h" @@ -51,13 +52,11 @@ class S3Client: private NonCopyable { virtual auto listBuckets() const -> std::vector = 0; - virtual void putObject(const std::string& bucket, const std::string& object) const = 0; - virtual auto putObject(const std::string& bucket, const std::string& object, const void* buffer, - uint64_t length) const -> long long = 0; + uint64_t length) const -> Length = 0; virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t offset, - uint64_t length) const -> long long = 0; + uint64_t length) const -> Length = 0; virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0; diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 372721658..82e5925ff 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -27,61 +27,70 @@ S3Handle::S3Handle(const S3Name& name): S3Handle(name, 0) { } S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { } void S3Handle::print(std::ostream& out) const { - out << "S3Handle[name=" << name_ << ",position=" << pos_ << ", open=" << open_ << ", writable=" << canWrite_ << "]"; + out << "S3Handle[name=" << name_ << ", position=" << pos_ << ", open=" << open_ << ", writable=" << canWrite_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- -void S3Handle::open(const Offset& position) { +void S3Handle::open(const bool canWrite) { ASSERT(!open_); - pos_ = position; - open_ = true; + pos_ = 0; + open_ = true; + canWrite_ = canWrite; } Length S3Handle::openForRead() { - ASSERT(name_.exists()); - - open(0); - - canWrite_ = false; + open(false); return size(); } void S3Handle::openForWrite(const Length& length) { - open(Offset(length)); + open(true); - canWrite_ = true; + if (name_.exists()) { ASSERT(size() == length); } } void S3Handle::openForAppend(const Length& length) { - open(Offset(length)); + open(true); + + const auto oSize = size(); - canWrite_ = true; + ASSERT(oSize == length); + + pos_ += oSize; } //---------------------------------------------------------------------------------------------------------------------- long S3Handle::read(void* buffer, const long length) { - if (!open_) { throw S3SeriousBug("S3 handle is not open!", Here()); } + ASSERT(open_); + ASSERT(!canWrite_); + + const auto oSize = size(); + + if (pos_ == oSize) { return 0; } + + const auto rLength = name_.get(buffer, pos_, length); - const auto len = name_.get(buffer, pos_, length); + pos_ += rLength; - pos_ += len; + ASSERT(pos_ >= Offset(0) || pos_ <= oSize); - return len; + return rLength; } long S3Handle::write(const void* buffer, const long length) { - ASSERT(!name_.exists()); + ASSERT(open_); + ASSERT(canWrite_); - if (!open_ || !canWrite_) { throw S3SeriousBug("S3 handle is not writable!", Here()); } + const auto rLength = name_.put(buffer, length); - const auto len = name_.put(buffer, length); + pos_ += rLength; - pos_ += len; + ASSERT(pos_ >= Offset(0) || pos_ <= size()); - return len; + return rLength; } void S3Handle::flush() { @@ -89,12 +98,13 @@ void S3Handle::flush() { } void S3Handle::close() { + pos_ = 0; open_ = false; canWrite_ = false; } Length S3Handle::size() { - return name_.size(); // asserts + return name_.size(); } Length S3Handle::estimate() { @@ -107,6 +117,7 @@ Offset S3Handle::position() { Offset S3Handle::seek(const Offset& offset) { pos_ = pos_ + offset; + ASSERT(pos_ >= Offset(0) || pos_ <= size()); return pos_; } diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index ec3d79729..a33b5db2a 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -59,7 +59,7 @@ class S3Handle: public DataHandle { private: // methods void print(std::ostream& out) const override; - void open(const Offset& position); + void open(bool canWrite); private: // members const S3Name name_; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 4b6559f06..955ebb60c 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -48,11 +48,16 @@ void S3Name::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- -auto S3Name::put(const void* buffer, const long length) const -> long { +// auto S3Name::create() const -> Length { +// if (exists()) { return size(); } +// return client_->putObject(bucket_, object_, nullptr, 0); +// } + +auto S3Name::put(const void* buffer, const long length) const -> Length { return client_->putObject(bucket_, object_, buffer, length); } -auto S3Name::get(void* buffer, const long offset, const long length) const -> long { +auto S3Name::get(void* buffer, const long offset, const long length) const -> Length { return client_->getObject(bucket_, object_, buffer, offset, length); } diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 104c0b598..d64b3f6a0 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -37,9 +37,11 @@ class S3Name { public: // methods explicit S3Name(const URI& uri); - auto put(const void* buffer, long length) const -> long; + // auto create() const -> Length; - auto get(void* buffer, long offset, long length) const -> long; + auto put(const void* buffer, long length) const -> Length; + + auto get(void* buffer, long offset, long length) const -> Length; auto bucket() const -> const std::string& { return bucket_; } diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 572579981..3e7bee586 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -120,12 +120,12 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { auto outcome = client_->CreateBucket(request); - if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; - } else { + if (!outcome.IsSuccess()) { auto msg = awsErrorMessage("Failed to create bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } + + LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; } void S3ClientAWS::emptyBucket(const std::string& bucket) const { @@ -138,88 +138,71 @@ void S3ClientAWS::deleteBucket(const std::string& bucket) const { auto outcome = client_->DeleteBucket(request); - if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucket << std::endl; - } else { + if (!outcome.IsSuccess()) { auto msg = awsErrorMessage("Failed to delete bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } + + LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucket << std::endl; } auto S3ClientAWS::bucketExists(const std::string& bucket) const -> bool { Aws::S3::Model::HeadBucketRequest request; + request.SetBucket(bucket); + return client_->HeadBucket(request).IsSuccess(); } auto S3ClientAWS::listBuckets() const -> std::vector { - std::vector buckets; - auto outcome = client_->ListBuckets(); if (outcome.IsSuccess()) { + std::vector buckets; for (const auto& bucket : outcome.GetResult().GetBuckets()) { buckets.emplace_back(bucket.GetName()); } - } else { - Log::warning() << "Failed to list buckets!" << outcome.GetError(); + return buckets; } - return buckets; + auto msg = awsErrorMessage("Failed list buckets!", outcome.GetError()); + throw S3SeriousBug(msg, Here()); } //---------------------------------------------------------------------------------------------------------------------- // PUT OBJECT -void S3ClientAWS::putObject(const std::string& bucket, const std::string& object) const { - Aws::S3::Model::PutObjectRequest request; - - request.SetBucket(bucket); - request.SetKey(object); - - // empty object - request.SetBody(Aws::MakeShared(ALLOC_TAG)); - - auto outcome = client_->PutObject(request); - - if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Added object=" << object << " to bucket=" << bucket << std::endl; - } else { - auto msg = awsErrorMessage("Failed to put object=" + object + " to bucket=" + bucket, outcome.GetError()); - throw S3SeriousBug(msg, Here()); - } -} - auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object, const void* buffer, - const uint64_t length) const -> long long { + const uint64_t length) const -> Length { Aws::S3::Model::PutObjectRequest request; request.SetBucket(bucket); request.SetKey(object); // request.SetContentLength(length); - auto streamBuffer = Aws::MakeShared(ALLOC_TAG, const_cast(buffer), length); - request.SetBody(streamBuffer); + if (buffer && length > 0) { + auto streamBuffer = Aws::MakeShared(ALLOC_TAG, const_cast(buffer), length); + request.SetBody(streamBuffer); + } else { + // empty object + request.SetBody(Aws::MakeShared(ALLOC_TAG)); + } auto outcome = client_->PutObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Added object=" << object << " to bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Put object=" << object << " [len=" << length << "] to bucket=" << bucket << std::endl; /// @todo actual size of written bytes return length; - // return request.GetContentLength(); - } else { - auto msg = awsErrorMessage("Failed to put object=" + object + " to bucket=" + bucket, outcome.GetError()); - throw S3SeriousBug(msg, Here()); } + + auto msg = awsErrorMessage("Failed to put object=" + object + " to bucket=" + bucket, outcome.GetError()); + throw S3SeriousBug(msg, Here()); } //---------------------------------------------------------------------------------------------------------------------- // GET OBJECT auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object, void* buffer, const uint64_t offset, - const uint64_t length) const -> long long { - // no throw; POSIX - if (length == 0) { return 0; } - + const uint64_t length) const -> Length { Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket); @@ -230,9 +213,9 @@ auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object auto outcome = client_->GetObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Retrieved object=" << object << " from bucket=" << bucket << std::endl; - - /// @todo (try to check) if EOF and return 0; according to POSIX + LOG_DEBUG_LIB(LibEcKit) << "Get object=" << object << " from bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Req. len=" << length << ", Obj. len=" << outcome.GetResult().GetContentLength() + << std::endl; return outcome.GetResult().GetContentLength(); } @@ -246,16 +229,17 @@ auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& object) const { Aws::S3::Model::DeleteObjectRequest request; - request.WithKey(object).WithBucket(bucket); + request.SetBucket(bucket); + request.SetKey(object); auto outcome = client_->DeleteObject(request); - if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << " in bucket=" << bucket << std::endl; - } else { + if (!outcome.IsSuccess()) { auto msg = awsErrorMessage("Failed to delete object=" + object + " in bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } + + LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << " in bucket=" << bucket << std::endl; } void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vector& objects) const { @@ -276,56 +260,56 @@ void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vectorDeleteObjects(request); - if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Deleted " << objects.size() << " objects in bucket=" << bucket << std::endl; - for (const auto& object : objects) { LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << std::endl; } - } else { + if (!outcome.IsSuccess()) { auto msg = awsErrorMessage("Failed to delete objects in bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } + + LOG_DEBUG_LIB(LibEcKit) << "Deleted " << objects.size() << " objects in bucket=" << bucket << std::endl; + for (const auto& object : objects) { LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << std::endl; } } auto S3ClientAWS::listObjects(const std::string& bucket) const -> std::vector { - std::vector result; - Aws::S3::Model::ListObjectsRequest request; - request.WithBucket(bucket); + request.SetBucket(bucket); auto outcome = client_->ListObjects(request); if (outcome.IsSuccess()) { + std::vector result; + const auto& objects = outcome.GetResult().GetContents(); for (const auto& object : objects) { result.emplace_back(object.GetKey()); } - } else { - Log::warning() << "Failed to list objects in bucket=" << bucket << outcome.GetError(); + + return result; } - return result; + auto msg = awsErrorMessage("Failed to list objects in bucket=" + bucket, outcome.GetError()); + throw S3SeriousBug(msg, Here()); } auto S3ClientAWS::objectExists(const std::string& bucket, const std::string& object) const -> bool { Aws::S3::Model::HeadObjectRequest request; - request.WithKey(object).WithBucket(bucket); + + request.SetBucket(bucket); + request.SetKey(object); + return client_->HeadObject(request).IsSuccess(); } auto S3ClientAWS::objectSize(const std::string& bucket, const std::string& object) const -> Length { - Length result; - Aws::S3::Model::HeadObjectRequest request; - request.WithKey(object).WithBucket(bucket); + + request.SetBucket(bucket); + request.SetKey(object); auto outcome = client_->HeadObject(request); - if (outcome.IsSuccess()) { - result = outcome.GetResult().GetContentLength(); - } else { - const auto msg = awsErrorMessage("Object '" + object + "' doesn't exist or no access!", outcome.GetError()); - throw S3SeriousBug(msg, Here()); - } + if (outcome.IsSuccess()) { return outcome.GetResult().GetContentLength(); } - return result; + const auto msg = awsErrorMessage("Object '" + object + "' doesn't exist or no access!", outcome.GetError()); + throw S3SeriousBug(msg, Here()); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 0aaf0f3e9..70d38a5ae 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -45,13 +45,11 @@ class S3ClientAWS: public S3Client { auto listBuckets() const -> std::vector override; - void putObject(const std::string& bucket, const std::string& object) const override; - auto putObject(const std::string& bucket, const std::string& object, const void* buffer, uint64_t length) const - -> long long override; + -> Length override; auto getObject(const std::string& bucket, const std::string& object, void* buffer, const uint64_t offset, - const uint64_t length) const -> long long override; + const uint64_t length) const -> Length override; void deleteObject(const std::string& bucket, const std::string& object) const override; diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 4479700c6..aae8ec473 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -57,12 +57,10 @@ void bucketSetup() { client->createBucket(TEST_BUCKET); } -CASE("S3Handle") { +CASE("bucket exists") { const void* buffer = TEST_DATA.data(); const long length = TEST_DATA.size(); - std::cout << "write buffer: " << TEST_DATA << std::endl; - URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); { @@ -76,14 +74,17 @@ CASE("S3Handle") { { std::unique_ptr h(uri.newReadHandle()); - h->openForRead(); + + const auto rlen = h->openForRead(); + EXPECT(rlen == Length(length)); std::string rbuf; rbuf.resize(length); const auto len = h->read(rbuf.data(), length); - - // EXPECT(len == 0); // POSIX + /// @todo this is odd with respect to saveInto issue below + // EXPECT(len == 0); // POSIX + EXPECT(len == length); EXPECT(rbuf == TEST_DATA); } @@ -92,7 +93,7 @@ CASE("S3Handle") { std::unique_ptr dh(uri.newReadHandle()); MemoryHandle mh(length); - /// @todo this creates issues because internal buffer size is set to 64*1024*1024 + /// @todo this creates request loop issues because default internal buffer size is 64MiB dh->saveInto(mh); EXPECT(mh.size() == Length(length)); @@ -100,6 +101,15 @@ CASE("S3Handle") { } } +CASE("bucket does not exist") { + URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + + ensureClean(); + + std::unique_ptr h(uri.newReadHandle()); + EXPECT_THROWS(h->openForRead()); +} + /// @todo Also check that it doesn't work if the bucket doesn't exist, etc. //---------------------------------------------------------------------------------------------------------------------- From bd245ef32e6f7f3c11f78c1df811c704216209d2 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 14 Feb 2024 23:33:52 +0100 Subject: [PATCH 46/90] refactor(S3): remove comment --- src/eckit/io/s3/S3Name.cc | 8 +------- src/eckit/io/s3/S3Name.h | 2 -- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index d4e533c5f..1297f4707 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -31,8 +31,7 @@ S3Name::S3Name(const URI& uri): client_(S3Client::makeShared({uri})) { parse(uri.name()); } -S3Name::S3Name(const eckit::S3Bucket& bucket, const std::string& key) : - bucket_(bucket.name()), object_(key) {} +S3Name::S3Name(const eckit::S3Bucket& bucket, const std::string& key): bucket_(bucket.name()), object_(key) { } //---------------------------------------------------------------------------------------------------------------------- @@ -52,11 +51,6 @@ void S3Name::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- -// auto S3Name::create() const -> Length { -// if (exists()) { return size(); } -// return client_->putObject(bucket_, object_, nullptr, 0); -// } - auto S3Name::put(const void* buffer, const long length) const -> Length { return client_->putObject(bucket_, object_, buffer, length); } diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 53f15b6c0..696406236 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -40,8 +40,6 @@ class S3Name { S3Name(const eckit::S3Bucket&, const std::string& key); - // auto create() const -> Length; - auto put(const void* buffer, long length) const -> Length; auto get(void* buffer, long offset, long length) const -> Length; From 0585ad765ec141b93da1f0f76ed4d8a9ba4426ed Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Wed, 14 Feb 2024 23:53:56 +0100 Subject: [PATCH 47/90] feat(S3): adds S3Bucket::ensureCreated and ensureDestroyed. --- src/eckit/io/s3/S3Bucket.cc | 14 +++++++++++++- src/eckit/io/s3/S3Bucket.h | 22 +++++++--------------- src/eckit/io/s3/S3Exception.cc | 11 +++++++++++ src/eckit/io/s3/S3Exception.h | 17 +++++++++++++++++ src/eckit/io/s3/aws/S3ClientAWS.cc | 9 +++++++++ 5 files changed, 57 insertions(+), 16 deletions(-) diff --git a/src/eckit/io/s3/S3Bucket.cc b/src/eckit/io/s3/S3Bucket.cc index 5e86e456e..5c964352a 100644 --- a/src/eckit/io/s3/S3Bucket.cc +++ b/src/eckit/io/s3/S3Bucket.cc @@ -18,7 +18,7 @@ // #include "eckit/config/LibEcKit.h" // #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" -// #include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Exception.h" // #include "eckit/io/s3/S3Handle.h" // #include "eckit/utils/Tokenizer.h" @@ -61,6 +61,18 @@ void S3Bucket::destroy() { return client_->deleteBucket(name_); } +void S3Bucket::ensureCreated() { + try { + create(); + } catch (eckit::S3EntityAlreadyExists& e) { } +} + +void S3Bucket::ensureDestroyed() { + try { + destroy(); + } catch (eckit::S3EntityNotFound& e) { } +} + std::vector S3Bucket::listObjects() const { std::vector objects; for (auto obj : client_->listObjects(name_)) { diff --git a/src/eckit/io/s3/S3Bucket.h b/src/eckit/io/s3/S3Bucket.h index d0c8bd883..83e73bfaf 100644 --- a/src/eckit/io/s3/S3Bucket.h +++ b/src/eckit/io/s3/S3Bucket.h @@ -47,27 +47,19 @@ class S3Bucket { auto exists() const -> bool; - // Create the bucket on the - // n.b. throws if bucket already exists + /// @note: throws if bucket already exists void create(); - // Destroy the bucket. - // n.b. throws if bucket does not exist + /// @note: throws if bucket does not exist void destroy(); - // Ensure that the bucket is created. Already existing is not a problem - // (this is very useful in a multi-client environment) - void ensureCreated() { NOTIMP; }; + void ensureCreated(); - // Similarly for destroying bucket - void ensureDestroyed() { NOTIMP; }; + void ensureDestroyed(); - // List the objects contained inside the bucket - // This function can have variants according to the various S3 functionality available - // e.g. selecting by prefix - // - // TO DISCUSS: How do we handle there being lots of keys. Really we want to return an - // iterator that internally handles the relevant pagination. + /// @todo: How do we handle there being lots of keys. + /// We want to return an iterator that internally handles the + /// relevant pagination. std::vector listObjects() const; // friend std::ostream& operator<<(std::ostream& out, const S3Bucket& name) { diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc index 0645ab2ac..2ad94d980 100644 --- a/src/eckit/io/s3/S3Exception.cc +++ b/src/eckit/io/s3/S3Exception.cc @@ -40,4 +40,15 @@ S3SeriousBug::S3SeriousBug(const std::string& msg, const int code, const CodeLoc //---------------------------------------------------------------------------------------------------------------------- +S3Exception::S3Exception(const std::string& w, const eckit::CodeLocation& l) : + Exception(w, l) {} + +S3EntityNotFound::S3EntityNotFound(const std::string& w, const eckit::CodeLocation& l) : + S3Exception(w, l) {} + +S3EntityAlreadyExists::S3EntityAlreadyExists(const std::string& w, const eckit::CodeLocation& l) : + S3Exception(w, l) {} + +//---------------------------------------------------------------------------------------------------------------------- + } // namespace eckit diff --git a/src/eckit/io/s3/S3Exception.h b/src/eckit/io/s3/S3Exception.h index d53b6272b..88f8bbb00 100644 --- a/src/eckit/io/s3/S3Exception.h +++ b/src/eckit/io/s3/S3Exception.h @@ -38,4 +38,21 @@ class S3SeriousBug: public SeriousBug { //---------------------------------------------------------------------------------------------------------------------- +class S3Exception : public eckit::Exception { +public: + S3Exception(const std::string&, const eckit::CodeLocation&); +}; + +class S3EntityNotFound : public S3Exception { +public: + S3EntityNotFound(const std::string&, const eckit::CodeLocation&); +}; + +class S3EntityAlreadyExists : public S3Exception { +public: + S3EntityAlreadyExists(const std::string&, const eckit::CodeLocation&); +}; + +//---------------------------------------------------------------------------------------------------------------------- + } // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 572579981..4e128fe2f 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -122,6 +122,12 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; + } else if ( + outcome.GetError().GetErrorType() == Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS || + outcome.GetError().GetErrorType() == Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU + ) { + auto msg = awsErrorMessage("Bucket already exists=" + bucket, outcome.GetError()); + throw S3EntityAlreadyExists(msg, Here()); } else { auto msg = awsErrorMessage("Failed to create bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); @@ -140,6 +146,9 @@ void S3ClientAWS::deleteBucket(const std::string& bucket) const { if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucket << std::endl; + } else if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_BUCKET) { + auto msg = awsErrorMessage("Bucket does not exist=" + bucket, outcome.GetError()); + throw S3EntityNotFound(msg, Here()); } else { auto msg = awsErrorMessage("Failed to delete bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); From ce3b2c76a76c48f87ff58a55b6c07310708819b1 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 15 Feb 2024 11:00:08 +0100 Subject: [PATCH 48/90] refactor(S3): cleanup test s3handle --- tests/io/test_s3handle.cc | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index aae8ec473..257fc36ab 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -51,13 +51,11 @@ void ensureClean() { } } -void bucketSetup() { +CASE("bucket exists") { ensureClean(); - auto client = S3Client::makeUnique(cfg); - client->createBucket(TEST_BUCKET); -} -CASE("bucket exists") { + S3Client::makeUnique(cfg)->createBucket(TEST_BUCKET); + const void* buffer = TEST_DATA.data(); const long length = TEST_DATA.size(); @@ -110,8 +108,6 @@ CASE("bucket does not exist") { EXPECT_THROWS(h->openForRead()); } -/// @todo Also check that it doesn't work if the bucket doesn't exist, etc. - //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test @@ -122,9 +118,8 @@ int main(int argc, char** argv) { int ret = -1; try { - eckit::test::bucketSetup(); - ret = run_tests(argc, argv); eckit::test::ensureClean(); + ret = run_tests(argc, argv); } catch (...) {} return ret; } From 5670806c961d29d54cbfd546953c2c7f7dc68f60 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 15 Feb 2024 21:51:30 +0100 Subject: [PATCH 49/90] fix(S3)!: client return type long long --- src/eckit/io/s3/S3Client.h | 7 +++---- src/eckit/io/s3/aws/S3ClientAWS.cc | 12 +++++------- src/eckit/io/s3/aws/S3ClientAWS.h | 6 +++--- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index b6b3fc651..8a198e78c 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -19,7 +19,6 @@ #pragma once -#include "eckit/io/Length.h" #include "eckit/io/s3/S3Config.h" #include "eckit/memory/NonCopyable.h" @@ -53,10 +52,10 @@ class S3Client: private NonCopyable { virtual auto listBuckets() const -> std::vector = 0; virtual auto putObject(const std::string& bucket, const std::string& object, const void* buffer, - uint64_t length) const -> Length = 0; + uint64_t length) const -> long long = 0; virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t offset, - uint64_t length) const -> Length = 0; + uint64_t length) const -> long long = 0; virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0; @@ -66,7 +65,7 @@ class S3Client: private NonCopyable { virtual auto objectExists(const std::string& bucket, const std::string& object) const -> bool = 0; - virtual auto objectSize(const std::string& bucket, const std::string& object) const -> Length = 0; + virtual auto objectSize(const std::string& bucket, const std::string& object) const -> long long = 0; friend std::ostream& operator<<(std::ostream& out, const S3Client& client) { client.print(out); diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 88b9da33b..cea6bd78a 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -122,10 +122,8 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; - } else if ( - outcome.GetError().GetErrorType() == Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS || - outcome.GetError().GetErrorType() == Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU - ) { + } else if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS || + outcome.GetError().GetErrorType() == Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) { auto msg = awsErrorMessage("Bucket already exists=" + bucket, outcome.GetError()); throw S3EntityAlreadyExists(msg, Here()); } else { @@ -184,7 +182,7 @@ auto S3ClientAWS::listBuckets() const -> std::vector { // PUT OBJECT auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object, const void* buffer, - const uint64_t length) const -> Length { + const uint64_t length) const -> long long { Aws::S3::Model::PutObjectRequest request; request.SetBucket(bucket); @@ -215,7 +213,7 @@ auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object // GET OBJECT auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object, void* buffer, const uint64_t offset, - const uint64_t length) const -> Length { + const uint64_t length) const -> long long { Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket); @@ -311,7 +309,7 @@ auto S3ClientAWS::objectExists(const std::string& bucket, const std::string& obj return client_->HeadObject(request).IsSuccess(); } -auto S3ClientAWS::objectSize(const std::string& bucket, const std::string& object) const -> Length { +auto S3ClientAWS::objectSize(const std::string& bucket, const std::string& object) const -> long long { Aws::S3::Model::HeadObjectRequest request; request.SetBucket(bucket); diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 70d38a5ae..aaf7a3c3d 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -46,10 +46,10 @@ class S3ClientAWS: public S3Client { auto listBuckets() const -> std::vector override; auto putObject(const std::string& bucket, const std::string& object, const void* buffer, uint64_t length) const - -> Length override; + -> long long override; auto getObject(const std::string& bucket, const std::string& object, void* buffer, const uint64_t offset, - const uint64_t length) const -> Length override; + const uint64_t length) const -> long long override; void deleteObject(const std::string& bucket, const std::string& object) const override; @@ -59,7 +59,7 @@ class S3ClientAWS: public S3Client { auto objectExists(const std::string& bucket, const std::string& object) const -> bool override; - auto objectSize(const std::string& bucket, const std::string& object) const -> Length override; + auto objectSize(const std::string& bucket, const std::string& object) const -> long long override; private: // methods void print(std::ostream& out) const override; From 077e64ae9c6c03d2de1cf7c48236b723294969da Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 15 Feb 2024 21:55:18 +0100 Subject: [PATCH 50/90] fix(S3): assert fix and tidy up S3Handle, fix S3Name Length type --- src/eckit/CMakeLists.txt | 4 +- src/eckit/io/s3/S3Handle.cc | 76 +++++++++++++------------------------ src/eckit/io/s3/S3Handle.h | 13 ++++--- src/eckit/io/s3/S3Name.cc | 8 ++-- src/eckit/io/s3/S3Name.h | 10 ++--- 5 files changed, 44 insertions(+), 67 deletions(-) diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index e6bd3aa7f..8ef426ec3 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -292,6 +292,8 @@ io/s3/aws/S3ClientAWS.cc io/s3/aws/S3ClientAWS.h io/s3/aws/S3ContextAWS.cc io/s3/aws/S3ContextAWS.h +io/s3/S3Bucket.cc +io/s3/S3Bucket.h io/s3/S3Client.cc io/s3/S3Client.h io/s3/S3Config.cc @@ -310,8 +312,6 @@ io/s3/S3Session.h io/s3/S3Macros.h io/s3/S3URIManager.cc io/s3/S3URIManager.h -io/s3/S3Bucket.cc -io/s3/S3Bucket.h ) endif(HAVE_AWS_S3) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 82e5925ff..2e5d0405c 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -27,81 +27,61 @@ S3Handle::S3Handle(const S3Name& name): S3Handle(name, 0) { } S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { } void S3Handle::print(std::ostream& out) const { - out << "S3Handle[name=" << name_ << ", position=" << pos_ << ", open=" << open_ << ", writable=" << canWrite_ << "]"; + out << "S3Handle[name=" << name_ << ", position=" << pos_ << ", open=" << open_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- -void S3Handle::open(const bool canWrite) { - ASSERT(!open_); - pos_ = 0; - open_ = true; - canWrite_ = canWrite; -} - Length S3Handle::openForRead() { - open(false); + open(Mode::READ); - return size(); + return estimate(); } void S3Handle::openForWrite(const Length& length) { - open(true); - - if (name_.exists()) { ASSERT(size() == length); } -} + open(Mode::WRITE); -void S3Handle::openForAppend(const Length& length) { - open(true); + ASSERT(name_.bucketExists()); - const auto oSize = size(); - - ASSERT(oSize == length); - - pos_ += oSize; + if (name_.exists()) { ASSERT(size() == length); } } //---------------------------------------------------------------------------------------------------------------------- long S3Handle::read(void* buffer, const long length) { - ASSERT(open_); - ASSERT(!canWrite_); - - const auto oSize = size(); - - if (pos_ == oSize) { return 0; } - - const auto rLength = name_.get(buffer, pos_, length); - - pos_ += rLength; + ASSERT(open_ && mode_ == Mode::READ); - ASSERT(pos_ >= Offset(0) || pos_ <= oSize); + if (size() == pos_) { return 0; } - return rLength; + return seek(name_.get(buffer, pos_, length)); } long S3Handle::write(const void* buffer, const long length) { - ASSERT(open_); - ASSERT(canWrite_); + ASSERT(open_ && mode_ == Mode::WRITE); - const auto rLength = name_.put(buffer, length); + return seek(name_.put(buffer, length)); +} - pos_ += rLength; +//---------------------------------------------------------------------------------------------------------------------- - ASSERT(pos_ >= Offset(0) || pos_ <= size()); +void S3Handle::open(const Mode mode) { + ASSERT(!open_); + pos_ = 0; + open_ = true; + mode_ = mode; +} - return rLength; +void S3Handle::close() { + pos_ = 0; + open_ = false; + mode_ = Mode::NONE; } void S3Handle::flush() { NOTIMP; } -void S3Handle::close() { - pos_ = 0; - open_ = false; - canWrite_ = false; -} +//---------------------------------------------------------------------------------------------------------------------- Length S3Handle::size() { return name_.size(); @@ -111,13 +91,11 @@ Length S3Handle::estimate() { return size(); } -Offset S3Handle::position() { - return pos_; -} - Offset S3Handle::seek(const Offset& offset) { pos_ = pos_ + offset; - ASSERT(pos_ >= Offset(0) || pos_ <= size()); + + ASSERT(0 <= pos_ && size() >= pos_); + return pos_; } diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index a33b5db2a..ff2238bc1 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -28,6 +28,8 @@ namespace eckit { class S3Handle: public DataHandle { public: // methods + enum class Mode { NONE, READ, WRITE }; + explicit S3Handle(const S3Name& name); S3Handle(const S3Name& name, const Offset& offset); @@ -36,8 +38,6 @@ class S3Handle: public DataHandle { void openForWrite(const Length& length) override; - void openForAppend(const Length& length) override; - long read(void* buffer, long length) override; long write(const void* buffer, long length) override; @@ -50,16 +50,16 @@ class S3Handle: public DataHandle { Length estimate() override; - Offset position() override; + Offset position() override { return pos_; } Offset seek(const Offset& offset) override; - bool canSeek() const override { return true; } + auto canSeek() const -> bool override { return true; } private: // methods void print(std::ostream& out) const override; - void open(bool canWrite); + void open(Mode mode); private: // members const S3Name name_; @@ -67,7 +67,8 @@ class S3Handle: public DataHandle { Offset pos_ {0}; bool open_ {false}; - bool canWrite_ {false}; + + Mode mode_ {Mode::NONE}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 1297f4707..656f6f422 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -17,10 +17,10 @@ #include "eckit/config/LibEcKit.h" #include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3Bucket.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Handle.h" -#include "eckit/io/s3/S3Bucket.h" #include "eckit/utils/Tokenizer.h" namespace eckit { @@ -51,11 +51,11 @@ void S3Name::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- -auto S3Name::put(const void* buffer, const long length) const -> Length { +auto S3Name::put(const void* buffer, const long length) const -> long long { return client_->putObject(bucket_, object_, buffer, length); } -auto S3Name::get(void* buffer, const long offset, const long length) const -> Length { +auto S3Name::get(void* buffer, const long offset, const long length) const -> long long { return client_->getObject(bucket_, object_, buffer, offset, length); } @@ -67,7 +67,7 @@ auto S3Name::exists() const -> bool { return client_->objectExists(bucket_, object_); } -auto S3Name::size() const -> Length { +auto S3Name::size() const -> long long { return client_->objectSize(bucket_, object_); } diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 696406236..04869f426 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -20,8 +20,6 @@ #pragma once -#include "eckit/io/Length.h" - #include #include @@ -38,11 +36,11 @@ class S3Name { public: // methods explicit S3Name(const URI& uri); - S3Name(const eckit::S3Bucket&, const std::string& key); + S3Name(const S3Bucket&, const std::string& key); - auto put(const void* buffer, long length) const -> Length; + auto put(const void* buffer, long length) const -> long long; - auto get(void* buffer, long offset, long length) const -> Length; + auto get(void* buffer, long offset, long length) const -> long long; auto bucket() const -> const std::string& { return bucket_; } @@ -52,7 +50,7 @@ class S3Name { auto exists() const -> bool; - auto size() const -> Length; + auto size() const -> long long; [[nodiscard]] auto dataHandle() -> DataHandle*; From b4cc9de38e5f49258eaf18febb29d4ce5a0d89d7 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 15 Feb 2024 21:57:43 +0100 Subject: [PATCH 51/90] feat(S3): improve s3handle tests --- tests/io/test_s3handle.cc | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 257fc36ab..dd9d82894 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -39,8 +39,8 @@ static const std::string TEST_OBJECT("eckit-s3handle-test-object"); //---------------------------------------------------------------------------------------------------------------------- void ensureClean() { - auto client = S3Client::makeUnique(cfg); - auto&& tmp = client->listBuckets(); + auto client = S3Client::makeUnique(cfg); + auto&& tmp = client->listBuckets(); std::set buckets(tmp.begin(), tmp.end()); for (const std::string& name : {TEST_BUCKET}) { @@ -80,8 +80,6 @@ CASE("bucket exists") { rbuf.resize(length); const auto len = h->read(rbuf.data(), length); - /// @todo this is odd with respect to saveInto issue below - // EXPECT(len == 0); // POSIX EXPECT(len == length); EXPECT(rbuf == TEST_DATA); @@ -91,7 +89,6 @@ CASE("bucket exists") { std::unique_ptr dh(uri.newReadHandle()); MemoryHandle mh(length); - /// @todo this creates request loop issues because default internal buffer size is 64MiB dh->saveInto(mh); EXPECT(mh.size() == Length(length)); @@ -99,9 +96,26 @@ CASE("bucket exists") { } } -CASE("bucket does not exist") { +CASE("S3Handle::openForWrite") { + std::cout << "===================" << std::endl; + + ensureClean(); + URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + { + std::unique_ptr h(uri.newWriteHandle()); + // no bucket + EXPECT_THROWS(h->openForWrite(0)); + // no read + EXPECT_THROWS(h->openForRead()); + } + + { + std::unique_ptr h(uri.newWriteHandle()); + EXPECT_THROWS(h->openForWrite(0)); + } +} ensureClean(); std::unique_ptr h(uri.newReadHandle()); @@ -120,6 +134,6 @@ int main(int argc, char** argv) { try { eckit::test::ensureClean(); ret = run_tests(argc, argv); - } catch (...) {} + } catch (...) { } return ret; } From bec47da43dce8348e3d5fdb316feaf65d70387c0 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Fri, 16 Feb 2024 02:52:54 +0100 Subject: [PATCH 52/90] feat(S3): adds a few S3Name features and minor API modifications. --- src/eckit/io/s3/S3Bucket.cc | 42 ++++++++++++++++++++++++++----- src/eckit/io/s3/S3Bucket.h | 12 ++++++++- src/eckit/io/s3/S3Handle.cc | 11 +++++++-- src/eckit/io/s3/S3Handle.h | 6 ++++- src/eckit/io/s3/S3Name.cc | 49 ++++++++++++++++++++++++------------- src/eckit/io/s3/S3Name.h | 20 +++++++++++---- 6 files changed, 108 insertions(+), 32 deletions(-) diff --git a/src/eckit/io/s3/S3Bucket.cc b/src/eckit/io/s3/S3Bucket.cc index 5c964352a..76507b7d7 100644 --- a/src/eckit/io/s3/S3Bucket.cc +++ b/src/eckit/io/s3/S3Bucket.cc @@ -13,28 +13,58 @@ * (Project ID: 955811) iosea-project.eu */ -#include "eckit/io/s3/S3Bucket.h" - // #include "eckit/config/LibEcKit.h" // #include "eckit/filesystem/URI.h" -#include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Exception.h" // #include "eckit/io/s3/S3Handle.h" // #include "eckit/utils/Tokenizer.h" +#include "eckit/io/s3/S3Bucket.h" +#include "eckit/io/s3/S3Name.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Exception.h" + namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Bucket::S3Bucket(const URI& uri): client_(eckit::S3Client::makeShared({uri})) { +S3Bucket::S3Bucket(const URI& uri): + client_(eckit::S3Client::makeShared({uri})), + endpoint_(uri) { + parse(uri.name()); + } S3Bucket::S3Bucket(const eckit::net::Endpoint& endpoint, const std::string& name): - client_(S3Client::makeShared({endpoint})), name_(name) { } + client_(S3Client::makeShared({endpoint})), + endpoint_(endpoint), + name_(name) { } + +S3Bucket::S3Bucket(const S3Bucket& other) : + client_(S3Client::makeShared({other.endpoint_})), + endpoint_(other.endpoint_), + name_(other.name_) { } + +S3Bucket& S3Bucket::operator=(S3Bucket&& other) { + + client_ = std::move(other.client_); + endpoint_ = std::move(other.endpoint_); + name_ = std::move(other.name_); + + return *this; + +} //---------------------------------------------------------------------------------------------------------------------- +auto S3Bucket::uri() const -> eckit::URI { + + eckit::URI u{"s3", endpoint_.host(), endpoint_.port()}; + u.path(name_); + return u; + +} + void S3Bucket::parse(const std::string& path) { ASSERT(path.find("/") == std::string::npos); diff --git a/src/eckit/io/s3/S3Bucket.h b/src/eckit/io/s3/S3Bucket.h index 83e73bfaf..19e75031e 100644 --- a/src/eckit/io/s3/S3Bucket.h +++ b/src/eckit/io/s3/S3Bucket.h @@ -27,13 +27,13 @@ #include "eckit/net/Endpoint.h" #include "eckit/io/s3/S3Exception.h" -#include "eckit/io/s3/S3Name.h" namespace eckit { class URI; class DataHandle; class S3Client; +class S3Name; //---------------------------------------------------------------------------------------------------------------------- @@ -42,9 +42,13 @@ class S3Bucket { explicit S3Bucket(const URI& uri); explicit S3Bucket(const eckit::net::Endpoint& endpoint, const std::string& name); + S3Bucket(const S3Bucket&); + S3Bucket& operator=(S3Bucket&&); auto name() const -> const std::string& { return name_; } + auto uri() const -> eckit::URI; + auto exists() const -> bool; /// @note: throws if bucket already exists @@ -70,12 +74,18 @@ class S3Bucket { auto asString() const -> const std::string& { return name_; } private: // methods + + friend class S3Name; + + S3Bucket() {} + void parse(const std::string& path); void print(std::ostream& out) const; private: // members std::shared_ptr client_; + eckit::net::Endpoint endpoint_; std::string name_; }; diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 2e5d0405c..044fe7458 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -22,9 +22,10 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Handle::S3Handle(const S3Name& name): S3Handle(name, 0) { } +S3Handle::S3Handle(const S3Name& name): name_(name), pos_(0), readonly_(false) { } -S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { } +S3Handle::S3Handle(const S3Name& name, const Offset& offset, const Length& length): + name_(name), pos_(offset), len_(length), readonly_(true) { } void S3Handle::print(std::ostream& out) const { out << "S3Handle[name=" << name_ << ", position=" << pos_ << ", open=" << open_ << "]"; @@ -39,6 +40,7 @@ Length S3Handle::openForRead() { } void S3Handle::openForWrite(const Length& length) { + ASSERT(!readonly_); open(Mode::WRITE); ASSERT(name_.bucketExists()); @@ -84,7 +86,12 @@ void S3Handle::flush() { //---------------------------------------------------------------------------------------------------------------------- Length S3Handle::size() { + + if (readonly_) + return len_; + return name_.size(); + } Length S3Handle::estimate() { diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index ff2238bc1..3322fbea8 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -32,7 +32,7 @@ class S3Handle: public DataHandle { explicit S3Handle(const S3Name& name); - S3Handle(const S3Name& name, const Offset& offset); + S3Handle(const S3Name& name, const Offset& offset, const Length& length); Length openForRead() override; @@ -66,6 +66,10 @@ class S3Handle: public DataHandle { Offset pos_ {0}; + Length len_; + + bool readonly_; + bool open_ {false}; Mode mode_ {Mode::NONE}; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 656f6f422..d902733dc 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -17,7 +17,6 @@ #include "eckit/config/LibEcKit.h" #include "eckit/filesystem/URI.h" -#include "eckit/io/s3/S3Bucket.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Handle.h" @@ -28,55 +27,71 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- S3Name::S3Name(const URI& uri): client_(S3Client::makeShared({uri})) { - parse(uri.name()); -} - -S3Name::S3Name(const eckit::S3Bucket& bucket, const std::string& key): bucket_(bucket.name()), object_(key) { } -//---------------------------------------------------------------------------------------------------------------------- - -void S3Name::parse(const std::string& path) { - const auto pairs = Tokenizer("/").tokenize(path); + const auto pairs = Tokenizer("/").tokenize(uri.name()); const auto pSize = pairs.size(); ASSERT(pSize == 1 || pSize == 2); - if (pSize > 0) { bucket_ = pairs[0]; } + bucket_ = S3Bucket{eckit::net::Endpoint{uri}, pairs[0]}; if (pSize > 1) { object_ = pairs[1]; } + } +S3Name::S3Name(const eckit::S3Bucket& bucket, const std::string& key): bucket_(bucket), object_(key) { } + +//---------------------------------------------------------------------------------------------------------------------- + void S3Name::print(std::ostream& out) const { - out << "S3Name[bucket=" << bucket_ << ",object=" << object_ << "]"; + out << "S3Name[bucket=" << bucket_.name() << ",object=" << object_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- auto S3Name::put(const void* buffer, const long length) const -> long long { - return client_->putObject(bucket_, object_, buffer, length); + return client_->putObject(bucket_.name(), object_, buffer, length); } auto S3Name::get(void* buffer, const long offset, const long length) const -> long long { - return client_->getObject(bucket_, object_, buffer, offset, length); + return client_->getObject(bucket_.name(), object_, buffer, offset, length); +} + +void S3Name::destroy() { + + client_->deleteObject(bucket_.name(), object_); + +} + +auto S3Name::uri() const -> eckit::URI { + + eckit::URI u{bucket_.uri()}; + u.path(u.name() + "/" + object_); + return u; + } auto S3Name::bucketExists() const -> bool { - return client_->bucketExists(bucket_); + return client_->bucketExists(bucket_.name()); } auto S3Name::exists() const -> bool { - return client_->objectExists(bucket_, object_); + return client_->objectExists(bucket_.name(), object_); } auto S3Name::size() const -> long long { - return client_->objectSize(bucket_, object_); + return client_->objectSize(bucket_.name(), object_); } auto S3Name::dataHandle() -> DataHandle* { return new S3Handle(*this); } +auto S3Name::dataHandle(const eckit::Offset& offset, const eckit::Length& length) -> DataHandle* { + return new S3Handle(*this, offset, length); +} + auto S3Name::asString() const -> std::string { - return bucket_ + "/" + object_; + return bucket_.name() + "/" + object_; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 04869f426..f2867bdf9 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -23,12 +23,13 @@ #include #include +#include "eckit/io/DataHandle.h" +#include "eckit/io/s3/S3Bucket.h" + namespace eckit { class URI; -class DataHandle; class S3Client; -class S3Bucket; //---------------------------------------------------------------------------------------------------------------------- @@ -38,13 +39,20 @@ class S3Name { S3Name(const S3Bucket&, const std::string& key); + S3Name(const net::Endpoint&, const std::string& bucket, const std::string& key); + auto put(const void* buffer, long length) const -> long long; auto get(void* buffer, long offset, long length) const -> long long; - auto bucket() const -> const std::string& { return bucket_; } + void destroy(); + + auto bucket() const -> const S3Bucket& { return bucket_; } auto object() const -> const std::string& { return object_; } + auto name() const -> const std::string& { return object_; } + + auto uri() const -> eckit::URI; auto bucketExists() const -> bool; @@ -55,6 +63,9 @@ class S3Name { [[nodiscard]] auto dataHandle() -> DataHandle*; + [[nodiscard]] + auto dataHandle(const eckit::Offset&, const eckit::Length&) -> DataHandle*; + auto asString() const -> std::string; friend std::ostream& operator<<(std::ostream& out, const S3Name& name) { @@ -63,12 +74,11 @@ class S3Name { } private: // methods - void parse(const std::string& path); void print(std::ostream& out) const; private: // members - std::string bucket_; + eckit::S3Bucket bucket_; std::string object_; std::shared_ptr client_; From 6d01bf8edac7a6f8a37184653aaeeb11583da6b0 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 16 Feb 2024 11:07:43 +0100 Subject: [PATCH 53/90] fix(S3): removed unnecessary stuff from S3Handle --- src/eckit/io/s3/S3Handle.cc | 31 +++++++++++++++---------------- src/eckit/io/s3/S3Handle.h | 12 +++--------- src/eckit/io/s3/S3Name.cc | 4 ++-- src/eckit/io/s3/S3Name.h | 2 +- 4 files changed, 21 insertions(+), 28 deletions(-) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 044fe7458..f3ddc6c2b 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -22,13 +22,20 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Handle::S3Handle(const S3Name& name): name_(name), pos_(0), readonly_(false) { } +S3Handle::S3Handle(const S3Name& name): name_(name), pos_(0) { } -S3Handle::S3Handle(const S3Name& name, const Offset& offset, const Length& length): - name_(name), pos_(offset), len_(length), readonly_(true) { } +S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { } void S3Handle::print(std::ostream& out) const { - out << "S3Handle[name=" << name_ << ", position=" << pos_ << ", open=" << open_ << "]"; + out << "S3Handle[name=" << name_ << ", position=" << pos_; + if (mode_ == Mode::CLOSED) { + out << ", mode=closed"; + } else if (mode_ == Mode::READ) { + out << ", mode=read"; + } else if (mode_ == Mode::WRITE) { + out << ", mode=write"; + } + out << "]"; } //---------------------------------------------------------------------------------------------------------------------- @@ -40,7 +47,6 @@ Length S3Handle::openForRead() { } void S3Handle::openForWrite(const Length& length) { - ASSERT(!readonly_); open(Mode::WRITE); ASSERT(name_.bucketExists()); @@ -51,7 +57,7 @@ void S3Handle::openForWrite(const Length& length) { //---------------------------------------------------------------------------------------------------------------------- long S3Handle::read(void* buffer, const long length) { - ASSERT(open_ && mode_ == Mode::READ); + ASSERT(mode_ == Mode::READ); if (size() == pos_) { return 0; } @@ -59,7 +65,7 @@ long S3Handle::read(void* buffer, const long length) { } long S3Handle::write(const void* buffer, const long length) { - ASSERT(open_ && mode_ == Mode::WRITE); + ASSERT(mode_ == Mode::WRITE); return seek(name_.put(buffer, length)); } @@ -67,16 +73,14 @@ long S3Handle::write(const void* buffer, const long length) { //---------------------------------------------------------------------------------------------------------------------- void S3Handle::open(const Mode mode) { - ASSERT(!open_); + ASSERT(mode_ == Mode::CLOSED); pos_ = 0; - open_ = true; mode_ = mode; } void S3Handle::close() { pos_ = 0; - open_ = false; - mode_ = Mode::NONE; + mode_ = Mode::CLOSED; } void S3Handle::flush() { @@ -86,12 +90,7 @@ void S3Handle::flush() { //---------------------------------------------------------------------------------------------------------------------- Length S3Handle::size() { - - if (readonly_) - return len_; - return name_.size(); - } Length S3Handle::estimate() { diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index 3322fbea8..0633f4fc8 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -28,11 +28,11 @@ namespace eckit { class S3Handle: public DataHandle { public: // methods - enum class Mode { NONE, READ, WRITE }; + enum class Mode { CLOSED, READ, WRITE }; explicit S3Handle(const S3Name& name); - S3Handle(const S3Name& name, const Offset& offset, const Length& length); + S3Handle(const S3Name& name, const Offset& offset); Length openForRead() override; @@ -66,13 +66,7 @@ class S3Handle: public DataHandle { Offset pos_ {0}; - Length len_; - - bool readonly_; - - bool open_ {false}; - - Mode mode_ {Mode::NONE}; + Mode mode_ {Mode::CLOSED}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index d902733dc..4796f596d 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -86,8 +86,8 @@ auto S3Name::dataHandle() -> DataHandle* { return new S3Handle(*this); } -auto S3Name::dataHandle(const eckit::Offset& offset, const eckit::Length& length) -> DataHandle* { - return new S3Handle(*this, offset, length); +auto S3Name::dataHandle(const eckit::Offset& offset) -> DataHandle* { + return new S3Handle(*this, offset); } auto S3Name::asString() const -> std::string { diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index f2867bdf9..f8db50349 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -64,7 +64,7 @@ class S3Name { auto dataHandle() -> DataHandle*; [[nodiscard]] - auto dataHandle(const eckit::Offset&, const eckit::Length&) -> DataHandle*; + auto dataHandle(const eckit::Offset&) -> DataHandle*; auto asString() const -> std::string; From 34b8f2c70464d707c4b44fe1fe8fc5bed7da8c44 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 16 Feb 2024 13:21:44 +0100 Subject: [PATCH 54/90] fix(S3): write throws if object exists --- src/eckit/io/s3/S3Handle.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index f3ddc6c2b..1538e7d8c 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -22,7 +22,7 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Handle::S3Handle(const S3Name& name): name_(name), pos_(0) { } +S3Handle::S3Handle(const S3Name& name): S3Handle(name, 0) { } S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { } @@ -66,6 +66,7 @@ long S3Handle::read(void* buffer, const long length) { long S3Handle::write(const void* buffer, const long length) { ASSERT(mode_ == Mode::WRITE); + ASSERT(!name_.exists()); return seek(name_.put(buffer, length)); } From 64c15a08a1a01519e90571a5060bfd597ca206dc Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 16 Feb 2024 13:23:33 +0100 Subject: [PATCH 55/90] feat(S3): added test for no bucket s3handle --- tests/io/test_s3handle.cc | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index dd9d82894..a7a4e99e0 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -103,23 +103,28 @@ CASE("S3Handle::openForWrite") { URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); - { - std::unique_ptr h(uri.newWriteHandle()); - // no bucket - EXPECT_THROWS(h->openForWrite(0)); - // no read - EXPECT_THROWS(h->openForRead()); - } + std::unique_ptr h(uri.newWriteHandle()); - { - std::unique_ptr h(uri.newWriteHandle()); - EXPECT_THROWS(h->openForWrite(0)); - } + // no bucket + EXPECT_NO_THROW(h->openForWrite(0)); + + // no read + EXPECT_THROWS(h->openForRead()); } + +CASE("S3Handle::openForRead") { + std::cout << "===================" << std::endl; + ensureClean(); - std::unique_ptr h(uri.newReadHandle()); - EXPECT_THROWS(h->openForRead()); + S3Client::makeUnique(cfg)->createBucket(TEST_BUCKET); + + URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + + { + std::unique_ptr h(uri.newReadHandle()); + EXPECT_THROWS(h->openForRead()); + } } //---------------------------------------------------------------------------------------------------------------------- From 2ffe3228db393874869ff18a6dc40736a4f85f0a Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 16 Feb 2024 13:25:17 +0100 Subject: [PATCH 56/90] fix(S3): commented expensive code in openForWrite --- src/eckit/io/s3/S3Handle.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 1538e7d8c..01736143f 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -49,9 +49,8 @@ Length S3Handle::openForRead() { void S3Handle::openForWrite(const Length& length) { open(Mode::WRITE); - ASSERT(name_.bucketExists()); - - if (name_.exists()) { ASSERT(size() == length); } + /// @todo slow code, needed ? + // if (length > 0 && name_.exists()) { ASSERT(size() == length); } } //---------------------------------------------------------------------------------------------------------------------- From e17e438ca0888b4bc0878db409a42d0048cf3840 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 16 Feb 2024 13:25:57 +0100 Subject: [PATCH 57/90] refactor(S3): removed eckit namespaces --- src/eckit/io/s3/S3Name.cc | 16 +++++----------- src/eckit/io/s3/S3Name.h | 7 +++---- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 4796f596d..dff2c727f 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -27,18 +27,16 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- S3Name::S3Name(const URI& uri): client_(S3Client::makeShared({uri})) { - const auto pairs = Tokenizer("/").tokenize(uri.name()); const auto pSize = pairs.size(); ASSERT(pSize == 1 || pSize == 2); - bucket_ = S3Bucket{eckit::net::Endpoint{uri}, pairs[0]}; + bucket_ = S3Bucket {net::Endpoint {uri}, pairs[0]}; if (pSize > 1) { object_ = pairs[1]; } - } -S3Name::S3Name(const eckit::S3Bucket& bucket, const std::string& key): bucket_(bucket), object_(key) { } +S3Name::S3Name(const S3Bucket& bucket, const std::string& key): bucket_(bucket), object_(key) { } //---------------------------------------------------------------------------------------------------------------------- @@ -57,17 +55,13 @@ auto S3Name::get(void* buffer, const long offset, const long length) const -> lo } void S3Name::destroy() { - client_->deleteObject(bucket_.name(), object_); - } -auto S3Name::uri() const -> eckit::URI { - - eckit::URI u{bucket_.uri()}; +auto S3Name::uri() const -> URI { + URI u {bucket_.uri()}; u.path(u.name() + "/" + object_); return u; - } auto S3Name::bucketExists() const -> bool { @@ -86,7 +80,7 @@ auto S3Name::dataHandle() -> DataHandle* { return new S3Handle(*this); } -auto S3Name::dataHandle(const eckit::Offset& offset) -> DataHandle* { +auto S3Name::dataHandle(const Offset& offset) -> DataHandle* { return new S3Handle(*this, offset); } diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index f8db50349..16135bacc 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -20,12 +20,12 @@ #pragma once -#include -#include - #include "eckit/io/DataHandle.h" #include "eckit/io/s3/S3Bucket.h" +#include +#include + namespace eckit { class URI; @@ -74,7 +74,6 @@ class S3Name { } private: // methods - void print(std::ostream& out) const; private: // members From ff3342e5456a4418b2de3fcbe2dfc8ffe97d1760 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 16 Feb 2024 20:56:44 +0100 Subject: [PATCH 58/90] refactor(S3)!: S3BucketName --- src/eckit/CMakeLists.txt | 4 +- src/eckit/io/s3/S3Bucket.cc | 116 -------------------------------- src/eckit/io/s3/S3Bucket.h | 94 -------------------------- src/eckit/io/s3/S3BucketName.cc | 76 +++++++++++++++++++++ src/eckit/io/s3/S3BucketName.h | 57 ++++++++++++++++ 5 files changed, 135 insertions(+), 212 deletions(-) delete mode 100644 src/eckit/io/s3/S3Bucket.cc delete mode 100644 src/eckit/io/s3/S3Bucket.h create mode 100644 src/eckit/io/s3/S3BucketName.cc create mode 100644 src/eckit/io/s3/S3BucketName.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 8ef426ec3..63a4d13bf 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -292,8 +292,8 @@ io/s3/aws/S3ClientAWS.cc io/s3/aws/S3ClientAWS.h io/s3/aws/S3ContextAWS.cc io/s3/aws/S3ContextAWS.h -io/s3/S3Bucket.cc -io/s3/S3Bucket.h +io/s3/S3BucketName.cc +io/s3/S3BucketName.h io/s3/S3Client.cc io/s3/S3Client.h io/s3/S3Config.cc diff --git a/src/eckit/io/s3/S3Bucket.cc b/src/eckit/io/s3/S3Bucket.cc deleted file mode 100644 index 76507b7d7..000000000 --- a/src/eckit/io/s3/S3Bucket.cc +++ /dev/null @@ -1,116 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/* - * This software was developed as part of the EC H2020 funded project IO-SEA - * (Project ID: 955811) iosea-project.eu - */ - -// #include "eckit/config/LibEcKit.h" -// #include "eckit/filesystem/URI.h" -// #include "eckit/io/s3/S3Handle.h" -// #include "eckit/utils/Tokenizer.h" - -#include "eckit/io/s3/S3Bucket.h" -#include "eckit/io/s3/S3Name.h" -#include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Exception.h" - -namespace eckit { - -//---------------------------------------------------------------------------------------------------------------------- - -S3Bucket::S3Bucket(const URI& uri): - client_(eckit::S3Client::makeShared({uri})), - endpoint_(uri) { - - parse(uri.name()); - -} - -S3Bucket::S3Bucket(const eckit::net::Endpoint& endpoint, const std::string& name): - client_(S3Client::makeShared({endpoint})), - endpoint_(endpoint), - name_(name) { } - -S3Bucket::S3Bucket(const S3Bucket& other) : - client_(S3Client::makeShared({other.endpoint_})), - endpoint_(other.endpoint_), - name_(other.name_) { } - -S3Bucket& S3Bucket::operator=(S3Bucket&& other) { - - client_ = std::move(other.client_); - endpoint_ = std::move(other.endpoint_); - name_ = std::move(other.name_); - - return *this; - -} - -//---------------------------------------------------------------------------------------------------------------------- - -auto S3Bucket::uri() const -> eckit::URI { - - eckit::URI u{"s3", endpoint_.host(), endpoint_.port()}; - u.path(name_); - return u; - -} - -void S3Bucket::parse(const std::string& path) { - - ASSERT(path.find("/") == std::string::npos); - - name_ = path; - -} - -void S3Bucket::print(std::ostream& out) const { - out << "S3Bucket[bucket=" << name_ << "]"; -} - -//---------------------------------------------------------------------------------------------------------------------- - -auto S3Bucket::exists() const -> bool { - return client_->bucketExists(name_); -} - -void S3Bucket::create() { - return client_->createBucket(name_); -} - -void S3Bucket::destroy() { - return client_->deleteBucket(name_); -} - -void S3Bucket::ensureCreated() { - try { - create(); - } catch (eckit::S3EntityAlreadyExists& e) { } -} - -void S3Bucket::ensureDestroyed() { - try { - destroy(); - } catch (eckit::S3EntityNotFound& e) { } -} - -std::vector S3Bucket::listObjects() const { - std::vector objects; - for (auto obj : client_->listObjects(name_)) { - objects.push_back(eckit::S3Name{*this, obj}); - } - return objects; -} - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace eckit diff --git a/src/eckit/io/s3/S3Bucket.h b/src/eckit/io/s3/S3Bucket.h deleted file mode 100644 index 19e75031e..000000000 --- a/src/eckit/io/s3/S3Bucket.h +++ /dev/null @@ -1,94 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/* - * This software was developed as part of the EC H2020 funded project IO-SEA - * (Project ID: 955811) iosea-project.eu - */ - -/// @file S3Bucket.h -/// @author Metin Cakircali -/// @author Simon Smart -/// @date Jan 2024 - -#pragma once - -#include -#include - -#include "eckit/io/Length.h" -#include "eckit/net/Endpoint.h" - -#include "eckit/io/s3/S3Exception.h" - -namespace eckit { - -class URI; -class DataHandle; -class S3Client; -class S3Name; - -//---------------------------------------------------------------------------------------------------------------------- - -class S3Bucket { -public: // methods - - explicit S3Bucket(const URI& uri); - explicit S3Bucket(const eckit::net::Endpoint& endpoint, const std::string& name); - S3Bucket(const S3Bucket&); - S3Bucket& operator=(S3Bucket&&); - - auto name() const -> const std::string& { return name_; } - - auto uri() const -> eckit::URI; - - auto exists() const -> bool; - - /// @note: throws if bucket already exists - void create(); - - /// @note: throws if bucket does not exist - void destroy(); - - void ensureCreated(); - - void ensureDestroyed(); - - /// @todo: How do we handle there being lots of keys. - /// We want to return an iterator that internally handles the - /// relevant pagination. - std::vector listObjects() const; - - // friend std::ostream& operator<<(std::ostream& out, const S3Bucket& name) { - // name.print(out); - // return out; - // } - - auto asString() const -> const std::string& { return name_; } - -private: // methods - - friend class S3Name; - - S3Bucket() {} - - void parse(const std::string& path); - - void print(std::ostream& out) const; - -private: // members - std::shared_ptr client_; - eckit::net::Endpoint endpoint_; - std::string name_; -}; - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace eckit diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc new file mode 100644 index 000000000..290221edf --- /dev/null +++ b/src/eckit/io/s3/S3BucketName.cc @@ -0,0 +1,76 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3BucketName.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Exception.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3BucketName::S3BucketName(const URI& uri): S3Name(uri) { + parse(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +void S3BucketName::print(std::ostream& out) const { + out << "S3BucketName[bucket=" << bucket_ << "]"; +} + +void S3BucketName::parse() { + const auto pairs = parseName(); + if (pairs.empty()) { throw S3SeriousBug("Could not parse bucket name!", Here()); } + bucket_ = pairs[0]; +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3BucketName::exists() const -> bool { + return client()->bucketExists(bucket_); +} + +void S3BucketName::create() { + client()->createBucket(bucket_); +} + +void S3BucketName::destroy() { + client()->deleteBucket(bucket_); +} + +void S3BucketName::ensureCreated() { + try { + create(); + } catch (S3EntityAlreadyExists& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } +} + +void S3BucketName::ensureDestroyed() { + try { + client()->emptyBucket(bucket_); + client()->deleteBucket(bucket_); + } catch (S3EntityNotFound& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } +} + +auto S3BucketName::listObjects() const -> std::vector { + return client()->listObjects(bucket_); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h new file mode 100644 index 000000000..2967267b1 --- /dev/null +++ b/src/eckit/io/s3/S3BucketName.h @@ -0,0 +1,57 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3BucketName.h +/// @author Metin Cakircali +/// @author Nicolau Manubens +/// @date Feb 2024 + +#pragma once + +#include "eckit/io/s3/S3Name.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3BucketName: public S3Name { +public: // methods + explicit S3BucketName(const URI& uri); + + auto exists() const -> bool override; + + void create(); + + void destroy(); + + void ensureCreated(); + + void ensureDestroyed(); + + /// @todo: return S3 object iterator + auto listObjects() const -> std::vector; + +private: // methods + void print(std::ostream& out) const override; + + void parse(); + +private: // members + std::string bucket_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit From 13a3d01dcf41c998b500ef0341b4176eefc8ca05 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 16 Feb 2024 20:57:21 +0100 Subject: [PATCH 59/90] refactor(S3)!: S3ClientAWS throws --- src/eckit/io/s3/aws/S3ClientAWS.cc | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index cea6bd78a..2ae111ab1 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -120,14 +120,13 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { auto outcome = client_->CreateBucket(request); - if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; - } else if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS || - outcome.GetError().GetErrorType() == Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) { - auto msg = awsErrorMessage("Bucket already exists=" + bucket, outcome.GetError()); - throw S3EntityAlreadyExists(msg, Here()); - } else { - auto msg = awsErrorMessage("Failed to create bucket=" + bucket, outcome.GetError()); + if (!outcome.IsSuccess()) { + const auto& err = outcome.GetError(); + const auto msg = awsErrorMessage("Failed to create bucket=" + bucket, err); + const auto eType = err.GetErrorType(); + if (eType == Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS || eType == Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) { + throw S3EntityAlreadyExists(msg, Here()); + } throw S3SeriousBug(msg, Here()); } @@ -144,13 +143,10 @@ void S3ClientAWS::deleteBucket(const std::string& bucket) const { auto outcome = client_->DeleteBucket(request); - if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucket << std::endl; - } else if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_BUCKET) { - auto msg = awsErrorMessage("Bucket does not exist=" + bucket, outcome.GetError()); - throw S3EntityNotFound(msg, Here()); - } else { - auto msg = awsErrorMessage("Failed to delete bucket=" + bucket, outcome.GetError()); + if (!outcome.IsSuccess()) { + const auto& err = outcome.GetError(); + const auto msg = awsErrorMessage("Failed to delete bucket=" + bucket, err); + if (err.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_BUCKET) { throw S3EntityNotFound(msg, Here()); } throw S3SeriousBug(msg, Here()); } From b4ae6a12b02791d88db9d90455ce05de9fdb8dde Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 16 Feb 2024 20:57:41 +0100 Subject: [PATCH 60/90] refactor(S3)!: S3Exception --- src/eckit/io/s3/S3Exception.cc | 9 +++------ src/eckit/io/s3/S3Exception.h | 13 +++++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc index 2ad94d980..c2788f975 100644 --- a/src/eckit/io/s3/S3Exception.cc +++ b/src/eckit/io/s3/S3Exception.cc @@ -40,14 +40,11 @@ S3SeriousBug::S3SeriousBug(const std::string& msg, const int code, const CodeLoc //---------------------------------------------------------------------------------------------------------------------- -S3Exception::S3Exception(const std::string& w, const eckit::CodeLocation& l) : - Exception(w, l) {} +S3Exception::S3Exception(const std::string& msg, const CodeLocation& loc): Exception("[S3 Exception] " + msg, loc) { } -S3EntityNotFound::S3EntityNotFound(const std::string& w, const eckit::CodeLocation& l) : - S3Exception(w, l) {} +S3EntityNotFound::S3EntityNotFound(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } -S3EntityAlreadyExists::S3EntityAlreadyExists(const std::string& w, const eckit::CodeLocation& l) : - S3Exception(w, l) {} +S3EntityAlreadyExists::S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Exception.h b/src/eckit/io/s3/S3Exception.h index 88f8bbb00..52dc59cba 100644 --- a/src/eckit/io/s3/S3Exception.h +++ b/src/eckit/io/s3/S3Exception.h @@ -15,6 +15,7 @@ /// @file S3Exception.h /// @author Metin Cakircali +/// @author Nicolau Manubens /// @date Jan 2024 #pragma once @@ -38,19 +39,19 @@ class S3SeriousBug: public SeriousBug { //---------------------------------------------------------------------------------------------------------------------- -class S3Exception : public eckit::Exception { +class S3Exception: public Exception { public: - S3Exception(const std::string&, const eckit::CodeLocation&); + S3Exception(const std::string& msg, const CodeLocation& loc); }; -class S3EntityNotFound : public S3Exception { +class S3EntityNotFound: public S3Exception { public: - S3EntityNotFound(const std::string&, const eckit::CodeLocation&); + S3EntityNotFound(const std::string& msg, const CodeLocation& loc); }; -class S3EntityAlreadyExists : public S3Exception { +class S3EntityAlreadyExists: public S3Exception { public: - S3EntityAlreadyExists(const std::string&, const eckit::CodeLocation&); + S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc); }; //---------------------------------------------------------------------------------------------------------------------- From 0c84e60ceb868d7b6555ef64d833c704dfb44c3c Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 16 Feb 2024 21:04:46 +0100 Subject: [PATCH 61/90] feat(S3)!: added S3ObjectName --- src/eckit/CMakeLists.txt | 2 + src/eckit/io/s3/S3BucketName.h | 2 +- src/eckit/io/s3/S3Handle.cc | 4 +- src/eckit/io/s3/S3Handle.h | 8 ++-- src/eckit/io/s3/S3Name.cc | 64 ++++++------------------- src/eckit/io/s3/S3Name.h | 51 ++++++-------------- src/eckit/io/s3/S3ObjectName.cc | 83 +++++++++++++++++++++++++++++++++ src/eckit/io/s3/S3ObjectName.h | 67 ++++++++++++++++++++++++++ src/eckit/io/s3/S3URIManager.cc | 16 +++---- tests/io/test_s3handle.cc | 2 +- 10 files changed, 195 insertions(+), 104 deletions(-) create mode 100644 src/eckit/io/s3/S3ObjectName.cc create mode 100644 src/eckit/io/s3/S3ObjectName.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 63a4d13bf..ce00cccb6 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -307,6 +307,8 @@ io/s3/S3Handle.cc io/s3/S3Handle.h io/s3/S3Name.cc io/s3/S3Name.h +io/s3/S3ObjectName.cc +io/s3/S3ObjectName.h io/s3/S3Session.cc io/s3/S3Session.h io/s3/S3Macros.h diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h index 2967267b1..8057bbbab 100644 --- a/src/eckit/io/s3/S3BucketName.h +++ b/src/eckit/io/s3/S3BucketName.h @@ -40,7 +40,7 @@ class S3BucketName: public S3Name { void ensureDestroyed(); - /// @todo: return S3 object iterator + /// @todo: return S3 object iterator but first add prefix auto listObjects() const -> std::vector; private: // methods diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 01736143f..9ef75cb7a 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -22,9 +22,9 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Handle::S3Handle(const S3Name& name): S3Handle(name, 0) { } +S3Handle::S3Handle(const S3ObjectName& name): S3Handle(name, 0) { } -S3Handle::S3Handle(const S3Name& name, const Offset& offset): name_(name), pos_(offset) { } +S3Handle::S3Handle(const S3ObjectName& name, const Offset& offset): name_(name), pos_(offset) { } void S3Handle::print(std::ostream& out) const { out << "S3Handle[name=" << name_ << ", position=" << pos_; diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index 0633f4fc8..ec6ceadd5 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -20,7 +20,7 @@ #pragma once #include "eckit/io/DataHandle.h" -#include "eckit/io/s3/S3Name.h" +#include "eckit/io/s3/S3ObjectName.h" namespace eckit { @@ -30,9 +30,9 @@ class S3Handle: public DataHandle { public: // methods enum class Mode { CLOSED, READ, WRITE }; - explicit S3Handle(const S3Name& name); + explicit S3Handle(const S3ObjectName& name); - S3Handle(const S3Name& name, const Offset& offset); + S3Handle(const S3ObjectName& name, const Offset& offset); Length openForRead() override; @@ -62,7 +62,7 @@ class S3Handle: public DataHandle { void open(Mode mode); private: // members - const S3Name name_; + const S3ObjectName name_; Offset pos_ {0}; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index dff2c727f..c567e1e5f 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -15,77 +15,41 @@ #include "eckit/io/s3/S3Name.h" -#include "eckit/config/LibEcKit.h" #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" -#include "eckit/io/s3/S3Handle.h" #include "eckit/utils/Tokenizer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Name::S3Name(const URI& uri): client_(S3Client::makeShared({uri})) { - const auto pairs = Tokenizer("/").tokenize(uri.name()); - const auto pSize = pairs.size(); - - ASSERT(pSize == 1 || pSize == 2); - - bucket_ = S3Bucket {net::Endpoint {uri}, pairs[0]}; - if (pSize > 1) { object_ = pairs[1]; } +S3Name::S3Name(const URI& uri): endpoint_(uri), name_(uri.name()) { + ASSERT(uri.scheme() == "s3"); } -S3Name::S3Name(const S3Bucket& bucket, const std::string& key): bucket_(bucket), object_(key) { } +S3Name::S3Name(const net::Endpoint& endpoint, const std::string& name): endpoint_(endpoint), name_(name) { } //---------------------------------------------------------------------------------------------------------------------- -void S3Name::print(std::ostream& out) const { - out << "S3Name[bucket=" << bucket_.name() << ",object=" << object_ << "]"; -} - -//---------------------------------------------------------------------------------------------------------------------- - -auto S3Name::put(const void* buffer, const long length) const -> long long { - return client_->putObject(bucket_.name(), object_, buffer, length); -} - -auto S3Name::get(void* buffer, const long offset, const long length) const -> long long { - return client_->getObject(bucket_.name(), object_, buffer, offset, length); -} - -void S3Name::destroy() { - client_->deleteObject(bucket_.name(), object_); -} - -auto S3Name::uri() const -> URI { - URI u {bucket_.uri()}; - u.path(u.name() + "/" + object_); - return u; -} - -auto S3Name::bucketExists() const -> bool { - return client_->bucketExists(bucket_.name()); -} - -auto S3Name::exists() const -> bool { - return client_->objectExists(bucket_.name(), object_); +auto S3Name::asString() const -> std::string { + std::ostringstream oss; + oss << "s3://" << endpoint_ << "/" << name_; + return oss.str(); } -auto S3Name::size() const -> long long { - return client_->objectSize(bucket_.name(), object_); +void S3Name::print(std::ostream& out) const { + out << "endpoint=" << endpoint_ << ",name=" << name_; } -auto S3Name::dataHandle() -> DataHandle* { - return new S3Handle(*this); -} +//---------------------------------------------------------------------------------------------------------------------- -auto S3Name::dataHandle(const Offset& offset) -> DataHandle* { - return new S3Handle(*this, offset); +auto S3Name::parseName() const -> std::vector { + return Tokenizer("/").tokenize(name_); } -auto S3Name::asString() const -> std::string { - return bucket_.name() + "/" + object_; +auto S3Name::client() const -> std::shared_ptr { + return S3Client::makeShared({endpoint_}); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 16135bacc..21c7a102a 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -20,15 +20,12 @@ #pragma once -#include "eckit/io/DataHandle.h" -#include "eckit/io/s3/S3Bucket.h" +#include "eckit/net/Endpoint.h" -#include #include namespace eckit { -class URI; class S3Client; //---------------------------------------------------------------------------------------------------------------------- @@ -37,50 +34,28 @@ class S3Name { public: // methods explicit S3Name(const URI& uri); - S3Name(const S3Bucket&, const std::string& key); + S3Name(const net::Endpoint& endpoint, const std::string& name); - S3Name(const net::Endpoint&, const std::string& bucket, const std::string& key); + virtual auto exists() const -> bool = 0; - auto put(const void* buffer, long length) const -> long long; - - auto get(void* buffer, long offset, long length) const -> long long; - - void destroy(); - - auto bucket() const -> const S3Bucket& { return bucket_; } - - auto object() const -> const std::string& { return object_; } - auto name() const -> const std::string& { return object_; } - - auto uri() const -> eckit::URI; - - auto bucketExists() const -> bool; - - auto exists() const -> bool; - - auto size() const -> long long; - - [[nodiscard]] - auto dataHandle() -> DataHandle*; - - [[nodiscard]] - auto dataHandle(const eckit::Offset&) -> DataHandle*; - - auto asString() const -> std::string; + virtual auto asString() const -> std::string; friend std::ostream& operator<<(std::ostream& out, const S3Name& name) { name.print(out); return out; } -private: // methods - void print(std::ostream& out) const; +protected: // methods + virtual void print(std::ostream& out) const; -private: // members - eckit::S3Bucket bucket_; - std::string object_; + [[nodiscard]] + auto parseName() const -> std::vector; - std::shared_ptr client_; + auto client() const -> std::shared_ptr; + +private: // members + const net::Endpoint endpoint_; + const std::string name_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc new file mode 100644 index 000000000..43334f7c8 --- /dev/null +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -0,0 +1,83 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3ObjectName.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Handle.h" +#include "eckit/utils/Tokenizer.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3ObjectName::S3ObjectName(const URI& uri): S3Name(uri) { + parse(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +void S3ObjectName::print(std::ostream& out) const { + out << "S3ObjectName[bucket=" << bucket_; + S3Name::print(out); + out << "]"; +} + +void S3ObjectName::parse() { + const auto pairs = parseName(); + if (pairs.size() != 2) { throw S3SeriousBug("Could not parse bucket and object names!", Here()); } + bucket_ = pairs[0]; + object_ = pairs[1]; +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ObjectName::uri() const -> URI { + return asString(); +} + +auto S3ObjectName::size() const -> long long { + return client()->objectSize(bucket_, object_); +} + +auto S3ObjectName::exists() const -> bool { + return client()->objectExists(bucket_, object_); +} + +void S3ObjectName::remove() { + client()->deleteObject(bucket_, object_); +} + +auto S3ObjectName::put(const void* buffer, const long length) const -> long long { + return client()->putObject(bucket_, object_, buffer, length); +} + +auto S3ObjectName::get(void* buffer, const long offset, const long length) const -> long long { + return client()->getObject(bucket_, object_, buffer, offset, length); +} + +auto S3ObjectName::dataHandle() -> DataHandle* { + return new S3Handle(*this); +} + +auto S3ObjectName::dataHandle(const Offset& offset) -> DataHandle* { + return new S3Handle(*this, offset); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h new file mode 100644 index 000000000..b0efb650c --- /dev/null +++ b/src/eckit/io/s3/S3ObjectName.h @@ -0,0 +1,67 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3ObjectName.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/s3/S3Name.h" + +namespace eckit { + +class Offset; +class DataHandle; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3ObjectName: public S3Name { +public: // methods + explicit S3ObjectName(const URI& uri); + + auto bucket() const -> const std::string& { return bucket_; } + + auto uri() const -> URI; + + auto size() const -> long long; + + auto exists() const -> bool override; + + void remove(); + + auto put(const void* buffer, long length) const -> long long; + + auto get(void* buffer, long offset, long length) const -> long long; + + [[nodiscard]] + auto dataHandle() -> DataHandle*; + + [[nodiscard]] + auto dataHandle(const Offset& offset) -> DataHandle*; + +private: // methods + void print(std::ostream& out) const override; + + void parse(); + +private: // members + std::string bucket_; + std::string object_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3URIManager.cc b/src/eckit/io/s3/S3URIManager.cc index ac9c67207..ef96dfdc4 100644 --- a/src/eckit/io/s3/S3URIManager.cc +++ b/src/eckit/io/s3/S3URIManager.cc @@ -13,9 +13,9 @@ * (Project ID: 955811) iosea-project.eu */ -#include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3URIManager.h" -#include "eckit/io/s3/S3Name.h" + +#include "eckit/io/s3/S3ObjectName.h" namespace eckit { @@ -28,25 +28,25 @@ S3URIManager::S3URIManager(const std::string& name): URIManager(name) { } S3URIManager::~S3URIManager() = default; bool S3URIManager::exists(const URI& uri) { - return S3Name(uri).exists(); + return S3ObjectName(uri).exists(); } DataHandle* S3URIManager::newWriteHandle(const URI& uri) { - return S3Name(uri).dataHandle(); + return S3ObjectName(uri).dataHandle(); } DataHandle* S3URIManager::newReadHandle(const URI& uri) { - return S3Name(uri).dataHandle(); + return S3ObjectName(uri).dataHandle(); } DataHandle* S3URIManager::newReadHandle(const URI& uri, const OffsetList&, const LengthList&) { - return S3Name(uri).dataHandle(); + return S3ObjectName(uri).dataHandle(); } std::string S3URIManager::asString(const URI& uri) const { - return std::string("s3://") + uri.name(); + return S3ObjectName(uri).asString(); } //---------------------------------------------------------------------------------------------------------------------- -} +} // namespace eckit diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index a7a4e99e0..d0dac75c7 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -62,7 +62,7 @@ CASE("bucket exists") { URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); { - S3Name name(uri); + S3ObjectName name(uri); std::unique_ptr h(name.dataHandle()); h->openForWrite(length); From 48797f64bd228677294b115efb5e85ecff37c998 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sat, 17 Feb 2024 14:32:43 +0100 Subject: [PATCH 62/90] fix(S3): memory leak S3Name --- src/eckit/io/s3/S3Name.cc | 2 ++ src/eckit/io/s3/S3Name.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index c567e1e5f..cb7052a0a 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -30,6 +30,8 @@ S3Name::S3Name(const URI& uri): endpoint_(uri), name_(uri.name()) { S3Name::S3Name(const net::Endpoint& endpoint, const std::string& name): endpoint_(endpoint), name_(name) { } +S3Name::~S3Name() = default; + //---------------------------------------------------------------------------------------------------------------------- auto S3Name::asString() const -> std::string { diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 21c7a102a..89113b643 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -36,6 +36,8 @@ class S3Name { S3Name(const net::Endpoint& endpoint, const std::string& name); + virtual ~S3Name(); + virtual auto exists() const -> bool = 0; virtual auto asString() const -> std::string; From f9382342c568008d8d28480c1334b8803cd096cd Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sat, 17 Feb 2024 14:33:52 +0100 Subject: [PATCH 63/90] feat(S3): added S3Name accessors --- src/eckit/io/s3/S3Name.cc | 6 ++---- src/eckit/io/s3/S3Name.h | 2 ++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index cb7052a0a..212ba3279 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -35,13 +35,11 @@ S3Name::~S3Name() = default; //---------------------------------------------------------------------------------------------------------------------- auto S3Name::asString() const -> std::string { - std::ostringstream oss; - oss << "s3://" << endpoint_ << "/" << name_; - return oss.str(); + return "s3://" + std::string(endpoint_); } void S3Name::print(std::ostream& out) const { - out << "endpoint=" << endpoint_ << ",name=" << name_; + out << ",endpoint=" << endpoint_ << ",name=" << name_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 89113b643..2a1bda6d9 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -38,6 +38,8 @@ class S3Name { virtual ~S3Name(); + auto endpoint() const -> const net::Endpoint& { return endpoint_; } + virtual auto exists() const -> bool = 0; virtual auto asString() const -> std::string; From 7a47f998648a6adc2a894476855fb9726e48a963 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Sat, 17 Feb 2024 14:41:34 +0100 Subject: [PATCH 64/90] refactor(S3): various log debug --- src/eckit/io/s3/S3BucketName.cc | 17 ++++++++++++++--- src/eckit/io/s3/S3Client.cc | 2 +- src/eckit/io/s3/S3ObjectName.cc | 7 +++++-- src/eckit/io/s3/S3ObjectName.h | 2 ++ src/eckit/io/s3/aws/S3ClientAWS.cc | 23 ++++++++++------------- 5 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc index 290221edf..b1586ee9c 100644 --- a/src/eckit/io/s3/S3BucketName.cc +++ b/src/eckit/io/s3/S3BucketName.cc @@ -30,8 +30,13 @@ S3BucketName::S3BucketName(const URI& uri): S3Name(uri) { //---------------------------------------------------------------------------------------------------------------------- +auto S3BucketName::asString() const -> std::string { + return S3Name::asString() + "/" + bucket_; +} + void S3BucketName::print(std::ostream& out) const { - out << "S3BucketName[bucket=" << bucket_ << "]"; + out << "S3BucketName[bucket=" << bucket_; + S3Name::print(out); } void S3BucketName::parse() { @@ -57,14 +62,20 @@ void S3BucketName::destroy() { void S3BucketName::ensureCreated() { try { create(); - } catch (S3EntityAlreadyExists& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } + } + catch (S3EntityAlreadyExists& e) { + LOG_DEBUG_LIB(LibEcKit) << e.what() << "\n"; + } } void S3BucketName::ensureDestroyed() { try { client()->emptyBucket(bucket_); client()->deleteBucket(bucket_); - } catch (S3EntityNotFound& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } + } + catch (S3EntityNotFound& e) { + LOG_DEBUG_LIB(LibEcKit) << e.what() << "\n"; + } } auto S3BucketName::listObjects() const -> std::vector { diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index 7e3cfcc34..bd8116e16 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -51,7 +51,7 @@ void S3Client::print(std::ostream& out) const { } else if (type == S3Types::REST) { out << "REST"; } else { - out << "REST"; + out << "NONE"; } out << "]"; } diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc index 43334f7c8..713953921 100644 --- a/src/eckit/io/s3/S3ObjectName.cc +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -32,9 +32,8 @@ S3ObjectName::S3ObjectName(const URI& uri): S3Name(uri) { //---------------------------------------------------------------------------------------------------------------------- void S3ObjectName::print(std::ostream& out) const { - out << "S3ObjectName[bucket=" << bucket_; + out << "S3ObjectName[object=" << object_ << ",bucket=" << bucket_; S3Name::print(out); - out << "]"; } void S3ObjectName::parse() { @@ -46,6 +45,10 @@ void S3ObjectName::parse() { //---------------------------------------------------------------------------------------------------------------------- +auto S3ObjectName::asString() const -> std::string { + return S3Name::asString() + "/" + bucket_ + "/" + object_; +} + auto S3ObjectName::uri() const -> URI { return asString(); } diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h index b0efb650c..82a7be512 100644 --- a/src/eckit/io/s3/S3ObjectName.h +++ b/src/eckit/io/s3/S3ObjectName.h @@ -52,6 +52,8 @@ class S3ObjectName: public S3Name { [[nodiscard]] auto dataHandle(const Offset& offset) -> DataHandle*; + auto asString() const -> std::string override; + private: // methods void print(std::ostream& out) const override; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 2ae111ab1..2436fba57 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -73,7 +73,7 @@ S3ClientAWS::~S3ClientAWS() = default; //---------------------------------------------------------------------------------------------------------------------- void S3ClientAWS::configure(const S3Config& config) { - LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client." << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client.\n"; Aws::Client::ClientConfiguration configuration; @@ -130,7 +130,7 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { throw S3SeriousBug(msg, Here()); } - LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << "\n"; } void S3ClientAWS::emptyBucket(const std::string& bucket) const { @@ -150,7 +150,7 @@ void S3ClientAWS::deleteBucket(const std::string& bucket) const { throw S3SeriousBug(msg, Here()); } - LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucket << "\n"; } auto S3ClientAWS::bucketExists(const std::string& bucket) const -> bool { @@ -196,7 +196,7 @@ auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object auto outcome = client_->PutObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Put object=" << object << " [len=" << length << "] to bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Put object=" << object << " [len=" << length << "] to bucket=" << bucket << "\n"; /// @todo actual size of written bytes return length; } @@ -220,9 +220,9 @@ auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object auto outcome = client_->GetObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Get object=" << object << " from bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Get object=" << object << " from bucket=" << bucket << "\n"; LOG_DEBUG_LIB(LibEcKit) << "Req. len=" << length << ", Obj. len=" << outcome.GetResult().GetContentLength() - << std::endl; + << "\n"; return outcome.GetResult().GetContentLength(); } @@ -246,14 +246,11 @@ void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& obj throw S3SeriousBug(msg, Here()); } - LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << " in bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << " in bucket=" << bucket << "\n"; } void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vector& objects) const { - if (objects.empty()) { - LOG_DEBUG_LIB(LibEcKit) << "No objects to delete in bucket=" << bucket << std::endl; - return; - } + if (objects.empty()) { return; } Aws::S3::Model::DeleteObjectsRequest request; @@ -272,8 +269,8 @@ void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vector std::vector { From 227db9fd350c76689334a89e40bf5602da86a162 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 19 Feb 2024 13:59:36 +0100 Subject: [PATCH 65/90] fix(S3)!: S3Handle bug read/write return --- src/eckit/io/s3/S3Handle.cc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 9ef75cb7a..928facac4 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -50,6 +50,9 @@ void S3Handle::openForWrite(const Length& length) { open(Mode::WRITE); /// @todo slow code, needed ? + ASSERT(name_.bucketExists()); + + /// @todo slow code, use of length ? // if (length > 0 && name_.exists()) { ASSERT(size() == length); } } @@ -57,17 +60,26 @@ void S3Handle::openForWrite(const Length& length) { long S3Handle::read(void* buffer, const long length) { ASSERT(mode_ == Mode::READ); + ASSERT(0 <= pos_); if (size() == pos_) { return 0; } - return seek(name_.get(buffer, pos_, length)); + const auto len = name_.get(buffer, pos_, length); + + pos_ += len; + + return len; } long S3Handle::write(const void* buffer, const long length) { ASSERT(mode_ == Mode::WRITE); ASSERT(!name_.exists()); - return seek(name_.put(buffer, length)); + const auto len = name_.put(buffer, length); + + pos_ += len; + + return len; } //---------------------------------------------------------------------------------------------------------------------- @@ -79,12 +91,13 @@ void S3Handle::open(const Mode mode) { } void S3Handle::close() { + if (mode_ == Mode::WRITE) { flush(); } pos_ = 0; mode_ = Mode::CLOSED; } void S3Handle::flush() { - NOTIMP; + LOG_DEBUG_LIB(LibEcKit) << "flush()!" << std::endl; } //---------------------------------------------------------------------------------------------------------------------- From c07f52c5fda2a7eb558a5268dc8a6b10caff73fd Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 19 Feb 2024 14:04:31 +0100 Subject: [PATCH 66/90] feat(S3): added S3Handle tests, S3Bucket/Object accessors and change to ListObjects V2 --- src/eckit/io/s3/S3BucketName.cc | 12 +- src/eckit/io/s3/S3BucketName.h | 8 ++ src/eckit/io/s3/S3Exception.cc | 2 + src/eckit/io/s3/S3Exception.h | 5 + src/eckit/io/s3/S3Handle.cc | 1 - src/eckit/io/s3/S3ObjectName.cc | 7 + src/eckit/io/s3/S3ObjectName.h | 8 +- src/eckit/io/s3/aws/S3ClientAWS.cc | 72 ++++++---- tests/io/test_s3handle.cc | 214 +++++++++++++++++++++++------ 9 files changed, 256 insertions(+), 73 deletions(-) diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc index b1586ee9c..b921b095b 100644 --- a/src/eckit/io/s3/S3BucketName.cc +++ b/src/eckit/io/s3/S3BucketName.cc @@ -19,6 +19,7 @@ #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3ObjectName.h" namespace eckit { @@ -28,6 +29,9 @@ S3BucketName::S3BucketName(const URI& uri): S3Name(uri) { parse(); } +S3BucketName::S3BucketName(const net::Endpoint& endpoint, const std::string& bucket): + S3Name(endpoint, "/"), bucket_(bucket) { } + //---------------------------------------------------------------------------------------------------------------------- auto S3BucketName::asString() const -> std::string { @@ -47,6 +51,10 @@ void S3BucketName::parse() { //---------------------------------------------------------------------------------------------------------------------- +auto S3BucketName::makeObject(const std::string& object) const -> std::unique_ptr { + return std::make_unique(endpoint(), bucket_, object); +} + auto S3BucketName::exists() const -> bool { return client()->bucketExists(bucket_); } @@ -64,7 +72,7 @@ void S3BucketName::ensureCreated() { create(); } catch (S3EntityAlreadyExists& e) { - LOG_DEBUG_LIB(LibEcKit) << e.what() << "\n"; + LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } } @@ -74,7 +82,7 @@ void S3BucketName::ensureDestroyed() { client()->deleteBucket(bucket_); } catch (S3EntityNotFound& e) { - LOG_DEBUG_LIB(LibEcKit) << e.what() << "\n"; + LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } } diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h index 8057bbbab..57fbb50cf 100644 --- a/src/eckit/io/s3/S3BucketName.h +++ b/src/eckit/io/s3/S3BucketName.h @@ -24,12 +24,18 @@ namespace eckit { +class S3ObjectName; + //---------------------------------------------------------------------------------------------------------------------- class S3BucketName: public S3Name { public: // methods explicit S3BucketName(const URI& uri); + S3BucketName(const net::Endpoint& endpoint, const std::string& bucket); + + auto makeObject(const std::string& object) const -> std::unique_ptr; + auto exists() const -> bool override; void create(); @@ -43,6 +49,8 @@ class S3BucketName: public S3Name { /// @todo: return S3 object iterator but first add prefix auto listObjects() const -> std::vector; + auto asString() const -> std::string override; + private: // methods void print(std::ostream& out) const override; diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc index c2788f975..1a60b8a35 100644 --- a/src/eckit/io/s3/S3Exception.cc +++ b/src/eckit/io/s3/S3Exception.cc @@ -42,6 +42,8 @@ S3SeriousBug::S3SeriousBug(const std::string& msg, const int code, const CodeLoc S3Exception::S3Exception(const std::string& msg, const CodeLocation& loc): Exception("[S3 Exception] " + msg, loc) { } +S3BucketNotEmpty::S3BucketNotEmpty(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } + S3EntityNotFound::S3EntityNotFound(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } S3EntityAlreadyExists::S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } diff --git a/src/eckit/io/s3/S3Exception.h b/src/eckit/io/s3/S3Exception.h index 52dc59cba..e49866710 100644 --- a/src/eckit/io/s3/S3Exception.h +++ b/src/eckit/io/s3/S3Exception.h @@ -44,6 +44,11 @@ class S3Exception: public Exception { S3Exception(const std::string& msg, const CodeLocation& loc); }; +class S3BucketNotEmpty: public S3Exception { +public: + S3BucketNotEmpty(const std::string& msg, const CodeLocation& loc); +}; + class S3EntityNotFound: public S3Exception { public: S3EntityNotFound(const std::string& msg, const CodeLocation& loc); diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 928facac4..bc88a7228 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -49,7 +49,6 @@ Length S3Handle::openForRead() { void S3Handle::openForWrite(const Length& length) { open(Mode::WRITE); - /// @todo slow code, needed ? ASSERT(name_.bucketExists()); /// @todo slow code, use of length ? diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc index 713953921..cf9c5b708 100644 --- a/src/eckit/io/s3/S3ObjectName.cc +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -29,6 +29,9 @@ S3ObjectName::S3ObjectName(const URI& uri): S3Name(uri) { parse(); } +S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, const std::string& bucket, const std::string& object): + S3Name(endpoint, "/"), bucket_(bucket), object_(object) { } + //---------------------------------------------------------------------------------------------------------------------- void S3ObjectName::print(std::ostream& out) const { @@ -61,6 +64,10 @@ auto S3ObjectName::exists() const -> bool { return client()->objectExists(bucket_, object_); } +auto S3ObjectName::bucketExists() const -> bool { + return client()->bucketExists(bucket_); +} + void S3ObjectName::remove() { client()->deleteObject(bucket_, object_); } diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h index 82a7be512..7a8f653ef 100644 --- a/src/eckit/io/s3/S3ObjectName.h +++ b/src/eckit/io/s3/S3ObjectName.h @@ -32,7 +32,7 @@ class S3ObjectName: public S3Name { public: // methods explicit S3ObjectName(const URI& uri); - auto bucket() const -> const std::string& { return bucket_; } + S3ObjectName(const net::Endpoint& endpoint, const std::string& bucket, const std::string& object); auto uri() const -> URI; @@ -54,6 +54,12 @@ class S3ObjectName: public S3Name { auto asString() const -> std::string override; + auto name() const -> const std::string& { return object_; } + + auto bucket() const -> const std::string& { return bucket_; } + + auto bucketExists() const -> bool; + private: // methods void print(std::ostream& out) const override; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 2436fba57..27c51d0ff 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include @@ -54,9 +54,9 @@ inline std::string awsErrorMessage(const std::string& msg, const Aws::S3::S3Erro class BufferIOStream: public Aws::IOStream { public: - /// @todo reinterpret_cast - BufferIOStream(void* buffer, const uint64_t size): - Aws::IOStream(new Aws::Utils::Stream::PreallocatedStreamBuf(reinterpret_cast(buffer), size)) { } + BufferIOStream(void* buffer, const uint64_t length): + Aws::IOStream(new Aws::Utils::Stream::PreallocatedStreamBuf(reinterpret_cast(buffer), length)) { + } ~BufferIOStream() override { delete rdbuf(); } }; @@ -73,7 +73,7 @@ S3ClientAWS::~S3ClientAWS() = default; //---------------------------------------------------------------------------------------------------------------------- void S3ClientAWS::configure(const S3Config& config) { - LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client.\n"; + LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client..." << std::endl; Aws::Client::ClientConfiguration configuration; @@ -130,13 +130,15 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { throw S3SeriousBug(msg, Here()); } - LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << "\n"; + LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; } void S3ClientAWS::emptyBucket(const std::string& bucket) const { deleteObjects(bucket, listObjects(bucket)); } +//---------------------------------------------------------------------------------------------------------------------- + void S3ClientAWS::deleteBucket(const std::string& bucket) const { Aws::S3::Model::DeleteBucketRequest request; request.SetBucket(bucket); @@ -147,12 +149,17 @@ void S3ClientAWS::deleteBucket(const std::string& bucket) const { const auto& err = outcome.GetError(); const auto msg = awsErrorMessage("Failed to delete bucket=" + bucket, err); if (err.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_BUCKET) { throw S3EntityNotFound(msg, Here()); } + if (err.GetErrorType() == Aws::S3::S3Errors::UNKNOWN && err.GetExceptionName() == "BucketNotEmpty") { + throw S3BucketNotEmpty(msg, Here()); + } throw S3SeriousBug(msg, Here()); } - LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucket << "\n"; + LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucket << std::endl; } +//---------------------------------------------------------------------------------------------------------------------- + auto S3ClientAWS::bucketExists(const std::string& bucket) const -> bool { Aws::S3::Model::HeadBucketRequest request; @@ -161,6 +168,8 @@ auto S3ClientAWS::bucketExists(const std::string& bucket) const -> bool { return client_->HeadBucket(request).IsSuccess(); } +//---------------------------------------------------------------------------------------------------------------------- + auto S3ClientAWS::listBuckets() const -> std::vector { auto outcome = client_->ListBuckets(); @@ -196,7 +205,7 @@ auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object auto outcome = client_->PutObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Put object=" << object << " [len=" << length << "] to bucket=" << bucket << "\n"; + LOG_DEBUG_LIB(LibEcKit) << "Put object=" << object << " [len=" << length << "] to bucket=" << bucket << std::endl; /// @todo actual size of written bytes return length; } @@ -215,23 +224,24 @@ auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object request.SetBucket(bucket); request.SetKey(object); request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(ALLOC_TAG, buffer, length); }); - if (offset > 0) { request.SetRange(std::to_string(offset) + "-" + std::to_string(offset + length)); } + /// @todo range and streambuf + // request.SetRange(std::to_string(offset) + "-" + std::to_string(offset + length)); auto outcome = client_->GetObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Get object=" << object << " from bucket=" << bucket << "\n"; + LOG_DEBUG_LIB(LibEcKit) << "Get object=" << object << " from bucket=" << bucket << std::endl; LOG_DEBUG_LIB(LibEcKit) << "Req. len=" << length << ", Obj. len=" << outcome.GetResult().GetContentLength() - << "\n"; + << std::endl; return outcome.GetResult().GetContentLength(); } - auto msg = awsErrorMessage("Failed to retrieve object=" + object + " to bucket=" + bucket, outcome.GetError()); + auto msg = awsErrorMessage("Failed to retrieve object=" + object + " from bucket=" + bucket, outcome.GetError()); throw S3SeriousBug(msg, Here()); } //---------------------------------------------------------------------------------------------------------------------- -// OBJECT +// DELETE OBJECT void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& object) const { Aws::S3::Model::DeleteObjectRequest request; @@ -246,9 +256,11 @@ void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& obj throw S3SeriousBug(msg, Here()); } - LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << " in bucket=" << bucket << "\n"; + LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << " in bucket=" << bucket << std::endl; } +//---------------------------------------------------------------------------------------------------------------------- + void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vector& objects) const { if (objects.empty()) { return; } @@ -269,30 +281,36 @@ void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vectorlistBuckets(); - std::set buckets(tmp.begin(), tmp.end()); - - for (const std::string& name : {TEST_BUCKET}) { - if (buckets.find(name) != buckets.end()) { + auto client = S3Client::makeUnique(cfg); + for (const auto& name : {TEST_BUCKET}) { + if (client->bucketExists(name)) { client->emptyBucket(name); client->deleteBucket(name); } } } -CASE("bucket exists") { +//---------------------------------------------------------------------------------------------------------------------- + +CASE("invalid bucket") { + { + EXPECT_THROWS(S3BucketName("http://127.0.0.1:9000/" + TEST_BUCKET)); + EXPECT_THROWS(S3BucketName("s3://127.0.0.1" + TEST_BUCKET)); + EXPECT_THROWS(S3BucketName("s3://127.0.0.1/" + TEST_BUCKET)); + } +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3BucketName: no bucket") { + ensureClean(); + + S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + + EXPECT_NOT(bucket.exists()); + + // LIST + EXPECT_THROWS(bucket.listObjects()); + + // CREATE OBJECT + EXPECT_THROWS(bucket.makeObject(TEST_OBJECT)->put(TEST_DATA.data(), TEST_DATA.size())); + + // DESTROY BUCKET + EXPECT_THROWS(bucket.destroy()); + EXPECT_NO_THROW(bucket.ensureDestroyed()); +} + +CASE("S3BucketName: create bucket") { + ensureClean(); + + S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + + EXPECT_NOT(bucket.exists()); + + // CREATE BUCKET + EXPECT_NO_THROW(bucket.create()); + EXPECT_THROWS(bucket.create()); + EXPECT_NO_THROW(bucket.ensureCreated()); + + // DESTROY BUCKET + EXPECT_NO_THROW(bucket.destroy()); +} + +CASE("S3BucketName: empty bucket") { + ensureClean(); + + S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + + // CREATE BUCKET + EXPECT_NO_THROW(bucket.ensureCreated()); + EXPECT(bucket.exists()); + + // LIST + EXPECT(bucket.listObjects().size() == 0); + + // DESTROY BUCKET + EXPECT_NO_THROW(bucket.destroy()); +} + +CASE("S3BucketName: bucket with object") { + ensureClean(); + + S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + + // CREATE BUCKET + EXPECT_NO_THROW(bucket.ensureCreated()); + + // CREATE OBJECT + EXPECT_NO_THROW(bucket.makeObject(TEST_OBJECT)->put(TEST_DATA.data(), TEST_DATA.size())); + + // LIST + EXPECT(bucket.listObjects().size() == 1); + + // DESTROY BUCKET + EXPECT_THROWS(bucket.destroy()); + EXPECT_NO_THROW(bucket.ensureDestroyed()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3Handle operations") { ensureClean(); S3Client::makeUnique(cfg)->createBucket(TEST_BUCKET); @@ -59,72 +140,118 @@ CASE("bucket exists") { const void* buffer = TEST_DATA.data(); const long length = TEST_DATA.size(); - URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); { - S3ObjectName name(uri); + S3ObjectName object(uri); - std::unique_ptr h(name.dataHandle()); - h->openForWrite(length); - AutoClose closer(*h); - EXPECT(h->write(buffer, length) == length); + std::unique_ptr handle(object.dataHandle()); + EXPECT_NO_THROW(handle->openForWrite(length)); + EXPECT(handle->write(buffer, length) == length); + EXPECT_NO_THROW(handle->close()); } { - std::unique_ptr h(uri.newReadHandle()); - - const auto rlen = h->openForRead(); - EXPECT(rlen == Length(length)); + std::unique_ptr handle(uri.newReadHandle()); + EXPECT_NO_THROW(handle->openForRead()); std::string rbuf; rbuf.resize(length); - const auto len = h->read(rbuf.data(), length); - EXPECT(len == length); + EXPECT_NO_THROW(handle->read(rbuf.data(), length)); EXPECT(rbuf == TEST_DATA); + + EXPECT_NO_THROW(handle->close()); } { - std::unique_ptr dh(uri.newReadHandle()); + std::unique_ptr handle(uri.newReadHandle()); - MemoryHandle mh(length); - dh->saveInto(mh); + MemoryHandle memHandle(length); + handle->saveInto(memHandle); - EXPECT(mh.size() == Length(length)); - EXPECT(::memcmp(mh.data(), buffer, length) == 0); + EXPECT(memHandle.size() == Length(length)); + EXPECT(::memcmp(memHandle.data(), buffer, length) == 0); + + EXPECT_NO_THROW(handle->close()); } } CASE("S3Handle::openForWrite") { - std::cout << "===================" << std::endl; - ensureClean(); - URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); - std::unique_ptr h(uri.newWriteHandle()); + { // NO BUCKET + std::unique_ptr handle(uri.newWriteHandle()); + EXPECT_THROWS(handle->openForWrite(0)); + EXPECT_NO_THROW(handle->close()); + } - // no bucket - EXPECT_NO_THROW(h->openForWrite(0)); + // CREATE BUCKET + EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); - // no read - EXPECT_THROWS(h->openForRead()); + { // BUCKET EXISTS + std::unique_ptr handle(uri.newWriteHandle()); + EXPECT_NO_THROW(handle->openForWrite(0)); + EXPECT_NO_THROW(handle->close()); + } } CASE("S3Handle::openForRead") { - std::cout << "===================" << std::endl; + ensureClean(); + + const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); + + std::unique_ptr handle(uri.newReadHandle()); + + // NO OBJECT + EXPECT_THROWS(handle->openForRead()); + + // CREATE OBJECT + EXPECT_NO_THROW(S3ObjectName(uri).put(nullptr, 0)); + + // DOUBLE OPEN + EXPECT_THROWS(handle->openForRead()); + + // RE-OPEN + EXPECT_NO_THROW(handle->close()); + EXPECT_NO_THROW(handle->openForRead()); +} + +CASE("S3Handle::read") { ensureClean(); - S3Client::makeUnique(cfg)->createBucket(TEST_BUCKET); + const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); - URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); - { - std::unique_ptr h(uri.newReadHandle()); - EXPECT_THROWS(h->openForRead()); + // CREATE OBJECT + EXPECT_NO_THROW(S3ObjectName(uri).put(TEST_DATA.data(), TEST_DATA.size())); + + std::unique_ptr handle(uri.newReadHandle()); + + // OPEN + EXPECT_NO_THROW(handle->openForRead()); + + { /// @todo range based read + + // const auto length = TEST_DATA.size(); + + // std::string rbuf; + // rbuf.resize(length); + // auto len = handle->read(rbuf.data(), 10); + // std::cout << "========" << rbuf << std::endl; + // len = handle->read(rbuf.data(), length - 10); + // std::cout << "========" << rbuf << std::endl; + // EXPECT(rbuf == TEST_DATA); } + + // CLOSE + EXPECT_NO_THROW(handle->close()); } //---------------------------------------------------------------------------------------------------------------------- @@ -137,8 +264,9 @@ int main(int argc, char** argv) { int ret = -1; try { - eckit::test::ensureClean(); ret = run_tests(argc, argv); - } catch (...) { } + } + catch (...) { + } return ret; } From 9cb443b64a1e7b2be5369ba188b7b74f735a2438 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 19 Feb 2024 19:51:45 +0100 Subject: [PATCH 67/90] fix(S3): S3Client and IMDS bugs --- src/eckit/io/s3/S3Client.cc | 24 ++-------- src/eckit/io/s3/S3Client.h | 2 - src/eckit/io/s3/aws/S3ClientAWS.cc | 70 ++++++++++++++++------------- src/eckit/io/s3/aws/S3ClientAWS.h | 6 ++- src/eckit/io/s3/aws/S3ContextAWS.cc | 7 +-- 5 files changed, 47 insertions(+), 62 deletions(-) diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index bd8116e16..d628e8cb8 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -21,20 +21,6 @@ namespace eckit { -namespace { - -inline auto makeSharedS3Client(const S3Types type) -> std::shared_ptr { - if (type == S3Types::AWS) { return std::make_shared(); } - throw S3SeriousBug("Unkown S3 client type!", Here()); -} - -inline auto makeUniqueS3Client(const S3Types type) -> std::unique_ptr { - if (type == S3Types::AWS) { return std::make_unique(); } - throw S3SeriousBug("Unkown S3 client type!", Here()); -} - -} // namespace - //---------------------------------------------------------------------------------------------------------------------- S3Client::S3Client(std::shared_ptr context): context_(context) { } @@ -59,17 +45,15 @@ void S3Client::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- auto S3Client::makeShared(const S3Config& config) -> std::shared_ptr { - auto client = makeSharedS3Client(config.type); - client->configure(config); - return client; + if (config.type == S3Types::AWS) { return std::make_shared(config); } + throw S3SeriousBug("Unkown S3 client type!", Here()); } //---------------------------------------------------------------------------------------------------------------------- auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { - auto client = makeUniqueS3Client(config.type); - client->configure(config); - return client; + if (config.type == S3Types::AWS) { return std::make_unique(config); } + throw S3SeriousBug("Unkown S3 client type!", Here()); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 8a198e78c..69f65b226 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -39,8 +39,6 @@ class S3Client: private NonCopyable { static auto makeUnique(const S3Config& config) -> std::unique_ptr; - virtual void configure(const S3Config& config) = 0; - virtual void createBucket(const std::string& bucket) const = 0; virtual void emptyBucket(const std::string& bucket) const = 0; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 27c51d0ff..d28cd6cb4 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -21,9 +21,8 @@ #include "eckit/io/s3/S3Session.h" #include "eckit/io/s3/aws/S3ContextAWS.h" -#include +#include #include -#include #include #include #include @@ -38,10 +37,6 @@ #include #include -namespace eckit { - -const auto ALLOC_TAG = "S3ClientAWS"; - //---------------------------------------------------------------------------------------------------------------------- namespace { @@ -65,8 +60,13 @@ class BufferIOStream: public Aws::IOStream { //---------------------------------------------------------------------------------------------------------------------- -S3ClientAWS::S3ClientAWS(): - S3Client(S3Session::instance().getContext(S3Types::AWS)), client_(std::make_unique()) { } +namespace eckit { + +const auto ALLOC_TAG = "S3ClientAWS"; + +S3ClientAWS::S3ClientAWS(const S3Config& config): S3Client(S3Session::instance().getContext(S3Types::AWS)) { + configure(config); +} S3ClientAWS::~S3ClientAWS() = default; @@ -75,16 +75,19 @@ S3ClientAWS::~S3ClientAWS() = default; void S3ClientAWS::configure(const S3Config& config) { LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client..." << std::endl; - Aws::Client::ClientConfiguration configuration; + Aws::Client::ClientConfigurationInitValues initVal; + initVal.shouldDisableIMDS = true; + Aws::Client::ClientConfiguration configuration(initVal); // we are not an ec2 instance - configuration.disableIMDS = true; + configuration.disableIMDS = true; + configuration.disableImdsV1 = true; // setup region // configuration.region = config.region; if (!config.region.empty()) { configuration.region = config.region; } - // configuration.proxyScheme = Aws::Http::Scheme::HTTPS; + // configuration.scheme = Aws::Http::Scheme::HTTPS; // configuration.verifySSL = false; // setup endpoint @@ -92,18 +95,16 @@ void S3ClientAWS::configure(const S3Config& config) { if (!config.endpoint.host().empty()) { configuration.endpointOverride = "http://" + config.endpoint.host(); } if (config.endpoint.port() > 0) { configuration.endpointOverride += ":" + std::to_string(config.endpoint.port()); } - // setup credentials - Aws::Auth::AWSCredentials credentials; if (auto cred = S3Session::instance().getCredentials(config.endpoint.host())) { - credentials.SetAWSAccessKeyId(cred->keyID); - credentials.SetAWSSecretKey(cred->secret); + // credentials provider + auto cProvider = Aws::MakeShared(ALLOC_TAG, cred->keyID, cred->secret); + // endpoint provider + auto eProvider = Aws::MakeShared(ALLOC_TAG); + // client + client_ = std::make_unique(cProvider, eProvider, configuration); + } else { + throw S3SeriousBug("No credentials found!", Here()); } - - // endpoint provider - auto provider = Aws::MakeShared(ALLOC_TAG); - - // finally client - client_ = std::make_unique(credentials, provider, configuration); } void S3ClientAWS::print(std::ostream& out) const { @@ -111,6 +112,11 @@ void S3ClientAWS::print(std::ostream& out) const { out << "S3ClientAWS[]"; } +auto S3ClientAWS::getClient() const -> Aws::S3::S3Client& { + if (client_) { return *client_; } + throw S3SeriousBug("Invalid client!", Here()); +} + //---------------------------------------------------------------------------------------------------------------------- // BUCKET @@ -118,7 +124,7 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { Aws::S3::Model::CreateBucketRequest request; request.SetBucket(bucket); - auto outcome = client_->CreateBucket(request); + auto outcome = getClient().CreateBucket(request); if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); @@ -143,7 +149,7 @@ void S3ClientAWS::deleteBucket(const std::string& bucket) const { Aws::S3::Model::DeleteBucketRequest request; request.SetBucket(bucket); - auto outcome = client_->DeleteBucket(request); + auto outcome = getClient().DeleteBucket(request); if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); @@ -165,13 +171,13 @@ auto S3ClientAWS::bucketExists(const std::string& bucket) const -> bool { request.SetBucket(bucket); - return client_->HeadBucket(request).IsSuccess(); + return getClient().HeadBucket(request).IsSuccess(); } //---------------------------------------------------------------------------------------------------------------------- auto S3ClientAWS::listBuckets() const -> std::vector { - auto outcome = client_->ListBuckets(); + auto outcome = getClient().ListBuckets(); if (outcome.IsSuccess()) { std::vector buckets; @@ -202,7 +208,7 @@ auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object request.SetBody(Aws::MakeShared(ALLOC_TAG)); } - auto outcome = client_->PutObject(request); + auto outcome = getClient().PutObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Put object=" << object << " [len=" << length << "] to bucket=" << bucket << std::endl; @@ -227,7 +233,7 @@ auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object /// @todo range and streambuf // request.SetRange(std::to_string(offset) + "-" + std::to_string(offset + length)); - auto outcome = client_->GetObject(request); + auto outcome = getClient().GetObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Get object=" << object << " from bucket=" << bucket << std::endl; @@ -249,7 +255,7 @@ void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& obj request.SetBucket(bucket); request.SetKey(object); - auto outcome = client_->DeleteObject(request); + auto outcome = getClient().DeleteObject(request); if (!outcome.IsSuccess()) { auto msg = awsErrorMessage("Failed to delete object=" + object + " in bucket=" + bucket, outcome.GetError()); @@ -274,7 +280,7 @@ void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vectorDeleteObjects(request); + auto outcome = getClient().DeleteObjects(request); if (!outcome.IsSuccess()) { auto msg = awsErrorMessage("Failed to delete objects in bucket=" + bucket, outcome.GetError()); @@ -292,7 +298,7 @@ auto S3ClientAWS::listObjects(const std::string& bucket) const -> std::vectorListObjectsV2(request); + auto outcome = getClient().ListObjectsV2(request); if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); @@ -317,7 +323,7 @@ auto S3ClientAWS::objectExists(const std::string& bucket, const std::string& obj request.SetBucket(bucket); request.SetKey(object); - return client_->HeadObject(request).IsSuccess(); + return getClient().HeadObject(request).IsSuccess(); } //---------------------------------------------------------------------------------------------------------------------- @@ -328,7 +334,7 @@ auto S3ClientAWS::objectSize(const std::string& bucket, const std::string& objec request.SetBucket(bucket); request.SetKey(object); - auto outcome = client_->HeadObject(request); + auto outcome = getClient().HeadObject(request); if (outcome.IsSuccess()) { return outcome.GetResult().GetContentLength(); } diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index aaf7a3c3d..2d4530e9c 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -29,11 +29,11 @@ namespace eckit { class S3ClientAWS: public S3Client { public: // methods - S3ClientAWS(); + S3ClientAWS(const S3Config& config); ~S3ClientAWS(); - void configure(const S3Config& config) override; + void configure(const S3Config& config); void createBucket(const std::string& bucket) const override; @@ -64,6 +64,8 @@ class S3ClientAWS: public S3Client { private: // methods void print(std::ostream& out) const override; + auto getClient() const -> Aws::S3::S3Client&; + private: // members std::unique_ptr client_; }; diff --git a/src/eckit/io/s3/aws/S3ContextAWS.cc b/src/eckit/io/s3/aws/S3ContextAWS.cc index b49303fac..3805d4282 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.cc +++ b/src/eckit/io/s3/aws/S3ContextAWS.cc @@ -15,23 +15,18 @@ #include "eckit/io/s3/aws/S3ContextAWS.h" -#include "eckit/config/LibEcKit.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/aws/S3ClientAWS.h" -#include - namespace eckit { //---------------------------------------------------------------------------------------------------------------------- S3ContextAWS::S3ContextAWS(const Aws::SDKOptions& options): S3Context(S3Types::AWS), options_(options) { - LOG_DEBUG_LIB(LibEcKit) << "Initialize AWS API!" << std::endl; Aws::InitAPI(options_); } S3ContextAWS::~S3ContextAWS() { - LOG_DEBUG_LIB(LibEcKit) << "Shutdown AWS API!" << std::endl; Aws::ShutdownAPI(options_); } @@ -39,7 +34,7 @@ S3ContextAWS::~S3ContextAWS() { auto S3ContextAWS::makeShared() -> std::shared_ptr { Aws::SDKOptions options; - /// @todo: remove debugging logs + /// @todo remove debug stuff below options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; return std::make_shared(options); } From b4c8f7fdd62b005c1c3c60cc609d29933ee37ba6 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 19 Feb 2024 20:12:30 +0100 Subject: [PATCH 68/90] feat(S3): tests 1000 object writes --- tests/io/test_s3handle.cc | 59 +++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 2778e9400..5af8a53eb 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -178,27 +178,6 @@ CASE("S3Handle operations") { } } -CASE("S3Handle::openForWrite") { - ensureClean(); - - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); - - { // NO BUCKET - std::unique_ptr handle(uri.newWriteHandle()); - EXPECT_THROWS(handle->openForWrite(0)); - EXPECT_NO_THROW(handle->close()); - } - - // CREATE BUCKET - EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); - - { // BUCKET EXISTS - std::unique_ptr handle(uri.newWriteHandle()); - EXPECT_NO_THROW(handle->openForWrite(0)); - EXPECT_NO_THROW(handle->close()); - } -} - CASE("S3Handle::openForRead") { ensureClean(); @@ -222,6 +201,27 @@ CASE("S3Handle::openForRead") { EXPECT_NO_THROW(handle->openForRead()); } +CASE("S3Handle::openForWrite") { + ensureClean(); + + const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + + { // NO BUCKET + std::unique_ptr handle(uri.newWriteHandle()); + EXPECT_THROWS(handle->openForWrite(0)); + EXPECT_NO_THROW(handle->close()); + } + + // CREATE BUCKET + EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); + + { // BUCKET EXISTS + std::unique_ptr handle(uri.newWriteHandle()); + EXPECT_NO_THROW(handle->openForWrite(0)); + EXPECT_NO_THROW(handle->close()); + } +} + CASE("S3Handle::read") { ensureClean(); @@ -254,6 +254,23 @@ CASE("S3Handle::read") { EXPECT_NO_THROW(handle->close()); } +CASE("write 1000 objects") { + ensureClean(); + + const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET); + + S3BucketName bucket(uri); + + EXPECT_NO_THROW(bucket.ensureCreated()); + + EXPECT_NO_THROW({ + for (int i = 0; i < 1000; i++) { + const auto objName = TEST_OBJECT + std::to_string(i); + bucket.makeObject(objName)->put(TEST_DATA.data(), TEST_DATA.size()); + } + }); +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test From b9e30d26c496aa67194c8953f2d2a529ae329f39 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 20 Feb 2024 09:58:54 +0100 Subject: [PATCH 69/90] feat(S3): write performance tests --- tests/io/test_s3handle.cc | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 5af8a53eb..6498939da 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -14,12 +14,15 @@ /// @date Jan 2024 #include "eckit/filesystem/URI.h" +#include "eckit/io/Buffer.h" #include "eckit/io/MemoryHandle.h" #include "eckit/io/s3/S3BucketName.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Handle.h" #include "eckit/io/s3/S3ObjectName.h" #include "eckit/io/s3/S3Session.h" +#include "eckit/log/Bytes.h" +#include "eckit/log/Timer.h" #include "eckit/testing/Test.h" #include @@ -39,6 +42,23 @@ S3Config cfg("eu-central-1", "127.0.0.1", 9000); //---------------------------------------------------------------------------------------------------------------------- +void writePerformance(S3BucketName& bucket, const int count) { + eckit::Timer timer; + + Buffer buffer(1024 * 1024); + buffer.zero(); + + timer.start(); + for (int i = 0; i < count; i++) { + const auto objName = TEST_OBJECT + std::to_string(i); + bucket.makeObject(objName)->put(buffer.data(), buffer.size()); + } + timer.stop(); + + std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count + << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; +} + void ensureClean() { auto client = S3Client::makeUnique(cfg); for (const auto& name : {TEST_BUCKET}) { @@ -254,7 +274,7 @@ CASE("S3Handle::read") { EXPECT_NO_THROW(handle->close()); } -CASE("write 1000 objects") { +CASE("performance: write 1000 objects") { ensureClean(); const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET); @@ -263,12 +283,11 @@ CASE("write 1000 objects") { EXPECT_NO_THROW(bucket.ensureCreated()); - EXPECT_NO_THROW({ - for (int i = 0; i < 1000; i++) { - const auto objName = TEST_OBJECT + std::to_string(i); - bucket.makeObject(objName)->put(TEST_DATA.data(), TEST_DATA.size()); - } - }); + writePerformance(bucket, 10); + + writePerformance(bucket, 100); + + writePerformance(bucket, 1000); } //---------------------------------------------------------------------------------------------------------------------- From a9f0cd1f5557545449277539e30cee1c235a030f Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 20 Feb 2024 20:55:22 +0100 Subject: [PATCH 70/90] feat(S3): improved perf 20% .. reuse S3Client --- src/eckit/io/s3/S3Client.cc | 7 +++++- src/eckit/io/s3/S3Client.h | 6 ++++- src/eckit/io/s3/S3Context.h | 12 ++++++---- src/eckit/io/s3/S3Handle.cc | 2 +- src/eckit/io/s3/S3Name.cc | 4 +++- src/eckit/io/s3/S3Session.cc | 37 ++++++++++++++++++++++++++++++ src/eckit/io/s3/S3Session.h | 26 ++++++++++++++++----- src/eckit/io/s3/aws/S3ClientAWS.cc | 2 +- src/eckit/io/s3/aws/S3ContextAWS.h | 6 ++++- 9 files changed, 86 insertions(+), 16 deletions(-) diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index d628e8cb8..0c4ec22cd 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -15,6 +15,7 @@ #include "eckit/io/s3/S3Client.h" +#include "S3Client.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Session.h" #include "eckit/io/s3/aws/S3ClientAWS.h" @@ -23,7 +24,7 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Client::S3Client(std::shared_ptr context): context_(context) { } +S3Client::S3Client(const S3Config& config, std::shared_ptr context): config_(config), context_(context) { } S3Client::~S3Client() = default; @@ -56,6 +57,10 @@ auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { throw S3SeriousBug("Unkown S3 client type!", Here()); } +auto S3Client::endpoint() const -> net::Endpoint { + return config_.endpoint; +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 69f65b226..b14388ccd 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -39,6 +39,8 @@ class S3Client: private NonCopyable { static auto makeUnique(const S3Config& config) -> std::unique_ptr; + virtual auto endpoint() const -> net::Endpoint; + virtual void createBucket(const std::string& bucket) const = 0; virtual void emptyBucket(const std::string& bucket) const = 0; @@ -71,11 +73,13 @@ class S3Client: private NonCopyable { } protected: // methods - explicit S3Client(std::shared_ptr context); + explicit S3Client(const S3Config& config, std::shared_ptr context); virtual void print(std::ostream& out) const; private: // members + S3Config config_; + std::shared_ptr context_; }; diff --git a/src/eckit/io/s3/S3Context.h b/src/eckit/io/s3/S3Context.h index a16788f12..19653e92c 100644 --- a/src/eckit/io/s3/S3Context.h +++ b/src/eckit/io/s3/S3Context.h @@ -20,7 +20,6 @@ #pragma once #include "eckit/io/s3/S3Config.h" -#include "eckit/io/s3/S3Macros.h" #include @@ -30,14 +29,19 @@ namespace eckit { class S3Context { public: // methods - NO_COPY_NO_MOVE(S3Context) + S3Context(const S3Context&) = delete; + S3Context& operator=(const S3Context&) = delete; + S3Context(S3Context&&) = delete; + S3Context& operator=(S3Context&&) = delete; explicit S3Context(S3Types type); virtual ~S3Context(); - NODISCARD - auto getType() const -> S3Types { return type_; } + [[nodiscard]] + auto getType() const -> S3Types { + return type_; + } private: // methods friend class S3Session; diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index bc88a7228..838f8d583 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -61,7 +61,7 @@ long S3Handle::read(void* buffer, const long length) { ASSERT(mode_ == Mode::READ); ASSERT(0 <= pos_); - if (size() == pos_) { return 0; } + if (size() <= pos_) { return 0; } const auto len = name_.get(buffer, pos_, length); diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 212ba3279..05609f4ab 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -18,6 +18,7 @@ #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Session.h" #include "eckit/utils/Tokenizer.h" namespace eckit { @@ -49,7 +50,8 @@ auto S3Name::parseName() const -> std::vector { } auto S3Name::client() const -> std::shared_ptr { - return S3Client::makeShared({endpoint_}); + /// @todo + return S3Session::instance().getClient({endpoint_}); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index e743546c6..d9e519034 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -24,6 +24,16 @@ namespace eckit { namespace { +/// @brief Functor for S3Context type +struct IsClientEndpoint { + IsClientEndpoint(const net::Endpoint& endpoint): endpoint_(endpoint) { } + + bool operator()(const std::shared_ptr& client) const { return client->endpoint() == endpoint_; } + +private: + const net::Endpoint& endpoint_; +}; + /// @brief Functor for S3Context type struct IsContextType { const S3Types type_; @@ -56,6 +66,33 @@ S3Session::S3Session() = default; S3Session::~S3Session() = default; +//---------------------------------------------------------------------------------------------------------------------- +// CLIENT + +auto S3Session::getClient(const S3Config& config) -> std::shared_ptr { + // return if found + if (auto client = findClient(config.endpoint)) { return client; } + + // not found + auto client = S3Client::makeShared(config); + client_.push_back(client); + + return client; +} + +auto S3Session::findClient(const net::Endpoint& endpoint) -> std::shared_ptr { + // search by type + const auto client = std::find_if(client_.begin(), client_.end(), IsClientEndpoint(endpoint)); + // found + if (client != client_.end()) { return *client; } + // not found + return {}; +} + +void S3Session::removeClient(const net::Endpoint& endpoint) { + client_.remove_if(IsClientEndpoint(endpoint)); +} + //---------------------------------------------------------------------------------------------------------------------- // CREDENTIALS diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h index 31232e8ca..16947b81b 100644 --- a/src/eckit/io/s3/S3Session.h +++ b/src/eckit/io/s3/S3Session.h @@ -27,37 +27,51 @@ namespace eckit { +class S3Client; + //---------------------------------------------------------------------------------------------------------------------- class S3Session { public: // methods - NO_COPY_NO_MOVE(S3Session) + S3Session(const S3Session&) = delete; + S3Session& operator=(const S3Session&) = delete; + S3Session(S3Session&&) = delete; + S3Session& operator=(S3Session&&) = delete; static S3Session& instance(); - NODISCARD + [[nodiscard]] auto getCredentials(const std::string& endpoint) const -> std::shared_ptr; void addCredentials(const S3Credential& credential); void removeCredentials(const std::string& endpoint); - NODISCARD + void clear(); + + [[nodiscard]] + auto getClient(const S3Config& config) -> std::shared_ptr; + + void removeClient(const net::Endpoint& endpoint); + + [[nodiscard]] auto getContext(S3Types type) -> std::shared_ptr; void removeContext(S3Types type); - void clear(); - private: // methods S3Session(); ~S3Session(); - NODISCARD + [[nodiscard]] auto findContext(S3Types type) -> std::shared_ptr; + [[nodiscard]] + auto findClient(const net::Endpoint& endpoint) -> std::shared_ptr; + private: // members + std::list> client_; std::list> contexts_; std::list> credentials_; }; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index d28cd6cb4..d3fda63d4 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -64,7 +64,7 @@ namespace eckit { const auto ALLOC_TAG = "S3ClientAWS"; -S3ClientAWS::S3ClientAWS(const S3Config& config): S3Client(S3Session::instance().getContext(S3Types::AWS)) { +S3ClientAWS::S3ClientAWS(const S3Config& config): S3Client(config, S3Session::instance().getContext(S3Types::AWS)) { configure(config); } diff --git a/src/eckit/io/s3/aws/S3ContextAWS.h b/src/eckit/io/s3/aws/S3ContextAWS.h index 7f760c936..5feb391a8 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.h +++ b/src/eckit/io/s3/aws/S3ContextAWS.h @@ -20,6 +20,7 @@ #pragma once #include "eckit/io/s3/S3Context.h" +#include "eckit/io/s3/S3Macros.h" #include @@ -31,7 +32,10 @@ namespace eckit { class S3ContextAWS: public S3Context { public: // methods - NO_MOVE(S3ContextAWS) + S3ContextAWS(const S3ContextAWS&) = delete; + S3ContextAWS& operator=(const S3ContextAWS&) = delete; + S3ContextAWS(S3ContextAWS&&) = delete; + S3ContextAWS& operator=(S3ContextAWS&&) = delete; explicit S3ContextAWS(const Aws::SDKOptions& options); From 17fd771f0923de4f76f0390ad9d1a5ca40bdffc3 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 20 Feb 2024 21:59:08 +0100 Subject: [PATCH 71/90] fix(S3): removed S3Context --- src/eckit/CMakeLists.txt | 2 - src/eckit/io/s3/S3Client.cc | 15 ++------ src/eckit/io/s3/S3Client.h | 6 +-- src/eckit/io/s3/S3Config.cc | 14 ++++++- src/eckit/io/s3/S3Config.h | 6 ++- src/eckit/io/s3/S3Context.cc | 43 ---------------------- src/eckit/io/s3/S3Context.h | 57 ----------------------------- src/eckit/io/s3/S3ObjectName.cc | 1 + src/eckit/io/s3/S3Session.cc | 49 ++++--------------------- src/eckit/io/s3/S3Session.h | 19 ++++------ src/eckit/io/s3/aws/S3ClientAWS.cc | 7 +--- src/eckit/io/s3/aws/S3ClientAWS.h | 8 ++-- src/eckit/io/s3/aws/S3ContextAWS.cc | 24 ++++++------ src/eckit/io/s3/aws/S3ContextAWS.h | 16 +++----- tests/io/test_s3handle.cc | 8 ++-- 15 files changed, 65 insertions(+), 210 deletions(-) delete mode 100644 src/eckit/io/s3/S3Context.cc delete mode 100644 src/eckit/io/s3/S3Context.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index ce00cccb6..68903fe9e 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -298,8 +298,6 @@ io/s3/S3Client.cc io/s3/S3Client.h io/s3/S3Config.cc io/s3/S3Config.h -io/s3/S3Context.cc -io/s3/S3Context.h io/s3/S3Credential.h io/s3/S3Exception.cc io/s3/S3Exception.h diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index 0c4ec22cd..2a269ef85 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -24,23 +24,14 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Client::S3Client(const S3Config& config, std::shared_ptr context): config_(config), context_(context) { } +S3Client::S3Client(const S3Config& config): config_(config) { } S3Client::~S3Client() = default; //---------------------------------------------------------------------------------------------------------------------- void S3Client::print(std::ostream& out) const { - out << "S3Client[type="; - const auto type = context_->getType(); - if (type == S3Types::AWS) { - out << "AWS"; - } else if (type == S3Types::REST) { - out << "REST"; - } else { - out << "NONE"; - } - out << "]"; + out << "S3Client[config=" << config_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- @@ -57,7 +48,7 @@ auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { throw S3SeriousBug("Unkown S3 client type!", Here()); } -auto S3Client::endpoint() const -> net::Endpoint { +auto S3Client::endpoint() const -> const net::Endpoint& { return config_.endpoint; } diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index b14388ccd..d42e3792a 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -39,7 +39,7 @@ class S3Client: private NonCopyable { static auto makeUnique(const S3Config& config) -> std::unique_ptr; - virtual auto endpoint() const -> net::Endpoint; + virtual auto endpoint() const -> const net::Endpoint&; virtual void createBucket(const std::string& bucket) const = 0; @@ -73,14 +73,12 @@ class S3Client: private NonCopyable { } protected: // methods - explicit S3Client(const S3Config& config, std::shared_ptr context); + explicit S3Client(const S3Config& config); virtual void print(std::ostream& out) const; private: // members S3Config config_; - - std::shared_ptr context_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Config.cc b/src/eckit/io/s3/S3Config.cc index b78aef069..8eedb780d 100644 --- a/src/eckit/io/s3/S3Config.cc +++ b/src/eckit/io/s3/S3Config.cc @@ -15,6 +15,8 @@ #include "eckit/io/s3/S3Config.h" +#include "eckit/filesystem/URI.h" + namespace eckit { //---------------------------------------------------------------------------------------------------------------------- @@ -31,7 +33,17 @@ S3Config::S3Config(const net::Endpoint& e): endpoint(e) { } //---------------------------------------------------------------------------------------------------------------------- void S3Config::print(std::ostream& out) const { - out << "S3Config[region=" << region << ",endpoint=" << endpoint << "]"; + out << "S3Config[type="; + if (type == S3Types::AWS) { + out << "AWS"; + } else if (type == S3Types::MINIO) { + out << "MinIO"; + } else if (type == S3Types::REST) { + out << "REST"; + } else { + out << "NONE"; + } + out << ",region=" << region << ",endpoint=" << endpoint << "]"; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 4ccf2eff8..5b5b981b9 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -19,13 +19,15 @@ #pragma once -#include "eckit/filesystem/URI.h" +#include "eckit/net/Endpoint.h" #include namespace eckit { -enum class S3Types { NONE, AWS, REST }; +class URI; + +enum class S3Types { NONE, AWS, MINIO, REST }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Context.cc b/src/eckit/io/s3/S3Context.cc deleted file mode 100644 index 69b7fd30e..000000000 --- a/src/eckit/io/s3/S3Context.cc +++ /dev/null @@ -1,43 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/* - * This software was developed as part of the EC H2020 funded project IO-SEA - * (Project ID: 955811) iosea-project.eu - */ - -#include "eckit/io/s3/S3Context.h" - -#include "eckit/exception/Exceptions.h" -#include "eckit/io/s3/aws/S3ContextAWS.h" - -namespace eckit { - -//---------------------------------------------------------------------------------------------------------------------- - -S3Context::S3Context(const S3Types type): type_(type) { } - -S3Context::~S3Context() = default; - -//---------------------------------------------------------------------------------------------------------------------- - -auto S3Context::makeShared(S3Types type) -> std::shared_ptr { - // AWS SDK API - if (type == S3Types::AWS) { return S3ContextAWS::makeShared(); } - - // REST API - if (type == S3Types::REST) { NOTIMP; } - - return {}; -} - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace eckit diff --git a/src/eckit/io/s3/S3Context.h b/src/eckit/io/s3/S3Context.h deleted file mode 100644 index 19653e92c..000000000 --- a/src/eckit/io/s3/S3Context.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/* - * This software was developed as part of the EC H2020 funded project IO-SEA - * (Project ID: 955811) iosea-project.eu - */ - -/// @file S3Context.h -/// @author Metin Cakircali -/// @date Jan 2024 - -#pragma once - -#include "eckit/io/s3/S3Config.h" - -#include - -namespace eckit { - -//---------------------------------------------------------------------------------------------------------------------- - -class S3Context { -public: // methods - S3Context(const S3Context&) = delete; - S3Context& operator=(const S3Context&) = delete; - S3Context(S3Context&&) = delete; - S3Context& operator=(S3Context&&) = delete; - - explicit S3Context(S3Types type); - - virtual ~S3Context(); - - [[nodiscard]] - auto getType() const -> S3Types { - return type_; - } - -private: // methods - friend class S3Session; - - static auto makeShared(S3Types type) -> std::shared_ptr; - -private: // members - const S3Types type_ {S3Types::NONE}; -}; - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace eckit diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc index cf9c5b708..36f5da7d1 100644 --- a/src/eckit/io/s3/S3ObjectName.cc +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -16,6 +16,7 @@ #include "eckit/io/s3/S3ObjectName.h" #include "eckit/config/LibEcKit.h" +#include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Handle.h" diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index d9e519034..73de457c5 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -15,10 +15,9 @@ #include "eckit/io/s3/S3Session.h" -#include "eckit/config/LibEcKit.h" #include "eckit/io/s3/S3Credential.h" -#include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/aws/S3ClientAWS.h" +#include "eckit/net/Endpoint.h" namespace eckit { @@ -34,13 +33,6 @@ struct IsClientEndpoint { const net::Endpoint& endpoint_; }; -/// @brief Functor for S3Context type -struct IsContextType { - const S3Types type_; - - bool operator()(const std::shared_ptr& ctx) const { return ctx->getType() == type_; } -}; - /// @brief Functor for S3Credential endpoint struct IsCredentialEndpoint { IsCredentialEndpoint(const std::string& endpoint): endpoint_(endpoint) { } @@ -66,6 +58,13 @@ S3Session::S3Session() = default; S3Session::~S3Session() = default; +//---------------------------------------------------------------------------------------------------------------------- + +void S3Session::clear() { + client_.clear(); + credentials_.clear(); +} + //---------------------------------------------------------------------------------------------------------------------- // CLIENT @@ -118,38 +117,6 @@ void S3Session::removeCredentials(const std::string& endpoint) { credentials_.remove_if(IsCredentialEndpoint(endpoint)); } -//---------------------------------------------------------------------------------------------------------------------- -// CONTEXT - -auto S3Session::getContext(const S3Types type) -> std::shared_ptr { - // return if found - if (auto context = findContext(type)) { return context; } - - // not found - auto context = S3Context::makeShared(type); - contexts_.push_back(context); - - return context; -} - -auto S3Session::findContext(const S3Types type) -> std::shared_ptr { - // search by type - const auto context = std::find_if(contexts_.begin(), contexts_.end(), IsContextType({type})); - // found - if (context != contexts_.end()) { return *context; } - // not found - return {}; -} - -void S3Session::removeContext(const S3Types type) { - contexts_.remove_if(IsContextType({type})); -} - -void S3Session::clear() { - contexts_.clear(); - credentials_.clear(); -} - //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h index 16947b81b..6a8af08d2 100644 --- a/src/eckit/io/s3/S3Session.h +++ b/src/eckit/io/s3/S3Session.h @@ -19,7 +19,6 @@ #pragma once -#include "eckit/io/s3/S3Context.h" #include "eckit/io/s3/S3Credential.h" #include @@ -28,6 +27,11 @@ namespace eckit { class S3Client; +struct S3Config; + +namespace net { +class Endpoint; +} // namespace net //---------------------------------------------------------------------------------------------------------------------- @@ -40,6 +44,8 @@ class S3Session { static S3Session& instance(); + void clear(); + [[nodiscard]] auto getCredentials(const std::string& endpoint) const -> std::shared_ptr; @@ -47,32 +53,21 @@ class S3Session { void removeCredentials(const std::string& endpoint); - void clear(); - [[nodiscard]] auto getClient(const S3Config& config) -> std::shared_ptr; void removeClient(const net::Endpoint& endpoint); - [[nodiscard]] - auto getContext(S3Types type) -> std::shared_ptr; - - void removeContext(S3Types type); - private: // methods S3Session(); ~S3Session(); - [[nodiscard]] - auto findContext(S3Types type) -> std::shared_ptr; - [[nodiscard]] auto findClient(const net::Endpoint& endpoint) -> std::shared_ptr; private: // members std::list> client_; - std::list> contexts_; std::list> credentials_; }; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index d3fda63d4..6ed858bd8 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -64,7 +64,7 @@ namespace eckit { const auto ALLOC_TAG = "S3ClientAWS"; -S3ClientAWS::S3ClientAWS(const S3Config& config): S3Client(config, S3Session::instance().getContext(S3Types::AWS)) { +S3ClientAWS::S3ClientAWS(const S3Config& config): S3Client(config), ctx_(S3ContextAWS::instance()) { configure(config); } @@ -107,11 +107,6 @@ void S3ClientAWS::configure(const S3Config& config) { } } -void S3ClientAWS::print(std::ostream& out) const { - S3Client::print(out); - out << "S3ClientAWS[]"; -} - auto S3ClientAWS::getClient() const -> Aws::S3::S3Client& { if (client_) { return *client_; } throw S3SeriousBug("Invalid client!", Here()); diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 2d4530e9c..2f104aeb8 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -25,11 +25,13 @@ namespace eckit { +class S3ContextAWS; + //---------------------------------------------------------------------------------------------------------------------- class S3ClientAWS: public S3Client { public: // methods - S3ClientAWS(const S3Config& config); + explicit S3ClientAWS(const S3Config& config); ~S3ClientAWS(); @@ -62,11 +64,11 @@ class S3ClientAWS: public S3Client { auto objectSize(const std::string& bucket, const std::string& object) const -> long long override; private: // methods - void print(std::ostream& out) const override; - auto getClient() const -> Aws::S3::S3Client&; private: // members + const S3ContextAWS& ctx_; + std::unique_ptr client_; }; diff --git a/src/eckit/io/s3/aws/S3ContextAWS.cc b/src/eckit/io/s3/aws/S3ContextAWS.cc index 3805d4282..0b9eb2f58 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.cc +++ b/src/eckit/io/s3/aws/S3ContextAWS.cc @@ -15,28 +15,30 @@ #include "eckit/io/s3/aws/S3ContextAWS.h" +#include "eckit/config/LibEcKit.h" #include "eckit/io/s3/S3Exception.h" -#include "eckit/io/s3/aws/S3ClientAWS.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3ContextAWS::S3ContextAWS(const Aws::SDKOptions& options): S3Context(S3Types::AWS), options_(options) { - Aws::InitAPI(options_); -} - -S3ContextAWS::~S3ContextAWS() { - Aws::ShutdownAPI(options_); +S3ContextAWS& S3ContextAWS::instance() { + static S3ContextAWS ctx; + return ctx; } //---------------------------------------------------------------------------------------------------------------------- -auto S3ContextAWS::makeShared() -> std::shared_ptr { - Aws::SDKOptions options; +S3ContextAWS::S3ContextAWS() { + LOG_DEBUG_LIB(LibEcKit) << "init AWS API\n"; /// @todo remove debug stuff below - options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; - return std::make_shared(options); + options_.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + Aws::InitAPI(options_); +} + +S3ContextAWS::~S3ContextAWS() { + LOG_DEBUG_LIB(LibEcKit) << "shutdown AWS API\n"; + Aws::ShutdownAPI(options_); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ContextAWS.h b/src/eckit/io/s3/aws/S3ContextAWS.h index 5feb391a8..cce8d281d 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.h +++ b/src/eckit/io/s3/aws/S3ContextAWS.h @@ -19,32 +19,26 @@ #pragma once -#include "eckit/io/s3/S3Context.h" -#include "eckit/io/s3/S3Macros.h" - #include -#include - namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -class S3ContextAWS: public S3Context { +/// @brief Provides singleton for AWS API init/shutdown +class S3ContextAWS { public: // methods S3ContextAWS(const S3ContextAWS&) = delete; S3ContextAWS& operator=(const S3ContextAWS&) = delete; S3ContextAWS(S3ContextAWS&&) = delete; S3ContextAWS& operator=(S3ContextAWS&&) = delete; - explicit S3ContextAWS(const Aws::SDKOptions& options); - - ~S3ContextAWS(); + static S3ContextAWS& instance(); private: // methods - friend S3Context; + explicit S3ContextAWS(); - static auto makeShared() -> std::shared_ptr; + ~S3ContextAWS(); private: // members Aws::SDKOptions options_; diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 6498939da..483d59074 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -72,11 +72,9 @@ void ensureClean() { //---------------------------------------------------------------------------------------------------------------------- CASE("invalid bucket") { - { - EXPECT_THROWS(S3BucketName("http://127.0.0.1:9000/" + TEST_BUCKET)); - EXPECT_THROWS(S3BucketName("s3://127.0.0.1" + TEST_BUCKET)); - EXPECT_THROWS(S3BucketName("s3://127.0.0.1/" + TEST_BUCKET)); - } + EXPECT_THROWS(S3BucketName("http://127.0.0.1:9000/" + TEST_BUCKET)); + EXPECT_THROWS(S3BucketName("s3://127.0.0.1" + TEST_BUCKET)); + EXPECT_THROWS(S3BucketName("s3://127.0.0.1/" + TEST_BUCKET)); } //---------------------------------------------------------------------------------------------------------------------- From b48159710a3d248b42784778757dbcea43d82483 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 21 Feb 2024 15:38:44 +0100 Subject: [PATCH 72/90] feat(S3): move uri to base --- src/eckit/io/s3/S3Handle.cc | 2 +- src/eckit/io/s3/S3Name.cc | 6 +++++- src/eckit/io/s3/S3Name.h | 2 ++ src/eckit/io/s3/S3ObjectName.cc | 4 ---- src/eckit/io/s3/S3ObjectName.h | 2 -- src/eckit/io/s3/S3Session.cc | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 838f8d583..072cc8400 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -46,7 +46,7 @@ Length S3Handle::openForRead() { return estimate(); } -void S3Handle::openForWrite(const Length& length) { +void S3Handle::openForWrite(const Length& /*length*/) { open(Mode::WRITE); ASSERT(name_.bucketExists()); diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 05609f4ab..aa6b4811e 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -35,8 +35,12 @@ S3Name::~S3Name() = default; //---------------------------------------------------------------------------------------------------------------------- +auto S3Name::uri() const -> URI { + return "s3://" + asString(); +} + auto S3Name::asString() const -> std::string { - return "s3://" + std::string(endpoint_); + return std::string(endpoint_); } void S3Name::print(std::ostream& out) const { diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 2a1bda6d9..e43f6fcdd 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -38,6 +38,8 @@ class S3Name { virtual ~S3Name(); + auto uri() const -> URI; + auto endpoint() const -> const net::Endpoint& { return endpoint_; } virtual auto exists() const -> bool = 0; diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc index 36f5da7d1..a859f82a9 100644 --- a/src/eckit/io/s3/S3ObjectName.cc +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -53,10 +53,6 @@ auto S3ObjectName::asString() const -> std::string { return S3Name::asString() + "/" + bucket_ + "/" + object_; } -auto S3ObjectName::uri() const -> URI { - return asString(); -} - auto S3ObjectName::size() const -> long long { return client()->objectSize(bucket_, object_); } diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h index 7a8f653ef..919289162 100644 --- a/src/eckit/io/s3/S3ObjectName.h +++ b/src/eckit/io/s3/S3ObjectName.h @@ -34,8 +34,6 @@ class S3ObjectName: public S3Name { S3ObjectName(const net::Endpoint& endpoint, const std::string& bucket, const std::string& object); - auto uri() const -> URI; - auto size() const -> long long; auto exists() const -> bool override; diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index 73de457c5..0cead0027 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -48,7 +48,7 @@ struct IsCredentialEndpoint { //---------------------------------------------------------------------------------------------------------------------- S3Session& S3Session::instance() { - thread_local S3Session session; // TODO: thread local vs static + thread_local S3Session session; return session; } From f46d47c3b546fe8943ac99dcd16634edd5ee5e34 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 22 Feb 2024 16:43:00 +0100 Subject: [PATCH 73/90] fix(S3): remove debug log --- src/eckit/io/s3/aws/S3ContextAWS.cc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/eckit/io/s3/aws/S3ContextAWS.cc b/src/eckit/io/s3/aws/S3ContextAWS.cc index 0b9eb2f58..67619c571 100644 --- a/src/eckit/io/s3/aws/S3ContextAWS.cc +++ b/src/eckit/io/s3/aws/S3ContextAWS.cc @@ -15,9 +15,6 @@ #include "eckit/io/s3/aws/S3ContextAWS.h" -#include "eckit/config/LibEcKit.h" -#include "eckit/io/s3/S3Exception.h" - namespace eckit { //---------------------------------------------------------------------------------------------------------------------- @@ -30,14 +27,10 @@ S3ContextAWS& S3ContextAWS::instance() { //---------------------------------------------------------------------------------------------------------------------- S3ContextAWS::S3ContextAWS() { - LOG_DEBUG_LIB(LibEcKit) << "init AWS API\n"; - /// @todo remove debug stuff below - options_.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; Aws::InitAPI(options_); } S3ContextAWS::~S3ContextAWS() { - LOG_DEBUG_LIB(LibEcKit) << "shutdown AWS API\n"; Aws::ShutdownAPI(options_); } From 2e53988b434dbaaf167b591b96a27a05f8f14640 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 26 Feb 2024 10:56:28 +0100 Subject: [PATCH 74/90] chore(S3): removed unused macros file --- src/eckit/CMakeLists.txt | 1 - src/eckit/io/s3/S3Macros.h | 44 -------------------------------------- 2 files changed, 45 deletions(-) delete mode 100644 src/eckit/io/s3/S3Macros.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 68903fe9e..12fead5ba 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -309,7 +309,6 @@ io/s3/S3ObjectName.cc io/s3/S3ObjectName.h io/s3/S3Session.cc io/s3/S3Session.h -io/s3/S3Macros.h io/s3/S3URIManager.cc io/s3/S3URIManager.h ) diff --git a/src/eckit/io/s3/S3Macros.h b/src/eckit/io/s3/S3Macros.h deleted file mode 100644 index e6dedf33c..000000000 --- a/src/eckit/io/s3/S3Macros.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2024- European Centre for Medium-Range Weather Forecasts (ECMWF). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/// @file S3Macros.h -/// @author Metin Cakircali -/// @date Jan 2024 - -#pragma once - -// nodiscard is C++17 or higher -#if defined(__has_cpp_attribute) -#if __has_cpp_attribute(nodiscard) -#define NODISCARD [[nodiscard]] -#endif -#elif !defined(_MSC_VER) -#define NODISCARD __attribute__((warn_unused_result)) -#else -#define NODISCARD -#endif - -#define NO_COPY(TypeName) \ - TypeName(const TypeName&) = delete; \ - TypeName& operator=(const TypeName&) = delete; - -#define NO_MOVE(TypeName) \ - TypeName(TypeName&&) = delete; \ - TypeName& operator=(TypeName&&) = delete; - -#define NO_COPY_NO_MOVE(TypeName) \ - NO_COPY(TypeName) \ - NO_MOVE(TypeName) From 7d76c425d8e73538dfef2e16a48fefd23f6ebaa0 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 27 Feb 2024 17:26:36 +0100 Subject: [PATCH 75/90] feat(S3): read credentials from user/default file --- src/eckit/io/s3/S3Credential.h | 2 +- src/eckit/io/s3/S3Session.cc | 21 +++++++++++++++++++-- src/eckit/io/s3/S3Session.h | 6 ++++-- src/eckit/io/s3/aws/S3ClientAWS.cc | 2 +- tests/io/test_s3client.cc | 2 +- tests/io/test_s3handle.cc | 5 +++-- 6 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/eckit/io/s3/S3Credential.h b/src/eckit/io/s3/S3Credential.h index 06fea910a..79eb6f975 100644 --- a/src/eckit/io/s3/S3Credential.h +++ b/src/eckit/io/s3/S3Credential.h @@ -26,9 +26,9 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- struct S3Credential { + std::string endpoint {"127.0.0.1"}; std::string keyID; std::string secret; - std::string endpoint {"127.0.0.1"}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index 0cead0027..270b0411c 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -15,9 +15,13 @@ #include "eckit/io/s3/S3Session.h" +#include "eckit/filesystem/LocalPathName.h" #include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/aws/S3ClientAWS.h" #include "eckit/net/Endpoint.h" +#include "eckit/parser/YAMLParser.h" + +#include namespace eckit { @@ -95,8 +99,21 @@ void S3Session::removeClient(const net::Endpoint& endpoint) { //---------------------------------------------------------------------------------------------------------------------- // CREDENTIALS -auto S3Session::getCredentials(const std::string& endpoint) const -> std::shared_ptr { - /// @todo check if all keyid and secret different +void S3Session::readCredentials(std::string path) { + if (path.empty()) { + path = Resource("$ECKIT_S3_CREDENTIALS_FILE", "~/.config/eckit/s3credentials.yaml"); + } + + const PathName credFile(path, true); + if (credFile.exists()) { + const ValueList creds = YAMLParser::decodeFile(credFile); + for (auto&& cred : creds) { addCredentials({cred["endpoint"], cred["accessKeyID"], cred["secretKey"]}); } + } else { + Log::warning() << ReadError(credFile).what() << std::endl; + } +} + +auto S3Session::getCredentials(const net::Endpoint& endpoint) const -> std::shared_ptr { // search by endpoint const auto cred = std::find_if(credentials_.begin(), credentials_.end(), IsCredentialEndpoint(endpoint)); // found diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h index 6a8af08d2..6542cf1ef 100644 --- a/src/eckit/io/s3/S3Session.h +++ b/src/eckit/io/s3/S3Session.h @@ -31,7 +31,7 @@ struct S3Config; namespace net { class Endpoint; -} // namespace net +} //---------------------------------------------------------------------------------------------------------------------- @@ -47,7 +47,9 @@ class S3Session { void clear(); [[nodiscard]] - auto getCredentials(const std::string& endpoint) const -> std::shared_ptr; + auto getCredentials(const net::Endpoint& endpoint) const -> std::shared_ptr; + + void readCredentials(std::string path = ""); void addCredentials(const S3Credential& credential); diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index 6ed858bd8..d5ba85776 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -95,7 +95,7 @@ void S3ClientAWS::configure(const S3Config& config) { if (!config.endpoint.host().empty()) { configuration.endpointOverride = "http://" + config.endpoint.host(); } if (config.endpoint.port() > 0) { configuration.endpointOverride += ":" + std::to_string(config.endpoint.port()); } - if (auto cred = S3Session::instance().getCredentials(config.endpoint.host())) { + if (auto cred = S3Session::instance().getCredentials(config.endpoint)) { // credentials provider auto cProvider = Aws::MakeShared(ALLOC_TAG, cred->keyID, cred->secret); // endpoint provider diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc index 191ae4e26..aa94427ca 100644 --- a/tests/io/test_s3client.cc +++ b/tests/io/test_s3client.cc @@ -124,7 +124,7 @@ CASE("list buckets") { } // namespace eckit::test int main(int argc, char** argv) { - const S3Credential cred {"minio", "minio1234", "127.0.0.1"}; + const S3Credential cred {"127.0.0.1:9000", "minio", "minio1234"}; S3Session::instance().addCredentials(cred); auto ret = run_tests(argc, argv); diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index 483d59074..b8dc6c857 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -13,6 +13,7 @@ /// @author Simon Smart /// @date Jan 2024 +#include "eckit/filesystem/PathName.h" #include "eckit/filesystem/URI.h" #include "eckit/io/Buffer.h" #include "eckit/io/MemoryHandle.h" @@ -272,7 +273,7 @@ CASE("S3Handle::read") { EXPECT_NO_THROW(handle->close()); } -CASE("performance: write 1000 objects") { +CASE("performance: write 10 100 1000 objects") { ensureClean(); const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET); @@ -293,7 +294,7 @@ CASE("performance: write 1000 objects") { } // namespace eckit::test int main(int argc, char** argv) { - const S3Credential cred {"minio", "minio1234", "127.0.0.1"}; + const S3Credential cred {"127.0.0.1:9000", "minio", "minio1234"}; S3Session::instance().addCredentials(cred); int ret = -1; From 650594d8726643306b1962668e53f39e5c7bdb9d Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 28 Feb 2024 12:58:25 +0100 Subject: [PATCH 76/90] fix(S3): missing headers --- src/eckit/io/s3/S3Exception.cc | 1 + src/eckit/io/s3/S3Name.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc index 1a60b8a35..7ec075753 100644 --- a/src/eckit/io/s3/S3Exception.cc +++ b/src/eckit/io/s3/S3Exception.cc @@ -16,6 +16,7 @@ #include "eckit/io/s3/S3Exception.h" #include +#include //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index e43f6fcdd..c13a82d0a 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -22,7 +22,9 @@ #include "eckit/net/Endpoint.h" +#include #include +#include namespace eckit { From a682dfa4b8dbf7016dc89c36bce4354363ba4cad Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 28 Feb 2024 13:00:24 +0100 Subject: [PATCH 77/90] fix(S3): more missing headers --- tests/io/test_s3handle.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/io/test_s3handle.cc b/tests/io/test_s3handle.cc index b8dc6c857..db5c84530 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/test_s3handle.cc @@ -26,6 +26,7 @@ #include "eckit/log/Timer.h" #include "eckit/testing/Test.h" +#include #include using namespace std; From 27e8f8a677982f2ca67ed8847cff3e0e60c439a6 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 4 Mar 2024 18:38:17 +0100 Subject: [PATCH 78/90] fix(S3): turned off verifySSL --- src/eckit/io/s3/aws/S3ClientAWS.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index d5ba85776..ee0baf80b 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -84,15 +84,13 @@ void S3ClientAWS::configure(const S3Config& config) { configuration.disableImdsV1 = true; // setup region - // configuration.region = config.region; if (!config.region.empty()) { configuration.region = config.region; } - // configuration.scheme = Aws::Http::Scheme::HTTPS; - // configuration.verifySSL = false; + // configuration.scheme = Aws::Http::Scheme::HTTPS; + configuration.verifySSL = false; // setup endpoint - /// @todo handle http/https possibly via scheme flag in S3Config - if (!config.endpoint.host().empty()) { configuration.endpointOverride = "http://" + config.endpoint.host(); } + if (!config.endpoint.host().empty()) { configuration.endpointOverride = config.endpoint.host(); } if (config.endpoint.port() > 0) { configuration.endpointOverride += ":" + std::to_string(config.endpoint.port()); } if (auto cred = S3Session::instance().getCredentials(config.endpoint)) { From 3647ce8de3d9e96d1c6cd508de9478ac3b550201 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 17 Dec 2024 21:56:38 +0100 Subject: [PATCH 79/90] feat(S3): add config and credentials from file fix several minor issues --- src/eckit/CMakeLists.txt | 57 ++++---- src/eckit/io/s3/S3BucketName.cc | 37 ++--- src/eckit/io/s3/S3BucketName.h | 12 +- src/eckit/io/s3/S3Client.cc | 33 ++--- src/eckit/io/s3/S3Client.h | 42 ++++-- src/eckit/io/s3/S3Config.cc | 103 +++++++++++-- src/eckit/io/s3/S3Config.h | 64 ++++++-- src/eckit/io/s3/S3Credential.cc | 72 +++++++++ src/eckit/io/s3/S3Credential.h | 38 ++++- src/eckit/io/s3/S3Exception.cc | 24 ++- src/eckit/io/s3/S3Exception.h | 21 +-- src/eckit/io/s3/S3Name.cc | 30 +++- src/eckit/io/s3/S3Name.h | 31 ++-- src/eckit/io/s3/S3ObjectName.cc | 27 ++-- src/eckit/io/s3/S3ObjectName.h | 8 +- src/eckit/io/s3/S3Session.cc | 125 ++++++++-------- src/eckit/io/s3/S3Session.h | 44 ++++-- src/eckit/io/s3/S3URIManager.cc | 11 +- src/eckit/io/s3/S3URIManager.h | 16 +- src/eckit/io/s3/aws/S3ClientAWS.cc | 124 +++++++++------- src/eckit/io/s3/aws/S3ClientAWS.h | 31 ++-- src/eckit/io/s3/aws/S3ContextAWS.cc | 2 + tests/io/CMakeLists.txt | 14 +- tests/io/s3/CMakeLists.txt | 23 +++ tests/io/s3/S3Config.yaml | 15 ++ tests/io/s3/S3Credentials.yaml | 13 ++ tests/io/s3/test_s3client.cc | 218 ++++++++++++++++++++++++++++ tests/io/{ => s3}/test_s3handle.cc | 163 +++++++++++++-------- tests/io/test_s3client.cc | 135 ----------------- 29 files changed, 1023 insertions(+), 510 deletions(-) create mode 100644 src/eckit/io/s3/S3Credential.cc create mode 100644 tests/io/s3/CMakeLists.txt create mode 100644 tests/io/s3/S3Config.yaml create mode 100644 tests/io/s3/S3Credentials.yaml create mode 100644 tests/io/s3/test_s3client.cc rename tests/io/{ => s3}/test_s3handle.cc (58%) delete mode 100644 tests/io/test_s3client.cc diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index d2817cc0b..7242598db 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -283,33 +283,34 @@ if(HAVE_RADOS) ) endif() -if(HAVE_AWS_S3) -list( APPEND eckit_io_srcs -io/s3/aws/S3ClientAWS.cc -io/s3/aws/S3ClientAWS.h -io/s3/aws/S3ContextAWS.cc -io/s3/aws/S3ContextAWS.h -io/s3/S3BucketName.cc -io/s3/S3BucketName.h -io/s3/S3Client.cc -io/s3/S3Client.h -io/s3/S3Config.cc -io/s3/S3Config.h -io/s3/S3Credential.h -io/s3/S3Exception.cc -io/s3/S3Exception.h -io/s3/S3Handle.cc -io/s3/S3Handle.h -io/s3/S3Name.cc -io/s3/S3Name.h -io/s3/S3ObjectName.cc -io/s3/S3ObjectName.h -io/s3/S3Session.cc -io/s3/S3Session.h -io/s3/S3URIManager.cc -io/s3/S3URIManager.h -) -endif(HAVE_AWS_S3) +if( eckit_HAVE_AWS_S3 ) + list( APPEND eckit_io_srcs + io/s3/aws/S3ClientAWS.cc + io/s3/aws/S3ClientAWS.h + io/s3/aws/S3ContextAWS.cc + io/s3/aws/S3ContextAWS.h + io/s3/S3BucketName.cc + io/s3/S3BucketName.h + io/s3/S3Client.cc + io/s3/S3Client.h + io/s3/S3Config.cc + io/s3/S3Config.h + io/s3/S3Credential.cc + io/s3/S3Credential.h + io/s3/S3Exception.cc + io/s3/S3Exception.h + io/s3/S3Handle.cc + io/s3/S3Handle.h + io/s3/S3Name.cc + io/s3/S3Name.h + io/s3/S3ObjectName.cc + io/s3/S3ObjectName.h + io/s3/S3Session.cc + io/s3/S3Session.h + io/s3/S3URIManager.cc + io/s3/S3URIManager.h + ) +endif( eckit_HAVE_AWS_S3 ) list( APPEND eckit_filesystem_srcs filesystem/BasePathName.cc @@ -960,9 +961,9 @@ ecbuild_add_library( "${BZIP2_INCLUDE_DIRS}" "${AEC_INCLUDE_DIRS}" "${RADOS_INCLUDE_DIRS}" - "${AWSSDK_INCLUDE_DIRS}" "${OPENSSL_INCLUDE_DIR}" "${AIO_INCLUDE_DIRS}" + "${AWSSDK_INCLUDE_DIRS}" PRIVATE_LIBS "${SNAPPY_LIBRARIES}" diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc index b921b095b..f90adfabc 100644 --- a/src/eckit/io/s3/S3BucketName.cc +++ b/src/eckit/io/s3/S3BucketName.cc @@ -19,18 +19,27 @@ #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Name.h" #include "eckit/io/s3/S3ObjectName.h" +#include "eckit/log/Log.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3BucketName::S3BucketName(const URI& uri): S3Name(uri) { +S3BucketName::S3BucketName(const URI& uri) : S3Name(uri) { parse(); } -S3BucketName::S3BucketName(const net::Endpoint& endpoint, const std::string& bucket): - S3Name(endpoint, "/"), bucket_(bucket) { } +S3BucketName::S3BucketName(const net::Endpoint& endpoint, std::string bucket) + : S3Name(endpoint, "/"), bucket_ {std::move(bucket)} { } //---------------------------------------------------------------------------------------------------------------------- @@ -56,38 +65,32 @@ auto S3BucketName::makeObject(const std::string& object) const -> std::unique_pt } auto S3BucketName::exists() const -> bool { - return client()->bucketExists(bucket_); + return client().bucketExists(bucket_); } void S3BucketName::create() { - client()->createBucket(bucket_); + client().createBucket(bucket_); } void S3BucketName::destroy() { - client()->deleteBucket(bucket_); + client().deleteBucket(bucket_); } void S3BucketName::ensureCreated() { try { create(); - } - catch (S3EntityAlreadyExists& e) { - LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; - } + } catch (S3EntityAlreadyExists& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } } void S3BucketName::ensureDestroyed() { try { - client()->emptyBucket(bucket_); - client()->deleteBucket(bucket_); - } - catch (S3EntityNotFound& e) { - LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; - } + client().emptyBucket(bucket_); + client().deleteBucket(bucket_); + } catch (S3EntityNotFound& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } } auto S3BucketName::listObjects() const -> std::vector { - return client()->listObjects(bucket_); + return client().listObjects(bucket_); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h index 57fbb50cf..fafbab992 100644 --- a/src/eckit/io/s3/S3BucketName.h +++ b/src/eckit/io/s3/S3BucketName.h @@ -21,6 +21,12 @@ #pragma once #include "eckit/io/s3/S3Name.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include namespace eckit { @@ -28,11 +34,11 @@ class S3ObjectName; //---------------------------------------------------------------------------------------------------------------------- -class S3BucketName: public S3Name { +class S3BucketName : public S3Name { public: // methods explicit S3BucketName(const URI& uri); - S3BucketName(const net::Endpoint& endpoint, const std::string& bucket); + S3BucketName(const net::Endpoint& endpoint, std::string bucket); auto makeObject(const std::string& object) const -> std::unique_ptr; @@ -46,7 +52,7 @@ class S3BucketName: public S3Name { void ensureDestroyed(); - /// @todo: return S3 object iterator but first add prefix + /// @todo return S3 object iterator but first add prefix auto listObjects() const -> std::vector; auto asString() const -> std::string override; diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc index 2a269ef85..894dc2a96 100644 --- a/src/eckit/io/s3/S3Client.cc +++ b/src/eckit/io/s3/S3Client.cc @@ -15,41 +15,40 @@ #include "eckit/io/s3/S3Client.h" -#include "S3Client.h" -#include "eckit/io/s3/S3Exception.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/io/s3/S3Config.h" #include "eckit/io/s3/S3Session.h" #include "eckit/io/s3/aws/S3ClientAWS.h" +#include "eckit/log/CodeLocation.h" + +#include +#include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Client::S3Client(const S3Config& config): config_(config) { } - -S3Client::~S3Client() = default; +S3Client::S3Client(S3Config config) : config_ {std::move(config)} { } //---------------------------------------------------------------------------------------------------------------------- -void S3Client::print(std::ostream& out) const { - out << "S3Client[config=" << config_ << "]"; -} +auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { -//---------------------------------------------------------------------------------------------------------------------- + if (config.backend == S3Backend::AWS) { return std::make_unique(config); } -auto S3Client::makeShared(const S3Config& config) -> std::shared_ptr { - if (config.type == S3Types::AWS) { return std::make_shared(config); } - throw S3SeriousBug("Unkown S3 client type!", Here()); + throw UserError("Unsupported S3 backend! Supported backend = AWS ", Here()); } //---------------------------------------------------------------------------------------------------------------------- -auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { - if (config.type == S3Types::AWS) { return std::make_unique(config); } - throw S3SeriousBug("Unkown S3 client type!", Here()); +void S3Client::print(std::ostream& out) const { + out << "S3Client[config=" << config_ << "]"; } -auto S3Client::endpoint() const -> const net::Endpoint& { - return config_.endpoint; +std::ostream& operator<<(std::ostream& out, const S3Client& client) { + client.print(out); + return out; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index d42e3792a..1d00c45de 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -20,9 +20,11 @@ #pragma once #include "eckit/io/s3/S3Config.h" -#include "eckit/memory/NonCopyable.h" +#include +#include #include +#include #include namespace eckit { @@ -31,15 +33,19 @@ class S3Context; //---------------------------------------------------------------------------------------------------------------------- -class S3Client: private NonCopyable { +class S3Client { public: // methods - virtual ~S3Client(); - - static auto makeShared(const S3Config& config) -> std::shared_ptr; + S3Client(const S3Client&) = delete; + S3Client& operator=(const S3Client&) = delete; + S3Client(S3Client&&) = default; + S3Client& operator=(S3Client&&) = default; + virtual ~S3Client() = default; static auto makeUnique(const S3Config& config) -> std::unique_ptr; - virtual auto endpoint() const -> const net::Endpoint&; + static auto makeShared(const S3Config& config) -> std::shared_ptr { return makeUnique(config); } + + auto config() const -> const S3Config& { return config_; } virtual void createBucket(const std::string& bucket) const = 0; @@ -51,11 +57,16 @@ class S3Client: private NonCopyable { virtual auto listBuckets() const -> std::vector = 0; - virtual auto putObject(const std::string& bucket, const std::string& object, const void* buffer, - uint64_t length) const -> long long = 0; + virtual auto putObject(const std::string& bucket, + const std::string& object, + const void* buffer, + uint64_t length) const -> long long = 0; - virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t offset, - uint64_t length) const -> long long = 0; + virtual auto getObject(const std::string& bucket, + const std::string& object, + void* buffer, + uint64_t offset, + uint64_t length) const -> long long = 0; virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0; @@ -67,16 +78,15 @@ class S3Client: private NonCopyable { virtual auto objectSize(const std::string& bucket, const std::string& object) const -> long long = 0; - friend std::ostream& operator<<(std::ostream& out, const S3Client& client) { - client.print(out); - return out; - } - protected: // methods - explicit S3Client(const S3Config& config); + S3Client(); + + explicit S3Client(S3Config config); virtual void print(std::ostream& out) const; + friend std::ostream& operator<<(std::ostream& out, const S3Client& client); + private: // members S3Config config_; }; diff --git a/src/eckit/io/s3/S3Config.cc b/src/eckit/io/s3/S3Config.cc index 8eedb780d..d857684f6 100644 --- a/src/eckit/io/s3/S3Config.cc +++ b/src/eckit/io/s3/S3Config.cc @@ -15,35 +15,112 @@ #include "eckit/io/s3/S3Config.h" +#include "eckit/config/LibEcKit.h" +#include "eckit/config/LocalConfiguration.h" +#include "eckit/config/Resource.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/container/DenseSet.h" +#include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include +#include +#include +#include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Config::S3Config(std::string region, const URI& uri): region(std::move(region)), endpoint(uri) { } +namespace { + +const std::string defaultConfigFile = "~/.config/eckit/S3Config.yaml"; + +S3Config fromYAML(const LocalConfiguration& config) { + const net::Endpoint endpoint {config.getString("endpoint")}; + + const auto region = config.getString("region", s3DefaultRegion); + + S3Config s3config(endpoint, region); + + if (config.has("backend")) { + S3Backend backend {S3Backend::AWS}; // Default to AWS + std::string backendStr = config.getString("backend"); + if (backendStr == "AWS") { + backend = S3Backend::AWS; + } else if (backendStr == "REST") { + backend = S3Backend::REST; + } else if (backendStr == "MinIO") { + backend = S3Backend::MINIO; + } else { + throw UserError("Invalid backend: " + backendStr, Here()); + } + s3config.backend = backend; + } + + if (config.has("secure")) { s3config.secure = config.getBool("secure"); } + + return s3config; +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Config::fromFile(std::string path) -> std::vector { -S3Config::S3Config(std::string region, const std::string& hostname, const int port): - region(std::move(region)), endpoint(hostname, port) { } + if (path.empty()) { path = Resource("s3ConfigFile;$ECKIT_S3_CONFIG_FILE", defaultConfigFile); } -S3Config::S3Config(const URI& uri): endpoint(uri) { } + PathName configFile(path); -S3Config::S3Config(const net::Endpoint& e): endpoint(e) { } + if (!configFile.exists()) { + Log::debug() << "S3 configuration file does not exist: " << configFile << std::endl; + return {}; + } + + const auto servers = YAMLConfiguration(configFile).getSubConfigurations("servers"); + + std::vector result; + result.reserve(servers.size()); + for (const auto& server : servers) { result.emplace_back(fromYAML(server)); } + + return result; +} + +S3Config::S3Config(const net::Endpoint& endpoint, std::string region) + : endpoint {endpoint}, region {std::move(region)} { } + +S3Config::S3Config(const std::string& host, const uint16_t port, std::string region) + : endpoint {host, port}, region {std::move(region)} { } + +S3Config::S3Config(const URI& uri) : S3Config(uri.host(), uri.port()) { } //---------------------------------------------------------------------------------------------------------------------- void S3Config::print(std::ostream& out) const { - out << "S3Config[type="; - if (type == S3Types::AWS) { + out << "S3Config[endpoint=" << endpoint << ",region=" << region << ",backend="; + if (backend == S3Backend::AWS) { out << "AWS"; - } else if (type == S3Types::MINIO) { - out << "MinIO"; - } else if (type == S3Types::REST) { + } else if (backend == S3Backend::REST) { out << "REST"; - } else { - out << "NONE"; + } else if (backend == S3Backend::MINIO) { + out << "MinIO"; } - out << ",region=" << region << ",endpoint=" << endpoint << "]"; + out << ",secure=" << secure << "]"; +} + +//---------------------------------------------------------------------------------------------------------------------- + +std::ostream& operator<<(std::ostream& out, const S3Config& config) { + config.print(out); + return out; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 5b5b981b9..dce4f0fc0 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -21,33 +21,69 @@ #include "eckit/net/Endpoint.h" +#include +#include #include +#include namespace eckit { -class URI; +enum class S3Backend : std::uint8_t { AWS, REST, MINIO }; -enum class S3Types { NONE, AWS, MINIO, REST }; +constexpr auto s3DefaultHost = "127.0.0.1"; +constexpr uint16_t s3DefaultPort = 443; +constexpr auto s3DefaultRegion = "default"; //---------------------------------------------------------------------------------------------------------------------- +/// @brief S3 configurations for a given endpoint +/// +/// @example Example YAML S3 configuration file: +/// +/// ECKIT_S3_CONFIG_FILE = ~/.config/eckit/S3Config.yaml +/// +/// --- +/// servers: +/// - endpoint: "127.0.0.1:9000" +/// region: "default" # (default) +/// secure: true # (default) +/// backend: "AWS" # (default) +/// +/// - endpoint: "minio:9000" +/// region: "eu-central-1" +/// +/// - endpoint: "https://localhost:9000" +/// region: "eu-central-1" +/// +/// # region is inferred from the endpoint +/// - endpoint: "https://eu-central-1.ecmwf.int:9000" +/// struct S3Config { - /// @todo region is part of hostname (s3express-control.region_code.amazonaws.com/bucket-name) - S3Config(std::string region, const URI& uri); - S3Config(std::string region, const std::string& hostname, int port); - S3Config(const URI& uri); - S3Config(const net::Endpoint&); - - friend std::ostream& operator<<(std::ostream& out, const S3Config& config) { - config.print(out); - return out; + + static auto fromFile(std::string path) -> std::vector; + + S3Config() = default; + + explicit S3Config(const net::Endpoint& endpoint, std::string region = s3DefaultRegion); + + explicit S3Config(const std::string& host, uint16_t port, std::string region = s3DefaultRegion); + + explicit S3Config(const URI& uri); + + bool operator==(const S3Config& other) const { + return backend == other.backend && endpoint == other.endpoint && region == other.region; } + bool operator!=(const S3Config& other) const { return !(*this == other); } + void print(std::ostream& out) const; - S3Types type {S3Types::AWS}; - std::string region {"eu-central-1"}; - net::Endpoint endpoint {"127.0.0.1", -1}; + friend std::ostream& operator<<(std::ostream& out, const S3Config& config); + + net::Endpoint endpoint {s3DefaultHost, s3DefaultPort}; + std::string region {s3DefaultRegion}; + S3Backend backend {S3Backend::AWS}; + bool secure {true}; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Credential.cc b/src/eckit/io/s3/S3Credential.cc new file mode 100644 index 000000000..cf33ac377 --- /dev/null +++ b/src/eckit/io/s3/S3Credential.cc @@ -0,0 +1,72 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Credential.h" + +#include "eckit/config/Configuration.h" +#include "eckit/config/LibEcKit.h" +#include "eckit/config/LocalConfiguration.h" +#include "eckit/config/Resource.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/log/Log.h" + +#include +#include +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +const std::string defaultCredFile = "~/.config/eckit/S3Credentials.yaml"; + +S3Credential fromYAML(const LocalConfiguration& config) { + const auto endpoint = config.getString("endpoint"); + const auto keyID = config.getString("accessKeyID"); + const auto secret = config.getString("secretKey"); + return {endpoint, keyID, secret}; +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Credential::fromFile(std::string path) -> std::vector { + + if (path.empty()) { path = Resource("s3CredentialsFile;$ECKIT_S3_CREDENTIALS_FILE", defaultCredFile); } + + PathName credFile(path); + + if (!credFile.exists()) { + Log::debug() << "S3 credentials file does not exist: " << credFile << std::endl; + return {}; + } + + std::vector result; + + const auto creds = YAMLConfiguration(credFile).getSubConfigurations("credentials"); + result.reserve(creds.size()); + for (const auto& cred : creds) { result.emplace_back(fromYAML(cred)); } + + return result; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Credential.h b/src/eckit/io/s3/S3Credential.h index 79eb6f975..4cb9c594b 100644 --- a/src/eckit/io/s3/S3Credential.h +++ b/src/eckit/io/s3/S3Credential.h @@ -19,16 +19,48 @@ #pragma once +#include "eckit/net/Endpoint.h" + #include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- +/// @brief S3 credential information for a given endpoint +/// +/// @example Example YAML S3 credential file: +/// +/// ECKIT_S3_CREDENTIALS_FILE = ~/.config/eckit/S3Credentials.yaml +/// +/// --- +/// credentials: +/// - endpoint: '127.0.0.1:9000' +/// accessKeyID: 'minio' +/// secretKey: 'minio1234' +/// +/// - endpoint: 'minio:9000' +/// accessKeyID: 'minio' +/// secretKey: 'minio1234' +/// +/// - endpoint: 'localhost:9000' +/// accessKeyID: 'asd2' +/// secretKey: 'asd2' +/// struct S3Credential { - std::string endpoint {"127.0.0.1"}; - std::string keyID; - std::string secret; + + static auto fromFile(std::string path) -> std::vector; + + auto operator==(const S3Credential& other) const -> bool { + return endpoint == other.endpoint && keyID == other.keyID && secret == other.secret; + } + + auto operator!=(const S3Credential& other) const -> bool { return !(*this == other); } + + net::Endpoint endpoint; + std::string keyID; + std::string secret; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc index 7ec075753..90d04798e 100644 --- a/src/eckit/io/s3/S3Exception.cc +++ b/src/eckit/io/s3/S3Exception.cc @@ -15,8 +15,12 @@ #include "eckit/io/s3/S3Exception.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/log/CodeLocation.h" + +#include #include -#include +#include //---------------------------------------------------------------------------------------------------------------------- @@ -34,20 +38,24 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3SeriousBug::S3SeriousBug(const std::string& msg, const CodeLocation& loc): SeriousBug("[S3 Error] " + msg, loc) { } +S3SeriousBug::S3SeriousBug(const std::string& msg, const CodeLocation& loc) + : SeriousBug("[S3 Serious Bug] " + msg, loc) { } -S3SeriousBug::S3SeriousBug(const std::string& msg, const int code, const CodeLocation& loc): - S3SeriousBug(addCode(msg, code), loc) { } +S3SeriousBug::S3SeriousBug(const std::string& msg, const int code, const CodeLocation& loc) + : S3SeriousBug(addCode(msg, code), loc) { } //---------------------------------------------------------------------------------------------------------------------- -S3Exception::S3Exception(const std::string& msg, const CodeLocation& loc): Exception("[S3 Exception] " + msg, loc) { } +S3Exception::S3Exception(const std::string& msg, const CodeLocation& loc) : Exception("[S3 Exception] " + msg, loc) { } -S3BucketNotEmpty::S3BucketNotEmpty(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } +S3BucketNotEmpty::S3BucketNotEmpty(const std::string& msg, const CodeLocation& loc) + : S3Exception("[Not Empty] " + msg, loc) { } -S3EntityNotFound::S3EntityNotFound(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } +S3EntityNotFound::S3EntityNotFound(const std::string& msg, const CodeLocation& loc) + : S3Exception("[Not Found] " + msg, loc) { } -S3EntityAlreadyExists::S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc): S3Exception(msg, loc) { } +S3EntityAlreadyExists::S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc) + : S3Exception("[Already Exists] " + msg, loc) { } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Exception.h b/src/eckit/io/s3/S3Exception.h index e49866710..173409ab7 100644 --- a/src/eckit/io/s3/S3Exception.h +++ b/src/eckit/io/s3/S3Exception.h @@ -15,46 +15,49 @@ /// @file S3Exception.h /// @author Metin Cakircali -/// @author Nicolau Manubens /// @date Jan 2024 #pragma once #include "eckit/exception/Exceptions.h" +#include "eckit/log/CodeLocation.h" +#include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -class S3SeriousBug: public SeriousBug { +class S3SeriousBug : public SeriousBug { public: S3SeriousBug(const std::string& msg, const CodeLocation& loc); S3SeriousBug(const std::string& msg, int code, const CodeLocation& loc); - S3SeriousBug(const std::ostringstream& msg, const CodeLocation& loc): S3SeriousBug(msg.str(), loc) { } - S3SeriousBug(const std::ostringstream& msg, int code, const CodeLocation& loc): - S3SeriousBug(msg.str(), code, loc) { } + + S3SeriousBug(const std::ostringstream& msg, const CodeLocation& loc) : S3SeriousBug(msg.str(), loc) { } + + S3SeriousBug(const std::ostringstream& msg, int code, const CodeLocation& loc) + : S3SeriousBug(msg.str(), code, loc) { } }; //---------------------------------------------------------------------------------------------------------------------- -class S3Exception: public Exception { +class S3Exception : public Exception { public: S3Exception(const std::string& msg, const CodeLocation& loc); }; -class S3BucketNotEmpty: public S3Exception { +class S3BucketNotEmpty : public S3Exception { public: S3BucketNotEmpty(const std::string& msg, const CodeLocation& loc); }; -class S3EntityNotFound: public S3Exception { +class S3EntityNotFound : public S3Exception { public: S3EntityNotFound(const std::string& msg, const CodeLocation& loc); }; -class S3EntityAlreadyExists: public S3Exception { +class S3EntityAlreadyExists : public S3Exception { public: S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc); }; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index aa6b4811e..6ffeafaa2 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -15,28 +15,35 @@ #include "eckit/io/s3/S3Name.h" +#include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Session.h" #include "eckit/utils/Tokenizer.h" +#include +#include +#include +#include +#include + namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Name::S3Name(const URI& uri): endpoint_(uri), name_(uri.name()) { - ASSERT(uri.scheme() == "s3"); +S3Name::S3Name(const URI& uri) : endpoint_ {uri}, name_ {uri.name()} { + /// @todo is "s3://endpoint/bucket/object" a valid URI ? + ASSERT(uri.scheme() == type); } -S3Name::S3Name(const net::Endpoint& endpoint, const std::string& name): endpoint_(endpoint), name_(name) { } +S3Name::S3Name(const net::Endpoint& endpoint, std::string name) : endpoint_ {endpoint}, name_ {std::move(name)} { } S3Name::~S3Name() = default; //---------------------------------------------------------------------------------------------------------------------- auto S3Name::uri() const -> URI { - return "s3://" + asString(); + return {type, asString()}; } auto S3Name::asString() const -> std::string { @@ -53,9 +60,16 @@ auto S3Name::parseName() const -> std::vector { return Tokenizer("/").tokenize(name_); } -auto S3Name::client() const -> std::shared_ptr { - /// @todo - return S3Session::instance().getClient({endpoint_}); +auto S3Name::client() const -> S3Client& { + if (!client_) { client_ = S3Session::instance().getClient(endpoint_); } + return *client_; +} + +//---------------------------------------------------------------------------------------------------------------------- + +std::ostream& operator<<(std::ostream& out, const S3Name& name) { + name.print(out); + return out; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index c13a82d0a..7dc3ccff2 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -22,6 +22,7 @@ #include "eckit/net/Endpoint.h" +#include #include #include #include @@ -33,13 +34,26 @@ class S3Client; //---------------------------------------------------------------------------------------------------------------------- class S3Name { +public: // types + static constexpr auto type = "s3"; + public: // methods explicit S3Name(const URI& uri); - S3Name(const net::Endpoint& endpoint, const std::string& name); + S3Name(const net::Endpoint& endpoint, std::string name); + + // rules + + S3Name(const S3Name&) = default; + S3Name& operator=(const S3Name&) = default; + + S3Name(S3Name&&) = delete; + S3Name& operator=(S3Name&&) = delete; virtual ~S3Name(); + // accessors + auto uri() const -> URI; auto endpoint() const -> const net::Endpoint& { return endpoint_; } @@ -48,22 +62,21 @@ class S3Name { virtual auto asString() const -> std::string; - friend std::ostream& operator<<(std::ostream& out, const S3Name& name) { - name.print(out); - return out; - } - protected: // methods virtual void print(std::ostream& out) const; + friend std::ostream& operator<<(std::ostream& out, const S3Name& name); + [[nodiscard]] auto parseName() const -> std::vector; - auto client() const -> std::shared_ptr; + auto client() const -> S3Client&; private: // members - const net::Endpoint endpoint_; - const std::string name_; + net::Endpoint endpoint_; + std::string name_; + + mutable std::shared_ptr client_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc index a859f82a9..27b89a6e2 100644 --- a/src/eckit/io/s3/S3ObjectName.cc +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -15,23 +15,28 @@ #include "eckit/io/s3/S3ObjectName.h" -#include "eckit/config/LibEcKit.h" #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Handle.h" -#include "eckit/utils/Tokenizer.h" +#include "eckit/io/s3/S3Name.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3ObjectName::S3ObjectName(const URI& uri): S3Name(uri) { +S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri) { parse(); } -S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, const std::string& bucket, const std::string& object): - S3Name(endpoint, "/"), bucket_(bucket), object_(object) { } +S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object) + : S3Name(endpoint, "/"), bucket_ {std::move(bucket)}, object_ {std::move(object)} { } //---------------------------------------------------------------------------------------------------------------------- @@ -54,27 +59,27 @@ auto S3ObjectName::asString() const -> std::string { } auto S3ObjectName::size() const -> long long { - return client()->objectSize(bucket_, object_); + return client().objectSize(bucket_, object_); } auto S3ObjectName::exists() const -> bool { - return client()->objectExists(bucket_, object_); + return client().objectExists(bucket_, object_); } auto S3ObjectName::bucketExists() const -> bool { - return client()->bucketExists(bucket_); + return client().bucketExists(bucket_); } void S3ObjectName::remove() { - client()->deleteObject(bucket_, object_); + client().deleteObject(bucket_, object_); } auto S3ObjectName::put(const void* buffer, const long length) const -> long long { - return client()->putObject(bucket_, object_, buffer, length); + return client().putObject(bucket_, object_, buffer, length); } auto S3ObjectName::get(void* buffer, const long offset, const long length) const -> long long { - return client()->getObject(bucket_, object_, buffer, offset, length); + return client().getObject(bucket_, object_, buffer, offset, length); } auto S3ObjectName::dataHandle() -> DataHandle* { diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h index 919289162..447fab093 100644 --- a/src/eckit/io/s3/S3ObjectName.h +++ b/src/eckit/io/s3/S3ObjectName.h @@ -20,6 +20,10 @@ #pragma once #include "eckit/io/s3/S3Name.h" +#include "eckit/net/Endpoint.h" + +#include +#include namespace eckit { @@ -28,11 +32,11 @@ class DataHandle; //---------------------------------------------------------------------------------------------------------------------- -class S3ObjectName: public S3Name { +class S3ObjectName : public S3Name { public: // methods explicit S3ObjectName(const URI& uri); - S3ObjectName(const net::Endpoint& endpoint, const std::string& bucket, const std::string& object); + S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object); auto size() const -> long long; diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index 270b0411c..9a899583f 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -15,36 +15,30 @@ #include "eckit/io/s3/S3Session.h" -#include "eckit/filesystem/LocalPathName.h" +#include "eckit/config/LibEcKit.h" #include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/aws/S3ClientAWS.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" #include "eckit/net/Endpoint.h" -#include "eckit/parser/YAMLParser.h" +#include "eckit/runtime/Main.h" -#include +#include +#include +#include namespace eckit { namespace { -/// @brief Functor for S3Context type -struct IsClientEndpoint { - IsClientEndpoint(const net::Endpoint& endpoint): endpoint_(endpoint) { } - - bool operator()(const std::shared_ptr& client) const { return client->endpoint() == endpoint_; } - -private: +/// @brief Predicate to find a client or credential by its endpoint +struct EndpointMatch { const net::Endpoint& endpoint_; -}; - -/// @brief Functor for S3Credential endpoint -struct IsCredentialEndpoint { - IsCredentialEndpoint(const std::string& endpoint): endpoint_(endpoint) { } bool operator()(const std::shared_ptr& cred) const { return cred->endpoint == endpoint_; } -private: - const std::string& endpoint_; + bool operator()(const std::shared_ptr& client) const { return client->config().endpoint == endpoint_; } }; } // namespace @@ -52,86 +46,101 @@ struct IsCredentialEndpoint { //---------------------------------------------------------------------------------------------------------------------- S3Session& S3Session::instance() { - thread_local S3Session session; + static S3Session session; return session; } //---------------------------------------------------------------------------------------------------------------------- -S3Session::S3Session() = default; +S3Session::S3Session() { + if (!Main::ready()) { + // inform when called before Main::initialise + Log::debug() << "Skipping initialization of S3Session instance.\n"; + return; + } + loadClients(); + loadCredentials(); +} S3Session::~S3Session() = default; //---------------------------------------------------------------------------------------------------------------------- void S3Session::clear() { - client_.clear(); + Log::debug() << "Clearing S3 clients and credentials.\n"; + clients_.clear(); credentials_.clear(); } //---------------------------------------------------------------------------------------------------------------------- // CLIENT -auto S3Session::getClient(const S3Config& config) -> std::shared_ptr { - // return if found - if (auto client = findClient(config.endpoint)) { return client; } +void S3Session::loadClients(const std::string& path) { + const auto configs = S3Config::fromFile(path); + for (const auto& config : configs) { addClient(config); } +} - // not found - auto client = S3Client::makeShared(config); - client_.push_back(client); +auto S3Session::findClient(const net::Endpoint& endpoint) const -> std::shared_ptr { - return client; -} + auto client = std::find_if(clients_.begin(), clients_.end(), EndpointMatch {endpoint}); + + if (client != clients_.end()) { return *client; } -auto S3Session::findClient(const net::Endpoint& endpoint) -> std::shared_ptr { - // search by type - const auto client = std::find_if(client_.begin(), client_.end(), IsClientEndpoint(endpoint)); - // found - if (client != client_.end()) { return *client; } // not found return {}; } +auto S3Session::getClient(const net::Endpoint& endpoint) const -> std::shared_ptr { + + if (auto client = findClient(endpoint)) { return client; } + + throw S3EntityNotFound("Could not find the client for " + std::string(endpoint), Here()); +} + +auto S3Session::addClient(const S3Config& config) -> std::shared_ptr { + if (auto client = findClient(config.endpoint)) { return client; } + // not found, add new item + return clients_.emplace_back(S3Client::makeShared(config)); +} + void S3Session::removeClient(const net::Endpoint& endpoint) { - client_.remove_if(IsClientEndpoint(endpoint)); + clients_.remove_if(EndpointMatch {endpoint}); } //---------------------------------------------------------------------------------------------------------------------- // CREDENTIALS -void S3Session::readCredentials(std::string path) { - if (path.empty()) { - path = Resource("$ECKIT_S3_CREDENTIALS_FILE", "~/.config/eckit/s3credentials.yaml"); - } - - const PathName credFile(path, true); - if (credFile.exists()) { - const ValueList creds = YAMLParser::decodeFile(credFile); - for (auto&& cred : creds) { addCredentials({cred["endpoint"], cred["accessKeyID"], cred["secretKey"]}); } - } else { - Log::warning() << ReadError(credFile).what() << std::endl; - } +void S3Session::loadCredentials(const std::string& path) { + const auto creds = S3Credential::fromFile(path); + for (const auto& cred : creds) { addCredential(cred); } } -auto S3Session::getCredentials(const net::Endpoint& endpoint) const -> std::shared_ptr { - // search by endpoint - const auto cred = std::find_if(credentials_.begin(), credentials_.end(), IsCredentialEndpoint(endpoint)); - // found +auto S3Session::findCredential(const net::Endpoint& endpoint) const -> std::shared_ptr { + + auto cred = std::find_if(credentials_.begin(), credentials_.end(), EndpointMatch {endpoint}); + if (cred != credentials_.end()) { return *cred; } + // not found return {}; } -void S3Session::addCredentials(const S3Credential& credential) { - // check if already exists - if (getCredentials(credential.endpoint)) { return; } - // add new item - auto cred = std::make_shared(credential); - credentials_.emplace_back(cred); +auto S3Session::getCredential(const net::Endpoint& endpoint) const -> std::shared_ptr { + + if (auto cred = findCredential(endpoint)) { return cred; } + + throw S3EntityNotFound("Could not find the credential for " + std::string(endpoint), Here()); +} + +auto S3Session::addCredential(const S3Credential& credential) -> std::shared_ptr { + // don't add duplicate credentials + if (auto cred = findCredential(credential.endpoint)) { return cred; } + // not found: add new item + return credentials_.emplace_back(std::make_shared(credential)); } -void S3Session::removeCredentials(const std::string& endpoint) { - credentials_.remove_if(IsCredentialEndpoint(endpoint)); +void S3Session::removeCredential(const net::Endpoint& endpoint) { + credentials_.remove_if(EndpointMatch {endpoint}); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h index 6542cf1ef..50a042b3a 100644 --- a/src/eckit/io/s3/S3Session.h +++ b/src/eckit/io/s3/S3Session.h @@ -19,14 +19,14 @@ #pragma once -#include "eckit/io/s3/S3Credential.h" - #include #include +#include namespace eckit { class S3Client; +struct S3Credential; struct S3Config; namespace net { @@ -35,7 +35,7 @@ class Endpoint; //---------------------------------------------------------------------------------------------------------------------- -class S3Session { +class S3Session final { public: // methods S3Session(const S3Session&) = delete; S3Session& operator=(const S3Session&) = delete; @@ -46,30 +46,50 @@ class S3Session { void clear(); + // clients + + void loadClients(const std::string& path = ""); + + /// @brief Get an S3 client for the given configuration + /// @param config S3 configuration + /// @return S3 client + /// @throws S3EntityNotFound if the client does not exist [[nodiscard]] - auto getCredentials(const net::Endpoint& endpoint) const -> std::shared_ptr; + auto getClient(const net::Endpoint& endpoint) const -> std::shared_ptr; - void readCredentials(std::string path = ""); + /// @brief Add an S3 client for the given configuration + /// @param config S3 configuration + /// @return S3 client + auto addClient(const S3Config& config) -> std::shared_ptr; + + void removeClient(const net::Endpoint& endpoint); - void addCredentials(const S3Credential& credential); + // credentials - void removeCredentials(const std::string& endpoint); + void loadCredentials(const std::string& path = ""); + /// @brief Get an S3 credential for the given configuration + /// @param endpoint S3 endpoint + /// @return S3 credential + /// @throws S3EntityNotFound if the credential does not exist [[nodiscard]] - auto getClient(const S3Config& config) -> std::shared_ptr; + auto getCredential(const net::Endpoint& endpoint) const -> std::shared_ptr; - void removeClient(const net::Endpoint& endpoint); + auto addCredential(const S3Credential& credential) -> std::shared_ptr; + + void removeCredential(const net::Endpoint& endpoint); private: // methods S3Session(); ~S3Session(); - [[nodiscard]] - auto findClient(const net::Endpoint& endpoint) -> std::shared_ptr; + auto findClient(const net::Endpoint& endpoint) const -> std::shared_ptr; + + auto findCredential(const net::Endpoint& endpoint) const -> std::shared_ptr; private: // members - std::list> client_; + std::list> clients_; std::list> credentials_; }; diff --git a/src/eckit/io/s3/S3URIManager.cc b/src/eckit/io/s3/S3URIManager.cc index ef96dfdc4..c8749f56a 100644 --- a/src/eckit/io/s3/S3URIManager.cc +++ b/src/eckit/io/s3/S3URIManager.cc @@ -15,17 +15,20 @@ #include "eckit/io/s3/S3URIManager.h" +#include "eckit/filesystem/URIManager.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" #include "eckit/io/s3/S3ObjectName.h" +#include + namespace eckit { static S3URIManager manager_s3("s3"); //---------------------------------------------------------------------------------------------------------------------- -S3URIManager::S3URIManager(const std::string& name): URIManager(name) { } - -S3URIManager::~S3URIManager() = default; +S3URIManager::S3URIManager(const std::string& name) : URIManager(name) { } bool S3URIManager::exists(const URI& uri) { return S3ObjectName(uri).exists(); @@ -39,7 +42,7 @@ DataHandle* S3URIManager::newReadHandle(const URI& uri) { return S3ObjectName(uri).dataHandle(); } -DataHandle* S3URIManager::newReadHandle(const URI& uri, const OffsetList&, const LengthList&) { +DataHandle* S3URIManager::newReadHandle(const URI& uri, const OffsetList& /*offsets*/, const LengthList& /*lengths*/) { return S3ObjectName(uri).dataHandle(); } diff --git a/src/eckit/io/s3/S3URIManager.h b/src/eckit/io/s3/S3URIManager.h index 7a5262e4e..9b3dc5e1a 100644 --- a/src/eckit/io/s3/S3URIManager.h +++ b/src/eckit/io/s3/S3URIManager.h @@ -22,24 +22,24 @@ #include "eckit/filesystem/URIManager.h" +#include + namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -class S3URIManager: public URIManager { +class S3URIManager : public URIManager { public: // methods - S3URIManager(const std::string& name); - - ~S3URIManager() override; + explicit S3URIManager(const std::string& name); bool authority() override { return true; } private: // methods - bool exists(const URI&) override; + bool exists(const URI& uri) override; - DataHandle* newWriteHandle(const URI&) override; - DataHandle* newReadHandle(const URI&) override; - DataHandle* newReadHandle(const URI&, const OffsetList&, const LengthList&) override; + DataHandle* newWriteHandle(const URI& uri) override; + DataHandle* newReadHandle(const URI& uri) override; + DataHandle* newReadHandle(const URI& uri, const OffsetList& offsets, const LengthList& lengths) override; std::string asString(const URI& uri) const override; }; diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index ee0baf80b..b1d4d9906 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -16,13 +16,26 @@ #include "eckit/io/s3/aws/S3ClientAWS.h" #include "eckit/config/LibEcKit.h" +#include "eckit/exception/Exceptions.h" #include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Session.h" #include "eckit/io/s3/aws/S3ContextAWS.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" +#include "eckit/net/IPAddress.h" #include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include #include #include #include @@ -32,10 +45,15 @@ #include #include #include +#include #include +#include #include #include +#include +#include +#include //---------------------------------------------------------------------------------------------------------------------- @@ -47,10 +65,10 @@ inline std::string awsErrorMessage(const std::string& msg, const Aws::S3::S3Erro return oss.str(); } -class BufferIOStream: public Aws::IOStream { +class BufferIOStream : public Aws::IOStream { public: - BufferIOStream(void* buffer, const uint64_t length): - Aws::IOStream(new Aws::Utils::Stream::PreallocatedStreamBuf(reinterpret_cast(buffer), length)) { + BufferIOStream(void* buffer, const uint64_t length) + : Aws::IOStream(new Aws::Utils::Stream::PreallocatedStreamBuf(reinterpret_cast(buffer), length)) { } ~BufferIOStream() override { delete rdbuf(); } @@ -62,52 +80,53 @@ class BufferIOStream: public Aws::IOStream { namespace eckit { -const auto ALLOC_TAG = "S3ClientAWS"; +const auto allocTag = "S3ClientAWS"; -S3ClientAWS::S3ClientAWS(const S3Config& config): S3Client(config), ctx_(S3ContextAWS::instance()) { - configure(config); -} - -S3ClientAWS::~S3ClientAWS() = default; +S3ClientAWS::S3ClientAWS(const S3Config& config) : S3Client(config), ctx_ {S3ContextAWS::instance()} { } //---------------------------------------------------------------------------------------------------------------------- -void S3ClientAWS::configure(const S3Config& config) { - LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client..." << std::endl; +void S3ClientAWS::configure() const { + + LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client... "; Aws::Client::ClientConfigurationInitValues initVal; initVal.shouldDisableIMDS = true; - Aws::Client::ClientConfiguration configuration(initVal); + Aws::Client::ClientConfiguration configAWS(initVal); // we are not an ec2 instance - configuration.disableIMDS = true; - configuration.disableImdsV1 = true; + configAWS.disableIMDS = true; + configAWS.disableImdsV1 = true; // setup region - if (!config.region.empty()) { configuration.region = config.region; } + if (!config().region.empty()) { configAWS.region = config().region; } // configuration.scheme = Aws::Http::Scheme::HTTPS; - configuration.verifySSL = false; + configAWS.verifySSL = false; // setup endpoint - if (!config.endpoint.host().empty()) { configuration.endpointOverride = config.endpoint.host(); } - if (config.endpoint.port() > 0) { configuration.endpointOverride += ":" + std::to_string(config.endpoint.port()); } - - if (auto cred = S3Session::instance().getCredentials(config.endpoint)) { - // credentials provider - auto cProvider = Aws::MakeShared(ALLOC_TAG, cred->keyID, cred->secret); - // endpoint provider - auto eProvider = Aws::MakeShared(ALLOC_TAG); - // client - client_ = std::make_unique(cProvider, eProvider, configuration); - } else { - throw S3SeriousBug("No credentials found!", Here()); - } + if (config().endpoint.host().empty()) { throw UserError("Empty endpoint hostname in configuration!", Here()); } + + configAWS.endpointOverride = net::IPAddress::hostAddress(config().endpoint.host()).asString(); + + ASSERT(config().endpoint.port() > 0); + configAWS.endpointOverride += ":" + std::to_string(config().endpoint.port()); + + LOG_DEBUG_LIB(LibEcKit) << "endpoint=" << configAWS.endpointOverride << std::endl; + + const auto cred = S3Session::instance().getCredential(config().endpoint); + // credentials provider + auto cProvider = Aws::MakeShared(allocTag, cred->keyID, cred->secret); + // endpoint provider + auto eProvider = Aws::MakeShared(allocTag); + // client + client_ = std::make_unique(cProvider, eProvider, configAWS); } -auto S3ClientAWS::getClient() const -> Aws::S3::S3Client& { - if (client_) { return *client_; } - throw S3SeriousBug("Invalid client!", Here()); +auto S3ClientAWS::client() const -> Aws::S3::S3Client& { + if (!client_) { configure(); } + if (!client_) { throw S3SeriousBug("Invalid client!", Here()); } + return *client_; } //---------------------------------------------------------------------------------------------------------------------- @@ -117,7 +136,7 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { Aws::S3::Model::CreateBucketRequest request; request.SetBucket(bucket); - auto outcome = getClient().CreateBucket(request); + auto outcome = client().CreateBucket(request); if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); @@ -129,6 +148,8 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { throw S3SeriousBug(msg, Here()); } + /// @todo do we wait for the bucket to propagate? + LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; } @@ -142,7 +163,7 @@ void S3ClientAWS::deleteBucket(const std::string& bucket) const { Aws::S3::Model::DeleteBucketRequest request; request.SetBucket(bucket); - auto outcome = getClient().DeleteBucket(request); + auto outcome = client().DeleteBucket(request); if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); @@ -164,13 +185,13 @@ auto S3ClientAWS::bucketExists(const std::string& bucket) const -> bool { request.SetBucket(bucket); - return getClient().HeadBucket(request).IsSuccess(); + return client().HeadBucket(request).IsSuccess(); } //---------------------------------------------------------------------------------------------------------------------- auto S3ClientAWS::listBuckets() const -> std::vector { - auto outcome = getClient().ListBuckets(); + auto outcome = client().ListBuckets(); if (outcome.IsSuccess()) { std::vector buckets; @@ -185,8 +206,10 @@ auto S3ClientAWS::listBuckets() const -> std::vector { //---------------------------------------------------------------------------------------------------------------------- // PUT OBJECT -auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object, const void* buffer, - const uint64_t length) const -> long long { +auto S3ClientAWS::putObject(const std::string& bucket, + const std::string& object, + const void* buffer, + const uint64_t length) const -> long long { Aws::S3::Model::PutObjectRequest request; request.SetBucket(bucket); @@ -194,14 +217,14 @@ auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object // request.SetContentLength(length); if (buffer && length > 0) { - auto streamBuffer = Aws::MakeShared(ALLOC_TAG, const_cast(buffer), length); + auto streamBuffer = Aws::MakeShared(allocTag, const_cast(buffer), length); request.SetBody(streamBuffer); } else { // empty object - request.SetBody(Aws::MakeShared(ALLOC_TAG)); + request.SetBody(Aws::MakeShared(allocTag)); } - auto outcome = getClient().PutObject(request); + auto outcome = client().PutObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Put object=" << object << " [len=" << length << "] to bucket=" << bucket << std::endl; @@ -216,17 +239,20 @@ auto S3ClientAWS::putObject(const std::string& bucket, const std::string& object //---------------------------------------------------------------------------------------------------------------------- // GET OBJECT -auto S3ClientAWS::getObject(const std::string& bucket, const std::string& object, void* buffer, const uint64_t offset, +auto S3ClientAWS::getObject(const std::string& bucket, + const std::string& object, + void* buffer, + const uint64_t /*offset*/, const uint64_t length) const -> long long { Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket); request.SetKey(object); - request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(ALLOC_TAG, buffer, length); }); + request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(allocTag, buffer, length); }); /// @todo range and streambuf // request.SetRange(std::to_string(offset) + "-" + std::to_string(offset + length)); - auto outcome = getClient().GetObject(request); + auto outcome = client().GetObject(request); if (outcome.IsSuccess()) { LOG_DEBUG_LIB(LibEcKit) << "Get object=" << object << " from bucket=" << bucket << std::endl; @@ -248,7 +274,7 @@ void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& obj request.SetBucket(bucket); request.SetKey(object); - auto outcome = getClient().DeleteObject(request); + auto outcome = client().DeleteObject(request); if (!outcome.IsSuccess()) { auto msg = awsErrorMessage("Failed to delete object=" + object + " in bucket=" + bucket, outcome.GetError()); @@ -273,7 +299,7 @@ void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vector + namespace eckit { //---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/io/CMakeLists.txt b/tests/io/CMakeLists.txt index fb9459f12..a567332d4 100644 --- a/tests/io/CMakeLists.txt +++ b/tests/io/CMakeLists.txt @@ -74,21 +74,11 @@ ecbuild_add_test( TARGET eckit_test_radoshandle INCLUDES ${RADOS_INCLUDE_DIRS} LIBS eckit ) -ecbuild_add_test( TARGET eckit_test_s3client - SOURCES test_s3client.cc - CONDITION HAVE_AWS_S3 - INCLUDES ${AWSSDK_INCLUDE_DIRS} - LIBS eckit ) - -ecbuild_add_test( TARGET eckit_test_s3handle - SOURCES test_s3handle.cc - CONDITION HAVE_AWS_S3 - INCLUDES ${AWSSDK_INCLUDE_DIRS} - LIBS eckit ) - ecbuild_add_test( TARGET eckit_rados-performance SOURCES rados-performance.cc CONDITION HAVE_EXTRA_TESTS AND HAVE_RADOS INCLUDES ${RADOS_INCLUDE_DIRS} TEST_DEPENDS get_eckit_io_test_data LIBS eckit ) + +add_subdirectory( s3 ) diff --git a/tests/io/s3/CMakeLists.txt b/tests/io/s3/CMakeLists.txt new file mode 100644 index 000000000..4adca6508 --- /dev/null +++ b/tests/io/s3/CMakeLists.txt @@ -0,0 +1,23 @@ + +if( eckit_HAVE_AWS_S3 ) + + file( + COPY S3Config.yaml S3Credentials.yaml + DESTINATION ${CMAKE_CURRENT_BINARY_DIR} + ) + + ecbuild_add_test( + TARGET eckit_test_s3client + SOURCES test_s3client.cc + INCLUDES ${AWSSDK_INCLUDE_DIRS} + LIBS eckit ${AWSSDK_LINK_LIBRARIES} + ) + + ecbuild_add_test( + TARGET eckit_test_s3handle + SOURCES test_s3handle.cc + INCLUDES ${AWSSDK_INCLUDE_DIRS} + LIBS eckit ${AWSSDK_LINK_LIBRARIES} + ) + +endif( eckit_HAVE_AWS_S3 ) diff --git a/tests/io/s3/S3Config.yaml b/tests/io/s3/S3Config.yaml new file mode 100644 index 000000000..fd2026ec9 --- /dev/null +++ b/tests/io/s3/S3Config.yaml @@ -0,0 +1,15 @@ +--- +servers: + - endpoint: "127.0.0.1:9000" + region: "default" # (default) + secure: true # (default) + backend: "AWS" # (default) + + - endpoint: "minio:9000" + region: "eu-central-1" + + - endpoint: "localhost:9000" + region: "eu-central-1" + + # region is inferred from the endpoint + - endpoint: "eu-central-1.ecmwf.int:9000" diff --git a/tests/io/s3/S3Credentials.yaml b/tests/io/s3/S3Credentials.yaml new file mode 100644 index 000000000..1935a4815 --- /dev/null +++ b/tests/io/s3/S3Credentials.yaml @@ -0,0 +1,13 @@ +--- +credentials: + - endpoint: '127.0.0.1:9000' + accessKeyID: 'minio' + secretKey: 'minio1234' + + - endpoint: 'minio:9000' + accessKeyID: 'minio' + secretKey: 'minio1234' + + - endpoint: 'localhost:9000' + accessKeyID: 'asd2' + secretKey: 'asd2' diff --git a/tests/io/s3/test_s3client.cc b/tests/io/s3/test_s3client.cc new file mode 100644 index 000000000..09be236ff --- /dev/null +++ b/tests/io/s3/test_s3client.cc @@ -0,0 +1,218 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @file test_s3client.cc +/// @author Metin Cakircali +/// @author Simon Smart +/// @date Jan 2024 + +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/net/Endpoint.h" +#include "eckit/testing/Test.h" + +#include +#include +#include + +using namespace eckit; +using namespace eckit::testing; + +// this test requires a working S3 endpoint and credentials. such as, a local docker container MinIO instance + +namespace eckit::test { + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +const net::Endpoint TEST_ENDPOINT {"minio", 9000}; + +const S3Config TEST_CONFIG {TEST_ENDPOINT, "eu-central-1"}; + +bool findString(const std::vector& list, const std::string& item) { + return (std::find(list.begin(), list.end(), item) != list.end()); +} + +void cleanup() { + auto client = S3Client::makeUnique(TEST_CONFIG); + for (const auto* name : {"test-bucket-1", "test-bucket-2"}) { + if (client->bucketExists(name)) { + client->emptyBucket(name); + client->deleteBucket(name); + } + } +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 client: API") { + + const S3Config config {TEST_ENDPOINT, "eu-central-1"}; + const S3Credential cred {TEST_ENDPOINT, "minio", "minio1234"}; + + EXPECT(S3Session::instance().addClient(config)); + EXPECT(S3Session::instance().addCredential(cred)); + + EXPECT_NO_THROW(cleanup()); + + EXPECT_NO_THROW(S3Session::instance().removeClient(config.endpoint)); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 client: read from file") { + + auto& session = S3Session::instance(); + + session.clear(); + + EXPECT_THROWS_AS(session.getClient({"127.0.0.1", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getClient({"minio", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getClient({"localhost", 9000}), S3EntityNotFound); + + session.loadClients("./S3Config.yaml"); + + EXPECT_NO_THROW(session.getClient({"127.0.0.1", 9000})); + EXPECT_NO_THROW(session.getClient({"minio", 9000})); + EXPECT_NO_THROW(session.getClient({"localhost", 9000})); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 credentials: API") { + + const S3Credential cred {TEST_ENDPOINT, "minio", "minio1234"}; + + EXPECT(S3Session::instance().addCredential(cred)); + + EXPECT_NO_THROW(cleanup()); + + EXPECT_NO_THROW(S3Session::instance().removeCredential(cred.endpoint)); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 credentials: read from file") { + + auto& session = S3Session::instance(); + + session.clear(); + + EXPECT_THROWS_AS(session.getCredential({"127.0.0.1", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getCredential({"minio", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getCredential({"localhost", 9000}), S3EntityNotFound); + + session.loadCredentials("./S3Credentials.yaml"); + + EXPECT_NO_THROW(session.getCredential({"127.0.0.1", 9000})); + EXPECT_NO_THROW(session.getCredential({"minio", 9000})); + EXPECT_NO_THROW(session.getCredential({"localhost", 9000})); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 backends") { + S3Config cfgTmp(TEST_CONFIG); + + cfgTmp.backend = S3Backend::AWS; + EXPECT_NO_THROW(S3Client::makeUnique(cfgTmp)); + + cfgTmp.backend = S3Backend::REST; + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); + + cfgTmp.backend = S3Backend::MINIO; + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); +} + +// CASE("wrong credentials") { +// ensureClean(); +// +// S3Config cfgTmp(cfg); +// cfgTmp.region = "no-region-random"; +// +// EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("failed-bucket")); +// } + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("create s3 bucket in non-existing region") { + EXPECT_NO_THROW(cleanup()); + + // this test requires an S3 endpoint that sets it's region + // a MinIO instance with empty region will not throw an exception + + auto cfgTmp = TEST_CONFIG; + cfgTmp.region = "non-existing-region-random"; + + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("test-bucket-1")); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("create s3 bucket") { + EXPECT_NO_THROW(cleanup()); + + auto client = S3Client::makeUnique(TEST_CONFIG); + + EXPECT_NO_THROW(client->createBucket("test-bucket-1")); + + EXPECT_THROWS(client->createBucket("test-bucket-1")); + + EXPECT_NO_THROW(client->createBucket("test-bucket-2")); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("list s3 buckets") { + EXPECT_NO_THROW(cleanup()); + + auto client = S3Client::makeUnique(TEST_CONFIG); + EXPECT_NO_THROW(client->createBucket("test-bucket-1")); + EXPECT_NO_THROW(client->createBucket("test-bucket-2")); + + { + const auto buckets = client->listBuckets(); + + EXPECT(findString(buckets, "test-bucket-1")); + EXPECT(findString(buckets, "test-bucket-2")); + + EXPECT_NO_THROW(client->deleteBucket("test-bucket-1")); + EXPECT_NO_THROW(client->deleteBucket("test-bucket-2")); + } + + { + const auto buckets = client->listBuckets(); + EXPECT_NOT(findString(buckets, "test-bucket-1")); + EXPECT_NOT(findString(buckets, "test-bucket-2")); + } + + EXPECT_THROWS(client->deleteBucket("test-bucket-1")); + EXPECT_THROWS(client->deleteBucket("test-bucket-2")); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test + +int main(int argc, char** argv) { + int ret = -1; + + ret = run_tests(argc, argv); + + test::cleanup(); + + return ret; +} diff --git a/tests/io/test_s3handle.cc b/tests/io/s3/test_s3handle.cc similarity index 58% rename from tests/io/test_s3handle.cc rename to tests/io/s3/test_s3handle.cc index db5c84530..8c5fcba2a 100644 --- a/tests/io/test_s3handle.cc +++ b/tests/io/s3/test_s3handle.cc @@ -19,28 +19,42 @@ #include "eckit/io/MemoryHandle.h" #include "eckit/io/s3/S3BucketName.h" #include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Handle.h" +#include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3ObjectName.h" #include "eckit/io/s3/S3Session.h" #include "eckit/log/Bytes.h" #include "eckit/log/Timer.h" +#include "eckit/net/Endpoint.h" #include "eckit/testing/Test.h" +#include + #include +#include +#include +#include #include +#include + +using namespace std::string_literals; -using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + constexpr std::string_view TEST_DATA = "abcdefghijklmnopqrstuvwxyz"; -static const std::string TEST_BUCKET("eckit-s3handle-test-bucket"); -static const std::string TEST_OBJECT("eckit-s3handle-test-object"); +const net::Endpoint TEST_ENDPOINT {"minio", 9000}; +const std::string TEST_BUCKET {"eckit-s3handle-test-bucket"}; +const std::string TEST_OBJECT {"eckit-s3handle-test-object"}; -S3Config cfg("eu-central-1", "127.0.0.1", 9000); +const S3Config TEST_CONFIG {TEST_ENDPOINT, "eu-central-1"}; +const S3Credential TEST_CREDENTIAL {TEST_ENDPOINT, "minio", "minio1234"}; //---------------------------------------------------------------------------------------------------------------------- @@ -61,8 +75,8 @@ void writePerformance(S3BucketName& bucket, const int count) { << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; } -void ensureClean() { - auto client = S3Client::makeUnique(cfg); +void cleanup() { + auto client = S3Client::makeUnique(TEST_CONFIG); for (const auto& name : {TEST_BUCKET}) { if (client->bucketExists(name)) { client->emptyBucket(name); @@ -71,20 +85,30 @@ void ensureClean() { } } +} // namespace + //---------------------------------------------------------------------------------------------------------------------- -CASE("invalid bucket") { - EXPECT_THROWS(S3BucketName("http://127.0.0.1:9000/" + TEST_BUCKET)); - EXPECT_THROWS(S3BucketName("s3://127.0.0.1" + TEST_BUCKET)); - EXPECT_THROWS(S3BucketName("s3://127.0.0.1/" + TEST_BUCKET)); +CASE("initialize s3 session") { + + EXPECT(S3Session::instance().addCredential(TEST_CREDENTIAL)); + EXPECT(S3Session::instance().addClient(TEST_CONFIG)); + + EXPECT_NO_THROW(cleanup()); +} + +CASE("invalid s3 bucket") { + EXPECT_THROWS(S3BucketName(URI {"http://127.0.0.1:9000/" + TEST_BUCKET})); + EXPECT_THROWS(S3BucketName(URI {"s3://127.0.0.1" + TEST_BUCKET})); + EXPECT_THROWS(S3BucketName(URI {"s3://127.0.0.1/" + TEST_BUCKET})); } //---------------------------------------------------------------------------------------------------------------------- CASE("S3BucketName: no bucket") { - ensureClean(); + EXPECT_NO_THROW(cleanup()); - S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); EXPECT_NOT(bucket.exists()); @@ -99,10 +123,12 @@ CASE("S3BucketName: no bucket") { EXPECT_NO_THROW(bucket.ensureDestroyed()); } +//---------------------------------------------------------------------------------------------------------------------- + CASE("S3BucketName: create bucket") { - ensureClean(); + EXPECT_NO_THROW(cleanup()); - S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); EXPECT_NOT(bucket.exists()); @@ -115,26 +141,30 @@ CASE("S3BucketName: create bucket") { EXPECT_NO_THROW(bucket.destroy()); } +//---------------------------------------------------------------------------------------------------------------------- + CASE("S3BucketName: empty bucket") { - ensureClean(); + EXPECT_NO_THROW(cleanup()); - S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); // CREATE BUCKET EXPECT_NO_THROW(bucket.ensureCreated()); EXPECT(bucket.exists()); // LIST - EXPECT(bucket.listObjects().size() == 0); + EXPECT_EQUAL(bucket.listObjects().size(), 0); // DESTROY BUCKET EXPECT_NO_THROW(bucket.destroy()); } +//---------------------------------------------------------------------------------------------------------------------- + CASE("S3BucketName: bucket with object") { - ensureClean(); + EXPECT_NO_THROW(cleanup()); - S3BucketName bucket("s3://127.0.0.1:9000/" + TEST_BUCKET); + S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); // CREATE BUCKET EXPECT_NO_THROW(bucket.ensureCreated()); @@ -143,7 +173,7 @@ CASE("S3BucketName: bucket with object") { EXPECT_NO_THROW(bucket.makeObject(TEST_OBJECT)->put(TEST_DATA.data(), TEST_DATA.size())); // LIST - EXPECT(bucket.listObjects().size() == 1); + EXPECT_EQUAL(bucket.listObjects().size(), 1); // DESTROY BUCKET EXPECT_THROWS(bucket.destroy()); @@ -152,22 +182,22 @@ CASE("S3BucketName: bucket with object") { //---------------------------------------------------------------------------------------------------------------------- -CASE("S3Handle operations") { - ensureClean(); +CASE("S3Handle: basic operations") { + EXPECT_NO_THROW(cleanup()); - S3Client::makeUnique(cfg)->createBucket(TEST_BUCKET); + S3Client::makeUnique(TEST_CONFIG)->createBucket(TEST_BUCKET); const void* buffer = TEST_DATA.data(); const long length = TEST_DATA.size(); - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); { S3ObjectName object(uri); std::unique_ptr handle(object.dataHandle()); EXPECT_NO_THROW(handle->openForWrite(length)); - EXPECT(handle->write(buffer, length) == length); + EXPECT_EQUAL(handle->write(buffer, length), length); EXPECT_NO_THROW(handle->close()); } @@ -180,7 +210,7 @@ CASE("S3Handle operations") { EXPECT_NO_THROW(handle->read(rbuf.data(), length)); - EXPECT(rbuf == TEST_DATA); + EXPECT_EQUAL(rbuf, TEST_DATA); EXPECT_NO_THROW(handle->close()); } @@ -191,17 +221,19 @@ CASE("S3Handle operations") { MemoryHandle memHandle(length); handle->saveInto(memHandle); - EXPECT(memHandle.size() == Length(length)); - EXPECT(::memcmp(memHandle.data(), buffer, length) == 0); + EXPECT_EQUAL(memHandle.size(), Length(length)); + EXPECT_EQUAL(::memcmp(memHandle.data(), buffer, length), 0); EXPECT_NO_THROW(handle->close()); } } -CASE("S3Handle::openForRead") { - ensureClean(); +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3Handle: openForRead") { + EXPECT_NO_THROW(cleanup()); - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); @@ -221,10 +253,12 @@ CASE("S3Handle::openForRead") { EXPECT_NO_THROW(handle->openForRead()); } -CASE("S3Handle::openForWrite") { - ensureClean(); +//---------------------------------------------------------------------------------------------------------------------- - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); +CASE("S3Handle: openForWrite") { + EXPECT_NO_THROW(cleanup()); + + const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); { // NO BUCKET std::unique_ptr handle(uri.newWriteHandle()); @@ -242,10 +276,12 @@ CASE("S3Handle::openForWrite") { } } -CASE("S3Handle::read") { - ensureClean(); +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3Handle: read") { + EXPECT_NO_THROW(cleanup()); - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); @@ -257,37 +293,39 @@ CASE("S3Handle::read") { // OPEN EXPECT_NO_THROW(handle->openForRead()); - { /// @todo range based read - - // const auto length = TEST_DATA.size(); - - // std::string rbuf; - // rbuf.resize(length); - // auto len = handle->read(rbuf.data(), 10); - // std::cout << "========" << rbuf << std::endl; - // len = handle->read(rbuf.data(), length - 10); - // std::cout << "========" << rbuf << std::endl; - // EXPECT(rbuf == TEST_DATA); - } + /// @todo range based read + // { + // const auto length = TEST_DATA.size(); + // + // std::string rbuf; + // rbuf.resize(length); + // auto len = handle->read(rbuf.data(), 10); + // std::cout << "========" << rbuf << std::endl; + // len = handle->read(rbuf.data(), length - 10); + // std::cout << "========" << rbuf << std::endl; + // EXPECT_EQUAL(rbuf, TEST_DATA); + // } // CLOSE EXPECT_NO_THROW(handle->close()); } -CASE("performance: write 10 100 1000 objects") { - ensureClean(); +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 performance: write 1 10 100 objects") { + EXPECT_NO_THROW(cleanup()); - const URI uri("s3://127.0.0.1:9000/" + TEST_BUCKET); + const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET); S3BucketName bucket(uri); EXPECT_NO_THROW(bucket.ensureCreated()); + writePerformance(bucket, 1); + writePerformance(bucket, 10); writePerformance(bucket, 100); - - writePerformance(bucket, 1000); } //---------------------------------------------------------------------------------------------------------------------- @@ -295,14 +333,13 @@ CASE("performance: write 10 100 1000 objects") { } // namespace eckit::test int main(int argc, char** argv) { - const S3Credential cred {"127.0.0.1:9000", "minio", "minio1234"}; - S3Session::instance().addCredentials(cred); - int ret = -1; - try { - ret = run_tests(argc, argv); - } - catch (...) { - } + + ret = run_tests(argc, argv); + + test::cleanup(); + return ret; } + +//---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/io/test_s3client.cc b/tests/io/test_s3client.cc deleted file mode 100644 index aa94427ca..000000000 --- a/tests/io/test_s3client.cc +++ /dev/null @@ -1,135 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/// @file test_s3client.cc -/// @author Metin Cakircali -/// @author Simon Smart -/// @date Jan 2024 - -#include "eckit/config/LibEcKit.h" -#include "eckit/io/Buffer.h" -#include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Session.h" -#include "eckit/testing/Test.h" - -using namespace std; -using namespace eckit; -using namespace eckit::testing; - -namespace eckit::test { - -const S3Config cfg("eu-central-1", "127.0.0.1", 9000); - -//---------------------------------------------------------------------------------------------------------------------- - -bool findString(const std::vector& list, const std::string& item) { - return (std::find(list.begin(), list.end(), item) != list.end()); -} - -void ensureClean() { - - auto client = S3Client::makeUnique(cfg); - auto&& tmp = client->listBuckets(); - std::set buckets(tmp.begin(), tmp.end()); - - for (const std::string& name : {"test-bucket-1", "test-bucket-2"}) { - if (buckets.find(name) != buckets.end()) { - client->emptyBucket(name); - client->deleteBucket(name); - } - } -} - -//---------------------------------------------------------------------------------------------------------------------- - -CASE("different types") { - S3Config cfgTmp(cfg); - - cfgTmp.type = S3Types::NONE; - EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); - - cfgTmp.type = S3Types::AWS; - EXPECT_NO_THROW(S3Client::makeUnique(cfgTmp)); - - cfgTmp.type = S3Types::REST; - EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); -} - -// CASE("wrong credentials") { -// ensureClean(); -// -// S3Config cfgTmp(cfg); -// cfgTmp.region = "no-region-random"; -// -// EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("failed-bucket")); -// } - -CASE("create bucket in non-existing region") { - ensureClean(); - - S3Config cfgTmp(cfg); - cfgTmp.region = "non-existing-region-random"; - - EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("test-bucket-1")); -} - -CASE("create bucket") { - ensureClean(); - - auto client = S3Client::makeUnique(cfg); - - EXPECT_NO_THROW(client->createBucket("test-bucket-1")); - - EXPECT_THROWS(client->createBucket("test-bucket-1")); - - EXPECT_NO_THROW(client->createBucket("test-bucket-2")); -} - -CASE("list buckets") { - ensureClean(); - - auto client = S3Client::makeUnique(cfg); - EXPECT_NO_THROW(client->createBucket("test-bucket-1")); - EXPECT_NO_THROW(client->createBucket("test-bucket-2")); - - { - const auto buckets = client->listBuckets(); - - EXPECT(findString(buckets, "test-bucket-1")); - EXPECT(findString(buckets, "test-bucket-2")); - - EXPECT_NO_THROW(client->deleteBucket("test-bucket-1")); - EXPECT_NO_THROW(client->deleteBucket("test-bucket-2")); - } - - { - const auto buckets = client->listBuckets(); - EXPECT_NOT(findString(buckets, "test-bucket-1")); - EXPECT_NOT(findString(buckets, "test-bucket-2")); - } - - EXPECT_THROWS(client->deleteBucket("test-bucket-1")); - EXPECT_THROWS(client->deleteBucket("test-bucket-2")); -} - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace eckit::test - -int main(int argc, char** argv) { - const S3Credential cred {"127.0.0.1:9000", "minio", "minio1234"}; - S3Session::instance().addCredentials(cred); - - auto ret = run_tests(argc, argv); - try { - eckit::test::ensureClean(); - } catch (...) { } - return ret; -} From 3adb373adfc8a97aa260c50fb818ad47c8c4c1d1 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 18 Dec 2024 16:51:53 +0100 Subject: [PATCH 80/90] fix(S3): uri bug --- src/eckit/io/s3/S3BucketName.cc | 18 ++++++++++-------- src/eckit/io/s3/S3BucketName.h | 4 ++-- src/eckit/io/s3/S3Name.cc | 21 +++++++++++---------- src/eckit/io/s3/S3Name.h | 14 ++++++-------- src/eckit/io/s3/S3ObjectName.cc | 20 +++++++++++--------- src/eckit/io/s3/S3ObjectName.h | 4 ++-- 6 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc index f90adfabc..fa70f5cc6 100644 --- a/src/eckit/io/s3/S3BucketName.cc +++ b/src/eckit/io/s3/S3BucketName.cc @@ -35,14 +35,22 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- S3BucketName::S3BucketName(const URI& uri) : S3Name(uri) { - parse(); + const auto pairs = parse(uri.name()); + if (pairs.empty()) { throw S3SeriousBug("Could not parse bucket name!", Here()); } + bucket_ = pairs[0]; } S3BucketName::S3BucketName(const net::Endpoint& endpoint, std::string bucket) - : S3Name(endpoint, "/"), bucket_ {std::move(bucket)} { } + : S3Name(endpoint), bucket_ {std::move(bucket)} { } //---------------------------------------------------------------------------------------------------------------------- +auto S3BucketName::uri() const -> URI { + auto uri = S3Name::uri(); + uri.path("/" + bucket_); + return uri; +} + auto S3BucketName::asString() const -> std::string { return S3Name::asString() + "/" + bucket_; } @@ -52,12 +60,6 @@ void S3BucketName::print(std::ostream& out) const { S3Name::print(out); } -void S3BucketName::parse() { - const auto pairs = parseName(); - if (pairs.empty()) { throw S3SeriousBug("Could not parse bucket name!", Here()); } - bucket_ = pairs[0]; -} - //---------------------------------------------------------------------------------------------------------------------- auto S3BucketName::makeObject(const std::string& object) const -> std::unique_ptr { diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h index fafbab992..99eecbeb5 100644 --- a/src/eckit/io/s3/S3BucketName.h +++ b/src/eckit/io/s3/S3BucketName.h @@ -42,6 +42,8 @@ class S3BucketName : public S3Name { auto makeObject(const std::string& object) const -> std::unique_ptr; + auto uri() const -> URI override; + auto exists() const -> bool override; void create(); @@ -60,8 +62,6 @@ class S3BucketName : public S3Name { private: // methods void print(std::ostream& out) const override; - void parse(); - private: // members std::string bucket_; }; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index 6ffeafaa2..dbf262f22 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -24,26 +24,31 @@ #include #include #include -#include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Name::S3Name(const URI& uri) : endpoint_ {uri}, name_ {uri.name()} { +auto S3Name::parse(const std::string& name) -> std::vector { + return Tokenizer("/").tokenize(name); +} + +//---------------------------------------------------------------------------------------------------------------------- + +S3Name::S3Name(const net::Endpoint& endpoint) : endpoint_ {endpoint} { } + +S3Name::S3Name(const URI& uri) : S3Name( {uri.host(), uri.port()}) { /// @todo is "s3://endpoint/bucket/object" a valid URI ? ASSERT(uri.scheme() == type); } -S3Name::S3Name(const net::Endpoint& endpoint, std::string name) : endpoint_ {endpoint}, name_ {std::move(name)} { } - S3Name::~S3Name() = default; //---------------------------------------------------------------------------------------------------------------------- auto S3Name::uri() const -> URI { - return {type, asString()}; + return {type, endpoint_.host(), endpoint_.port()}; } auto S3Name::asString() const -> std::string { @@ -51,15 +56,11 @@ auto S3Name::asString() const -> std::string { } void S3Name::print(std::ostream& out) const { - out << ",endpoint=" << endpoint_ << ",name=" << name_ << "]"; + out << ",endpoint=" << endpoint_ << ",]"; } //---------------------------------------------------------------------------------------------------------------------- -auto S3Name::parseName() const -> std::vector { - return Tokenizer("/").tokenize(name_); -} - auto S3Name::client() const -> S3Client& { if (!client_) { client_ = S3Session::instance().getClient(endpoint_); } return *client_; diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 7dc3ccff2..5f59c0229 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -37,10 +37,12 @@ class S3Name { public: // types static constexpr auto type = "s3"; + static auto parse(const std::string& name) -> std::vector; + public: // methods - explicit S3Name(const URI& uri); + explicit S3Name(const net::Endpoint& endpoint); - S3Name(const net::Endpoint& endpoint, std::string name); + explicit S3Name(const URI& uri); // rules @@ -54,10 +56,10 @@ class S3Name { // accessors - auto uri() const -> URI; - auto endpoint() const -> const net::Endpoint& { return endpoint_; } + virtual auto uri() const -> URI; + virtual auto exists() const -> bool = 0; virtual auto asString() const -> std::string; @@ -67,14 +69,10 @@ class S3Name { friend std::ostream& operator<<(std::ostream& out, const S3Name& name); - [[nodiscard]] - auto parseName() const -> std::vector; - auto client() const -> S3Client&; private: // members net::Endpoint endpoint_; - std::string name_; mutable std::shared_ptr client_; }; diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc index 27b89a6e2..985290beb 100644 --- a/src/eckit/io/s3/S3ObjectName.cc +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -32,11 +32,14 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri) { - parse(); + const auto pairs = parse(uri.name()); + if (pairs.size() != 2) { throw S3SeriousBug("Could not parse bucket and object names!", Here()); } + bucket_ = pairs[0]; + object_ = pairs[1]; } S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object) - : S3Name(endpoint, "/"), bucket_ {std::move(bucket)}, object_ {std::move(object)} { } + : S3Name(endpoint), bucket_ {std::move(bucket)}, object_ {std::move(object)} { } //---------------------------------------------------------------------------------------------------------------------- @@ -45,15 +48,14 @@ void S3ObjectName::print(std::ostream& out) const { S3Name::print(out); } -void S3ObjectName::parse() { - const auto pairs = parseName(); - if (pairs.size() != 2) { throw S3SeriousBug("Could not parse bucket and object names!", Here()); } - bucket_ = pairs[0]; - object_ = pairs[1]; -} - //---------------------------------------------------------------------------------------------------------------------- +auto S3ObjectName::uri() const -> URI { + auto uri = S3Name::uri(); + uri.path("/" + bucket_ + "/" + object_); + return uri; +} + auto S3ObjectName::asString() const -> std::string { return S3Name::asString() + "/" + bucket_ + "/" + object_; } diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h index 447fab093..557dc4806 100644 --- a/src/eckit/io/s3/S3ObjectName.h +++ b/src/eckit/io/s3/S3ObjectName.h @@ -38,6 +38,8 @@ class S3ObjectName : public S3Name { S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object); + auto uri() const -> URI override; + auto size() const -> long long; auto exists() const -> bool override; @@ -65,8 +67,6 @@ class S3ObjectName : public S3Name { private: // methods void print(std::ostream& out) const override; - void parse(); - private: // members std::string bucket_; std::string object_; From 14f357944393014fc1af04ae812c2ce66591136e Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Thu, 19 Dec 2024 22:38:48 +0100 Subject: [PATCH 81/90] feat(S3): added S3ObjectPath --- src/eckit/CMakeLists.txt | 2 ++ src/eckit/io/s3/S3BucketName.cc | 15 +++++--- src/eckit/io/s3/S3BucketName.h | 3 ++ src/eckit/io/s3/S3Client.h | 22 ++++++------ src/eckit/io/s3/S3Name.cc | 17 +++++++-- src/eckit/io/s3/S3Name.h | 8 +++-- src/eckit/io/s3/S3ObjectName.cc | 36 ++++++++++--------- src/eckit/io/s3/S3ObjectName.h | 18 ++++++---- src/eckit/io/s3/S3ObjectPath.cc | 32 +++++++++++++++++ src/eckit/io/s3/S3ObjectPath.h | 40 ++++++++++++++++++++++ src/eckit/io/s3/aws/S3ClientAWS.cc | 55 ++++++++++++++---------------- src/eckit/io/s3/aws/S3ClientAWS.h | 26 +++++++------- 12 files changed, 185 insertions(+), 89 deletions(-) create mode 100644 src/eckit/io/s3/S3ObjectPath.cc create mode 100644 src/eckit/io/s3/S3ObjectPath.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 7242598db..02e7c78be 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -305,6 +305,8 @@ if( eckit_HAVE_AWS_S3 ) io/s3/S3Name.h io/s3/S3ObjectName.cc io/s3/S3ObjectName.h + io/s3/S3ObjectPath.cc + io/s3/S3ObjectPath.h io/s3/S3Session.cc io/s3/S3Session.h io/s3/S3URIManager.cc diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc index fa70f5cc6..735012f5f 100644 --- a/src/eckit/io/s3/S3BucketName.cc +++ b/src/eckit/io/s3/S3BucketName.cc @@ -21,6 +21,7 @@ #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Name.h" #include "eckit/io/s3/S3ObjectName.h" +#include "eckit/io/s3/S3ObjectPath.h" #include "eckit/log/Log.h" #include "eckit/net/Endpoint.h" @@ -34,15 +35,19 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3BucketName::S3BucketName(const URI& uri) : S3Name(uri) { - const auto pairs = parse(uri.name()); - if (pairs.empty()) { throw S3SeriousBug("Could not parse bucket name!", Here()); } - bucket_ = pairs[0]; +auto S3BucketName::parse(const std::string& name) -> std::string { + const auto parsed = S3Name::parse(name); + if (parsed.size() != 1) { throw S3SeriousBug("Could not parse bucket from name: " + name, Here()); } + return {parsed[0]}; } +//---------------------------------------------------------------------------------------------------------------------- + S3BucketName::S3BucketName(const net::Endpoint& endpoint, std::string bucket) : S3Name(endpoint), bucket_ {std::move(bucket)} { } +S3BucketName::S3BucketName(const URI& uri) : S3Name(uri), bucket_ {parse(uri.name())} { } + //---------------------------------------------------------------------------------------------------------------------- auto S3BucketName::uri() const -> URI { @@ -63,7 +68,7 @@ void S3BucketName::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- auto S3BucketName::makeObject(const std::string& object) const -> std::unique_ptr { - return std::make_unique(endpoint(), bucket_, object); + return std::make_unique(endpoint(), S3ObjectPath {bucket_, object}); } auto S3BucketName::exists() const -> bool { diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h index 99eecbeb5..ab5f2a102 100644 --- a/src/eckit/io/s3/S3BucketName.h +++ b/src/eckit/io/s3/S3BucketName.h @@ -35,6 +35,9 @@ class S3ObjectName; //---------------------------------------------------------------------------------------------------------------------- class S3BucketName : public S3Name { +public: // methods + static auto parse(const std::string& name) -> std::string; + public: // methods explicit S3BucketName(const URI& uri); diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index 1d00c45de..aea094f45 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -20,6 +20,7 @@ #pragma once #include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3ObjectPath.h" #include #include @@ -47,6 +48,8 @@ class S3Client { auto config() const -> const S3Config& { return config_; } + // bucket operations + virtual void createBucket(const std::string& bucket) const = 0; virtual void emptyBucket(const std::string& bucket) const = 0; @@ -57,26 +60,21 @@ class S3Client { virtual auto listBuckets() const -> std::vector = 0; - virtual auto putObject(const std::string& bucket, - const std::string& object, - const void* buffer, - uint64_t length) const -> long long = 0; + // object operations + + virtual auto putObject(const S3ObjectPath& path, const void* buffer, uint64_t length) const -> long long = 0; - virtual auto getObject(const std::string& bucket, - const std::string& object, - void* buffer, - uint64_t offset, - uint64_t length) const -> long long = 0; + virtual auto getObject(const S3ObjectPath& path, void* buffer, uint64_t offset, uint64_t length) const -> long long = 0; - virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0; + virtual void deleteObject(const S3ObjectPath& path) const = 0; virtual void deleteObjects(const std::string& bucket, const std::vector& objects) const = 0; virtual auto listObjects(const std::string& bucket) const -> std::vector = 0; - virtual auto objectExists(const std::string& bucket, const std::string& object) const -> bool = 0; + virtual auto objectExists(const S3ObjectPath& path) const -> bool = 0; - virtual auto objectSize(const std::string& bucket, const std::string& object) const -> long long = 0; + virtual auto objectSize(const S3ObjectPath& path) const -> long long = 0; protected: // methods S3Client(); diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index dbf262f22..e1897fa23 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -17,7 +17,10 @@ #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3BucketName.h" #include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/io/s3/S3ObjectPath.h" #include "eckit/io/s3/S3Session.h" #include "eckit/utils/Tokenizer.h" @@ -34,11 +37,20 @@ auto S3Name::parse(const std::string& name) -> std::vector { return Tokenizer("/").tokenize(name); } +auto S3Name::make(const net::Endpoint& endpoint, const std::string& name) -> std::unique_ptr { + const auto names = parse(name); + switch (names.size()) { + case 1: return std::make_unique(endpoint, names[0]); + case 2: return std::make_unique(endpoint, S3ObjectPath {names[0], names[1]}); + default: throw SeriousBug("Could not parse S3 name: " + name, Here()); + } +} + //---------------------------------------------------------------------------------------------------------------------- S3Name::S3Name(const net::Endpoint& endpoint) : endpoint_ {endpoint} { } -S3Name::S3Name(const URI& uri) : S3Name( {uri.host(), uri.port()}) { +S3Name::S3Name(const URI& uri) : endpoint_ {uri.host(), uri.port()} { /// @todo is "s3://endpoint/bucket/object" a valid URI ? ASSERT(uri.scheme() == type); } @@ -62,8 +74,7 @@ void S3Name::print(std::ostream& out) const { //---------------------------------------------------------------------------------------------------------------------- auto S3Name::client() const -> S3Client& { - if (!client_) { client_ = S3Session::instance().getClient(endpoint_); } - return *client_; + return *S3Session::instance().getClient(endpoint_); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 5f59c0229..5c4f7d440 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -34,11 +34,13 @@ class S3Client; //---------------------------------------------------------------------------------------------------------------------- class S3Name { -public: // types +public: // statics static constexpr auto type = "s3"; static auto parse(const std::string& name) -> std::vector; + static auto make(const net::Endpoint& endpoint, const std::string& name) -> std::unique_ptr; + public: // methods explicit S3Name(const net::Endpoint& endpoint); @@ -56,6 +58,8 @@ class S3Name { // accessors + void endpoint(const net::Endpoint& endpoint) { endpoint_ = endpoint; } + auto endpoint() const -> const net::Endpoint& { return endpoint_; } virtual auto uri() const -> URI; @@ -73,8 +77,6 @@ class S3Name { private: // members net::Endpoint endpoint_; - - mutable std::shared_ptr client_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc index 985290beb..73fd335e4 100644 --- a/src/eckit/io/s3/S3ObjectName.cc +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -20,6 +20,7 @@ #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Handle.h" #include "eckit/io/s3/S3Name.h" +#include "eckit/io/s3/S3ObjectPath.h" #include "eckit/log/CodeLocation.h" #include "eckit/net/Endpoint.h" @@ -31,20 +32,23 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri) { - const auto pairs = parse(uri.name()); - if (pairs.size() != 2) { throw S3SeriousBug("Could not parse bucket and object names!", Here()); } - bucket_ = pairs[0]; - object_ = pairs[1]; +auto S3ObjectName::parse(const std::string& name) -> S3ObjectPath { + const auto parsed = S3Name::parse(name); + if (parsed.size() != 2) { throw S3SeriousBug("Could not parse bucket/object from name: " + name, Here()); } + return {parsed[0], parsed[1]}; } -S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object) - : S3Name(endpoint), bucket_ {std::move(bucket)}, object_ {std::move(object)} { } +//---------------------------------------------------------------------------------------------------------------------- + +S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, S3ObjectPath path) + : S3Name(endpoint), path_ {std::move(path)} { } + +S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri), path_ {parse(uri.name())} { } //---------------------------------------------------------------------------------------------------------------------- void S3ObjectName::print(std::ostream& out) const { - out << "S3ObjectName[object=" << object_ << ",bucket=" << bucket_; + out << "S3ObjectName[path=" << path_; S3Name::print(out); } @@ -52,36 +56,36 @@ void S3ObjectName::print(std::ostream& out) const { auto S3ObjectName::uri() const -> URI { auto uri = S3Name::uri(); - uri.path("/" + bucket_ + "/" + object_); + uri.path(path_); return uri; } auto S3ObjectName::asString() const -> std::string { - return S3Name::asString() + "/" + bucket_ + "/" + object_; + return S3Name::asString() + '/' + path_.asString(); } auto S3ObjectName::size() const -> long long { - return client().objectSize(bucket_, object_); + return client().objectSize(path_); } auto S3ObjectName::exists() const -> bool { - return client().objectExists(bucket_, object_); + return client().objectExists(path_); } auto S3ObjectName::bucketExists() const -> bool { - return client().bucketExists(bucket_); + return client().bucketExists(path_.bucket); } void S3ObjectName::remove() { - client().deleteObject(bucket_, object_); + client().deleteObject(path_); } auto S3ObjectName::put(const void* buffer, const long length) const -> long long { - return client().putObject(bucket_, object_, buffer, length); + return client().putObject(path_, buffer, length); } auto S3ObjectName::get(void* buffer, const long offset, const long length) const -> long long { - return client().getObject(bucket_, object_, buffer, offset, length); + return client().getObject(path_, buffer, offset, length); } auto S3ObjectName::dataHandle() -> DataHandle* { diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h index 557dc4806..8508b7574 100644 --- a/src/eckit/io/s3/S3ObjectName.h +++ b/src/eckit/io/s3/S3ObjectName.h @@ -20,7 +20,7 @@ #pragma once #include "eckit/io/s3/S3Name.h" -#include "eckit/net/Endpoint.h" +#include "eckit/io/s3/S3ObjectPath.h" #include #include @@ -33,10 +33,13 @@ class DataHandle; //---------------------------------------------------------------------------------------------------------------------- class S3ObjectName : public S3Name { +public: // helpers + static auto parse(const std::string& name) -> S3ObjectPath; + public: // methods - explicit S3ObjectName(const URI& uri); + S3ObjectName(const net::Endpoint& endpoint, S3ObjectPath path); - S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object); + explicit S3ObjectName(const URI& uri); auto uri() const -> URI override; @@ -58,9 +61,11 @@ class S3ObjectName : public S3Name { auto asString() const -> std::string override; - auto name() const -> const std::string& { return object_; } + auto path() const -> const S3ObjectPath& { return path_; } + + auto name() const -> const std::string& { return path_.object; } - auto bucket() const -> const std::string& { return bucket_; } + auto bucket() const -> const std::string& { return path_.bucket; } auto bucketExists() const -> bool; @@ -68,8 +73,7 @@ class S3ObjectName : public S3Name { void print(std::ostream& out) const override; private: // members - std::string bucket_; - std::string object_; + S3ObjectPath path_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3ObjectPath.cc b/src/eckit/io/s3/S3ObjectPath.cc new file mode 100644 index 000000000..86fb6fe4d --- /dev/null +++ b/src/eckit/io/s3/S3ObjectPath.cc @@ -0,0 +1,32 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3ObjectPath.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path) { + out << "object=" << path.object << ", bucket=" << path.bucket; + return out; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit + diff --git a/src/eckit/io/s3/S3ObjectPath.h b/src/eckit/io/s3/S3ObjectPath.h new file mode 100644 index 000000000..d87a00d0e --- /dev/null +++ b/src/eckit/io/s3/S3ObjectPath.h @@ -0,0 +1,40 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3ObjectPath.h +/// @author Metin Cakircali +/// @date Dec 2024 + +#pragma once + +#include +#include + +namespace eckit { + +struct S3ObjectPath { + std::string bucket; + std::string object; + + auto asString() const -> std::string { return bucket + '/' + object; } + + operator std::string() const { return asString(); } + + friend std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path); +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index b1d4d9906..c12021424 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -19,6 +19,7 @@ #include "eckit/exception/Exceptions.h" #include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3ObjectPath.h" #include "eckit/io/s3/S3Session.h" #include "eckit/io/s3/aws/S3ContextAWS.h" #include "eckit/log/CodeLocation.h" @@ -199,21 +200,18 @@ auto S3ClientAWS::listBuckets() const -> std::vector { return buckets; } - auto msg = awsErrorMessage("Failed list buckets!", outcome.GetError()); + auto msg = awsErrorMessage("Failed to list buckets!", outcome.GetError()); throw S3SeriousBug(msg, Here()); } //---------------------------------------------------------------------------------------------------------------------- // PUT OBJECT -auto S3ClientAWS::putObject(const std::string& bucket, - const std::string& object, - const void* buffer, - const uint64_t length) const -> long long { +auto S3ClientAWS::putObject(const S3ObjectPath& path, const void* buffer, const uint64_t length) const -> long long { Aws::S3::Model::PutObjectRequest request; - request.SetBucket(bucket); - request.SetKey(object); + request.SetBucket(path.bucket); + request.SetKey(path.object); // request.SetContentLength(length); if (buffer && length > 0) { @@ -227,27 +225,26 @@ auto S3ClientAWS::putObject(const std::string& bucket, auto outcome = client().PutObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Put object=" << object << " [len=" << length << "] to bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Put " << path << " with len=" << length << '\n'; /// @todo actual size of written bytes return length; } - auto msg = awsErrorMessage("Failed to put object=" + object + " to bucket=" + bucket, outcome.GetError()); + auto msg = awsErrorMessage("Failed to put " + std::string(path), outcome.GetError()); throw S3SeriousBug(msg, Here()); } //---------------------------------------------------------------------------------------------------------------------- // GET OBJECT -auto S3ClientAWS::getObject(const std::string& bucket, - const std::string& object, - void* buffer, +auto S3ClientAWS::getObject(const S3ObjectPath& path, + void* buffer, const uint64_t /*offset*/, const uint64_t length) const -> long long { Aws::S3::Model::GetObjectRequest request; - request.SetBucket(bucket); - request.SetKey(object); + request.SetBucket(path.bucket); + request.SetKey(path.object); request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(allocTag, buffer, length); }); /// @todo range and streambuf // request.SetRange(std::to_string(offset) + "-" + std::to_string(offset + length)); @@ -255,33 +252,33 @@ auto S3ClientAWS::getObject(const std::string& bucket, auto outcome = client().GetObject(request); if (outcome.IsSuccess()) { - LOG_DEBUG_LIB(LibEcKit) << "Get object=" << object << " from bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Get " << path << '\n'; LOG_DEBUG_LIB(LibEcKit) << "Req. len=" << length << ", Obj. len=" << outcome.GetResult().GetContentLength() - << std::endl; + << '\n'; return outcome.GetResult().GetContentLength(); } - auto msg = awsErrorMessage("Failed to retrieve object=" + object + " from bucket=" + bucket, outcome.GetError()); + auto msg = awsErrorMessage("Failed to retrieve " + std::string(path), outcome.GetError()); throw S3SeriousBug(msg, Here()); } //---------------------------------------------------------------------------------------------------------------------- // DELETE OBJECT -void S3ClientAWS::deleteObject(const std::string& bucket, const std::string& object) const { +void S3ClientAWS::deleteObject(const S3ObjectPath& path) const { Aws::S3::Model::DeleteObjectRequest request; - request.SetBucket(bucket); - request.SetKey(object); + request.SetBucket(path.bucket); + request.SetKey(path.object); auto outcome = client().DeleteObject(request); if (!outcome.IsSuccess()) { - auto msg = awsErrorMessage("Failed to delete object=" + object + " in bucket=" + bucket, outcome.GetError()); + auto msg = awsErrorMessage("Failed to delete " + std::string(path), outcome.GetError()); throw S3SeriousBug(msg, Here()); } - LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << " in bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Deleted " << path << '\n'; } //---------------------------------------------------------------------------------------------------------------------- @@ -336,28 +333,28 @@ auto S3ClientAWS::listObjects(const std::string& bucket) const -> std::vector bool { +auto S3ClientAWS::objectExists(const S3ObjectPath& path) const -> bool { Aws::S3::Model::HeadObjectRequest request; - request.SetBucket(bucket); - request.SetKey(object); + request.SetBucket(path.bucket); + request.SetKey(path.object); return client().HeadObject(request).IsSuccess(); } //---------------------------------------------------------------------------------------------------------------------- -auto S3ClientAWS::objectSize(const std::string& bucket, const std::string& object) const -> long long { +auto S3ClientAWS::objectSize(const S3ObjectPath& path) const -> long long { Aws::S3::Model::HeadObjectRequest request; - request.SetBucket(bucket); - request.SetKey(object); + request.SetBucket(path.bucket); + request.SetKey(path.object); auto outcome = client().HeadObject(request); if (outcome.IsSuccess()) { return outcome.GetResult().GetContentLength(); } - const auto msg = awsErrorMessage("Object '" + object + "' doesn't exist or no access!", outcome.GetError()); + const auto msg = awsErrorMessage("Object '" + path.object + "' doesn't exist or no access!", outcome.GetError()); throw S3SeriousBug(msg, Here()); } diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h index 732a593d1..ec2c0eba2 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.h +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -20,6 +20,7 @@ #pragma once #include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3ObjectPath.h" #include @@ -39,6 +40,8 @@ class S3ClientAWS final : public S3Client { public: // methods explicit S3ClientAWS(const S3Config& config); + // bucket operations + void createBucket(const std::string& bucket) const override; void emptyBucket(const std::string& bucket) const override; @@ -49,26 +52,21 @@ class S3ClientAWS final : public S3Client { auto listBuckets() const -> std::vector override; - auto putObject(const std::string& bucket, - const std::string& object, - const void* buffer, - uint64_t length) const -> long long override; + auto listObjects(const std::string& bucket) const -> std::vector override; - auto getObject(const std::string& bucket, - const std::string& object, - void* buffer, - uint64_t offset, - uint64_t length) const -> long long override; + // object operations - void deleteObject(const std::string& bucket, const std::string& object) const override; + auto putObject(const S3ObjectPath& path, const void* buffer, uint64_t length) const -> long long override; - void deleteObjects(const std::string& bucket, const std::vector& objects) const override; + auto getObject(const S3ObjectPath& path, void* buffer, uint64_t offset, uint64_t length) const -> long long override; - auto listObjects(const std::string& bucket) const -> std::vector override; + void deleteObject(const S3ObjectPath& path) const override; + + void deleteObjects(const std::string& bucket, const std::vector& objects) const override; - auto objectExists(const std::string& bucket, const std::string& object) const -> bool override; + auto objectExists(const S3ObjectPath& path) const -> bool override; - auto objectSize(const std::string& bucket, const std::string& object) const -> long long override; + auto objectSize(const S3ObjectPath& path) const -> long long override; private: // methods void configure() const; From 701fe9e018f2608ffbb6975948efe377195534ca Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 20 Dec 2024 22:02:51 +0100 Subject: [PATCH 82/90] fix(S3): bucket name parse --- src/eckit/io/s3/S3BucketName.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc index 735012f5f..dd96d5fcb 100644 --- a/src/eckit/io/s3/S3BucketName.cc +++ b/src/eckit/io/s3/S3BucketName.cc @@ -37,7 +37,9 @@ namespace eckit { auto S3BucketName::parse(const std::string& name) -> std::string { const auto parsed = S3Name::parse(name); - if (parsed.size() != 1) { throw S3SeriousBug("Could not parse bucket from name: " + name, Here()); } + if (const auto size = parsed.size(); size < 1 || size > 2) { + throw S3SeriousBug("Could not parse bucket from name: " + name, Here()); + } return {parsed[0]}; } From 3c29ecfc700e1ac890e24165eedb867699b7cc1d Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 20 Dec 2024 22:04:00 +0100 Subject: [PATCH 83/90] feat(S3): added make to S3Config --- src/eckit/io/s3/S3Config.cc | 17 +++++++---------- src/eckit/io/s3/S3Config.h | 14 +++++++++++++- src/eckit/io/s3/S3Session.cc | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/eckit/io/s3/S3Config.cc b/src/eckit/io/s3/S3Config.cc index d857684f6..0fb40d6fd 100644 --- a/src/eckit/io/s3/S3Config.cc +++ b/src/eckit/io/s3/S3Config.cc @@ -19,7 +19,6 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/config/Resource.h" #include "eckit/config/YAMLConfiguration.h" -#include "eckit/container/DenseSet.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/log/CodeLocation.h" @@ -28,9 +27,7 @@ #include #include -#include #include -#include #include #include #include @@ -43,7 +40,11 @@ namespace { const std::string defaultConfigFile = "~/.config/eckit/S3Config.yaml"; -S3Config fromYAML(const LocalConfiguration& config) { +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +S3Config S3Config::make(const LocalConfiguration& config) { const net::Endpoint endpoint {config.getString("endpoint")}; const auto region = config.getString("region", s3DefaultRegion); @@ -70,11 +71,7 @@ S3Config fromYAML(const LocalConfiguration& config) { return s3config; } -} // namespace - -//---------------------------------------------------------------------------------------------------------------------- - -auto S3Config::fromFile(std::string path) -> std::vector { +auto S3Config::make(std::string path) -> std::vector { if (path.empty()) { path = Resource("s3ConfigFile;$ECKIT_S3_CONFIG_FILE", defaultConfigFile); } @@ -89,7 +86,7 @@ auto S3Config::fromFile(std::string path) -> std::vector { std::vector result; result.reserve(servers.size()); - for (const auto& server : servers) { result.emplace_back(fromYAML(server)); } + for (const auto& server : servers) { result.emplace_back(make(server)); } return result; } diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index dce4f0fc0..9a3fcf60e 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -28,6 +28,8 @@ namespace eckit { +class LocalConfiguration; + enum class S3Backend : std::uint8_t { AWS, REST, MINIO }; constexpr auto s3DefaultHost = "127.0.0.1"; @@ -60,7 +62,13 @@ constexpr auto s3DefaultRegion = "default"; /// struct S3Config { - static auto fromFile(std::string path) -> std::vector; + // static methods + + static auto make(const LocalConfiguration& config) -> S3Config; + + static auto make(std::string path) -> std::vector; + + // constructors S3Config() = default; @@ -70,6 +78,8 @@ struct S3Config { explicit S3Config(const URI& uri); + // operators + bool operator==(const S3Config& other) const { return backend == other.backend && endpoint == other.endpoint && region == other.region; } @@ -80,6 +90,8 @@ struct S3Config { friend std::ostream& operator<<(std::ostream& out, const S3Config& config); + // members + net::Endpoint endpoint {s3DefaultHost, s3DefaultPort}; std::string region {s3DefaultRegion}; S3Backend backend {S3Backend::AWS}; diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc index 9a899583f..6bf1c6387 100644 --- a/src/eckit/io/s3/S3Session.cc +++ b/src/eckit/io/s3/S3Session.cc @@ -76,7 +76,7 @@ void S3Session::clear() { // CLIENT void S3Session::loadClients(const std::string& path) { - const auto configs = S3Config::fromFile(path); + const auto configs = S3Config::make(path); for (const auto& config : configs) { addClient(config); } } From 042667c8ba0a9b1a06657899dfe784773cf80557 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Fri, 20 Dec 2024 23:58:08 +0100 Subject: [PATCH 84/90] fix(StringContent): compareString problem strcmp returns wrong result --- src/eckit/value/StringContent.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eckit/value/StringContent.cc b/src/eckit/value/StringContent.cc index a39496af4..6f79021ac 100644 --- a/src/eckit/value/StringContent.cc +++ b/src/eckit/value/StringContent.cc @@ -66,7 +66,7 @@ int StringContent::compare(const Content& other) const { } int StringContent::compareString(const StringContent& other) const { - return ::strcmp(value_.c_str(), other.value_.c_str()); + return value_.compare(other.value_); } void StringContent::value(std::string& s) const { From 9771afa679bdbd5ce67f377f2420902098956f5c Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 24 Dec 2024 12:40:24 +0100 Subject: [PATCH 85/90] feat(S3): made strongly typed --- CMakeLists.txt | 10 +++-- src/eckit/CMakeLists.txt | 21 ++++++---- src/eckit/io/s3/S3BucketName.cc | 31 ++++++++------- src/eckit/io/s3/S3BucketName.h | 29 +++++++++----- .../io/s3/{S3ObjectPath.cc => S3BucketPath.h} | 34 ++++++++++++---- src/eckit/io/s3/S3Client.h | 13 ++++--- src/eckit/io/s3/S3Config.cc | 28 ++++++------- src/eckit/io/s3/S3Config.h | 18 ++++----- src/eckit/io/s3/S3Handle.cc | 12 ++++-- src/eckit/io/s3/S3Handle.h | 14 ++++--- src/eckit/io/s3/S3Name.cc | 16 ++++---- src/eckit/io/s3/S3Name.h | 6 +-- src/eckit/io/s3/S3ObjectName.cc | 21 +++++----- src/eckit/io/s3/S3ObjectName.h | 36 ++++++++++------- src/eckit/io/s3/S3ObjectPath.h | 23 ++++++++--- src/eckit/io/s3/aws/S3ClientAWS.cc | 39 ++++++++++--------- src/eckit/io/s3/aws/S3ClientAWS.h | 13 ++++--- tests/io/s3/CMakeLists.txt | 8 ++-- tests/io/s3/test_s3client.cc | 27 ++++++------- 19 files changed, 230 insertions(+), 169 deletions(-) rename src/eckit/io/s3/{S3ObjectPath.cc => S3BucketPath.h} (50%) diff --git a/CMakeLists.txt b/CMakeLists.txt index e309f760b..7545b4e9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,10 +115,14 @@ ecbuild_add_option( FEATURE RADOS ### S3 Support -ecbuild_add_option( FEATURE AWS_S3 +ecbuild_add_option( FEATURE AWSSDK_S3 DEFAULT OFF - REQUIRED_PACKAGES "AWSSDK REQUIRED COMPONENTS s3" - DESCRIPTION "Enables AWS S3 support" ) + REQUIRED_PACKAGES "AWSSDK COMPONENTS s3 QUIET" + DESCRIPTION "Enables S3 storage support via AWS SDK C++ library" ) + +ecbuild_add_option( FEATURE S3 + CONDITION HAVE_AWSSDK_S3 + DESCRIPTION "Enables S3 storage support" ) ### Armadillo diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 02e7c78be..5d70231ab 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -283,14 +283,21 @@ if(HAVE_RADOS) ) endif() -if( eckit_HAVE_AWS_S3 ) +if( eckit_HAVE_S3 ) + + if( eckit_HAVE_AWSSDK_S3 ) + list( APPEND eckit_io_srcs + io/s3/aws/S3ClientAWS.cc + io/s3/aws/S3ClientAWS.h + io/s3/aws/S3ContextAWS.cc + io/s3/aws/S3ContextAWS.h + ) + endif( eckit_HAVE_AWSSDK_S3 ) + list( APPEND eckit_io_srcs - io/s3/aws/S3ClientAWS.cc - io/s3/aws/S3ClientAWS.h - io/s3/aws/S3ContextAWS.cc - io/s3/aws/S3ContextAWS.h io/s3/S3BucketName.cc io/s3/S3BucketName.h + io/s3/S3BucketPath.h io/s3/S3Client.cc io/s3/S3Client.h io/s3/S3Config.cc @@ -305,14 +312,14 @@ if( eckit_HAVE_AWS_S3 ) io/s3/S3Name.h io/s3/S3ObjectName.cc io/s3/S3ObjectName.h - io/s3/S3ObjectPath.cc io/s3/S3ObjectPath.h io/s3/S3Session.cc io/s3/S3Session.h io/s3/S3URIManager.cc io/s3/S3URIManager.h ) -endif( eckit_HAVE_AWS_S3 ) + +endif( eckit_HAVE_S3 ) list( APPEND eckit_filesystem_srcs filesystem/BasePathName.cc diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc index dd96d5fcb..1ef0f54a8 100644 --- a/src/eckit/io/s3/S3BucketName.cc +++ b/src/eckit/io/s3/S3BucketName.cc @@ -17,11 +17,13 @@ #include "eckit/config/LibEcKit.h" #include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3BucketPath.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Name.h" #include "eckit/io/s3/S3ObjectName.h" #include "eckit/io/s3/S3ObjectPath.h" +#include "eckit/log/CodeLocation.h" #include "eckit/log/Log.h" #include "eckit/net/Endpoint.h" @@ -35,7 +37,7 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -auto S3BucketName::parse(const std::string& name) -> std::string { +auto S3BucketName::parse(const std::string& name) -> S3BucketPath { const auto parsed = S3Name::parse(name); if (const auto size = parsed.size(); size < 1 || size > 2) { throw S3SeriousBug("Could not parse bucket from name: " + name, Here()); @@ -45,44 +47,45 @@ auto S3BucketName::parse(const std::string& name) -> std::string { //---------------------------------------------------------------------------------------------------------------------- -S3BucketName::S3BucketName(const net::Endpoint& endpoint, std::string bucket) - : S3Name(endpoint), bucket_ {std::move(bucket)} { } +S3BucketName::S3BucketName(const net::Endpoint& endpoint, S3BucketPath path) + : S3Name(endpoint), path_ {std::move(path)} { } -S3BucketName::S3BucketName(const URI& uri) : S3Name(uri), bucket_ {parse(uri.name())} { } +S3BucketName::S3BucketName(const URI& uri) : S3Name(uri), path_ {parse(uri.name())} { } //---------------------------------------------------------------------------------------------------------------------- auto S3BucketName::uri() const -> URI { auto uri = S3Name::uri(); - uri.path("/" + bucket_); + uri.path(path_); return uri; } auto S3BucketName::asString() const -> std::string { - return S3Name::asString() + "/" + bucket_; + return S3Name::asString() + '/' + path_.asString(); } void S3BucketName::print(std::ostream& out) const { - out << "S3BucketName[bucket=" << bucket_; + out << "S3BucketName["; S3Name::print(out); + out << ',' << path_ << ']'; } //---------------------------------------------------------------------------------------------------------------------- auto S3BucketName::makeObject(const std::string& object) const -> std::unique_ptr { - return std::make_unique(endpoint(), S3ObjectPath {bucket_, object}); + return std::make_unique(endpoint(), S3ObjectPath {path_, object}); } auto S3BucketName::exists() const -> bool { - return client().bucketExists(bucket_); + return client().bucketExists(path_); } void S3BucketName::create() { - client().createBucket(bucket_); + client().createBucket(path_); } void S3BucketName::destroy() { - client().deleteBucket(bucket_); + client().deleteBucket(path_); } void S3BucketName::ensureCreated() { @@ -93,13 +96,13 @@ void S3BucketName::ensureCreated() { void S3BucketName::ensureDestroyed() { try { - client().emptyBucket(bucket_); - client().deleteBucket(bucket_); + client().emptyBucket(path_); + client().deleteBucket(path_); } catch (S3EntityNotFound& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } } auto S3BucketName::listObjects() const -> std::vector { - return client().listObjects(bucket_); + return client().listObjects(path_); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h index ab5f2a102..79686bf45 100644 --- a/src/eckit/io/s3/S3BucketName.h +++ b/src/eckit/io/s3/S3BucketName.h @@ -20,6 +20,7 @@ #pragma once +#include "eckit/io/s3/S3BucketPath.h" #include "eckit/io/s3/S3Name.h" #include "eckit/net/Endpoint.h" @@ -35,20 +36,33 @@ class S3ObjectName; //---------------------------------------------------------------------------------------------------------------------- class S3BucketName : public S3Name { -public: // methods - static auto parse(const std::string& name) -> std::string; +public: // factory + static auto parse(const std::string& name) -> S3BucketPath; + + auto makeObject(const std::string& object) const -> std::unique_ptr; public: // methods + S3BucketName(const net::Endpoint& endpoint, S3BucketPath path); + explicit S3BucketName(const URI& uri); - S3BucketName(const net::Endpoint& endpoint, std::string bucket); + // accessors - auto makeObject(const std::string& object) const -> std::unique_ptr; + auto asString() const -> std::string override; auto uri() const -> URI override; auto exists() const -> bool override; + auto path() const -> const S3BucketPath& { return path_; } + + auto bucket() const -> const std::string& { return path_.bucket; } + + /// @todo return S3 object iterator but first add prefix + auto listObjects() const -> std::vector; + + // modifiers + void create(); void destroy(); @@ -57,16 +71,11 @@ class S3BucketName : public S3Name { void ensureDestroyed(); - /// @todo return S3 object iterator but first add prefix - auto listObjects() const -> std::vector; - - auto asString() const -> std::string override; - private: // methods void print(std::ostream& out) const override; private: // members - std::string bucket_; + S3BucketPath path_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3ObjectPath.cc b/src/eckit/io/s3/S3BucketPath.h similarity index 50% rename from src/eckit/io/s3/S3ObjectPath.cc rename to src/eckit/io/s3/S3BucketPath.h index 86fb6fe4d..3d3cf9507 100644 --- a/src/eckit/io/s3/S3ObjectPath.cc +++ b/src/eckit/io/s3/S3BucketPath.h @@ -9,24 +9,42 @@ */ /* - * This software was developed as part of the EC H2020 funded project IO-SEA - * (Project ID: 955811) iosea-project.eu + * This software was developed as part of the Horizon Europe programme funded project DaFab + * (Grant agreement: 101128693) https://www.dafab-ai.eu/ */ -#include "eckit/io/s3/S3ObjectPath.h" +/// @file S3BucketPath.h +/// @author Metin Cakircali +/// @date Dec 2024 + +#pragma once #include +#include +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path) { - out << "object=" << path.object << ", bucket=" << path.bucket; - return out; -} +struct S3BucketPath { + + S3BucketPath(std::string bucket) : bucket {std::move(bucket)} { } + + auto asString() const -> std::string { return '/' + bucket; } + + operator std::string() const { return asString(); } + + friend std::ostream& operator<<(std::ostream& out, const S3BucketPath& path) { + out << "bucket=" << path.bucket; + return out; + } + + // members + + std::string bucket; +}; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit - diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h index aea094f45..338d345b0 100644 --- a/src/eckit/io/s3/S3Client.h +++ b/src/eckit/io/s3/S3Client.h @@ -19,6 +19,7 @@ #pragma once +#include "eckit/io/s3/S3BucketPath.h" #include "eckit/io/s3/S3Config.h" #include "eckit/io/s3/S3ObjectPath.h" @@ -50,13 +51,13 @@ class S3Client { // bucket operations - virtual void createBucket(const std::string& bucket) const = 0; + virtual void createBucket(const S3BucketPath& path) const = 0; - virtual void emptyBucket(const std::string& bucket) const = 0; + virtual void emptyBucket(const S3BucketPath& path) const = 0; - virtual void deleteBucket(const std::string& bucket) const = 0; + virtual void deleteBucket(const S3BucketPath& path) const = 0; - virtual auto bucketExists(const std::string& bucket) const -> bool = 0; + virtual auto bucketExists(const S3BucketPath& path) const -> bool = 0; virtual auto listBuckets() const -> std::vector = 0; @@ -68,9 +69,9 @@ class S3Client { virtual void deleteObject(const S3ObjectPath& path) const = 0; - virtual void deleteObjects(const std::string& bucket, const std::vector& objects) const = 0; + virtual void deleteObjects(const S3BucketPath& path, const std::vector& objects) const = 0; - virtual auto listObjects(const std::string& bucket) const -> std::vector = 0; + virtual auto listObjects(const S3BucketPath& path) const -> std::vector = 0; virtual auto objectExists(const S3ObjectPath& path) const -> bool = 0; diff --git a/src/eckit/io/s3/S3Config.cc b/src/eckit/io/s3/S3Config.cc index 0fb40d6fd..79b13fe41 100644 --- a/src/eckit/io/s3/S3Config.cc +++ b/src/eckit/io/s3/S3Config.cc @@ -36,34 +36,26 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -namespace { +auto S3Config::make(const LocalConfiguration& config) -> S3Config { + const auto endpoint = config.getString("endpoint"); -const std::string defaultConfigFile = "~/.config/eckit/S3Config.yaml"; + if (endpoint.empty()) { throw eckit::UserError("No S3 endpoint found in configuration!", Here()); } -} // namespace - -//---------------------------------------------------------------------------------------------------------------------- - -S3Config S3Config::make(const LocalConfiguration& config) { - const net::Endpoint endpoint {config.getString("endpoint")}; - - const auto region = config.getString("region", s3DefaultRegion); + const auto region = config.getString("region", defaultRegion); S3Config s3config(endpoint, region); if (config.has("backend")) { - S3Backend backend {S3Backend::AWS}; // Default to AWS - std::string backendStr = config.getString("backend"); + const auto backendStr = config.getString("backend"); if (backendStr == "AWS") { - backend = S3Backend::AWS; + s3config.backend = S3Backend::AWS; } else if (backendStr == "REST") { - backend = S3Backend::REST; + s3config.backend = S3Backend::REST; } else if (backendStr == "MinIO") { - backend = S3Backend::MINIO; + s3config.backend = S3Backend::MINIO; } else { throw UserError("Invalid backend: " + backendStr, Here()); } - s3config.backend = backend; } if (config.has("secure")) { s3config.secure = config.getBool("secure"); } @@ -73,7 +65,9 @@ S3Config S3Config::make(const LocalConfiguration& config) { auto S3Config::make(std::string path) -> std::vector { - if (path.empty()) { path = Resource("s3ConfigFile;$ECKIT_S3_CONFIG_FILE", defaultConfigFile); } + if (path.empty()) { + path = Resource("s3ConfigFile;$ECKIT_S3_CONFIG_FILE", "~/.config/eckit/S3Config.yaml"); + } PathName configFile(path); diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h index 9a3fcf60e..3a9a69824 100644 --- a/src/eckit/io/s3/S3Config.h +++ b/src/eckit/io/s3/S3Config.h @@ -32,10 +32,6 @@ class LocalConfiguration; enum class S3Backend : std::uint8_t { AWS, REST, MINIO }; -constexpr auto s3DefaultHost = "127.0.0.1"; -constexpr uint16_t s3DefaultPort = 443; -constexpr auto s3DefaultRegion = "default"; - //---------------------------------------------------------------------------------------------------------------------- /// @brief S3 configurations for a given endpoint @@ -61,6 +57,8 @@ constexpr auto s3DefaultRegion = "default"; /// - endpoint: "https://eu-central-1.ecmwf.int:9000" /// struct S3Config { + static constexpr auto defaultRegion = "default"; + static constexpr auto defaultBackend = S3Backend::AWS; // static methods @@ -70,11 +68,9 @@ struct S3Config { // constructors - S3Config() = default; - - explicit S3Config(const net::Endpoint& endpoint, std::string region = s3DefaultRegion); + explicit S3Config(const net::Endpoint& endpoint, std::string region = defaultRegion); - explicit S3Config(const std::string& host, uint16_t port, std::string region = s3DefaultRegion); + explicit S3Config(const std::string& host, uint16_t port, std::string region = defaultRegion); explicit S3Config(const URI& uri); @@ -92,9 +88,9 @@ struct S3Config { // members - net::Endpoint endpoint {s3DefaultHost, s3DefaultPort}; - std::string region {s3DefaultRegion}; - S3Backend backend {S3Backend::AWS}; + net::Endpoint endpoint; + std::string region {defaultRegion}; + S3Backend backend {defaultBackend}; bool secure {true}; }; diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc index 072cc8400..5416af8ce 100644 --- a/src/eckit/io/s3/S3Handle.cc +++ b/src/eckit/io/s3/S3Handle.cc @@ -16,15 +16,21 @@ #include "eckit/io/s3/S3Handle.h" #include "eckit/config/LibEcKit.h" -#include "eckit/io/s3/S3Exception.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/log/Log.h" + +#include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -S3Handle::S3Handle(const S3ObjectName& name): S3Handle(name, 0) { } +S3Handle::S3Handle(const S3ObjectName& name, const Offset& offset) : name_(name), pos_(offset) { } -S3Handle::S3Handle(const S3ObjectName& name, const Offset& offset): name_(name), pos_(offset) { } +S3Handle::S3Handle(const S3ObjectName& name) : S3Handle(name, 0) { } void S3Handle::print(std::ostream& out) const { out << "S3Handle[name=" << name_ << ", position=" << pos_; diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h index ec6ceadd5..30cb65d6d 100644 --- a/src/eckit/io/s3/S3Handle.h +++ b/src/eckit/io/s3/S3Handle.h @@ -20,20 +20,24 @@ #pragma once #include "eckit/io/DataHandle.h" +#include "eckit/io/Length.h" #include "eckit/io/s3/S3ObjectName.h" +#include +#include + namespace eckit { //---------------------------------------------------------------------------------------------------------------------- -class S3Handle: public DataHandle { +class S3Handle : public DataHandle { public: // methods - enum class Mode { CLOSED, READ, WRITE }; - - explicit S3Handle(const S3ObjectName& name); + enum class Mode : std::uint8_t { CLOSED, READ, WRITE }; S3Handle(const S3ObjectName& name, const Offset& offset); + explicit S3Handle(const S3ObjectName& name); + Length openForRead() override; void openForWrite(const Length& length) override; @@ -62,7 +66,7 @@ class S3Handle: public DataHandle { void open(Mode mode); private: // members - const S3ObjectName name_; + S3ObjectName name_; Offset pos_ {0}; diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc index e1897fa23..96c74fe73 100644 --- a/src/eckit/io/s3/S3Name.cc +++ b/src/eckit/io/s3/S3Name.cc @@ -18,10 +18,12 @@ #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" #include "eckit/io/s3/S3BucketName.h" +#include "eckit/io/s3/S3BucketPath.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3ObjectName.h" #include "eckit/io/s3/S3ObjectPath.h" #include "eckit/io/s3/S3Session.h" +#include "eckit/log/CodeLocation.h" #include "eckit/utils/Tokenizer.h" #include @@ -40,7 +42,7 @@ auto S3Name::parse(const std::string& name) -> std::vector { auto S3Name::make(const net::Endpoint& endpoint, const std::string& name) -> std::unique_ptr { const auto names = parse(name); switch (names.size()) { - case 1: return std::make_unique(endpoint, names[0]); + case 1: return std::make_unique(endpoint, S3BucketPath {names[0]}); case 2: return std::make_unique(endpoint, S3ObjectPath {names[0], names[1]}); default: throw SeriousBug("Could not parse S3 name: " + name, Here()); } @@ -59,6 +61,10 @@ S3Name::~S3Name() = default; //---------------------------------------------------------------------------------------------------------------------- +auto S3Name::client() const -> S3Client& { + return *S3Session::instance().getClient(endpoint_); +} + auto S3Name::uri() const -> URI { return {type, endpoint_.host(), endpoint_.port()}; } @@ -68,13 +74,7 @@ auto S3Name::asString() const -> std::string { } void S3Name::print(std::ostream& out) const { - out << ",endpoint=" << endpoint_ << ",]"; -} - -//---------------------------------------------------------------------------------------------------------------------- - -auto S3Name::client() const -> S3Client& { - return *S3Session::instance().getClient(endpoint_); + out << "endpoint=" << endpoint_; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h index 5c4f7d440..dbd2357fc 100644 --- a/src/eckit/io/s3/S3Name.h +++ b/src/eckit/io/s3/S3Name.h @@ -58,16 +58,14 @@ class S3Name { // accessors - void endpoint(const net::Endpoint& endpoint) { endpoint_ = endpoint; } - - auto endpoint() const -> const net::Endpoint& { return endpoint_; } - virtual auto uri() const -> URI; virtual auto exists() const -> bool = 0; virtual auto asString() const -> std::string; + auto endpoint() const -> const net::Endpoint& { return endpoint_; } + protected: // methods virtual void print(std::ostream& out) const; diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc index 73fd335e4..507084bde 100644 --- a/src/eckit/io/s3/S3ObjectName.cc +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -47,13 +47,6 @@ S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri), path_ {parse(uri.name( //---------------------------------------------------------------------------------------------------------------------- -void S3ObjectName::print(std::ostream& out) const { - out << "S3ObjectName[path=" << path_; - S3Name::print(out); -} - -//---------------------------------------------------------------------------------------------------------------------- - auto S3ObjectName::uri() const -> URI { auto uri = S3Name::uri(); uri.path(path_); @@ -61,9 +54,17 @@ auto S3ObjectName::uri() const -> URI { } auto S3ObjectName::asString() const -> std::string { - return S3Name::asString() + '/' + path_.asString(); + return S3Name::asString() + path_.asString(); } +void S3ObjectName::print(std::ostream& out) const { + out << "S3ObjectName["; + S3Name::print(out); + out << ',' << path_ << ']'; +} + +//---------------------------------------------------------------------------------------------------------------------- + auto S3ObjectName::size() const -> long long { return client().objectSize(path_); } @@ -88,11 +89,11 @@ auto S3ObjectName::get(void* buffer, const long offset, const long length) const return client().getObject(path_, buffer, offset, length); } -auto S3ObjectName::dataHandle() -> DataHandle* { +auto S3ObjectName::dataHandle() const -> DataHandle* { return new S3Handle(*this); } -auto S3ObjectName::dataHandle(const Offset& offset) -> DataHandle* { +auto S3ObjectName::dataHandle(const Offset& offset) const -> DataHandle* { return new S3Handle(*this, offset); } diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h index 8508b7574..083cbc442 100644 --- a/src/eckit/io/s3/S3ObjectName.h +++ b/src/eckit/io/s3/S3ObjectName.h @@ -22,7 +22,7 @@ #include "eckit/io/s3/S3Name.h" #include "eckit/io/s3/S3ObjectPath.h" -#include +#include #include namespace eckit { @@ -33,7 +33,7 @@ class DataHandle; //---------------------------------------------------------------------------------------------------------------------- class S3ObjectName : public S3Name { -public: // helpers +public: // factory static auto parse(const std::string& name) -> S3ObjectPath; public: // methods @@ -41,33 +41,39 @@ class S3ObjectName : public S3Name { explicit S3ObjectName(const URI& uri); + // accessors + auto uri() const -> URI override; + auto exists() const -> bool override; + + auto asString() const -> std::string override; + auto size() const -> long long; - auto exists() const -> bool override; + auto path() const -> const S3ObjectPath& { return path_; } - void remove(); + auto name() const -> const std::string& { return path_.object; } - auto put(const void* buffer, long length) const -> long long; + auto bucket() const -> const std::string& { return path_.bucket; } - auto get(void* buffer, long offset, long length) const -> long long; + auto bucketExists() const -> bool; - [[nodiscard]] - auto dataHandle() -> DataHandle*; + // modifiers - [[nodiscard]] - auto dataHandle(const Offset& offset) -> DataHandle*; + void remove(); - auto asString() const -> std::string override; + // I/O - auto path() const -> const S3ObjectPath& { return path_; } + auto put(const void* buffer, long length) const -> long long; - auto name() const -> const std::string& { return path_.object; } + auto get(void* buffer, long offset, long length) const -> long long; - auto bucket() const -> const std::string& { return path_.bucket; } + [[nodiscard]] + auto dataHandle() const -> DataHandle*; - auto bucketExists() const -> bool; + [[nodiscard]] + auto dataHandle(const Offset& offset) const -> DataHandle*; private: // methods void print(std::ostream& out) const override; diff --git a/src/eckit/io/s3/S3ObjectPath.h b/src/eckit/io/s3/S3ObjectPath.h index d87a00d0e..478d26706 100644 --- a/src/eckit/io/s3/S3ObjectPath.h +++ b/src/eckit/io/s3/S3ObjectPath.h @@ -9,8 +9,8 @@ */ /* - * This software was developed as part of the EC H2020 funded project IO-SEA - * (Project ID: 955811) iosea-project.eu + * This software was developed as part of the Horizon Europe programme funded project DaFab + * (Grant agreement: 101128693) https://www.dafab-ai.eu/ */ /// @file S3ObjectPath.h @@ -21,18 +21,29 @@ #include #include +#include namespace eckit { +//---------------------------------------------------------------------------------------------------------------------- + struct S3ObjectPath { - std::string bucket; - std::string object; - auto asString() const -> std::string { return bucket + '/' + object; } + S3ObjectPath(std::string bucket, std::string object) : bucket {std::move(bucket)}, object {std::move(object)} { } + + auto asString() const -> std::string { return '/' + bucket + '/' + object; } operator std::string() const { return asString(); } - friend std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path); + friend std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path) { + out << "bucket=" << path.bucket << ",object=" << path.object; + return out; + } + + // members + + std::string bucket; + std::string object; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc index c12021424..5ccfc943b 100644 --- a/src/eckit/io/s3/aws/S3ClientAWS.cc +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -17,6 +17,7 @@ #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" +#include "eckit/io/s3/S3BucketPath.h" #include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3ObjectPath.h" @@ -133,15 +134,15 @@ auto S3ClientAWS::client() const -> Aws::S3::S3Client& { //---------------------------------------------------------------------------------------------------------------------- // BUCKET -void S3ClientAWS::createBucket(const std::string& bucket) const { +void S3ClientAWS::createBucket(const S3BucketPath& path) const { Aws::S3::Model::CreateBucketRequest request; - request.SetBucket(bucket); + request.SetBucket(path); auto outcome = client().CreateBucket(request); if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); - const auto msg = awsErrorMessage("Failed to create bucket=" + bucket, err); + const auto msg = awsErrorMessage("Failed to create bucket=" + path.asString(), err); const auto eType = err.GetErrorType(); if (eType == Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS || eType == Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) { throw S3EntityAlreadyExists(msg, Here()); @@ -151,24 +152,24 @@ void S3ClientAWS::createBucket(const std::string& bucket) const { /// @todo do we wait for the bucket to propagate? - LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << path << std::endl; } -void S3ClientAWS::emptyBucket(const std::string& bucket) const { - deleteObjects(bucket, listObjects(bucket)); +void S3ClientAWS::emptyBucket(const S3BucketPath& path) const { + deleteObjects(path, listObjects(path)); } //---------------------------------------------------------------------------------------------------------------------- -void S3ClientAWS::deleteBucket(const std::string& bucket) const { +void S3ClientAWS::deleteBucket(const S3BucketPath& path) const { Aws::S3::Model::DeleteBucketRequest request; - request.SetBucket(bucket); + request.SetBucket(path); auto outcome = client().DeleteBucket(request); if (!outcome.IsSuccess()) { const auto& err = outcome.GetError(); - const auto msg = awsErrorMessage("Failed to delete bucket=" + bucket, err); + const auto msg = awsErrorMessage("Failed to delete bucket=" + path.asString(), err); if (err.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_BUCKET) { throw S3EntityNotFound(msg, Here()); } if (err.GetErrorType() == Aws::S3::S3Errors::UNKNOWN && err.GetExceptionName() == "BucketNotEmpty") { throw S3BucketNotEmpty(msg, Here()); @@ -176,15 +177,15 @@ void S3ClientAWS::deleteBucket(const std::string& bucket) const { throw S3SeriousBug(msg, Here()); } - LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << bucket << std::endl; + LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << path << std::endl; } //---------------------------------------------------------------------------------------------------------------------- -auto S3ClientAWS::bucketExists(const std::string& bucket) const -> bool { +auto S3ClientAWS::bucketExists(const S3BucketPath& path) const -> bool { Aws::S3::Model::HeadBucketRequest request; - request.SetBucket(bucket); + request.SetBucket(path); return client().HeadBucket(request).IsSuccess(); } @@ -283,12 +284,12 @@ void S3ClientAWS::deleteObject(const S3ObjectPath& path) const { //---------------------------------------------------------------------------------------------------------------------- -void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vector& objects) const { +void S3ClientAWS::deleteObjects(const S3BucketPath& path, const std::vector& objects) const { if (objects.empty()) { return; } Aws::S3::Model::DeleteObjectsRequest request; - request.SetBucket(bucket); + request.SetBucket(path); Aws::S3::Model::Delete deleteObject; for (const auto& object : objects) { deleteObject.AddObjects(Aws::S3::Model::ObjectIdentifier().WithKey(object)); } @@ -299,26 +300,26 @@ void S3ClientAWS::deleteObjects(const std::string& bucket, const std::vector bool override; + auto bucketExists(const S3BucketPath& path) const -> bool override; auto listBuckets() const -> std::vector override; - auto listObjects(const std::string& bucket) const -> std::vector override; + auto listObjects(const S3BucketPath& path) const -> std::vector override; // object operations @@ -62,7 +63,7 @@ class S3ClientAWS final : public S3Client { void deleteObject(const S3ObjectPath& path) const override; - void deleteObjects(const std::string& bucket, const std::vector& objects) const override; + void deleteObjects(const S3BucketPath& path, const std::vector& objects) const override; auto objectExists(const S3ObjectPath& path) const -> bool override; diff --git a/tests/io/s3/CMakeLists.txt b/tests/io/s3/CMakeLists.txt index 4adca6508..d5f1aa9c4 100644 --- a/tests/io/s3/CMakeLists.txt +++ b/tests/io/s3/CMakeLists.txt @@ -1,9 +1,9 @@ -if( eckit_HAVE_AWS_S3 ) +if( eckit_HAVE_S3 ) file( - COPY S3Config.yaml S3Credentials.yaml - DESTINATION ${CMAKE_CURRENT_BINARY_DIR} + COPY S3Config.yaml S3Credentials.yaml + DESTINATION ${CMAKE_CURRENT_BINARY_DIR} ) ecbuild_add_test( @@ -20,4 +20,4 @@ if( eckit_HAVE_AWS_S3 ) LIBS eckit ${AWSSDK_LINK_LIBRARIES} ) -endif( eckit_HAVE_AWS_S3 ) +endif( eckit_HAVE_S3 ) diff --git a/tests/io/s3/test_s3client.cc b/tests/io/s3/test_s3client.cc index 09be236ff..038adc871 100644 --- a/tests/io/s3/test_s3client.cc +++ b/tests/io/s3/test_s3client.cc @@ -13,6 +13,7 @@ /// @author Simon Smart /// @date Jan 2024 +#include "eckit/io/s3/S3BucketPath.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Config.h" #include "eckit/io/s3/S3Credential.h" @@ -46,7 +47,7 @@ bool findString(const std::vector& list, const std::string& item) { void cleanup() { auto client = S3Client::makeUnique(TEST_CONFIG); - for (const auto* name : {"test-bucket-1", "test-bucket-2"}) { + for (const auto& name : {S3BucketPath {"test-bucket-1"}, S3BucketPath {{"test-bucket-2"}}}) { if (client->bucketExists(name)) { client->emptyBucket(name); client->deleteBucket(name); @@ -157,7 +158,7 @@ CASE("create s3 bucket in non-existing region") { auto cfgTmp = TEST_CONFIG; cfgTmp.region = "non-existing-region-random"; - EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("test-bucket-1")); + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket({"test-bucket-1"})); } //---------------------------------------------------------------------------------------------------------------------- @@ -167,11 +168,11 @@ CASE("create s3 bucket") { auto client = S3Client::makeUnique(TEST_CONFIG); - EXPECT_NO_THROW(client->createBucket("test-bucket-1")); + EXPECT_NO_THROW(client->createBucket({"test-bucket-1"})); - EXPECT_THROWS(client->createBucket("test-bucket-1")); + EXPECT_THROWS(client->createBucket({"test-bucket-1"})); - EXPECT_NO_THROW(client->createBucket("test-bucket-2")); + EXPECT_NO_THROW(client->createBucket({"test-bucket-2"})); } //---------------------------------------------------------------------------------------------------------------------- @@ -180,27 +181,27 @@ CASE("list s3 buckets") { EXPECT_NO_THROW(cleanup()); auto client = S3Client::makeUnique(TEST_CONFIG); - EXPECT_NO_THROW(client->createBucket("test-bucket-1")); - EXPECT_NO_THROW(client->createBucket("test-bucket-2")); + EXPECT_NO_THROW(client->createBucket({"test-bucket-1"})); + EXPECT_NO_THROW(client->createBucket({"test-bucket-2"})); { const auto buckets = client->listBuckets(); EXPECT(findString(buckets, "test-bucket-1")); - EXPECT(findString(buckets, "test-bucket-2")); + EXPECT(findString(buckets, {"test-bucket-2"})); - EXPECT_NO_THROW(client->deleteBucket("test-bucket-1")); - EXPECT_NO_THROW(client->deleteBucket("test-bucket-2")); + EXPECT_NO_THROW(client->deleteBucket({"test-bucket-1"})); + EXPECT_NO_THROW(client->deleteBucket({"test-bucket-2"})); } { const auto buckets = client->listBuckets(); EXPECT_NOT(findString(buckets, "test-bucket-1")); - EXPECT_NOT(findString(buckets, "test-bucket-2")); + EXPECT_NOT(findString(buckets, {"test-bucket-2"})); } - EXPECT_THROWS(client->deleteBucket("test-bucket-1")); - EXPECT_THROWS(client->deleteBucket("test-bucket-2")); + EXPECT_THROWS(client->deleteBucket({"test-bucket-1"})); + EXPECT_THROWS(client->deleteBucket({"test-bucket-2"})); } //---------------------------------------------------------------------------------------------------------------------- From 2cc56af091364c865becb6c42253c9ea8fe65f65 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 25 Dec 2024 10:52:34 +0100 Subject: [PATCH 86/90] test(S3): improved --- src/eckit/io/s3/S3Config.cc | 2 + src/eckit/io/s3/S3Credential.cc | 2 + tests/io/s3/CMakeLists.txt | 42 +++++++++++-------- .../io/s3/{S3Config.yaml => S3Config.yaml.in} | 12 +++--- tests/io/s3/S3Credentials.yaml | 13 ------ tests/io/s3/S3Credentials.yaml.in | 13 ++++++ .../{test_s3client.cc => test_s3_client.cc} | 22 +++++----- tests/io/s3/test_s3_config.h | 17 ++++++++ tests/io/s3/test_s3_config.h.in | 17 ++++++++ .../{test_s3handle.cc => test_s3_handle.cc} | 15 ++++--- 10 files changed, 101 insertions(+), 54 deletions(-) rename tests/io/s3/{S3Config.yaml => S3Config.yaml.in} (68%) delete mode 100644 tests/io/s3/S3Credentials.yaml create mode 100644 tests/io/s3/S3Credentials.yaml.in rename tests/io/s3/{test_s3client.cc => test_s3_client.cc} (90%) create mode 100644 tests/io/s3/test_s3_config.h create mode 100644 tests/io/s3/test_s3_config.h.in rename tests/io/s3/{test_s3handle.cc => test_s3_handle.cc} (96%) diff --git a/src/eckit/io/s3/S3Config.cc b/src/eckit/io/s3/S3Config.cc index 79b13fe41..8cb000bc2 100644 --- a/src/eckit/io/s3/S3Config.cc +++ b/src/eckit/io/s3/S3Config.cc @@ -76,6 +76,8 @@ auto S3Config::make(std::string path) -> std::vector { return {}; } + LOG_DEBUG_LIB(LibEcKit) << "Reading S3 configuration from: " << configFile << std::endl; + const auto servers = YAMLConfiguration(configFile).getSubConfigurations("servers"); std::vector result; diff --git a/src/eckit/io/s3/S3Credential.cc b/src/eckit/io/s3/S3Credential.cc index cf33ac377..4465f8265 100644 --- a/src/eckit/io/s3/S3Credential.cc +++ b/src/eckit/io/s3/S3Credential.cc @@ -58,6 +58,8 @@ auto S3Credential::fromFile(std::string path) -> std::vector { return {}; } + LOG_DEBUG_LIB(LibEcKit) << "Reading S3 credentials from: " << credFile << std::endl; + std::vector result; const auto creds = YAMLConfiguration(credFile).getSubConfigurations("credentials"); diff --git a/tests/io/s3/CMakeLists.txt b/tests/io/s3/CMakeLists.txt index d5f1aa9c4..8ade180f1 100644 --- a/tests/io/s3/CMakeLists.txt +++ b/tests/io/s3/CMakeLists.txt @@ -1,23 +1,29 @@ if( eckit_HAVE_S3 ) - file( - COPY S3Config.yaml S3Credentials.yaml - DESTINATION ${CMAKE_CURRENT_BINARY_DIR} - ) - - ecbuild_add_test( - TARGET eckit_test_s3client - SOURCES test_s3client.cc - INCLUDES ${AWSSDK_INCLUDE_DIRS} - LIBS eckit ${AWSSDK_LINK_LIBRARIES} - ) - - ecbuild_add_test( - TARGET eckit_test_s3handle - SOURCES test_s3handle.cc - INCLUDES ${AWSSDK_INCLUDE_DIRS} - LIBS eckit ${AWSSDK_LINK_LIBRARIES} - ) + set( S3_TEST_HOST "minio" ) + set( S3_TEST_PORT "9000" ) + set( S3_TEST_ENDPOINT ${S3_TEST_HOST}:${S3_TEST_PORT} ) + set( S3_TEST_REGION "eu-central-1" ) + set( S3_TEST_BUCKET "eckit-test-bucket" ) + + configure_file( test_s3_config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/test_s3_config.h @ONLY ) + + configure_file( S3Config.yaml.in S3Config.yaml @ONLY ) + configure_file( S3Credentials.yaml.in S3Credentials.yaml @ONLY ) + + set( test_srcs client handle ) + + foreach( _test ${test_srcs} ) + + ecbuild_add_test( + TARGET eckit_test_s3_${_test} + SOURCES test_s3_${_test}.cc + INCLUDES test_s3_config.h ${AWSSDK_INCLUDE_DIRS} + LIBS eckit ${AWSSDK_LINK_LIBRARIES} + ENVIRONMENT "${_test_environment}" + ) + + endforeach() endif( eckit_HAVE_S3 ) diff --git a/tests/io/s3/S3Config.yaml b/tests/io/s3/S3Config.yaml.in similarity index 68% rename from tests/io/s3/S3Config.yaml rename to tests/io/s3/S3Config.yaml.in index fd2026ec9..217dfb2d9 100644 --- a/tests/io/s3/S3Config.yaml +++ b/tests/io/s3/S3Config.yaml.in @@ -1,15 +1,17 @@ --- servers: + - endpoint: @S3_TEST_ENDPOINT@ + region: @S3_TEST_REGION@ + - endpoint: "127.0.0.1:9000" region: "default" # (default) - secure: true # (default) - backend: "AWS" # (default) - - - endpoint: "minio:9000" - region: "eu-central-1" + secure: true # (default) + backend: "AWS" # (default) - endpoint: "localhost:9000" region: "eu-central-1" + secure: true # (default) + backend: "AWS" # (default) # region is inferred from the endpoint - endpoint: "eu-central-1.ecmwf.int:9000" diff --git a/tests/io/s3/S3Credentials.yaml b/tests/io/s3/S3Credentials.yaml deleted file mode 100644 index 1935a4815..000000000 --- a/tests/io/s3/S3Credentials.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -credentials: - - endpoint: '127.0.0.1:9000' - accessKeyID: 'minio' - secretKey: 'minio1234' - - - endpoint: 'minio:9000' - accessKeyID: 'minio' - secretKey: 'minio1234' - - - endpoint: 'localhost:9000' - accessKeyID: 'asd2' - secretKey: 'asd2' diff --git a/tests/io/s3/S3Credentials.yaml.in b/tests/io/s3/S3Credentials.yaml.in new file mode 100644 index 000000000..3269ce3b7 --- /dev/null +++ b/tests/io/s3/S3Credentials.yaml.in @@ -0,0 +1,13 @@ +--- +credentials: + - endpoint: @S3_TEST_ENDPOINT@ + accessKeyID: minio + secretKey: minio1234 + + - endpoint: localhost:9000 + accessKeyID: asd1 + secretKey: asd1 + + - endpoint: 127.0.0.1:9000 + accessKeyID: asd2 + secretKey: asd2 diff --git a/tests/io/s3/test_s3client.cc b/tests/io/s3/test_s3_client.cc similarity index 90% rename from tests/io/s3/test_s3client.cc rename to tests/io/s3/test_s3_client.cc index 038adc871..b5ff7f3c3 100644 --- a/tests/io/s3/test_s3client.cc +++ b/tests/io/s3/test_s3_client.cc @@ -21,6 +21,7 @@ #include "eckit/io/s3/S3Session.h" #include "eckit/net/Endpoint.h" #include "eckit/testing/Test.h" +#include "test_s3_config.h" #include #include @@ -37,9 +38,11 @@ namespace eckit::test { namespace { -const net::Endpoint TEST_ENDPOINT {"minio", 9000}; +const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; -const S3Config TEST_CONFIG {TEST_ENDPOINT, "eu-central-1"}; +const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; + +const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; bool findString(const std::vector& list, const std::string& item) { return (std::find(list.begin(), list.end(), item) != list.end()); @@ -61,15 +64,12 @@ void cleanup() { CASE("s3 client: API") { - const S3Config config {TEST_ENDPOINT, "eu-central-1"}; - const S3Credential cred {TEST_ENDPOINT, "minio", "minio1234"}; - - EXPECT(S3Session::instance().addClient(config)); - EXPECT(S3Session::instance().addCredential(cred)); + EXPECT(S3Session::instance().addClient(TEST_CONFIG)); + EXPECT(S3Session::instance().addCredential(TEST_CRED)); EXPECT_NO_THROW(cleanup()); - EXPECT_NO_THROW(S3Session::instance().removeClient(config.endpoint)); + EXPECT_NO_THROW(S3Session::instance().removeClient(TEST_ENDPOINT)); } //---------------------------------------------------------------------------------------------------------------------- @@ -95,13 +95,11 @@ CASE("s3 client: read from file") { CASE("s3 credentials: API") { - const S3Credential cred {TEST_ENDPOINT, "minio", "minio1234"}; - - EXPECT(S3Session::instance().addCredential(cred)); + EXPECT(S3Session::instance().addCredential(TEST_CRED)); EXPECT_NO_THROW(cleanup()); - EXPECT_NO_THROW(S3Session::instance().removeCredential(cred.endpoint)); + EXPECT_NO_THROW(S3Session::instance().removeCredential(TEST_ENDPOINT)); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/io/s3/test_s3_config.h b/tests/io/s3/test_s3_config.h new file mode 100644 index 000000000..e5333149d --- /dev/null +++ b/tests/io/s3/test_s3_config.h @@ -0,0 +1,17 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#pragma once + +#define S3_TEST_HOST "minio" +#define S3_TEST_PORT "9000" +#define S3_TEST_ENDPOINT "minio:9000" +#define S3_TEST_REGION "eu-central-1" +#define S3_TEST_BUCKET "eckit-test-bucket" diff --git a/tests/io/s3/test_s3_config.h.in b/tests/io/s3/test_s3_config.h.in new file mode 100644 index 000000000..4b67ff8e5 --- /dev/null +++ b/tests/io/s3/test_s3_config.h.in @@ -0,0 +1,17 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#pragma once + +#cmakedefine S3_TEST_HOST "@S3_TEST_HOST@" +#cmakedefine S3_TEST_PORT "@S3_TEST_PORT@" +#cmakedefine S3_TEST_ENDPOINT "@S3_TEST_ENDPOINT@" +#cmakedefine S3_TEST_REGION "@S3_TEST_REGION@" +#cmakedefine S3_TEST_BUCKET "@S3_TEST_BUCKET@" diff --git a/tests/io/s3/test_s3handle.cc b/tests/io/s3/test_s3_handle.cc similarity index 96% rename from tests/io/s3/test_s3handle.cc rename to tests/io/s3/test_s3_handle.cc index 8c5fcba2a..868634e66 100644 --- a/tests/io/s3/test_s3handle.cc +++ b/tests/io/s3/test_s3_handle.cc @@ -26,6 +26,7 @@ #include "eckit/log/Timer.h" #include "eckit/net/Endpoint.h" #include "eckit/testing/Test.h" +#include "test_s3_config.h" #include @@ -49,12 +50,14 @@ namespace { constexpr std::string_view TEST_DATA = "abcdefghijklmnopqrstuvwxyz"; -const net::Endpoint TEST_ENDPOINT {"minio", 9000}; -const std::string TEST_BUCKET {"eckit-s3handle-test-bucket"}; -const std::string TEST_OBJECT {"eckit-s3handle-test-object"}; +const std::string TEST_BUCKET {"eckit-s3handle-test-bucket"}; +const std::string TEST_OBJECT {"eckit-s3handle-test-object"}; -const S3Config TEST_CONFIG {TEST_ENDPOINT, "eu-central-1"}; -const S3Credential TEST_CREDENTIAL {TEST_ENDPOINT, "minio", "minio1234"}; +const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; + +const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; + +const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; //---------------------------------------------------------------------------------------------------------------------- @@ -91,7 +94,7 @@ void cleanup() { CASE("initialize s3 session") { - EXPECT(S3Session::instance().addCredential(TEST_CREDENTIAL)); + EXPECT(S3Session::instance().addCredential(TEST_CRED)); EXPECT(S3Session::instance().addClient(TEST_CONFIG)); EXPECT_NO_THROW(cleanup()); From fcc5f379de7eb2c36a0e01b97e930bc13c3c55de Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 25 Dec 2024 11:49:02 +0100 Subject: [PATCH 87/90] test(S3): improved common config --- tests/io/s3/CMakeLists.txt | 3 +- tests/io/s3/test_s3_client.cc | 64 +++++++------------- tests/io/s3/test_s3_config.h | 64 +++++++++++++++++++- tests/io/s3/test_s3_config.h.in | 62 ++++++++++++++++++++ tests/io/s3/test_s3_handle.cc | 100 +++++++++----------------------- 5 files changed, 176 insertions(+), 117 deletions(-) diff --git a/tests/io/s3/CMakeLists.txt b/tests/io/s3/CMakeLists.txt index 8ade180f1..069af9f09 100644 --- a/tests/io/s3/CMakeLists.txt +++ b/tests/io/s3/CMakeLists.txt @@ -5,7 +5,8 @@ if( eckit_HAVE_S3 ) set( S3_TEST_PORT "9000" ) set( S3_TEST_ENDPOINT ${S3_TEST_HOST}:${S3_TEST_PORT} ) set( S3_TEST_REGION "eu-central-1" ) - set( S3_TEST_BUCKET "eckit-test-bucket" ) + set( S3_TEST_BUCKET "eckit-test-s3-bucket" ) + set( S3_TEST_OBJECT "eckit-test-s3-object" ) configure_file( test_s3_config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/test_s3_config.h @ONLY ) diff --git a/tests/io/s3/test_s3_client.cc b/tests/io/s3/test_s3_client.cc index b5ff7f3c3..142e1ac89 100644 --- a/tests/io/s3/test_s3_client.cc +++ b/tests/io/s3/test_s3_client.cc @@ -16,14 +16,12 @@ #include "eckit/io/s3/S3BucketPath.h" #include "eckit/io/s3/S3Client.h" #include "eckit/io/s3/S3Config.h" -#include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3Exception.h" #include "eckit/io/s3/S3Session.h" #include "eckit/net/Endpoint.h" #include "eckit/testing/Test.h" #include "test_s3_config.h" -#include #include #include @@ -36,40 +34,18 @@ namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- -namespace { - -const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; - -const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; - -const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; - -bool findString(const std::vector& list, const std::string& item) { - return (std::find(list.begin(), list.end(), item) != list.end()); -} - -void cleanup() { - auto client = S3Client::makeUnique(TEST_CONFIG); - for (const auto& name : {S3BucketPath {"test-bucket-1"}, S3BucketPath {{"test-bucket-2"}}}) { - if (client->bucketExists(name)) { - client->emptyBucket(name); - client->deleteBucket(name); - } - } -} - -} // namespace +const std::vector testBuckets {"test-bucket-1", "test-bucket-2"}; //---------------------------------------------------------------------------------------------------------------------- CASE("s3 client: API") { - EXPECT(S3Session::instance().addClient(TEST_CONFIG)); - EXPECT(S3Session::instance().addCredential(TEST_CRED)); + EXPECT(S3Session::instance().addClient(s3::TEST_CONFIG)); + EXPECT(S3Session::instance().addCredential(s3::TEST_CRED)); - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - EXPECT_NO_THROW(S3Session::instance().removeClient(TEST_ENDPOINT)); + EXPECT_NO_THROW(S3Session::instance().removeClient(s3::TEST_ENDPOINT)); } //---------------------------------------------------------------------------------------------------------------------- @@ -95,11 +71,11 @@ CASE("s3 client: read from file") { CASE("s3 credentials: API") { - EXPECT(S3Session::instance().addCredential(TEST_CRED)); + EXPECT(S3Session::instance().addCredential(s3::TEST_CRED)); - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - EXPECT_NO_THROW(S3Session::instance().removeCredential(TEST_ENDPOINT)); + EXPECT_NO_THROW(S3Session::instance().removeCredential(s3::TEST_ENDPOINT)); } //---------------------------------------------------------------------------------------------------------------------- @@ -124,7 +100,7 @@ CASE("s3 credentials: read from file") { //---------------------------------------------------------------------------------------------------------------------- CASE("s3 backends") { - S3Config cfgTmp(TEST_CONFIG); + S3Config cfgTmp(s3::TEST_CONFIG); cfgTmp.backend = S3Backend::AWS; EXPECT_NO_THROW(S3Client::makeUnique(cfgTmp)); @@ -148,12 +124,12 @@ CASE("s3 backends") { //---------------------------------------------------------------------------------------------------------------------- CASE("create s3 bucket in non-existing region") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); // this test requires an S3 endpoint that sets it's region // a MinIO instance with empty region will not throw an exception - auto cfgTmp = TEST_CONFIG; + auto cfgTmp = s3::TEST_CONFIG; cfgTmp.region = "non-existing-region-random"; EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket({"test-bucket-1"})); @@ -162,9 +138,9 @@ CASE("create s3 bucket in non-existing region") { //---------------------------------------------------------------------------------------------------------------------- CASE("create s3 bucket") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - auto client = S3Client::makeUnique(TEST_CONFIG); + auto client = S3Client::makeUnique(s3::TEST_CONFIG); EXPECT_NO_THROW(client->createBucket({"test-bucket-1"})); @@ -176,17 +152,17 @@ CASE("create s3 bucket") { //---------------------------------------------------------------------------------------------------------------------- CASE("list s3 buckets") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - auto client = S3Client::makeUnique(TEST_CONFIG); + auto client = S3Client::makeUnique(s3::TEST_CONFIG); EXPECT_NO_THROW(client->createBucket({"test-bucket-1"})); EXPECT_NO_THROW(client->createBucket({"test-bucket-2"})); { const auto buckets = client->listBuckets(); - EXPECT(findString(buckets, "test-bucket-1")); - EXPECT(findString(buckets, {"test-bucket-2"})); + EXPECT(s3::findString(buckets, "test-bucket-1")); + EXPECT(s3::findString(buckets, {"test-bucket-2"})); EXPECT_NO_THROW(client->deleteBucket({"test-bucket-1"})); EXPECT_NO_THROW(client->deleteBucket({"test-bucket-2"})); @@ -194,8 +170,8 @@ CASE("list s3 buckets") { { const auto buckets = client->listBuckets(); - EXPECT_NOT(findString(buckets, "test-bucket-1")); - EXPECT_NOT(findString(buckets, {"test-bucket-2"})); + EXPECT_NOT(s3::findString(buckets, "test-bucket-1")); + EXPECT_NOT(s3::findString(buckets, {"test-bucket-2"})); } EXPECT_THROWS(client->deleteBucket({"test-bucket-1"})); @@ -211,7 +187,7 @@ int main(int argc, char** argv) { ret = run_tests(argc, argv); - test::cleanup(); + test::s3::cleanup(test::testBuckets); return ret; } diff --git a/tests/io/s3/test_s3_config.h b/tests/io/s3/test_s3_config.h index e5333149d..004d1320e 100644 --- a/tests/io/s3/test_s3_config.h +++ b/tests/io/s3/test_s3_config.h @@ -10,8 +10,70 @@ #pragma once +#include "eckit/io/Buffer.h" +#include "eckit/io/s3/S3BucketName.h" +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/log/Bytes.h" +#include "eckit/log/Timer.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include + #define S3_TEST_HOST "minio" #define S3_TEST_PORT "9000" #define S3_TEST_ENDPOINT "minio:9000" #define S3_TEST_REGION "eu-central-1" -#define S3_TEST_BUCKET "eckit-test-bucket" +#define S3_TEST_BUCKET "eckit-test-s3-bucket" +#define S3_TEST_OBJECT "eckit-test-s3-object" + +namespace eckit::test::s3 { + +//---------------------------------------------------------------------------------------------------------------------- + +const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; + +const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; + +const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; + +inline bool findString(const std::vector& list, const std::string& item) { + return (std::find(list.begin(), list.end(), item) != list.end()); +} + +inline void cleanup(const std::vector& bucketPaths) { + auto client = S3Client::makeUnique(TEST_CONFIG); + for (const auto& path : bucketPaths) { + if (client->bucketExists(path)) { + client->emptyBucket(path); + client->deleteBucket(path); + } + } +} + +void writePerformance(S3BucketName& bucket, const int count) { + eckit::Timer timer; + + Buffer buffer(1024 * 1024); + buffer.zero(); + + timer.start(); + for (int i = 0; i < count; i++) { + const auto objName = S3_TEST_OBJECT + std::to_string(i); + bucket.makeObject(objName)->put(buffer.data(), buffer.size()); + } + timer.stop(); + + std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count + << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test::s3 diff --git a/tests/io/s3/test_s3_config.h.in b/tests/io/s3/test_s3_config.h.in index 4b67ff8e5..3d05cdc84 100644 --- a/tests/io/s3/test_s3_config.h.in +++ b/tests/io/s3/test_s3_config.h.in @@ -10,8 +10,70 @@ #pragma once +#include "eckit/io/Buffer.h" +#include "eckit/io/s3/S3BucketName.h" +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/log/Bytes.h" +#include "eckit/log/Timer.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include + #cmakedefine S3_TEST_HOST "@S3_TEST_HOST@" #cmakedefine S3_TEST_PORT "@S3_TEST_PORT@" #cmakedefine S3_TEST_ENDPOINT "@S3_TEST_ENDPOINT@" #cmakedefine S3_TEST_REGION "@S3_TEST_REGION@" #cmakedefine S3_TEST_BUCKET "@S3_TEST_BUCKET@" +#cmakedefine S3_TEST_OBJECT "@S3_TEST_OBJECT@" + +namespace eckit::test::s3 { + +//---------------------------------------------------------------------------------------------------------------------- + +const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; + +const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; + +const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; + +inline bool findString(const std::vector& list, const std::string& item) { + return (std::find(list.begin(), list.end(), item) != list.end()); +} + +inline void cleanup(const std::vector& bucketPaths) { + auto client = S3Client::makeUnique(TEST_CONFIG); + for (const auto& path : bucketPaths) { + if (client->bucketExists(path)) { + client->emptyBucket(path); + client->deleteBucket(path); + } + } +} + +void writePerformance(S3BucketName& bucket, const int count) { + eckit::Timer timer; + + Buffer buffer(1024 * 1024); + buffer.zero(); + + timer.start(); + for (int i = 0; i < count; i++) { + const auto objName = S3_TEST_OBJECT + std::to_string(i); + bucket.makeObject(objName)->put(buffer.data(), buffer.size()); + } + timer.stop(); + + std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count + << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test::s3 diff --git a/tests/io/s3/test_s3_handle.cc b/tests/io/s3/test_s3_handle.cc index 868634e66..bf61df6df 100644 --- a/tests/io/s3/test_s3_handle.cc +++ b/tests/io/s3/test_s3_handle.cc @@ -15,15 +15,11 @@ #include "eckit/filesystem/PathName.h" #include "eckit/filesystem/URI.h" -#include "eckit/io/Buffer.h" #include "eckit/io/MemoryHandle.h" #include "eckit/io/s3/S3BucketName.h" #include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Credential.h" #include "eckit/io/s3/S3ObjectName.h" #include "eckit/io/s3/S3Session.h" -#include "eckit/log/Bytes.h" -#include "eckit/log/Timer.h" #include "eckit/net/Endpoint.h" #include "eckit/testing/Test.h" #include "test_s3_config.h" @@ -31,11 +27,10 @@ #include #include -#include #include -#include #include #include +#include using namespace std::string_literals; @@ -46,58 +41,21 @@ namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- -namespace { - -constexpr std::string_view TEST_DATA = "abcdefghijklmnopqrstuvwxyz"; +constexpr std::string_view TEST_DATA {"abcdefghijklmnopqrstuvwxyz"}; const std::string TEST_BUCKET {"eckit-s3handle-test-bucket"}; const std::string TEST_OBJECT {"eckit-s3handle-test-object"}; -const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; - -const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; - -const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; - -//---------------------------------------------------------------------------------------------------------------------- - -void writePerformance(S3BucketName& bucket, const int count) { - eckit::Timer timer; - - Buffer buffer(1024 * 1024); - buffer.zero(); - - timer.start(); - for (int i = 0; i < count; i++) { - const auto objName = TEST_OBJECT + std::to_string(i); - bucket.makeObject(objName)->put(buffer.data(), buffer.size()); - } - timer.stop(); - - std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count - << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; -} - -void cleanup() { - auto client = S3Client::makeUnique(TEST_CONFIG); - for (const auto& name : {TEST_BUCKET}) { - if (client->bucketExists(name)) { - client->emptyBucket(name); - client->deleteBucket(name); - } - } -} - -} // namespace +const std::vector testBuckets {TEST_BUCKET}; //---------------------------------------------------------------------------------------------------------------------- CASE("initialize s3 session") { - EXPECT(S3Session::instance().addCredential(TEST_CRED)); - EXPECT(S3Session::instance().addClient(TEST_CONFIG)); + EXPECT(S3Session::instance().addCredential(s3::TEST_CRED)); + EXPECT(S3Session::instance().addClient(s3::TEST_CONFIG)); - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); } CASE("invalid s3 bucket") { @@ -109,9 +67,9 @@ CASE("invalid s3 bucket") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3BucketName: no bucket") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); EXPECT_NOT(bucket.exists()); @@ -129,9 +87,9 @@ CASE("S3BucketName: no bucket") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3BucketName: create bucket") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); EXPECT_NOT(bucket.exists()); @@ -147,9 +105,9 @@ CASE("S3BucketName: create bucket") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3BucketName: empty bucket") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); // CREATE BUCKET EXPECT_NO_THROW(bucket.ensureCreated()); @@ -165,9 +123,9 @@ CASE("S3BucketName: empty bucket") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3BucketName: bucket with object") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - S3BucketName bucket(TEST_ENDPOINT, TEST_BUCKET); + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); // CREATE BUCKET EXPECT_NO_THROW(bucket.ensureCreated()); @@ -186,14 +144,14 @@ CASE("S3BucketName: bucket with object") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3Handle: basic operations") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - S3Client::makeUnique(TEST_CONFIG)->createBucket(TEST_BUCKET); + S3Client::makeUnique(s3::TEST_CONFIG)->createBucket(TEST_BUCKET); const void* buffer = TEST_DATA.data(); const long length = TEST_DATA.size(); - const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); { S3ObjectName object(uri); @@ -234,9 +192,9 @@ CASE("S3Handle: basic operations") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3Handle: openForRead") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); @@ -259,9 +217,9 @@ CASE("S3Handle: openForRead") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3Handle: openForWrite") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); { // NO BUCKET std::unique_ptr handle(uri.newWriteHandle()); @@ -282,9 +240,9 @@ CASE("S3Handle: openForWrite") { //---------------------------------------------------------------------------------------------------------------------- CASE("S3Handle: read") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); @@ -316,19 +274,19 @@ CASE("S3Handle: read") { //---------------------------------------------------------------------------------------------------------------------- CASE("s3 performance: write 1 10 100 objects") { - EXPECT_NO_THROW(cleanup()); + EXPECT_NO_THROW(s3::cleanup(testBuckets)); - const URI uri("s3://" + std::string(TEST_ENDPOINT) + "/" + TEST_BUCKET); + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET); S3BucketName bucket(uri); EXPECT_NO_THROW(bucket.ensureCreated()); - writePerformance(bucket, 1); + s3::writePerformance(bucket, 1); - writePerformance(bucket, 10); + s3::writePerformance(bucket, 10); - writePerformance(bucket, 100); + s3::writePerformance(bucket, 100); } //---------------------------------------------------------------------------------------------------------------------- @@ -340,7 +298,7 @@ int main(int argc, char** argv) { ret = run_tests(argc, argv); - test::cleanup(); + test::s3::cleanup(test::testBuckets); return ret; } From 07b936b5ee1e7733cdd5c05582dda6aa5f3aaff6 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 25 Dec 2024 15:23:39 +0100 Subject: [PATCH 88/90] fix(S3): asString output and test config --- src/eckit/io/s3/S3BucketPath.h | 2 +- src/eckit/io/s3/S3Config.cc | 13 ++++++++----- src/eckit/io/s3/S3Credential.cc | 15 +++++++++------ src/eckit/io/s3/S3ObjectPath.h | 2 +- tests/io/s3/test_s3_config.h | 2 +- tests/io/s3/test_s3_config.h.in | 2 +- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/eckit/io/s3/S3BucketPath.h b/src/eckit/io/s3/S3BucketPath.h index 3d3cf9507..5c9663a4b 100644 --- a/src/eckit/io/s3/S3BucketPath.h +++ b/src/eckit/io/s3/S3BucketPath.h @@ -31,7 +31,7 @@ struct S3BucketPath { S3BucketPath(std::string bucket) : bucket {std::move(bucket)} { } - auto asString() const -> std::string { return '/' + bucket; } + auto asString() const -> std::string { return bucket; } operator std::string() const { return asString(); } diff --git a/src/eckit/io/s3/S3Config.cc b/src/eckit/io/s3/S3Config.cc index 8cb000bc2..63973ae16 100644 --- a/src/eckit/io/s3/S3Config.cc +++ b/src/eckit/io/s3/S3Config.cc @@ -69,16 +69,19 @@ auto S3Config::make(std::string path) -> std::vector { path = Resource("s3ConfigFile;$ECKIT_S3_CONFIG_FILE", "~/.config/eckit/S3Config.yaml"); } - PathName configFile(path); + PathName configPath(path); - if (!configFile.exists()) { - Log::debug() << "S3 configuration file does not exist: " << configFile << std::endl; + if (!configPath.exists()) { + Log::debug() << "S3 configuration file does not exist: " << configPath << std::endl; return {}; } - LOG_DEBUG_LIB(LibEcKit) << "Reading S3 configuration from: " << configFile << std::endl; + if (configPath.isDir()) { + Log::debug() << "Path " << configPath << " is a directory. Expecting a file!" << std::endl; + return {}; + } - const auto servers = YAMLConfiguration(configFile).getSubConfigurations("servers"); + const auto servers = YAMLConfiguration(configPath).getSubConfigurations("servers"); std::vector result; result.reserve(servers.size()); diff --git a/src/eckit/io/s3/S3Credential.cc b/src/eckit/io/s3/S3Credential.cc index 4465f8265..762fe8bdc 100644 --- a/src/eckit/io/s3/S3Credential.cc +++ b/src/eckit/io/s3/S3Credential.cc @@ -51,18 +51,21 @@ auto S3Credential::fromFile(std::string path) -> std::vector { if (path.empty()) { path = Resource("s3CredentialsFile;$ECKIT_S3_CREDENTIALS_FILE", defaultCredFile); } - PathName credFile(path); + PathName credPath(path); - if (!credFile.exists()) { - Log::debug() << "S3 credentials file does not exist: " << credFile << std::endl; + if (!credPath.exists()) { + Log::debug() << "S3 credentials file does not exist: " << credPath << std::endl; return {}; } - LOG_DEBUG_LIB(LibEcKit) << "Reading S3 credentials from: " << credFile << std::endl; + if (credPath.isDir()) { + Log::debug() << "Path " << credPath << " is a directory. Expecting a file!" << std::endl; + return {}; + } - std::vector result; + const auto creds = YAMLConfiguration(credPath).getSubConfigurations("credentials"); - const auto creds = YAMLConfiguration(credFile).getSubConfigurations("credentials"); + std::vector result; result.reserve(creds.size()); for (const auto& cred : creds) { result.emplace_back(fromYAML(cred)); } diff --git a/src/eckit/io/s3/S3ObjectPath.h b/src/eckit/io/s3/S3ObjectPath.h index 478d26706..304379229 100644 --- a/src/eckit/io/s3/S3ObjectPath.h +++ b/src/eckit/io/s3/S3ObjectPath.h @@ -31,7 +31,7 @@ struct S3ObjectPath { S3ObjectPath(std::string bucket, std::string object) : bucket {std::move(bucket)}, object {std::move(object)} { } - auto asString() const -> std::string { return '/' + bucket + '/' + object; } + auto asString() const -> std::string { return bucket + '/' + object; } operator std::string() const { return asString(); } diff --git a/tests/io/s3/test_s3_config.h b/tests/io/s3/test_s3_config.h index 004d1320e..1864adcbb 100644 --- a/tests/io/s3/test_s3_config.h +++ b/tests/io/s3/test_s3_config.h @@ -57,7 +57,7 @@ inline void cleanup(const std::vector& bucketPaths) { } } -void writePerformance(S3BucketName& bucket, const int count) { +inline void writePerformance(S3BucketName& bucket, const int count) { eckit::Timer timer; Buffer buffer(1024 * 1024); diff --git a/tests/io/s3/test_s3_config.h.in b/tests/io/s3/test_s3_config.h.in index 3d05cdc84..aedef16a0 100644 --- a/tests/io/s3/test_s3_config.h.in +++ b/tests/io/s3/test_s3_config.h.in @@ -57,7 +57,7 @@ inline void cleanup(const std::vector& bucketPaths) { } } -void writePerformance(S3BucketName& bucket, const int count) { +inline void writePerformance(S3BucketName& bucket, const int count) { eckit::Timer timer; Buffer buffer(1024 * 1024); From 25d98c7099267b7484500cbb214df2143387fb94 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 25 Dec 2024 15:51:50 +0100 Subject: [PATCH 89/90] test(S3): removed generated header [skip ci] --- tests/io/s3/CMakeLists.txt | 6 +-- tests/io/s3/test_s3_config.h | 79 ------------------------------------ 2 files changed, 3 insertions(+), 82 deletions(-) delete mode 100644 tests/io/s3/test_s3_config.h diff --git a/tests/io/s3/CMakeLists.txt b/tests/io/s3/CMakeLists.txt index 069af9f09..405059c16 100644 --- a/tests/io/s3/CMakeLists.txt +++ b/tests/io/s3/CMakeLists.txt @@ -8,7 +8,7 @@ if( eckit_HAVE_S3 ) set( S3_TEST_BUCKET "eckit-test-s3-bucket" ) set( S3_TEST_OBJECT "eckit-test-s3-object" ) - configure_file( test_s3_config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/test_s3_config.h @ONLY ) + configure_file( test_s3_config.h.in test_s3_config.h @ONLY ) configure_file( S3Config.yaml.in S3Config.yaml @ONLY ) configure_file( S3Credentials.yaml.in S3Credentials.yaml @ONLY ) @@ -20,8 +20,8 @@ if( eckit_HAVE_S3 ) ecbuild_add_test( TARGET eckit_test_s3_${_test} SOURCES test_s3_${_test}.cc - INCLUDES test_s3_config.h ${AWSSDK_INCLUDE_DIRS} - LIBS eckit ${AWSSDK_LINK_LIBRARIES} + INCLUDES ${CMAKE_CURRENT_BINARY_DIR} + LIBS eckit ENVIRONMENT "${_test_environment}" ) diff --git a/tests/io/s3/test_s3_config.h b/tests/io/s3/test_s3_config.h deleted file mode 100644 index 1864adcbb..000000000 --- a/tests/io/s3/test_s3_config.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#pragma once - -#include "eckit/io/Buffer.h" -#include "eckit/io/s3/S3BucketName.h" -#include "eckit/io/s3/S3BucketPath.h" -#include "eckit/io/s3/S3Client.h" -#include "eckit/io/s3/S3Config.h" -#include "eckit/io/s3/S3Credential.h" -#include "eckit/io/s3/S3ObjectName.h" -#include "eckit/log/Bytes.h" -#include "eckit/log/Timer.h" -#include "eckit/net/Endpoint.h" - -#include -#include -#include -#include - -#define S3_TEST_HOST "minio" -#define S3_TEST_PORT "9000" -#define S3_TEST_ENDPOINT "minio:9000" -#define S3_TEST_REGION "eu-central-1" -#define S3_TEST_BUCKET "eckit-test-s3-bucket" -#define S3_TEST_OBJECT "eckit-test-s3-object" - -namespace eckit::test::s3 { - -//---------------------------------------------------------------------------------------------------------------------- - -const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; - -const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; - -const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; - -inline bool findString(const std::vector& list, const std::string& item) { - return (std::find(list.begin(), list.end(), item) != list.end()); -} - -inline void cleanup(const std::vector& bucketPaths) { - auto client = S3Client::makeUnique(TEST_CONFIG); - for (const auto& path : bucketPaths) { - if (client->bucketExists(path)) { - client->emptyBucket(path); - client->deleteBucket(path); - } - } -} - -inline void writePerformance(S3BucketName& bucket, const int count) { - eckit::Timer timer; - - Buffer buffer(1024 * 1024); - buffer.zero(); - - timer.start(); - for (int i = 0; i < count; i++) { - const auto objName = S3_TEST_OBJECT + std::to_string(i); - bucket.makeObject(objName)->put(buffer.data(), buffer.size()); - } - timer.stop(); - - std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count - << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; -} - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace eckit::test::s3 From 50e3a25ecc087a99a71587714bbc3c9dae4574a8 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Mon, 30 Dec 2024 20:57:00 +0100 Subject: [PATCH 90/90] feat(S3): added compare operator --- src/eckit/io/s3/S3BucketName.h | 2 -- src/eckit/io/s3/S3BucketPath.h | 4 ++++ src/eckit/io/s3/S3ObjectName.h | 4 ---- src/eckit/io/s3/S3ObjectPath.h | 4 ++++ 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h index 79686bf45..26ba193df 100644 --- a/src/eckit/io/s3/S3BucketName.h +++ b/src/eckit/io/s3/S3BucketName.h @@ -56,8 +56,6 @@ class S3BucketName : public S3Name { auto path() const -> const S3BucketPath& { return path_; } - auto bucket() const -> const std::string& { return path_.bucket; } - /// @todo return S3 object iterator but first add prefix auto listObjects() const -> std::vector; diff --git a/src/eckit/io/s3/S3BucketPath.h b/src/eckit/io/s3/S3BucketPath.h index 5c9663a4b..8eba5f9f7 100644 --- a/src/eckit/io/s3/S3BucketPath.h +++ b/src/eckit/io/s3/S3BucketPath.h @@ -35,6 +35,10 @@ struct S3BucketPath { operator std::string() const { return asString(); } + bool operator==(const S3BucketPath& other) const { return bucket == other.bucket; } + + bool operator!=(const S3BucketPath& other) const { return !(*this == other); } + friend std::ostream& operator<<(std::ostream& out, const S3BucketPath& path) { out << "bucket=" << path.bucket; return out; diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h index 083cbc442..4699ba8b5 100644 --- a/src/eckit/io/s3/S3ObjectName.h +++ b/src/eckit/io/s3/S3ObjectName.h @@ -53,10 +53,6 @@ class S3ObjectName : public S3Name { auto path() const -> const S3ObjectPath& { return path_; } - auto name() const -> const std::string& { return path_.object; } - - auto bucket() const -> const std::string& { return path_.bucket; } - auto bucketExists() const -> bool; // modifiers diff --git a/src/eckit/io/s3/S3ObjectPath.h b/src/eckit/io/s3/S3ObjectPath.h index 304379229..2fbc60d92 100644 --- a/src/eckit/io/s3/S3ObjectPath.h +++ b/src/eckit/io/s3/S3ObjectPath.h @@ -35,6 +35,10 @@ struct S3ObjectPath { operator std::string() const { return asString(); } + bool operator==(const S3ObjectPath& other) const { return bucket == other.bucket && object == other.object; } + + bool operator!=(const S3ObjectPath& other) const { return !(*this == other); } + friend std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path) { out << "bucket=" << path.bucket << ",object=" << path.object; return out;