Skip to content

Commit 536785a

Browse files
mcakircalimcakircali
committed
feat(S3): add config and credentials from file
fix several minor issues
1 parent 45b7135 commit 536785a

29 files changed

+1023
-510
lines changed

‎src/eckit/CMakeLists.txt

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -283,33 +283,34 @@ if(HAVE_RADOS)
283283
)
284284
endif()
285285

286-
if(HAVE_AWS_S3)
287-
list( APPEND eckit_io_srcs
288-
io/s3/aws/S3ClientAWS.cc
289-
io/s3/aws/S3ClientAWS.h
290-
io/s3/aws/S3ContextAWS.cc
291-
io/s3/aws/S3ContextAWS.h
292-
io/s3/S3BucketName.cc
293-
io/s3/S3BucketName.h
294-
io/s3/S3Client.cc
295-
io/s3/S3Client.h
296-
io/s3/S3Config.cc
297-
io/s3/S3Config.h
298-
io/s3/S3Credential.h
299-
io/s3/S3Exception.cc
300-
io/s3/S3Exception.h
301-
io/s3/S3Handle.cc
302-
io/s3/S3Handle.h
303-
io/s3/S3Name.cc
304-
io/s3/S3Name.h
305-
io/s3/S3ObjectName.cc
306-
io/s3/S3ObjectName.h
307-
io/s3/S3Session.cc
308-
io/s3/S3Session.h
309-
io/s3/S3URIManager.cc
310-
io/s3/S3URIManager.h
311-
)
312-
endif(HAVE_AWS_S3)
286+
if( eckit_HAVE_AWS_S3 )
287+
list( APPEND eckit_io_srcs
288+
io/s3/aws/S3ClientAWS.cc
289+
io/s3/aws/S3ClientAWS.h
290+
io/s3/aws/S3ContextAWS.cc
291+
io/s3/aws/S3ContextAWS.h
292+
io/s3/S3BucketName.cc
293+
io/s3/S3BucketName.h
294+
io/s3/S3Client.cc
295+
io/s3/S3Client.h
296+
io/s3/S3Config.cc
297+
io/s3/S3Config.h
298+
io/s3/S3Credential.cc
299+
io/s3/S3Credential.h
300+
io/s3/S3Exception.cc
301+
io/s3/S3Exception.h
302+
io/s3/S3Handle.cc
303+
io/s3/S3Handle.h
304+
io/s3/S3Name.cc
305+
io/s3/S3Name.h
306+
io/s3/S3ObjectName.cc
307+
io/s3/S3ObjectName.h
308+
io/s3/S3Session.cc
309+
io/s3/S3Session.h
310+
io/s3/S3URIManager.cc
311+
io/s3/S3URIManager.h
312+
)
313+
endif( eckit_HAVE_AWS_S3 )
313314

314315
list( APPEND eckit_filesystem_srcs
315316
filesystem/BasePathName.cc
@@ -960,9 +961,9 @@ ecbuild_add_library(
960961
"${BZIP2_INCLUDE_DIRS}"
961962
"${AEC_INCLUDE_DIRS}"
962963
"${RADOS_INCLUDE_DIRS}"
963-
"${AWSSDK_INCLUDE_DIRS}"
964964
"${OPENSSL_INCLUDE_DIR}"
965965
"${AIO_INCLUDE_DIRS}"
966+
"${AWSSDK_INCLUDE_DIRS}"
966967

967968
PRIVATE_LIBS
968969
"${SNAPPY_LIBRARIES}"

‎src/eckit/io/s3/S3BucketName.cc

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,27 @@
1919
#include "eckit/filesystem/URI.h"
2020
#include "eckit/io/s3/S3Client.h"
2121
#include "eckit/io/s3/S3Exception.h"
22+
#include "eckit/io/s3/S3Name.h"
2223
#include "eckit/io/s3/S3ObjectName.h"
24+
#include "eckit/log/Log.h"
25+
#include "eckit/net/Endpoint.h"
26+
27+
#include <memory>
28+
#include <ostream>
29+
#include <string>
30+
#include <utility>
31+
#include <vector>
2332

2433
namespace eckit {
2534

2635
//----------------------------------------------------------------------------------------------------------------------
2736

28-
S3BucketName::S3BucketName(const URI& uri): S3Name(uri) {
37+
S3BucketName::S3BucketName(const URI& uri) : S3Name(uri) {
2938
parse();
3039
}
3140

32-
S3BucketName::S3BucketName(const net::Endpoint& endpoint, const std::string& bucket):
33-
S3Name(endpoint, "/"), bucket_(bucket) { }
41+
S3BucketName::S3BucketName(const net::Endpoint& endpoint, std::string bucket)
42+
: S3Name(endpoint, "/"), bucket_ {std::move(bucket)} { }
3443

3544
//----------------------------------------------------------------------------------------------------------------------
3645

@@ -56,38 +65,32 @@ auto S3BucketName::makeObject(const std::string& object) const -> std::unique_pt
5665
}
5766

5867
auto S3BucketName::exists() const -> bool {
59-
return client()->bucketExists(bucket_);
68+
return client().bucketExists(bucket_);
6069
}
6170

6271
void S3BucketName::create() {
63-
client()->createBucket(bucket_);
72+
client().createBucket(bucket_);
6473
}
6574

6675
void S3BucketName::destroy() {
67-
client()->deleteBucket(bucket_);
76+
client().deleteBucket(bucket_);
6877
}
6978

7079
void S3BucketName::ensureCreated() {
7180
try {
7281
create();
73-
}
74-
catch (S3EntityAlreadyExists& e) {
75-
LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl;
76-
}
82+
} catch (S3EntityAlreadyExists& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; }
7783
}
7884

7985
void S3BucketName::ensureDestroyed() {
8086
try {
81-
client()->emptyBucket(bucket_);
82-
client()->deleteBucket(bucket_);
83-
}
84-
catch (S3EntityNotFound& e) {
85-
LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl;
86-
}
87+
client().emptyBucket(bucket_);
88+
client().deleteBucket(bucket_);
89+
} catch (S3EntityNotFound& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; }
8790
}
8891

8992
auto S3BucketName::listObjects() const -> std::vector<std::string> {
90-
return client()->listObjects(bucket_);
93+
return client().listObjects(bucket_);
9194
}
9295

9396
//----------------------------------------------------------------------------------------------------------------------

‎src/eckit/io/s3/S3BucketName.h

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,24 @@
2121
#pragma once
2222

2323
#include "eckit/io/s3/S3Name.h"
24+
#include "eckit/net/Endpoint.h"
25+
26+
#include <iosfwd>
27+
#include <memory>
28+
#include <string>
29+
#include <vector>
2430

2531
namespace eckit {
2632

2733
class S3ObjectName;
2834

2935
//----------------------------------------------------------------------------------------------------------------------
3036

31-
class S3BucketName: public S3Name {
37+
class S3BucketName : public S3Name {
3238
public: // methods
3339
explicit S3BucketName(const URI& uri);
3440

35-
S3BucketName(const net::Endpoint& endpoint, const std::string& bucket);
41+
S3BucketName(const net::Endpoint& endpoint, std::string bucket);
3642

3743
auto makeObject(const std::string& object) const -> std::unique_ptr<S3ObjectName>;
3844

@@ -46,7 +52,7 @@ class S3BucketName: public S3Name {
4652

4753
void ensureDestroyed();
4854

49-
/// @todo: return S3 object iterator but first add prefix
55+
/// @todo return S3 object iterator but first add prefix
5056
auto listObjects() const -> std::vector<std::string>;
5157

5258
auto asString() const -> std::string override;

‎src/eckit/io/s3/S3Client.cc

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,41 +15,40 @@
1515

1616
#include "eckit/io/s3/S3Client.h"
1717

18-
#include "S3Client.h"
19-
#include "eckit/io/s3/S3Exception.h"
18+
#include "eckit/exception/Exceptions.h"
19+
#include "eckit/io/s3/S3Config.h"
2020
#include "eckit/io/s3/S3Session.h"
2121
#include "eckit/io/s3/aws/S3ClientAWS.h"
22+
#include "eckit/log/CodeLocation.h"
23+
24+
#include <memory>
25+
#include <ostream>
26+
#include <utility>
2227

2328
namespace eckit {
2429

2530
//----------------------------------------------------------------------------------------------------------------------
2631

27-
S3Client::S3Client(const S3Config& config): config_(config) { }
28-
29-
S3Client::~S3Client() = default;
32+
S3Client::S3Client(S3Config config) : config_ {std::move(config)} { }
3033

3134
//----------------------------------------------------------------------------------------------------------------------
3235

33-
void S3Client::print(std::ostream& out) const {
34-
out << "S3Client[config=" << config_ << "]";
35-
}
36+
auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr<S3Client> {
3637

37-
//----------------------------------------------------------------------------------------------------------------------
38+
if (config.backend == S3Backend::AWS) { return std::make_unique<S3ClientAWS>(config); }
3839

39-
auto S3Client::makeShared(const S3Config& config) -> std::shared_ptr<S3Client> {
40-
if (config.type == S3Types::AWS) { return std::make_shared<S3ClientAWS>(config); }
41-
throw S3SeriousBug("Unkown S3 client type!", Here());
40+
throw UserError("Unsupported S3 backend! Supported backend = AWS ", Here());
4241
}
4342

4443
//----------------------------------------------------------------------------------------------------------------------
4544

46-
auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr<S3Client> {
47-
if (config.type == S3Types::AWS) { return std::make_unique<S3ClientAWS>(config); }
48-
throw S3SeriousBug("Unkown S3 client type!", Here());
45+
void S3Client::print(std::ostream& out) const {
46+
out << "S3Client[config=" << config_ << "]";
4947
}
5048

51-
auto S3Client::endpoint() const -> const net::Endpoint& {
52-
return config_.endpoint;
49+
std::ostream& operator<<(std::ostream& out, const S3Client& client) {
50+
client.print(out);
51+
return out;
5352
}
5453

5554
//----------------------------------------------------------------------------------------------------------------------

‎src/eckit/io/s3/S3Client.h

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@
2020
#pragma once
2121

2222
#include "eckit/io/s3/S3Config.h"
23-
#include "eckit/memory/NonCopyable.h"
2423

24+
#include <cstdint>
25+
#include <iosfwd>
2526
#include <memory>
27+
#include <string>
2628
#include <vector>
2729

2830
namespace eckit {
@@ -31,15 +33,19 @@ class S3Context;
3133

3234
//----------------------------------------------------------------------------------------------------------------------
3335

34-
class S3Client: private NonCopyable {
36+
class S3Client {
3537
public: // methods
36-
virtual ~S3Client();
37-
38-
static auto makeShared(const S3Config& config) -> std::shared_ptr<S3Client>;
38+
S3Client(const S3Client&) = delete;
39+
S3Client& operator=(const S3Client&) = delete;
40+
S3Client(S3Client&&) = default;
41+
S3Client& operator=(S3Client&&) = default;
42+
virtual ~S3Client() = default;
3943

4044
static auto makeUnique(const S3Config& config) -> std::unique_ptr<S3Client>;
4145

42-
virtual auto endpoint() const -> const net::Endpoint&;
46+
static auto makeShared(const S3Config& config) -> std::shared_ptr<S3Client> { return makeUnique(config); }
47+
48+
auto config() const -> const S3Config& { return config_; }
4349

4450
virtual void createBucket(const std::string& bucket) const = 0;
4551

@@ -51,11 +57,16 @@ class S3Client: private NonCopyable {
5157

5258
virtual auto listBuckets() const -> std::vector<std::string> = 0;
5359

54-
virtual auto putObject(const std::string& bucket, const std::string& object, const void* buffer,
55-
uint64_t length) const -> long long = 0;
60+
virtual auto putObject(const std::string& bucket,
61+
const std::string& object,
62+
const void* buffer,
63+
uint64_t length) const -> long long = 0;
5664

57-
virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t offset,
58-
uint64_t length) const -> long long = 0;
65+
virtual auto getObject(const std::string& bucket,
66+
const std::string& object,
67+
void* buffer,
68+
uint64_t offset,
69+
uint64_t length) const -> long long = 0;
5970

6071
virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0;
6172

@@ -67,16 +78,15 @@ class S3Client: private NonCopyable {
6778

6879
virtual auto objectSize(const std::string& bucket, const std::string& object) const -> long long = 0;
6980

70-
friend std::ostream& operator<<(std::ostream& out, const S3Client& client) {
71-
client.print(out);
72-
return out;
73-
}
74-
7581
protected: // methods
76-
explicit S3Client(const S3Config& config);
82+
S3Client();
83+
84+
explicit S3Client(S3Config config);
7785

7886
virtual void print(std::ostream& out) const;
7987

88+
friend std::ostream& operator<<(std::ostream& out, const S3Client& client);
89+
8090
private: // members
8191
S3Config config_;
8292
};

0 commit comments

Comments
 (0)