diff --git a/mtr/binlog_streaming/include/set_up_binsrv_environment.inc b/mtr/binlog_streaming/include/set_up_binsrv_environment.inc index 6f8d5e1..4688443 100644 --- a/mtr/binlog_streaming/include/set_up_binsrv_environment.inc +++ b/mtr/binlog_streaming/include/set_up_binsrv_environment.inc @@ -42,7 +42,7 @@ if ($storage_backend == s3) --let $aws_s3_bucket = $MTR_BINSRV_AWS_S3_BUCKET if ($MTR_BINSRV_AWS_S3_ENDPOINT != '') { - eval SET @storage_uri = CONCAT('http://', '$MTR_BINSRV_AWS_ACCESS_KEY_ID', ':', '$MTR_BINSRV_AWS_SECRET_ACCESS_KEY', '@', '$MTR_BINSRV_AWS_S3_ENDPOINT', '/$MTR_BINSRV_AWS_S3_BUCKET', '$binsrv_storage_path'); + eval SET @storage_uri = CONCAT('http://', '$MTR_BINSRV_AWS_ACCESS_KEY_ID', ':', REPLACE('$MTR_BINSRV_AWS_SECRET_ACCESS_KEY', '/', '%2F'), '@', '$MTR_BINSRV_AWS_S3_ENDPOINT', '/', '$MTR_BINSRV_AWS_S3_BUCKET', '$binsrv_storage_path'); } if ($MTR_BINSRV_AWS_S3_ENDPOINT == '') { @@ -51,7 +51,7 @@ if ($storage_backend == s3) { --let $qualified_bucket = $qualified_bucket.$MTR_BINSRV_AWS_S3_REGION } - eval SET @storage_uri = CONCAT('s3://', '$MTR_BINSRV_AWS_ACCESS_KEY_ID', ':', '$MTR_BINSRV_AWS_SECRET_ACCESS_KEY', '@', '$qualified_bucket', '$binsrv_storage_path'); + eval SET @storage_uri = CONCAT('s3://', '$MTR_BINSRV_AWS_ACCESS_KEY_ID', ':', REPLACE('$MTR_BINSRV_AWS_SECRET_ACCESS_KEY', '/', '%2F'), '@', '$qualified_bucket', '$binsrv_storage_path'); } } diff --git a/mtr/binlog_streaming/t/purge_binlogs.test b/mtr/binlog_streaming/t/purge_binlogs.test index a07a26d..559e972 100644 --- a/mtr/binlog_streaming/t/purge_binlogs.test +++ b/mtr/binlog_streaming/t/purge_binlogs.test @@ -2,15 +2,8 @@ --source ../include/v80_v84_compatibility_defines.inc -# identifying backend storage type ('file' or 's3') early - 'purge_binlogs' -# is supported only on the local filesystem backend --source ../include/identify_storage_backend.inc -if ($storage_backend == s3) -{ - --skip purge_binlogs is supported only on the local filesystem backend -} - # in case of --repeat=N, we need to start from a fresh binary log to make # this test deterministic --echo *** Resetting replication at the very beginning of the test. diff --git a/src/app.cpp b/src/app.cpp index c9d6f7c..c5d52b3 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -1058,13 +1058,6 @@ bool handle_purge_binlogs(std::string_view config_file_path, const auto &replication_config = config.root().get<"replication">(); const auto replication_mode{replication_config.get<"mode">()}; - // for now, only file backend supported - if (storage_config.get<"backend">() != binsrv::storage_backend_type::file) { - throw std::runtime_error( - "purge_binlogs is only supported on the local filesystem storage " - "backend"); - } - binsrv::storage storage{storage_config, binsrv::storage_construction_mode_type::purging, replication_mode}; diff --git a/src/binsrv/s3_storage_backend.cpp b/src/binsrv/s3_storage_backend.cpp index 87a6977..4227b4f 100644 --- a/src/binsrv/s3_storage_backend.cpp +++ b/src/binsrv/s3_storage_backend.cpp @@ -53,6 +53,7 @@ #include #include +#include #include #include #include @@ -171,6 +172,8 @@ class s3_storage_backend::aws_context : private aws_context_base { void put_object_from_span(const qualified_object_path &dest, util::const_byte_span content) const; + void delete_object(const qualified_object_path &target) const; + [[nodiscard]] storage_object_name_container list_objects(const qualified_object_path &prefix); @@ -346,6 +349,23 @@ void s3_storage_backend::aws_context::put_object_from_span( put_object_from_stream(dest, content_stream); } +// TODO: Consider deleting several objects in one request by using +// `DeleteObjects()` AWS SDK C++ API call +void s3_storage_backend::aws_context::delete_object( + const qualified_object_path &target) const { + Aws::S3Crt::Model::DeleteObjectRequest delete_object_request; + delete_object_request.SetBucket(target.bucket); + delete_object_request.SetKey(target.object_path.generic_string()); + + const auto delete_object_outcome{ + client_->DeleteObject(delete_object_request)}; + + if (!delete_object_outcome.IsSuccess()) { + raise_s3_error_from_outcome("cannot delete object from S3 bucket", + delete_object_outcome.GetError()); + } +} + [[nodiscard]] storage_object_name_container s3_storage_backend::aws_context::list_objects( const qualified_object_path &prefix) { @@ -679,10 +699,9 @@ void s3_storage_backend::do_put_object(std::string_view name, {.bucket = bucket_, .object_path = get_object_path(name)}, content); } -void s3_storage_backend::do_remove_object( - [[maybe_unused]] std::string_view name) { - util::exception_location().raise( - "remove_object is not supported on the S3 storage backend"); +void s3_storage_backend::do_remove_object(std::string_view name) { + impl_->delete_object( + {.bucket = bucket_, .object_path = get_object_path(name)}); } void s3_storage_backend::do_fsync() { diff --git a/src/binsrv/storage.cpp b/src/binsrv/storage.cpp index 6f03a37..48128b9 100644 --- a/src/binsrv/storage.cpp +++ b/src/binsrv/storage.cpp @@ -33,7 +33,6 @@ #include "binsrv/binlog_file_metadata.hpp" #include "binsrv/replication_mode_type.hpp" #include "binsrv/storage_backend_factory.hpp" -#include "binsrv/storage_backend_type.hpp" #include "binsrv/storage_config.hpp" #include "binsrv/storage_metadata.hpp" @@ -54,13 +53,6 @@ storage::storage(const storage_config &config, replication_mode_type replication_mode) : construction_mode_{construction_mode}, backend_{}, replication_mode_{replication_mode} { - if (construction_mode_ == storage_construction_mode_type::purging && - config.get<"backend">() != storage_backend_type::file) { - util::exception_location().raise( - "purge_binlogs is only supported on the local filesystem storage " - "backend"); - } - const auto &checkpoint_size_opt{config.get<"checkpoint_size">()}; if (checkpoint_size_opt.has_value()) { checkpoint_size_bytes_ = checkpoint_size_opt->get_value();