WIP: Feature: Add S3 Storage Support for Stream Queues

extras/lavinmq.ini

@@ -28,3 +28,12 @@ bind = ::
 ;port = 5679
 ;advertised_uri = tcp://hostname.local:5679
 ;etcd_endpoints = localhost:2379
+
+[s3-storage]
+;enabled = false
+;region = eu-north-1
+; Use virtual-hosted-style URL with bucket in hostname:
+;endpoint = https://my-bucket.s3.eu-north-1.amazonaws.com
+;access_key_id = AKIAIOSFODNN7EXAMPLE
+;secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+;local_segments_per_stream = 50

shard.lock

@@ -12,6 +12,10 @@ shards:
     git: https://github.com/cloudamqp/amqp-client.cr.git
     version: 1.3.2
 
+  awscr-signer:
+    git: https://github.com/taylorfinnell/awscr-signer.git
+    version: 0.9.0
+
   lz4:
     git: https://github.com/84codes/lz4.cr.git
     version: 1.0.0+git.commit.96d714f7593c66ca7425872fd26c7b1286806d3d

shard.yml

@@ -34,6 +34,8 @@ dependencies:
     github: 84codes/lz4.cr
   mqtt-protocol:
     github: 84codes/mqtt-protocol.cr
+  awscr-signer:
+    github: taylorfinnell/awscr-signer
 
 development_dependencies:
   ameba:

spec/config_spec.cr

@@ -163,6 +163,14 @@ describe LavinMQ::Config do
           advertised_uri = lavinmq://localhost:5680
           on_leader_elected = echo "Leader elected"
           on_leader_lost = echo "Leader lost"
+
+          [s3-storage]
+          enabled = true
+          region = us-east-1
+          access_key_id = AKIAIOSFODNN7EXAMPLE
+          secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+          endpoint = https://s3.example.com
+          local_segments_per_stream = 100
         CONFIG
       end
       config = LavinMQ::Config.new
@@ -245,6 +253,14 @@ describe LavinMQ::Config do
       config.clustering_advertised_uri.should eq "lavinmq://localhost:5680"
       config.clustering_on_leader_elected.should eq "echo \"Leader elected\""
       config.clustering_on_leader_lost.should eq "echo \"Leader lost\""
+
+      # S3 Storage section
+      config.streams_s3_storage?.should be_true
+      config.streams_s3_storage_region.should eq "us-east-1"
+      config.streams_s3_storage_access_key_id.should eq "AKIAIOSFODNN7EXAMPLE"
+      config.streams_s3_storage_secret_access_key.should eq "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+      config.streams_s3_storage_endpoint.should eq "https://s3.example.com"
+      config.streams_s3_storage_local_segments_per_stream.should eq 100
     ensure
       # Reset log level to default for other specs
       Log.setup(:fatal)
@@ -284,6 +300,12 @@ describe LavinMQ::Config do
       "--clustering-etcd-prefix=cli-prefix",
       "--clustering-max-unsynced-actions=4096",
       "--clustering-port=5680",
+      "--s3-storage=true",
+      "--s3-storage-region=eu-west-1",
+      "--s3-storage-access-key-id=AKIACLIEXAMPLE",
+      "--s3-storage-secret-access-key=cliSecretKey123",
+      "--s3-storage-endpoint=https://s3.cli.example.com",
+      "--s3-storage-local-segments=75",
     ]
     config.parse(argv)
 
@@ -317,6 +339,12 @@ describe LavinMQ::Config do
     config.clustering_etcd_prefix.should eq "cli-prefix"
     config.clustering_max_unsynced_actions.should eq 4096
     config.clustering_port.should eq 5680
+    config.streams_s3_storage?.should be_true
+    config.streams_s3_storage_region.should eq "eu-west-1"
+    config.streams_s3_storage_access_key_id.should eq "AKIACLIEXAMPLE"
+    config.streams_s3_storage_secret_access_key.should eq "cliSecretKey123"
+    config.streams_s3_storage_endpoint.should eq "https://s3.cli.example.com"
+    config.streams_s3_storage_local_segments_per_stream.should eq 75
   end
 
   it "can parse -d/--debug flag for verbose logging" do