[pgmoneta#916]support listobjc2 for s3

Signed-off-by: Amr-Shams <amr.shams2015.as@gmail.com>

Author: Amr-Shams

doc/CLI.md

@@ -43,6 +43,8 @@ Commands:
                            - 'reload' to reload the configuration
                            - 'set' to modify a configuration value;
                              conf set <parameter_name> <parameter_value>;
+  s3  <action>             Manage S3, with one of subcommands:
+                           - 'ls' to get the list of files in s3
   decompress               Decompress a file using configured method
   decrypt                  Decrypt a file using master-key
   delete                   Delete a backup from a server
@@ -104,7 +106,6 @@ Example
 ```sh
 pgmoneta-cli list-backup primary
 ```
-
 ## restore
 
 Restore a backup from a server
@@ -440,7 +441,37 @@ Configuration change requires manual restart
    Requested value: 192.168.1.100 (cannot be applied to live instance)
    Status: Requires full service restart
 ```
+## s3 
+
+Manage the s3 storage 
+
+Command
 
+```sh
+pgmoneta-cli s3 [ls] <server>
+```
+
+Subcommand
+
+- `ls` : List all the files in s3
+
+Example
+
+```sh
+pgmoneta-cli s3 ls primary
+```
+### s3 ls
+
+Get the list of server files/objects in the remote storage s3
+
+- you can set the server or use the global one in the config
+
+Examples
+
+```sh
+pgmoneta-cli s3 ls primary
+pgmoneta-cli s3 ls
+```
 ## clear
 
 Clear data/statistics

doc/S3.md

@@ -92,6 +92,45 @@ s3_endpoint = <endpoint_url>
 s3_port = <port>
 s3_region = <region>
 ```
+
+### Garage Tutorial
+
+If you already have Garage downloaded and your S3 access key and secret key created, the flow is:
+
+1. Start Garage so the S3 endpoint is reachable.
+2. Configure pgmoneta to use your Garage endpoint.
+3. Start pgmoneta.
+4. Run `pgmoneta-cli s3 ls primary`.
+
+Minimal configuration example:
+
+```ini
+[pgmoneta]
+storage_engine = s3
+s3_access_key_id = <garage_access_key_id>
+s3_secret_access_key = <garage_secret_access_key>
+s3_bucket = <garage_bucket_name>
+s3_base_dir = pgmoneta
+s3_endpoint = <garage_endpoint_host>
+s3_port = <garage_endpoint_port>
+s3_region = garage
+s3_use_tls = off
+```
+
+Start pgmoneta:
+
+```sh
+pgmoneta -c pgmoneta.conf -u pgmoneta_users.conf
+```
+
+List remote objects:
+
+```sh
+pgmoneta-cli s3 ls primary
+```
+
+`pgmoneta-cli s3 ls` lists objects recursively under the configured server path and returns all objects in one result set.
+
 ## Per-Server Configuration
 
 All S3 configuration settings can be specified within a server's configuration section. This allows you to use different S3 buckets, credentials, or even regions for different PostgreSQL servers.

doc/manual/en/05-cli.md

@@ -46,6 +46,9 @@ Commands:
                            - 'reload' to reload the configuration
                            - 'set' to modify a configuration value;
                              conf set <parameter_name> <parameter_value>;
+  s3  <action>             Manage S3, with one of subcommands:
+                           - 'ls' to get the list of files in s3
+
   decompress               Decompress a file using configured method
   decrypt                  Decrypt a file using master-key
   delete                   Delete a backup from a server
@@ -474,6 +477,39 @@ pgmoneta-cli conf get host
 - On restart, pgmoneta always reads from the configuration files on disk
 - Without file updates, restart will revert to the original file-based values
 
+## s3 
+
+Manage the s3 storage 
+
+Command
+
+```sh
+pgmoneta-cli s3 [ls] <server>
+```
+
+Subcommand
+
+- `ls` : List all the files in s3
+
+Example
+
+```sh
+pgmoneta-cli s3 ls primary
+```
+
+### s3 ls
+
+Get the list of server files/objects in the remote storage s3
+
+- you can set the server or use the [pgmoneta] section in the config
+
+Examples
+
+```sh
+pgmoneta-cli s3 ls primary
+pgmoneta-cli s3 ls
+```
+
 
 ## clear
 

doc/manual/en/16-s3.md

@@ -54,3 +54,40 @@ s3_base_dir = directory-where-backups-will-be-stored-in
 ```
 
 under the `[pgmoneta]` section.
+
+## Garage tutorial 
+
+If Garage is already downloaded and configured with an S3 access key, secret key, and bucket, the flow is:
+
+1. Start Garage.
+2. Configure pgmoneta for the Garage S3 endpoint.
+3. Start pgmoneta.
+4. Run `pgmoneta-cli s3 ls primary`.
+
+Example `pgmoneta.conf` entries under `[pgmoneta]`:
+
+``` ini
+storage_engine = s3
+s3_access_key_id = your-garage-access-key-id
+s3_secret_access_key = your-garage-secret-access-key
+s3_bucket = your-garage-bucket
+s3_base_dir = pgmoneta
+s3_endpoint = your-garage-endpoint-host
+s3_port = 3900
+s3_region = garage
+s3_use_tls = off
+```
+
+Start pgmoneta:
+
+``` sh
+pgmoneta -c pgmoneta.conf -u pgmoneta_users.conf
+```
+
+List remote objects:
+
+``` sh
+pgmoneta-cli s3 ls primary
+```
+
+`pgmoneta-cli s3 ls` lists objects recursively in the configured remote path and prints all discovered objects in one output.

src/cli.c

@@ -87,6 +87,7 @@
 #define COMMAND_STATUS         "status"
 #define COMMAND_STATUS_DETAILS "status-details"
 #define COMMAND_VERIFY         "verify"
+#define COMMAND_S3             "s3"
 
 #define OUTPUT_FORMAT_JSON     "json"
 #define OUTPUT_FORMAT_TEXT     "text"
@@ -118,6 +119,7 @@ static void display_helper(char* command);
 
 static int backup(SSL* ssl, int socket, char* server, uint8_t compression, uint8_t encryption, char* incremental, int32_t output_format);
 static int list_backup(SSL* ssl, int socket, char* server, char* sort_order, uint8_t compression, uint8_t encryption, int32_t output_format);
+static int list_s3_objects(SSL* ssl, int socket, char* server, uint8_t compression, uint8_t encryption, int32_t output_format);
 static int restore(SSL* ssl, int socket, char* server, char* backup_id, char* position, char* directory, uint8_t compression, uint8_t encryption, int32_t output_format);
 static int verify(SSL* ssl, int socket, char* server, char* backup_id, char* directory, char* files, uint8_t compression, uint8_t encryption, int32_t output_format);
 static int archive(SSL* ssl, int socket, char* server, char* backup_id, char* position, char* directory, uint8_t compression, uint8_t encryption, int32_t output_format);
@@ -253,6 +255,12 @@ struct pgmoneta_command command_table[] = {
       .deprecated = false,
       .log_message = "<list-backup> [%s]",
    },
+   {.command = "s3",
+    .subcommand = "ls",
+    .accepted_argument_count = {1, 2},
+    .action = MANAGEMENT_S3_LS,
+    .deprecated = false,
+    .log_message = "<s3 ls>"},
    {
       .command = "restore",
       .subcommand = "",
@@ -894,6 +902,10 @@ main(int argc, char** argv)
    {
       exit_code = list_backup(s_ssl, socket, parsed.args[0], sort_option, compression, encryption, output_format);
    }
+   else if (parsed.cmd->action == MANAGEMENT_S3_LS)
+   {
+      exit_code = list_s3_objects(s_ssl, socket, parsed.args[0], compression, encryption, output_format);
+   }
    else if (parsed.cmd->action == MANAGEMENT_RESTORE)
    {
       if (parsed.args[3])
@@ -1092,6 +1104,12 @@ help_list_backup(void)
    printf("List backups for a server\n");
    printf("  pgmoneta-cli list-backup <server> [--sort asc|desc]\n");
 }
+static void
+help_s3(void)
+{
+   printf("Manage the s3\n");
+   printf("  pgmoneta-cli s3 [ls] <server>\n");
+}
 
 static void
 help_restore(void)
@@ -1233,6 +1251,10 @@ display_helper(char* command)
    {
       help_list_backup();
    }
+   else if (!strcmp(command, COMMAND_S3))
+   {
+      help_s3();
+   }
    else if (!strcmp(command, COMMAND_RESTORE))
    {
       help_restore();
@@ -1350,7 +1372,21 @@ list_backup(SSL* ssl, int socket, char* server, char* sort_order, uint8_t compre
 
    return 1;
 }
-
+static int
+list_s3_objects(SSL* ssl, int socket, char* server, uint8_t compression, uint8_t encryption, int32_t output_format)
+{
+   if (pgmoneta_management_request_list_s3_objects(ssl, socket, server, compression, encryption, output_format))
+   {
+      goto error;
+   }
+   if (process_result(ssl, socket, output_format))
+   {
+      goto error;
+   }
+   return 0;
+error:
+   return 1;
+}
 static int
 restore(SSL* ssl, int socket, char* server, char* backup_id, char* position, char* directory, uint8_t compression, uint8_t encryption, int32_t output_format)
 {
@@ -2498,6 +2534,11 @@ translate_command(int32_t cmd_code)
       case MANAGEMENT_LIST_BACKUP:
          command_output = pgmoneta_append(command_output, COMMAND_LIST_BACKUP);
          break;
+      case MANAGEMENT_S3_LS:
+         command_output = pgmoneta_append(command_output, COMMAND_S3);
+         command_output = pgmoneta_append_char(command_output, ' ');
+         command_output = pgmoneta_append(command_output, "ls");
+         break;
       case MANAGEMENT_RESTORE:
          command_output = pgmoneta_append(command_output, COMMAND_RESTORE);
          break;
@@ -3169,6 +3210,16 @@ translate_json_object(struct json* j)
                }
                pgmoneta_json_iterator_destroy(backup_it);
                break;
+            case MANAGEMENT_S3_LS:
+               translate_response_argument(response);
+               servers = (struct json*)pgmoneta_json_get(response, MANAGEMENT_ARGUMENT_SERVERS);
+               pgmoneta_json_iterator_create(servers, &server_it);
+               while (pgmoneta_json_iterator_next(server_it))
+               {
+                  translate_servers_argument((struct json*)pgmoneta_value_data(server_it->value));
+               }
+               pgmoneta_json_iterator_destroy(server_it);
+               break;
             case MANAGEMENT_STATUS_DETAILS:
                translate_response_argument(response);
                servers = (struct json*)pgmoneta_json_get(response, MANAGEMENT_ARGUMENT_SERVERS);

src/include/management.h

@@ -93,6 +93,8 @@ extern "C" {
 #define MANAGEMENT_REMOVE_USER    27
 #define MANAGEMENT_LIST_USERS     28
 
+#define MANAGEMENT_S3_LS          100
+
 /**
  * Management categories
  */
@@ -165,6 +167,8 @@ extern "C" {
 #define MANAGEMENT_ARGUMENT_RETENTION_MONTHS      "RetentionMonths"
 #define MANAGEMENT_ARGUMENT_RETENTION_WEEKS       "RetentionWeeks"
 #define MANAGEMENT_ARGUMENT_RETENTION_YEARS       "RetentionYears"
+#define MANAGEMENT_ARGUMENT_S3_OBJECTS            "S3Objects"
+#define MANAGEMENT_ARGUMENT_S3_KEY                "S3Key"
 #define MANAGEMENT_ARGUMENT_SERVER                "Server"
 #define MANAGEMENT_ARGUMENT_SERVERS               "Servers"
 #define MANAGEMENT_ARGUMENT_SERVER_SIZE           "ServerSize"
@@ -372,6 +376,13 @@ extern "C" {
 #define MANAGEMENT_ERROR_MODE_ERROR                         2804
 #define MANAGEMENT_ERROR_MODE_UNKNOWN_ACTION                2805
 
+#define MANAGEMENT_ERROR_LIST_S3_NOSERVER                   2900
+#define MANAGEMENT_ERROR_LIST_S3_NOFORK                     2901
+#define MANAGEMENT_ERROR_LIST_S3_WORKFLOW                   2902
+#define MANAGEMENT_ERROR_LIST_S3_JSON_VALUE                 2903
+#define MANAGEMENT_ERROR_LIST_S3_NETWORK                    2904
+#define MANAGEMENT_ERROR_LIST_S3_ERROR                      2905
+
 /**
  * Output formats
  */
@@ -450,6 +461,19 @@ pgmoneta_management_request_backup(SSL* ssl, int socket, char* server, uint8_t c
 int
 pgmoneta_management_request_list_backup(SSL* ssl, int socket, char* server, char* sort_order, uint8_t compression, uint8_t encryption, int32_t output_format);
 
+/**
+ * Create a list s3 objects request
+ * @param ssl The SSL connection
+ * @param socket The socket descriptor
+ * @param server The server
+ * @param compression The compress method for wire protocol
+ * @param encryption The encrypt method for wire protocol
+ * @param output_format The output format
+ * @return 0 upon success, otherwise 1
+ */
+int
+pgmoneta_management_request_list_s3_objects(SSL* ssl, int socket, char* server, uint8_t compression, uint8_t encryption, int32_t output_format);
+
 /**
  * Create a restore request
  * @param ssl The SSL connection