S3 Storage - Fargate Credentials

README.md

@@ -184,6 +184,12 @@ export IMMUDB_S3_ENDPOINT="https://${IMMUDB_S3_BUCKET_NAME}.s3.${IMMUDB_S3_LOCAT
 ./immudb
 ```
 
+If using Fargate, the credentials URL can be sourced automatically:
+
+```bash
+export IMMUDB_S3_USE_FARGATE_CREDENTIALS=true
+```
+
 Optionally, you can specify the exact role immudb should be using with:
 
 ```bash

cmd/immudb/command/init.go

@@ -89,6 +89,7 @@ func (cl *Commandline) setupFlags(cmd *cobra.Command, options *server.Options) {
 	cmd.Flags().String("s3-path-prefix", "", "s3 path prefix (multiple immudb instances can share the same bucket if they have different prefixes)")
 	cmd.Flags().Bool("s3-external-identifier", false, "use the remote identifier if there is no local identifier")
 	cmd.Flags().String("s3-instance-metadata-url", "http://169.254.169.254", "s3 instance metadata url")
+	cmd.Flags().String("s3-use-fargate-credentials", "false", "use fargate credentials for s3 authentication: true/false")
 	cmd.Flags().Int("max-sessions", 100, "maximum number of simultaneously opened sessions")
 	cmd.Flags().Duration("max-session-inactivity-time", 3*time.Minute, "max session inactivity time is a duration after which an active session is declared inactive by the server. A session is kept active if server is still receiving requests from client (keep-alive or other methods)")
 	cmd.Flags().Duration("max-session-age-time", 0, "the current default value is infinity. max session age time is a duration after which session will be forcibly closed")

cmd/immudb/command/parse_options.go

@@ -106,6 +106,7 @@ func parseOptions() (options *server.Options, err error) {
 	s3PathPrefix := viper.GetString("s3-path-prefix")
 	s3ExternalIdentifier := viper.GetBool("s3-external-identifier")
 	s3MetadataURL := viper.GetString("s3-instance-metadata-url")
+	s3UseFargateCredentials := viper.GetBool("s3-use-fargate-credentials")
 
 	remoteStorageOptions := server.DefaultRemoteStorageOptions().
 		WithS3Storage(s3Storage).
@@ -118,7 +119,8 @@ func parseOptions() (options *server.Options, err error) {
 		WithS3Location(s3Location).
 		WithS3PathPrefix(s3PathPrefix).
 		WithS3ExternalIdentifier(s3ExternalIdentifier).
-		WithS3InstanceMetadataURL(s3MetadataURL)
+		WithS3InstanceMetadataURL(s3MetadataURL).
+		WithS3UseFargateCredentials(s3UseFargateCredentials)
 
 	sessionOptions := sessions.DefaultOptions().
 		WithMaxSessions(viper.GetInt("max-sessions")).

embedded/remotestorage/s3/s3.go

@@ -56,6 +56,8 @@ type Storage struct {
 
 	awsInstanceMetadataURL string
 	awsCredsRefreshPeriod  time.Duration
+
+	useFargateCredentials bool
 }
 
 var (
@@ -99,6 +101,7 @@ func Open(
 	location string,
 	prefix string,
 	awsInstanceMetadataURL string,
+	useFargateCredentials bool,
 ) (remotestorage.Storage, error) {
 
 	// Endpoint must always end with '/'
@@ -137,6 +140,7 @@ func Open(
 		},
 		awsInstanceMetadataURL: awsInstanceMetadataURL,
 		awsCredsRefreshPeriod:  time.Minute,
+		useFargateCredentials:     useFargateCredentials,
 	}
 
 	err := s3storage.getRoleCredentials()
@@ -807,6 +811,45 @@ func (s *Storage) getRoleCredentials() error {
 }
 
 func (s *Storage) requestCredentials() (string, string, string, error) {
+	if s.useFargateCredentials {
+		// Use Fargate credentials
+
+		const fargateMetadataEndpoint = "http://169.254.170.2"
+
+		fargateCredentialsRelativeURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")
+		if fargateCredentialsRelativeURI == "" {
+			return "", "", "", errors.New("environment variable AWS_CONTAINER_CREDENTIALS_RELATIVE_URI is not set or empty")
+		}
+		fargateCredentialsURL := fargateMetadataEndpoint + fargateCredentialsRelativeURI
+
+		fargateReq, err := http.NewRequest("GET", fargateCredentialsURL, nil)
+		if err != nil {
+			return "", "", "", errors.New("cannot form fargate credentials request")
+		}
+
+		fargateResp, err := http.DefaultClient.Do(fargateReq)
+		if err != nil {
+			return "", "", "", errors.New("cannot get fargate credentials")
+		}
+		defer fargateResp.Body.Close()
+
+		creds, err := io.ReadAll(fargateResp.Body)
+		if err != nil {
+			return "", "", "", errors.New("cannot read fargate credentials")
+		}
+
+		var credentials struct {
+			AccessKeyID     string `json:"AccessKeyId"`
+			SecretAccessKey string `json:"SecretAccessKey"`
+			SessionToken    string `json:"Token"`
+		}
+		if err := json.Unmarshal(creds, &credentials); err != nil {
+			return "", "", "", errors.New("cannot parse fargate credentials")
+		}
+
+		return credentials.AccessKeyID, credentials.SecretAccessKey, credentials.SessionToken, nil
+	}
+
 	tokenReq, err := http.NewRequest("PUT", fmt.Sprintf("%s%s",
 		s.awsInstanceMetadataURL,
 		"/latest/api/token",

embedded/remotestorage/s3/s3_test.go

@@ -41,6 +41,7 @@ func TestOpen(t *testing.T) {
 		"",
 		"prefix",
 		"",
+		false,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, s)
@@ -90,6 +91,7 @@ func TestCornerCases(t *testing.T) {
 			"",
 			"",
 			"",
+			false,
 		)
 		require.ErrorIs(t, err, ErrInvalidArguments)
 		require.ErrorIs(t, err, ErrInvalidArgumentsBucketEmpty)
@@ -107,6 +109,7 @@ func TestCornerCases(t *testing.T) {
 			"",
 			"",
 			"",
+			false,
 		)
 		require.ErrorIs(t, err, ErrInvalidArguments)
 		require.ErrorIs(t, err, ErrInvalidArgumentsBucketSlash)
@@ -124,6 +127,7 @@ func TestCornerCases(t *testing.T) {
 			"",
 			"",
 			"",
+			false,
 		)
 		require.NoError(t, err)
 		require.Equal(t, "", s.(*Storage).prefix)
@@ -138,6 +142,7 @@ func TestCornerCases(t *testing.T) {
 			"",
 			"/test/",
 			"",
+			false,
 		)
 		require.NoError(t, err)
 		require.Equal(t, "test/", s.(*Storage).prefix)
@@ -152,6 +157,7 @@ func TestCornerCases(t *testing.T) {
 			"",
 			"/test",
 			"",
+			false,
 		)
 		require.NoError(t, err)
 		require.Equal(t, "test/", s.(*Storage).prefix)
@@ -168,6 +174,7 @@ func TestCornerCases(t *testing.T) {
 			"",
 			"",
 			"",
+			false,
 		)
 		require.NoError(t, err)
 		require.Equal(t, "s3(misconfigured)", s.String())
@@ -184,6 +191,7 @@ func TestCornerCases(t *testing.T) {
 			"",
 			"",
 			"",
+			false,
 		)
 		require.NoError(t, err)
 
@@ -211,6 +219,7 @@ func TestCornerCases(t *testing.T) {
 			"",
 			"",
 			"",
+			false,
 		)
 		require.NoError(t, err)
 
@@ -230,6 +239,7 @@ func TestCornerCases(t *testing.T) {
 			"",
 			"",
 			"",
+			false,
 		)
 		require.Error(t, err)
 		require.Nil(t, s)
@@ -246,6 +256,7 @@ func TestCornerCases(t *testing.T) {
 			"",
 			"",
 			"",
+			false,
 		)
 		require.NoError(t, err)
 
@@ -262,7 +273,7 @@ func TestCornerCases(t *testing.T) {
 		}))
 		defer ts.Close()
 
-		s, err := Open(ts.URL, false, "", "", "", "bucket", "", "", "")
+		s, err := Open(ts.URL, false, "", "", "", "bucket", "", "", "", false)
 		require.NoError(t, err)
 
 		ctx := context.Background()
@@ -277,7 +288,7 @@ func TestCornerCases(t *testing.T) {
 		}))
 		defer ts.Close()
 
-		s, err := Open(ts.URL, false, "", "", "", "bucket", "", "", "")
+		s, err := Open(ts.URL, false, "", "", "", "bucket", "", "", "", false)
 		require.NoError(t, err)
 
 		ctx := context.Background()
@@ -300,6 +311,7 @@ func TestSignatureV4(t *testing.T) {
 		"us-east-1",
 		"",
 		"",
+		false,
 	)
 	require.NoError(t, err)
 
@@ -378,7 +390,7 @@ func TestHandlingRedirects(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	s, err := Open(ts.URL, false, "", "", "", "bucket", "", "", "")
+	s, err := Open(ts.URL, false, "", "", "", "bucket", "", "", "", false)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -761,7 +773,7 @@ func TestListEntries(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	s, err := Open(ts.URL, false, "", "", "", "bucket", "", "", "")
+	s, err := Open(ts.URL, false, "", "", "", "bucket", "", "", "", false)
 	require.NoError(t, err)
 
 	ctx := context.Background()

embedded/remotestorage/s3/s3_with_minio_test.go

@@ -45,6 +45,7 @@ func TestS3WithServer(t *testing.T) {
 		"",
 		fmt.Sprintf("prefix_%x", randomBytes),
 		"",
+		false,
 	)
 	require.NoError(t, err)
 

pkg/server/options.go

@@ -87,17 +87,18 @@ type Options struct {
 }
 
 type RemoteStorageOptions struct {
-	S3Storage             bool
-	S3RoleEnabled         bool
-	S3Role                string
-	S3Endpoint            string
-	S3AccessKeyID         string
-	S3SecretKey           string `json:"-"`
-	S3BucketName          string
-	S3Location            string
-	S3PathPrefix          string
-	S3ExternalIdentifier  bool
-	S3InstanceMetadataURL string
+	S3Storage               bool
+	S3RoleEnabled           bool
+	S3Role                  string
+	S3Endpoint              string
+	S3AccessKeyID           string
+	S3SecretKey             string `json:"-"`
+	S3BucketName            string
+	S3Location              string
+	S3PathPrefix            string
+	S3ExternalIdentifier    bool
+	S3InstanceMetadataURL   string
+	S3UseFargateCredentials bool
 }
 
 type ReplicationOptions struct {
@@ -370,7 +371,11 @@ func (o *Options) String() string {
 		opts = append(opts, "S3 storage")
 		if o.RemoteStorageOptions.S3RoleEnabled {
 			opts = append(opts, rightPad("   role auth", o.RemoteStorageOptions.S3RoleEnabled))
-			opts = append(opts, rightPad("   role name", o.RemoteStorageOptions.S3Role))
+			if o.RemoteStorageOptions.S3UseFargateCredentials {
+				opts = append(opts, rightPad("   fargate creds", o.RemoteStorageOptions.S3UseFargateCredentials))
+			} else {
+				opts = append(opts, rightPad("   role name", o.RemoteStorageOptions.S3Role))
+			}
 		}
 		opts = append(opts, rightPad("   endpoint", o.RemoteStorageOptions.S3Endpoint))
 		opts = append(opts, rightPad("   bucket name", o.RemoteStorageOptions.S3BucketName))
@@ -379,7 +384,9 @@ func (o *Options) String() string {
 		}
 		opts = append(opts, rightPad("   prefix", o.RemoteStorageOptions.S3PathPrefix))
 		opts = append(opts, rightPad("   external id", o.RemoteStorageOptions.S3ExternalIdentifier))
-		opts = append(opts, rightPad("   metadata url", o.RemoteStorageOptions.S3InstanceMetadataURL))
+		if !o.RemoteStorageOptions.S3UseFargateCredentials {
+		  opts = append(opts, rightPad("   metadata url", o.RemoteStorageOptions.S3InstanceMetadataURL))
+		}
 	}
 	if o.AdminPassword == auth.SysAdminPassword {
 		opts = append(opts, "----------------------------------------")
@@ -599,6 +606,11 @@ func (opts *RemoteStorageOptions) WithS3InstanceMetadataURL(url string) *RemoteS
 	return opts
 }
 
+func (opts *RemoteStorageOptions) WithS3UseFargateCredentials(s3UseFargateCredentials bool) *RemoteStorageOptions {
+	opts.S3UseFargateCredentials = s3UseFargateCredentials
+	return opts
+}
+
 // ReplicationOptions
 
 func (opts *ReplicationOptions) WithIsReplica(isReplica bool) *ReplicationOptions {

pkg/server/remote_storage.go

@@ -60,6 +60,7 @@ func (s *ImmuServer) createRemoteStorageInstance() (remotestorage.Storage, error
 			s.Options.RemoteStorageOptions.S3Location,
 			s.Options.RemoteStorageOptions.S3PathPrefix,
 			s.Options.RemoteStorageOptions.S3InstanceMetadataURL,
+			s.Options.RemoteStorageOptions.S3UseFargateCredentials,
 		)
 	}