feat: download object as json and csv (#17)

Author: hgiasac

Dockerfile

@@ -18,9 +18,9 @@ RUN mkdir /data && chmod 755 -R /data
 FROM gcr.io/distroless/static-debian12:nonroot
 
 # Copy the binary to the production image from the builder stage.
+COPY --from=builder /etc/mime.types /etc/mime.types
 COPY --from=builder /app/ndc-cli /ndc-cli
 COPY --from=builder --chown=65532:65532 /data /home/nonroot/data
-
 ENV HASURA_CONFIGURATION_DIRECTORY=/etc/connector
 
 ENTRYPOINT ["/ndc-cli"]

compose.yaml

@@ -96,7 +96,7 @@ services:
 
   # https://github.com/fsouza/fake-gcs-server
   gcp-storage-emulator:
-    image: fsouza/fake-gcs-server:1.52.1
+    image: fsouza/fake-gcs-server:1.52.2
     command:
       [
         "-public-host",

connector/functions/object.go

@@ -2,13 +2,15 @@ package functions
 
 import (
 	"context"
+	"fmt"
 	"io"
 
 	"github.com/hasura/ndc-sdk-go/scalar"
 	"github.com/hasura/ndc-sdk-go/schema"
 	"github.com/hasura/ndc-sdk-go/utils"
 	"github.com/hasura/ndc-storage/connector/collection"
 	"github.com/hasura/ndc-storage/connector/storage/common"
+	"github.com/hasura/ndc-storage/connector/storage/common/encoding"
 	"github.com/hasura/ndc-storage/connector/types"
 )
 
@@ -104,7 +106,7 @@ func FunctionStorageObject(ctx context.Context, state *types.State, args *common
 
 // FunctionDownloadStorageObjectAsBase64 returns a stream of the object data. Most of the common errors occur when reading the stream.
 func FunctionDownloadStorageObjectAsBase64(ctx context.Context, state *types.State, args *common.GetStorageObjectArguments) (*DownloadStorageObjectResponse, error) {
-	reader, err := downloadStorageObject(ctx, state, args)
+	_, reader, err := downloadStorageObject(ctx, state, args)
 	if err != nil {
 		return nil, err
 	}
@@ -127,7 +129,7 @@ func FunctionDownloadStorageObjectAsBase64(ctx context.Context, state *types.Sta
 
 // FunctionDownloadStorageObjectAsText returns the object content in plain text. Use this function only if you know exactly the file as an text file.
 func FunctionDownloadStorageObjectAsText(ctx context.Context, state *types.State, args *common.GetStorageObjectArguments) (*DownloadStorageObjectTextResponse, error) {
-	reader, err := downloadStorageObject(ctx, state, args)
+	_, reader, err := downloadStorageObject(ctx, state, args)
 	if err != nil {
 		return nil, err
 	}
@@ -148,17 +150,89 @@ func FunctionDownloadStorageObjectAsText(ctx context.Context, state *types.State
 	return &DownloadStorageObjectTextResponse{Data: dataStr}, nil
 }
 
-func downloadStorageObject(ctx context.Context, state *types.State, args *common.GetStorageObjectArguments) (io.ReadCloser, error) {
+// FunctionDownloadStorageObjectAsJson returns the object content in arbitrary json. Returns error if the content is unable to be decoded.
+func FunctionDownloadStorageObjectAsJson(ctx context.Context, state *types.State, args *common.GetStorageObjectArguments) (*DownloadStorageObjectJsonResponse, error) {
+	stat, reader, err := downloadStorageObject(ctx, state, args)
+	if err != nil {
+		return nil, err
+	}
+
+	if reader == nil {
+		return nil, nil
+	}
+
+	defer reader.Close()
+
+	var contentType string
+	if stat.ContentType != nil {
+		contentType = *stat.ContentType
+	}
+
+	data, err := encoding.DecodeArbitraryData(ctx, stat.Name, contentType, reader)
+	if err != nil {
+		return nil, schema.UnprocessableContentError(err.Error(), nil)
+	}
+
+	return &DownloadStorageObjectJsonResponse{Data: data}, nil
+}
+
+// FunctionDownloadStorageObjectAsCsv downloads and decode the object content from CSV. Returns error if the content is unable to be decoded.
+func FunctionDownloadStorageObjectAsCsv(ctx context.Context, state *types.State, args *common.DownloadStorageObjectAsCsvArguments) (*DownloadStorageObjectJsonResponse, error) {
+	getArgs := args.GetStorageObjectArguments
+	getArgs.PreValidate = func(so *common.StorageObject) error {
+		var contentType string
+		if so.ContentType != nil {
+			contentType = *so.ContentType
+		}
+
+		if !encoding.IsValidCSVObject(so.Name, contentType) {
+			return fmt.Errorf("failed to decode file %s to csv, unsupported content type %s", so.Name, contentType)
+		}
+
+		return nil
+	}
+
+	stat, reader, err := downloadStorageObject(ctx, state, &args.GetStorageObjectArguments)
+	if err != nil {
+		return nil, err
+	}
+
+	if reader == nil {
+		return nil, nil
+	}
+
+	defer reader.Close()
+
+	decodeOptions := args.Options
+
+	if decodeOptions.Delimiter == "" {
+		var contentType string
+		if stat.ContentType != nil {
+			contentType = *stat.ContentType
+		}
+
+		decodeOptions.Delimiter = encoding.CSVCommaFromContentType(stat.Name, contentType)
+	}
+
+	data, err := encoding.DecodeCSV(ctx, reader, decodeOptions)
+	if err != nil {
+		return nil, schema.UnprocessableContentError(err.Error(), nil)
+	}
+
+	return &DownloadStorageObjectJsonResponse{Data: data}, nil
+}
+
+func downloadStorageObject(ctx context.Context, state *types.State, args *common.GetStorageObjectArguments) (*common.StorageObject, io.ReadCloser, error) {
 	request, err := collection.EvalObjectPredicate(args.StorageBucketArguments, &collection.StringComparisonOperator{
 		Value:    args.Name,
 		Operator: collection.OperatorEqual,
 	}, args.Where, types.QueryVariablesFromContext(ctx))
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if !request.IsValid {
-		return nil, nil
+		return nil, nil, nil
 	}
 
 	return state.Storage.GetObject(ctx, request.GetBucketArguments(), request.ObjectNamePredicate.GetPrefix(), args.GetStorageObjectOptions)

connector/functions/types.generated.go

@@ -45,6 +45,11 @@ type DownloadStorageObjectTextResponse struct {
 	Data string `json:"data"`
 }
 
+// DownloadStorageObjectJsonResponse represents the object data response in arbitrary JSON format.
+type DownloadStorageObjectJsonResponse struct {
+	Data any `json:"data"`
+}
+
 // PutStorageObjectArguments represents input arguments of the PutObject method.
 type PutStorageObjectBase64Arguments struct {
 	common.PutStorageObjectArguments