Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add CsvExporter and JsonExporter #647

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions export/csv_exporter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package export

import (
"context"
"fmt"
"strings"

"github.com/turbot/pipe-fittings/v2/constants"
"github.com/turbot/pipe-fittings/v2/querydisplay"
"github.com/turbot/pipe-fittings/v2/queryresult"
)

type CsvExporter struct {
ExporterBase
}

// Export processes the query result and writes CSV output to a file
func (e *CsvExporter) Export(ctx context.Context, input ExportSourceData, filePath string) error {
result, ok := input.(*queryresult.Result[*queryresult.QueryTimingMetadata])
if !ok {
return fmt.Errorf("CsvExporter input must be a queryresult.Result")
}

// Generate csv output
jsonString, _, _ := querydisplay.BuildCSV(ctx, result)

// Write to file
return Write(filePath, strings.NewReader(jsonString))
}

func (e *CsvExporter) FileExtension() string {
return constants.CsvExtension
}

func (e *CsvExporter) Name() string {
return constants.OutputFormatCSV
}
37 changes: 37 additions & 0 deletions export/json_exporter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package export

import (
"context"
"fmt"
"strings"

"github.com/turbot/pipe-fittings/v2/constants"
"github.com/turbot/pipe-fittings/v2/querydisplay"
"github.com/turbot/pipe-fittings/v2/queryresult"
)

type JsonExporter struct {
ExporterBase
}

// Export processes the query result and writes JSON output to a file
func (e *JsonExporter) Export(ctx context.Context, input ExportSourceData, filePath string) error {
result, ok := input.(*queryresult.Result[*queryresult.QueryTimingMetadata])
if !ok {
return fmt.Errorf("JsonExporter input must be a queryresult.Result")
}

// Generate JSON output
jsonString, _, _ := querydisplay.BuildJSON(ctx, result)

// Write to file
return Write(filePath, strings.NewReader(jsonString))
}

func (e *JsonExporter) FileExtension() string {
return constants.JsonExtension
}

func (e *JsonExporter) Name() string {
return constants.OutputFormatJSON
}
38 changes: 26 additions & 12 deletions querydisplay/display.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,13 @@ func newJSONOutput() *jsonOutput {
}
}

func displayJSON[T queryresult.TimingContainer](ctx context.Context, result *queryresult.Result[T]) (rowCount, rowErrors int) {
func displayJSON[T queryresult.TimingContainer](ctx context.Context, result *queryresult.Result[T]) (int, int) {
jsonOutput, rowCount, rowErrors := BuildJSON(ctx, result)
fmt.Println(jsonOutput) //nolint:forbidigo // intentional use of fmt
return rowCount, rowErrors
}

func BuildJSON[T queryresult.TimingContainer](ctx context.Context, result *queryresult.Result[T]) (op string, rowCount, rowErrors int) {
jsonOutput := newJSONOutput()

// add column defs to the JSON output
Expand Down Expand Up @@ -244,29 +250,37 @@ func displayJSON[T queryresult.TimingContainer](ctx context.Context, result *que
if err != nil {
error_helpers.ShowError(ctx, err)
rowErrors++
return 0, rowErrors
return "", 0, rowErrors
}

// now we have iterated the rows, get the timing
if viper.IsSet(constants.ArgTiming) {
jsonOutput.Metadata = result.Timing.GetTiming()
}

// display the JSON
encoder := json.NewEncoder(os.Stdout)
// Encode the JSON output
var buf bytes.Buffer
encoder := json.NewEncoder(&buf)
encoder.SetIndent("", " ")
encoder.SetEscapeHTML(false)
if err := encoder.Encode(jsonOutput); err != nil {
//nolint:forbidigo // acceptable
fmt.Print("Error displaying result as JSON", err)
return 0, 0
fmt.Print("Error building JSON result", err)
return "", 0, 0
}
return count, rowErrors
return buf.String(), count, rowErrors
}

func displayCSV[T queryresult.TimingContainer](ctx context.Context, result *queryresult.Result[T]) (rowCount, rowErrors int) {
func displayCSV[T queryresult.TimingContainer](ctx context.Context, result *queryresult.Result[T]) (int, int) {
csvOutput, rowCount, rowErrors := BuildCSV(ctx, result)
fmt.Print(csvOutput) //nolint:forbidigo // Print to stdout
return rowCount, rowErrors
}

func BuildCSV[T queryresult.TimingContainer](ctx context.Context, result *queryresult.Result[T]) (op string, rowCount, rowErrors int) {

csvWriter := csv.NewWriter(os.Stdout)
var buf bytes.Buffer
csvWriter := csv.NewWriter(&buf)
csvWriter.Comma = []rune(viper.GetString(pconstants.ArgSeparator))[0]

if viper.GetBool(constants.ArgHeader) {
Expand All @@ -285,15 +299,15 @@ func displayCSV[T queryresult.TimingContainer](ctx context.Context, result *quer
if err != nil {
error_helpers.ShowError(ctx, err)
rowErrors++
return 0, rowErrors
return "", 0, rowErrors
}

csvWriter.Flush()
if csvWriter.Error() != nil {
error_helpers.ShowErrorWithMessage(ctx, csvWriter.Error(), "unable to print csv")
error_helpers.ShowErrorWithMessage(ctx, csvWriter.Error(), "unable to build csv")
}

return count, rowErrors
return buf.String(), count, rowErrors
}

func displayLine[T queryresult.TimingContainer](ctx context.Context, result *queryresult.Result[T]) (rowCount, rowErrors int) {
Expand Down
16 changes: 13 additions & 3 deletions queryresult/result.go
Original file line number Diff line number Diff line change
@@ -1,17 +1,27 @@
package queryresult

import (
"time"
)
import "time"

type RowResult struct {
Data []interface{}
Error error
}

type TimingMetadata struct {
Duration time.Duration
}

type QueryTimingMetadata struct {
RowsReturned int `json:"rows_returned"`
Duration string `json:"duration_ms"`
}

// GetTiming implements TimingContainer - we implement this interface
// to allow QueryTimingMetadata to be used to parameterize the ResultStreamer
func (t QueryTimingMetadata) GetTiming() any {
return t
}

// TimingContainer is an interface that allows us to parameterize the Result struct
// it must handle case of a query returning a stream of timing data OR a timing data struct directly
// the GetTiming func is used to populate the timing data in the JSON output (as we cannot serialise a stream!)
Expand Down