Merge pull request #6 from spatialcurrent/pipe

Compatibility with go-pipe

Author: pjdufour

.circleci/config.yml

@@ -2,14 +2,14 @@ version: 2.1
 executors:
   base:
     docker:
-      - image: circleci/golang:1.12.1
+      - image: circleci/golang:1.12
     working_directory: /go/src/github.com/spatialcurrent/go-simple-serializer
 jobs:
   pre_deps_golang:
     executor: base
     steps:
       - checkout
-      - run: go get -d ./...
+      - run: make deps
       - run: sudo chown -R circleci /go/src
       - save_cache:
           key: v1-go-src-{{ .Branch }}-{{ .Revision }}
@@ -40,7 +40,7 @@ jobs:
       - run:
           name: Download and install staticheck
           command: go get -u honnef.co/go/tools/cmd/staticcheck
-      - run: bash scripts/test.sh
+      - run: make test
   validate:
     executor: base
     steps:
@@ -68,7 +68,7 @@ jobs:
           keys:
             - v1-go-src-{{ .Branch }}-{{ .Revision }}
       - run: go get github.com/inconshreveable/mousetrap # for windows CLI builds
-      - run: bash scripts/build_cli.sh
+      - run: make build_cli
       - store_artifacts:
           path: bin
           destination: /
@@ -82,7 +82,7 @@ jobs:
       - run:
           name: Install GopherJS
           command: go get -u github.com/gopherjs/gopherjs
-      - run: bash scripts/build_javascript.sh
+      - run: make build_javascript
       - store_artifacts:
           path: bin
           destination: /
@@ -93,7 +93,7 @@ jobs:
       - restore_cache:
           keys:
             - v1-go-src-{{ .Branch }}-{{ .Revision }}
-      - run: bash scripts/build_so.sh
+      - run: make build_so
       - store_artifacts:
           path: bin
           destination: /

Makefile

@@ -0,0 +1,74 @@
+# =================================================================
+#
+# Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved
+# Released as open source under the MIT License.  See LICENSE file.
+#
+# =================================================================
+
+ifdef GOPATH
+GCFLAGS=-trimpath=$(shell printenv GOPATH)/src
+else
+GCFLAGS=-trimpath=$(shell go env GOPATH)/src
+endif
+
+LDFLAGS=-X main.gitBranch=$(shell git branch | grep \* | cut -d ' ' -f2) -X main.gitCommit=$(shell git rev-list -1 HEAD)
+
+ifndef DEST
+DEST=bin
+endif
+
+deps:
+	go get -d -t ./...
+
+fmt:
+	go fmt $$(go list ./... )
+
+vet:
+	go vet $$(go list ./...)
+
+test:
+	bash scripts/test.sh
+
+bin/gss_darwin_amd64:
+	GOOS=darwin GOARCH=amd64 go build -o $(DEST)/gss_darwin_amd64 -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/go-simple-serializer/cmd/gss
+
+bin/gss_linux_amd64:
+	GOOS=linux GOARCH=amd64 go build -o $(DEST)/gss_linux_amd64 -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/go-simple-serializer/cmd/gss
+
+bin/gss_windows_amd64.exe:
+	GOOS=windows GOARCH=amd64 go build -o $(DEST)/gss_windows_amd64.exe -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/go-simple-serializer/cmd/gss
+
+bin/gss_linux_arm64:
+	GOOS=linux GOARCH=arm64 go build -o $(DEST)/gss_linux_arm64 -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/go-simple-serializer/cmd/gss
+
+# Build Shared Object
+bin/gss.so:
+	go build -o $(DEST)/gss.so -buildmode=c-shared -ldflags "$(LDFLAGS)" -gcflags="$(GCFLAGS)" github.com/spatialcurrent/go-simple-serializer/plugins/gss
+
+# Build JavaScript Library
+bin/gss.js:
+	gopherjs build -o $(DEST)/gss.js github.com/spatialcurrent/go-simple-serializer/cmd/gss.js
+
+# Build Minified JavaScript Library
+bin/gss.min.js:
+	gopherjs build -m -o $(DEST)/gss.min.js github.com/spatialcurrent/go-simple-serializer/cmd/gss.js
+
+# Build Android Archive Library
+bin/gss.aar:
+	gomobile bind -target android -javapkg=com.spatialcurrent -o $(DEST)/gss.aar -gcflags="$(GCFLAGS)" github.com/spatialcurrent/go-simple-serializer/gss
+
+build_cli: bin/gss_darwin_amd64 bin/gss_linux_amd64 bin/gss_windows_amd64.exe bin/gss_linux_arm64
+
+build_javascript: bin/gss.js bin/gss.min.js
+
+build_android: bin/gss.arr
+
+build_so: bin/gss.so
+
+build: build_cli build_javascript build_so build_android
+
+install:
+	go install -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/go-simple-serializer/cmd/gss
+
+clean:
+	rm -fr bin

README.md

@@ -14,23 +14,38 @@ Using cross compilers, this library can also be called by other languages.  This
 
 **CLI**
 
-You can use the command line tool to convert between formats.
+You can use the command line tool to convert between serialization formats.
 
 ```
-Usage: gss -i INPUT_FORMAT -o OUTPUT_FORMAT [-h HEADER] [-c COMMENT]
-Options:
-  -c string
-    	The input comment character, e.g., #.  Commented lines are not sent to output.
-  -h string
-    	The input header if the stdin input has no header.
-  -help
-    	Print help.
-  -i string
-    	The input format: bson, csv, tsv, hcl, hcl2, json, jsonl, properties, toml, yaml
-  -o string
-    	The output format: bson, csv, tsv, hcl, hcl2, json, jsonl, properties, toml, yaml
-  -version
-    	Prints version to stdout.
+gss is a simple program for serializing/deserializing data.
+
+Usage:
+  gss [flags]
+  gss [command]
+
+Available Commands:
+  completion  Generates bash completion scripts
+  help        Help about any command
+  version     print version information to stdout
+
+Flags:
+  -a, --async                   async processing
+  -h, --help                    help for gss
+  -c, --input-comment string    The input comment character, e.g., #.  Commented lines are not sent to output.
+  -i, --input-format string     The input format: bson, csv, tsv, hcl, hcl2, json, jsonl, properties, toml, yaml
+      --input-header strings    The input header if the stdin input has no header.
+      --input-lazy-quotes       allows lazy quotes for CSV and TSV
+  -l, --input-limit int         The input limit (default -1)
+      --input-skip-lines int    The number of lines to skip before processing
+  -t, --input-trim              trim input lines
+  -o, --output-format string    The output format: bson, csv, tsv, hcl, hcl2, json, jsonl, properties, toml, yaml
+      --output-header strings   The output header if the stdout output has no header.
+  -n, --output-limit int        the output limit (default -1)
+  -p, --output-pretty           print pretty output
+  -s, --output-sorted           sort output
+      --verbose                 Print debug info to stdout
+
+Use "gss [command] --help" for more information about a command.
 ```
 
 **Go**
@@ -131,23 +146,23 @@ language=$(cat .travis.yml | ./gss_linux_amd64 -i yaml -o json -c '#' | jq .lang
 
 **CLI**
 
-The `build_cli.sh` script is used to build executables for Linux and Windows.
+The `make build_cli` script is used to build executables for Linux and Windows.
 
 **JavaScript**
 
-You can compile GSS to pure JavaScript with the `scripts/build_javascript.sh` script.
+You can compile GSS to pure JavaScript with the `make build_javascript` script.
 
 **Android**
 
-The `build_android.sh` script is used to build an [Android Archive](https://developer.android.com/studio/projects/android-library) (AAR) file and associated Javadocs.
+The `make build_android` script is used to build an [Android Archive](https://developer.android.com/studio/projects/android-library) (AAR) file and associated Javadocs.
 
 **Shared Object**
 
-The `build_so.sh` script is used to build a Shared Object (`*.go`), which can be called by `C`, `C++`, and `Python` on Linux machines.
+The `make build_so` script is used to build a Shared Object (`*.go`), which can be called by `C`, `C++`, and `Python` on Linux machines.
 
 **Changing Destination**
 
-The default destination for build artifacts is `go-simple-serializer/bin`, but you can change the destination with a CLI argument.  For building on a Chromebook consider saving the artifacts in `/usr/local/go/bin`, e.g., `bash scripts/build_cli.sh /usr/local/go/bin`
+The default destination for build artifacts is `go-simple-serializer/bin`, but you can change the destination with an environment variable.  For building on a Chromebook consider saving the artifacts in `/usr/local/go/bin`, e.g., `DEST=/usr/local/go/bin make build_cli`
 
 # Contributing
 

cmd/gss/main.go

@@ -11,6 +11,7 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"strings"
@@ -23,13 +24,52 @@ import (
 )
 
 import (
+	"github.com/spatialcurrent/go-pipe/pkg/pipe"
 	"github.com/spatialcurrent/go-simple-serializer/gss"
+	"github.com/spatialcurrent/go-simple-serializer/pkg/iterator"
+	"github.com/spatialcurrent/go-simple-serializer/pkg/jsonl"
+	"github.com/spatialcurrent/go-simple-serializer/pkg/sv"
 )
 
 var gitTag string
 var gitBranch string
 var gitCommit string
 
+func buildValueSerializer(decimal bool, noDataValue string) func(object interface{}) (string, error) {
+	if decimal {
+		return sv.DecimalValueSerializer(noDataValue)
+	}
+	return sv.DefaultValueSerializer(noDataValue)
+}
+
+func buildWriter(outputWriter io.Writer, outputFormat string, outputHeader []interface{}, outputValueSerializer func(object interface{}) (string, error)) (pipe.Writer, error) {
+	if outputFormat == "csv" || outputFormat == "tsv" {
+		separator, err := sv.FormatToSeparator(outputFormat)
+		if err != nil {
+			return nil, err
+		}
+		return sv.NewWriter(outputWriter, separator, outputHeader, outputValueSerializer), nil
+	} else if outputFormat == "jsonl" {
+		return jsonl.NewWriter(outputWriter), nil
+	}
+	return nil, fmt.Errorf("invalid format %q", outputFormat)
+}
+
+func canStream(inputFormat string, outputFormat string, outputSorted bool) bool {
+	if !outputSorted {
+		if inputFormat == "csv" || inputFormat == "tsv" {
+			if outputFormat == "csv" || outputFormat == "tsv" || outputFormat == "jsonl" {
+				return true
+			}
+		} else if inputFormat == "jsonl" {
+			if outputFormat == "jsonl" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func main() {
 	rootCommand := &cobra.Command{
 		Use:   "gss",
@@ -57,25 +97,84 @@ func main() {
 				return errors.New("output-format is required")
 			}
 
+			outputHeader := v.GetStringSlice("output-header")
+
+			outputSorted := v.GetBool("output-sorted")
+
+			outputValueSerializer := buildValueSerializer(v.GetBool("output-decimal"), v.GetString("output-no-data-value"))
+
+			if canStream(inputFormat, outputFormat, outputSorted) {
+
+				p := pipe.NewBuilder()
+
+				it, errorIterator := iterator.NewIterator(&iterator.NewIteratorInput{
+					Reader:       os.Stdin,
+					Format:       inputFormat,
+					SkipLines:    v.GetInt("input-skip-lines"),
+					SkipBlanks:   true,
+					SkipComments: true,
+					Comment:      v.GetString("input-comment"),
+					Trim:         v.GetBool("input-trim"),
+				})
+				if it == nil {
+					return errors.New(fmt.Sprintf("error building input iterator with format %q", inputFormat))
+				}
+				if errorIterator != nil {
+					return errors.Wrap(errorIterator, "error creating input iterator")
+				}
+				p = p.Input(it)
+
+				if inputLimit := v.GetInt("input-limit"); inputLimit >= 0 {
+					p = p.InputLimit(inputLimit)
+				}
+
+				outputColumns := make([]interface{}, 0, len(outputHeader))
+				for _, str := range outputHeader {
+					outputColumns = append(outputColumns, str)
+				}
+
+				w, errorWriter := buildWriter(
+					os.Stdout,
+					outputFormat,
+					outputColumns,
+					outputValueSerializer)
+				if errorWriter != nil {
+					return errors.Wrap(errorWriter, "error building output writer")
+				}
+				p = p.Output(w)
+
+				if outputLimit := v.GetInt("output-limit"); outputLimit >= 0 {
+					p = p.OutputLimit(outputLimit)
+				}
+
+				errorRun := p.Run()
+				if errorRun != nil {
+					return errors.Wrap(errorRun, "error piping data")
+				}
+				return nil
+			}
+
 			inputBytes, err := ioutil.ReadAll(os.Stdin)
 			if err != nil {
 				return errors.Wrap(err, "error reading from stdin")
 			}
 
 			outputString, err := gss.Convert(&gss.ConvertInput{
-				InputBytes:      inputBytes,
-				InputFormat:     inputFormat,
-				InputHeader:     v.GetStringSlice("input-header"),
-				InputComment:    v.GetString("input-comment"),
-				InputLazyQuotes: v.GetBool("input-lazy-quotes"),
-				InputSkipLines:  v.GetInt("input-skip-lines"),
-				InputLimit:      v.GetInt("input-limit"),
-				OutputFormat:    outputFormat,
-				OutputHeader:    v.GetStringSlice("output-header"),
-				OutputLimit:     v.GetInt("output-limit"),
-				OutputPretty:    v.GetBool("output-pretty"),
-				Async:           v.GetBool("async"),
-				Verbose:         v.GetBool("verbose"),
+				InputBytes:            inputBytes,
+				InputFormat:           inputFormat,
+				InputHeader:           v.GetStringSlice("input-header"),
+				InputComment:          v.GetString("input-comment"),
+				InputLazyQuotes:       v.GetBool("input-lazy-quotes"),
+				InputSkipLines:        v.GetInt("input-skip-lines"),
+				InputLimit:            v.GetInt("input-limit"),
+				OutputFormat:          outputFormat,
+				OutputHeader:          v.GetStringSlice("output-header"),
+				OutputLimit:           v.GetInt("output-limit"),
+				OutputPretty:          v.GetBool("output-pretty"),
+				OutputSorted:          v.GetBool("output-sorted"),
+				OutputValueSerializer: outputValueSerializer,
+				Async:                 v.GetBool("async"),
+				Verbose:               v.GetBool("verbose"),
 			})
 			if err != nil {
 				return errors.Wrap(err, "error converting")
@@ -91,14 +190,17 @@ func main() {
 	flags.Bool("input-lazy-quotes", false, "allows lazy quotes for CSV and TSV")
 	flags.Int("input-skip-lines", gss.NoSkip, "The number of lines to skip before processing")
 	flags.IntP("input-limit", "l", gss.NoLimit, "The input limit")
+	flags.BoolP("input-trim", "t", false, "trim input lines")
 	flags.StringP("output-format", "o", "", "The output format: "+strings.Join(gss.Formats, ", "))
 	flags.StringSlice("output-header", []string{}, "The output header if the stdout output has no header.")
-	flags.Int("output-limit", gss.NoLimit, "the output limit")
+	flags.IntP("output-limit", "n", gss.NoLimit, "the output limit")
 	flags.BoolP("output-pretty", "p", false, "print pretty output")
+	flags.BoolP("output-sorted", "s", false, "sort output")
+	flags.BoolP("output-decimal", "d", false, "when converting floats to strings use decimals rather than scientific notation")
+	flags.StringP("output-no-data-value", "0", "", "no data value, e.g., used for missing values when converting JSON to CSV")
 	flags.BoolP("async", "a", false, "async processing")
 	flags.Bool("verbose", false, "Print debug info to stdout")
 
-
 	completionCommandLong := ""
 	if _, err := os.Stat("/etc/bash_completion.d/"); !os.IsNotExist(err) {
 		completionCommandLong = "To install completion scripts run:\ngss completion > /etc/bash_completion.d/gss"