Use multireader (#2)

use multireader

Author: pjdufour

.circleci/config.yml

@@ -9,77 +9,40 @@ jobs:
     executor: base
     steps:
       - checkout
-      - run: go get -d ./...
-      - run: go get github.com/inconshreveable/mousetrap # for windows CLI builds
+      - run: make deps_go
       - run: sudo chown -R circleci /go/src
       - save_cache:
           key: v1-go-src-{{ .Branch }}-{{ .Revision }}
           paths:
             - /go/src
-  test:
-    executor: base
-    steps:
-      - run: sudo chown -R circleci /go/src
-      - restore_cache:
-          keys:
-            - v1-go-src-{{ .Branch }}-{{ .Revision }}
-      - run:
-          name: Get shadow
-          command: go get golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
-      - run:
-          name: Install shadow
-          command: go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
-      - run:
-          name: Download and install errcheck
-          command: go get -u github.com/kisielk/errcheck
-      - run:
-          name: Download and install misspell
-          command: go get -u github.com/client9/misspell/cmd/misspell
-      - run:
-          name: Download and install ineffassign
-          command: go get -u github.com/gordonklaus/ineffassign
-      - run:
-          name: Download and install staticheck
-          command: go get -u honnef.co/go/tools/cmd/staticcheck
-      - run: bash scripts/test.sh
-  validate:
+  test_go:
     executor: base
     steps:
       - run: sudo chown -R circleci /go/src
       - restore_cache:
           keys:
             - v1-go-src-{{ .Branch }}-{{ .Revision }}
-      - run:
-          name: Install Dig
-          command: sudo apt update && sudo apt install dnsutils
-      - run:
-          name: "Update ~/.ssh/known_hosts"
-          command: |
-            mkdir ~/.ssh/
-            touch ~/.ssh/known_hosts
-            for ip in $(dig @8.8.8.8 github.com +short); do ssh-keyscan github.com,$ip; ssh-keyscan $ip; done 2>/dev/null >> ~/.ssh/known_hosts
-      - run: go get github.com/spatialcurrent/go-header/...
-      - run: go install github.com/spatialcurrent/go-header/cmd/goheader
-      - run: goheader fix --fix-year 2019 --exit-code-on-changes 1 --verbose
+      - run: make deps_go_test
+      - run: make test_go
+      - run: make imports
+      - run: git diff --exit-code
   build_cli:
     executor: base
     steps:
       - run: sudo chown -R circleci /go/src
       - restore_cache:
           keys:
             - v1-go-src-{{ .Branch }}-{{ .Revision }}
-      - run: make build
+      - run: go get github.com/inconshreveable/mousetrap # for windows CLI builds
+      - run: make build_cli
       - store_artifacts:
           path: bin
           destination: /
 workflows:
   main:
     jobs:
       - pre_deps_golang
-      - test:
-          requires:
-            - pre_deps_golang
-      - validate:
+      - test_go:
           requires:
             - pre_deps_golang
       - build_cli:

Makefile

@@ -4,32 +4,88 @@
 # Released as open source under the MIT License.  See LICENSE file.
 #
 # =================================================================
-bin/gocat_darwin_amd64:
-	GOOS=darwin GOARCH=amd64 go build -o bin/gocat_darwin_amd64 $$(go list ./...)
 
-bin/gocat_linux_amd64:
-	GOOS=linux GOARCH=amd64 go build -o bin/gocat_linux_amd64 $$(go list ./...)
+ifdef GOPATH
+GCFLAGS=-trimpath=$(shell printenv GOPATH)/src
+else
+GCFLAGS=-trimpath=$(shell go env GOPATH)/src
+endif
 
-bin/gocat_windows_amd64.exe:
-	GOOS=windows GOARCH=amd64 go build -o bin/gocat_windows_amd64.exe $$(go list ./...)
+LDFLAGS=-X main.gitBranch=$(shell git branch | grep \* | cut -d ' ' -f2) -X main.gitCommit=$(shell git rev-list -1 HEAD)
 
-bin/gocat_linux_arm64:
-	GOOS=linux GOARCH=arm64 go build -o bin/gocat_linux_arm64 $$(go list ./...)
+ifndef DEST
+DEST=bin
+endif
 
-build: \
-bin/gocat_darwin_amd64 \
-bin/gocat_linux_amd64 \
-bin/gocat_windows_amd64.exe \
-bin/gocat_linux_arm64
+.PHONY: help
 
-fmt:
+help:  ## Print the help documentation
+	@grep -E '^[a-zA-Z0-9_-\]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+#
+# Dependencies
+#
+
+deps_go:  ## Install Go dependencies
+	go get -d -t ./...
+
+.PHONY: deps_go_test
+deps_go_test: ## Download Go dependencies for tests
+	go get golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow # download shadow
+	go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow # install shadow
+	go get -u github.com/kisielk/errcheck # download and install errcheck
+	go get -u github.com/client9/misspell/cmd/misspell # download and install misspell
+	go get -u github.com/gordonklaus/ineffassign # download and install ineffassign
+	go get -u honnef.co/go/tools/cmd/staticcheck # download and instal staticcheck
+	go get -u golang.org/x/tools/cmd/goimports # download and install goimports
+
+deps_arm:  ## Install dependencies to cross-compile to ARM
+	# ARMv7
+	apt-get install -y libc6-armel-cross libc6-dev-armel-cross binutils-arm-linux-gnueabi libncurses5-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi
+  # ARMv8
+	apt-get install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+
+#
+# Go building, formatting, testing, and installing
+#
+
+fmt:  ## Format Go source code
 	go fmt $$(go list ./... )
 
-install:
-	go install $$(go list ./...)
+imports: ## Update imports in Go source code
+	# If missing, install goimports with: go get golang.org/x/tools/cmd/goimports
+	goimports -w $$(find . -iname '*.go')
+
+vet: ## Vet Go source code
+	go vet $$(go list ./...)
 
-test:
+test_go: ## Run Go tests
 	bash scripts/test.sh
 
-clean:
+build: build_cli build_javascript build_so build_android  ## Build CLI, Shared Objects (.so), JavaScript, and Android
+
+install:  ## Install gocat CLI on current platform
+	go install -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/gocat
+
+#
+# Command line Programs
+#
+
+bin/gocat_darwin_amd64: ## Build gocat CLI for Darwin / amd64
+	GOOS=darwin GOARCH=amd64 go build -o $(DEST)/gocat_darwin_amd64 -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/gocat
+
+bin/gocat_linux_amd64: ## Build gocat CLI for Linux / amd64
+	GOOS=linux GOARCH=amd64 go build -o $(DEST)/gocat_linux_amd64 -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/gocat
+
+bin/gocat_windows_amd64.exe:  ## Build gocat CLI for Windows / amd64
+	GOOS=windows GOARCH=amd64 go build -o $(DEST)/gocat_windows_amd64.exe -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/gocat
+
+bin/gocat_linux_arm64: ## Build gocat CLI for Linux / arm64
+	GOOS=linux GOARCH=arm64 go build -o $(DEST)/gocat_linux_arm64 -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/gocat
+
+build_cli: bin/gocat_darwin_amd64 bin/gocat_linux_amd64 bin/gocat_windows_amd64.exe bin/gocat_linux_arm64  ## Build command line programs
+
+## Clean
+
+clean:  ## Clean artifacts
 	rm -fr bin

README.md

@@ -23,7 +23,7 @@ If you do have go already installed, you can just run using `go run main.go` or
 
 See the usage below or the following examples.
 
-```
+```shell
 gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.  Supports stdin (aka "-"), local files (path/to/file or file://path/to/file), remote files (http://path/to/file), or files on AWS S3 (s3://path/to/file).
 
 Usage:

main.go

@@ -8,7 +8,10 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"os"
 	"strings"
 )
@@ -22,7 +25,6 @@ import (
 
 import (
 	"github.com/aws/aws-sdk-go/aws"
-	//"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	awssession "github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -52,7 +54,7 @@ func initFlags(flag *pflag.FlagSet) {
 	flag.StringP(flagAWSSecretAccessKey, "", "", "AWS Secret Access Key")
 	flag.StringP(flagAWSSessionToken, "", "", "AWS Session Token")
 	flag.IntP(flagBufferSize, "b", 4096, "buffer size for file reader")
-	flag.BoolP(flagAppendNewlines, "a", false, "append new lines to files that do not end in new lines characters")
+	flag.BoolP(flagAppendNewlines, "a", false, "append new lines to files")
 }
 
 func initViper(cmd *cobra.Command) (*viper.Viper, error) {
@@ -76,9 +78,10 @@ func checkConfig(v *viper.Viper) error {
 
 func main() {
 	cmd := &cobra.Command{
-		Use:   "gocat [-|stdin|FILE|URI]...",
-		Short: "gocat",
-		Long:  `gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.  Supports stdin (aka "-"), local files (path/to/file or file://path/to/file), remote files (http://path/to/file), or files on AWS S3 (s3://path/to/file).`,
+		Use:                   "gocat [flags] [-|stdin|FILE|URI]...",
+		Short:                 "gocat",
+		DisableFlagsInUseLine: true,
+		Long:                  `gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.  Supports stdin (aka "-"), local files (path/to/file or file://path/to/file), remote files (http://path/to/file), or files on AWS S3 (s3://path/to/file).`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v, err := initViper(cmd)
 			if err != nil {
@@ -94,13 +97,15 @@ func main() {
 			}
 
 			bufferSize := v.GetInt(flagBufferSize)
+			appendNewlines := v.GetBool(flagAppendNewlines)
 
 			stdinBytes := make([]byte, 0)
 
 			var session *awssession.Session
 
 			var s3Client *s3.S3
 
+			inputReaders := make([]io.Reader, 0)
 			for _, uri := range args {
 
 				if uri == "-" {
@@ -141,49 +146,35 @@ func main() {
 					}
 				}
 
-				inputBytes := make([]byte, 0)
-
-				// if not reading from stdin or stdin hasn't been read yet.
-				if uri != "stdin" || len(stdinBytes) == 0 {
-					inputReader, _, inputError := grw.ReadFromResource(uri, "none", bufferSize, false, s3Client)
-					if inputError != nil {
-						return errors.Wrap(inputError, fmt.Sprintf("error reading from uri %q", uri))
+				if uri == "stdin" {
+					if len(stdinBytes) == 0 {
+						b, err := ioutil.ReadAll(os.Stdin)
+						if err != nil {
+							return errors.Wrap(err, "error reading from stdin")
+						}
+						stdinBytes = b
 					}
-					b, inputError := inputReader.ReadAllAndClose()
-					if inputError != nil {
-						return errors.Wrap(inputError, fmt.Sprintf("error reading from uri %q", uri))
+					if len(stdinBytes) > 0 {
+						inputReaders = append(inputReaders, bytes.NewReader(stdinBytes))
 					}
-					inputBytes = b
-				}
-
-				// if reading from stdin and stdin bytes are already cached
-				if uri == "stdin" && len(inputBytes) == 0 && len(stdinBytes) > 0 {
-					inputBytes = stdinBytes
-				}
-
-				if len(inputBytes) > 0 {
-
-					_, err = os.Stdout.Write(inputBytes)
+				} else {
+					inputReader, _, err := grw.ReadFromResource(uri, "none", bufferSize, false, s3Client)
 					if err != nil {
-						return errors.Wrap(err, fmt.Sprintf("error writing bytes from uri %q", uri))
-					}
-
-					if v.GetBool(flagAppendNewlines) {
-						if inputBytes[len(inputBytes)-1] != '\n' {
-							_, err := os.Stdout.Write([]byte("\n"))
-							if err != nil {
-								return errors.Wrap(err, fmt.Sprintf("error writing new line from uri %q", uri))
-							}
-						}
-					}
-
-					if uri == "stdin" {
-						stdinBytes = inputBytes
+						return errors.Wrap(err, fmt.Sprintf("error reading from uri %q", uri))
 					}
+					inputReaders = append(inputReaders, inputReader)
+				}
 
+				if appendNewlines {
+					inputReaders = append(inputReaders, bytes.NewReader([]byte("\n")))
 				}
 
 			}
+
+			if _, err := io.Copy(os.Stdout, io.MultiReader(inputReaders...)); err != nil {
+				return errors.Wrap(err, "error copying input")
+			}
+
 			return nil
 		},
 	}