Use lazy reader (#5)

Author: pjdufour

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2.1
 executors:
   base:
     docker:
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.13
     working_directory: /go/src/github.com/spatialcurrent/gocat
 jobs:
   pre_deps_golang:

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Spatial Current, Inc.
+Copyright (c) 2020 Spatial Current, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -41,7 +41,7 @@ See the usage below or the following examples.
 ```shell
 gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.
 Supports stdin (aka "-"), local files (path/to/file or file://path/to/file), remote files (http://path/to/file), or files on AWS S3 (s3://path/to/file).
-Supports the following compression algorithms: bzip2, flate, gzip, none, snappy, zip, zlib
+Supports the following compression algorithms: bzip2, flate, gzip, none, snappy, zip, zlib.
 
 Usage:
   gocat [flags] [-|stdin|FILE|URI]...
@@ -92,7 +92,7 @@ The `make build_cli` script is used to build executables for Linux and Windows.
 
 **Changing Destination**
 
-The default destination for build artifacts is `go-reader-writer/bin`, but you can change the destination with an environment variable.  For building on a Chromebook consider saving the artifacts in `/usr/local/go/bin`, e.g., `DEST=/usr/local/go/bin make build_cli`
+The default destination for build artifacts is `gocat/bin`, but you can change the destination with an environment variable.  For building on a Chromebook consider saving the artifacts in `/usr/local/go/bin`, e.g., `DEST=/usr/local/go/bin make build_cli`
 
 ## Testing
 

main.go

@@ -1,6 +1,6 @@
 // =================================================================
 //
-// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved
+// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved
 // Released as open source under the MIT License.  See LICENSE file.
 //
 // =================================================================
@@ -11,19 +11,18 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	awssession "github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 
+	"github.com/spatialcurrent/go-lazy/pkg/lazy"
 	"github.com/spatialcurrent/go-reader-writer/pkg/grw"
 )
 
@@ -54,7 +53,7 @@ func initViper(cmd *cobra.Command) (*viper.Viper, error) {
 	v := viper.New()
 	err := v.BindPFlags(cmd.Flags())
 	if err != nil {
-		return v, errors.Wrap(err, "error binding flag set to viper")
+		return v, fmt.Errorf("error binding flag set to viper: %w", err)
 	}
 	v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 	v.AutomaticEnv() // set environment variables to overwrite config
@@ -76,11 +75,11 @@ func main() {
 		Short:                 "gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.",
 		Long: `gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.
 Supports stdin (aka "-"), local files (path/to/file or file://path/to/file), remote files (http://path/to/file), or files on AWS S3 (s3://path/to/file).
-Supports the following compression algorithms: ` + strings.Join(grw.Algorithms, ", "),
+Supports the following compression algorithms: ` + strings.Join(grw.Algorithms, ", ") + `.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v, err := initViper(cmd)
 			if err != nil {
-				return errors.Wrap(err, "error initializing viper")
+				return fmt.Errorf("error initializing viper: %w", err)
 			}
 
 			if len(args) == 0 {
@@ -94,14 +93,13 @@ Supports the following compression algorithms: ` + strings.Join(grw.Algorithms,
 			bufferSize := v.GetInt(flagBufferSize)
 			appendNewlines := v.GetBool(flagAppendNewlines)
 
-			stdinBytes := make([]byte, 0)
-
 			var session *awssession.Session
 
 			var s3Client *s3.S3
 
 			inputReaders := make([]io.Reader, 0)
 			for _, uri := range args {
+				uri := uri
 
 				if uri == "-" {
 					uri = "stdin"
@@ -142,28 +140,21 @@ Supports the following compression algorithms: ` + strings.Join(grw.Algorithms,
 				}
 
 				if uri == "stdin" {
-					if len(stdinBytes) == 0 {
-						b, err := ioutil.ReadAll(os.Stdin)
+					inputReaders = append(inputReaders, os.Stdin)
+				} else {
+					inputReaders = append(inputReaders, lazy.NewLazyReader(func() (io.Reader, error) {
+						r, _, err := grw.ReadFromResource(&grw.ReadFromResourceInput{
+							Uri:        uri,
+							Alg:        "none",
+							Dict:       grw.NoDict,
+							BufferSize: bufferSize,
+							S3Client:   s3Client,
+						})
 						if err != nil {
-							return errors.Wrap(err, "error reading from stdin")
+							return nil, fmt.Errorf("error reading from uri %q: %w", uri, err)
 						}
-						stdinBytes = b
-					}
-					if len(stdinBytes) > 0 {
-						inputReaders = append(inputReaders, bytes.NewReader(stdinBytes))
-					}
-				} else {
-					inputReader, _, err := grw.ReadFromResource(&grw.ReadFromResourceInput{
-						Uri:        uri,
-						Alg:        "none",
-						Dict:       grw.NoDict,
-						BufferSize: bufferSize,
-						S3Client:   s3Client,
-					})
-					if err != nil {
-						return errors.Wrap(err, fmt.Sprintf("error reading from uri %q", uri))
-					}
-					inputReaders = append(inputReaders, inputReader)
+						return r, nil
+					}))
 				}
 
 				if appendNewlines {
@@ -173,7 +164,7 @@ Supports the following compression algorithms: ` + strings.Join(grw.Algorithms,
 			}
 
 			if _, err := io.Copy(os.Stdout, io.MultiReader(inputReaders...)); err != nil {
-				return errors.Wrap(err, "error copying input")
+				return fmt.Errorf("error copying input: %w", err)
 			}
 
 			return nil