update GRW; cli tests (#4)

Author: pjdufour

.circleci/config.yml

@@ -26,6 +26,16 @@ jobs:
       - run: make test_go
       - run: make imports
       - run: git diff --exit-code
+  test_cli:
+    executor: base
+    steps:
+      - run: sudo chown -R circleci /go/src
+      - restore_cache:
+          keys:
+            - v1-go-src-{{ .Branch }}-{{ .Revision }}
+      - run: make deps_go
+      - run: make install
+      - run: make test_cli
   build_cli:
     executor: base
     steps:
@@ -48,3 +58,6 @@ workflows:
       - build_cli:
           requires:
             - pre_deps_golang
+      - test_cli:
+          requires:
+            - pre_deps_golang

Makefile

@@ -52,16 +52,23 @@ deps_arm:  ## Install dependencies to cross-compile to ARM
 fmt:  ## Format Go source code
 	go fmt $$(go list ./... )
 
+.PHONY: imports
 imports: ## Update imports in Go source code
 	# If missing, install goimports with: go get golang.org/x/tools/cmd/goimports
-	goimports -w $$(find . -iname '*.go')
+	goimports -w -local github.com/spatialcurrent/gocat,github.com/spatialcurrent $$(find . -iname '*.go')
 
 vet: ## Vet Go source code
 	go vet $$(go list ./...)
 
+.PHONY: test_go
 test_go: ## Run Go tests
 	bash scripts/test.sh
 
+.PHONY: test_cli
+test_cli: ## Run CLI tests
+	bash scripts/test-cli.sh
+
+.PHONY: build
 build: build_cli build_javascript build_so build_android  ## Build CLI, Shared Objects (.so), JavaScript, and Android
 
 install:  ## Install gocat CLI on current platform
@@ -83,6 +90,7 @@ bin/gocat_windows_amd64.exe:  ## Build gocat CLI for Windows / amd64
 bin/gocat_linux_arm64: ## Build gocat CLI for Linux / arm64
 	GOOS=linux GOARCH=arm64 go build -o $(DEST)/gocat_linux_arm64 -gcflags="$(GCFLAGS)" -ldflags="$(LDFLAGS)" github.com/spatialcurrent/gocat
 
+.PHONY: build_cli
 build_cli: bin/gocat_darwin_amd64 bin/gocat_linux_amd64 bin/gocat_windows_amd64.exe bin/gocat_linux_arm64  ## Build command line programs
 
 ## Clean

README.md

@@ -6,25 +6,42 @@
 
 **gocat** is a super simple command line program for concatenating files.  **gocat** supports the following operating systems and architectures.
 
+## Platforms
+
+The following platforms are supported.  Pull requests to support other platforms are welcome!
+
 | GOOS | GOARCH |
 | ---- | ------ |
 | darwin | amd64 |
 | linux | amd64 |
 | windows | amd64 |
 | linux | arm64 |
 
-# Installation
+## Releases
+
+Find releases at [https://github.com/spatialcurrent/gocat/releases](https://github.com/spatialcurrent/gocat/releases).  You might want to rename your binary to just `gocat` (or `cat`) for convenience.  See the **Building** section below to build from scratch.
+
+**Darwin**
+
+- `gocat_darwin_amd64` - CLI for Darwin on amd64 (includes `macOS` and `iOS` platforms)
 
-No installation is required.  Just grab a [release](https://github.com/spatialcurrent/gocat/releases).  You might want to rename your binary to just `gocat` (or `cat`) for convenience.
+**Linux**
 
-If you do have go already installed, you can just run using `go run main.go` or install with `make install`
+- `gocat_linux_amd64` - CLI for Linux on amd64
+- `gocat_linux_amd64` - CLI for Linux on arm64
 
-# Usage
+**Windows**
+
+- `gocat_windows_amd64.exe` - CLI for Windows on amd64
+
+## Usage
 
 See the usage below or the following examples.
 
 ```shell
-gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.  Supports stdin (aka "-"), local files (path/to/file or file://path/to/file), remote files (http://path/to/file), or files on AWS S3 (s3://path/to/file).
+gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.
+Supports stdin (aka "-"), local files (path/to/file or file://path/to/file), remote files (http://path/to/file), or files on AWS S3 (s3://path/to/file).
+Supports the following compression algorithms: bzip2, flate, gzip, none, snappy, zip, zlib
 
 Usage:
   gocat [flags] [-|stdin|FILE|URI]...
@@ -41,42 +58,60 @@ Flags:
   -h, --help                           help for gocat
 ```
 
-# Examples
+## Examples
 
-**Shell**
+### Shell
 
 ```shell
 gocat "~/.bash_aliases" "~/.bashrc"
 ```
 
-**Remote File**
+### Remote File
 
 `gocat` arguments can point to remote files
 
 ```shell
 gocat https://spatialcurrent.io
 ```
 
-**All Files With Extension**
+### All Files With Extension
 
 `gocat` will print every file given as a positional argument, which can be useful when looking through thousands of files.
 
 ```shell
 gocat $( find . -print | grep -i '.*[.]md')
 ```
 
-# Building
+## Building
+
+Use `make help` to see help information for each target.
+
+**CLI**
 
-You can build all the released artifacts using `make build` or run the make target for a specific operating system and architecture.
+The `make build_cli` script is used to build executables for Linux and Windows.  Use `make install` for standard installation as a go executable.
+
+**Changing Destination**
+
+The default destination for build artifacts is `go-reader-writer/bin`, but you can change the destination with an environment variable.  For building on a Chromebook consider saving the artifacts in `/usr/local/go/bin`, e.g., `DEST=/usr/local/go/bin make build_cli`
+
+## Testing
+
+**CLI**
+
+To run CLI testes use `make test_cli`, which uses [shUnit2](https://github.com/kward/shunit2).  If you recive a `shunit2:FATAL Please declare TMPDIR with path on partition with exec permission.` error, you can modify the `TMPDIR` environment variable in line or with `export TMPDIR=<YOUR TEMP DIRECTORY HERE>`. For example:
+
+```
+TMPDIR="/usr/local/tmp" make test_cli
+```
 
-# Testing
+**Go**
 
-Run test using `bash scripts/test.sh` or `make test`, which runs unit tests, `go vet`, `go vet with shadow`, [errcheck](https://github.com/kisielk/errcheck), [ineffassign](https://github.com/gordonklaus/ineffassign), [staticcheck](https://staticcheck.io/), and [misspell](https://github.com/client9/misspell).
+To run Go tests use `make test_go` (or `bash scripts/test.sh`), which runs unit tests, `go vet`, `go vet with shadow`, [errcheck](https://github.com/kisielk/errcheck), [ineffassign](https://github.com/gordonklaus/ineffassign), [staticcheck](https://staticcheck.io/), and [misspell](https://github.com/client9/misspell).
 
-# Contributing
+## Contributing
 
 [Spatial Current, Inc.](https://spatialcurrent.io) is currently accepting pull requests for this repository.  We'd love to have your contributions!  Please see [Contributing.md](https://github.com/spatialcurrent/gocat/blob/master/CONTRIBUTING.md) for how to get started.
 
-# License
+## License
 
 This work is distributed under the **MIT License**.  See **LICENSE** file.

main.go

@@ -14,24 +14,17 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
-)
-
-import (
-	"github.com/pkg/errors"
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-	"github.com/spf13/viper"
-)
 
-import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	awssession "github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
-)
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
 
-import (
-	"github.com/spatialcurrent/go-reader-writer/grw"
+	"github.com/spatialcurrent/go-reader-writer/pkg/grw"
 )
 
 const (
@@ -78,10 +71,12 @@ func checkConfig(v *viper.Viper) error {
 
 func main() {
 	cmd := &cobra.Command{
-		Use:                   "gocat [flags] [-|stdin|FILE|URI]...",
-		Short:                 "gocat",
+		Use:                   `gocat [flags] [-|stdin|FILE|URI]...`,
 		DisableFlagsInUseLine: true,
-		Long:                  `gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.  Supports stdin (aka "-"), local files (path/to/file or file://path/to/file), remote files (http://path/to/file), or files on AWS S3 (s3://path/to/file).`,
+		Short:                 "gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.",
+		Long: `gocat is a super simple utility to concatenate files (local, remote, or on AWS S3) provided as positional arguments.
+Supports stdin (aka "-"), local files (path/to/file or file://path/to/file), remote files (http://path/to/file), or files on AWS S3 (s3://path/to/file).
+Supports the following compression algorithms: ` + strings.Join(grw.Algorithms, ", "),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			v, err := initViper(cmd)
 			if err != nil {
@@ -158,7 +153,13 @@ func main() {
 						inputReaders = append(inputReaders, bytes.NewReader(stdinBytes))
 					}
 				} else {
-					inputReader, _, err := grw.ReadFromResource(uri, "none", bufferSize, false, s3Client)
+					inputReader, _, err := grw.ReadFromResource(&grw.ReadFromResourceInput{
+						Uri:        uri,
+						Alg:        "none",
+						Dict:       grw.NoDict,
+						BufferSize: bufferSize,
+						S3Client:   s3Client,
+					})
 					if err != nil {
 						return errors.Wrap(err, fmt.Sprintf("error reading from uri %q", uri))
 					}
@@ -181,6 +182,8 @@ func main() {
 	initFlags(cmd.Flags())
 
 	if err := cmd.Execute(); err != nil {
-		panic(err)
+		fmt.Fprintln(os.Stderr, "gocat: "+err.Error())
+		fmt.Fprintln(os.Stderr, "Try gocat --help for more information.")
+		os.Exit(1)
 	}
 }