0.0.1 release

Author: pjdufour

.gitignore

@@ -1,3 +1,5 @@
 bin
+vendor
+Gopkg.lock
 *.so
 *.h

.travis.yml

@@ -8,15 +8,31 @@ go:
 
 os:
   - linux
-  - osx
+  #- osx
 
 matrix:
   allow_failures:
     - go: tip
   fast_finish: true
 
+before_install:
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then curl -L -s https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64 -o $GOPATH/bin/dep; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then wget https://github.com/golang/dep/releases/download/v0.5.0/dep-darwin-amd64 -O $GOPATH/bin/dep; fi
+  - chmod +x $GOPATH/bin/dep
+
+install:
+  - dep ensure
+  - go get -u github.com/alecthomas/gometalinter
+  - gometalinter --install
+
 script:
-  - cd railgun
+  - gometalinter --misspell-locale=US --disable-all --enable=misspell --enable=vet ./railgun/
+  #- gometalinter --misspell-locale=US --disable-all --enable=misspell --enable=vet ./plugins/railgun/
+  - gometalinter --misspell-locale=US --disable-all --enable=misspell --enable=vet ./cmd/railgun/
+  - gometalinter --misspell-locale=US --disable-all --enable=misspell --enable=vet ./cmd/railgun.js/
+  - cd gss
   - go fmt
   - cd ./../cmd/railgun
   - go fmt
+  - cd ./../../cmd/railgun.js
+  - go fmt

Gopkg.toml

@@ -0,0 +1,3 @@
+[[constraint]]
+  name = "github.com/colinmarc/hdfs"
+  revision = "279cc88949e23da3364c94a9ee440f5beee3ca65"

README.md

@@ -1,82 +1,101 @@
-[![Build Status](https://travis-ci.org/spatialcurrent/railgun.svg)](https://travis-ci.org/spatialcurrent/railgun) [![GoDoc](https://godoc.org/github.com/spatialcurrent/railgun?status.svg)](https://godoc.org/github.com/spatialcurrent/railgun)
+[![Build Status](https://travis-ci.org/spatialcurrent/railgun.svg)](https://travis-ci.org/spatialcurrent/railgun) [![Go Report Card](https://goreportcard.com/badge/spatialcurrent/railgun)](https://goreportcard.com/report/spatialcurrent/railgun)  [![GoDoc](https://godoc.org/github.com/spatialcurrent/railgun?status.svg)](https://godoc.org/github.com/spatialcurrent/railgun) [![license](http://img.shields.io/badge/license-MIT-red.svg?style=flat)](https://github.com/spatialcurrent/railgun/blob/master/LICENSE)
 
 # Railgun
 
 # Description
 
-**Railgun** is a simple and fast data processing tool.  **Railgun** uses [go-simple-serializer](https://github.com/spatialcurrent/go-simple-serializer) (GSS) for reading/writing objects to standard formats.  **Railgun** uses [go-dfl](https://github.com/spatialcurrent/go-dfl) for filtering.
+**Railgun** is a simple and fast data processing tool.  **Railgun** uses:
+- [go-reader](https://github.com/spatialcurrent/go-reader) for opening and reading from URIs,
+- [go-simple-serializer](https://github.com/spatialcurrent/go-simple-serializer) (GSS) for reading/writing objects to standard formats, and
+- [go-dfl](https://github.com/spatialcurrent/go-dfl) for filtering and transforming data.
 
-GSS supports `bson`, `csv`, `tsv`, `hcl`, `hcl2`, `json`, `jsonl`, `properties`, `toml`, `yaml`.  `hcl` and `hcl2` implementation is fragile and very much in `alpha`.
+go-reader can read from `stdin`, `http/https`, the local filesystem, [AWS S3](https://aws.amazon.com/s3/), and [HDFS](https://hortonworks.com/apache/hdfs/).
+
+go-simple-serializer (GSS) supports `bson`, `csv`, `tsv`, `hcl`, `hcl2`, `json`, `jsonl`, `properties`, `toml`, `yaml`.  `hcl` and `hcl2` implementation is fragile and very much in `alpha`.
 
 # Usage
 
 **CLI**
 
-You can use the command line tool to convert between formats.
+You can use the command line tool to process data.
 
 ```
-Usage: railgun -input_format INPUT_FORMAT -o OUTPUT_FORMAT [-input_uri INPUT_URI] [-input_compression [bzip2|gzip|snappy]] [-h HEADER] [-c COMMENT] [-object_path PATH] [-f FILTER] [-output_path OUTPUT_PATH] [-max MAX_COUNT]
+Usage: railgun -input_format INPUT_FORMAT -o OUTPUT_FORMAT [-input_uri INPUT_URI] [-input_compression [bzip2|gzip|snappy]] [-h HEADER] [-c COMMENT] [-object_path PATH] [-dfl_exp DFL_EXPRESSION] [-dfl_file DFL_FILE] [-output_path OUTPUT_PATH] [-max MAX_COUNT]
 Options:
+  -aws_access_key_id string
+    	Defaults to value of environment variable AWS_ACCESS_KEY_ID
+  -aws_default_region string
+    	Defaults to value of environment variable AWS_DEFAULT_REGION.
+  -aws_secret_access_key string
+    	Defaults to value of environment variable AWS_SECRET_ACCESS_KEY.
+  -aws_session_token string
+    	Defaults to value of environment variable AWS_SESSION_TOKEN.
   -c string
     	The input comment character, e.g., #.  Commented lines are not sent to output.
-  -f string
-    	The output filter
+  -dfl_exp string
+    	Process using dfl expression
+  -dfl_file string
+    	Process using dfl file.
   -h string
     	The input header if the stdin input has no header.
+  -hdfs_name_node string
+    	Defaults to value of environment variable HDFS_DEFAULT_NAME_NODE.
   -help
     	Print help.
   -input_compression string
-    	The input compression: none, gzip, snappy (default "none")
+    	The input compression: none, bzip2, gzip, snappy (default "none")
   -input_format string
-    	The input format: csv, tsv, hcl, hcl2, json, jsonl, properties, toml, yaml
+    	The input format: bson, csv, tsv, hcl, hcl2, json, jsonl, properties, toml, yaml
+  -input_reader_buffer_size int
+    	The input reader buffer size (default 4096)
   -input_uri string
     	The input uri (default "stdin")
   -max int
     	The maximum number of objects to output (default -1)
-  -o string
-    	The output format: csv, tsv, hcl, hcl2, json, jsonl, properties, toml, yaml
-  -object_path string
-    	The output path
-  -output_path string
-    	The output path
+  -output_format string
+    	The output format: bson, csv, tsv, hcl, hcl2, json, jsonl, properties, toml, yaml
+  -output_uri string
+    	The output uri (default "stdout")
   -version
     	Prints version to stdout.
 ```
 
-**Go**
+# Releases
 
-You can import **railgun** as a library with:
+**Railgun** is currently in **alpha**.  See releases at https://github.com/spatialcurrent/railgun/releases.
 
-```go
-import (
-  "github.com/spatialcurrent/go-railgun/railgun"
-)
-```
+# Examples
 
-The `Process` function is the core functions to use.
+**Search for Cuisine**
 
-```go
-...
-  output_object, err := railgun.Process(input_object, object_path, filter, funcs, max_count, output_path)
-...
-  output_string, err := gss.Serialize(output_object, output_format)
-...
+```
+~/go/src/github.com/spatialcurrent/go-osm/bin/osm_linux_amd64 -input_uri 'http://download.geofabrik.de/north-america/us/district-of-columbia-latest.osm.bz2' -ways_to_nodes -output_format geojsonl -filter_keys_keep amenity -output_uri stdout | railgun -input_format jsonl  -output_format json -dfl_file ~/go/src/github.com/spatialcurrent/railgun/examples/mexican.dfl -output_uri mexican.json
 ```
 
-# Releases
-
-**Railgun** is currently in **alpha**.  See releases at https://github.com/spatialcurrent/railgun/releases.
-
-# Examples
+**Tsunami Feed**
 
-TBD
+```
+const pipeline = ["filter(@features, '(@properties?.tsunami != null) and (@properties.tsunami == 1)')", "sort(@, '@properties?.mag', true)", "map(@, '@properties?.place ?: \"\"')", "limit(@, 10)"];
+(await fetch("https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/2.5_month.geojson")).json().then(earthquakes => {
+  result = railgun.process(earthquakes, {"dfl": pipeline, "output_format": "yaml"});
+  console.log(result);
+})
+```
 
 # Building
 
 **CLI**
 
 The `build_cli.sh` script is used to build executables for Linux and Windows.
 
+**JavaScript**
+
+You can compile GSS to pure JavaScript with the `scripts/build_javascript.sh` script.
+
+**Changing Destination**
+
+The default destination for build artifacts is `railgun/bin`, but you can change the destination with a CLI argument.  For building on a Chromebook consider saving the artifacts in `/usr/local/go/bin`, e.g., `bash scripts/build_cli.sh /usr/local/go/bin`
+
 # Contributing
 
 [Spatial Current, Inc.](https://spatialcurrent.io) is currently accepting pull requests for this repository.  We'd love to have your contributions!  Please see [Contributing.md](https://github.com/spatialcurrent/railgun/blob/master/CONTRIBUTING.md) for how to get started.

cmd/railgun.js/main.go

@@ -0,0 +1,170 @@
+// =================================================================
+//
+// Copyright (C) 2018 Spatial Current, Inc. - All Rights Reserved
+// Released as open source under the MIT License.  See LICENSE file.
+//
+// =================================================================
+
+// railgun.js is the Javascript version of Railgun.
+//
+package main
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+)
+
+import (
+	"github.com/pkg/errors"
+)
+
+import (
+	"github.com/spatialcurrent/go-dfl/dfl"
+	"github.com/spatialcurrent/go-simple-serializer/gss"
+	"github.com/spatialcurrent/railgun/railgun"
+)
+
+import (
+	"github.com/gopherjs/gopherjs/js"
+	"honnef.co/go/js/console"
+)
+
+var GO_RAILGUN_COMPRESSION_ALGORITHMS = []string{"none", "gzip", "snappy"}
+var GO_RAILGUN_FORMATS = []string{"csv", "tsv", "hcl", "hcl2", "json", "jsonl", "properties", "toml", "yaml"}
+
+func main() {
+	js.Global.Set("railgun", map[string]interface{}{
+		"version": railgun.VERSION,
+		"process": Process,
+	})
+}
+
+func Process(in interface{}, options *js.Object) interface{} {
+
+	// Convert Javascript options object into Golang map
+	m := map[string]interface{}{}
+	for _, key := range js.Keys(options) {
+		m[key] = options.Get(key).Interface()
+	}
+
+	input_header := []string{}
+	input_comment := ""
+	input_format := ""
+	output_format := ""
+	dfl_exp := ""
+
+	if v, ok := m["header"]; ok {
+		switch v.(type) {
+		case []string:
+			input_header = v.([]string)
+		case []interface{}:
+			input_header = make([]string, 0, len(v.([]interface{})))
+			for _, h := range v.([]interface{}) {
+				input_header = append(input_header, fmt.Sprint(h))
+			}
+		}
+	}
+
+	if v, ok := m["input_comment"]; ok {
+		switch v.(type) {
+		case string:
+			input_comment = v.(string)
+		}
+	}
+
+	if v, ok := m["input_format"]; ok {
+		switch v.(type) {
+		case string:
+			input_format = v.(string)
+		}
+	}
+
+	if v, ok := m["output_format"]; ok {
+		switch v.(type) {
+		case string:
+			output_format = v.(string)
+		}
+	}
+
+	var ctx interface{}
+
+	switch in.(type) {
+	case string:
+		input_object, err := gss.NewObject(in.(string), input_format)
+		if err != nil {
+			console.Log(errors.Wrap(err, "error creating new object for format "+input_format))
+			return ""
+		}
+		switch input_object_typed := input_object.(type) {
+		case []map[string]interface{}:
+			err = gss.Deserialize(in.(string), input_format, input_header, input_comment, &input_object_typed)
+			if err != nil {
+				console.Log(errors.Wrap(err, "error deserializing input using format "+input_format).Error())
+				return ""
+			}
+			ctx = input_object_typed // This is a critical line, otherwise the type information is lost.
+		default:
+			err = gss.Deserialize(in.(string), input_format, input_header, input_comment, &input_object)
+			if err != nil {
+				console.Log(errors.Wrap(err, "error deserializing input using format "+input_format).Error())
+				return ""
+			}
+			ctx = input_object
+		}
+	case *js.Object:
+		ctx = in.(*js.Object).Interface()
+	case map[string]interface{}:
+		ctx = in.(map[string]interface{})
+	default:
+		console.Log("Unknown input type", fmt.Sprint(reflect.TypeOf(in)))
+	}
+
+	if v, ok := m["dfl"]; ok {
+		switch v.(type) {
+		case string:
+			dfl_exp = v.(string)
+		case []interface{}:
+			arr := dfl.TryConvertArray(v.([]interface{}))
+			switch arr.(type) {
+			case []string:
+				dfl_exp = strings.Join(arr.([]string), " | ")
+			}
+		}
+	}
+
+	var dfl_node dfl.Node
+	if len(dfl_exp) > 0 {
+		n, err := dfl.Parse(dfl_exp)
+		if err != nil {
+			console.Log(errors.Wrap(err, "Error parsing dfl node.").Error())
+			return ""
+		}
+		dfl_node = n.Compile()
+	}
+
+	var output interface{}
+	if dfl_node != nil {
+		o, err := dfl_node.Evaluate(ctx, dfl.NewFuntionMapWithDefaults())
+		if err != nil {
+			console.Log(errors.Wrap(err, "error processing").Error())
+			return ""
+		}
+		output = o
+	} else {
+		output = ctx
+	}
+
+	output = gss.StringifyMapKeys(output)
+
+	if len(output_format) > 0 {
+		output_string, err := gss.Serialize(output, output_format)
+		if err != nil {
+			console.Log(errors.Wrap(err, "Error converting to output format "+output_format).Error())
+			return ""
+		}
+		return output_string
+	}
+
+	return output
+}