0.0.4 release

Author: pjdufour

README.md

@@ -9,10 +9,14 @@
 - [go-simple-serializer](https://github.com/spatialcurrent/go-simple-serializer) (GSS) for reading/writing objects to standard formats, and
 - [go-dfl](https://github.com/spatialcurrent/go-dfl) for filtering and transforming data.
 
+**Railgun** uses the **Dynamic Filter Language** through **go-dfl**.  See the `*_test` files in the [dfl](https://github.com/spatialcurrent/go-dfl/tree/master/dfl) source folder on GitHub for comprehensive examples of the syntax.
+
 go-reader can read from `stdin`, `http/https`, the local filesystem, [AWS S3](https://aws.amazon.com/s3/), and [HDFS](https://hortonworks.com/apache/hdfs/).
 
 go-simple-serializer (GSS) supports `bson`, `csv`, `tsv`, `hcl`, `hcl2`, `json`, `jsonl`, `properties`, `toml`, `yaml`.  `hcl` and `hcl2` implementation is fragile and very much in `alpha`.
 
+For an interactive demo, see the [railgun notebook](https://beta.observablehq.com/@pjdufour/railgun) on [ObservableHQ](http://observablehq.com). It is very heavy, so only use WiFi.
+
 # Usage
 
 **CLI**

cmd/railgun.js/main.go

@@ -5,7 +5,7 @@
 //
 // =================================================================
 
-// railgun.js is the Javascript version of Railgun.
+// railgun.js is the Javascript package for Railgun.
 //
 package main
 
@@ -20,10 +20,10 @@ import (
 )
 
 import (
-	dfljs "github.com/spatialcurrent/go-dfl/cmd/dfl.js"
 	"github.com/spatialcurrent/go-dfl/dfl"
-	gssjs "github.com/spatialcurrent/go-simple-serializer/cmd/gss.js"
+	"github.com/spatialcurrent/go-dfl/dfljs"
 	"github.com/spatialcurrent/go-simple-serializer/gss"
+	"github.com/spatialcurrent/go-simple-serializer/gssjs"
 	"github.com/spatialcurrent/railgun/railgun"
 )
 
@@ -32,9 +32,6 @@ import (
 	"honnef.co/go/js/console"
 )
 
-var GO_RAILGUN_COMPRESSION_ALGORITHMS = []string{"none", "gzip", "snappy"}
-var GO_RAILGUN_FORMATS = []string{"csv", "tsv", "hcl", "hcl2", "json", "jsonl", "properties", "toml", "yaml"}
-
 func main() {
 
 	js.Global.Set("railgun", map[string]interface{}{
@@ -49,8 +46,11 @@ func main() {
 			"EvaluateString": dfljs.EvaluateString,
 		},
 		"gss": map[string]interface{}{
-			"version": gss.VERSION,
-			"convert": gssjs.Convert,
+			"version":     gss.VERSION,
+			"formats":     gss.Formats,
+			"convert":     gssjs.Convert,
+			"deserialize": gssjs.Deserialize,
+			"serialize":   gssjs.Serialize,
 		},
 	})
 }
@@ -147,15 +147,15 @@ func Process(in interface{}, options *js.Object) interface{} {
 	if len(dfl_exp) > 0 {
 		n, err := dfl.Parse(dfl_exp)
 		if err != nil {
-			console.Error(errors.Wrap(err, "Error parsing dfl node.").Error())
+			console.Error(errors.Wrap(err, "Error parsing DFL node.").Error())
 			return ""
 		}
 		dfl_node = n.Compile()
 	}
 
 	var output interface{}
 	if dfl_node != nil {
-		o, err := dfl_node.Evaluate(ctx, dfl.NewFuntionMapWithDefaults(), []string{"\"", "'", "`"})
+		_, o, err := dfl_node.Evaluate(map[string]interface{}{}, ctx, dfl.NewFuntionMapWithDefaults(), []string{"\"", "'", "`"})
 		if err != nil {
 			console.Error(errors.Wrap(err, "error processing").Error())
 			return ""

cmd/railgun/main.go

@@ -5,6 +5,8 @@
 //
 // =================================================================
 
+// railgun is the command line program for Railgun.
+//
 package main
 
 import (
@@ -42,7 +44,6 @@ import (
 )
 
 var GO_RAILGUN_COMPRESSION_ALGORITHMS = []string{"none", "bzip2", "gzip", "snappy"}
-var GO_RAILGUN_FORMATS = []string{"bson", "csv", "tsv", "hcl", "hcl2", "json", "jsonl", "properties", "toml", "yaml"}
 var GO_RAILGUN_DEFAULT_SALT = "4F56C8C88B38CD8CD96BF8A9724F4BFE"
 
 func printUsage() {
@@ -116,19 +117,19 @@ func main() {
 
 	flag.StringVar(&input_uri, "input_uri", "stdin", "The input uri")
 	flag.StringVar(&input_compression, "input_compression", "none", "The input compression: "+strings.Join(GO_RAILGUN_COMPRESSION_ALGORITHMS, ", "))
-	flag.StringVar(&input_format, "input_format", "", "The input format: "+strings.Join(GO_RAILGUN_FORMATS, ", "))
+	flag.StringVar(&input_format, "input_format", "", "The input format: "+strings.Join(gss.Formats, ", "))
 	flag.StringVar(&input_header_text, "h", "", "The input header if the stdin input has no header.")
 	flag.StringVar(&input_comment, "c", "", "The input comment character, e.g., #.  Commented lines are not sent to output.")
 	flag.StringVar(&input_passphrase_string, "input_passphrase", "", "The input passphrase.")
-	flag.StringVar(&input_salt_string, "input_salt", GO_RAILGUN_DEFAULT_SALT, "The input salt as hexidecimal.")
+	flag.StringVar(&input_salt_string, "input_salt", GO_RAILGUN_DEFAULT_SALT, "The input salt as hexadecimal.")
 
 	flag.IntVar(&input_reader_buffer_size, "input_reader_buffer_size", 4096, "The input reader buffer size") // default from https://golang.org/src/bufio/bufio.go
 
 	flag.StringVar(&output_uri, "output_uri", "stdout", "The output uri")
 	flag.StringVar(&output_compression, "output_compression", "none", "The output compression: "+strings.Join(GO_RAILGUN_COMPRESSION_ALGORITHMS, ", "))
-	flag.StringVar(&output_format, "output_format", "", "The output format: "+strings.Join(GO_RAILGUN_FORMATS, ", "))
+	flag.StringVar(&output_format, "output_format", "", "The output format: "+strings.Join(gss.Formats, ", "))
 	flag.StringVar(&output_passphrase_string, "output_passphrase", "", "The output passphrase.")
-	flag.StringVar(&output_salt_string, "output_salt", GO_RAILGUN_DEFAULT_SALT, "The output salt as hexidecimal.")
+	flag.StringVar(&output_salt_string, "output_salt", GO_RAILGUN_DEFAULT_SALT, "The output salt as hexadecimal.")
 
 	flag.StringVar(&dfl_exp, "dfl_exp", "", "Process using dfl expression")
 	flag.StringVar(&dfl_file, "dfl_file", "", "Process using dfl file.")
@@ -312,23 +313,25 @@ func main() {
 				log.Fatal(errors.Wrap(err, "error dumping dfl_node as yaml to stdout"))
 			}
 			fmt.Println(dfl_node_yaml)
+			fmt.Println(dfl_node.Dfl(dfl.DefaultQuotes, true, 0))
 		}
 
 		input_type, err := gss.GetType(input_string, input_format)
 		if err != nil {
 			log.Fatal(errors.Wrap(err, "error geting type for input"))
 		}
-		
+
 		input_object, err := gss.Deserialize(input_string, input_format, input_header, input_comment, input_type, verbose)
 		if err != nil {
 			log.Fatal(errors.Wrap(err, "error deserializing input using format "+input_format))
 		}
-		
+
 		var output interface{}
 		if dfl_node != nil {
-			o, err := dfl_node.Evaluate(input_object, funcs, []string{"'", "\"", "`"})
+			_, o, err := dfl_node.Evaluate(map[string]interface{}{}, input_object, funcs, []string{"'", "\"", "`"})
 			if err != nil {
-				log.Fatal(errors.Wrap(err, "error processing"))
+				fmt.Fprintf(os.Stdout, "%+v", err)
+				log.Fatal(errors.Wrap(err, "error evaluating filter"))
 			}
 			output = o
 		} else {

railgun/InferFormatAndCompression.go

@@ -11,7 +11,7 @@ import (
 	"path/filepath"
 )
 
-// InferFormatAndCompression infers the format and compresion of the file.
+// InferFormatAndCompression infers the format and compression of the file.
 //  - *.json => ("json", "") // JSON File
 //  - *.json.bz2 => ("json", "bzip2") // bzip2-compressed JSON file
 //  - *.json.gz => ("json", "gzip") // gzip-compressed JSON file

railgun/Version.go

@@ -7,4 +7,4 @@
 
 package railgun
 
-var VERSION = "0.0.3"
+var VERSION = "0.0.4"