initial commit

Author: PS-davetemplin

.gitignore

@@ -3,6 +3,8 @@
 *.dll
 *.so
 *.dylib
+bigboy
+debug
 
 # Test binary, build with `go test -c`
 *.test

.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "remotePath": "",
+            "port": 2345,
+            "host": "127.0.0.1",
+            "program": "${fileDirname}",
+            "env": {
+                "GOPATH": "c:/Users/Dave/go"
+            },
+            "cwd": "../bigdata",
+            "showLog": true
+        }
+    ]
+}

README.md

@@ -0,0 +1,171 @@
+# Bigboy
+Extract data from SQL Server, PostgreSQL, or MySQL, transforming SQL-to-JSON or JSON-to-JSON.
+
+Written by Dave Templin
+
+# Overview
+Bigboy is a tool that extracts data from SQL Server, PostgreSQL, or MySQL databases and transforms SQL-to-JSON or JSON-to-JSON; basically performing the **E** and the **T** part of **ETL** *(Extract/Transform/Load)*. The tool provides a simple model for configuring SQL extraction queries and optionally Javascript functions for transformations. A simple but powerful command-line interface (CLI) makes it easy to perform both adhoc and batch processing scenarios (BASH, CRON, etc.). The tool is also designed to maximize available local compute resources to extract and transform massive data volumes in a time-efficient way.
+
+## Features
+* Extract data from SQL Server, PostgreSQL, or MySQL
+* Perform SQL-to-JSON or JSON-to-JSON transformations
+* Nest rows to form complex hierarchical (or document-oriented) data
+* Leverage Javascript functions to perform arbitrarily complex data transformations
+* Define command driven parameters to create dynamic queries and scripts
+* Combine data from multiple different database sources
+* Apply timezone to dates stored without a timezone
+* Configure the tool to maximize local compute resources and minimize processing time
+
+## Quickstart
+
+
+
+# Concepts
+
+## Connections
+
+## Targets
+
+## Fetching and Prefetching
+fetch, prefetch
+
+## Transforms
+nest, script, split, timezone
+
+
+
+# Reference
+
+## Command Arguments
+
+* `-e` Maximum overall number of errors before aborting *(default=100)*
+* `-n` Include nulls in output *(default=false)*
+* `-o` Output directory *(creates \"out\" directory if not specified)*
+* `-p` Number of rows per page extracted *(default=1000)*
+* `-q` Supress informational output *(default=false)*
+* `-r` Number of consecutive errors before aborting *(default=3)*
+* `-v` Print version info about bigboy and exit
+* `-w` Number of background workers *(default=4)*
+
+> Above defaults can also be configured in the `config.json` file.
+
+## config.json
+This section describes the `config.json` file format.
+
+| Name | Description |
+| --- | --- |
+| `connections` | ... |
+| `errors` | ... |
+| `nulls` | ... |
+| `page` | ... |
+| `quiet` | ... |
+| `retries` | ... |
+| `workers` | ... |
+
+### connections
+| Name | Description |
+| --- | --- |
+| `driver` | ... |
+| `server` | ... |
+| `database` | ... |
+| `dsn` | ... |
+| `port` | ... |
+| `user` | ... |
+| `password` | ... |
+| `max` | ... |
+| `timezone` | ... |
+
+
+## target.json
+This section describes the `target.json` file format.
+
+| Name | Description |
+| --- | --- |
+| `connection` | ... |
+| `fetch` | ... |
+| `params` | ... |
+| `prefetch` | ... |
+| `nest` | ... |
+| `script` | ... |
+| `split` | ... |
+| `timezone` | ... |
+
+### nest
+| Name | Description |
+| --- | --- |
+| `connection` | ... |
+| `childKey` | ... |
+| `parentKey` | ... |
+| `fetch` | ... |
+| `timezone` | ... |
+
+### param
+| Name | Description |
+| --- | --- |
+| `name` | ... |
+| `type` | ... |
+| `default` | ... |
+
+### split
+| Name | Description |
+| --- | --- |
+| `by` | ... |
+| `value` | ... |
+
+
+## Date Format
+
+All dates are assumed to be in GMT unless a timezone is specified.
+If a time is not specified then midnight GMT is assumed.
+Examples below illustrate various scenarios of specifying a date or date-range.
+
+The following examples assume there is a target named `log` with a single paramter of type `date` representing a start date for the extraction.
+
+| Example                               | Comments
+| ------------------------------------- | ------------------------------------------------------- |
+| `bigboy log 2017-07-21`               | 7/21/2017 at midnight GMT
+| `bigboy log "2017-07-21 15:00:00"`    | 7/21/2017 at 3pm GMT
+| `bigboy log today`                    | Midnight GMT of the current day
+| `bigboy log yesterday`                |   Midnight GMT of the previous day
+
+The following examples assume there is a target named `sales` with two paramters of type `date` representing a date range for the extraction.
+
+| Example                               | Comments
+| ------------------------------------- | ------------------------------------------------------- |
+| `bigboy sales 2017-07-21 2017-07-23`  | From 7/21/2017 to 7/23/2017 midnight-to-midnight GMT
+| `bigboy sales 2017-07-21 2d`          | Midnight GMT of the previous day.
+
+
+> The time zone database needed by LoadLocation may not be present on all systems, especially non-Unix systems. LoadLocation looks in the directory or uncompressed zip file named by the ZONEINFO environment variable, if any, then looks in known installation locations on Unix systems, and finally looks in $GOROOT/lib/time/zoneinfo.zip.
+
+
+
+# Build
+
+Install [golang](https://golang.org/dl/)
+
+```
+$ go get github.com/denisenkom/go-mssqldb
+$ go get github.com/lib/pq
+$ go get github.com/go-sql-driver/mysql
+$ git clone https://github.com/davetemplin/bigboy.git
+$ go build
+```
+
+## Cross compile
+```
+$ build windows
+$ build linux
+$ build mac
+```
+
+
+
+# References
+
+There are lots of ways to approach ETL, and lots of vendors that want to sell you a solution!
+Here are some additional references that may be helpful...
+
+* [Wikipedia article on ETL](https://en.wikipedia.org/wiki/Extract,_transform,_load)
+* [Performing ETL from a Relational Database into BigQuery](https://cloud.google.com/solutions/performing-etl-from-relational-database-into-bigquery)
+* [ETL Software: Top 63](https://www.predictiveanalyticstoday.com/top-free-extract-transform-load-etl-software/)

build.cmd

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+IF "%1"=="windows" GOTO windows
+IF "%1"=="linux" GOTO linux
+IF "%1"=="mac" GOTO mac
+GOTO error
+
+:windows
+SETLOCAL
+SET GOOS=windows
+SET GOARCH=amd64
+go build
+ENDLOCAL
+GOTO end
+
+:linux
+SETLOCAL
+SET GOOS=linux
+SET GOARCH=amd64
+go build
+ENDLOCAL
+GOTO end
+
+:mac
+SETLOCAL
+SET GOOS=darwin
+SET GOARCH=amd64
+go build
+ENDLOCAL
+GOTO end
+
+:error
+ECHO Specify build target: windows, linux, or mac
+GOTO end
+
+:end

config.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+)
+
+// Config ...
+type Config struct {
+	Connections  map[string]*Connection  `json:"connections"`
+	Errors       uint64                  `json:"errors"`
+	Nulls        bool                    `json:"nulls"`
+	Page         int                     `json:"page"`
+	Quiet        bool                    `json:"quiet"`
+	Retries      uint64                  `json:"retries"`
+	Workers      int                     `json:"workers"`
+}
+
+var config Config
+const undefined = ^uint64(0)
+
+func loadConfig() {
+	config.Errors = undefined
+	config.Retries = undefined
+
+	name := "config.json"
+	if fileExists(name) {
+		buffer, err := ioutil.ReadFile(name)
+		check(err)
+		json.Unmarshal(buffer, &config)
+	}
+
+	if config.Errors == undefined {
+		config.Errors = 100
+	}
+	if config.Page == 0 {
+		config.Page = 1000
+	}
+	if config.Retries == undefined {
+		config.Retries = 3
+	}
+	if config.Workers == 0 {
+		config.Workers = 4
+	}
+}

connection.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+	"time"
+	_ "github.com/denisenkom/go-mssqldb"
+	_ "github.com/lib/pq"
+	_ "github.com/go-sql-driver/mysql"
+)
+
+// Connection ...
+type Connection struct {
+	Driver          string  `json:"driver"`
+	Server          string  `json:"server"`
+	Database        string  `json:"database"`
+	Dsn             string  `json:"dsn"`
+	Port            int     `json:"port"`
+	User            string  `json:"user"`
+	Password        string  `json:"password"`
+	Timezone        string  `json:"timezone"`
+	Max             int     `json:"max"`
+	db *sql.DB
+	location *time.Location
+}
+
+func connect(key string) *Connection {
+	connection, ok := config.Connections[key]
+
+	if !ok {
+		stop(fmt.Sprintf("Invalid connection key: '%s'", key), 1)
+	}
+
+	dsn := formatDsn(connection)
+	if dsn == "" {
+		stop(fmt.Sprintf("Invalid driver specified for connection: '%s'", key), 1)
+	}
+
+	if connection.db == nil {
+		var err error
+		connection.db, err = sql.Open(connection.Driver, dsn)
+		check(err)
+
+		err = connection.db.Ping()
+		if err != nil {
+			stop(fmt.Sprintf("Unable to establish connection to server \"%s\"", connection.Server), 3)
+		}
+
+		if connection.Max != 0 {
+			connection.db.SetMaxOpenConns(connection.Max)
+		}
+
+		if connection.Timezone != "" {
+			connection.location, err = time.LoadLocation(connection.Timezone)
+			check(err)
+		}
+	}
+
+	return connection
+}
+
+func disconnect() {
+	for _, connection := range config.Connections {
+		if connection.db != nil {
+			err := connection.db.Close()
+			check(err)
+			connection.db = nil
+		}
+	}
+}
+
+func formatDsn(connection *Connection) string {
+	if connection.Dsn != "" && !strings.HasPrefix(connection.Dsn, "...") {
+		return connection.Dsn
+	}
+		
+	var dsn string
+	if connection.Driver == "mssql" {
+		dsn = fmt.Sprintf("server=%s;user id=%s;password=%s;port=%d",
+			connection.Server,
+			connection.User,
+			connection.Password,
+			connection.Port)
+	} else if connection.Driver == "postgres" {
+		dsn = fmt.Sprintf("host=%s user=%s password='%s' port=%d dbname=%s",
+			strings.Replace(connection.Server, " ", "\\ ", -1),
+			strings.Replace(connection.User, " ", "\\ ", -1),
+			strings.Replace(connection.Password, "'", "\\'", -1),
+			connection.Port,
+			strings.Replace(connection.Database, " ", "\\ ", -1))
+	} else if connection.Driver == "mysql" {
+		dsn = fmt.Sprintf("%s:%s@%s/%s",
+			connection.User,
+			connection.Password,
+			connection.Server,
+			connection.Database)
+	}
+
+	if strings.HasPrefix(connection.Dsn, "...") {
+		dsn += connection.Dsn[3:]
+	}
+
+	return dsn
+}

constants.go

@@ -0,0 +1,3 @@
+package main
+
+const version = "1.0.0"