Initial beta release

Author: hartfordfive

.gitignore

@@ -0,0 +1,10 @@
+/.idea
+/build
+
+.DS_Store
+/cloudflarebeat
+/cloudflarebeat.test
+*.pyc
+cloudflarebeat.local.yml
+/cloudflarelogs.state
+*.state

.travis.yml

@@ -0,0 +1,43 @@
+sudo: required
+dist: trusty
+services:
+  - docker
+
+language: go
+
+go:
+  - 1.6
+
+os:
+  - linux
+  - osx
+
+env:
+  matrix:
+    - TARGETS="check"
+    - TARGETS="testsuite"
+
+  global:
+    # Cross-compile for amd64 only to speed up testing.
+    - GOX_FLAGS="-arch amd64"
+
+addons:
+  apt:
+    packages:
+      - python-virtualenv
+
+before_install:
+  # Redo the travis setup but with the elastic/libbeat path. This is needed so the package path is correct
+  - mkdir -p $HOME/gopath/src/github.com/hartfordfive/cloudflarebeat/
+  - rsync -az ${TRAVIS_BUILD_DIR}/ $HOME/gopath/src/github.com/hartfordfive/cloudflarebeat/
+  - export TRAVIS_BUILD_DIR=$HOME/gopath/src/github.com/hartfordfive/cloudflarebeat/
+  - cd $HOME/gopath/src/github.com/hartfordfive/cloudflarebeat/
+
+install:
+  - true
+
+script:
+  - make $TARGETS
+
+after_success:
+  # Copy full.cov to coverage.txt because codecov.io requires this file

CONTRIBUTING.md

@@ -0,0 +1,41 @@
+BEATNAME=cloudflarebeat
+BEAT_DIR=github.com/hartfordfive/cloudflarebeat
+SYSTEM_TESTS=false
+TEST_ENVIRONMENT=false
+ES_BEATS?=./vendor/github.com/elastic/beats
+GOPACKAGES=$(shell glide novendor)
+PREFIX?=.
+
+# Path to the libbeat Makefile
+-include $(ES_BEATS)/libbeat/scripts/Makefile
+
+# Initial beat setup
+.PHONY: setup
+setup: copy-vendor
+	make update
+
+# Copy beats into vendor directory
+.PHONY: copy-vendor
+copy-vendor:
+	mkdir -p vendor/github.com/elastic/
+	cp -R ${GOPATH}/src/github.com/elastic/beats vendor/github.com/elastic/
+	rm -rf vendor/github.com/elastic/beats/.git
+
+.PHONY: git-init
+git-init:
+	git init
+	git add README.md CONTRIBUTING.md
+	git commit -m "Initial commit"
+	git add LICENSE
+	git commit -m "Add the LICENSE"
+	git add .gitignore
+	git commit -m "Add git settings"
+	git add .
+	git reset -- .travis.yml
+	git commit -m "Add cloudflarebeat"
+	git add .travis.yml
+	git commit -m "Add Travis CI"
+
+# This is called by the beats packer before building starts
+.PHONY: before-build
+before-build:

Makefile

@@ -0,0 +1,137 @@
+# Cloudflarebeat
+
+Welcome to Cloudflarebeat.
+
+Ensure that this folder is at the following location:
+`${GOPATH}/github.com/hartfordfive`
+
+## Getting Started with Cloudflarebeat
+
+### Requirements
+
+* [Golang](https://golang.org/dl/) 1.7
+
+### Init Project
+To get running with Cloudflarebeat and also install the
+dependencies, run the following command:
+
+```
+make setup
+```
+
+It will create a clean git history for each major step. Note that you can always rewrite the history if you wish before pushing your changes.
+
+To push Cloudflarebeat in the git repository, run the following commands:
+
+```
+git remote set-url origin https://github.com/hartfordfive/cloudflarebeat
+git push origin master
+```
+
+For further development, check out the [beat developer guide](https://www.elastic.co/guide/en/beats/libbeat/current/new-beat.html).
+
+### Build
+
+To build the binary for Cloudflarebeat run the command below. This will generate a binary
+in the same directory with the name cloudflarebeat.
+
+```
+make
+```
+
+
+### Run
+
+To run Cloudflarebeat with debugging output enabled, run:
+
+```
+./cloudflarebeat -c cloudflarebeat.yml -e -d "*"
+```
+
+For details of command line options, view the following links:
+
+- https://www.elastic.co/guide/en/beats/libbeat/master/config-file-format-cli.html
+- https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-command-line.html
+
+
+## Cloudflarebeat specific configuration options
+
+- `cloudflarebeat.period` : The period at which the cloudflare logs will be fetched.  (Default value is 1800s/30mins which is the default suggested by the Enterprise Log Share API documentation page.)
+- `cloudflarebeat.api_key` : The API key of the user account (mandatory)
+- `cloudflarebeat.email` : The email address of the user account (mandatory)
+- `cloudflarebeat.zone_tag` : The zone tag of the domain for which you want to access the enterpise logs (mandatory)
+- `cloudflarebeat.state_file_storage_type` : The type of storage for the state file, either `disk`, `s3`, or `consul`, which keeps track of the current progress. (Defau)
+- `cloudflarebeat.aws_access_key` : The user AWS access key, if S3 storage selected.
+- `cloudflarebeat.aws_secret_access_key` : The user AWS secret access key, if S3 storage selected.
+
+## Filtering out specific logs and/or log properties
+
+
+### Test
+
+To test Cloudflarebeat, run the following command:
+
+```
+make testsuite
+```
+
+alternatively:
+```
+make unit-tests
+make system-tests
+make integration-tests
+make coverage-report
+```
+
+The test coverage is reported in the folder `./build/coverage/`
+
+### Update
+
+Each beat has a template for the mapping in elasticsearch and a documentation for the fields
+which is automatically generated based on `etc/fields.yml`.
+To generate etc/cloudflarebeat.template.json and etc/cloudflarebeat.asciidoc
+
+```
+make update
+```
+
+
+### Cleanup
+
+To clean  Cloudflarebeat source code, run the following commands:
+
+```
+make fmt
+make simplify
+```
+
+To clean up the build directory and generated artifacts, run:
+
+```
+make clean
+```
+
+
+### Clone
+
+To clone Cloudflarebeat from the git repository, run the following commands:
+
+```
+mkdir -p ${GOPATH}/github.com/hartfordfive
+cd ${GOPATH}/github.com/hartfordfive
+git clone https://github.com/hartfordfive/cloudflarebeat
+```
+
+
+For further development, check out the [beat developer guide](https://www.elastic.co/guide/en/beats/libbeat/current/new-beat.html).
+
+
+## Packaging
+
+The beat frameworks provides tools to crosscompile and package your beat for different platforms. This requires [docker](https://www.docker.com/) and vendoring as described above. To build packages of your beat, run the following command:
+
+```
+make package
+```
+
+This will fetch and create all images required for the build process. The hole process to finish can take several minutes.

README.md

@@ -0,0 +1,121 @@
+package beater
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/libbeat/publisher"
+
+	"github.com/hartfordfive/cloudflarebeat/cloudflare"
+	"github.com/hartfordfive/cloudflarebeat/config"
+)
+
+type Cloudflarebeat struct {
+	done   chan struct{}
+	config config.Config
+	client publisher.Client
+}
+
+// Creates beater
+func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
+	config := config.DefaultConfig
+	if err := cfg.Unpack(&config); err != nil {
+		return nil, fmt.Errorf("Error reading config file: %v", err)
+	}
+
+	bt := &Cloudflarebeat{
+		done:   make(chan struct{}),
+		config: config,
+	}
+	return bt, nil
+}
+
+func (bt *Cloudflarebeat) Run(b *beat.Beat) error {
+	logp.Info("cloudflarebeat is running! Hit CTRL-C to stop it.")
+
+	bt.client = b.Publisher.Connect()
+	ticker := time.NewTicker(bt.config.Period)
+	counter := 1
+
+	// Initialize the Cloudflare client here
+	cc := cloudflare.NewClient(map[string]interface{}{
+		"api_key": bt.config.APIKey,
+		"email":   bt.config.Email,
+		"debug":   bt.config.Debug,
+		"exclude": bt.config.Exclude,
+	})
+
+	sf := cloudflare.NewStateFile("cloudflarebeat.state", bt.config.StateFileStorageType)
+	logp.Info("Initializing state file 'cloudflarebeats.state' with storage type '" + bt.config.StateFileStorageType + "'")
+	err := sf.Initialize()
+
+	if err != nil {
+		logp.Err("Could not load statefile: %s", err.Error())
+		os.Exit(1)
+	}
+
+	if sf.GetLastStartTS() != 0 {
+		logp.Info("Start time loaded from state file: %s", time.Unix(int64(sf.GetLastStartTS()), 0).Format(time.RFC3339))
+		logp.Info("  End time loaded from state file: %s", time.Unix(int64(sf.GetLastEndTS()), 0).Format(time.RFC3339))
+	}
+
+	var timeStart, timeEnd, timeNow int
+
+	for {
+		select {
+		case <-bt.done:
+			return nil
+		case <-ticker.C:
+		}
+
+		timeNow = int(time.Now().UTC().Unix())
+
+		if sf.GetLastStartTS() != 0 {
+			timeStart = sf.GetLastEndTS() + 1       // last time start
+			timeEnd = sf.GetLastEndTS() + (30 * 60) // to 30 minutes later, minus 1 second
+		} else {
+			timeStart = timeNow - (30 * 60) // last end TS as per statefile
+			timeEnd = timeNow               // to 1 second ago
+		}
+
+		logp.Info("Next request start time: %s", time.Unix(int64(timeStart), 0).Format(time.RFC3339))
+		logp.Info("  Next request end Time: %s", time.Unix(int64(timeEnd), 0).Format(time.RFC3339))
+
+		logs, err := cc.GetLogRangeFromTimestamp(map[string]interface{}{
+			"zone_tag":   bt.config.ZoneTag,
+			"time_start": timeStart,
+			"time_end":   timeEnd,
+			//"count":      10,
+		})
+
+		if err != nil {
+			logp.Err("GetLogRangeFromTimestamp: %s", err.Error())
+			sf.UpdateLastRequestTS(timeNow)
+		} else {
+			bt.client.PublishEvents(logs)
+			logp.Info("Total events sent this period: %d", len(logs))
+			// Now need to update the disk-based state file that keeps track of the current state
+			sf.UpdateLastStartTS(timeStart)
+			sf.UpdateLastEndTS(timeEnd)
+			sf.UpdateLastCount(len(logs))
+			sf.UpdateLastRequestTS(timeNow)
+		}
+
+		err = sf.Save()
+		if err != nil {
+			logp.Err("Could not persist state file to storage: %s", err.Error())
+		}
+
+		counter++
+	}
+
+}
+
+func (bt *Cloudflarebeat) Stop() {
+	bt.client.Close()
+	close(bt.done)
+}