Merge pull request #461 from gliderlabs/master

release 3.2.7

Author: michaelshobbs

.golangci.yml

@@ -0,0 +1,77 @@
+linters-settings:
+  errcheck:
+    # ignore cases where we truly don't care about the returned error
+    ignore: net/http:^Write$,io:^WriteString$
+  govet:
+    check-shadowing: true
+    settings:
+      printf:
+        funcs:
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+  golint:
+    min-confidence: 0
+  gocyclo:
+    min-complexity: 12
+  maligned:
+    suggest-new: true
+  dupl:
+    threshold: 100
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  depguard:
+    list-type: blacklist
+  misspell:
+    locale: US
+  lll:
+    line-length: 160
+  goimports:
+    local-prefixes: github.com/gliderlabs/logspout
+  gocritic:
+    settings:
+      hugeParam:
+        sizeThreshold: 160
+    enabled-tags:
+      - performance
+  nakedret:
+      max-func-lines: 65
+
+linters:
+  enable-all: true
+  disable:
+    - maligned
+    - prealloc
+    - gochecknoglobals
+    - funlen
+    - gochecknoinits
+
+issues:
+  # Excluding configuration per-path, per-linter, per-text and per-source
+  exclude-rules:
+    # Exclude some linters from running on tests files.
+    - path: _test\.go
+      linters:
+        - dupl
+        - errcheck
+        - goconst
+        - gocyclo
+        - gosec
+        - lll
+        - nakedret
+        - unparam
+        - funlen
+
+run:
+  deadline: 2m
+  issues-exit-code: 1
+
+# golangci.com configuration
+# https://github.com/golangci/golangci/wiki/Configuration
+service:
+  golangci-lint-version: 1.18.x # use the fixed version to not introduce new linters unexpectedly
+  prepare:
+    - echo "here I can run custom commands, but no preparation needed for this repo"
+

CHANGELOG.md

@@ -10,6 +10,18 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 
+## [v3.2.7] - 2020-04-03
+### Fixed
+- @CodeLingoBot @gbolo Fix function comments based on best practices from Effective Go
+
+### Changed
+- @michaelshobbs update alpine to 3.10/go 1.12.12-r0 and fix linting
+- @whoisteri DOC Document accessible data in RAW_FORMAT template
+- @tiagorlampert DOC typos
+- @michaelshobbs DOC CHANGLELOG formatting
+- @tomlankhorst DOC Suggest to disable userns-remap for logspout
+- @StudioEtrange DOC add link to logspout-fluentd
+
 ## [v3.2.6] - 2018-10-04
 ### Fixed
 - @jdgiotta Spelling corrections and fixed stack compose formatting in example
@@ -28,13 +40,12 @@ All notable changes to this project will be documented in this file.
 - @gbolo enforced the use of `go 1.8+` in order to accommodate some TLS settings
 
 ## [v3.2.5] - 2018-06-05
+### Fixed
+- @michaelshobbs fix working_directory so we don't duplicate test runs
 - @gmelika panic if reconnect fails
 - @masterada Added multiline adapter
 - @billimek sleeping and syncing to fix issues with docker hub builds
 
-### Fixed
-- @michaelshobbs fix working_directory so we don't duplicate test runs
-
 ### Added
 - @chris7444 take the hostname from /etc/host_hostname if the file is there
 - @chris7444 update README.md for swarm deployments PR #329
@@ -199,7 +210,8 @@ All notable changes to this project will be documented in this file.
 - Base container is now Alpine
 - Moved to gliderlabs organization
 
-[unreleased]: https://github.com/gliderlabs/logspout/compare/v3.2.6...HEAD
+[unreleased]: https://github.com/gliderlabs/logspout/compare/v3.2.7...HEAD
+[v3.2.7]: https://github.com/gliderlabs/logspout/compare/v3.2.6...v3.2.7
 [v3.2.6]: https://github.com/gliderlabs/logspout/compare/v3.2.5...v3.2.6
 [v3.2.5]: https://github.com/gliderlabs/logspout/compare/v3.2.4...v3.2.5
 [v3.2.4]: https://github.com/gliderlabs/logspout/compare/v3.2.3...v3.2.4

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.8
+FROM alpine:3.10
 ENTRYPOINT ["/bin/logspout"]
 VOLUME /mnt/routes
 EXPOSE 80

Dockerfile.dev

@@ -1,12 +1,9 @@
-FROM alpine:3.8
+FROM alpine:3.10
 VOLUME /mnt/routes
 EXPOSE 80
 
-ENV GOPATH /go
-RUN apk --no-cache add go build-base git mercurial ca-certificates
-COPY . /go/src/github.com/gliderlabs/logspout
-WORKDIR /go/src/github.com/gliderlabs/logspout
-RUN go get
-CMD go get \
-	&& go build -ldflags "-X main.Version=dev" -o /bin/logspout \
-	&& exec /bin/logspout
+RUN apk --no-cache add go build-base git mercurial ca-certificates curl
+COPY . /src
+WORKDIR /src
+CMD go build -ldflags "-X main.Version=dev" -o /bin/logspout \
+		&& exec /bin/logspout

Makefile

@@ -5,11 +5,13 @@ VERSION=$(shell cat VERSION)
 # max image size of 40MB
 MAX_IMAGE_SIZE := 40000000
 
+GOBIN := $(shell go env GOPATH | awk -F ":" '{ print $$1 }')/bin
+GOLANGCI_LINT_VERSION := v1.18.0
+
 ifeq ($(shell uname), Darwin)
 	XARGS_ARG="-L1"
 endif
 GOPACKAGES ?= $(shell go list ./... | egrep -v 'custom|vendor')
-GOLINT := go list ./... | egrep -v '/custom/|/vendor/' | xargs $(XARGS_ARG) golint | egrep -v 'extpoints.go|types.go'
 TEST_ARGS ?= -race
 
 ifdef TEST_RUN
@@ -37,12 +39,21 @@ build-custom:
 	docker tag $(NAME):$(VERSION) gliderlabs/$(NAME):master
 	cd custom && docker build -t $(NAME):custom .
 
-lint:
-	test -x $(GOPATH)/bin/golint || go get github.com/golang/lint/golint
-	go get \
-		&& go install $(GOPACKAGES) \
-		&& go tool vet -v $(shell ls -d */ | egrep -v 'custom|vendor/' | xargs $(XARGS_ARG))
-	@if [ -n "$(shell $(GOLINT) | cut -d ':' -f 1)" ]; then $(GOLINT) && exit 1 ; fi
+lint-requirements:
+ifeq ($(shell which golangci-lint), )
+	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(GOBIN) $(GOLANGCI_LINT_VERSION)
+endif
+
+lint: lint-requirements
+	$(GOBIN)/golangci-lint run
+
+lint-ci-direct: lint-requirements
+	$(GOBIN)/golangci-lint --verbose run
+
+lint-ci: build-dev
+	docker run \
+		-v $(PWD):/go/src/github.com/gliderlabs/logspout \
+		$(NAME):dev make -e lint-ci-direct
 
 test: build-dev
 	docker run \
@@ -76,7 +87,7 @@ test-healthcheck:
 		-p 8000:80 \
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		$(NAME):$(VERSION)
-	sleep 2
+	sleep 5
 	docker logs $(NAME)-healthcheck
 	docker inspect --format='{{ .State.Running }}' $(NAME)-healthcheck | grep true
 	curl --head --silent localhost:8000/health | grep "200 OK"
@@ -85,7 +96,7 @@ test-healthcheck:
 
 test-custom:
 	docker run --name $(NAME)-custom $(NAME):custom || true
-	docker logs $(NAME)-custom | grep -q logstash
+	docker logs $(NAME)-custom 2>&1 | grep -q logstash
 	docker rmi gliderlabs/$(NAME):master || true
 	docker rm $(NAME)-custom || true
 

README.md

@@ -37,6 +37,8 @@ logspout will gather logs from other containers that are started **without the `
 
 To see what data is used for syslog messages, see the [syslog adapter](http://github.com/gliderlabs/logspout/blob/master/adapters) docs.
 
+The container must be able to access the Docker Unix socket to mount it. This is typically a problem when [namespace remapping](https://docs.docker.com/engine/security/userns-remap/) is enabled. To disable remapping for the logspout container, pass the `--userns=host` flag to `docker run`, `.. create`, etc. 
+
 #### Ignoring specific containers
 
 You can tell logspout to ignore specific containers by setting an environment variable when starting your container, like so:-
@@ -192,6 +194,14 @@ If you use multiline logging with raw, it's recommended to json encode the Data
 
 The raw adapter has a function `toJSON` that can be used to format the message/fields to generate JSON-like output in a simple way, or full JSON output.
 
+The RAW_FORMAT env variable is used as a [Go template](https://golang.org/pkg/text/template/) with a [`Message` struct](https://github.com/gliderlabs/logspout/blob/master/router/types.go#L52) passed as data. You can access the following fields
+
+* `Source` - source stream name ("stdout", "stderr", ...)
+* `Data` - original log message 
+* `Time` - a Go [`Time` struct](https://golang.org/pkg/time/#Time)
+* `Container` - a [go-dockerclient](https://github.com/fsouza/go-dockerclient) `Container` struct (see [container.go](https://github.com/fsouza/go-dockerclient/blob/master/container.go#L443) source file for accessible fields)
+
+
 Use examples:
 
 ##### Mixed JSON + generic:
@@ -273,9 +283,9 @@ logspout supports modification of the client TLS settings via environment variab
 | Environment Variable  | Description |
 | :---                  |  :---       |
 | `LOGSPOUT_TLS_DISABLE_SYSTEM_ROOTS` | when set to `true` it disables loading the system trust store into the trust store of logspout |
-| `LOGSPOUT_TLS_CA_CERTS` | a comma seperated list of filesystem paths to pem encoded CA certificates that should be added to logsput's TLS trust store. Each pem file can contain more than one certificate |
-| `LOGSPOUT_TLS_CLIENT_CERT` | filesytem path to pem encoded x509 client certificate to load when TLS mutual authentication is desired |
-| `LOGSPOUT_TLS_CLIENT_KEY` | filesytem path to pem encoded client private key to load when TLS mutual authentication is desired |
+| `LOGSPOUT_TLS_CA_CERTS` | a comma separated list of filesystem paths to pem encoded CA certificates that should be added to logsput's TLS trust store. Each pem file can contain more than one certificate |
+| `LOGSPOUT_TLS_CLIENT_CERT` | filesystem path to pem encoded x509 client certificate to load when TLS mutual authentication is desired |
+| `LOGSPOUT_TLS_CLIENT_KEY` | filesystem path to pem encoded client private key to load when TLS mutual authentication is desired |
 | `LOGSPOUT_TLS_HARDENING` | when set to `true` it enables stricter client TLS settings designed to mitigate some known TLS vulnerabilities |
 
 #### Example TLS settings
@@ -329,6 +339,8 @@ The standard distribution of logspout comes with all modules defined in this rep
  * [logspout-logstash](https://github.com/looplab/logspout-logstash)
  * [logspout-redis-logstash](https://github.com/rtoma/logspout-redis-logstash)
  * [logspout-gelf](https://github.com/micahhausler/logspout-gelf) for Graylog
+ * [logspout-fluentd](https://github.com/dsouzajude/logspout-fluentd) for fluentd or fluent-bit - instead of using fluentd log driver
+ 
 
 ### Loggly support
 

VERSION

@@ -1 +1 @@
-v3.2.6
+v3.2.7

adapters/multiline/multiline.go

@@ -9,7 +9,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/fsouza/go-dockerclient"
+	docker "github.com/fsouza/go-dockerclient"
+
 	"github.com/gliderlabs/logspout/router"
 )
 
@@ -40,11 +41,10 @@ type Adapter struct {
 }
 
 // NewMultilineAdapter returns a configured multiline.Adapter
-func NewMultilineAdapter(route *router.Route) (a router.LogAdapter, err error) {
+func NewMultilineAdapter(route *router.Route) (a router.LogAdapter, err error) { //nolint:gocyclo
 	enableByDefault := true
 	enableStr := os.Getenv("MULTILINE_ENABLE_DEFAULT")
 	if enableStr != "" {
-		var err error
 		enableByDefault, err = strconv.ParseBool(enableStr)
 		if err != nil {
 			return nil, errors.New("multiline: invalid value for MULTILINE_ENABLE_DEFAULT (must be true|false): " + enableStr)
@@ -92,8 +92,8 @@ func NewMultilineAdapter(route *router.Route) (a router.LogAdapter, err error) {
 	flushAfter := 500 * time.Millisecond
 	flushAfterStr := os.Getenv("MULTILINE_FLUSH_AFTER")
 	if flushAfterStr != "" {
-		timeoutMS, err := strconv.Atoi(flushAfterStr)
-		if err != nil {
+		timeoutMS, errConv := strconv.Atoi(flushAfterStr)
+		if errConv != nil {
 			return nil, errors.New("multiline: invalid value for multiline_timeout (must be number): " + flushAfterStr)
 		}
 		flushAfter = time.Duration(timeoutMS) * time.Millisecond
@@ -135,7 +135,7 @@ func NewMultilineAdapter(route *router.Route) (a router.LogAdapter, err error) {
 }
 
 // Stream sends log data to the next adapter
-func (a *Adapter) Stream(logstream chan *router.Message) {
+func (a *Adapter) Stream(logstream chan *router.Message) { //nolint:gocyclo
 	wg := sync.WaitGroup{}
 	wg.Add(1)
 	go func() {
@@ -173,7 +173,7 @@ func (a *Adapter) Stream(logstream chan *router.Message) {
 				a.buffers[cID] = message
 			} else {
 				isLastLine := a.isLastLine(message)
-				
+
 				if oldExists {
 					old.Data += a.separator + message.Data
 					message = old

adapters/multiline/multiline_test.go

@@ -7,7 +7,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/fsouza/go-dockerclient"
+	docker "github.com/fsouza/go-dockerclient"
+
 	"github.com/gliderlabs/logspout/router"
 )
 

adapters/raw/raw.go

@@ -21,7 +21,7 @@ var funcs = template.FuncMap{
 	"toJSON": func(value interface{}) string {
 		bytes, err := json.Marshal(value)
 		if err != nil {
-			log.Println("error marshalling to JSON: ", err)
+			log.Println("error marshaling to JSON: ", err)
 			return "null"
 		}
 		return string(bytes)

adapters/syslog/syslog.go

@@ -15,6 +15,7 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/gliderlabs/logspout/cfg"
 	"github.com/gliderlabs/logspout/router"
 )
 
@@ -34,22 +35,14 @@ func init() {
 }
 
 func setRetryCount() {
-	if count, err := strconv.Atoi(getopt("RETRY_COUNT", strconv.Itoa(defaultRetryCount))); err != nil {
+	if count, err := strconv.Atoi(cfg.GetEnvDefault("RETRY_COUNT", strconv.Itoa(defaultRetryCount))); err != nil {
 		retryCount = uint(defaultRetryCount)
 	} else {
 		retryCount = uint(count)
 	}
 	debug("setting retryCount to:", retryCount)
 }
 
-func getopt(name, dfault string) string {
-	value := os.Getenv(name)
-	if value == "" {
-		value = dfault
-	}
-	return value
-}
-
 func debug(v ...interface{}) {
 	if os.Getenv("DEBUG") != "" {
 		log.Println(v...)
@@ -61,7 +54,7 @@ func getHostname() string {
 	if err == nil && len(content) > 0 {
 		hostname = strings.TrimRight(string(content), "\r\n")
 	} else {
-		hostname = getopt("SYSLOG_HOSTNAME", "{{.Container.Config.Hostname}}")
+		hostname = cfg.GetEnvDefault("SYSLOG_HOSTNAME", "{{.Container.Config.Hostname}}")
 	}
 	return hostname
 }
@@ -77,18 +70,18 @@ func NewSyslogAdapter(route *router.Route) (router.LogAdapter, error) {
 		return nil, err
 	}
 
-	format := getopt("SYSLOG_FORMAT", "rfc5424")
-	priority := getopt("SYSLOG_PRIORITY", "{{.Priority}}")
-	pid := getopt("SYSLOG_PID", "{{.Container.State.Pid}}")
+	format := cfg.GetEnvDefault("SYSLOG_FORMAT", "rfc5424")
+	priority := cfg.GetEnvDefault("SYSLOG_PRIORITY", "{{.Priority}}")
+	pid := cfg.GetEnvDefault("SYSLOG_PID", "{{.Container.State.Pid}}")
 	hostname = getHostname()
 
-	tag := getopt("SYSLOG_TAG", "{{.ContainerName}}"+route.Options["append_tag"])
-	structuredData := getopt("SYSLOG_STRUCTURED_DATA", "")
+	tag := cfg.GetEnvDefault("SYSLOG_TAG", "{{.ContainerName}}"+route.Options["append_tag"])
+	structuredData := cfg.GetEnvDefault("SYSLOG_STRUCTURED_DATA", "")
 	if route.Options["structured_data"] != "" {
 		structuredData = route.Options["structured_data"]
 	}
-	data := getopt("SYSLOG_DATA", "{{.Data}}")
-	timestamp := getopt("SYSLOG_TIMESTAMP", "{{.Timestamp}}")
+	data := cfg.GetEnvDefault("SYSLOG_DATA", "{{.Data}}")
+	timestamp := cfg.GetEnvDefault("SYSLOG_TIMESTAMP", "{{.Timestamp}}")
 
 	if structuredData == "" {
 		structuredData = "-"