Initial commit

Author: deniszh

Dockerfile

@@ -0,0 +1,70 @@
+FROM phusion/baseimage:0.9.22
+MAINTAINER Denys Zhdanov <[email protected]>
+
+RUN apt-get -y update \
+  && apt-get -y upgrade \
+  && apt-get -y --force-yes install ca-certificates \
+  apt-transport-https \
+  wget \
+  nginx \
+  git \
+  sqlite3 \
+  libcairo2 \
+  libcairo2-dev \
+  && curl -s https://packagecloud.io/install/repositories/go-graphite/stable/script.deb.sh | bash \
+  && apt-get install -y carbonapi carbonzipper \
+  && mkdir /etc/carbonapi/ \
+  && rm -rf /var/lib/apt/lists/*
+
+# install go-carbon
+RUN wget https://github.com/lomik/go-carbon/releases/download/v0.10.1/go-carbon_0.10.1_amd64.deb \
+  && dpkg -i go-carbon_0.10.1_amd64.deb \
+  && rm /go-carbon_0.10.1_amd64.deb \
+  && mkdir -p /var/lib/graphite/whisper \
+  && mkdir -p /var/lib/graphite/dump \
+  && service go-carbon stop
+
+# install grafana
+RUN wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_4.4.1_amd64.deb \
+  && dpkg -i grafana_4.4.1_amd64.deb \
+  && rm /grafana_4.4.1_amd64.deb \
+  && service grafana-server restart \
+  && sleep 5 \
+  && curl -X POST -H 'Content-Type: application/json' -u 'admin:admin' \
+  -d '{ "name": "carbonapi", "type": "graphite", "url": "https://127.0.0.1:8081", "access": "proxy", "basicAuth": false }' \
+  "http://127.0.0.1:3000/api/datasources" \
+  && service grafana-server stop
+
+# config nginx
+RUN rm /etc/nginx/sites-enabled/default
+ADD conf/etc/nginx/nginx.conf /etc/nginx/nginx.conf
+ADD conf/etc/nginx/sites-enabled/go-graphite.conf /etc/nginx/sites-enabled/go-graphite.conf
+
+# config go-carbon
+ADD conf/etc/go-carbon/go-carbon.conf /etc/go-carbon/go-carbon.conf
+ADD conf/etc/go-carbon/storage-aggregation.conf /etc/go-carbon/storage-aggregation.conf
+ADD conf/etc/go-carbon/storage-schemas.conf /etc/go-carbon/storage-schemas.conf
+
+# config carbonapi
+ADD conf/etc/carbonapi/carbonapi.yaml /etc/carbonapi/carbonapi.yaml
+
+# logging support
+RUN mkdir -p /var/log/go-carbon /var/log/carbonapi /var/log/nginx
+ADD conf/etc/logrotate.d/go-graphite.conf /etc/logrotate.d/go-graphite.conf
+
+# daemons
+ADD conf/etc/service/go-carbon/run /etc/service/go-carbon/run
+ADD conf/etc/service/carbonapi/run /etc/service/carbonapi/run
+ADD conf/etc/service/grafana/run /etc/service/grafana/run
+ADD conf/etc/service/nginx/run /etc/service/nginx/run
+
+# cleanup
+RUN apt-get clean\
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# defaults
+EXPOSE 80 2003 2004 8080 8081
+VOLUME ["/etc/go-carbon", "/etc/carbonapi", "/var/lib/graphite", "/etc/nginx", "/etc/logrotate.d", "/var/log"]
+WORKDIR /
+ENV HOME /root
+CMD ["/sbin/my_init"]

README.md

@@ -1,4 +1,4 @@
 # docker-go-graphite
 Docker image for go-carbon + carbonapi + grafana (zipper included)
 
-WIP
+WIP, not ready yet

conf/etc/carbonapi/carbonapi.yaml

@@ -0,0 +1,68 @@
+# Need to be URL, http or https
+# This url specifies the backend or a loadbalancer
+#
+# Is you are using carbonzipper you should set it to
+# zipper's url
+#
+# If you are using plain go-carbon or graphite-clickhouse
+# you should set it to URL of go-carbon's carbonserver module
+# or graphite-clickhouse's http url.
+zipper: "http://localhost:8080"
+# Listen address, should always include hostname or ip address and a port.
+listen: "localhost:8081"
+# Max concurrent requests to CarbonZipper
+concurency: 20
+cache:
+   # Type of caching. Valid: "mem", "memcache", "null"
+   type: "mem"
+   # Cache limit in megabytes
+   size_mb: 0
+   # Default cache timeout value. Identical to DEFAULT_CACHE_DURATION in graphite-web.
+   defaultTimeoutSec: 60
+   # Only used by memcache type of cache. List of memcache servers.
+   memcachedServers:
+       - "127.0.0.1:1234"
+       - "127.0.0.2:1235"
+# Amount of CPUs to use. 0 - unlimited
+cpus: 0
+# Timezone, default - local
+tz: ""
+# If 'true', carbonapi will send requests as is, with globs and braces
+# Otherwise for each request it will generate /metrics/find and then /render
+# individual metrics.
+# true --- faster, but will cause carbonzipper to consume much more RAM.
+#
+# For some backends (e.x. graphite-clickhouse) this *MUST* be set to true in order
+# to get reasonable performance
+#
+# For go-carbon --- it depends on how you use it.
+sendGlobsAsIs: false
+# If sendGlobsAsIs is set and resulting response will be larger than maxBatchSize
+# carbonapi will revert to old behavir. This allows you to use benifits of passing
+# globs as is but keep memory usage in sane limits.
+#
+# For go-carbon you might want it to keep in some reasonable limits, 100 is good "safe" defaults
+#
+# For some backends (e.x. graphite-clickhouse) you might want to set it to some insanly high value, like 100000
+maxBatchSize: 100
+graphite:
+    # Host:port where to send internal metrics
+    # Empty = disabled
+    host: "localhost:2003"
+    interval: "60s"
+    prefix: "carbon.api"
+# Maximium idle connections to carbonzipper
+idleConnections: 10
+pidFile: ""
+# See https://github.com/go-graphite/carbonzipper/blob/master/example.conf#L70-L108 for format explanation
+logger:
+    - logger: ""
+      file: "stderr"
+      level: "debug"
+      encoding: "console"
+      encodingTime: "iso8601"
+      encodingDuration: "seconds"
+    - logger: ""
+      file: "carbonapi.log"
+      level: "info"
+      encoding: "json"

conf/etc/go-carbon/go-carbon.conf

@@ -0,0 +1,149 @@
+[common]
+# Run as user. Works only in daemon mode
+user = "carbon"
+# Prefix for store all internal go-carbon graphs. Supported macroses: {host}
+graph-prefix = "carbon.agents.{host}"
+# Endpoint for store internal carbon metrics. Valid values: "" or "local", "tcp://host:port", "udp://host:port"
+metric-endpoint = "local"
+# Interval of storing internal metrics. Like CARBON_METRIC_INTERVAL
+metric-interval = "1m0s"
+# Increase for configuration with multi persister workers
+max-cpu = 4
+
+[whisper]
+data-dir = "/var/lib/graphite/whisper"
+# http://graphite.readthedocs.org/en/latest/config-carbon.html#storage-schemas-conf. Required
+schemas-file = "/etc/go-carbon/storage-schemas.conf"
+# http://graphite.readthedocs.org/en/latest/config-carbon.html#storage-aggregation-conf. Optional
+aggregation-file = "/etc/go-carbon/storage-aggregation.conf"
+# Worker threads count. Metrics sharded by "crc32(metricName) % workers"
+workers = 8
+# Limits the number of whisper update_many() calls per second. 0 - no limit
+max-updates-per-second = 0
+# Sparse file creation
+sparse-create = true
+enabled = true
+
+[cache]
+# Limit of in-memory stored points (not metrics)
+max-size = 1000000
+# Capacity of queue between receivers and cache
+# Strategy to persist metrics. Values: "max","sorted","noop"
+#   "max" - write metrics with most unwritten datapoints first
+#   "sorted" - sort by timestamp of first unwritten datapoint.
+#   "noop" - pick metrics to write in unspecified order,
+#            requires least CPU and improves cache responsiveness
+write-strategy = "max"
+
+[udp]
+listen = ":2003"
+enabled = true
+# Enable optional logging of incomplete messages (chunked by max UDP packet size)
+log-incomplete = false
+# Optional internal queue between receiver and cache
+buffer-size = 0
+
+[tcp]
+listen = ":2003"
+enabled = true
+# Optional internal queue between receiver and cache
+buffer-size = 0
+
+[pickle]
+listen = ":2004"
+# Limit message size for prevent memory overflow
+max-message-size = 67108864
+enabled = true
+# Optional internal queue between receiver and cache
+buffer-size = 0
+
+[carbonlink]
+listen = "127.0.0.1:7002"
+enabled = false
+# Close inactive connections after "read-timeout"
+read-timeout = "30s"
+
+[carbonserver]
+# Please NOTE: carbonserver is not intended to fully replace graphite-web
+# It acts as a "REMOTE_STORAGE" for graphite-web or carbonzipper/carbonapi
+listen = "127.0.0.1:8080"
+# Carbonserver support is still experimental and may contain bugs
+# Or be incompatible with github.com/grobian/carbonserver
+enabled = true
+# Buckets to track response times
+buckets = 10
+# carbonserver-specific metrics will be sent as counters
+# For compatibility with grobian/carbonserver
+metrics-as-counters = false
+# Read and Write timeouts for HTTP server
+read-timeout = "60s"
+write-timeout = "60s"
+# Enable /render cache, it will cache the result for 1 minute
+query-cache-enabled = true
+# 0 for unlimited
+query-cache-size-mb = 0
+# Enable /metrics/find cache, it will cache the result for 5 minutes
+find-cache-enabled = true
+# Control trigram index
+#  This index is used to speed-up /find requests
+#  However, it will lead to increased memory consumption
+#  Estimated memory consumption is approx. 500 bytes per each metric on disk
+#  Another drawback is that it will recreate index every scan-frequency interval
+#  All new/deleted metrics will still be searchable until index is recreated
+trigram-index = true
+# carbonserver keeps track of all available whisper files
+# in memory. This determines how often it will check FS
+# for new or deleted metrics.
+scan-frequency = "5m0s"
+# Maximum amount of globs in a single metric in index
+# This value is used to speed-up /find requests with
+# a lot of globs, but will lead to increased memory consumption
+max-globs = 100
+# graphite-web-10-mode
+# Use Graphite-web 1.0 native structs for pickle response
+# This mode will break compatibility with graphite-web 0.9.x
+# If false, carbonserver won't send graphite-web 1.0 specific structs
+# That might degrade performance of the cluster
+# But will be compatible with both graphite-web 1.0 and 0.9.x
+graphite-web-10-strict-mode = true
+
+[dump]
+# Enable dump/restore function on USR2 signal
+enabled = true
+# Directory for store dump data. Should be writeable for carbon
+path = "/var/lib/graphite/dump/"
+# Restore speed. 0 - unlimited
+restore-per-second = 0
+
+[pprof]
+listen = "localhost:7007"
+enabled = false
+
+# Default logger
+[[logging]]
+# logger name
+# available loggers:
+# * "" - default logger for all messages without configured special logger
+# @TODO
+logger = ""
+# Log output: filename, "stderr", "stdout", "none", "" (same as "stderr")
+file = "stdout"
+# Log level: "debug", "info", "warn", "error", "dpanic", "panic", and "fatal"
+level = "info"
+# Log format: "json", "console", "mixed"
+encoding = "mixed"
+# Log time format: "millis", "nanos", "epoch", "iso8601"
+encoding-time = "iso8601"
+# Log duration format: "seconds", "nanos", "string"
+encoding-duration = "seconds"
+
+# You can define multiply loggers:
+
+# Copy errors to stderr for systemd
+# [[logging]]
+# logger = ""
+# file = "stderr"
+# level = "error"
+# encoding = "mixed"
+# encoding-time = "iso8601"
+# encoding-duration = "seconds"

conf/etc/go-carbon/storage-aggregation.conf

@@ -0,0 +1,7 @@
+# Documentation:
+# http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-aggregation-conf
+
+[default]
+pattern = .*
+xFilesFactor = 0.5
+aggregationMethod = average

conf/etc/go-carbon/storage-schemas.conf

@@ -0,0 +1,6 @@
+# Documentation:
+# http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf
+
+[default]
+pattern = .*
+retentions = 60s:30d,1h:5y

conf/etc/logrotate.d/go-graphite.conf

@@ -0,0 +1,11 @@
+/var/log/*.log /var/log/*/*.log {
+  weekly
+  size 50M
+  missingok
+  rotate 10
+  compress
+  delaycompress
+  notifempty
+  copytruncate
+  su root syslog
+}