[Q] short statsd flush interval causes missing data in grafana dashboard

[rebrowning]

My current setup is statsd -> go-carbon -> carbonapi -> grafana

carbonapi version is 16.0
go-carbon version is 17.1

If I set the flushInterval in statsd to 60000, I see all expected values in my grafana dashboard. However if I lower that interval to something like 10000, even though I see all the data being sent to go-carbon with a tcpdump I do not see the correct metrics in grafana. In fact it looks like only the last metric received from statsd for each minute time period is the one displayed. I'm not sure the correct way to run statsd is to have the flushInterval set so high, and I would expect to see all metrics sent to go-carbon displayed in grafana. However I cannot figure out the appropriate configuration to make it work. I don't know if the issue is on the go-carbon side or the carbonapi side.

Any pointers would be great. This is a very simple setup with all apps running on the same box, with one instance of each app.

My configuration files:

statsd config

{
  graphitePort: 2003
, graphiteHost: "127.0.0.1"
, port: 8125
, backends: [ "./backends/graphite" ]
, flushInterval: 60000
}

carbonapi.yaml

listen: "0.0.0.0:8081"
prefix: ""
notFoundStatusCode: 404
concurency: 1000

maxBatchSize: 100
graphite:
    host: ""
    interval: "60s"
    prefix: "carbon.api"
    pattern: "{prefix}.{fqdn}"
idleConnections: 10
pidFile: ""
upstreams:
    buckets: 10
    timeouts:
        find: "10s"
        render: "10s"
        connect: "200ms"
    concurrencyLimitPerServer: 0
    keepAliveInterval: "30s"
    maxIdleConnsPerHost: 100
    backends:
        - "http://localhost:8080"
    graphite09compat: false
expireDelaySec: 10
logger:
    - logger: ""
      file: "stderr"
      level: "debug"
      encoding: "console"
      encodingTime: "iso8601"
      encodingDuration: "seconds"

go-carbon.conf

[common]
user = "carbon"
graph-prefix = "carbon.agents.{host}"
metric-endpoint = "local"
metric-interval = "1m0s"
max-cpu = 1

[whisper]
data-dir = "/var/lib/whisper"
schemas-file = "/etc/go-carbon/storage-schemas.conf"
workers = 8
max-updates-per-second = 0
max-creates-per-second = 0
hard-max-creates-per-second = false
sparse-create = false
flock = true
enabled = true
hash-filenames = true
compressed = false
remove-empty-file = false

[cache]
max-size = 1000000
write-strategy = "max"

[udp]
listen = ":2003"
enabled = true
log-incomplete = false
buffer-size = 1000

[grpc]
listen = "127.0.0.1:7003"
enabled = true

[carbonserver]
listen = "127.0.0.1:8080"
enabled = true
buckets = 10
metrics-as-counters = false
read-timeout = "60s"
write-timeout = "60s"
query-cache-enabled = true
query-cache-size-mb = 0
max-creates-per-second = 0
find-cache-enabled = true
trigram-index = true
scan-frequency = "5m0s"
max-globs = 100
fail-on-max-globs = false
graphite-web-10-strict-mode = true
internal-stats-dir = ""
stats-percentiles = [99, 98, 95, 75, 50]

# Default logger
[[logging]]
logger = ""
file = "stdout"
level = "debug"
encoding = "mixed"
encoding-time = "iso8601"

/etc/go-carbon/storage-schemas.conf

[default]
pattern = .*
retentions = 10s:14d,1h:5y
compressed = false