[BUG] target missing in response #844
Description
Describe the bug
A request with multiple similar targets is not returning all results if they use multiple metrics and one of them does not exist.
CarbonAPI Version
0.16.0 and 0.17.0
Logs
will try to fetch data {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "max_connections": 0}
got slot {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
sending request {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
will try to fetch data {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "max_connections": 0}
got slot {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
sending request {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
trying to get slot {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host01.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host01.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got slot for server {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host01.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host01.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got response {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response (after merge) {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got some fetch responses {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "backends_count": 1, "response_count": 1, "have_errors": false, "errors": null, "metrics_in_response": 1}
got response {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response (after merge) {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got some fetch responses {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "backends_count": 1, "response_count": 1, "have_errors": false, "errors": null, "metrics_in_response": 1}
will try to fetch data {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "max_connections": 0}
got slot {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
sending request {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
will try to fetch data {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "max_connections": 0}
got slot {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
sending request {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
trying to get slot {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got slot for server {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got response {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 0, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response (after merge) {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 0, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "response_is_nil": true, "stats_is_nil": false, "err": "metric not found", "errVerbose": "metric not found\nHTTP Code: 404\n\ngithub.com/go-graphite/carbonapi/zipper/types.init\n\t/root/go/src/github.com/go-graphite/carbonapi/zipper/types/errors.go:22\nruntime.doInit1\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7176\nruntime.doInit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7143\nruntime.main\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:253\nruntime.goexit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/asm_amd64.s:1695"}
got response (after merge) {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "metrics_in_response": 0, "errors_count": 1, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
had errors while fetching result {"function": "FetchProtoV3", "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "errors": "metric not found", "errorsVerbose": "metric not found\nHTTP Code: 404\n\ngithub.com/go-graphite/carbonapi/zipper/types.init\n\t/root/go/src/github.com/go-graphite/carbonapi/zipper/types/errors.go:22\nruntime.doInit1\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7176\nruntime.doInit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7143\nruntime.main\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:253\nruntime.goexit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/asm_amd64.s:1695", "httpCode": 404}
no metrics fetched {"function": "FetchProtoV3", "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "errors": "metric not found", "errorsVerbose": "metric not found\nHTTP Code: 404\n\ngithub.com/go-graphite/carbonapi/zipper/types.init\n\t/root/go/src/github.com/go-graphite/carbonapi/zipper/types/errors.go:22\nruntime.doInit1\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7176\nruntime.doInit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7143\nruntime.main\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:253\nruntime.goexit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/asm_amd64.s:1695"}
request served {"data": {"handler":"render","carbonapi_uuid":"de03d1ee-6381-404c-92d9-0c95ce04c74b","url":"/render","peer_ip":"198.51.100.24","host":"198.51.100.25:8081","format":"json","use_cache":true,"targets":["alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'1')","alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'3')"],"cache_timeout":60,"metrics":["alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'1')","alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'3')"],"have_non_fatal_errors":true,"runtime":0.015166965,"http_code":200,"carbonzipper_response_size_bytes":344717,"carbonapi_response_size_bytes":22261,"from":1727733600,"until":1730318365,"max_data_points":1306,"from_raw":"1727733600","until_raw":"1730318365","uri":"/render","from_cache":false,"used_backend_cache":false,"request_headers":{}}}
CarbonAPI Configuration:
listen: "[::]:8081"
concurency: 1000
cache:
type: "null"
size_mb: 0
defaultTimeoutSec: 60
cpus: 0
tz: "Europe/Berlin,0"
maxBatchSize: 0
graphite:
host: "localhost:2003"
interval: "60s"
prefix: "carbon.api"
pattern: "{prefix}.{fqdn}"
idleConnections: 10
pidFile: ""
# See https://github.com/go-graphite/carbonzipper/blob/master/example.conf#L70-L108 for format explanation
upstreams:
tldCacheDisabled: true
buckets: 10
slowLogThreshold: "60s"
timeouts:
find: "2s"
render: "300s"
connect: "200ms"
concurrencyLimitPerServer: 0
keepAliveInterval: "30s"
maxIdleConnsPerHost: 100
doMultipleRequestsIfSplit: false
backendsv2:
backends:
-
groupName: "localhost"
protocol: "carbonapi_v3_pb"
lbMethod: "broadcast"
maxTries: 3
maxBatchSize: 0
keepAliveInterval: "10s"
concurrencyLimit: 0
maxIdleConnsPerHost: 1000
forceAttemptHTTP2: false
doMultipleRequestsIfSplit: false
servers:
- "http://127.0.0.1:8080"
expvar:
enabled: true
pprofEnabled: true
listen: "[::]:8082"
logger:
- logger: ""
file: "stderr"
level: "debug"
encoding: "console"
encodingTime: "iso8601"
encodingDuration: "seconds"Simplified query (if applicable)
?target=alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'1')
&target=alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'3')
&from=1727733600
&until=1730318937
&format=json
&maxDataPoints=1306
In this query we have have two targets which only differ in the alias. The first metric carbon.api.host01.requests does exist, but the second carbon.api.host02.requests does not exist.
The response only contains the result for the first target. In our real example query we have three such targets and only get the result for the first and second.
[
{
"target":"1",
"datapoints":[[156,1727733660],[...],[80.28571428571429,1730317560]],
"tags":{"name":"carbon.api.host01.requests"}
}
]