etcd: add metrics for watch and paginated list requests

Previously there were no metrics emitted for paginated list calls or the
corresponding watch events. This leads to a blind spot that could hide
real bugs or performance issues.

One example was the issue where quorum was down and the list request
were executed with "WithSerializable" while the watch required quorum.
That resulted in the watch failing and the list starting again - all in
a tight loop. That issue was fixed in 1c82d58
("etcd: require quorum on etcd range requests"), and is no longer
present

The watch metric only measures the setup time and not the actual time a
watch request is open.

Signed-off-by: Odin Ugedal <ougedal@palantir.com>
Signed-off-by: Odin Ugedal <odin@uged.al>

Author: odinuge

pkg/kvstore/etcd.go

@@ -796,8 +796,11 @@ reList:
 			}
 		}
 
+		watcherDuration := spanstat.Start()
 		etcdWatch := e.client.Watch(client.WithRequireLeader(ctx), prefix,
 			client.WithPrefix(), client.WithRev(nextRev))
+		// This does not measure the actual time a watcher is open, but just the fact that it was opened
+		increaseMetric(prefix, metricRead, "WatchStart", watcherDuration.EndError(nil).Total(), nil)
 		lr.Done()
 
 		for {
@@ -889,10 +892,12 @@ func (e *etcdClient) paginatedList(ctx context.Context, log *slog.Logger, prefix
 	start, end := prefix, client.GetPrefixRangeEnd(prefix)
 
 	for {
+		duration := spanstat.Start()
 		res, err := e.client.Get(ctx, start, client.WithRange(end),
 			client.WithSort(client.SortByKey, client.SortAscend),
 			client.WithRev(revision), client.WithLimit(int64(e.listBatchSize)),
 		)
+		increaseMetric(prefix, metricRead, "ListPrefixPaginated", duration.EndError(err).Total(), err)
 		if err != nil {
 			return nil, 0, err
 		}