app/vlselect/logsql: consistently return zero hits on the selected time range from the /select/logsql/hits endpoint

Previously the endpoint could return responses with missing steps if there were no hits at these steps.
This complicated handling of such responses at the client side.

See #700

Author: valyala

app/vlselect/logsql/hits_response.qtpl

@@ -49,9 +49,9 @@
 	"fields":{%s= k %},
 	"timestamps":[
 		{% if len(timestamps) > 0 %}
-			{%q= timestamps[0] %}
+			{%q= timestampToString(timestamps[0]) %}
 			{% for _, ts := range timestamps[1:] %}
-				,{%q= ts %}
+				,{%q= timestampToString(ts) %}
 			{% endfor %}
 		{% endif %}
 	],

app/vlselect/logsql/hits_response.qtpl.go

@@ -257,6 +257,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 
 	// Add a pipe, which calculates hits over time with the given step and offset for the given fields.
 	ca.q.AddCountByTimePipe(int64(step), int64(offset), fields)
+	start, end := ca.q.GetFilterTimeRange()
 
 	var mLock sync.Mutex
 	m := make(map[string]*hitsSeries)
@@ -273,7 +274,10 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 
 		bb := blockResultPool.Get()
 		for i := 0; i < rowsCount; i++ {
-			timestampStr := strings.Clone(timestampValues[i])
+			timestampNsec, ok := logstorage.TryParseTimestampRFC3339Nano(timestampValues[i])
+			if !ok {
+				logger.Panicf("BUG: cannot parse timestamp=%q", timestampValues[i])
+			}
 			hitsStr := strings.Clone(hitsValues[i])
 			hits, err := strconv.ParseUint(hitsStr, 10, 64)
 			if err != nil {
@@ -286,11 +290,10 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 			mLock.Lock()
 			hs, ok := m[string(bb.B)]
 			if !ok {
-				k := string(bb.B)
 				hs = &hitsSeries{}
-				m[k] = hs
+				m[string(bb.B)] = hs
 			}
-			hs.timestamps = append(hs.timestamps, timestampStr)
+			hs.timestamps = append(hs.timestamps, timestampNsec)
 			hs.hits = append(hs.hits, hits)
 			hs.hitsTotal += hits
 			mLock.Unlock()
@@ -309,6 +312,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	}
 
 	m = getTopHitsSeries(m, fieldsLimit)
+	addMissingZeroHits(m, start, end, int64(step), int64(offset))
 
 	// Write response headers
 	h := w.Header()
@@ -320,6 +324,47 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	WriteHitsSeries(w, m)
 }
 
+func addMissingZeroHits(m map[string]*hitsSeries, start, end, step, offset int64) {
+	if start == math.MinInt64 {
+		start = math.MaxInt64
+		for _, hs := range m {
+			start = min(start, slices.Min(hs.timestamps))
+		}
+	} else {
+		start -= start%step - offset
+	}
+
+	if end == math.MaxInt64 {
+		end = math.MinInt64
+		for _, hs := range m {
+			end = max(end, slices.Max(hs.timestamps))
+		}
+	} else {
+		end -= start%step - offset
+	}
+
+	if start > end {
+		// nothing to do
+		return
+	}
+
+	for _, hs := range m {
+		ts := start
+		for ts <= end {
+			if !slices.Contains(hs.timestamps, ts) {
+				hs.timestamps = append(hs.timestamps, ts)
+				hs.hits = append(hs.hits, 0)
+			}
+
+			if ts+step < ts {
+				// stop on int64 overflow
+				break
+			}
+			ts += step
+		}
+	}
+}
+
 var blockResultPool bytesutil.ByteBufferPool
 
 func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hitsSeries {
@@ -342,15 +387,15 @@ func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hit
 		return a[i].hs.hitsTotal > a[j].hs.hitsTotal
 	})
 
-	hitsOther := make(map[string]uint64)
+	hitsOther := make(map[int64]uint64)
 	for _, x := range a[fieldsLimit:] {
-		for i, timestampStr := range x.hs.timestamps {
-			hitsOther[timestampStr] += x.hs.hits[i]
+		for i, timestamp := range x.hs.timestamps {
+			hitsOther[timestamp] += x.hs.hits[i]
 		}
 	}
 	var hsOther hitsSeries
-	for timestampStr, hits := range hitsOther {
-		hsOther.timestamps = append(hsOther.timestamps, timestampStr)
+	for timestamp, hits := range hitsOther {
+		hsOther.timestamps = append(hsOther.timestamps, timestamp)
 		hsOther.hits = append(hsOther.hits, hits)
 		hsOther.hitsTotal += hits
 	}
@@ -366,7 +411,7 @@ func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hit
 
 type hitsSeries struct {
 	hitsTotal  uint64
-	timestamps []string
+	timestamps []int64
 	hits       []uint64
 }
 

app/vlselect/logsql/logsql.go

@@ -24,6 +24,7 @@ according to the follosing docs:
 * FEATURE: [syslog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/): add support for automatic parsing of [`@cee` messages](http://cee.mitre.org/language/1.0-beta1/clt.html#syslog). Thanks to @exherb for the pull request [#842](https://github.com/VictoriaMetrics/VictoriaLogs/pull/842).
 * FEATURE: [HTTP querying APIs](https://docs.victoriametrics.com/victorialogs/querying/#http-api): return `AccountID` and `ProjectID` response headers, which match the corresponding request headers when executing [queries for the particular tenant](https://docs.victoriametrics.com/victorialogs/#multitenancy). These response headers can be used by client such as [built-in web UI for VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/#web-ui) for the proper displaying of the queried tenant. See [#656](https://github.com/VictoriaMetrics/VictoriaLogs/issues/656).
 * FETURE: [Elasticsearch data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/#elasticsearch-bulk-api): return the first parse error to the client, so the error could be quickly fixed. Previously the parse error was logged into VictoriaLogs internal logs and `success` response was returned to the client. This could complicate troubleshooting, since users need to have access to VictoriaLogs internal logs for the troubleshooting. See [#817](https://github.com/VictoriaMetrics/VictoriaLogs/issues/817) and [#60](https://github.com/VictoriaMetrics/VictoriaLogs/issues/60).
+* FEATURE: [hits API](https://docs.victoriametrics.com/victorialogs/querying/#querying-hits-stats): consistently return zero hits on the selected `[start .. end)` time range. Previously zero hits weren't returned, so this complicated processing of the hits response by clients. See [#700](https://github.com/VictoriaMetrics/VictoriaLogs/issues/700).
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add the help button with shortcuts reference and controls for charts and query input. See [#77](https://github.com/VictoriaMetrics/VictoriaLogs/issues/77).
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): auto-sync time picker with `_time` filter from the query (can be disabled). See [#558](https://github.com/VictoriaMetrics/VictoriaLogs/issues/558).