lib/logstorage: properly apply time_offset option to subqueries.

Previously the time_offset option from the top query was mistakenly applied to all the subqueries.

This is a follow-up for 8b22bc6

Updates #78
Updates #414

Author: valyala

lib/logstorage/parser.go

@@ -592,42 +592,40 @@ func (q *Query) GetFilterTimeRange() (int64, int64) {
 
 // AddTimeFilter adds global filter _time:[start ... end] to q.
 func (q *Query) AddTimeFilter(start, end int64) {
-	startStr := marshalTimestampRFC3339NanoString(nil, start)
-	endStr := marshalTimestampRFC3339NanoString(nil, end)
-
-	start = subNoOverflowInt64(start, q.opts.timeOffset)
-	end = subNoOverflowInt64(end, q.opts.timeOffset)
-
-	end = getMatchingEndTime(end, string(endStr))
-	ft := &filterTime{
-		minTimestamp: start,
-		maxTimestamp: end,
-		stringRepr:   fmt.Sprintf("[%s,%s]", startStr, endStr), // should be matched with parsing logic
-	}
-
 	q.visitSubqueries(func(q *Query) {
-		q.addTimeFilterNoSubqueries(ft)
+		q.addTimeFilterNoSubqueries(start, end)
 	})
 }
 
-func (q *Query) addTimeFilterNoSubqueries(ft *filterTime) {
+func (q *Query) addTimeFilterNoSubqueries(start, end int64) {
 	if q.opts.ignoreGlobalTimeFilter != nil && *q.opts.ignoreGlobalTimeFilter {
 		return
 	}
 
-	ftCopy := *ft
+	timeOffset := q.opts.timeOffset
+
+	startStr := marshalTimestampRFC3339NanoString(nil, start)
+	endStr := marshalTimestampRFC3339NanoString(nil, end)
+	ft := &filterTime{
+		minTimestamp: subNoOverflowInt64(start, timeOffset),
+		maxTimestamp: getMatchingEndTime(subNoOverflowInt64(end, timeOffset), string(endStr)),
+		stringRepr:   fmt.Sprintf("[%s,%s]", startStr, endStr),
+	}
+
 	fa, ok := q.f.(*filterAnd)
 	if ok {
 		filters := make([]filter, len(fa.filters)+1)
-		filters[0] = &ftCopy
+		filters[0] = ft
 		copy(filters[1:], fa.filters)
 		fa.filters = filters
 	} else {
 		q.f = &filterAnd{
-			filters: []filter{&ftCopy, q.f},
+			filters: []filter{ft, q.f},
 		}
 	}
 
+	q.f = flattenFiltersAnd(q.f)
+
 	// Remove `*` filters after adding the `_time` filter, since they are no longer needed.
 	q.f = removeStarFilters(q.f)
 

lib/logstorage/parser_test.go

@@ -220,12 +220,12 @@ func TestQuery_AddTimeFilter(t *testing.T) {
 
 	// time_offset option
 	f(`options(time_offset=1d3h534ms) *`, `options(time_offset=1d3h534ms) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z]`)
-	f(`options(time_offset = -1.5h) _time:1.3d`, `options(time_offset=-1.5h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:1.3d`)
-	f(`options(time_offset = -1.5h) _time:1.3d offset 3.5h`, `options(time_offset=-1.5h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:1.3d offset 3.5h`)
-	f(`options(time_offset = -1.5h) _time:1.3d offset -3.5h`, `options(time_offset=-1.5h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:1.3d offset -3.5h`)
-	f(`options(time_offset=1h) id:in(_time:5m | keep id)`, `options(time_offset=1h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] id:in(_time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:5m | fields id)`)
-	f(`id:in(options(time_offset=1h) _time:5m | keep id)`, `_time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] id:in(options(time_offset=1h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:5m | fields id)`)
-	f(`options(time_offset=1d) id:in(options(time_offset=1h) _time:5m | keep id)`, `options(time_offset=1d) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] id:in(options(time_offset=1h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:5m | fields id)`)
+	f(`options(time_offset = -1.5h) _time:2024Z`, `options(time_offset=-1.5h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:2024Z`)
+	f(`options(time_offset = -1.5h) _time:2025Z offset 3.5h`, `options(time_offset=-1.5h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:2025Z offset 3.5h`)
+	f(`options(time_offset = -1.5h) _time:2025Z offset -3.5h`, `options(time_offset=-1.5h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:2025Z offset -3.5h`)
+	f(`options(time_offset=1h) id:in(_time:2025Z | keep id)`, `options(time_offset=1h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] id:in(_time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:2025Z | fields id)`)
+	f(`id:in(options(time_offset=1h) _time:2025Z | keep id)`, `_time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] id:in(options(time_offset=1h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:2025Z | fields id)`)
+	f(`options(time_offset=1d) id:in(options(time_offset=1h) _time:2025Z | keep id)`, `options(time_offset=1d) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] id:in(options(time_offset=1h) _time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] _time:2025Z | fields id)`)
 
 	// join pipe
 	f(`foo | join by (x) (bar)`, `_time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] foo | join by (x) (_time:[2024-12-25T14:56:43Z,2025-01-13T12:45:34Z] bar)`)
@@ -1319,6 +1319,13 @@ func TestParseQuery_Success(t *testing.T) {
 	f(`_time:1h (Offset)`, `_time:1h "Offset"`) // "offset" is a search word, since it is in parens
 	f(`_time:1h "and"`, `_time:1h "and"`)       // "and" is a search word, since it is quoted
 
+	// multiple _time filters
+	f(`_time:1h _time:2025Z`, `_time:1h _time:2025Z`)
+	f(`_time:<10h _time:2025Z`, `_time:<10h _time:2025Z`)
+	f(`_time:2025Z _time:[2024-10Z, 2025-03Z]`, `_time:2025Z _time:[2024-10Z,2025-03Z]`)
+	f(`options(time_offset=1h) _time:2025Z _time:[2024-10Z, 2025-03Z]`, `options(time_offset=1h) _time:2025Z _time:[2024-10Z,2025-03Z]`)
+	f(`_time:2025Z _time:2024Z _time:10y`, `_time:2025Z _time:2024Z _time:10y`)
+
 	// dayRange filters
 	f(`_time:day_range[08:00, 20:30)`, `_time:day_range[08:00, 20:30)`)
 	f(`_time:day_range(08:00, 20:30)`, `_time:day_range(08:00, 20:30)`)