planner: recheck LIKE/ILIKE with a non-default ESCAPE on memtable scans (#69670)

ref #69653

Author: vismaytiwari

pkg/executor/memtable_reader_test.go

@@ -895,6 +895,30 @@ func TestTiDBClusterLog(t *testing.T) {
 			},
 			expected: [][]string{},
 		},
+		{
+			// ILIKE is pushed down as a case-insensitive prefilter, so an uppercase
+			// pattern still matches a lowercase message through the remote search.
+			// This also covers the retriever guard: a query whose only message
+			// filter is ILIKE must still produce a pattern, otherwise the scan is
+			// rejected with "denied to scan full logs".
+			conditions: []string{
+				"time>='2019/08/26 06:22:17.011'",
+				"time<='2019/08/26 06:22:17.011'",
+				"message ilike '%FOO%'",
+			},
+			expected: [][]string{
+				{"2019/08/26 06:22:17.011", "pd", "CRITICAL", "[test log message pd 5, foo]"},
+			},
+		},
+		{
+			// The prefilter plus the retained scalar recheck must not over-match.
+			conditions: []string{
+				"time>='2019/08/26 06:18:13.011'",
+				"time<='2019/08/26 06:28:19.011'",
+				"message ilike '%NoSuchMessage%'",
+			},
+			expected: [][]string{},
+		},
 		{
 			conditions: []string{
 				"time>='2019/08/26 06:18:13.011'",

pkg/planner/core/memtable_predicate_extractor.go

@@ -383,22 +383,25 @@ func (helper *extractHelper) extractLikePatternCol(
 			continue
 		}
 
-		var canBuildPattern bool
+		var canBuildPattern, isPrefilter bool
 		var pattern string
 		// We use '|' to combine DNF regular expression: .*a.*|.*b.*
 		// e.g:
 		// SELECT * FROM t WHERE c LIKE '%a%' OR c LIKE '%b%'
 		if fn.FuncName.L == ast.LogicOr && !toLower {
-			canBuildPattern, pattern = helper.extractOrLikePattern(ctx, fn, extractColName, extractCols, needLike2Regexp)
+			canBuildPattern, pattern, isPrefilter = helper.extractOrLikePattern(ctx, fn, extractColName, extractCols, needLike2Regexp, toLower)
 		} else {
-			canBuildPattern, pattern = helper.extractLikePattern(ctx, fn, extractColName, extractCols, needLike2Regexp)
+			canBuildPattern, pattern, isPrefilter = helper.extractLikePattern(ctx, fn, extractColName, extractCols, needLike2Regexp, toLower)
 		}
 		if canBuildPattern && toLower {
 			pattern = strings.ToLower(pattern)
 		}
 		if canBuildPattern {
 			patterns = append(patterns, pattern)
-		} else {
+		}
+		// A prefilter narrows the scan but is not exactly equivalent to the
+		// predicate, so it is pushed down *and* kept for a scalar recheck.
+		if !canBuildPattern || isPrefilter {
 			remained = append(remained, expr)
 		}
 	}
@@ -411,40 +414,51 @@ func (helper extractHelper) extractOrLikePattern(
 	extractColName string,
 	extractCols map[int64]*types.FieldName,
 	needLike2Regexp bool,
+	toLower bool,
 ) (
 	ok bool,
 	pattern string,
+	isPrefilter bool,
 ) {
 	predicates := expression.SplitDNFItems(orFunc)
 	if len(predicates) == 0 {
-		return false, ""
+		return false, "", false
 	}
 
 	patternBuilder := make([]string, 0, len(predicates))
 	for _, predicate := range predicates {
 		fn, ok := predicate.(*expression.ScalarFunction)
 		if !ok {
-			return false, ""
+			return false, "", false
 		}
 
-		ok, partPattern := helper.extractLikePattern(ctx, fn, extractColName, extractCols, needLike2Regexp)
+		ok, partPattern, partIsPrefilter := helper.extractLikePattern(ctx, fn, extractColName, extractCols, needLike2Regexp, toLower)
 		if !ok {
-			return false, ""
+			return false, "", false
 		}
+		// One inexact branch makes the whole disjunction inexact.
+		isPrefilter = isPrefilter || partIsPrefilter
 		patternBuilder = append(patternBuilder, partPattern)
 	}
-	return true, strings.Join(patternBuilder, "|")
+	return true, strings.Join(patternBuilder, "|"), isPrefilter
 }
 
+// extractLikePattern builds the pushed-down pattern for a single predicate.
+// toLower reports whether the caller folds case on both the pattern and the
+// scanned value, which decides how a case-insensitive ILIKE is represented.
+// isPrefilter reports that the pattern only narrows the scan and is not exactly
+// equivalent to the predicate, so the caller must keep a scalar recheck.
 func (helper extractHelper) extractLikePattern(
 	ctx base.PlanContext,
 	fn *expression.ScalarFunction,
 	extractColName string,
 	extractCols map[int64]*types.FieldName,
 	needLike2Regexp bool,
+	toLower bool,
 ) (
 	ok bool,
 	pattern string,
+	isPrefilter bool,
 ) {
 	var colName string
 	var datums []types.Datum
@@ -453,21 +467,62 @@ func (helper extractHelper) extractLikePattern(
 		colName, datums, _ = helper.extractColBinaryOpConsExpr(ctx, extractCols, fn)
 	}
 	if colName != extractColName {
-		return false, ""
+		return false, "", false
 	}
 	switch fn.FuncName.L {
 	case ast.EQ:
-		return true, "^" + regexp.QuoteMeta(datums[0].GetString()) + "$"
+		return true, "^" + regexp.QuoteMeta(datums[0].GetString()) + "$", false
 	case ast.Like, ast.Ilike:
-		if needLike2Regexp {
-			return true, stringutil.CompileLike2Regexp(datums[0].GetString())
+		// The pushed-down pattern must honour the LIKE ESCAPE so it does not
+		// diverge from the original predicate (issue #69653). The escape has to
+		// be resolvable at plan time: for a constant escape (including the
+		// default '\' and the empty escape of NO_BACKSLASH_ESCAPES) we compile
+		// the regexp with it; a non-constant/deferred escape can't be resolved
+		// here, so we skip extraction and let the scalar predicate be rechecked.
+		escape, ok := likeEscapeConst(fn)
+		if !ok {
+			return false, "", false
+		}
+		if !needLike2Regexp {
+			return true, datums[0].GetString(), false
 		}
-		return true, datums[0].GetString()
+		pattern = stringutil.CompileLike2Regexp(datums[0].GetString(), escape)
+		// ILIKE matches case-insensitively while the pattern above is
+		// case-sensitive. Callers that fold case (toLower) lower both the pattern
+		// and the scanned value, so it stays equivalent there. Callers that do
+		// not -- the cluster log path sends the pattern verbatim in
+		// SearchLogRequest.Patterns -- get a case-insensitive group instead, so
+		// an "ERROR" line still matches `message ILIKE '%error%'`. The scoped
+		// (?i:...) form keeps the flag from leaking across a '|' when several
+		// patterns are combined into one disjunction. Case folding there is the
+		// regexp engine's rather than ILIKE's, so it is only a prefilter and the
+		// predicate is rechecked.
+		if fn.FuncName.L == ast.Ilike && !toLower {
+			return true, "(?i:" + pattern + ")", true
+		}
+		return true, pattern, false
 	case ast.Regexp, ast.RegexpLike:
-		return true, datums[0].GetString()
+		return true, datums[0].GetString(), false
 	default:
-		return false, ""
+		return false, "", false
+	}
+}
+
+// likeEscapeConst returns the ESCAPE byte of a LIKE/ILIKE ScalarFunction when
+// it is a plan-time constant (ok=true). The escape must be constant so the
+// pushed-down pattern can be compiled to match the predicate exactly (issue
+// #69653); a non-constant, deferred, or parameterized escape returns ok=false,
+// telling the caller to skip pushdown and keep a scalar recheck.
+func likeEscapeConst(fn *expression.ScalarFunction) (byte, bool) {
+	args := fn.GetArgs()
+	if len(args) < 3 {
+		return 0, false
+	}
+	escape, ok := args[2].(*expression.Constant)
+	if !ok || escape.DeferredExpr != nil || escape.ParamMarker != nil {
+		return 0, false
 	}
+	return byte(escape.Value.GetInt64()), true
 }
 
 func (extractHelper) findColumn(schema *expression.Schema, names []*types.FieldName, colName string) map[int64]*types.FieldName {

pkg/planner/core/operator/logicalop/logicalop_test/logical_mem_table_predicate_extractor_test.go

@@ -540,6 +540,41 @@ func TestClusterLogTableExtractor(t *testing.T) {
 			instances: set.NewStringSet(),
 			level:     set.NewStringSet("debug", "error"),
 		},
+		{
+			// ILIKE is case-insensitive while the pattern is matched
+			// case-sensitively by the remote log search, so it is pushed down as a
+			// case-insensitive group -- an "ERROR" line still matches '%error%'.
+			// The predicate stays for a scalar recheck (see the executor test in
+			// TestClusterLogTableIlike), and a pattern is still produced so the
+			// retriever does not reject the scan as a full-log scan.
+			sql:       "select * from information_schema.cluster_log where message ilike '%error%'",
+			nodeTypes: set.NewStringSet(),
+			instances: set.NewStringSet(),
+			patterns:  []string{"(?i:^.*error.*$)"},
+		},
+		{
+			// Same for an ILIKE carrying a non-default ESCAPE ('#%' is a literal '%').
+			sql:       "select * from information_schema.cluster_log where message ilike '%error#%%' escape '#'",
+			nodeTypes: set.NewStringSet(),
+			instances: set.NewStringSet(),
+			patterns:  []string{"(?i:^.*error%.*$)"},
+		},
+		{
+			// In a DNF the scoped (?i:...) keeps the case-insensitive branch from
+			// leaking case folding onto the case-sensitive one across the '|'.
+			sql:       "select * from information_schema.cluster_log where (message ilike '%pd%' or message like '%tikv%')",
+			nodeTypes: set.NewStringSet(),
+			instances: set.NewStringSet(),
+			patterns:  []string{"(?i:^.*pd.*$)|^.*tikv.*$"},
+		},
+		{
+			// A case-sensitive LIKE still pushes down, honouring a custom ESCAPE
+			// so the pattern matches the predicate ('#%' is a literal '%').
+			sql:       "select * from information_schema.cluster_log where message like '%a#%b%' escape '#'",
+			nodeTypes: set.NewStringSet(),
+			instances: set.NewStringSet(),
+			patterns:  []string{"^.*a%b.*$"},
+		},
 	}
 	for _, ca := range cases {
 		logicalMemTable, ok := getLogicalMemTable(t, dom, se, parser, ca.sql)

pkg/planner/core/tests/extractor/BUILD.bazel

@@ -8,7 +8,7 @@ go_test(
         "memtable_infoschema_extractor_test.go",
     ],
     flaky = True,
-    shard_count = 4,
+    shard_count = 5,
     deps = [
         "//pkg/infoschema",
         "//pkg/planner/core",

pkg/planner/core/tests/extractor/memtable_infoschema_extractor_test.go

@@ -510,3 +510,26 @@ func TestMemtableInfoschemaExtractorPart4(t *testing.T) {
 	}
 	testMemtableInfoschemaExtractor(t, tcs)
 }
+
+func TestInfoSchemaTableNameLikeEscape(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("create database like_escape")
+	tk.MustExec("use like_escape")
+	tk.MustExec("create table `abc_def` (a int)")
+	tk.MustExec("create table `abc#x` (a int)")
+
+	// With ESCAPE '#', "#_" is a literal underscore, so only `abc_def` matches
+	// (not `abc#x`). The extractor pushes the pattern down compiled with the '#'
+	// escape, so the memtable scan matches exactly the LIKE ... ESCAPE predicate
+	// instead of diverging under the default '\' escape. See issue #69653.
+	tk.MustQuery("select table_name, table_name like '%#_%' escape '#' as self_true " +
+		"from information_schema.tables " +
+		"where table_schema = 'like_escape' and table_name like '%#_%' escape '#'").
+		Check(testkit.Rows("abc_def 1"))
+
+	// A default-escape LIKE is unaffected and still matches both tables.
+	tk.MustQuery("select table_name from information_schema.tables " +
+		"where table_schema = 'like_escape' and table_name like 'abc%'").
+		Sort().Check(testkit.Rows("abc#x", "abc_def"))
+}

pkg/util/stringutil/string_util.go

@@ -257,9 +257,10 @@ func matchRune(a, b rune) bool {
 	*/
 }
 
-// CompileLike2Regexp convert a like `lhs` to a regular expression
-func CompileLike2Regexp(str string) string {
-	patChars, patTypes := CompilePattern(str, '\\')
+// CompileLike2Regexp convert a like `lhs` to a regular expression, compiling
+// the pattern with the given LIKE escape byte (pass '\\' for the SQL default).
+func CompileLike2Regexp(str string, escape byte) string {
+	patChars, patTypes := CompilePattern(str, escape)
 	var result strings.Builder
 	result.Grow(len(patChars)*2 + 2)
 	result.WriteByte('^')

pkg/util/stringutil/string_util_test.go

@@ -133,7 +133,7 @@ func TestCompileLike2Regexp(t *testing.T) {
 		{`%_%_aA`, "^...*aA$"},
 	}
 	for _, v := range tbl {
-		result := CompileLike2Regexp(v.pattern)
+		result := CompileLike2Regexp(v.pattern, '\\')
 		require.Equalf(t, v.regexp, result, "source %v", v)
 	}
 }