collate: do not register latin1_swedish_ci yet

Address review feedback on two counts.

Registering the collation let GetCollationByName accept it, so DDL could create
columns whose sort key TiKV and TiFlash cannot reproduce, and nothing stops an
expression carrying that collation from being pushed down. Drop the
newCollatorMap and newCollatorIDMap entries so the collation is rejected exactly
as before this branch. The collator and its weight table stay, and the unit tests
exercise the type directly.

That also removes the second problem. IsCICollation gates fast paths in
pkg/expression that case-fold with strings.ToLower, which assumes UTF-8: every
latin1 byte >= 0x80 becomes U+FFFD, so distinct characters fold together and
LOCATE reports matches that do not exist, while the 1-to-3 byte inflation breaks
the surrounding rune-offset arithmetic. Folding for a byte-oriented collation has
to go through the collator, as locateStringWithCollation already does. Revert the
IsCICollation change rather than half-fix it; it belongs with the work that turns
the collation on.

The collator doc comment now lists what enabling it requires, so the next reader
does not have to rediscover it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TgAxo4cK4qCpuLrhroHWy1

Author: terry1purcell

pkg/expression/collation_test.go

@@ -179,7 +179,9 @@ func TestInferCollation(t *testing.T) {
 		},
 		// latin1_bin mixed with latin1_swedish_ci. At equal coercibility the _bin
 		// collation wins regardless of argument order, so latin1_swedish_ci must be
-		// recognized as the non-bin side (see isBinCollation).
+		// recognized as the non-bin side (see isBinCollation). The collation is not
+		// registered yet and so cannot reach these paths from SQL; these cases pin the
+		// classification so it is not changed by accident before it can be.
 		{
 			[]Expression{
 				newExpression(CoercibilityImplicit, UNICODE, charset.CharsetLatin1, charset.CollationLatin1),
@@ -911,19 +913,11 @@ func TestCompareString(t *testing.T) {
 
 	// latin1 collations are byte-oriented, so the arguments are spelled as cp1252
 	// bytes rather than as Go literals, which the compiler would encode as UTF-8.
-	require.Equal(t, 0, types.CompareString("a", "A", "latin1_swedish_ci"))
-	require.Equal(t, 0, types.CompareString("\xE9", "E", "latin1_swedish_ci"))
-	require.Equal(t, 0, types.CompareString("a ", "a  ", "latin1_swedish_ci"))
-	require.Equal(t, 0, types.CompareString("\xDC", "Y", "latin1_swedish_ci"))
-	// Å, Ä and Ö sort after Z rather than folding onto A and O.
-	require.NotEqual(t, 0, types.CompareString("\xC5", "A", "latin1_swedish_ci"))
-	require.NotEqual(t, 0, types.CompareString("\xD6", "O", "latin1_swedish_ci"))
-	require.Equal(t, 1, types.CompareString("\xC5", "Z", "latin1_swedish_ci"))
-	// ß folds to nothing here, unlike in the utf8 collations above.
-	require.NotEqual(t, 0, types.CompareString("\xDF", "s", "latin1_swedish_ci"))
-	require.NotEqual(t, 0, types.CompareString("\xDF", "ss", "latin1_swedish_ci"))
-
+	// latin1_swedish_ci is not covered here because it is not registered yet, so
+	// CompareString would silently fall back to another collator; its semantics are
+	// tested directly in pkg/util/collate.
 	require.NotEqual(t, 0, types.CompareString("a", "A", "latin1_bin"))
+	require.NotEqual(t, 0, types.CompareString("\xE9", "E", "latin1_bin"))
 	require.Equal(t, 0, types.CompareString("a ", "a  ", "latin1_bin"))
 
 	require.NotEqual(t, 0, types.CompareString("a", "A", "binary"))

pkg/util/collate/collate.go

@@ -251,9 +251,7 @@ func GetSupportedCollations() []*charset.Collation {
 		newSupportedCollations := make([]*charset.Collation, 0, len(newCollatorMap))
 		for name := range newCollatorMap {
 			// utf8mb4_zh_pinyin_tidb_as_cs is under developing, should not be shown to user.
-			// latin1_swedish_ci is under developing too: its sort key has no counterpart in
-			// TiKV/TiFlash yet, so pushed-down comparisons cannot reproduce it.
-			if name == "utf8mb4_zh_pinyin_tidb_as_cs" || name == "latin1_swedish_ci" {
+			if name == "utf8mb4_zh_pinyin_tidb_as_cs" {
 				continue
 			}
 			if coll, err := charset.GetCollationByName(name); err != nil {
@@ -313,8 +311,7 @@ func IsDefaultCollationForUTF8MB4(collate string) bool {
 func IsCICollation(collate string) bool {
 	return collate == "utf8_general_ci" || collate == "utf8mb4_general_ci" ||
 		collate == "utf8_unicode_ci" || collate == "utf8mb4_unicode_ci" || collate == "gbk_chinese_ci" ||
-		collate == "utf8mb4_0900_ai_ci" || collate == "gb18030_chinese_ci" ||
-		collate == "latin1_swedish_ci"
+		collate == "utf8mb4_0900_ai_ci" || collate == "gb18030_chinese_ci"
 }
 
 // ConvertAndGetBinCollation converts collation to binary collation
@@ -334,8 +331,6 @@ func ConvertAndGetBinCollation(collate string) string {
 		return "gbk_bin"
 	case "gb18030_chinese_ci":
 		return "gb18030_bin"
-	case "latin1_swedish_ci":
-		return "latin1_bin"
 	}
 
 	return collate
@@ -454,8 +449,6 @@ func init() {
 	newCollatorIDMap[CollationName2ID("ascii_bin")] = &binPaddingCollator{}
 	newCollatorMap["latin1_bin"] = &binPaddingCollator{}
 	newCollatorIDMap[CollationName2ID("latin1_bin")] = &binPaddingCollator{}
-	newCollatorMap["latin1_swedish_ci"] = &latin1SwedishCICollator{}
-	newCollatorIDMap[CollationName2ID("latin1_swedish_ci")] = &latin1SwedishCICollator{}
 	newCollatorMap["utf8mb4_bin"] = &binPaddingCollator{}
 	newCollatorIDMap[CollationName2ID("utf8mb4_bin")] = &binPaddingCollator{}
 	newCollatorMap["utf8_bin"] = &binPaddingCollator{}

pkg/util/collate/collate_test.go

@@ -145,13 +145,21 @@ func TestUTF8CollatorKey(t *testing.T) {
 	testKeyTable(t, collations, tests)
 }
 
+// latin1_swedish_ci is deliberately not registered in newCollatorMap yet, so the
+// latin1 tests below construct it directly instead of going through GetCollator.
+// See the doc comment on latin1SwedishCICollator.
+func latin1TestCollators() (collators []Collator, names []string) {
+	return []Collator{GetCollator("latin1_bin"), &latin1SwedishCICollator{}},
+		[]string{"latin1_bin", "latin1_swedish_ci"}
+}
+
 // The latin1 collators are byte-oriented, so the test data below spells the
 // interesting characters as explicit cp1252 bytes ("\xC4" for Ä) rather than as Go
 // source literals, which the compiler would encode as UTF-8.
 func TestLatin1CollatorCompare(t *testing.T) {
 	SetNewCollationEnabledForTest(true)
 	defer SetNewCollationEnabledForTest(false)
-	collations := []string{"latin1_bin", "latin1_swedish_ci"}
+	collators, names := latin1TestCollators()
 	tests := []compareTable{
 		{"a", "b", []int{-1, -1}},
 		{"a", "A", []int{1, 0}},
@@ -187,13 +195,18 @@ func TestLatin1CollatorCompare(t *testing.T) {
 		// limitation rather than endorsing it.
 		{"\xC3\x84", "a\x84", []int{1, 0}},
 	}
-	testCompareTable(t, collations, tests)
+	for i, collator := range collators {
+		for _, table := range tests {
+			comment := fmt.Sprintf("Compare Left: %q Right: %q, Using %v", table.Left, table.Right, names[i])
+			require.Equal(t, table.Expect[i], collator.Compare(table.Left, table.Right), comment)
+		}
+	}
 }
 
 func TestLatin1CollatorKey(t *testing.T) {
 	SetNewCollationEnabledForTest(true)
 	defer SetNewCollationEnabledForTest(false)
-	collations := []string{"latin1_bin", "latin1_swedish_ci"}
+	collators, names := latin1TestCollators()
 	tests := []keyTable{
 		{"a", [][]byte{{0x61}, {0x41}}},
 		{"A", [][]byte{{0x41}, {0x41}}},
@@ -204,12 +217,14 @@ func TestLatin1CollatorKey(t *testing.T) {
 		{"\xFF\xDD", [][]byte{{0xFF, 0xDD}, {0xFF, 0x59}}},
 		{"", [][]byte{{}, {}}},
 	}
-	testKeyTable(t, collations, tests)
-
-	// One weight byte per input byte, so the sort key never outgrows the raw data.
-	collator := GetCollator("latin1_swedish_ci")
-	for _, test := range tests {
-		require.LessOrEqual(t, len(collator.Key(test.Str)), collator.MaxKeyLen(test.Str), test.Str)
+	for i, collator := range collators {
+		for _, test := range tests {
+			comment := fmt.Sprintf("key %q, using %v", test.Str, names[i])
+			require.Equal(t, test.Expect[i], collator.Key(test.Str), comment)
+			require.Equal(t, test.Expect[i], collator.ImmutableKey(test.Str), comment)
+			// One weight byte per input byte, so the sort key never outgrows the raw data.
+			require.LessOrEqual(t, len(collator.Key(test.Str)), collator.MaxKeyLen(test.Str), comment)
+		}
 	}
 }
 
@@ -232,7 +247,7 @@ func TestLatin1SwedishCIPattern(t *testing.T) {
 		{"a\\%b", '\\', "AxB", false},
 	}
 	for _, test := range tests {
-		pattern := GetCollator("latin1_swedish_ci").Pattern()
+		pattern := (&latin1SwedishCICollator{}).Pattern()
 		pattern.Compile(test.pattern, test.escape)
 		require.Equal(t, test.match, pattern.DoMatch(test.str),
 			fmt.Sprintf("pattern %q, str %q", test.pattern, test.str))
@@ -283,8 +298,6 @@ func TestGetCollator(t *testing.T) {
 	require.IsType(t, &unicodeCICollator{}, GetCollatorByID(192))
 	require.IsType(t, &unicode0900AICICollator{}, GetCollatorByID(255))
 	require.IsType(t, &zhPinyinTiDBASCSCollator{}, GetCollatorByID(2048))
-	require.IsType(t, &latin1SwedishCICollator{}, GetCollator("latin1_swedish_ci"))
-	require.IsType(t, &latin1SwedishCICollator{}, GetCollatorByID(8))
 	require.IsType(t, &binPaddingCollator{}, GetCollatorByID(9999))
 
 	SetNewCollationEnabledForTest(false)

pkg/util/collate/latin1_swedish_ci.go

@@ -22,6 +22,19 @@ import (
 
 // latin1SwedishCICollator is the collator for latin1_swedish_ci.
 //
+// It is deliberately NOT registered in newCollatorMap/newCollatorIDMap yet, so
+// GetCollationByName still rejects the collation and no column can be created with
+// it. Enabling it needs three things that are not in place:
+//   - TiKV and TiFlash collators, so pushed-down expression evaluation agrees with
+//     the sort key TiDB writes, plus a guard that keeps expressions in TiDB until
+//     the storage layer understands the collation.
+//   - A byte-safe case fold for the collate.IsCICollation fast paths in
+//     pkg/expression. Those apply strings.ToLower, which assumes UTF-8 and maps
+//     every latin1 byte >= 0x80 to U+FFFD, making distinct characters compare equal.
+//   - The documentation updates tracked in the feature request.
+//
+// Until then this type stands alone and is exercised directly by its unit tests.
+//
 // Unlike the utf8mb4 collators this one is byte-oriented rather than rune-oriented.
 // MySQL defines latin1_swedish_ci as a 256-entry weight table indexed by the cp1252
 // byte value (strings/ctype-latin1.c, sort_order_latin1), and TiDB's latin1 charset