collate: add a latin1_swedish_ci collator

Implement latin1_swedish_ci as a byte-oriented collator. MySQL defines it as a
256-entry weight table indexed by the cp1252 byte value and drives it through
my_collation_8bit_simple_ci_handler with strxfrm_multiply 1, and TiDB's latin1
charset stores raw client bytes (charset.EncodingLatin1Impl uses encoding.Nop
with a no-op Transform), so staying byte-oriented matches both MySQL and the
latin1_bin collator already available on the same column. One weight byte per
input byte also means the sort key never outgrows the raw data.

The weight table is transcribed from sort_order_latin1 in strings/ctype-latin1.cc
and was diffed entry by entry against that source. Note that ß (0xDF) folds to
nothing here and so equals neither "s" nor "ss"; the expansion belongs to
latin1_german2_ci, which uses the separate sort_order_latin1_de table.

stringutil gains DoMatchCustomizedBinary, the byte-oriented counterpart of
DoMatchCustomized, for the LIKE path: DoMatchBinary only offers exact byte
equality, which cannot express a case-insensitive match.

The collation is registered so it can be exercised end to end, but it is kept
out of GetSupportedCollations for now, following the utf8mb4_zh_pinyin_tidb_as_cs
precedent. TiKV and TiFlash have no latin1_swedish_ci collator, so they cannot
reproduce the sort key that TiDB writes into indexes, and nothing yet prevents a
predicate on such a column from being pushed down. That has to be resolved before
the collation is offered to users.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TgAxo4cK4qCpuLrhroHWy1

Author: terry1purcell

pkg/util/collate/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "gbk_chinese_ci.go",
         "gbk_chinese_ci_data.go",
         "general_ci.go",
+        "latin1_swedish_ci.go",
         "pinyin_tidb_as_cs.go",
         "unicode_0400_ci_generated.go",
         "unicode_0400_ci_impl.go",

pkg/util/collate/collate.go

@@ -251,7 +251,9 @@ func GetSupportedCollations() []*charset.Collation {
 		newSupportedCollations := make([]*charset.Collation, 0, len(newCollatorMap))
 		for name := range newCollatorMap {
 			// utf8mb4_zh_pinyin_tidb_as_cs is under developing, should not be shown to user.
-			if name == "utf8mb4_zh_pinyin_tidb_as_cs" {
+			// latin1_swedish_ci is under developing too: its sort key has no counterpart in
+			// TiKV/TiFlash yet, so pushed-down comparisons cannot reproduce it.
+			if name == "utf8mb4_zh_pinyin_tidb_as_cs" || name == "latin1_swedish_ci" {
 				continue
 			}
 			if coll, err := charset.GetCollationByName(name); err != nil {
@@ -311,7 +313,8 @@ func IsDefaultCollationForUTF8MB4(collate string) bool {
 func IsCICollation(collate string) bool {
 	return collate == "utf8_general_ci" || collate == "utf8mb4_general_ci" ||
 		collate == "utf8_unicode_ci" || collate == "utf8mb4_unicode_ci" || collate == "gbk_chinese_ci" ||
-		collate == "utf8mb4_0900_ai_ci" || collate == "gb18030_chinese_ci"
+		collate == "utf8mb4_0900_ai_ci" || collate == "gb18030_chinese_ci" ||
+		collate == "latin1_swedish_ci"
 }
 
 // ConvertAndGetBinCollation converts collation to binary collation
@@ -331,6 +334,8 @@ func ConvertAndGetBinCollation(collate string) string {
 		return "gbk_bin"
 	case "gb18030_chinese_ci":
 		return "gb18030_bin"
+	case "latin1_swedish_ci":
+		return "latin1_bin"
 	}
 
 	return collate
@@ -449,6 +454,8 @@ func init() {
 	newCollatorIDMap[CollationName2ID("ascii_bin")] = &binPaddingCollator{}
 	newCollatorMap["latin1_bin"] = &binPaddingCollator{}
 	newCollatorIDMap[CollationName2ID("latin1_bin")] = &binPaddingCollator{}
+	newCollatorMap["latin1_swedish_ci"] = &latin1SwedishCICollator{}
+	newCollatorIDMap[CollationName2ID("latin1_swedish_ci")] = &latin1SwedishCICollator{}
 	newCollatorMap["utf8mb4_bin"] = &binPaddingCollator{}
 	newCollatorIDMap[CollationName2ID("utf8mb4_bin")] = &binPaddingCollator{}
 	newCollatorMap["utf8_bin"] = &binPaddingCollator{}

pkg/util/collate/collate_test.go

@@ -145,6 +145,100 @@ func TestUTF8CollatorKey(t *testing.T) {
 	testKeyTable(t, collations, tests)
 }
 
+// The latin1 collators are byte-oriented, so the test data below spells the
+// interesting characters as explicit cp1252 bytes ("\xC4" for Ä) rather than as Go
+// source literals, which the compiler would encode as UTF-8.
+func TestLatin1CollatorCompare(t *testing.T) {
+	SetNewCollationEnabledForTest(true)
+	defer SetNewCollationEnabledForTest(false)
+	collations := []string{"latin1_bin", "latin1_swedish_ci"}
+	tests := []compareTable{
+		{"a", "b", []int{-1, -1}},
+		{"a", "A", []int{1, 0}},
+		{"abc", "abc", []int{0, 0}},
+		{"abc", "ab", []int{1, 1}},
+		// Both latin1 collations are PAD SPACE.
+		{"a", "a ", []int{0, 0}},
+		{"a\t", "a", []int{1, 1}},
+		{"\xE9", "E", []int{1, 0}}, // é folds to E
+		{"\xE0\xE9", "AE", []int{1, 0}},
+		// Swedish letters sort after Z in the order Å, Ä, Ö instead of folding to A and O.
+		{"\xC5", "A", []int{1, 1}}, // Å
+		{"\xC5", "Z", []int{1, 1}},
+		{"\xC4", "\xC5", []int{-1, 1}}, // Ä after Å, opposite of the byte order
+		{"\xD6", "\xC4", []int{1, 1}},  // Ö after Ä
+		{"\xC6", "\xC4", []int{1, 0}},  // Æ collates as Ä
+		// MySQL quirks worth pinning down: Ü and Ý fold to Y but ÿ does not; ß folds to
+		// nothing, so it is equal to neither "s" nor "ss" (that is latin1_german2_ci);
+		// and Ø/ø fold to each other but not to O.
+		{"\xDC", "Y", []int{1, 0}},
+		{"\xDD", "y", []int{1, 0}},
+		{"\xFF", "Y", []int{1, 1}},
+		{"\xDF", "s", []int{1, 1}},
+		{"\xDF", "ss", []int{1, 1}},
+		{"\xDF", "\xDF", []int{0, 0}},
+		{"\xF8", "\xD8", []int{1, 0}},
+		{"\xD8", "O", []int{1, 1}},
+		// Bytes without a letter weight keep their own value, so they still compare
+		// equal to themselves.
+		{"\x80", "\x80", []int{0, 0}},
+		// UTF-8 data stored in a latin1 column is weighted byte by byte: "Ä" encoded as
+		// UTF-8 is 0xC3 0x84, whose leading byte folds to A. This documents the known
+		// limitation rather than endorsing it.
+		{"\xC3\x84", "a\x84", []int{1, 0}},
+	}
+	testCompareTable(t, collations, tests)
+}
+
+func TestLatin1CollatorKey(t *testing.T) {
+	SetNewCollationEnabledForTest(true)
+	defer SetNewCollationEnabledForTest(false)
+	collations := []string{"latin1_bin", "latin1_swedish_ci"}
+	tests := []keyTable{
+		{"a", [][]byte{{0x61}, {0x41}}},
+		{"A", [][]byte{{0x41}, {0x41}}},
+		{"a  ", [][]byte{{0x61}, {0x41}}},
+		{"Hello", [][]byte{{0x48, 0x65, 0x6C, 0x6C, 0x6F}, {0x48, 0x45, 0x4C, 0x4C, 0x4F}}},
+		{"\xC4\xC5\xD6", [][]byte{{0xC4, 0xC5, 0xD6}, {0x5C, 0x5B, 0x5D}}},
+		{"\xDF", [][]byte{{0xDF}, {0xDF}}},
+		{"\xFF\xDD", [][]byte{{0xFF, 0xDD}, {0xFF, 0x59}}},
+		{"", [][]byte{{}, {}}},
+	}
+	testKeyTable(t, collations, tests)
+
+	// One weight byte per input byte, so the sort key never outgrows the raw data.
+	collator := GetCollator("latin1_swedish_ci")
+	for _, test := range tests {
+		require.LessOrEqual(t, len(collator.Key(test.Str)), collator.MaxKeyLen(test.Str), test.Str)
+	}
+}
+
+func TestLatin1SwedishCIPattern(t *testing.T) {
+	SetNewCollationEnabledForTest(true)
+	defer SetNewCollationEnabledForTest(false)
+	tests := []struct {
+		pattern string
+		escape  byte
+		str     string
+		match   bool
+	}{
+		{"a%", '\\', "ABC", true},
+		{"A_C", '\\', "abc", true},
+		{"A_C", '\\', "abcd", false},
+		{"%\xC4%", '\\', "x\xE4y", true},  // Ä matches ä
+		{"%\xC4%", '\\', "x\xC5y", false}, // but not Å
+		{"\xDC", '\\', "y", true},         // Ü matches y, same as Compare
+		{"a\\%b", '\\', "A%B", true},      // the escaped wildcard is still a literal
+		{"a\\%b", '\\', "AxB", false},
+	}
+	for _, test := range tests {
+		pattern := GetCollator("latin1_swedish_ci").Pattern()
+		pattern.Compile(test.pattern, test.escape)
+		require.Equal(t, test.match, pattern.DoMatch(test.str),
+			fmt.Sprintf("pattern %q, str %q", test.pattern, test.str))
+	}
+}
+
 func TestSetNewCollateEnabled(t *testing.T) {
 	defer SetNewCollationEnabledForTest(false)
 
@@ -189,6 +283,8 @@ func TestGetCollator(t *testing.T) {
 	require.IsType(t, &unicodeCICollator{}, GetCollatorByID(192))
 	require.IsType(t, &unicode0900AICICollator{}, GetCollatorByID(255))
 	require.IsType(t, &zhPinyinTiDBASCSCollator{}, GetCollatorByID(2048))
+	require.IsType(t, &latin1SwedishCICollator{}, GetCollator("latin1_swedish_ci"))
+	require.IsType(t, &latin1SwedishCICollator{}, GetCollatorByID(8))
 	require.IsType(t, &binPaddingCollator{}, GetCollatorByID(9999))
 
 	SetNewCollationEnabledForTest(false)

pkg/util/collate/latin1_swedish_ci.go

@@ -0,0 +1,148 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collate
+
+import (
+	"cmp"
+
+	"github.com/pingcap/tidb/pkg/util/stringutil"
+)
+
+// latin1SwedishCICollator is the collator for latin1_swedish_ci.
+//
+// Unlike the utf8mb4 collators this one is byte-oriented rather than rune-oriented.
+// MySQL defines latin1_swedish_ci as a 256-entry weight table indexed by the cp1252
+// byte value (strings/ctype-latin1.c, sort_order_latin1), and TiDB's latin1 charset
+// stores whatever bytes the client sent: charset.EncodingLatin1Impl uses encoding.Nop
+// and a no-op Transform. Staying byte-oriented therefore matches both MySQL and the
+// existing latin1_bin collator for the same column.
+//
+// The consequence is that a multi-byte UTF-8 sequence stored in a latin1 column is
+// weighted byte by byte rather than as one character. That is the same treatment
+// latin1_bin already gives it, so it is not a new divergence, but it does mean the
+// collation is only MySQL-faithful for genuinely cp1252-encoded data.
+//
+// Because every input byte maps to exactly one weight byte, the sort key has the same
+// length as the (space-trimmed) input and index key sizes are unchanged relative to
+// latin1_bin. The sort key is not the raw data though, so latin1_swedish_ci columns
+// need restored data in indexes - see types.NeedRestoredDataWithCollate.
+type latin1SwedishCICollator struct{}
+
+// Compare implements Collator interface.
+func (*latin1SwedishCICollator) Compare(a, b string) int {
+	a = truncateTailingSpace(a)
+	b = truncateTailingSpace(b)
+	for i := 0; i < len(a) && i < len(b); i++ {
+		if c := cmp.Compare(latin1SwedishCISortOrder[a[i]], latin1SwedishCISortOrder[b[i]]); c != 0 {
+			return c
+		}
+	}
+	return cmp.Compare(len(a), len(b))
+}
+
+// Key implements Collator interface.
+func (c *latin1SwedishCICollator) Key(str string) []byte {
+	return c.KeyWithoutTrimRightSpace(truncateTailingSpace(str))
+}
+
+// ImmutableKey implements Collator interface.
+func (c *latin1SwedishCICollator) ImmutableKey(str string) []byte {
+	return c.KeyWithoutTrimRightSpace(truncateTailingSpace(str))
+}
+
+// KeyWithoutTrimRightSpace implements Collator interface.
+func (*latin1SwedishCICollator) KeyWithoutTrimRightSpace(str string) []byte {
+	buf := make([]byte, 0, len(str))
+	for i := range len(str) {
+		buf = append(buf, latin1SwedishCISortOrder[str[i]])
+	}
+	return buf
+}
+
+// MaxKeyLen implements Collator interface.
+func (*latin1SwedishCICollator) MaxKeyLen(s string) int {
+	return len(s)
+}
+
+// Pattern implements Collator interface.
+func (*latin1SwedishCICollator) Pattern() WildcardPattern {
+	return &latin1SwedishCIPattern{}
+}
+
+// Clone implements Collator interface.
+func (*latin1SwedishCICollator) Clone() Collator {
+	return new(latin1SwedishCICollator)
+}
+
+type latin1SwedishCIPattern struct {
+	patChars []byte
+	patTypes []byte
+}
+
+// Compile implements WildcardPattern interface.
+func (p *latin1SwedishCIPattern) Compile(patternStr string, escape byte) {
+	p.patChars, p.patTypes = stringutil.CompilePatternBinary(patternStr, escape)
+}
+
+// DoMatch implements WildcardPattern interface.
+func (p *latin1SwedishCIPattern) DoMatch(str string) bool {
+	return stringutil.DoMatchCustomizedBinary(str, p.patChars, p.patTypes, func(a, b byte) bool {
+		return latin1SwedishCISortOrder[a] == latin1SwedishCISortOrder[b]
+	})
+}
+
+// latin1SwedishCISortOrder is MySQL's sort_order_latin1, transcribed verbatim from
+// strings/ctype-latin1.cc in mysql-server. That is the table my_charset_latin1
+// (collation id 8, latin1_swedish_ci) hands to my_collation_8bit_simple_ci_handler.
+//
+// The entries carrying the actual Swedish semantics, and the ones to check first if a
+// comparison looks wrong: Å (0xC5) -> 0x5B, Ä/Æ (0xC4/0xC6) -> 0x5C, Ö (0xD6) -> 0x5D.
+// These sit just above 'Z' (0x5A), so the Swedish letters sort after the ASCII
+// alphabet rather than folding into A and O.
+//
+// Several accented characters deliberately do not fold to a base letter, which is
+// easy to get wrong by reasoning from Unicode case folding instead of from this table:
+// Ü (0xDC) and Ý (0xDD) both fold to 'Y', but ÿ (0xFF) keeps weight 0xFF; ß (0xDF)
+// keeps weight 0xDF and is therefore NOT equal to 's' or "ss" (that expansion belongs
+// to latin1_german2_ci, which uses the separate sort_order_latin1_de table); and
+// Þ/þ and Ø/ø fold to each other but to no letter.
+var latin1SwedishCISortOrder = [256]byte{
+	// 0x00 - 0x3F: control characters, punctuation and digits keep their own value.
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+	0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F,
+	0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F,
+	0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F,
+	// 0x40 - 0x5F: uppercase ASCII, unchanged.
+	0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,
+	0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
+	// 0x60 - 0x7F: lowercase ASCII folds onto uppercase.
+	0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,
+	0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F,
+	// 0x80 - 0xBF: cp1252 extras and symbols keep their own value.
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F,
+	0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF,
+	0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
+	// 0xC0 - 0xDF: accented uppercase letters.
+	// À Á Â Ã fold to A; Ä Å Æ take the Swedish weights; Ç -> C; È..Ë -> E; Ì..Ï -> I.
+	0x41, 0x41, 0x41, 0x41, 0x5C, 0x5B, 0x5C, 0x43, 0x45, 0x45, 0x45, 0x45, 0x49, 0x49, 0x49, 0x49,
+	// Ð -> D; Ñ -> N; Ò..Õ -> O; Ö -> 0x5D; × and Ø keep their own value;
+	// Ù..Û -> U; Ü and Ý -> Y; Þ and ß keep their own value.
+	0x44, 0x4E, 0x4F, 0x4F, 0x4F, 0x4F, 0x5D, 0xD7, 0xD8, 0x55, 0x55, 0x55, 0x59, 0x59, 0xDE, 0xDF,
+	// 0xE0 - 0xFF: accented lowercase letters, mirroring 0xC0 - 0xDF except for
+	// ÷ (0xF7) and ÿ (0xFF), which keep their own value.
+	0x41, 0x41, 0x41, 0x41, 0x5C, 0x5B, 0x5C, 0x43, 0x45, 0x45, 0x45, 0x45, 0x49, 0x49, 0x49, 0x49,
+	0x44, 0x4E, 0x4F, 0x4F, 0x4F, 0x4F, 0x5D, 0xF7, 0xD8, 0x55, 0x55, 0x55, 0x59, 0x59, 0xDE, 0xFF,
+}

pkg/util/stringutil/string_util.go

@@ -286,6 +286,14 @@ func DoMatchBinary(str string, patChars, patTypes []byte) bool {
 	return doMatchInner(lenPatWeights, lenBytes, patTypes, func(a, b int) bool { return bytes[a] == patChars[b] })
 }
 
+// DoMatchCustomizedBinary is the byte-oriented counterpart of `DoMatchCustomized`:
+// it matches `str` one byte at a time using a caller-supplied comparator. It exists
+// for collations whose weights are defined per byte rather than per rune, such as
+// latin1_swedish_ci.
+func DoMatchCustomizedBinary(str string, patWeights, patTypes []byte, matcher func(a, b byte) bool) bool {
+	return doMatchInner(len(patWeights), len(str), patTypes, func(a, b int) bool { return matcher(str[a], patWeights[b]) })
+}
+
 // DoMatch is an adapter for `DoMatchCustomized`, `str` can be any unicode string.
 func DoMatch(str string, patChars []rune, patTypes []byte) bool {
 	return DoMatchCustomized(str, patChars, patTypes, matchRune)