Allow non-ASCII characters in identifiers

Ruby accepts identifiers with non-ASCII code points: `Ú` and `Ω` are
valid constants, `únicos` and `日本語` are valid local identifiers. The
RBS lexer only recognised ASCII, so the same names could not be written
in a signature file. See rubocop/rubocop-on-rbs#151.

`rbs_next_char` already folded every non-ASCII character into a single
sentinel code point. It now reports one of two, chosen by the encoding's
`isupper_char` — the test Ruby's `rb_sym_constant_char_p` uses to tell a
constant from a local — and the grammar matches those two classes
directly. Widening `word` also lets every existing rule accept non-ASCII
continuation characters.

Deciding this in `rbs_next_char` rather than in a lexer action covers
encodings where a non-ASCII character is a single byte: in ISO-8859-1,
`À` is 0xC0 and `à` is 0xE0, and the old `byte_len == 1` branch passed
both through as raw bytes that matched no identifier rule. It costs one
`isupper_char` call per non-ASCII character rather than one per
identifier; over the bundled core and stdlib that is below the noise
floor, and even a corpus that is 46% Japanese prose is only ~12% slower.

Builds on #2983: the two sentinels are distinct from the invalid-byte
sentinel, so an invalid byte inside an identifier still surfaces as an
ErrorToken rather than being absorbed.

ASCII-8BIT needs an exception. It carries bytes rather than characters
and its `char_width` reports 1 for every byte, so folding "every valid
non-ASCII character" would turn arbitrary binary into identifiers.

`TypeName#kind` answers the same question for names that never pass
through the lexer, so it has to agree. Ask Prism — already a hard
dependency — instead of restating Ruby's rule with regexps, which covers
titlecase letters and non-UTF-8 names that a regexp got wrong. Only the
leading `_` case stays, since that is RBS's own interface convention.
`rust/ruby-rbs` has no Prism to defer to and keeps an explicit test.

src/lexer.c is regenerated with re2c 4.5.1.

Author: ksss

include/rbs/lexer.h

@@ -100,6 +100,10 @@ enum RBSTokenType {
     tANNOTATION, /* Annotation */
 };
 
+/* Arbitrary values, kept in sync by hand with the classes in src/lexer.re. */
+#define RBS_MB_UPPER_CODE_POINT 0x30EB /* ル */
+#define RBS_MB_OTHER_CODE_POINT 0x30D3 /* ビ */
+
 /**
  * The `byte_pos` (or `char_pos`) is the primary data.
  * The rest are cache.

lib/rbs/type_name.rb

@@ -9,17 +9,16 @@ class TypeName
     def initialize(namespace:, name:)
       @namespace = namespace
       @name = name
-      @kind = case
-              when name.match?(/\A[A-Z]/)
-                :class
-              when name.match?(/\A[a-z]/)
-                :alias
-              when name.start_with?("_")
-                :interface
-              else
-                # Defaults to :class
-                :class
-              end
+      # Not a regexp: `\p{...}` raises on a non-UTF-8 name, and no spelling of
+      # "uppercase" covers titlecase. Not `to_s`: `name` is already frozen.
+      @kind =
+        if name.start_with?("_")
+          :interface
+        elsif Prism::StringQuery.constant?(name.name)
+          :class
+        else
+          :alias
+        end
     end
 
     # Process-wide flyweight cache. Two-level Hash keyed by canonical

rust/ruby-rbs/src/type_name.rs

@@ -60,6 +60,24 @@ pub enum Kind {
     Interface,
 }
 
+/// `char::is_uppercase` excludes `Lt` and `std` cannot query a general
+/// category. Unicode has added no `Lt` since 3.0.
+const fn is_titlecase(c: char) -> bool {
+    matches!(
+        c,
+        '\u{01C5}'
+            | '\u{01C8}'
+            | '\u{01CB}'
+            | '\u{01F2}'
+            | '\u{1F88}'..='\u{1F8F}'
+            | '\u{1F98}'..='\u{1F9F}'
+            | '\u{1FA8}'..='\u{1FAF}'
+            | '\u{1FBC}'
+            | '\u{1FCC}'
+            | '\u{1FFC}'
+    )
+}
+
 /// An entry rooted in the absolute namespace (`::`).
 #[derive(Copy, Clone, Debug)]
 struct AbsoluteTypeNameEntry {
@@ -306,11 +324,21 @@ impl TypeNameInterner {
     #[must_use]
     pub fn kind(&self, name: TypeName, strings: &StringInterner) -> Option<Kind> {
         let seg = self.last_segment(name)?;
-        let bytes = strings.resolve(seg).as_bytes();
-        let first = *bytes.first()?;
-        Some(if first == b'_' {
+        let s = strings.resolve(seg);
+        let first_byte = *s.as_bytes().first()?;
+        Some(if first_byte == b'_' {
             Kind::Interface
-        } else if first.is_ascii_uppercase() {
+        } else if first_byte < 0x80 {
+            if first_byte.is_ascii_uppercase() {
+                Kind::Class
+            } else {
+                Kind::Alias
+            }
+        } else if s
+            .chars()
+            .next()
+            .is_some_and(|c| c.is_uppercase() || is_titlecase(c))
+        {
             Kind::Class
         } else {
             Kind::Alias
@@ -610,6 +638,32 @@ mod tests {
         assert_eq!(t.kind(root, &s), None);
     }
 
+    #[test]
+    fn kind_uses_unicode_uppercase_property() {
+        let (mut s, mut t) = setup();
+
+        let ultima = t.parse(&mut s, "Última");
+        let omega = t.parse(&mut s, "Ωmega");
+        let n_tilde = t.parse(&mut s, "Ñoño");
+        assert_eq!(t.kind(ultima, &s), Some(Kind::Class));
+        assert_eq!(t.kind(omega, &s), Some(Kind::Class));
+        assert_eq!(t.kind(n_tilde, &s), Some(Kind::Class));
+
+        // Lt is outside `Uppercase`, but Ruby starts a constant on it.
+        let dz = t.parse(&mut s, "Džfoo");
+        let greek_iota = t.parse(&mut s, "ᾼfoo");
+        assert_eq!(t.kind(dz, &s), Some(Kind::Class));
+        assert_eq!(t.kind(greek_iota, &s), Some(Kind::Class));
+
+        let alpha = t.parse(&mut s, "αlpha");
+        let kanji = t.parse(&mut s, "日本語");
+        assert_eq!(t.kind(alpha, &s), Some(Kind::Alias));
+        assert_eq!(t.kind(kanji, &s), Some(Kind::Alias));
+
+        let iface = t.parse(&mut s, "_Únicos");
+        assert_eq!(t.kind(iface, &s), Some(Kind::Interface));
+    }
+
     #[test]
     fn to_absolute_to_relative() {
         let (mut s, mut t) = setup();