[Py] add missing unicode categories in python library (#3872)

* add missing unicode categories in python library

* update changelogs

---------

Co-authored-by: Maxime Mangel <[email protected]>

Author: joprice

src/Fable.Cli/CHANGELOG.md

@@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * [All] Updated Fable-FCS to latest F# 9.0 (by @ncave)
 * [All] Updated metadata to latest .NET 9.0 (by @ncave)
 
+### Fixed
+
+* [Py] Add missing unicode categories in python library (by @joprice)
+
 ## 5.0.0-alpha.5 - 2025-01-09
 
 ### Added

src/Fable.Compiler/CHANGELOG.md

@@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * [All] Updated Fable-FCS to latest F# 9.0 (by @ncave)
 * [All] Updated metadata to latest .NET 9.0 (by @ncave)
 
+### Fixed
+
+* [Py] Add missing unicode categories in python library (by @joprice)
+
 ## 5.0.0-alpha.5 - 2025-01-09
 
 ### Added

src/fable-library-py/fable_library/char.py

@@ -60,7 +60,19 @@ class UnicodeCategory(IntEnum):
     "Sk": UnicodeCategory.ModifierSymbol,
     "Mn": UnicodeCategory.NonSpacingMark,
     "Lo": UnicodeCategory.OtherLetter,
-    "No": UnicodeCategory.OtherLetter,
+    "No": UnicodeCategory.OtherNumber,
+    "Lt": UnicodeCategory.TitlecaseLetter,
+    "Cn": UnicodeCategory.OtherNotAssigned,
+    "Co": UnicodeCategory.PrivateUse,
+    "Cs": UnicodeCategory.Surrogate,
+    "Zp": UnicodeCategory.ParagraphSeparator,
+    "Lm": UnicodeCategory.ModifierLetter,
+    "Mc": UnicodeCategory.SpacingCombiningMark,
+    "Me": UnicodeCategory.EnclosingMark,
+    "Pe": UnicodeCategory.ClosePunctuation,
+    "Pf": UnicodeCategory.FinalQuotePunctuation,
+    "Ps": UnicodeCategory.OpenPunctuation,
+    "So": UnicodeCategory.OtherSymbol,
 }
 
 

tests/Python/TestString.fs

@@ -991,3 +991,54 @@ let ``test calling ToString(CultureInfo.InvariantCulture) works`` () =
     (1).ToString(CultureInfo.InvariantCulture) |> equal "1"
     (7923209L).ToString(CultureInfo.InvariantCulture) |> equal "7923209"
     (7923209UL).ToString(CultureInfo.InvariantCulture) |> equal "7923209"
+
+
+#if FABLE_COMPILER
+open Fable.Core
+
+[<Import("category", "unicodedata")>]
+let unicodeCategory: char -> string = nativeOnly
+
+[<Fact>]
+let ``test unicode categories`` () =
+    let chars = [
+      "\x00", "Cc"
+      " ", "Zs"
+      "!", "Po"
+      "$", "Sc"
+      "(", "Ps"
+      ")", "Pe"
+      "+", "Sm"
+      "-", "Pd"
+      "0", "Nd"
+      "A", "Lu"
+      "^", "Sk"
+      "_", "Pc"
+      "a", "Ll"
+      "¦", "So"
+      "ª", "Lo"
+      "«", "Pi"
+      "\xad", "Cf"
+      "²", "No"
+      "»", "Pf"
+      "Dž", "Lt"
+      "ʰ", "Lm"
+      "", "Mn"
+      "\u0378", "Cn"
+      "\u0488", "Me"
+      "\u0903", "Mc"
+      "\u16ee", "Nl"
+      "\u2028", "Zl"
+      "\u2029", "Zp"
+     //TODO: this fails with error EXCEPTION: Unable to translate Unicode character \\uD800 at index 116 to specified code page.
+      //"\ud800" , "Cs"
+      "\ue000", "Co"
+    ]
+    for (s, cat) in chars do
+      s
+      |> String.iter (fun c ->
+        // this ensures that the character is from the expected category
+        cat |> equal (unicodeCategory c)
+        Char.IsLetterOrDigit c |> ignore
+      )
+#endif