mitigate expander hanging when non-UTF8 characters included

Author: james-willis

src/numex.c

@@ -724,7 +724,7 @@ numex_result_array *convert_numeric_expressions(char *str, char *lang) {
 
     while (idx < len) {
         if (state.state == NUMEX_SEARCH_STATE_SKIP_TOKEN) {
-            char_len = utf8proc_iterate(ptr, len, &codepoint);
+            char_len = utf8proc_iterate_non_negative(ptr, len, &codepoint);
             cat = utf8proc_category(codepoint);
 
             if (codepoint == 0) break;

src/string_utils.c

@@ -188,6 +188,11 @@ ssize_t utf8proc_iterate_reversed(const uint8_t *str, ssize_t start, int32_t *ds
     return ret_len;
 }
 
+ssize_t utf8proc_iterate_reversed_non_negative(const uint8_t *str, ssize_t start, int32_t *dst) {
+    ssize_t ret = utf8proc_iterate_reversed(str, start, dst);
+    return (ret < 1) ? 1 : ret;
+}
+
 char *utf8_reversed_string(const char *s) {
     int32_t unich;
     ssize_t len, remaining;

src/string_utils.h

@@ -69,6 +69,7 @@ uint32_t string_translate(char *str, size_t len, char *word_chars, char *word_re
 // UTF-8 string methods
 char *utf8_reversed_string(const char *s); // returns a copy, caller frees
 ssize_t utf8proc_iterate_reversed(const uint8_t *str, ssize_t start, int32_t *dst);
+ssize_t utf8proc_iterate_reversed_non_negative(const uint8_t *str, ssize_t start, int32_t *dst);
 
 // Casing functions return a copy, caller frees
 char *utf8_lower_options(const char *s, utf8proc_option_t options);

src/unicode_scripts.c

@@ -30,7 +30,7 @@ string_script_t get_string_script(char *str, size_t len) {
     bool is_ascii = true;
 
     while (idx < len) {
-        ssize_t char_len = utf8proc_iterate(ptr, len, &ch);
+        ssize_t char_len = utf8proc_iterate_non_negative(ptr, len, &ch);
 
         if (ch == 0) break;
 
@@ -43,7 +43,7 @@ string_script_t get_string_script(char *str, size_t len) {
         if (last_script != script && last_script != SCRIPT_UNKNOWN && !is_common_script(last_script)) {
             if (script_len < len) {
                 while (true) {
-                    char_len = utf8proc_iterate_reversed((const uint8_t *)str, idx, &ch);
+                    char_len = utf8proc_iterate_reversed_non_negative((const uint8_t *)str, idx, &ch);
                     if (ch == 0) break;
 
                     script = get_char_script((uint32_t)ch);

src/utf8proc/utf8proc.c

@@ -157,6 +157,13 @@ utf8proc_ssize_t utf8proc_iterate(
   return 4;
 }
 
+utf8proc_ssize_t utf8proc_iterate_non_negative(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
+) {
+  utf8proc_ssize_t ret = utf8proc_iterate(str, strlen, dst);
+  return (ret < 1) ? 1 : ret;
+}
+
 utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
     return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
 }
@@ -639,4 +646,3 @@ utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
     UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
   return retval;
 }
-

src/utf8proc/utf8proc.h

@@ -380,6 +380,11 @@ const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
  */
 utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
 
+/**
+ * Functions the same as @ref utf8proc_iterate, but does not ever return a value less than 1.
+ */
+utf8proc_ssize_t utf8proc_iterate_non_negative(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
+
 /**
  * Check if a codepoint is valid (regardless of whether it has been
  * assigned a value by the current Unicode standard).

test/test_expand.c

@@ -315,6 +315,15 @@ TEST test_expansions_no_options(void) {
     PASS();
 }
 
+TEST tests_utf16_case(void) {
+    libpostal_normalize_options_t options = libpostal_get_default_options();
+    options.address_components = LIBPOSTAL_ADDRESS_STREET | LIBPOSTAL_ADDRESS_ANY;
+
+    // This first case really should be "5-19 nakamachi". idk why the N is uppercase.
+    CHECK_CALL(test_root_expansion_contains("5-19�� Nakamachi", "5-19 Nakamachi", options));
+    CHECK_CALL(test_root_expansion_contains("No. 𝟣𝟣", "no 𝟣𝟣", options));
+}
+
 
 SUITE(libpostal_expansion_tests) {
     if (!libpostal_setup() || !libpostal_setup_language_classifier()) {
@@ -331,6 +340,7 @@ SUITE(libpostal_expansion_tests) {
     RUN_TEST(test_expansions_language_classifier);
     RUN_TEST(test_expansions_no_options);
     RUN_TEST(test_expansion_for_non_address_input);
+    RUN_TEST(tests_utf16_case);
 
     libpostal_teardown();
     libpostal_teardown_language_classifier();