feat(funbox): added support for cyrillic and arabic charset (@m4dd0c)

frontend/scripts/json-validation.cjs

@@ -402,6 +402,9 @@ function validateLanguages() {
         rightToLeft: { type: "boolean" },
         noLazyMode: { type: "boolean" },
         bcp47: { type: "string" },
+        charset: {
+          type: "string",
+        },
         words: {
           type: "array",
           items: { type: "string", minLength: 1 },

frontend/src/ts/test/funbox/funbox-functions.ts

@@ -417,8 +417,17 @@ const list: Partial<Record<FunboxName, FunboxFunctions>> = {
     },
   },
   gibberish: {
-    getWord(): string {
-      return GetText.getGibberish();
+    async withWords(words): Promise<Wordset> {
+      if (!words || words.length === 0) {
+        return new Wordset([]);
+      }
+
+      const lang = await JSONData.getLanguage(Config.language);
+      const gibberishWords = words.map(() =>
+        GetText.getGibberish(lang?.charset || "latin")
+      );
+
+      return new Wordset(gibberishWords);
     },
   },
   ascii: {

frontend/src/ts/utils/charsetRange.ts

@@ -0,0 +1,85 @@
+export const charsetRanges = {
+  arabic: [
+    { start: 1569, end: 1594 }, // U+0621–U+063A (ء to غ)
+    { start: 1601, end: 1608 }, // U+0641–U+0648 (ف to و)
+    { start: 1610, end: 1610 }, // U+064A (ي)
+  ],
+  latin: [
+    { start: 97, end: 122 }, // U+0061-U+007A (a to z)
+  ],
+  cyrillic: [
+    { start: 1072, end: 1103 }, //  U+0430-U+044F (а to я)
+  ],
+  devanagari: [
+    { start: 2309, end: 2361 }, // U+0905–U+0939 (अ to ह)
+    { start: 2366, end: 2376 }, // U+093E–U+0948 (vowel signs आ to ऐ)
+  ],
+  gujarati: [
+    { start: 2693, end: 2702 }, // U+0A85–U+0A94 (અ to ઔ)
+    { start: 2705, end: 2745 }, // U+0A95–U+0AB9 (ક to હ)
+    { start: 2750, end: 2764 }, // U+0ABE–U+0ACC (vowel signs ા to ૌ)
+  ],
+  geez: [
+    { start: 4768, end: 4960 }, // U+1200–U+135F (ሀ to ፟)
+  ],
+  tamil: [
+    { start: 2949, end: 3020 }, // U+0B85–U+0BBC (அ to ஔ)
+    { start: 3006, end: 3028 }, // U+0BBE–U+0BCC (vowel signs ா to ௌ)
+  ],
+  telugu: [
+    { start: 3077, end: 3148 }, // U+0C05–U+0C4C (అ to ౌ)
+    { start: 3158, end: 3160 }, // U+0C56–U+0C58 (additional vowels ౖ to ౘ)
+  ],
+  bengali: [
+    { start: 2437, end: 2489 }, // U+0985–U+09B9 (অ to হ)
+    { start: 2494, end: 2508 }, // U+09BE–U+09CC (vowel signs া to ৌ)
+  ],
+  malayalam: [
+    { start: 3333, end: 3396 }, // U+0D05–U+0D3C (അ to ഹ)
+    { start: 3398, end: 3404 }, // U+0D3E–U+0D44 (vowel signs ാ to ൄ)
+  ],
+  kannada: [
+    { start: 3205, end: 3268 }, // U+0C85–U+0CBC (ಅ to ಹ)
+    { start: 3270, end: 3276 }, // U+0CBE–U+0CC4 (vowel signs ಾ to ೄ)
+  ],
+  burmese: [
+    { start: 4096, end: 4138 }, // U+1000–U+102A (က to ဪ)
+  ],
+  tibetan: [
+    { start: 3904, end: 3911 }, // U+0F40–U+0F47 (ཀ to ཧ)
+  ],
+  sinhala: [
+    { start: 3461, end: 3516 }, // U+0D85–U+0DBC (අ to හ)
+    { start: 3535, end: 3551 }, // U+0DCF–U+0DDF (vowel signs ඾ to ෟ)
+  ],
+  hebrew: [
+    { start: 1488, end: 1514 }, // U+05D0-U+05EA (א to ת)
+  ],
+  thai: [
+    { start: 3585, end: 3631 }, // U+0E01–U+0E2F (ก to ๏)
+  ],
+  greek: [
+    { start: 945, end: 969 }, // U+03B1-U+03C9 (α to ω)
+  ],
+  han: [
+    { start: 19968, end: 27903 }, // U+4E00–U+6CAF (common CJK ideographs)
+  ],
+  hangul: [
+    { start: 44032, end: 55203 }, // U+AC00-U+D7A3 (가 to 힣)
+  ],
+  khmer: [
+    { start: 6016, end: 6067 }, // U+1780–U+17B3 (ក to ឳ)
+  ],
+  ol_chiki: [
+    { start: 7248, end: 7293 }, // U+1C5A–U+1C7D (ᱚ to ᱽ)
+  ],
+  hiragana: [
+    { start: 12353, end: 12438 }, // U+3041-U+3096 (あ to ん)
+  ],
+  katakana: [
+    { start: 12449, end: 12538 }, // U+30A1-U+30FA (ア to ン)
+  ],
+} as const;
+
+// Charset type
+export type Charset = keyof typeof charsetRanges;

frontend/src/ts/utils/generate.ts

@@ -1,6 +1,7 @@
 import { randomIntFromRange } from "@monkeytype/util/numbers";
 import * as Arrays from "./arrays";
 import * as Strings from "./strings";
+import { Charset, charsetRanges } from "./charsetRange";
 
 /**
  * Generates a random binary string of length 8.
@@ -101,15 +102,28 @@ export function getMorse(word: string): string {
  * Generates a random gibberish string of lowercase letters.
  * @returns The generated gibberish string.
  */
-export function getGibberish(): string {
+export function getGibberish(charset: Charset): string {
   const randLen = randomIntFromRange(1, 7);
   let ret = "";
-  for (let i = 0; i < randLen; i++) {
-    ret += String.fromCharCode(97 + randomIntFromRange(0, 25));
+  const ranges = charsetRanges[charset];
+
+  const chars = [];
+  for (let range of ranges) {
+    for (let i = range.start; i < range.end; i++) {
+      chars.push(i);
+    }
+  }
+
+  while (ret.length < randLen) {
+    const ch = String.fromCharCode(Arrays.randomElementFromArray(chars));
+
+    // Sanitizing the character
+    // keeping letters and vowels, killing viramas
+    // ref: https://www.regular-expressions.info/unicode.html
+    if (/\p{L}|\p{Mc}/u.test(ch)) ret += ch;
   }
   return ret;
 }
-
 /**
  * Generates a random ASCII string of printable characters.
  * @returns The generated ASCII string.

frontend/src/ts/utils/json-data.ts

@@ -1,6 +1,7 @@
-import { FunboxName } from "@monkeytype/contracts/schemas/configs";
 import { Language } from "@monkeytype/contracts/schemas/languages";
 import { Accents } from "../test/lazy-mode";
+import { Charset } from "./charsetRange";
+import { FunboxName } from "@monkeytype/contracts/schemas/configs";
 
 /**
  * Fetches JSON data from the specified URL using the fetch API.
@@ -98,6 +99,7 @@ export type LanguageObject = {
   noLazyMode?: boolean;
   ligatures?: boolean;
   orderedByFrequency?: boolean;
+  charset?: Charset;
   words: string[];
   additionalAccents: Accents;
   bcp47?: string;
@@ -111,8 +113,7 @@ let currentLanguage: LanguageObject;
  * @param lang The language code.
  * @returns A promise that resolves to the language object.
  */
-export async function getLanguage(lang: Language): Promise<LanguageObject> {
-  // try {
+export async function getLanguage(lang: string): Promise<LanguageObject> {
   if (currentLanguage === undefined || currentLanguage.name !== lang) {
     currentLanguage = await cachedFetchJson<LanguageObject>(
       `/languages/${lang}.json`

frontend/static/languages/amharic.json

@@ -2,6 +2,7 @@
   "name": "amharic",
   "ligatures": false,
   "bcp47": "am-ET",
+  "charset": "geez",
   "words": [
     "እግዚአብሔር",
     "መጽሐፍ",

frontend/static/languages/amharic_1k.json

@@ -2,6 +2,7 @@
   "name": "amharic_1k",
   "ligatures": false,
   "bcp47": "am-ET",
+  "charset": "geez",
   "words": [
     "መለየት",
     "ማሰብ",

frontend/static/languages/amharic_5k.json

@@ -2,6 +2,7 @@
   "name": "amharic_5k",
   "ligatures": false,
   "bcp47": "am-ET",
+  "charset": "geez",
   "words": [
     "ሙዚቀኝነት",
     "የሚባለው",

frontend/static/languages/arabic.json

@@ -3,6 +3,7 @@
   "rightToLeft": true,
   "ligatures": true,
   "bcp47": "ar-SA",
+  "charset": "arabic",
   "words": [
     "أَتَمَنَّى",
     "أَثِقْ",

frontend/static/languages/arabic_10k.json

@@ -3,6 +3,7 @@
   "rightToLeft": true,
   "ligatures": true,
   "bcp47": "ar-SA",
+  "charset": "arabic",
   "words": [
     " اِكْتَشَفَ",
     " فَيَجِبُ",

frontend/static/languages/arabic_egypt.json

@@ -3,6 +3,7 @@
   "rightToLeft": true,
   "ligatures": true,
   "bcp47": "ar-EG",
+  "charset": "arabic",
   "words": [
     "ازيك",
     "ايه",

frontend/static/languages/arabic_egypt_1k.json

@@ -3,6 +3,7 @@
   "rightToLeft": true,
   "ligatures": true,
   "bcp47": "ar-EG",
+  "charset": "arabic",
   "words": [
     "بلاش",
     "بسرعة",

frontend/static/languages/armenian.json

@@ -2,6 +2,7 @@
   "name": "armenian",
   "noLazyMode": true,
   "orderedByFrequency": false,
+  "charset": "armenian",
   "words": [
     "ազատ",
     "քաղաք",

frontend/static/languages/armenian_1k.json

@@ -2,6 +2,7 @@
   "name": "armenian_1k",
   "noLazyMode": true,
   "orderedByFrequency": false,
+  "charset": "armenian",
   "words": [
     "ազատ",
     "քաղաք",

frontend/static/languages/armenian_western.json

@@ -1,6 +1,7 @@
 {
   "name": "armenian_western",
   "bcp47": "hyw",
+  "charset": "armenian",
   "words": [
     "կանանց",
     "իրեն",

frontend/static/languages/armenian_western_1k.json

@@ -1,6 +1,7 @@
 {
   "name": "armenian_western_1k",
   "bcp47": "hyw",
+  "charset": "armenian",
   "words": [
     "թարգմանուած",
     "անոր",

frontend/static/languages/bangla.json

@@ -3,6 +3,7 @@
   "ligatures": true,
   "noLazyMode": true,
   "bcp47": "bn-BD",
+  "charset": "bengali",
   "words": [
     "।",
     "আমি",

frontend/static/languages/bangla_10k.json

@@ -3,6 +3,7 @@
   "ligatures": true,
   "noLazyMode": true,
   "bcp47": "bn_BD",
+  "charset": "bengali",
   "words": [
     "।",
     "আমি",

frontend/static/languages/bangla_letters.json

@@ -3,6 +3,7 @@
   "ligatures": true,
   "noLazyMode": true,
   "bcp47": "bn-BD",
+  "charset": "bengali",
   "words": [
     "অ",
     "আ",

frontend/static/languages/bashkir.json

@@ -2,6 +2,7 @@
   "name": "bashkir",
   "bcp-47": "ba",
   "orderedByFrequency": true,
+  "charset": "cyrillic",
   "words": [
     "бер",
     "һәм",

frontend/static/languages/belarusian.json

@@ -2,6 +2,7 @@
   "name": "belarusian",
   "noLazyMode": true,
   "bcp47": "be-BY",
+  "charset": "cyrillic",
   "words": [
     "ён",
     "і",

frontend/static/languages/belarusian_100k.json

@@ -1,6 +1,7 @@
 {
   "name": "belarusian_100k",
   "bcp47": "be-BY",
+  "charset": "cyrillic",
   "words": [
     "а",
     "аагамія",

frontend/static/languages/belarusian_10k.json

@@ -1,6 +1,7 @@
 {
   "name": "belarusian_10k",
   "bcp47": "be-BY",
+  "charset": "cyrillic",
   "words": [
     "а",
     "аазіс",

frontend/static/languages/belarusian_1k.json

@@ -2,6 +2,7 @@
   "name": "belarusian_1k",
   "noLazyMode": true,
   "bcp47": "be-BY",
+  "charset": "cyrillic",
   "words": [
     "ён",
     "і",

frontend/static/languages/belarusian_25k.json

@@ -1,6 +1,7 @@
 {
   "name": "belarusian_25k",
   "bcp47": "be-BY",
+  "charset": "cyrillic",
   "words": [
     "а",
     "аагамія",

frontend/static/languages/belarusian_50k.json

@@ -1,6 +1,7 @@
 {
   "name": "belarusian_50k",
   "bcp47": "be-BY",
+  "charset": "cyrillic",
   "words": [
     "а",
     "аагамія",

frontend/static/languages/belarusian_5k.json

@@ -1,6 +1,7 @@
 {
   "name": "belarusian_5k",
   "bcp47": "be-BY",
+  "charset": "cyrillic",
   "words": [
     "аазіс",
     "ааліт",

frontend/static/languages/bulgarian.json

@@ -1,6 +1,7 @@
 {
   "name": "bulgarian",
   "noLazyMode": true,
+  "charset": "cyrillic",
   "words": [
     "а",
     "аз",

frontend/static/languages/chinese_simplified.json

@@ -2,6 +2,7 @@
   "name": "chinese_simplified",
   "_comment": "Sourced from https://gist.github.com/indiejoseph/eae09c673460aa0b56db",
   "bcp47": "zh-CN",
+  "charset": "han",
   "words": [
     "我们",
     "他们",

frontend/static/languages/chinese_simplified_10k.json

@@ -2,6 +2,7 @@
   "name": "chinese_simplified_10k",
   "_comment": "Sourced from https://gist.github.com/indiejoseph/eae09c673460aa0b56db",
   "bcp47": "zh-CN",
+  "charset": "han",
   "words": [
     "我们",
     "他们",

frontend/static/languages/chinese_simplified_1k.json

@@ -2,6 +2,7 @@
   "name": "chinese_simplified_1k",
   "_comment": "Sourced from https://gist.github.com/indiejoseph/eae09c673460aa0b56db",
   "bcp47": "zh-CN",
+  "charset": "han",
   "words": [
     "我们",
     "他们",

frontend/static/languages/chinese_simplified_50k.json

@@ -2,6 +2,7 @@
   "name": "chinese_simplified_50k",
   "_comment": "Sourced from https://gist.github.com/indiejoseph/eae09c673460aa0b56db",
   "bcp47": "zh-CN",
+  "charset": "han",
   "words": [
     "我们",
     "他们",