feat(format-po-gettext): respect Plural-Forms header (#2070)

Author: garikkh

packages/format-po-gettext/package.json

@@ -45,6 +45,7 @@
     "@lingui/format-po": "4.13.0",
     "@lingui/message-utils": "4.13.0",
     "@messageformat/parser": "^5.0.0",
+    "cldr-core": "^45.0.0",
     "node-gettext": "^3.0.0",
     "plurals-cldr": "^2.0.1",
     "pofile": "^1.1.4"

packages/format-po-gettext/src/plural-samples.test.ts

@@ -0,0 +1,187 @@
+import {
+  createLocaleTest,
+  createSamples,
+  fillRange,
+  renameKeys,
+} from "./plural-samples"
+
+describe("Plural samples generation util", () => {
+  test.each([
+    [{ "pluralRule-count-zero": null }, { zero: null }],
+    [{ "pluralRule-count-one": null }, { one: null }],
+    [{ "pluralRule-count-two": null }, { two: null }],
+    [{ "pluralRule-count-few": null }, { few: null }],
+    [{ "pluralRule-count-many": null }, { many: null }],
+    [{ "pluralRule-count-other": null }, { other: null }],
+  ])("renameKeys", (original, expected) => {
+    expect(renameKeys(original)).toEqual(expected)
+  })
+
+  test("renameKeys multiple", () => {
+    const original = {
+      "pluralRule-count-zero":
+        "n = 0 @integer 0 @decimal 0.0, 0.00, 0.000, 0.0000",
+      "pluralRule-count-one":
+        "n = 1 @integer 1 @decimal 1.0, 1.00, 1.000, 1.0000",
+      "pluralRule-count-two":
+        "n = 2 @integer 2 @decimal 2.0, 2.00, 2.000, 2.0000",
+      "pluralRule-count-few":
+        "n % 100 = 3..10 @integer 3~10, 103~110, 1003, … @decimal 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 103.0, 1003.0, …",
+      "pluralRule-count-many":
+        "n % 100 = 11..99 @integer 11~26, 111, 1011, … @decimal 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 111.0, 1011.0, …",
+      "pluralRule-count-other":
+        " @integer 100~102, 200~202, 300~302, 400~402, 500~502, 600, 1000, 10000, 100000, 1000000, … @decimal 0.1~0.9, 1.1~1.7, 10.1, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, …",
+    }
+    expect(renameKeys(original)).toEqual({
+      zero: "n = 0 @integer 0 @decimal 0.0, 0.00, 0.000, 0.0000",
+      one: "n = 1 @integer 1 @decimal 1.0, 1.00, 1.000, 1.0000",
+      two: "n = 2 @integer 2 @decimal 2.0, 2.00, 2.000, 2.0000",
+      few: "n % 100 = 3..10 @integer 3~10, 103~110, 1003, … @decimal 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 103.0, 1003.0, …",
+      many: "n % 100 = 11..99 @integer 11~26, 111, 1011, … @decimal 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 111.0, 1011.0, …",
+      other:
+        " @integer 100~102, 200~202, 300~302, 400~402, 500~502, 600, 1000, 10000, 100000, 1000000, … @decimal 0.1~0.9, 1.1~1.7, 10.1, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, …",
+    })
+  })
+
+  test.each([
+    ["0~1", [0, 1]],
+    ["2~19", [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]],
+    ["100~102", [100, 101, 102]],
+  ])("fillRange - integer ranges", (range, values) => {
+    expect(fillRange(range)).toEqual(values)
+  })
+
+  test.each([
+    ["0.0~1.0", [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]],
+    // partials
+    [
+      "0.4~1.6",
+      [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6],
+    ],
+    ["0.04~0.09", [0.04, 0.05, 0.06, 0.07, 0.08, 0.09]],
+    [
+      "0.04~0.29",
+      [
+        0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.12, 0.13, 0.14, 0.15,
+        0.16, 0.17, 0.18, 0.19, 0.2, 0.21, 0.22, 0.23, 0.24, 0.25, 0.26, 0.27,
+        0.28, 0.29,
+      ],
+    ],
+  ])("fillRange - decimal ranges", (range, values) => {
+    expect(fillRange(range)).toEqual(values)
+  })
+
+  test("createSamples - single values", () => {
+    expect(createSamples("0")).toEqual([0])
+    expect(createSamples("0, 1, 2")).toEqual([0, 1, 2])
+    expect(createSamples("0, 1.0, 2.0")).toEqual([0, 1, 2])
+  })
+
+  test("createSamples - integer ranges", () => {
+    expect(createSamples("0~1")).toEqual([0, 1])
+    expect(createSamples("0~2")).toEqual([0, 1, 2])
+    expect(createSamples("0~10")).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+    expect(createSamples("2~17, 100, 1000, 10000, 100000, 1000000")).toEqual([
+      2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 100, 1000, 10000,
+      100000, 1000000,
+    ])
+  })
+
+  test("createSamples - mixed src", () => {
+    expect(createSamples("0.1~0.9")).toEqual([
+      0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9,
+    ])
+    // with ...
+    expect(
+      createSamples("0, 2~16, 100, 1000, 10000, 100000, 1000000, …")
+    ).toEqual([
+      0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 100, 1000, 10000,
+      100000, 1000000,
+    ])
+    // mixed with integer ranges
+    expect(
+      createSamples("0.1~0.9, 1.1~1.7, 10.0, 100.0, 1000.0, 10000.0, 100000.0")
+    ).toEqual([
+      0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6,
+      1.7, 10, 100, 1000, 10000, 100000,
+    ])
+    // trailing comma
+    expect(
+      createSamples("0.1~0.9, 1.1~1.7, 10.0, 100.0, 1000.0, 10000.0, 100000.0,")
+    ).toEqual([
+      0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6,
+      1.7, 10, 100, 1000, 10000, 100000,
+    ])
+  })
+
+  test("Run on ruleset", () => {
+    // ruleset for cs
+    const ruleset = {
+      "pluralRule-count-one": "i = 1 and v = 0 @integer 1",
+      "pluralRule-count-few": "i = 2..4 and v = 0 @integer 2~4",
+      "pluralRule-count-many":
+        "v != 0   @decimal 0.0~1.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, …",
+      "pluralRule-count-other":
+        " @integer 0, 5~19, 100, 1000, 10000, 100000, 1000000, …",
+    }
+    expect(createLocaleTest(ruleset)).toMatchInlineSnapshot(`
+      {
+        pluralRule-count-few: [
+          2,
+          3,
+          4,
+        ],
+        pluralRule-count-many: [
+          0,
+          0.1,
+          0.2,
+          0.3,
+          0.4,
+          0.5,
+          0.6,
+          0.7,
+          0.8,
+          0.9,
+          1,
+          1.1,
+          1.2,
+          1.3,
+          1.4,
+          1.5,
+          10,
+          100,
+          1000,
+          10000,
+          100000,
+          1000000,
+        ],
+        pluralRule-count-one: [
+          1,
+        ],
+        pluralRule-count-other: [
+          0,
+          5,
+          6,
+          7,
+          8,
+          9,
+          10,
+          11,
+          12,
+          13,
+          14,
+          15,
+          16,
+          17,
+          18,
+          19,
+          100,
+          1000,
+          10000,
+          100000,
+          1000000,
+        ],
+      }
+    `)
+  })
+})

packages/format-po-gettext/src/plural-samples.ts

@@ -0,0 +1,107 @@
+import cardinals from "cldr-core/supplemental/plurals.json"
+
+/*
+This script is heavily influenced by one that is used to generate plural samples
+found here: https://github.com/nodeca/plurals-cldr/blob/master/support/generate.js
+
+Ordinals were removed, and the original script supported strings and numbers,
+but for the use case of lingui-gettext formatter, we only want numbers.
+*/
+
+type PluralForm = "zero" | "one" | "two" | "few" | "many" | "other"
+type FormattedRuleset = Record<PluralForm, string>
+
+// Strip key prefixes to get clear names: zero / one / two / few / many / other
+// pluralRule-count-other -> other
+export function renameKeys(rules: Record<string, string>): FormattedRuleset {
+  const result = {}
+  Object.keys(rules).forEach((k) => {
+    const newKey = k.match(/[^-]+$/)[0]
+    result[newKey] = rules[k]
+  })
+  return result as FormattedRuleset
+}
+
+// Create array of sample values for single range
+// 5~16, 0.04~0.09. Both string & integer forms (when possible)
+export function fillRange(value: string): number[] {
+  let [start, end] = value.split("~")
+
+  const decimals = (start.split(".")[1] || "").length
+  // for example 0.1~0.9 has 10 values, need to add that many to list
+  // 0.004~0.009 has 100 values
+  let mult = Math.pow(10, decimals)
+
+  const startNum = Number(start)
+  const endNum = Number(end)
+
+  let range = Array(Math.ceil(endNum * mult - startNum * mult + 1))
+    .fill(0)
+    .map((v, idx) => (idx + startNum * mult) / mult)
+
+  let last = range[range.length - 1]
+
+  // Number defined in the range should be the last one, i.e. 5~16 should have 16
+  if (endNum !== last) {
+    throw new Error(`Range create error for ${value}: last value is ${last}`)
+  }
+
+  return range.map((v) => Number(v))
+}
+
+// Create array of test values for @integer or @decimal
+export function createSamples(src: string): number[] {
+  let result: number[] = []
+
+  src
+    .replace(/…/, "")
+    .trim()
+    .replace(/,$/, "")
+    .split(",")
+    .map(function (val) {
+      return val.trim()
+    })
+    .forEach((val) => {
+      if (val.indexOf("~") !== -1) {
+        result = result.concat(fillRange(val))
+      } else {
+        result.push(Number(val))
+      }
+    })
+
+  return result
+}
+
+// Create fixtures for single locale rules
+export function createLocaleTest(rules) {
+  let result = {}
+
+  Object.keys(rules).forEach((form) => {
+    let samples = rules[form].split(/@integer|@decimal/).slice(1)
+
+    result[form] = []
+    samples.forEach((sample) => {
+      result[form] = result[form].concat(createSamples(sample))
+    })
+  })
+
+  return result
+}
+
+export function getCldrPluralSamples(): Record<
+  string,
+  Record<PluralForm, number[]>
+> {
+  const pluralRules = {}
+
+  // Parse plural rules
+  Object.entries(cardinals.supplemental["plurals-type-cardinal"]).forEach(
+    ([loc, ruleset]) => {
+      let rules = renameKeys(ruleset)
+
+      pluralRules[loc.toLowerCase()] = createLocaleTest(rules)
+    }
+  )
+
+  return pluralRules
+}

packages/format-po-gettext/src/po-gettext.test.ts

@@ -231,6 +231,81 @@ msgstr[2] "# dní"
     expect(catalog).toMatchSnapshot()
   })
 
+  test("should use respect Plural-Forms header", () => {
+    const po = `
+msgid ""
+msgstr ""
+"Language: fr\\n"
+"Plural-Forms: nplurals=3; plural=(n == 0 || n == 1) ? 0 : n != 0 && n % 1000000 == 0 ? 1 : 2;\\n"
+
+#. js-lingui:icu=%7B0%2C+plural%2C+one+%7B%7Bcount%7D+day%7D+other+%7B%7Bcount%7D+days%7D%7D&pluralize_on=0
+msgid "{count} day"
+msgid_plural "{count} days"
+msgstr[0] "{count} jour"
+msgstr[1] "{count} jours"
+msgstr[2] "{count} jours"
+        `
+
+    const parsed = format.parse(po, defaultParseCtx)
+
+    // Note that the last case must be `other` (the 4th CLDR case name) instead of `many` (the 3rd CLDR case name).
+    expect(parsed).toMatchInlineSnapshot(`
+      {
+        ZETJEQ: {
+          comments: [],
+          context: null,
+          extra: {
+            flags: [],
+            translatorComments: [],
+          },
+          message: {0, plural, one {{count} day} other {{count} days}},
+          obsolete: false,
+          origin: [],
+          translation: {0, plural, one {{count} jour} many {{count} jours} other {{count} jours}},
+        },
+      }
+    `)
+  })
+
+  it("should correctly handle skipped form", () => {
+    // in this test Plural-Forms header defines 4 forms via `nplurals=4`
+    // but expression never returns 2 form, only [0, 1, 3]
+    const po = `
+msgid ""
+msgstr ""
+"Language: cs\n"
+"Plural-Forms: nplurals=4; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 3;\n"
+
+#. js-lingui:icu=%7Bcount%2C+plural%2C+one+%7B%7Bcount%7D+day%7D+few+%7B%7Bcount%7D+days%7D+many+%7B%7Bcount%7D+days%7D+other+%7B%7Bcount%7D+days%7D%7D&pluralize_on=#
+msgid "# day"
+msgid_plural "# days"
+msgstr[0] "# den"
+msgstr[1] "# dny"
+msgstr[2] "# dne"
+msgstr[3] "# dní"
+        `
+
+    const parsed = format.parse(po, defaultParseCtx)
+
+    // Note that the last case must be `other` (the 4th CLDR case name) instead of `many` (the 3rd CLDR case name).
+    expect(parsed).toMatchInlineSnapshot(`
+      {
+        GMnlGy: {
+          comments: [],
+          context: null,
+          extra: {
+            flags: [],
+            translatorComments: [],
+          },
+          message: {count, plural, one {{count} day} few {{count} days} many {{count} days} other {{count} days}},
+          obsolete: false,
+          origin: [],
+          translation: {#, plural, one {# den} few {# dny}  other {# dní}},
+        },
+      }
+    `)
+  })
+
   describe("using custom prefix", () => {
     it("parses plurals correctly", () => {
       const defaultProfile = fs