Unit: fix compound-symbol parsing, times-compound formatting, and derived-base custom units (#42, #43, #44); gitignore inflection data

Author: kipcole9

.gitignore

@@ -35,6 +35,11 @@ mise.toml
 !/priv/localize/locales/en.etf
 !/priv/localize/locales/und.etf
 
+# Inflection data (sources and generated ETFs) — belongs to the
+# inflection feature branch, not main
+/data/inflection/
+/priv/localize/inflection/
+
 # Experiment files
 bench_unit_parser.sh
 

data/data.ex

@@ -68,6 +68,7 @@ defmodule Localize.Data do
     {"common/supplemental/pluralRanges.xml", "pluralRanges.xml"},
     {"common/supplemental/subdivisions.xml", "subdivisions.xml"},
     {"common/supplemental/units.xml", "units.xml"},
+    {"common/supplemental/grammaticalFeatures.xml", "grammaticalFeatures.xml"},
     {"common/bcp47/timezone.xml", "bcp47_timezone.xml"}
   ]
 
@@ -123,6 +124,8 @@ defmodule Localize.Data do
     {"territory_subdivision_containment.etf",
      &Localize.Data.XmlExtractors.generate_territory_subdivision_containment/0},
     {"unit_data.etf", &Localize.Data.XmlExtractors.generate_unit_data/0},
+    {"unit_grammatical_derivations.etf",
+     &Localize.Data.XmlExtractors.generate_unit_grammatical_derivations/0},
     {"collation_tailoring.etf", &Localize.Data.Collation.generate_collation_tailoring/0},
     {"coverage_levels.etf", &Localize.Data.Supplemental.generate_coverage_levels/0},
     {"measurement_systems.etf", &Localize.Data.XmlExtractors.generate_measurement_systems/0},

data/xml_extractors.ex

@@ -319,6 +319,103 @@ defmodule Localize.Data.XmlExtractors do
     }
   end
 
+  @doc """
+  Generates the compound-unit grammatical-derivation table from
+  `grammaticalFeatures.xml`.
+
+  CLDR derives the plural category, grammatical case, and gender of a
+  compound unit's components from the compound as a whole (TR35
+  "Compound Units"). For example the default "times" derivation makes
+  the leading component singular and lets the trailing component carry
+  the count (`newton-meters`), while French pluralizes every component
+  (`tonnes-kilomètres`). The unit formatter reads this table to compose
+  the localized name of a compound unit that has no precomposed pattern.
+
+  Returns a map keyed by the base language subtag string (plus `"root"`
+  for the default), each value a map of the form:
+
+      %{
+        plural: %{times: {:one, :compound}, per: {:compound, :one}, ...},
+        case: %{times: {:nominative, :compound}, ...},
+        gender: %{times: 1, per: 0, ...}
+      }
+
+  For `deriveComponent` features (`:plural`, `:case`) the value is a
+  `{value0, value1}` tuple where `:compound` means "use the compound's
+  own category" and any other atom is a fixed category. For the
+  `deriveCompound` `:gender` feature the value is the `0`/`1` index of
+  the component whose gender the compound inherits. Each locale's block
+  is merged over `"root"` so per-feature fallbacks are already resolved.
+
+  """
+  def generate_unit_grammatical_derivations do
+    raw =
+      "grammatical_features.xml"
+      |> read_xml()
+      |> SweetXml.parse()
+      |> xpath(~x"//grammaticalDerivations"l,
+        locales: ~x"./@locales"s,
+        components: [
+          ~x"./deriveComponent"l,
+          feature: ~x"./@feature"s,
+          structure: ~x"./@structure"s,
+          value0: ~x"./@value0"s,
+          value1: ~x"./@value1"s
+        ],
+        compounds: [
+          ~x"./deriveCompound"l,
+          feature: ~x"./@feature"s,
+          structure: ~x"./@structure"s,
+          value: ~x"./@value"s
+        ]
+      )
+      |> Map.new(fn %{locales: locales, components: components, compounds: compounds} ->
+        {locales, build_derivation_map(components, compounds)}
+      end)
+
+    root = Map.get(raw, "root", %{})
+
+    raw
+    |> Enum.flat_map(fn {locales, derivations} ->
+      merged = deep_merge_derivations(root, derivations)
+      for locale <- String.split(locales), do: {locale, merged}
+    end)
+    |> Map.new()
+  end
+
+  # Folds a block's `deriveComponent`/`deriveCompound` rows into the
+  # nested `%{feature => %{structure => value}}` shape.
+  defp build_derivation_map(components, compounds) do
+    from_components =
+      Enum.reduce(components, %{}, fn %{feature: feature, structure: structure} = row, acc ->
+        value = {derivation_atom(row.value0), derivation_atom(row.value1)}
+        put_derivation(acc, feature, structure, value)
+      end)
+
+    Enum.reduce(compounds, from_components, fn %{feature: feature, structure: structure} = row,
+                                               acc ->
+      put_derivation(acc, feature, structure, String.to_integer(row.value))
+    end)
+  end
+
+  defp put_derivation(acc, feature, structure, value) do
+    feature = String.to_atom(feature)
+    structure = String.to_atom(structure)
+    Map.update(acc, feature, %{structure => value}, &Map.put(&1, structure, value))
+  end
+
+  defp derivation_atom(""), do: nil
+  defp derivation_atom(value), do: String.to_atom(value)
+
+  # Merges a locale block over `root` at the feature level, so a locale
+  # that overrides only some features (French overrides plural but not
+  # case) inherits the rest from root.
+  defp deep_merge_derivations(root, override) do
+    Map.merge(root, override, fn _feature, root_structures, override_structures ->
+      Map.merge(root_structures, override_structures)
+    end)
+  end
+
   @doc """
   Generates measurement system data from `bcp47/measure.xml`.
 
@@ -453,6 +550,7 @@ defmodule Localize.Data.XmlExtractors do
     "bcp47/timezone.xml" => {:bcp47, "timezone.xml"},
     "bcp47/measure.xml" => {:bcp47, "measure.xml"},
     "units.xml" => {:supplemental, "units.xml"},
+    "grammatical_features.xml" => {:supplemental, "grammaticalFeatures.xml"},
     "validity/unit.xml" => {:validity, "unit.xml"}
   }
 

lib/localize/supplemental_data.ex

@@ -39,6 +39,16 @@ defmodule Localize.SupplementalData do
     load_supplemental("likely_subtags.etf")
   end
 
+  @doc false
+  # The CLDR compound-unit grammatical-derivation table, keyed by base
+  # language subtag string (with `"root"` as the default). Read by the
+  # unit formatter to derive each component's plural/case when composing
+  # a compound unit that has no precomposed pattern.
+  @spec unit_grammatical_derivations() :: %{String.t() => map()}
+  def unit_grammatical_derivations do
+    load_supplemental("unit_grammatical_derivations.etf")
+  end
+
   @doc false
   @spec aliases() :: map()
   def aliases do

lib/localize/unit/custom_registry.ex

@@ -11,8 +11,12 @@ defmodule Localize.Unit.CustomRegistry do
 
   Each definition is a map with the following keys:
 
-  * `:base_unit` (required) — the CLDR base unit this custom unit converts to
-    (e.g., `"meter"`, `"kilogram"`, `"second"`).
+  * `:base_unit` (required) — the CLDR unit this custom unit converts to
+    (e.g., `"meter"`, `"kilogram"`, `"second"`). It need not be a
+    fundamental base unit: a derived unit such as `"day"` is accepted and
+    folded down to its fundamental base (`"second"`) at registration, with
+    the factor and offset adjusted accordingly, so the custom unit stays
+    convertible against every CLDR unit in the same category.
 
   * `:factor` (required) — the conversion factor:
     `1 custom_unit = factor * base_unit`.
@@ -129,9 +133,10 @@ defmodule Localize.Unit.CustomRegistry do
   def register(name, definition) do
     with :ok <- validate_name(name),
          :ok <- validate_definition(definition),
-         :ok <- validate_no_collision(name) do
+         :ok <- validate_no_collision(name),
+         {:ok, normalized} <- normalize_base(definition) do
       current = all()
-      :persistent_term.put(@persistent_term_key, Map.put(current, name, definition))
+      :persistent_term.put(@persistent_term_key, Map.put(current, name, normalized))
       :ok
     end
   end
@@ -173,8 +178,9 @@ defmodule Localize.Unit.CustomRegistry do
       Enum.reduce(definitions, %{}, fn {name, definition}, acc ->
         with :ok <- validate_name(name),
              :ok <- validate_definition(definition),
-             :ok <- validate_no_collision(name) do
-          Map.put(acc, name, definition)
+             :ok <- validate_no_collision(name),
+             {:ok, normalized} <- normalize_base(definition) do
+          Map.put(acc, name, normalized)
         else
           {:error, _reason} -> acc
         end
@@ -322,6 +328,67 @@ defmodule Localize.Unit.CustomRegistry do
     :ok
   end
 
+  # ── Base-unit normalization ──
+
+  # A custom unit's `:base_unit` must be a fundamental CLDR base unit,
+  # because compatibility and conversion compare fully-reduced base
+  # units: `compatible?/2` succeeds only when both units reduce to the
+  # same base. When a definition is expressed in terms of a *derived*
+  # unit (e.g. `%{base_unit: "day", factor: 1}`), storing "day"
+  # verbatim leaves the custom unit reducing to "day" while every CLDR
+  # unit reduces "day" to "second" — so the two never match.
+  #
+  # Fold the derived unit's own conversion into the custom factor and
+  # offset so the stored base becomes the fundamental base with an
+  # equivalent factor: `guestnight` given as `base_unit: "day",
+  # factor: 1` is stored as `base_unit: "second", factor: 86400.0`.
+  # A definition already expressed against a fundamental base is stored
+  # unchanged, and `:special` (function-based) definitions — which do
+  # not convert via a linear factor — are left untouched.
+  defp normalize_base(%{factor: factor} = definition) when is_number(factor) do
+    base = definition.base_unit
+
+    case Localize.Unit.BaseUnit.base_unit(base) do
+      {:ok, ^base} ->
+        {:ok, definition}
+
+      {:ok, true_base} ->
+        with {:ok, base_factor, base_offset} <- base_conversion(base, true_base) do
+          offset = Map.get(definition, :offset, 0.0)
+
+          {:ok,
+           definition
+           |> Map.put(:base_unit, true_base)
+           |> Map.put(:factor, factor * base_factor)
+           |> Map.put(:offset, offset * base_factor + base_offset)}
+        end
+
+      {:error, _reason} ->
+        {:error, "unknown base unit: #{inspect(base)}"}
+    end
+  end
+
+  defp normalize_base(definition), do: {:ok, definition}
+
+  # The linear parameters `{factor, offset}` that convert one unit of
+  # `base` into `true_base`, such that
+  # `value_in_true_base = value_in_base * factor + offset`. For a simple
+  # derived unit CLDR stores these directly; for a compound derived unit
+  # they are recovered from two reference conversions (the offset is the
+  # image of zero, the factor the difference across one unit).
+  defp base_conversion(base, true_base) do
+    case Localize.Unit.Data.conversion_factor_raw(base) do
+      %{factor: factor, offset: offset} when is_number(factor) ->
+        {:ok, factor, offset}
+
+      _other ->
+        with {:ok, at_zero} <- Localize.Unit.Conversion.convert(0, base, true_base),
+             {:ok, at_one} <- Localize.Unit.Conversion.convert(1, base, true_base) do
+          {:ok, at_one - at_zero, at_zero}
+        end
+    end
+  end
+
   # ── Validation ──
 
   @name_pattern ~r/^[a-z][a-z0-9_-]*$/