Merge remote-tracking branch 'origin/main' into inflection

Author: kipcole9

.gitignore

@@ -43,6 +43,11 @@ mise.toml
 !/data/inflection/test/
 !/priv/localize/locales/und.etf
 
+# Inflection data (sources and generated ETFs) — belongs to the
+# inflection feature branch, not main
+/data/inflection/
+/priv/localize/inflection/
+
 # Experiment files
 bench_unit_parser.sh
 

CHANGELOG.md

@@ -14,6 +14,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 * The BCP 47 validity data generator no longer crashes on deprecated timezone codes (those with a preferred replacement, such as `camtr` → `cator`); the generated validity data is unchanged, but it can once again be regenerated from source.
 
+* `Localize.Unit.parse/2` resolves unit symbols that map to compound identifiers — `"1 m/s"` → `meter-per-second`, `"3 kWh"` → `kilowatt-hour` — instead of failing on the internal underscore key form ([#42](https://github.com/elixir-localize/localize/issues/42)).
+
+* Times-compound units with no precomposed CLDR pattern (`tonne-kilometer`) now format as their localized name (`"5 metric ton-kilometers"`, `"5 t⋅km"`, `"5 tonnes-kilomètres"`) instead of the raw identifier, composed per the CLDR grammatical derivation loaded from `grammaticalFeatures.xml` ([#43](https://github.com/elixir-localize/localize/issues/43)).
+
+* Per-compound units whose denominator has no per-unit pattern now localize through the locale's `compound.per` pattern (Polish `"5 jardów na milimetr"`) instead of falling back to an English `"per"`.
+
+* `Localize.Unit.define_unit/2` accepts a derived unit as `base_unit` (e.g. `"day"`) and folds it to its fundamental base at registration, so the custom unit is convertible and `compatible?/2` returns `true` ([#44](https://github.com/elixir-localize/localize/issues/44)).
+
+* Person-duration units (`year-person`, `month-person`, `week-person`, `day-person`) format as their base unit (`"5 years"`), matching ICU, instead of the raw identifier.
+
+* SI-prefixed units with no precomposed CLDR pattern (`megajoule`, `gigajoule`, `nanogram`) now compose their display from the prefix and base unit (`"5 MJ"`, `"5 megajoules"`, with CLDR `combineLowercasing` for capitalising locales) instead of the raw identifier ([#46](https://github.com/elixir-localize/localize/issues/46)).
+
 ### Changed
 
 * Generated locale ETF files and the download-integrity hash manifest are now serialized with the `term_to_binary` `:deterministic` option, making the bytes reproducible across operating systems, architectures, and OTP versions. This lets the manifest generated during development match the data generated in CI and served from the CDN, so runtime download verification succeeds regardless of where each was produced. All bundled supplemental, validity, and collation ETF data is regenerated with the same option.

data/data.ex

@@ -68,6 +68,7 @@ defmodule Localize.Data do
     {"common/supplemental/pluralRanges.xml", "pluralRanges.xml"},
     {"common/supplemental/subdivisions.xml", "subdivisions.xml"},
     {"common/supplemental/units.xml", "units.xml"},
+    {"common/supplemental/grammaticalFeatures.xml", "grammaticalFeatures.xml"},
     {"common/bcp47/timezone.xml", "bcp47_timezone.xml"}
   ]
 
@@ -123,6 +124,8 @@ defmodule Localize.Data do
     {"territory_subdivision_containment.etf",
      &Localize.Data.XmlExtractors.generate_territory_subdivision_containment/0},
     {"unit_data.etf", &Localize.Data.XmlExtractors.generate_unit_data/0},
+    {"unit_grammatical_derivations.etf",
+     &Localize.Data.XmlExtractors.generate_unit_grammatical_derivations/0},
     {"collation_tailoring.etf", &Localize.Data.Collation.generate_collation_tailoring/0},
     {"coverage_levels.etf", &Localize.Data.Supplemental.generate_coverage_levels/0},
     {"measurement_systems.etf", &Localize.Data.XmlExtractors.generate_measurement_systems/0},

data/xml_extractors.ex

@@ -319,6 +319,103 @@ defmodule Localize.Data.XmlExtractors do
     }
   end
 
+  @doc """
+  Generates the compound-unit grammatical-derivation table from
+  `grammaticalFeatures.xml`.
+
+  CLDR derives the plural category, grammatical case, and gender of a
+  compound unit's components from the compound as a whole (TR35
+  "Compound Units"). For example the default "times" derivation makes
+  the leading component singular and lets the trailing component carry
+  the count (`newton-meters`), while French pluralizes every component
+  (`tonnes-kilomètres`). The unit formatter reads this table to compose
+  the localized name of a compound unit that has no precomposed pattern.
+
+  Returns a map keyed by the base language subtag string (plus `"root"`
+  for the default), each value a map of the form:
+
+      %{
+        plural: %{times: {:one, :compound}, per: {:compound, :one}, ...},
+        case: %{times: {:nominative, :compound}, ...},
+        gender: %{times: 1, per: 0, ...}
+      }
+
+  For `deriveComponent` features (`:plural`, `:case`) the value is a
+  `{value0, value1}` tuple where `:compound` means "use the compound's
+  own category" and any other atom is a fixed category. For the
+  `deriveCompound` `:gender` feature the value is the `0`/`1` index of
+  the component whose gender the compound inherits. Each locale's block
+  is merged over `"root"` so per-feature fallbacks are already resolved.
+
+  """
+  def generate_unit_grammatical_derivations do
+    raw =
+      "grammatical_features.xml"
+      |> read_xml()
+      |> SweetXml.parse()
+      |> xpath(~x"//grammaticalDerivations"l,
+        locales: ~x"./@locales"s,
+        components: [
+          ~x"./deriveComponent"l,
+          feature: ~x"./@feature"s,
+          structure: ~x"./@structure"s,
+          value0: ~x"./@value0"s,
+          value1: ~x"./@value1"s
+        ],
+        compounds: [
+          ~x"./deriveCompound"l,
+          feature: ~x"./@feature"s,
+          structure: ~x"./@structure"s,
+          value: ~x"./@value"s
+        ]
+      )
+      |> Map.new(fn %{locales: locales, components: components, compounds: compounds} ->
+        {locales, build_derivation_map(components, compounds)}
+      end)
+
+    root = Map.get(raw, "root", %{})
+
+    raw
+    |> Enum.flat_map(fn {locales, derivations} ->
+      merged = deep_merge_derivations(root, derivations)
+      for locale <- String.split(locales), do: {locale, merged}
+    end)
+    |> Map.new()
+  end
+
+  # Folds a block's `deriveComponent`/`deriveCompound` rows into the
+  # nested `%{feature => %{structure => value}}` shape.
+  defp build_derivation_map(components, compounds) do
+    from_components =
+      Enum.reduce(components, %{}, fn %{feature: feature, structure: structure} = row, acc ->
+        value = {derivation_atom(row.value0), derivation_atom(row.value1)}
+        put_derivation(acc, feature, structure, value)
+      end)
+
+    Enum.reduce(compounds, from_components, fn %{feature: feature, structure: structure} = row,
+                                               acc ->
+      put_derivation(acc, feature, structure, String.to_integer(row.value))
+    end)
+  end
+
+  defp put_derivation(acc, feature, structure, value) do
+    feature = String.to_atom(feature)
+    structure = String.to_atom(structure)
+    Map.update(acc, feature, %{structure => value}, &Map.put(&1, structure, value))
+  end
+
+  defp derivation_atom(""), do: nil
+  defp derivation_atom(value), do: String.to_atom(value)
+
+  # Merges a locale block over `root` at the feature level, so a locale
+  # that overrides only some features (French overrides plural but not
+  # case) inherits the rest from root.
+  defp deep_merge_derivations(root, override) do
+    Map.merge(root, override, fn _feature, root_structures, override_structures ->
+      Map.merge(root_structures, override_structures)
+    end)
+  end
+
   @doc """
   Generates measurement system data from `bcp47/measure.xml`.
 
@@ -453,6 +550,7 @@ defmodule Localize.Data.XmlExtractors do
     "bcp47/timezone.xml" => {:bcp47, "timezone.xml"},
     "bcp47/measure.xml" => {:bcp47, "measure.xml"},
     "units.xml" => {:supplemental, "units.xml"},
+    "grammatical_features.xml" => {:supplemental, "grammaticalFeatures.xml"},
     "validity/unit.xml" => {:validity, "unit.xml"}
   }
 

lib/localize/supplemental_data.ex

@@ -39,6 +39,16 @@ defmodule Localize.SupplementalData do
     load_supplemental("likely_subtags.etf")
   end
 
+  @doc false
+  # The CLDR compound-unit grammatical-derivation table, keyed by base
+  # language subtag string (with `"root"` as the default). Read by the
+  # unit formatter to derive each component's plural/case when composing
+  # a compound unit that has no precomposed pattern.
+  @spec unit_grammatical_derivations() :: %{String.t() => map()}
+  def unit_grammatical_derivations do
+    load_supplemental("unit_grammatical_derivations.etf")
+  end
+
   @doc false
   @spec aliases() :: map()
   def aliases do

lib/localize/unit.ex

@@ -490,6 +490,16 @@ defmodule Localize.Unit do
   * `{:error, reason}` if the unit has no value, the target cannot
     be parsed, or the units are not convertible.
 
+  ### Value type
+
+  The converted value is a `Decimal` when the source value is a `Decimal`
+  (results stay exact) and a float otherwise. Integer and float values
+  both yield a float, because unit conversion is real-valued — an integer
+  input cannot represent a converted quantity such as `1` mile in
+  kilometers, so promoting to float is preferred over integer arithmetic,
+  which would silently lose precision. Pass a `Decimal` value when you
+  need an exact result.
+
   ### Examples
 
       iex> {:ok, meters} = Localize.Unit.new(1, "kilometer")
@@ -499,6 +509,11 @@ defmodule Localize.Unit do
       iex> result.name
       "meter"
 
+      iex> {:ok, tonnes} = Localize.Unit.new(Decimal.new(1), "tonne")
+      iex> {:ok, result} = Localize.Unit.convert(tonnes, "kilogram")
+      iex> match?(%Decimal{}, result.value)
+      true
+
   """
   @spec convert(t(), String.t()) :: {:ok, t()} | {:error, Exception.t()}
 

lib/localize/unit/custom_registry.ex

@@ -11,8 +11,12 @@ defmodule Localize.Unit.CustomRegistry do
 
   Each definition is a map with the following keys:
 
-  * `:base_unit` (required) — the CLDR base unit this custom unit converts to
-    (e.g., `"meter"`, `"kilogram"`, `"second"`).
+  * `:base_unit` (required) — the CLDR unit this custom unit converts to
+    (e.g., `"meter"`, `"kilogram"`, `"second"`). It need not be a
+    fundamental base unit: a derived unit such as `"day"` is accepted and
+    folded down to its fundamental base (`"second"`) at registration, with
+    the factor and offset adjusted accordingly, so the custom unit stays
+    convertible against every CLDR unit in the same category.
 
   * `:factor` (required) — the conversion factor:
     `1 custom_unit = factor * base_unit`.
@@ -136,9 +140,10 @@ defmodule Localize.Unit.CustomRegistry do
   def register(name, definition) do
     with :ok <- validate_name(name),
          :ok <- validate_definition(definition),
-         :ok <- validate_no_collision(name) do
+         :ok <- validate_no_collision(name),
+         {:ok, normalized} <- normalize_base(definition) do
       current = all()
-      :persistent_term.put(@persistent_term_key, Map.put(current, name, definition))
+      :persistent_term.put(@persistent_term_key, Map.put(current, name, normalized))
       :ok
     end
   end
@@ -180,8 +185,9 @@ defmodule Localize.Unit.CustomRegistry do
       Enum.reduce(definitions, %{}, fn {name, definition}, acc ->
         with :ok <- validate_name(name),
              :ok <- validate_definition(definition),
-             :ok <- validate_no_collision(name) do
-          Map.put(acc, name, definition)
+             :ok <- validate_no_collision(name),
+             {:ok, normalized} <- normalize_base(definition) do
+          Map.put(acc, name, normalized)
         else
           {:error, _reason} -> acc
         end
@@ -329,6 +335,67 @@ defmodule Localize.Unit.CustomRegistry do
     :ok
   end
 
+  # ── Base-unit normalization ──
+
+  # A custom unit's `:base_unit` must be a fundamental CLDR base unit,
+  # because compatibility and conversion compare fully-reduced base
+  # units: `compatible?/2` succeeds only when both units reduce to the
+  # same base. When a definition is expressed in terms of a *derived*
+  # unit (e.g. `%{base_unit: "day", factor: 1}`), storing "day"
+  # verbatim leaves the custom unit reducing to "day" while every CLDR
+  # unit reduces "day" to "second" — so the two never match.
+  #
+  # Fold the derived unit's own conversion into the custom factor and
+  # offset so the stored base becomes the fundamental base with an
+  # equivalent factor: `guestnight` given as `base_unit: "day",
+  # factor: 1` is stored as `base_unit: "second", factor: 86400.0`.
+  # A definition already expressed against a fundamental base is stored
+  # unchanged, and `:special` (function-based) definitions — which do
+  # not convert via a linear factor — are left untouched.
+  defp normalize_base(%{factor: factor} = definition) when is_number(factor) do
+    base = definition.base_unit
+
+    case Localize.Unit.BaseUnit.base_unit(base) do
+      {:ok, ^base} ->
+        {:ok, definition}
+
+      {:ok, true_base} ->
+        with {:ok, base_factor, base_offset} <- base_conversion(base, true_base) do
+          offset = Map.get(definition, :offset, 0.0)
+
+          {:ok,
+           definition
+           |> Map.put(:base_unit, true_base)
+           |> Map.put(:factor, factor * base_factor)
+           |> Map.put(:offset, offset * base_factor + base_offset)}
+        end
+
+      {:error, _reason} ->
+        {:error, "unknown base unit: #{inspect(base)}"}
+    end
+  end
+
+  defp normalize_base(definition), do: {:ok, definition}
+
+  # The linear parameters `{factor, offset}` that convert one unit of
+  # `base` into `true_base`, such that
+  # `value_in_true_base = value_in_base * factor + offset`. For a simple
+  # derived unit CLDR stores these directly; for a compound derived unit
+  # they are recovered from two reference conversions (the offset is the
+  # image of zero, the factor the difference across one unit).
+  defp base_conversion(base, true_base) do
+    case Localize.Unit.Data.conversion_factor_raw(base) do
+      %{factor: factor, offset: offset} when is_number(factor) ->
+        {:ok, factor, offset}
+
+      _other ->
+        with {:ok, at_zero} <- Localize.Unit.Conversion.convert(0, base, true_base),
+             {:ok, at_one} <- Localize.Unit.Conversion.convert(1, base, true_base) do
+          {:ok, at_one - at_zero, at_zero}
+        end
+    end
+  end
+
   # ── Validation ──
 
   @name_pattern ~r/^[a-z][a-z0-9_-]*$/