From 7e12d4e99a44049bcca84323f6c7e80536e8ea94 Mon Sep 17 00:00:00 2001 From: TKaltofen Date: Tue, 21 Jul 2026 11:28:45 +0200 Subject: [PATCH] chore(deps): bump mloda to 0.10.0 (#11) * chore(deps): bump mloda to 0.10.0 --- README.md | 13 ++++---- demos/govdata_demo.py | 14 ++++---- .../feature_groups/govdata/reader.py | 9 ++--- .../govdata/tests/test_reader.py | 33 ++++++++++++++----- pyproject.toml | 2 +- uv.lock | 6 ++-- 6 files changed, 47 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 50e44c6..67685cd 100644 --- a/README.md +++ b/README.md @@ -25,15 +25,16 @@ from mloda_plugin_govdata.feature_groups.govdata import StuttgartPopulationReade slug = "einwohner-nach-altersgruppen-und-stadtbezirken" result = mloda.run_all( [ - Feature("Einwohner", options={StuttgartPopulationReader.__name__: slug}), - Feature("Stadtbezirk", options={StuttgartPopulationReader.__name__: slug}), + Feature("Einwohner", options={StuttgartPopulationReader: slug}), + Feature("Stadtbezirk", options={StuttgartPopulationReader: slug}), ], compute_frameworks=["PyArrowTable"], ) table = result[0] # pyarrow.Table with the requested columns +result.plan # resolved execution steps: which FeatureGroup ran on which framework ``` -The option value is a GovData dataset slug or a direct distribution URL. The license is read from the CKAN distribution metadata. Set `BaseGovDataReader.cache_dir` to control where downloads are cached. For any other GovData CSV dataset, `GovDataReader` works out of the box and reads every column as a string; subclass it and set `schema` for typed columns. +The options key is the reader class or its class-name string; both select the same reader. The option value is a GovData dataset slug or a direct distribution URL. The license is read from the CKAN distribution metadata. Set `BaseGovDataReader.cache_dir` to control where downloads are cached. For any other GovData CSV dataset, `GovDataReader` works out of the box and reads every column as a string; subclass it and set `schema` for typed columns. Don't know the slug yet? Search GovData with the paginated CKAN `package_search` API: @@ -62,7 +63,7 @@ from mloda_plugin_govdata.feature_groups.govdata import BundeswahlleiterinReader kerg = "https://www.bundeswahlleiterin.de/bundestagswahlen/2025/ergebnisse/opendata/btw25/csv/kerg.csv" result = mloda.run_all( - [Feature("Gebiet", options={BundeswahlleiterinReader.__name__: kerg})], + [Feature("Gebiet", options={BundeswahlleiterinReader: kerg})], compute_frameworks=["PyArrowTable"], ) ``` @@ -75,8 +76,8 @@ from mloda_plugin_govdata.feature_groups.govdata import UbaAirReader, uba_measur url = uba_measures_url(station=143, component=3, scope=2, date_from="2025-01-01", date_to="2025-01-01") result = mloda.run_all( [ - Feature("date_start", options={UbaAirReader.__name__: url}), - Feature("value", options={UbaAirReader.__name__: url}), + Feature("date_start", options={UbaAirReader: url}), + Feature("value", options={UbaAirReader: url}), ], compute_frameworks=["PyArrowTable"], ) diff --git a/demos/govdata_demo.py b/demos/govdata_demo.py index 2bb00d6..391dc93 100644 --- a/demos/govdata_demo.py +++ b/demos/govdata_demo.py @@ -94,10 +94,10 @@ def _(Feature, StuttgartPopulationReader, mloda): _slug = "einwohner-nach-altersgruppen-und-stadtbezirken" _result = mloda.run_all( [ - Feature("Stichtag", options={StuttgartPopulationReader.__name__: _slug}), - Feature("Stadtbezirk", options={StuttgartPopulationReader.__name__: _slug}), - Feature("Alter in 10 Gruppen", options={StuttgartPopulationReader.__name__: _slug}), - Feature("Einwohner", options={StuttgartPopulationReader.__name__: _slug}), + Feature("Stichtag", options={StuttgartPopulationReader: _slug}), + Feature("Stadtbezirk", options={StuttgartPopulationReader: _slug}), + Feature("Alter in 10 Gruppen", options={StuttgartPopulationReader: _slug}), + Feature("Einwohner", options={StuttgartPopulationReader: _slug}), ], compute_frameworks=["PyArrowTable"], ) @@ -116,7 +116,7 @@ def _(mo): def _(BundeswahlleiterinReader, Feature, mloda): _kerg = "https://www.bundeswahlleiterin.de/bundestagswahlen/2025/ergebnisse/opendata/btw25/csv/kerg.csv" _result = mloda.run_all( - [Feature("Gebiet", options={BundeswahlleiterinReader.__name__: _kerg})], + [Feature("Gebiet", options={BundeswahlleiterinReader: _kerg})], compute_frameworks=["PyArrowTable"], ) elections = _result[0].to_pandas() @@ -135,8 +135,8 @@ def _(Feature, UbaAirReader, mloda, uba_measures_url): _url = uba_measures_url(station=143, component=3, scope=2, date_from="2025-01-01", date_to="2025-01-01") _result = mloda.run_all( [ - Feature("date_start", options={UbaAirReader.__name__: _url}), - Feature("value", options={UbaAirReader.__name__: _url}), + Feature("date_start", options={UbaAirReader: _url}), + Feature("value", options={UbaAirReader: _url}), ], compute_frameworks=["PyArrowTable"], ) diff --git a/mloda_plugin_govdata/feature_groups/govdata/reader.py b/mloda_plugin_govdata/feature_groups/govdata/reader.py index b50706c..409b47c 100644 --- a/mloda_plugin_govdata/feature_groups/govdata/reader.py +++ b/mloda_plugin_govdata/feature_groups/govdata/reader.py @@ -19,7 +19,8 @@ from mloda.provider import FeatureSet # Imported for its registration side effect so "PyArrowTable" resolves; the reader returns a pyarrow Table. -from mloda_plugins.compute_framework.base_implementations.pyarrow.table import PyArrowTable # noqa: F401 +# Public path since mloda 0.10.0 (lazy export); resolving it imports and registers the framework class. +from mloda.user import PyArrowTable # noqa: F401 from mloda_plugins.feature_group.input_data.read_file import ReadFile from .core.cache import DownloadCache @@ -64,9 +65,9 @@ def match_subclass_data_access(cls, data_access: Any, feature_names: list[str], @classmethod def load_data(cls, data_access: Any, features: FeatureSet) -> Any: - # Touch features first: the framework probes load_data(None, None) and - # only tolerates AttributeError to detect scoped-access support. - requested = sorted(features.get_all_names()) # deterministic column order + # Overriding load_data wholesale classifies this as a final reader (mloda >=0.10.0 + # is_final_reader, structural, no runtime probe). + requested = list(features.get_all_names()) # sorted tuple since mloda 0.10.0; deterministic column order locator = cls._coerce_locator(data_access) table = cls._read_table(locator) available = set(table.column_names) diff --git a/mloda_plugin_govdata/feature_groups/govdata/tests/test_reader.py b/mloda_plugin_govdata/feature_groups/govdata/tests/test_reader.py index 12f8750..4bd4824 100644 --- a/mloda_plugin_govdata/feature_groups/govdata/tests/test_reader.py +++ b/mloda_plugin_govdata/feature_groups/govdata/tests/test_reader.py @@ -2,7 +2,7 @@ import json from pathlib import Path -from typing import cast +from typing import Any, cast import httpx import pyarrow as pa @@ -10,7 +10,8 @@ import respx from mloda.provider import FeatureSet -from mloda.user import Feature, mloda +from mloda.user import Feature, Options, mloda +from mloda_plugins.feature_group.input_data.read_file import ReadFile from mloda_plugin_govdata.feature_groups.govdata.bundeswahlleiterin import BundeswahlleiterinReader from mloda_plugin_govdata.feature_groups.govdata.feature import GovDataFeature @@ -33,12 +34,12 @@ def _mock_population_endpoints(fixtures_dir: Path) -> None: class _FakeFeatureSet: - """Just enough FeatureSet surface for load_data.""" + """Just enough FeatureSet surface for load_data; mirrors the sorted-tuple contract.""" def __init__(self, names: set[str]) -> None: - self._names = names + self._names = tuple(sorted(names)) - def get_all_names(self) -> set[str]: + def get_all_names(self) -> tuple[str, ...]: return self._names @@ -46,6 +47,13 @@ def test_feature_group_uses_base_govdata_reader() -> None: assert isinstance(GovDataFeature.input_data(), BaseGovDataReader) +def test_class_options_key_normalizes_to_reader_name() -> None: + # The reader class works as an options key; it normalizes to the class-name string (mloda >=0.10.0). + # The cast bridges a 0.10.0 typing gap: the runtime accepts class keys, the hints only admit str. + options = Options(cast(dict[str, Any], {GovDataReader: SLUG})) + assert options.get(GovDataReader.__name__) == SLUG + + @respx.mock def test_load_data_level2(fixtures_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(StuttgartPopulationReader, "cache_dir", str(tmp_path)) @@ -67,6 +75,11 @@ def test_load_data_level2(fixtures_dir: Path, tmp_path: Path, monkeypatch: pytes assert set(table.schema.names) == {"Einwohner", "Stadtbezirk"} assert table.num_rows == 1000 assert table.schema.field("Einwohner").type == pa.int64() + # RunResult.plan (mloda >=0.10.0): the resolved execution steps name our group and framework. + compute_steps = [step for step in result.plan if step.step_kind == "compute"] + assert [step.feature_group_name for step in compute_steps] == ["GovDataFeature"] + assert compute_steps[0].compute_framework_name == "PyArrowTable" + assert set(compute_steps[0].requested_feature_names) == {"Einwohner", "Stadtbezirk"} @respx.mock @@ -184,10 +197,12 @@ def test_peek_rejects_unusable_data_access() -> None: "reader", [BaseGovDataReader, GovDataReader, StuttgartPopulationReader, BundeswahlleiterinReader, UbaAirReader], ) -def test_load_data_probe_still_raises_attribute_error(reader: type[BaseGovDataReader]) -> None: - # The framework probes load_data(None, None) and only tolerates AttributeError. - with pytest.raises(AttributeError): - reader.load_data(None, cast(FeatureSet, None)) +def test_readers_classify_as_final_readers(reader: type[BaseGovDataReader]) -> None: + # mloda >=0.10.0 classifies readers structurally: overriding load_data wholesale + # relative to the ReadFile anchor makes each reader final; the ReadFile base is not. + assert reader.final_reader_anchor() is ReadFile + assert reader.is_final_reader() is True + assert ReadFile.is_final_reader() is False @respx.mock diff --git a/pyproject.toml b/pyproject.toml index 36f4978..71cbc06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ description = "mloda plugin providing connectors and harmonization for German go license = "Apache-2.0" authors = [{ name = "Tom Kaltofen", email = "tomkaltofen@mloda.ai" }] dependencies = [ - "mloda>=0.9.0", + "mloda>=0.10.0,<0.11.0", "mloda-testing>=0.3.1", "httpx>=0.27", "pandas>=2.0", diff --git a/uv.lock b/uv.lock index 0ec38db..1625bc2 100644 --- a/uv.lock +++ b/uv.lock @@ -607,10 +607,10 @@ wheels = [ [[package]] name = "mloda" -version = "0.9.0" +version = "0.10.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/82/a7893ddd9ab26dd839671c594e9c16437364a8b8ed5dffb02e9b2de52d7f/mloda-0.9.0-py3-none-any.whl", hash = "sha256:d193b03622e6c8670ccde500c9e6d74eba9ef4ed70ca3bd6adbcb9330a370585", size = 430826, upload-time = "2026-07-07T08:48:32.032Z" }, + { url = "https://files.pythonhosted.org/packages/d6/9d/2251f947bfde5e168b9f6759079c8b54af0393c3e854987268459b860776/mloda-0.10.0-py3-none-any.whl", hash = "sha256:13fd1482566de203cac661c29f5736423ce7d1cc8e43ec26f8d24d19e6386229", size = 456550, upload-time = "2026-07-13T18:35:02.322Z" }, ] [[package]] @@ -652,7 +652,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.27" }, { name = "hypothesis", marker = "extra == 'dev'" }, { name = "marimo", marker = "extra == 'demo'", specifier = ">=0.14" }, - { name = "mloda", specifier = ">=0.9.0" }, + { name = "mloda", specifier = ">=0.10.0,<0.11.0" }, { name = "mloda-testing", specifier = ">=0.3.1" }, { name = "mypy", marker = "extra == 'dev'" }, { name = "pandas", specifier = ">=2.0" },