Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,16 @@ from mloda_plugin_govdata.feature_groups.govdata import StuttgartPopulationReade
slug = "einwohner-nach-altersgruppen-und-stadtbezirken"
result = mloda.run_all(
[
Feature("Einwohner", options={StuttgartPopulationReader.__name__: slug}),
Feature("Stadtbezirk", options={StuttgartPopulationReader.__name__: slug}),
Feature("Einwohner", options={StuttgartPopulationReader: slug}),
Feature("Stadtbezirk", options={StuttgartPopulationReader: slug}),
],
compute_frameworks=["PyArrowTable"],
)
table = result[0] # pyarrow.Table with the requested columns
result.plan # resolved execution steps: which FeatureGroup ran on which framework
```

The option value is a GovData dataset slug or a direct distribution URL. The license is read from the CKAN distribution metadata. Set `BaseGovDataReader.cache_dir` to control where downloads are cached. For any other GovData CSV dataset, `GovDataReader` works out of the box and reads every column as a string; subclass it and set `schema` for typed columns.
The options key is the reader class or its class-name string; both select the same reader. The option value is a GovData dataset slug or a direct distribution URL. The license is read from the CKAN distribution metadata. Set `BaseGovDataReader.cache_dir` to control where downloads are cached. For any other GovData CSV dataset, `GovDataReader` works out of the box and reads every column as a string; subclass it and set `schema` for typed columns.

Don't know the slug yet? Search GovData with the paginated CKAN `package_search` API:

Expand Down Expand Up @@ -62,7 +63,7 @@ from mloda_plugin_govdata.feature_groups.govdata import BundeswahlleiterinReader

kerg = "https://www.bundeswahlleiterin.de/bundestagswahlen/2025/ergebnisse/opendata/btw25/csv/kerg.csv"
result = mloda.run_all(
[Feature("Gebiet", options={BundeswahlleiterinReader.__name__: kerg})],
[Feature("Gebiet", options={BundeswahlleiterinReader: kerg})],
compute_frameworks=["PyArrowTable"],
)
```
Expand All @@ -75,8 +76,8 @@ from mloda_plugin_govdata.feature_groups.govdata import UbaAirReader, uba_measur
url = uba_measures_url(station=143, component=3, scope=2, date_from="2025-01-01", date_to="2025-01-01")
result = mloda.run_all(
[
Feature("date_start", options={UbaAirReader.__name__: url}),
Feature("value", options={UbaAirReader.__name__: url}),
Feature("date_start", options={UbaAirReader: url}),
Feature("value", options={UbaAirReader: url}),
],
compute_frameworks=["PyArrowTable"],
)
Expand Down
14 changes: 7 additions & 7 deletions demos/govdata_demo.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,10 @@ def _(Feature, StuttgartPopulationReader, mloda):
_slug = "einwohner-nach-altersgruppen-und-stadtbezirken"
_result = mloda.run_all(
[
Feature("Stichtag", options={StuttgartPopulationReader.__name__: _slug}),
Feature("Stadtbezirk", options={StuttgartPopulationReader.__name__: _slug}),
Feature("Alter in 10 Gruppen", options={StuttgartPopulationReader.__name__: _slug}),
Feature("Einwohner", options={StuttgartPopulationReader.__name__: _slug}),
Feature("Stichtag", options={StuttgartPopulationReader: _slug}),
Feature("Stadtbezirk", options={StuttgartPopulationReader: _slug}),
Feature("Alter in 10 Gruppen", options={StuttgartPopulationReader: _slug}),
Feature("Einwohner", options={StuttgartPopulationReader: _slug}),
],
compute_frameworks=["PyArrowTable"],
)
Expand All @@ -116,7 +116,7 @@ def _(mo):
def _(BundeswahlleiterinReader, Feature, mloda):
_kerg = "https://www.bundeswahlleiterin.de/bundestagswahlen/2025/ergebnisse/opendata/btw25/csv/kerg.csv"
_result = mloda.run_all(
[Feature("Gebiet", options={BundeswahlleiterinReader.__name__: _kerg})],
[Feature("Gebiet", options={BundeswahlleiterinReader: _kerg})],
compute_frameworks=["PyArrowTable"],
)
elections = _result[0].to_pandas()
Expand All @@ -135,8 +135,8 @@ def _(Feature, UbaAirReader, mloda, uba_measures_url):
_url = uba_measures_url(station=143, component=3, scope=2, date_from="2025-01-01", date_to="2025-01-01")
_result = mloda.run_all(
[
Feature("date_start", options={UbaAirReader.__name__: _url}),
Feature("value", options={UbaAirReader.__name__: _url}),
Feature("date_start", options={UbaAirReader: _url}),
Feature("value", options={UbaAirReader: _url}),
],
compute_frameworks=["PyArrowTable"],
)
Expand Down
9 changes: 5 additions & 4 deletions mloda_plugin_govdata/feature_groups/govdata/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
from mloda.provider import FeatureSet

# Imported for its registration side effect so "PyArrowTable" resolves; the reader returns a pyarrow Table.
from mloda_plugins.compute_framework.base_implementations.pyarrow.table import PyArrowTable # noqa: F401
# Public path since mloda 0.10.0 (lazy export); resolving it imports and registers the framework class.
from mloda.user import PyArrowTable # noqa: F401
from mloda_plugins.feature_group.input_data.read_file import ReadFile

from .core.cache import DownloadCache
Expand Down Expand Up @@ -64,9 +65,9 @@ def match_subclass_data_access(cls, data_access: Any, feature_names: list[str],

@classmethod
def load_data(cls, data_access: Any, features: FeatureSet) -> Any:
# Touch features first: the framework probes load_data(None, None) and
# only tolerates AttributeError to detect scoped-access support.
requested = sorted(features.get_all_names()) # deterministic column order
# Overriding load_data wholesale classifies this as a final reader (mloda >=0.10.0
# is_final_reader, structural, no runtime probe).
requested = list(features.get_all_names()) # sorted tuple since mloda 0.10.0; deterministic column order
locator = cls._coerce_locator(data_access)
table = cls._read_table(locator)
available = set(table.column_names)
Expand Down
33 changes: 24 additions & 9 deletions mloda_plugin_govdata/feature_groups/govdata/tests/test_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,16 @@

import json
from pathlib import Path
from typing import cast
from typing import Any, cast

import httpx
import pyarrow as pa
import pytest
import respx

from mloda.provider import FeatureSet
from mloda.user import Feature, mloda
from mloda.user import Feature, Options, mloda
from mloda_plugins.feature_group.input_data.read_file import ReadFile

from mloda_plugin_govdata.feature_groups.govdata.bundeswahlleiterin import BundeswahlleiterinReader
from mloda_plugin_govdata.feature_groups.govdata.feature import GovDataFeature
Expand All @@ -33,19 +34,26 @@ def _mock_population_endpoints(fixtures_dir: Path) -> None:


class _FakeFeatureSet:
"""Just enough FeatureSet surface for load_data."""
"""Just enough FeatureSet surface for load_data; mirrors the sorted-tuple contract."""

def __init__(self, names: set[str]) -> None:
self._names = names
self._names = tuple(sorted(names))

def get_all_names(self) -> set[str]:
def get_all_names(self) -> tuple[str, ...]:
return self._names


def test_feature_group_uses_base_govdata_reader() -> None:
assert isinstance(GovDataFeature.input_data(), BaseGovDataReader)


def test_class_options_key_normalizes_to_reader_name() -> None:
# The reader class works as an options key; it normalizes to the class-name string (mloda >=0.10.0).
# The cast bridges a 0.10.0 typing gap: the runtime accepts class keys, the hints only admit str.
options = Options(cast(dict[str, Any], {GovDataReader: SLUG}))
assert options.get(GovDataReader.__name__) == SLUG


@respx.mock
def test_load_data_level2(fixtures_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(StuttgartPopulationReader, "cache_dir", str(tmp_path))
Expand All @@ -67,6 +75,11 @@ def test_load_data_level2(fixtures_dir: Path, tmp_path: Path, monkeypatch: pytes
assert set(table.schema.names) == {"Einwohner", "Stadtbezirk"}
assert table.num_rows == 1000
assert table.schema.field("Einwohner").type == pa.int64()
# RunResult.plan (mloda >=0.10.0): the resolved execution steps name our group and framework.
compute_steps = [step for step in result.plan if step.step_kind == "compute"]
assert [step.feature_group_name for step in compute_steps] == ["GovDataFeature"]
assert compute_steps[0].compute_framework_name == "PyArrowTable"
assert set(compute_steps[0].requested_feature_names) == {"Einwohner", "Stadtbezirk"}


@respx.mock
Expand Down Expand Up @@ -184,10 +197,12 @@ def test_peek_rejects_unusable_data_access() -> None:
"reader",
[BaseGovDataReader, GovDataReader, StuttgartPopulationReader, BundeswahlleiterinReader, UbaAirReader],
)
def test_load_data_probe_still_raises_attribute_error(reader: type[BaseGovDataReader]) -> None:
# The framework probes load_data(None, None) and only tolerates AttributeError.
with pytest.raises(AttributeError):
reader.load_data(None, cast(FeatureSet, None))
def test_readers_classify_as_final_readers(reader: type[BaseGovDataReader]) -> None:
# mloda >=0.10.0 classifies readers structurally: overriding load_data wholesale
# relative to the ReadFile anchor makes each reader final; the ReadFile base is not.
assert reader.final_reader_anchor() is ReadFile
assert reader.is_final_reader() is True
assert ReadFile.is_final_reader() is False


@respx.mock
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ description = "mloda plugin providing connectors and harmonization for German go
license = "Apache-2.0"
authors = [{ name = "Tom Kaltofen", email = "tomkaltofen@mloda.ai" }]
dependencies = [
"mloda>=0.9.0",
"mloda>=0.10.0,<0.11.0",
"mloda-testing>=0.3.1",
"httpx>=0.27",
"pandas>=2.0",
Expand Down
6 changes: 3 additions & 3 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading