Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
4432e20
docs(specs): add missing_cache_policy design spec (ITL-604)
kurodo3[bot] Aug 6, 2026
77d86c5
docs(plans): add missing_cache_policy implementation plan (ITL-604)
kurodo3[bot] Aug 6, 2026
3ab5aa8
feat(config): add missing_cache_policy to NodeConfig and CacheMissErr…
kurodo3[bot] Aug 6, 2026
ec4d894
feat(function_node): strict missing_cache_policy raises CacheMissErro…
kurodo3[bot] Aug 6, 2026
a7e8ccd
fix(function_node): fix stale docstring, restore hash-selection comme…
kurodo3[bot] Aug 6, 2026
2d4993d
feat(function_node): as_empty policy + ephemeral INFO log in _fetch_j…
kurodo3[bot] Aug 6, 2026
5c036cf
feat(function_node): emit EmptyData in execute() for as_empty/strict …
kurodo3[bot] Aug 6, 2026
86609c8
feat(function_node): align async_execute route_inputs with missing_ca…
kurodo3[bot] Aug 6, 2026
fdf5f5d
test(function_node): CACHE_ONLY missing_cache_policy coverage (ITL-604)
kurodo3[bot] Aug 6, 2026
f118001
test(function_node): end-to-end as_empty downstream cache hit (ITL-604)
kurodo3[bot] Aug 6, 2026
5b85624
test(function_node): recompute default regression guard (ITL-604)
kurodo3[bot] Aug 6, 2026
187bf65
test(function_node): add async strict mode non-ephemeral miss test (I…
kurodo3[bot] Aug 6, 2026
26effb1
test(function_node): fix inline DB wipes and clarify strict ephemeral…
kurodo3[bot] Aug 7, 2026
37fee6e
refactor(function_node): rename get_cached_results→load_cached_result…
kurodo3[bot] Aug 9, 2026
a09aa9e
test(missing_cache_policy): add coverage for Branch B, async non-ephe…
kurodo3[bot] Aug 9, 2026
d4166c5
Merge branch 'main' into eywalker/itl-604-non-ephemeral-result-store-…
eywalker Aug 13, 2026
bc2d93e
Merge branch 'main' into eywalker/itl-604-non-ephemeral-result-store-…
eywalker Aug 22, 2026
4a01dc3
fix(function_node): scope empty_data_tokens to base_entry_ids before …
kurodo3[bot] Aug 22, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
298 changes: 214 additions & 84 deletions src/orcapod/core/nodes/function_node.py

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions src/orcapod/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,3 +195,14 @@ def __init__(
f"node={node_identity_path!r} "
f"cached_content_hash={cached_content_hash!r}"
)


class CacheMissError(Exception):
"""Raised when a persistent (non-ephemeral) result-store entry is absent
and ``NodeConfig.missing_cache_policy`` is ``"strict"``.

A missing durable result indicates data loss or corruption. Set
``missing_cache_policy="recompute"`` (the default) to fall back to
recomputation, or ``"as_empty"`` to propagate an ``EmptyData`` token
downstream instead of raising.
"""
27 changes: 26 additions & 1 deletion src/orcapod/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from datetime import date, datetime
from enum import Enum
from types import UnionType
from typing import TYPE_CHECKING, Any, Generic, Self, TypeAlias, TypeVar
from typing import TYPE_CHECKING, Any, Generic, Literal, Self, TypeAlias, TypeVar

if TYPE_CHECKING:
import pyarrow as pa
Expand Down Expand Up @@ -360,10 +360,30 @@ class NodeConfig:
means no old schema is tolerated — any detected v0 table raises
``SchemaVersionError``. Pass ``("v0",)`` to suppress the error
and allow the node to recompute all results from scratch.
missing_cache_policy: Controls how the node reacts when the pipeline
table has an entry for an input but the result store does not.
``None`` inherits the default (``"recompute"``).

* ``"recompute"`` *(default)* — WARNING logged; the entry falls
through to recomputation from the original upstream data.
For ephemeral stores, ``EmptyData`` acts as a recompute sentinel
(same as current behaviour).
* ``"as_empty"`` — WARNING logged (non-ephemeral) or INFO logged
(ephemeral); an ``EmptyData`` token is emitted directly. The
downstream node attempts to serve the result from its own cache.
Only use when partial gaps are semantically expected (e.g.
shared read-only stores, exploratory pipelines).
* ``"strict"`` — ERROR logged and ``CacheMissError`` raised for
non-ephemeral misses. Ephemeral misses still degrade gracefully
to ``EmptyData`` (raising on an ephemeral miss would contradict
the semantics of ephemeral storage). Use in production pipelines
where a missing durable result always indicates a bug or data
loss.
"""

is_result_ephemeral: bool | None = None
ignore_schema: tuple[str, ...] | None = None
missing_cache_policy: Literal["recompute", "as_empty", "strict"] | None = None

def merge(self, other: "NodeConfig") -> "NodeConfig":
"""Return a new ``NodeConfig`` with ``other``'s non-``None`` fields overriding self.
Expand Down Expand Up @@ -395,6 +415,11 @@ def merge(self, other: "NodeConfig") -> "NodeConfig":
if other.ignore_schema is not None
else self.ignore_schema
),
missing_cache_policy=(
other.missing_cache_policy
if other.missing_cache_policy is not None
else self.missing_cache_policy
),
)


Expand Down
Loading