Skip to content

Commit a06633e

Browse files
kurodo3[bot]claude
andcommitted
fix(side-effects): add SideEffectNode blueprint class and fix record_id uniqueness
Introduce a lightweight SideEffectNode (blueprint) class so that Pipeline uses schema-only nodes without DB state, matching the FunctionNode / FunctionJobNode split. SideEffectJobNode now extends SideEffectNode and inherits node_uri, making Pipeline.save() and PipelineJob.as_pipeline() / to_invocations() work correctly on pipelines containing SideEffectPod. Also append uuid4() to the _write_invocation_row SHA-256 input to eliminate same-microsecond record_id collisions. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 00c095c commit a06633e

4 files changed

Lines changed: 68 additions & 23 deletions

File tree

src/orcapod/pipeline/base.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
PodInvocation,
1616
SideEffectInvocation,
1717
)
18+
from orcapod.side_effects import SideEffectNode
1819
from orcapod.protocols import core_protocols as cp
1920
from orcapod.utils.lazy_module import LazyModule
2021

@@ -536,6 +537,12 @@ def to_invocations(self) -> InvocationGraph:
536537
input_streams=(node.upstreams[0],),
537538
label=node._label,
538539
)
540+
elif isinstance(node, SideEffectNode):
541+
inv_by_node_hash[node_hash] = SideEffectInvocation(
542+
pod=node._pod,
543+
input_streams=(node.upstreams[0],),
544+
label=node._label,
545+
)
539546
else:
540547
if node._operator is None:
541548
raise RuntimeError(

src/orcapod/pipeline/graph.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
OperatorNode,
1313
SourceNode,
1414
)
15-
from orcapod.side_effects import SideEffectJobNode
15+
from orcapod.side_effects import SideEffectNode
1616
from orcapod.core.tracker import AutoRegisteringContextBasedTracker
1717
from orcapod.pipeline.base import AbstractPipelineBase
1818
from orcapod.pipeline.dag import OrcaDAG
@@ -65,7 +65,7 @@ class Pipeline(AbstractPipelineBase[GraphNode]):
6565
source_node_class = SourceNode
6666
function_node_class = FunctionNode
6767
operator_node_class = OperatorNode
68-
side_effect_node_class = SideEffectJobNode
68+
side_effect_node_class = SideEffectNode
6969

7070
def __init__(
7171
self,

src/orcapod/pipeline/job.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,7 @@ def as_pipeline(self) -> "Pipeline":
468468
from orcapod.core.nodes.function_node import FunctionNode
469469
from orcapod.core.nodes.operator_node import OperatorNode
470470
from orcapod.pipeline.graph import Pipeline
471+
from orcapod.side_effects import SideEffectNode
471472

472473
if not self._compiled:
473474
raise RuntimeError(
@@ -519,6 +520,14 @@ def as_pipeline(self) -> "Pipeline":
519520
tracker_manager=job_node.tracker_manager,
520521
)
521522

523+
elif isinstance(job_node, SideEffectJobNode):
524+
upstream_bp_hash = job_id_to_bp_hash[id(job_node._input_stream)]
525+
node_map[node_hash] = SideEffectNode(
526+
side_effect_pod=job_node._pod,
527+
input_stream=node_map[upstream_bp_hash],
528+
label=job_node._label,
529+
)
530+
522531
else:
523532
node_map[node_hash] = job_node.as_node()
524533

src/orcapod/side_effects.py

Lines changed: 50 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import datetime
1414
import hashlib
1515
import logging
16+
import uuid
1617
from collections.abc import Callable, Collection, Iterator, Sequence
1718
from typing import TYPE_CHECKING, Any, Literal
1819

@@ -418,7 +419,7 @@ def _write_invocation_row(
418419
executed_at = datetime.datetime.now(datetime.timezone.utc)
419420
executed_at_str = executed_at.isoformat()
420421
record_id_src = (
421-
f"{fip_hash_str}::{pod_content_hash_str}::{run_id}::{status}::{executed_at_str}"
422+
f"{fip_hash_str}::{pod_content_hash_str}::{run_id}::{status}::{executed_at_str}::{uuid.uuid4()}"
422423
).encode("utf-8")
423424
record_id = hashlib.sha256(record_id_src).digest()
424425

@@ -711,20 +712,16 @@ def _merge_config(
711712

712713

713714
# ---------------------------------------------------------------------------
714-
# SideEffectJobNode
715+
# SideEffectNode — lightweight blueprint node (no DB)
715716
# ---------------------------------------------------------------------------
716717

717718

718-
class SideEffectJobNode(StreamBase):
719-
"""DB-backed execution node for side-effect pods.
720-
721-
Created at pipeline compile time by ``PipelineJob``. Receives a
722-
``pipeline_database`` via ``attach_databases()``. ``run_id`` is passed
723-
as a call-time keyword argument from the orchestrator.
719+
class SideEffectNode(StreamBase):
720+
"""Lightweight blueprint node for side-effect pods.
724721
725-
Inherits from ``StreamBase`` for identity infrastructure and to satisfy
726-
the ``producer`` / ``upstreams`` / ``output_schema`` contract required
727-
by ``SyncPipelineOrchestrator._materialize_as_stream``.
722+
Used by ``Pipeline`` (the blueprint) to represent a side-effect pod
723+
invocation without any DB attachment or execution logic. Analogous to
724+
``FunctionNode`` in the function pod hierarchy.
728725
729726
Args:
730727
side_effect_pod: The ``SideEffectPod`` this node wraps.
@@ -743,8 +740,6 @@ def __init__(
743740
self._pod = side_effect_pod
744741
self._input_stream = input_stream
745742
super().__init__(label=label)
746-
self._pipeline_database: ArrowDatabaseProtocol | None = None
747-
self._table_path: tuple[str, ...] | None = None
748743

749744
# ------------------------------------------------------------------
750745
# StreamBase interface
@@ -808,14 +803,7 @@ def as_table(
808803
columns: ColumnConfig | dict[str, Any] | None = None,
809804
all_info: bool = False,
810805
) -> pa.Table:
811-
"""Collect all rows from ``iter_data()`` into an Arrow table.
812-
813-
Warning:
814-
Calling ``as_table()`` on a ``SideEffectJobNode`` iterates via
815-
``iter_data()``, which re-invokes the side-effect function for each
816-
row with no DB logging and no ``run_id``. Use ``execute()`` for
817-
orchestrated execution.
818-
"""
806+
"""Collect all rows from ``iter_data()`` into an Arrow table."""
819807
from orcapod.types import ColumnConfig as _ColumnConfig
820808
from orcapod.utils import arrow_utils
821809

@@ -832,6 +820,47 @@ def as_table(
832820
pa.concat_tables(data_tables),
833821
)
834822

823+
@property
824+
def node_uri(self) -> tuple[str, ...]:
825+
"""Canonical URI tuple identifying this side-effect node.
826+
827+
Returns:
828+
A tuple of the form ``("side_effect", label, content_hash_string)``.
829+
"""
830+
return ("side_effect", self._pod.label, self._pod.content_hash().to_string())
831+
832+
833+
# ---------------------------------------------------------------------------
834+
# SideEffectJobNode — DB-backed execution node
835+
# ---------------------------------------------------------------------------
836+
837+
838+
class SideEffectJobNode(SideEffectNode):
839+
"""DB-backed execution node for side-effect pods.
840+
841+
Created at pipeline compile time by ``PipelineJob``. Receives a
842+
``pipeline_database`` via ``attach_databases()``. ``run_id`` is passed
843+
as a call-time keyword argument from the orchestrator.
844+
845+
Extends ``SideEffectNode`` with DB attachment and orchestrated execution
846+
methods. Analogous to ``FunctionJobNode`` in the function pod hierarchy.
847+
848+
Args:
849+
side_effect_pod: The ``SideEffectPod`` this node wraps.
850+
input_stream: The upstream stream at compile time.
851+
label: Optional display label.
852+
"""
853+
854+
def __init__(
855+
self,
856+
side_effect_pod: SideEffectPod,
857+
input_stream: StreamProtocol,
858+
label: str | None = None,
859+
) -> None:
860+
super().__init__(side_effect_pod=side_effect_pod, input_stream=input_stream, label=label)
861+
self._pipeline_database: ArrowDatabaseProtocol | None = None
862+
self._table_path: tuple[str, ...] | None = None
863+
835864
# ------------------------------------------------------------------
836865
# DB attachment
837866
# ------------------------------------------------------------------

0 commit comments

Comments
 (0)