refactor(pipeline): extract PipelineRunner, RetryManager & ExecutionContext; simplify Pipeline

- Introduce PipelineRunner to encapsulate pipeline execution flow (sync & async).
- Add RetryManager to centralize retry/backoff logic for runs (sync + async).
- Add ExecutionContextBuilder to create executors, shutdown hooks and adapters.
- Add telemetry helper (initialize_telemetry) and ensure logging initialization helper.
- Simplify Pipeline: delegate run/run_async to PipelineRunner, expose adapter_manager & executor_factory, lazily create Runner.
- Extend RunConfig handling: add executor_override_raw and utilities to resolve/merge executor overrides.
- Harden RunConfig/RetryConfig serialization and exception string handling.
- Add examples for hello-world (pipelines, parallel pipeline, project config, python pipeline modules).
- Add tests for runner and updated pipeline tests to assert runner delegation and async parity.
- Remove docs cleanup plan/tasks (moved/removed) and update .gitignore to ignore docs/refactoring and cleanup plan.
- Bump versions: pyproject -> 0.32.6; uv.lock -> 0.32.5

Author: legout

.gitignore

@@ -82,3 +82,6 @@ flowerpower_llm_context7.md
 flowerpower_llm_doc_1.md
 flowerpower_llm_doc_2.md
 flowerpower_llm_doc_3.md
+docs/refactoring/
+docs/cleanup-refactor-plan.md
+.gitignore

docs/cleanup-refactor-plan.md

@@ -0,0 +1,44 @@
+adapter:
+  hamilton_tracker:
+    capture_data_statistics: true
+    dag_name: null
+    max_dict_length_capture: 10
+    max_list_length_capture: 50
+    project_id: null
+    tags: {}
+  mlflow:
+    experiment_description: null
+    experiment_name: null
+    experiment_tags: {}
+    run_description: null
+    run_id: null
+    run_name: null
+    run_tags: {}
+params:
+  avg_x_wk_spend:
+    rolling: 3
+  spend_zero_mean:
+    offset: 0
+run:
+  cache: false
+  config:
+    range: 10_000
+  executor:
+    max_workers: 40
+    num_cpus: 8
+    type: local
+  final_vars:
+    - spend
+    - signups
+    - avg_x_wk_spend
+    - spend_per_signup
+    - spend_zero_mean_unit_variance
+  inputs: {}
+  log_level: null
+  with_adapter:
+    future: false
+    mlflow: false
+    opentelemetry: false
+    progressbar: false
+    ray: false
+    tracker: false

docs/cleanup-tasks.md

@@ -0,0 +1,45 @@
+adapter:
+  hamilton_tracker:
+    capture_data_statistics: true
+    dag_name: null
+    max_dict_length_capture: 10
+    max_list_length_capture: 50
+    project_id: null
+    tags: {}
+  mlflow:
+    experiment_description: null
+    experiment_name: null
+    experiment_tags: {}
+    run_description: null
+    run_id: null
+    run_name: null
+    run_tags: {}
+params: {}
+run:
+  adapter: null
+  cache: false
+  config: {}
+  executor:
+    max_workers: 60
+    num_cpus: 12
+    type: threadpool
+  final_vars: []
+  inputs: {}
+  jitter_factor: 0.1
+  log_level: INFO
+  max_retries: 3
+  on_failure: null
+  on_success: null
+  pipeline_adapter_cfg: null
+  project_adapter_cfg: null
+  reload: false
+  retry_delay: 1
+  retry_exceptions:
+  - <class 'Exception'>
+  with_adapter:
+    future: false
+    hamilton_tracker: false
+    mlflow: false
+    opentelemetry: false
+    progressbar: false
+    ray: false

examples/hello-world/conf/pipelines/hello_world.yml

@@ -0,0 +1,18 @@
+name: hello-world
+adapter:
+  hamilton_tracker:
+    username: null
+    api_url: http://localhost:8241
+    ui_url: http://localhost:8242
+    api_key: null
+    verify: false
+  mlflow:
+    tracking_uri: null
+    registry_uri: null
+    artifact_location: null
+  ray:
+    ray_init_config: null
+    shutdown_ray_on_completion: false
+  opentelemetry:
+    host: localhost
+    port: 6831

examples/hello-world/conf/pipelines/hello_world_parallel.yml

@@ -0,0 +1,90 @@
+# FlowerPower pipeline hello_world.py
+# Created on 2024-10-26 12:44:27
+
+
+import sys
+import time
+from pathlib import Path
+
+import pandas as pd
+from hamilton.function_modifiers import config, parameterize
+from loguru import logger
+
+from flowerpower.cfg import Config
+
+PARAMS = Config.load(
+    Path(__file__).parents[1], pipeline_name="hello_world"
+).pipeline.h_params
+
+
+@config.when(range=10_000)
+def spend__10000() -> pd.Series:
+    """Returns a series of spend data."""
+    # time.sleep(2)
+    return pd.Series(range(10_000)) * 10
+
+
+@config.when(range=10_000)
+def signups__10000() -> pd.Series:
+    """Returns a series of signups data."""
+    time.sleep(1)
+    print(10_000)
+    return pd.Series(range(10_000))
+
+
+@config.when(range=1_000)
+def spend__1000() -> pd.Series:
+    """Returns a series of spend data."""
+    # time.sleep(2)
+    print(1_000)
+    return pd.Series(range(10_000)) * 10
+
+
+@config.when(range=1_000)
+def signups__1000() -> pd.Series:
+    """Returns a series of signups data."""
+    time.sleep(1)
+    print(1_000)
+    return pd.Series(range(10_000))
+
+
+@parameterize(
+    **PARAMS.avg_x_wk_spend
+)  # (avg_x_wk_spend={"rolling": value(3)})  #
+def avg_x_wk_spend(spend: pd.Series, rolling: int) -> pd.Series:
+    """Rolling x week average spend."""
+    # time.sleep(2)
+    return spend.rolling(rolling).mean()
+
+
+def spend_per_signup(spend: pd.Series, signups: pd.Series) -> pd.Series:
+    """The cost per signup in relation to spend."""
+    time.sleep(1)
+    return spend / signups
+
+
+def spend_mean(spend: pd.Series) -> float:
+    """Shows function creating a scalar. In this case it computes the mean of the entire column."""
+    return spend.mean()
+
+
+@parameterize(
+    **PARAMS.spend_zero_mean
+)  # (spend_zero_mean={"offset": value(0)})  #
+def spend_zero_mean(spend: pd.Series, spend_mean: float, offset: int) -> pd.Series:
+    """Shows function that takes a scalar. In this case to zero mean spend."""
+    return spend - spend_mean + offset
+
+
+def spend_std_dev(spend: pd.Series) -> float:
+    """Function that computes the standard deviation of the spend column."""
+    return spend.std()
+
+
+def spend_zero_mean_unit_variance(
+    spend_zero_mean: pd.Series, spend_std_dev: float, verbose: bool = False
+) -> pd.Series:
+    """Function showing one way to make spend have zero mean and unit variance."""
+    if verbose:
+        logger.info(f"spend_zero_mean_unit_variance {spend_zero_mean / spend_std_dev}")
+    return spend_zero_mean / spend_std_dev

examples/hello-world/conf/project.yml

@@ -0,0 +1,92 @@
+# FlowerPower pipeline hello_world_parallel.py
+# Created on 2025-10-14 02:39:22
+
+####################################################################################################
+# Import necessary libraries
+# NOTE: Remove or comment out imports that are not used in the pipeline
+
+from hamilton.function_modifiers import parameterize, dataloader, datasaver
+from hamilton.htypes import Parallelizable, Collect
+
+from pathlib import Path
+
+from flowerpower.cfg import Config
+import requests
+
+####################################################################################################
+# Load pipeline parameters. Do not modify this section.
+
+PARAMS = Config.load(
+    Path(__file__).parents[1], pipeline_name="hello_world_parallel"
+).pipeline.h_params
+
+
+####################################################################################################
+# Helper functions.
+# This functions have to start with an underscore (_).
+
+
+def _list_all_urls() -> list[str]:
+    return [
+        "https://www.gutenberg.org/files/1342/1342-0.txt",  # Pride and Prejudice
+        "https://www.gutenberg.org/files/11/11-0.txt",  # Alice's Adventures in Wonderland
+        "https://www.gutenberg.org/files/84/84-0.txt",  # Frankenstein
+        "https://www.gutenberg.org/files/98/98-0.txt",  # A Tale of Two Cities
+        "https://www.gutenberg.org/files/2701/2701-0.txt",  #
+        "https://www.gutenberg.org/files/1232/1232-0.txt",  # The Prince
+        "https://www.gutenberg.org/files/74/74-0.txt",  # The Adventures of Tom Sawyer
+        "https://www.gutenberg.org/files/5200/5200-0.txt",  # Metamorphosis
+        "https://www.gutenberg.org/files/16328/16328-0.txt",  # Beowulf
+        "https://www.gutenberg.org/files/55/55-0.txt",  # The Wonderful Wizard of Oz
+        "https://www.gutenberg.org/files/1080/1080-0.txt",  # A Modest Proposal
+        "https://www.gutenberg.org/files/345/345-0.txt",  # Dracula by Bram Stoker
+        "https://www.gutenberg.org/files/174/174-0.txt",  # The Picture of Dorian Gray
+        "https://www.gutenberg.org/files/23/23-0.txt",  # The Scarlet Letter
+        "https://www.gutenberg.org/files/768/768-0.txt",  # Wuthering Heights by Emily Brontë
+        "https://www.gutenberg.org/files/1260/1260-0.txt",  # Jane Eyre by Charlotte Brontë
+        "https://www.gutenberg.org/files/1399/1399-0.txt",  # The Iliad by Homer
+        "https://www.gutenberg.org/files/135/135-0.txt",  # The Odyssey by Homer
+        "https://www.gutenberg.org/files/author/1342.txt",  # The Complete Works of William Shakespeare
+        "https://www.gutenberg.org/files/author/11.txt",  # The Complete Works of Lewis Carroll
+        "https://www.gutenberg.org/files/author/84.txt",  # The Complete Works of Mary Shelley
+        "https://www.gutenberg.org/files/author/98.txt",  # The Complete Works of Charles Dickens
+        "https://www.gutenberg.org/files/author/2701.txt",  # The Complete Works of Herman Melville
+        "https://www.gutenberg.org/files/author/1232.txt",  # The Complete Works of Niccolò Machiavelli
+        "https://www.gutenberg.org/files/author/74.txt",  # The Complete Works of Mark Twain
+        "https://www.gutenberg.org/files/author/5200.txt",  # The Complete Works of Franz Kafka
+        "https://www.gutenberg.org/files/author/16328.txt",  # The Complete Works of Anonymous
+        "https://www.gutenberg.org/files/author/55.txt",  # The Complete Works of L. Frank Baum
+        "https://www.gutenberg.org/files/author/1080.txt",  # The Complete Works of Jonathan Swift
+        "https://www.gutenberg.org/files/author/345.txt",  # The Complete Works of Bram Stoker
+        "https://www.gutenberg.org/files/author/174.txt",  # The Complete Works of Oscar Wilde
+        "https://www.gutenberg.org/files/author/23.txt",  # The Complete Works of Nathaniel Hawthorne
+        "https://www.gutenberg.org/files/author/768.txt",  # The Complete Works of Emily Brontë
+        "https://www.gutenberg.org/files/author/1260.txt",  # The Complete Works of Charlotte Brontë
+    ]
+
+
+def _load(url: str) -> str:
+    response = requests.get(url)
+    response.raise_for_status()
+    return response.text
+
+
+####################################################################################################
+# Pipeline functions
+
+
+def url() -> Parallelizable[str]:
+    for url_ in _list_all_urls():
+        yield url_
+
+
+def url_loaded(url: str) -> str:
+    return _load(url)
+
+
+def counts(url_loaded: str) -> int:
+    return len(url_loaded.split(" "))
+
+
+def total_words(counts: Collect[int]) -> int:
+    return sum(counts)