fix: Init engines lazily (#181)

rmitsch · Raphael Mitsch · web-flow · commit 6a84fe086340 · 2025-10-12T18:58:13.000+02:00
* feat: HF classification uv script.

* chore: Correct package name.

* fix: Fix label normalization in classification HF dataset generation.

* feat: Example uv script for zero-shot HF classification dataset creation and upload.

* chore: Clean up.

* fix: Fix classification dataset test.

* chore: Rename OCR to Ingestion to better reflect task role.

* chore: Migrate to sieves &gt;= 0.15.

* chore: Align with changes in v0.16.0.

* fix: Make ingestion import conditional.

* chore: Bump required sieves version.

* fix: Use defensive approach to fetching objects from modules with optional support.

* fix: Fix import. Remove HF jobs example.

---------

Co-authored-by: Raphael Mitsch &lt;raphael@climatiq.com&gt;
diff --git a/docs/guides/distillation.md b/docs/guides/distillation.md
@@ -40,7 +40,7 @@ Here's how to distill a classification task using SetFit:
 import os
 import dspy
 from sieves.tasks import Classification
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves import Pipeline, Doc
 
 # 1. Run zero-shot classification task
@@ -219,7 +219,7 @@ output_path/
 Model2Vec is ideal for extremely fast inference with minimal resource usage:
 
 ```python
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 
 task.distill(
     base_model_id="minishlab/potion-base-8M",
diff --git a/sieves/engines/utils.py b/sieves/engines/utils.py
@@ -41,14 +41,17 @@ def init_engine(
     """
     model_type = type(model)
     module_engine_map = {
-        dspy_: dspy_.DSPy,
-        glix_: glix_.GliX,
-        huggingface_: huggingface_.HuggingFace,
-        langchain_: langchain_.LangChain,
-        outlines_: outlines_.Outlines,
+        dspy_: getattr(dspy_, "DSPy"),
+        glix_: getattr(glix_, "GliX"),
+        huggingface_: getattr(huggingface_, "HuggingFace"),
+        langchain_: getattr(langchain_, "LangChain"),
+        outlines_: getattr(outlines_, "Outlines"),
     }
 
     for module, engine_type in module_engine_map.items():
+        if engine_type is None:
+            continue
+
         try:
             module_model_types = module.Model.__args__
         except AttributeError:
@@ -64,6 +67,6 @@ def init_engine(
             return internal_engine
 
     raise ValueError(
-        f"Model type {model.__class__} is not supported. Please check the documentation and ensure you're "
-        f"providing a supported model type."
+        f"Model type {model.__class__} is not supported. Please check the documentation and ensure that (1) you're "
+        f"providing a supported model type and that (2) the corresponding library is installed in your environment."
     )
diff --git a/sieves/tasks/__init__.py b/sieves/tasks/__init__.py
@@ -2,7 +2,7 @@
 
 from sieves.tasks import predictive, preprocessing
 from sieves.tasks.core import Task
-from sieves.tasks.postprocessing import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves.tasks.predictive import (
     NER,
     Classification,
diff --git a/sieves/tasks/distillation/__init__.py b/sieves/tasks/distillation/__init__.py
diff --git a/sieves/tasks/distillation/distillation_import.py b/sieves/tasks/distillation/distillation_import.py
diff --git a/sieves/tasks/distillation/types.py b/sieves/tasks/distillation/types.py
@@ -1,17 +1,22 @@
+"""Distillation framework types."""
+
 from __future__ import annotations
 
 import enum
 from typing import Literal
 
 
 class DistillationFramework(enum.Enum):
+    """Available distillation frameworks."""
+
     model2vec = "model2vec"
     sentence_transformers = "sentence_transformers"
     setfit = "setfit"
 
     @classmethod
     def all(cls) -> tuple[DistillationFramework, ...]:
-        """Returns all available engine types.
+        """Return all available engine types.
+
         :return tuple[EngineType, ...]: All available engine types.
         """
         return tuple(dist_type for dist_type in DistillationFramework)
diff --git a/sieves/tasks/postprocessing/__init__.py b/sieves/tasks/postprocessing/__init__.py
diff --git a/sieves/tasks/predictive/classification/core.py b/sieves/tasks/predictive/classification/core.py
@@ -15,8 +15,8 @@
 from sieves.engines import EngineType, dspy_, glix_, huggingface_, langchain_, outlines_
 from sieves.engines.types import GenerationSettings
 from sieves.serialization import Config
-from sieves.tasks.postprocessing.distillation.distillation_import import model2vec, setfit
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.distillation_import import model2vec, setfit
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves.tasks.predictive.bridges import GliXBridge
 from sieves.tasks.predictive.classification.bridges import (
     DSPyClassification,
diff --git a/sieves/tasks/predictive/core.py b/sieves/tasks/predictive/core.py
@@ -24,7 +24,7 @@
 from sieves.serialization import Config
 from sieves.tasks import optimization
 from sieves.tasks.core import Task
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves.tasks.predictive.bridges import TaskBridge, TaskPromptSignature, TaskResult
 from sieves.tasks.types import Model
 
diff --git a/sieves/tasks/predictive/information_extraction/core.py b/sieves/tasks/predictive/information_extraction/core.py
@@ -15,7 +15,7 @@
 from sieves.engines import EngineType, dspy_, glix_, langchain_, outlines_
 from sieves.engines.types import GenerationSettings
 from sieves.serialization import Config
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves.tasks.predictive.core import FewshotExample as BaseFewshotExample
 from sieves.tasks.predictive.core import PredictiveTask
 from sieves.tasks.predictive.information_extraction.bridges import (
diff --git a/sieves/tasks/predictive/ner/core.py b/sieves/tasks/predictive/ner/core.py
@@ -21,7 +21,7 @@
 )
 from sieves.engines.types import GenerationSettings
 from sieves.serialization import Config
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves.tasks.predictive.core import FewshotExample as BaseFewshotExample
 from sieves.tasks.predictive.core import PredictiveTask
 from sieves.tasks.predictive.ner.bridges import (
diff --git a/sieves/tasks/predictive/pii_masking/core.py b/sieves/tasks/predictive/pii_masking/core.py
@@ -12,7 +12,7 @@
 from sieves.engines import EngineType, dspy_, langchain_, outlines_
 from sieves.engines.types import GenerationSettings
 from sieves.serialization import Config
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves.tasks.predictive.core import FewshotExample as BaseFewshotExample
 from sieves.tasks.predictive.core import PredictiveTask
 from sieves.tasks.predictive.pii_masking.bridges import (
diff --git a/sieves/tasks/predictive/question_answering/core.py b/sieves/tasks/predictive/question_answering/core.py
@@ -13,7 +13,7 @@
 from sieves.engines import EngineType, dspy_, glix_, langchain_, outlines_
 from sieves.engines.types import GenerationSettings
 from sieves.serialization import Config
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves.tasks.predictive.bridges import GliXBridge
 from sieves.tasks.predictive.core import FewshotExample as BaseFewshotExample
 from sieves.tasks.predictive.core import PredictiveTask
diff --git a/sieves/tasks/predictive/sentiment_analysis/core.py b/sieves/tasks/predictive/sentiment_analysis/core.py
@@ -14,7 +14,7 @@
 from sieves.engines import EngineType, dspy_, langchain_, outlines_
 from sieves.engines.types import GenerationSettings
 from sieves.serialization import Config
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves.tasks.predictive.core import FewshotExample as BaseFewshotExample
 from sieves.tasks.predictive.core import PredictiveTask
 from sieves.tasks.predictive.sentiment_analysis.bridges import (
diff --git a/sieves/tasks/predictive/summarization/core.py b/sieves/tasks/predictive/summarization/core.py
@@ -13,7 +13,7 @@
 from sieves.engines import EngineType, dspy_, glix_, langchain_, outlines_
 from sieves.engines.types import GenerationSettings
 from sieves.serialization import Config
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves.tasks.predictive.bridges import GliXBridge
 from sieves.tasks.predictive.core import FewshotExample as BaseFewshotExample
 from sieves.tasks.predictive.core import PredictiveTask
diff --git a/sieves/tasks/predictive/translation/core.py b/sieves/tasks/predictive/translation/core.py
@@ -13,7 +13,7 @@
 from sieves.engines import EngineType, dspy_, langchain_, outlines_
 from sieves.engines.types import GenerationSettings
 from sieves.serialization import Config
-from sieves.tasks.postprocessing.distillation.types import DistillationFramework
+from sieves.tasks.distillation.types import DistillationFramework
 from sieves.tasks.predictive.core import FewshotExample as BaseFewshotExample
 from sieves.tasks.predictive.core import PredictiveTask
 from sieves.tasks.predictive.translation.bridges import (
diff --git a/sieves/tasks/preprocessing/ingestion/core.py b/sieves/tasks/preprocessing/ingestion/core.py
@@ -69,11 +69,14 @@ def _init_ingestion_task(self) -> Task:
 
         # Identify the ingestion task that maps to the specified converter's type.
         converter_module_map = {
-            docling_: docling_.Docling,
-            marker_: marker_.Marker,
+            docling_: getattr(docling_, "Docling"),
+            marker_: getattr(marker_, "Marker"),
         }
 
         for module, ingestion_task_type in converter_module_map.items():
+            if ingestion_task_type is None:
+                continue
+
             try:
                 module_converter_types = module.Converter.__args__
             except AttributeError:
@@ -93,8 +96,9 @@ def _init_ingestion_task(self) -> Task:
                 return ingestion_task
 
         raise ValueError(
-            f"converter type {self._converter} is not supported. Please check the documentation "
-            f"and ensure you're providing a supported converter type."
+            f"converter type {self._converter} is not supported. Please check the documentation and ensure that (1) "
+            f"you're providing a supported converter type and that (2) the corresponding library is installed in your "
+            f"environment."
         )
 
     def __call__(self, docs: Iterable[Doc]) -> Iterable[Doc]:
