Merge pull request #23 from aldbr/metadata-model-semantics

Rename MetadataProcessor to ExecutionHooks

Author: aldbr

scripts/generate_schemas.py

@@ -4,7 +4,7 @@
 This script automatically generates JSON schemas for all registered metadata
 plugins in the DIRAC CWL prototype system. It includes schemas for:
 
-1. Core metadata models (BaseMetadataModel, MetadataDescriptor, TaskDescriptor)
+1. Core metadata models (ExecutionHooksBasePlugin, MetadataDescriptor, TaskDescriptor)
 2. Submission models (JobSubmissionModel, etc.)
 3. All registered metadata plugins (user plugins included)
 
@@ -38,18 +38,18 @@ def collect_pydantic_models() -> Dict[str, Any]:
     # Import core models
     try:
         from dirac_cwl_proto.metadata.core import (
-            BaseMetadataModel,
-            DataManager,
-            JobExecutor,
-            TransformationDataManager,
+            ExecutionHooksBasePlugin,
+            ExecutionHooksHint,
+            SchedulingHint,
+            TransformationExecutionHooksHint,
         )
 
         models.update(
             {
-                "BaseMetadataModel": BaseMetadataModel,
-                "DataManager": DataManager,
-                "JobExecutor": JobExecutor,
-                "TransformationDataManager": TransformationDataManager,
+                "ExecutionHooksBasePlugin": ExecutionHooksBasePlugin,
+                "ExecutionHooksHint": ExecutionHooksHint,
+                "SchedulingHint": SchedulingHint,
+                "TransformationExecutionHooksHint": TransformationExecutionHooksHint,
             }
         )
         logger.info("Collected core metadata models")
@@ -59,22 +59,18 @@ def collect_pydantic_models() -> Dict[str, Any]:
     # Import submission models
     try:
         from dirac_cwl_proto.submission_models import (
-            JobParameterModel,
+            JobInputModel,
             JobSubmissionModel,
-            ProductionStepMetadataModel,
             ProductionSubmissionModel,
-            TaskDescriptionModel,
             TransformationSubmissionModel,
         )
 
         models.update(
             {
-                "TaskDescriptionModel": TaskDescriptionModel,
-                "JobParameterModel": JobParameterModel,
+                "JobInputModel": JobInputModel,
                 "JobSubmissionModel": JobSubmissionModel,
                 "TransformationSubmissionModel": TransformationSubmissionModel,
                 "ProductionSubmissionModel": ProductionSubmissionModel,
-                "ProductionStepMetadataModel": ProductionStepMetadataModel,
             }
         )
         logger.info("Collected submission models")
@@ -103,9 +99,6 @@ def collect_pydantic_models() -> Dict[str, Any]:
 
         excluded_prefixes = (
             "BaseMetadata",
-            "MetadataDescriptor",
-            "TaskDescriptor",
-            "TaskDescription",
             "Job",
             "Transformation",
             "Production",
@@ -208,8 +201,8 @@ def generate_schema(model_class: Any, model_name: str) -> Dict[str, Any]:
             schema["dirac_description"] = model_class.description
         if hasattr(model_class, "vo"):
             schema["dirac_vo"] = model_class.vo
-        if hasattr(model_class, "get_metadata_class"):
-            schema["dirac_metadata_class"] = model_class.get_metadata_class()
+        if hasattr(model_class, "get_hook_plugin"):
+            schema["dirac_hook_plugin"] = model_class.name()
 
         # Set title if not present
         if "title" not in schema:
@@ -345,17 +338,15 @@ def main():
         save_schema(unified_schema, unified_file, args.format)
 
     # Generate plugin summary
-    plugin_models = {
-        k: v for k, v in models.items() if hasattr(v, "get_metadata_class")
-    }
+    plugin_models = {k: v for k, v in models.items() if hasattr(v, "name")}
     if plugin_models:
         summary = {
             "plugins": {
                 name: {
                     "class": model_class.__name__,
-                    "metadata_class": (
-                        model_class.get_metadata_class()
-                        if hasattr(model_class, "get_metadata_class")
+                    "hook_plugin": (
+                        model_class.name()
+                        if hasattr(model_class, "get_hook_plugin")
                         else None
                     ),
                     "description": getattr(model_class, "description", None),

src/dirac_cwl_proto/job/__init__.py

@@ -27,9 +27,9 @@
 from ruamel.yaml import YAML
 from schema_salad.exceptions import ValidationException
 
-from dirac_cwl_proto.metadata.core import BaseMetadataModel
+from dirac_cwl_proto.metadata.core import ExecutionHooksBasePlugin
 from dirac_cwl_proto.submission_models import (
-    JobParameterModel,
+    JobInputModel,
     JobSubmissionModel,
     extract_dirac_hints,
 )
@@ -81,7 +81,7 @@ def submit_job_client(
 
     # Extract and validate dirac hints; unknown hints are logged as warnings.
     try:
-        job_metadata, job_description = extract_dirac_hints(task)
+        job_metadata, job_scheduling = extract_dirac_hints(task)
     except Exception as exc:
         console.print(
             f"[red]:heavy_multiplication_x:[/red] [bold]CLI:[/bold] Invalid DIRAC hints:\n{exc}"
@@ -106,18 +106,18 @@ def submit_job_client(
             if overrides:
                 override_hints = overrides[next(iter(overrides))].get("hints", {})
                 if override_hints:
-                    job_description = job_description.model_copy(
-                        update=override_hints.pop("dirac:job-execution", {})
+                    job_scheduling = job_scheduling.model_copy(
+                        update=override_hints.pop("dirac:scheduling", {})
                     )
                     job_metadata = job_metadata.model_copy(
-                        update=override_hints.pop("dirac:data-management", {})
+                        update=override_hints.pop("dirac:execution-hooks", {})
                     )
 
             # Upload the local files to the sandbox store
             sandbox_id = upload_local_input_files(parameter)
 
             parameters.append(
-                JobParameterModel(
+                JobInputModel(
                     sandbox=[sandbox_id] if sandbox_id else None,
                     cwl=parameter,
                 )
@@ -129,8 +129,8 @@ def submit_job_client(
     job = JobSubmissionModel(
         task=task,
         parameters=parameters,
-        description=job_description,
-        metadata=job_metadata,
+        scheduling=job_scheduling,
+        execution_hooks=job_metadata,
     )
     console.print(
         "[green]:heavy_check_mark:[/green] [bold]CLI:[/bold] Job(s) validated."
@@ -230,8 +230,8 @@ def submit_job_router(job: JobSubmissionModel) -> bool:
                 JobSubmissionModel(
                     task=job.task,
                     parameters=[parameter],
-                    description=job.description,
-                    metadata=job.metadata,
+                    scheduling=job.scheduling,
+                    execution_hooks=job.execution_hooks,
                 )
             )
     logger.info("Job(s) validated!")
@@ -255,8 +255,8 @@ def submit_job_router(job: JobSubmissionModel) -> bool:
 
 def _pre_process(
     executable: CommandLineTool | Workflow | ExpressionTool,
-    arguments: JobParameterModel | None,
-    runtime_metadata: BaseMetadataModel | None,
+    arguments: JobInputModel | None,
+    runtime_metadata: ExecutionHooksBasePlugin | None,
     job_path: Path,
 ) -> list[str]:
     """
@@ -334,7 +334,7 @@ def _post_process(
     stdout: str,
     stderr: str,
     job_path: Path,
-    runtime_metadata: BaseMetadataModel | None,
+    runtime_metadata: ExecutionHooksBasePlugin | None,
 ):
     """
     Post-process the job after execution.
@@ -364,7 +364,9 @@ def run_job(job: JobSubmissionModel) -> bool:
     logger = logging.getLogger("JobWrapper")
     # Instantiate runtime metadata from the serializable descriptor and
     # the job context so implementations can access task inputs/overrides.
-    runtime_metadata = job.metadata.to_runtime(job) if job.metadata else None
+    runtime_metadata = (
+        job.execution_hooks.to_runtime(job) if job.execution_hooks else None
+    )
 
     # Isolate the job in a specific directory
     job_path = Path(".") / "workernode" / f"{random.randint(1000, 9999)}"

src/dirac_cwl_proto/metadata/__init__.py

@@ -8,15 +8,13 @@
 """
 
 from .core import (
-    BaseMetadataModel,
     DataCatalogInterface,
-    DataManager,
-    JobExecutor,
-    MetadataProcessor,
-    TransformationDataManager,
+    ExecutionHooksBasePlugin,
+    ExecutionHooksHint,
+    SchedulingHint,
+    TransformationExecutionHooksHint,
 )
 from .registry import (
-    MetadataPluginRegistry,
     discover_plugins,
     get_registry,
 )
@@ -32,15 +30,12 @@
     pass
 
 __all__ = [
-    # Core classes
-    "BaseMetadataModel",
+    # Core metadata and plugins
+    "ExecutionHooksHint",
+    "TransformationExecutionHooksHint",
+    "ExecutionHooksBasePlugin",
+    "SchedulingHint",
     "DataCatalogInterface",
-    "DataManager",
-    "MetadataProcessor",
-    "JobExecutor",
-    "TransformationDataManager",
-    # Registry functions
-    "MetadataPluginRegistry",
-    "discover_plugins",
+    "MetadataRegistry",
     "get_registry",
 ]