feat(outputs-separation): separate native tasks and output

packages/nemo-evaluator/src/nemo_evaluator/api/api_dataclasses.py

@@ -18,7 +18,7 @@
 from typing import Any, Dict, Optional
 
 import jinja2
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, computed_field
 
 from nemo_evaluator.adapters.adapter_config import AdapterConfig
 
@@ -116,6 +116,18 @@ class EvaluationConfig(BaseModel):
     )
     type: Optional[str] = Field(description="Type of the task", default=None)
 
+    @computed_field
+    @property
+    def task_output_dir(self) -> Optional[str]:
+        """Output directory with task name appended.
+
+        Returns the output_dir with the task name (type) appended.
+        This is used by both command rendering and output parsing.
+        """
+        if self.output_dir and self.type:
+            return f"{self.output_dir}/{self.type}"
+        return self.output_dir
+
 
 class EvaluationMetadata(dict):
     """We put here various evaluation metadata that does not influence the evaluation."""
@@ -133,6 +145,12 @@ class Evaluation(BaseModel):
     def render_command(self):
         values = self.model_dump()
 
+        # Add task-specific output directory for use in templates
+        if values.get("config", {}).get("output_dir") and values.get("config", {}).get(
+            "type"
+        ):
+            values["config"]["output_dir"] = self.config.task_output_dir
+
         def recursive_render(tpl):
             prev = tpl
             while True:

packages/nemo-evaluator/src/nemo_evaluator/core/evaluate.py

@@ -48,7 +48,8 @@
 def parse_output(evaluation: Evaluation) -> EvaluationResult:
     # create a module name that is importable
     output_module = importlib.import_module(f"core_evals.{evaluation.pkg_name}.output")
-    return output_module.parse_output(evaluation.config.output_dir)
+    # Use task-specific output directory (matches the rendered command path)
+    return output_module.parse_output(evaluation.config.task_output_dir)
 
 
 def evaluate(

packages/nemo-evaluator/src/nemo_evaluator/core/input.py

@@ -425,6 +425,9 @@ def check_type_compatibility(evaluation: Evaluation):
 def prepare_output_directory(evaluation: Evaluation):
     try:
         os.makedirs(evaluation.config.output_dir, exist_ok=True)
+        # Also create task-specific output directory
+        if evaluation.config.task_output_dir != evaluation.config.output_dir:
+            os.makedirs(evaluation.config.task_output_dir, exist_ok=True)
     except OSError as error:
         print(f"An error occurred while creating output directory: {error}")