add llama pipeline pkg

Signed-off-by: Jaideep Rao <jrao@redhat.com>

Author: jaideepr97

src/instructlab/sdg/__init__.py

@@ -28,6 +28,7 @@
     "SetToMajorityValueBlock",
     "FULL_PIPELINES_PACKAGE",
     "SIMPLE_PIPELINES_PACKAGE",
+    "LLAMA_PIPELINES_PKG",
     "generate_data",
     "mix_datasets",
 )
@@ -54,6 +55,7 @@
 from .generate_data import generate_data, mix_datasets
 from .pipeline import (
     FULL_PIPELINES_PACKAGE,
+    LLAMA_PIPELINES_PKG,
     SIMPLE_PIPELINES_PACKAGE,
     EmptyDatasetError,
     Pipeline,

src/instructlab/sdg/generate_data.py

@@ -29,6 +29,7 @@
 from instructlab.sdg.eval_data import generate_eval_task_data, mmlubench_pipe_init
 from instructlab.sdg.pipeline import (
     FULL_PIPELINES_PACKAGE,
+    LLAMA_PIPELINES_PKG,
     SIMPLE_PIPELINES_PACKAGE,
     Pipeline,
     PipelineContext,
@@ -253,6 +254,8 @@ def _sdg_init(ctx, pipeline):
             pipeline_pkg = FULL_PIPELINES_PACKAGE
         elif pipeline == "simple":
             pipeline_pkg = SIMPLE_PIPELINES_PACKAGE
+        elif pipeline == "llama":
+            pipeline_pkg = LLAMA_PIPELINES_PKG
         else:
             # Validate that pipeline is a valid directory and that it contains the required files
             if not os.path.exists(pipeline):

src/instructlab/sdg/pipeline.py

@@ -323,3 +323,4 @@ def _parse_pipeline_config_file(pipeline_yaml):
 SIMPLE_PIPELINES_PACKAGE = "instructlab.sdg.pipelines.simple"
 FULL_PIPELINES_PACKAGE = "instructlab.sdg.pipelines.full"
 EVAL_PIPELINES_PKG = "instructlab.sdg.pipelines.eval"
+LLAMA_PIPELINES_PKG = "instructlab.sdg.pipelines.llama"