Merge pull request #597 from jaideepr97/add-llama-pkg

feat: add llama pipeline pkg

Author: mergify[bot]

src/instructlab/sdg/__init__.py

@@ -28,6 +28,7 @@
     "SetToMajorityValueBlock",
     "FULL_PIPELINES_PACKAGE",
     "SIMPLE_PIPELINES_PACKAGE",
+    "LLAMA_PIPELINES_PKG",
     "generate_data",
     "mix_datasets",
 )
@@ -54,6 +55,7 @@
 from .generate_data import generate_data, mix_datasets
 from .pipeline import (
     FULL_PIPELINES_PACKAGE,
+    LLAMA_PIPELINES_PKG,
     SIMPLE_PIPELINES_PACKAGE,
     EmptyDatasetError,
     Pipeline,

src/instructlab/sdg/generate_data.py

@@ -30,6 +30,7 @@
 from instructlab.sdg.eval_data import generate_eval_task_data, mmlubench_pipe_init
 from instructlab.sdg.pipeline import (
     FULL_PIPELINES_PACKAGE,
+    LLAMA_PIPELINES_PKG,
     SIMPLE_PIPELINES_PACKAGE,
     Pipeline,
     PipelineContext,
@@ -254,6 +255,8 @@ def _sdg_init(ctx, pipeline):
             pipeline_pkg = FULL_PIPELINES_PACKAGE
         elif pipeline == "simple":
             pipeline_pkg = SIMPLE_PIPELINES_PACKAGE
+        elif pipeline == "llama":
+            pipeline_pkg = LLAMA_PIPELINES_PKG
         else:
             # Validate that pipeline is a valid directory and that it contains the required files
             if not os.path.exists(pipeline):

src/instructlab/sdg/pipeline.py

@@ -323,3 +323,4 @@ def _parse_pipeline_config_file(pipeline_yaml):
 SIMPLE_PIPELINES_PACKAGE = "instructlab.sdg.pipelines.simple"
 FULL_PIPELINES_PACKAGE = "instructlab.sdg.pipelines.full"
 EVAL_PIPELINES_PKG = "instructlab.sdg.pipelines.eval"
+LLAMA_PIPELINES_PKG = "instructlab.sdg.pipelines.llama"