Huggingface direct download to format command

deepfabric/format_command.py

@@ -6,7 +6,10 @@
 
 
 def format_command(
-    input_file: str,
+    input_file: str | None = None,
+    *,
+    repo: str | None = None,
+    split: str | None = None,
     config_file: str | None = None,
     formatter: str | None = None,
     output: str | None = None,
@@ -16,16 +19,51 @@ def format_command(
 
     Args:
         input_file: Path to the input JSONL dataset file
+        repo: Optional Hugging Face dataset repo id (e.g., "org/dataset-name")
+        split: Optional split to load from the Hugging Face dataset (default: train)
         config_file: Optional YAML config file with formatter settings
         formatter: Optional formatter name (e.g., 'im_format')
         output: Optional output file path
     """
     tui = get_tui()
 
-    # Load the existing dataset
-    tui.info(f"Loading dataset from {input_file}...")
-    dataset = Dataset.from_jsonl(input_file)
-    tui.success(f"Loaded {len(dataset)} samples")
+    if (input_file is None and repo is None) or (input_file and repo):
+        raise ValueError("Specify exactly one of INPUT_FILE or --repo")
+
+    # Load the existing dataset from local file or Hugging Face repo
+    if input_file:
+        tui.info(f"Loading dataset from {input_file}...")
+        dataset = Dataset.from_jsonl(input_file)
+        tui.success(f"Loaded {len(dataset)} samples")
+    else:
+        # Lazy import to avoid overhead when not needed
+        try:
+            from datasets import load_dataset  # type: ignore  # noqa: PLC0415
+            from datasets.exceptions import (  # type: ignore  # noqa: PLC0415
+                DatasetNotFoundError,
+                UnexpectedSplitsError,
+            )
+        except ImportError as e:  # pragma: no cover - import path
+            raise RuntimeError(
+                "The 'datasets' library is required to load from --repo. Please install it."
+            ) from e
+
+        hf_split = split or "train"
+        tui.info(f"Loading dataset from Hugging Face repo '{repo}' (split: {hf_split})...")
+        try:
+            # Bandit nosec, as no digest is set.
+            hf_ds = load_dataset(str(repo), split=hf_split) #  nosec
+        except (DatasetNotFoundError, UnexpectedSplitsError) as e:
+            msg = (
+                "Failed to load dataset from Hugging Face repo "
+                f"'{repo}' with split '{hf_split}': {e}"
+            )
+            raise RuntimeError(msg) from e
+
+        # Convert to DeepFabric Dataset from list of dicts
+        samples = list(hf_ds)
+        dataset = Dataset.from_list(samples)
+        tui.success(f"Loaded {len(dataset)} samples from {repo}:{hf_split}")
 
     # Determine formatter configuration
     formatter_configs = []
@@ -42,7 +80,11 @@ def format_command(
             raise ValueError("No formatters found in config file")
     elif formatter:
         # Use specified formatter with default settings
-        output_file = output or f"{input_file.rsplit('.', 1)[0]}_{formatter}.jsonl"
+        if input_file:
+            output_file = output or f"{input_file.rsplit('.', 1)[0]}_{formatter}.jsonl"
+        else:
+            # When loading from --repo, default to a simple formatted.jsonl unless specified
+            output_file = output or "formatted.jsonl"
 
         # Default configs for common formatters
         default_configs = {
@@ -79,13 +121,21 @@ def format_command(
                 "reasoning_level": "high",
                 "include_metadata": True,
             },
+            # TRL SFT Tools formatter defaults
+            "trl_sft_tools": {},
+            "trl": {},  # alias
             "xlam_v2": {},
         }
 
+        # Map aliases to actual builtin module names
+        template_name = formatter
+        if formatter == "trl":
+            template_name = "trl_sft_tools"
+
         formatter_configs = [
             {
                 "name": formatter,
-                "template": f"builtin://{formatter}.py",
+                "template": f"builtin://{template_name}.py",
                 "output": output_file,
                 "config": default_configs.get(formatter, {}),
             }
@@ -109,7 +159,15 @@ def format_command(
 
 
 @click.command(name="format")
-@click.argument("input_file", type=click.Path(exists=True))
+@click.argument("input_file", type=click.Path(exists=True), required=False)
+@click.option(
+    "--repo",
+    help="Hugging Face dataset repo id (e.g., 'org/dataset-name')",
+)
+@click.option(
+    "--split",
+    help="Split to load from Hugging Face dataset (default: train)",
+)
 @click.option(
     "--config-file",
     "-c",
@@ -119,7 +177,19 @@ def format_command(
 @click.option(
     "--formatter",
     "-f",
-    type=click.Choice(["im_format", "unsloth", "alpaca", "chatml", "grpo", "harmony", "xlam_v2"]),
+    type=click.Choice(
+        [
+            "im_format",
+            "unsloth",
+            "alpaca",
+            "chatml",
+            "grpo",
+            "harmony",
+            "trl",
+            "trl_sft_tools",
+            "xlam_v2",
+        ]
+    ),
     help="Formatter to apply",
 )
 @click.option(
@@ -129,11 +199,24 @@ def format_command(
 )
 @click.pass_context
 def format_cli(
-    ctx, input_file: str, config_file: str | None, formatter: str | None, output: str | None
+    ctx,
+    input_file: str | None,
+    repo: str | None,
+    split: str | None,
+    config_file: str | None,
+    formatter: str | None,
+    output: str | None,
 ) -> None:
     """Apply formatters to an existing dataset."""
     try:
-        format_command(input_file, config_file, formatter, output)
+        format_command(
+            input_file,
+            repo=repo,
+            split=split,
+            config_file=config_file,
+            formatter=formatter,
+            output=output,
+        )
     except FileNotFoundError as e:
         ctx.fail(f"Input file not found: {e}")
     except Exception as e:

docs/api/config.md

@@ -64,7 +64,7 @@ tree_args = config.get_tree_args(
     degree=5,
     temperature=0.9,
     provider="anthropic",
-    model="claude-3-opus"
+    model="claude-sonnet-4-5"
 )
 ```
 
@@ -228,8 +228,8 @@ from deepfabric.config import construct_model_string
 model_string = construct_model_string("openai", "gpt-4")
 # Returns: "openai/gpt-4"
 
-model_string = construct_model_string("anthropic", "claude-3-opus")
-# Returns: "anthropic/claude-3-opus"
+model_string = construct_model_string("anthropic", "claude-sonnet-4-5")
+# Returns: "anthropic/claude-sonnet-4-5"
 ```
 
 #### get_provider_config(provider: str)
@@ -311,7 +311,7 @@ generator = create_generator_with_overrides(
     "base_config.yaml",
     temperature=0.8,
     provider="anthropic",
-    model="claude-3-opus"
+    model="claude-sonnet-4-5"
 )
 
 # Multi-environment configuration

docs/api/generator.md

@@ -30,7 +30,7 @@ generator = DataSetGenerator(
 
 **provider** (str, required): LLM provider name, e.g., `openai`, `anthropic`, `gemini`, `ollama`.
 
-**model_name** (str, required): Model name specific to the provider, e.g., `gpt-4`, `claude-3-opus`.
+**model_name** (str, required): Model name specific to the provider, e.g., `gpt-4`, `claude-sonnet-4-5`.
 
 **temperature** (float, optional): Controls creativity and diversity in content generation. Range 0.0-2.0, typically 0.7-0.9. Default: 0.7.
 
@@ -112,7 +112,7 @@ Generate a single batch of examples for fine-grained control:
 batch = generator.create_batch(
     topics=selected_topics,
     batch_size=3,
-    model_name="openai/gpt-3.5-turbo"
+    model_name="openai/gpt-4-turbo"
 )
 ```
 
@@ -314,7 +314,7 @@ complex_generator = DataSetGenerator(
     instructions="Create advanced technical content",
     generation_system_prompt="You are an expert technical writer",
     provider="anthropic",
-    model_name="claude-3-opus",
+    model_name="claude-sonnet-4-5",
     temperature=0.7
 )
 
@@ -323,7 +323,7 @@ simple_generator = DataSetGenerator(
     instructions="Create basic explanations",
     generation_system_prompt="You are a teacher for beginners",
     provider="openai",
-    model_name="gpt-3.5-turbo",
+    model_name="gpt-4-turbo",
     temperature=0.8
 )
 

docs/api/graph.md

@@ -21,7 +21,7 @@ from deepfabric import Graph
 
 graph = Graph(
     topic_prompt="Artificial intelligence research areas",
-    model_name="anthropic/claude-3-opus",
+    model_name="anthropic/claude-sonnet-4-5",
     topic_system_prompt="You are mapping interconnected research concepts.",
     degree=4,        # Connections per node
     depth=3,         # Maximum distance from root
@@ -56,7 +56,7 @@ from deepfabric import Graph
 # Create and build a graph
 graph = Graph(
     topic_prompt="Artificial intelligence research areas",
-    model_name="anthropic/claude-3-opus",
+    model_name="anthropic/claude-sonnet-4-5",
     degree=4,
     depth=3,
     temperature=0.8
@@ -157,7 +157,7 @@ Reconstructs graph from previously saved JSON files:
 ```python
 graph = Graph(
     topic_prompt="Default prompt",
-    model_name="anthropic/claude-3-opus"
+    model_name="anthropic/claude-sonnet-4-5"
 )
 graph.load("existing_graph.json")
 ```
@@ -170,7 +170,7 @@ Class method for loading graphs with specific configuration:
 graph = Graph.from_json(
     "saved_graph.json",
     topic_prompt="Research areas",
-    model_name="anthropic/claude-3-opus"
+    model_name="anthropic/claude-sonnet-4-5"
 )
 ```
 
@@ -211,7 +211,7 @@ Control graph construction through individual phases:
 ```python
 graph = Graph(
     topic_prompt="Complex domain",
-    model_name="anthropic/claude-3-opus",
+    model_name="anthropic/claude-sonnet-4-5",
     degree=4,
     depth=3
 )
@@ -274,7 +274,7 @@ Graphs integrate seamlessly with dataset generation:
 # Generate dataset from graph
 generator = DataSetGenerator(
     instructions="Create interconnected explanations",
-    model_name="anthropic/claude-3-opus",
+    model_name="anthropic/claude-sonnet-4-5",
     temperature=0.7
 )
 dataset = generator.create_data(

docs/cli/format.md

@@ -5,18 +5,27 @@ The `format` command allows you to apply formatters to existing datasets without
 ## Usage
 
 ```bash
+# From a local JSONL file
 deepfabric format INPUT_FILE [OPTIONS]
+
+# Or directly from a Hugging Face dataset repo
+deepfabric format --repo ORG/DATASET [OPTIONS]
 ```
 
 ## Arguments
 
 - `INPUT_FILE` - Path to the input JSONL dataset file to format
+  - Alternatively, use `--repo ORG/DATASET` to load from the Hugging Face Hub
 
 ## Options
 
 - `-c, --config-file PATH` - YAML config file containing formatter settings
-- `-f, --formatter [im_format|unsloth|alpaca|chatml|grpo]` - Quick formatter selection with default settings
-- `-o, --output TEXT` - Output file path (default: `input_file_formatter.jsonl`)
+- `-f, --formatter [im_format|unsloth|alpaca|chatml|grpo|harmony|trl|xlam_v2]` - Quick formatter selection with default settings
+- `-o, --output TEXT` - Output file path
+  - Local file input: defaults to `input_file_formatter.jsonl`
+  - `--repo` input: defaults to `formatted.jsonl`
+- `--repo TEXT` - Hugging Face dataset repo id (e.g., `org/dataset-name`)
+- `--split TEXT` - Dataset split to load when using `--repo` (default: `train`)
 - `--help` - Show help message
 
 ## Examples
@@ -153,9 +162,22 @@ The command expects a JSONL file where each line is a JSON object. Supported for
 }
 ```
 
-### Working with HuggingFace Datasets
+### Working with Hugging Face Datasets
+
+You can now format datasets directly from the Hugging Face Hub using `--repo`, or continue using local JSONL files. Many HF datasets come in compatible formats:
+
+**Format directly from a Hub repo:**
+```bash
+# Pull from Hub, format to Harmony, write to formatted.jsonl
+deepfabric format --repo "org/dataset-name" --format harmony
 
-The format command works seamlessly with datasets downloaded from HuggingFace Hub. Many HF datasets come in compatible formats:
+# Load with datasets library
+python - <<'PY'
+from datasets import load_dataset
+ds = load_dataset("json", data_files="formatted.jsonl")
+print(ds)
+PY
+```
 
 **For datasets with `messages` field (e.g., chat datasets):**
 ```bash
@@ -185,7 +207,7 @@ deepfabric format alpaca_dataset.jsonl -f im_format
 - ShareGPT format (`conversations`)
 - Q&A format (`question`, `answer` or `response`)
 
-**Example conversion workflow:**
+**Example conversion workflow (local):**
 ```bash
 # 1. Download from HuggingFace
 huggingface-cli download tatsu-lab/alpaca --repo-type dataset
@@ -218,3 +240,14 @@ deepfabric format dataset_raw.jsonl -f grpo -o dataset_grpo.jsonl
 ```
 
 This allows you to prepare the same dataset for different training frameworks without regenerating the data.
+### TRL SFT Tools
+
+Use `-f trl` to convert agent/tool datasets to the Hugging Face TRL SFT tool-calling format. This maps to the built-in `trl_sft_tools` formatter.
+
+```bash
+# Format from a local file
+deepfabric format dataset.jsonl -f trl -o trl_sft_tools.jsonl
+
+# Or format directly from a Hub repo
+deepfabric format --repo org/dataset -f trl -o trl_sft_tools.jsonl
+```

docs/cli/generate.md

@@ -21,7 +21,7 @@ Override specific configuration parameters without modifying the configuration f
 ```bash
 deepfabric generate config.yaml \
   --provider anthropic \
-  --model claude-3-opus \
+  --model claude-sonnet-4-5 \
   --temperature 0.8 \
   --num-steps 100 \
   --batch-size 5
@@ -118,7 +118,7 @@ deepfabric generate research-dataset.yaml \
   --save-tree research_topics.jsonl \
   --dataset-save-as research_examples.jsonl \
   --provider anthropic \
-  --model claude-3-opus \
+  --model claude-sonnet-4-5 \
   --degree 4 \
   --depth 3 \
   --num-steps 200 \

docs/cli/validate.md

@@ -91,7 +91,7 @@ deepfabric validate config.yaml
 Provider Validation:
    OpenAI API key detected (OPENAI_API_KEY)
    Model gpt-4 is available
-  ⚠️  Model gpt-4 has higher costs than gpt-3.5-turbo
+  ⚠️  Model gpt-4 has higher costs than gpt-4-turbo
 ```
 
 ## Development Workflow Integration