diff --git a/deepfabric/format_command.py b/deepfabric/format_command.py index 40d14096..ccd1bcc2 100644 --- a/deepfabric/format_command.py +++ b/deepfabric/format_command.py @@ -6,7 +6,10 @@ def format_command( - input_file: str, + input_file: str | None = None, + *, + repo: str | None = None, + split: str | None = None, config_file: str | None = None, formatter: str | None = None, output: str | None = None, @@ -16,16 +19,51 @@ def format_command( Args: input_file: Path to the input JSONL dataset file + repo: Optional Hugging Face dataset repo id (e.g., "org/dataset-name") + split: Optional split to load from the Hugging Face dataset (default: train) config_file: Optional YAML config file with formatter settings formatter: Optional formatter name (e.g., 'im_format') output: Optional output file path """ tui = get_tui() - # Load the existing dataset - tui.info(f"Loading dataset from {input_file}...") - dataset = Dataset.from_jsonl(input_file) - tui.success(f"Loaded {len(dataset)} samples") + if (input_file is None and repo is None) or (input_file and repo): + raise ValueError("Specify exactly one of INPUT_FILE or --repo") + + # Load the existing dataset from local file or Hugging Face repo + if input_file: + tui.info(f"Loading dataset from {input_file}...") + dataset = Dataset.from_jsonl(input_file) + tui.success(f"Loaded {len(dataset)} samples") + else: + # Lazy import to avoid overhead when not needed + try: + from datasets import load_dataset # type: ignore # noqa: PLC0415 + from datasets.exceptions import ( # type: ignore # noqa: PLC0415 + DatasetNotFoundError, + UnexpectedSplitsError, + ) + except ImportError as e: # pragma: no cover - import path + raise RuntimeError( + "The 'datasets' library is required to load from --repo. Please install it." + ) from e + + hf_split = split or "train" + tui.info(f"Loading dataset from Hugging Face repo '{repo}' (split: {hf_split})...") + try: + # Bandit nosec, as no digest is set. + hf_ds = load_dataset(str(repo), split=hf_split) # nosec + except (DatasetNotFoundError, UnexpectedSplitsError) as e: + msg = ( + "Failed to load dataset from Hugging Face repo " + f"'{repo}' with split '{hf_split}': {e}" + ) + raise RuntimeError(msg) from e + + # Convert to DeepFabric Dataset from list of dicts + samples = list(hf_ds) + dataset = Dataset.from_list(samples) + tui.success(f"Loaded {len(dataset)} samples from {repo}:{hf_split}") # Determine formatter configuration formatter_configs = [] @@ -42,7 +80,11 @@ def format_command( raise ValueError("No formatters found in config file") elif formatter: # Use specified formatter with default settings - output_file = output or f"{input_file.rsplit('.', 1)[0]}_{formatter}.jsonl" + if input_file: + output_file = output or f"{input_file.rsplit('.', 1)[0]}_{formatter}.jsonl" + else: + # When loading from --repo, default to a simple formatted.jsonl unless specified + output_file = output or "formatted.jsonl" # Default configs for common formatters default_configs = { @@ -79,13 +121,21 @@ def format_command( "reasoning_level": "high", "include_metadata": True, }, + # TRL SFT Tools formatter defaults + "trl_sft_tools": {}, + "trl": {}, # alias "xlam_v2": {}, } + # Map aliases to actual builtin module names + template_name = formatter + if formatter == "trl": + template_name = "trl_sft_tools" + formatter_configs = [ { "name": formatter, - "template": f"builtin://{formatter}.py", + "template": f"builtin://{template_name}.py", "output": output_file, "config": default_configs.get(formatter, {}), } @@ -109,7 +159,15 @@ def format_command( @click.command(name="format") -@click.argument("input_file", type=click.Path(exists=True)) +@click.argument("input_file", type=click.Path(exists=True), required=False) +@click.option( + "--repo", + help="Hugging Face dataset repo id (e.g., 'org/dataset-name')", +) +@click.option( + "--split", + help="Split to load from Hugging Face dataset (default: train)", +) @click.option( "--config-file", "-c", @@ -119,7 +177,19 @@ def format_command( @click.option( "--formatter", "-f", - type=click.Choice(["im_format", "unsloth", "alpaca", "chatml", "grpo", "harmony", "xlam_v2"]), + type=click.Choice( + [ + "im_format", + "unsloth", + "alpaca", + "chatml", + "grpo", + "harmony", + "trl", + "trl_sft_tools", + "xlam_v2", + ] + ), help="Formatter to apply", ) @click.option( @@ -129,11 +199,24 @@ def format_command( ) @click.pass_context def format_cli( - ctx, input_file: str, config_file: str | None, formatter: str | None, output: str | None + ctx, + input_file: str | None, + repo: str | None, + split: str | None, + config_file: str | None, + formatter: str | None, + output: str | None, ) -> None: """Apply formatters to an existing dataset.""" try: - format_command(input_file, config_file, formatter, output) + format_command( + input_file, + repo=repo, + split=split, + config_file=config_file, + formatter=formatter, + output=output, + ) except FileNotFoundError as e: ctx.fail(f"Input file not found: {e}") except Exception as e: diff --git a/docs/api/config.md b/docs/api/config.md index 7c51faf0..0fcaad8d 100644 --- a/docs/api/config.md +++ b/docs/api/config.md @@ -64,7 +64,7 @@ tree_args = config.get_tree_args( degree=5, temperature=0.9, provider="anthropic", - model="claude-3-opus" + model="claude-sonnet-4-5" ) ``` @@ -228,8 +228,8 @@ from deepfabric.config import construct_model_string model_string = construct_model_string("openai", "gpt-4") # Returns: "openai/gpt-4" -model_string = construct_model_string("anthropic", "claude-3-opus") -# Returns: "anthropic/claude-3-opus" +model_string = construct_model_string("anthropic", "claude-sonnet-4-5") +# Returns: "anthropic/claude-sonnet-4-5" ``` #### get_provider_config(provider: str) @@ -311,7 +311,7 @@ generator = create_generator_with_overrides( "base_config.yaml", temperature=0.8, provider="anthropic", - model="claude-3-opus" + model="claude-sonnet-4-5" ) # Multi-environment configuration diff --git a/docs/api/generator.md b/docs/api/generator.md index 7e8048da..41d919db 100644 --- a/docs/api/generator.md +++ b/docs/api/generator.md @@ -30,7 +30,7 @@ generator = DataSetGenerator( **provider** (str, required): LLM provider name, e.g., `openai`, `anthropic`, `gemini`, `ollama`. -**model_name** (str, required): Model name specific to the provider, e.g., `gpt-4`, `claude-3-opus`. +**model_name** (str, required): Model name specific to the provider, e.g., `gpt-4`, `claude-sonnet-4-5`. **temperature** (float, optional): Controls creativity and diversity in content generation. Range 0.0-2.0, typically 0.7-0.9. Default: 0.7. @@ -112,7 +112,7 @@ Generate a single batch of examples for fine-grained control: batch = generator.create_batch( topics=selected_topics, batch_size=3, - model_name="openai/gpt-3.5-turbo" + model_name="openai/gpt-4-turbo" ) ``` @@ -314,7 +314,7 @@ complex_generator = DataSetGenerator( instructions="Create advanced technical content", generation_system_prompt="You are an expert technical writer", provider="anthropic", - model_name="claude-3-opus", + model_name="claude-sonnet-4-5", temperature=0.7 ) @@ -323,7 +323,7 @@ simple_generator = DataSetGenerator( instructions="Create basic explanations", generation_system_prompt="You are a teacher for beginners", provider="openai", - model_name="gpt-3.5-turbo", + model_name="gpt-4-turbo", temperature=0.8 ) diff --git a/docs/api/graph.md b/docs/api/graph.md index cd3fae92..1e5340c3 100644 --- a/docs/api/graph.md +++ b/docs/api/graph.md @@ -21,7 +21,7 @@ from deepfabric import Graph graph = Graph( topic_prompt="Artificial intelligence research areas", - model_name="anthropic/claude-3-opus", + model_name="anthropic/claude-sonnet-4-5", topic_system_prompt="You are mapping interconnected research concepts.", degree=4, # Connections per node depth=3, # Maximum distance from root @@ -56,7 +56,7 @@ from deepfabric import Graph # Create and build a graph graph = Graph( topic_prompt="Artificial intelligence research areas", - model_name="anthropic/claude-3-opus", + model_name="anthropic/claude-sonnet-4-5", degree=4, depth=3, temperature=0.8 @@ -157,7 +157,7 @@ Reconstructs graph from previously saved JSON files: ```python graph = Graph( topic_prompt="Default prompt", - model_name="anthropic/claude-3-opus" + model_name="anthropic/claude-sonnet-4-5" ) graph.load("existing_graph.json") ``` @@ -170,7 +170,7 @@ Class method for loading graphs with specific configuration: graph = Graph.from_json( "saved_graph.json", topic_prompt="Research areas", - model_name="anthropic/claude-3-opus" + model_name="anthropic/claude-sonnet-4-5" ) ``` @@ -211,7 +211,7 @@ Control graph construction through individual phases: ```python graph = Graph( topic_prompt="Complex domain", - model_name="anthropic/claude-3-opus", + model_name="anthropic/claude-sonnet-4-5", degree=4, depth=3 ) @@ -274,7 +274,7 @@ Graphs integrate seamlessly with dataset generation: # Generate dataset from graph generator = DataSetGenerator( instructions="Create interconnected explanations", - model_name="anthropic/claude-3-opus", + model_name="anthropic/claude-sonnet-4-5", temperature=0.7 ) dataset = generator.create_data( diff --git a/docs/cli/format.md b/docs/cli/format.md index 3c34c638..d16be124 100644 --- a/docs/cli/format.md +++ b/docs/cli/format.md @@ -5,18 +5,27 @@ The `format` command allows you to apply formatters to existing datasets without ## Usage ```bash +# From a local JSONL file deepfabric format INPUT_FILE [OPTIONS] + +# Or directly from a Hugging Face dataset repo +deepfabric format --repo ORG/DATASET [OPTIONS] ``` ## Arguments - `INPUT_FILE` - Path to the input JSONL dataset file to format + - Alternatively, use `--repo ORG/DATASET` to load from the Hugging Face Hub ## Options - `-c, --config-file PATH` - YAML config file containing formatter settings -- `-f, --formatter [im_format|unsloth|alpaca|chatml|grpo]` - Quick formatter selection with default settings -- `-o, --output TEXT` - Output file path (default: `input_file_formatter.jsonl`) +- `-f, --formatter [im_format|unsloth|alpaca|chatml|grpo|harmony|trl|xlam_v2]` - Quick formatter selection with default settings +- `-o, --output TEXT` - Output file path + - Local file input: defaults to `input_file_formatter.jsonl` + - `--repo` input: defaults to `formatted.jsonl` +- `--repo TEXT` - Hugging Face dataset repo id (e.g., `org/dataset-name`) +- `--split TEXT` - Dataset split to load when using `--repo` (default: `train`) - `--help` - Show help message ## Examples @@ -153,9 +162,22 @@ The command expects a JSONL file where each line is a JSON object. Supported for } ``` -### Working with HuggingFace Datasets +### Working with Hugging Face Datasets + +You can now format datasets directly from the Hugging Face Hub using `--repo`, or continue using local JSONL files. Many HF datasets come in compatible formats: + +**Format directly from a Hub repo:** +```bash +# Pull from Hub, format to Harmony, write to formatted.jsonl +deepfabric format --repo "org/dataset-name" --format harmony -The format command works seamlessly with datasets downloaded from HuggingFace Hub. Many HF datasets come in compatible formats: +# Load with datasets library +python - <<'PY' +from datasets import load_dataset +ds = load_dataset("json", data_files="formatted.jsonl") +print(ds) +PY +``` **For datasets with `messages` field (e.g., chat datasets):** ```bash @@ -185,7 +207,7 @@ deepfabric format alpaca_dataset.jsonl -f im_format - ShareGPT format (`conversations`) - Q&A format (`question`, `answer` or `response`) -**Example conversion workflow:** +**Example conversion workflow (local):** ```bash # 1. Download from HuggingFace huggingface-cli download tatsu-lab/alpaca --repo-type dataset @@ -218,3 +240,14 @@ deepfabric format dataset_raw.jsonl -f grpo -o dataset_grpo.jsonl ``` This allows you to prepare the same dataset for different training frameworks without regenerating the data. +### TRL SFT Tools + +Use `-f trl` to convert agent/tool datasets to the Hugging Face TRL SFT tool-calling format. This maps to the built-in `trl_sft_tools` formatter. + +```bash +# Format from a local file +deepfabric format dataset.jsonl -f trl -o trl_sft_tools.jsonl + +# Or format directly from a Hub repo +deepfabric format --repo org/dataset -f trl -o trl_sft_tools.jsonl +``` diff --git a/docs/cli/generate.md b/docs/cli/generate.md index 1b522043..87913068 100644 --- a/docs/cli/generate.md +++ b/docs/cli/generate.md @@ -21,7 +21,7 @@ Override specific configuration parameters without modifying the configuration f ```bash deepfabric generate config.yaml \ --provider anthropic \ - --model claude-3-opus \ + --model claude-sonnet-4-5 \ --temperature 0.8 \ --num-steps 100 \ --batch-size 5 @@ -118,7 +118,7 @@ deepfabric generate research-dataset.yaml \ --save-tree research_topics.jsonl \ --dataset-save-as research_examples.jsonl \ --provider anthropic \ - --model claude-3-opus \ + --model claude-sonnet-4-5 \ --degree 4 \ --depth 3 \ --num-steps 200 \ diff --git a/docs/cli/validate.md b/docs/cli/validate.md index 3146013c..9a6fa237 100644 --- a/docs/cli/validate.md +++ b/docs/cli/validate.md @@ -91,7 +91,7 @@ deepfabric validate config.yaml Provider Validation: OpenAI API key detected (OPENAI_API_KEY) Model gpt-4 is available - ⚠️ Model gpt-4 has higher costs than gpt-3.5-turbo + ⚠️ Model gpt-4 has higher costs than gpt-4-turbo ``` ## Development Workflow Integration diff --git a/docs/examples/advanced-workflows.md b/docs/examples/advanced-workflows.md index 897ce635..a1f05f11 100644 --- a/docs/examples/advanced-workflows.md +++ b/docs/examples/advanced-workflows.md @@ -1,6 +1,8 @@ # Advanced Workflows -Advanced DeepFabric workflows demonstrate sophisticated patterns for complex dataset generation scenarios, including multi-stage processing, quality control pipelines, and large-scale production deployments. These examples showcase techniques that go beyond basic configuration to leverage the full capabilities of the system. +Advanced DeepFabric workflows demonstrate patterns for complex dataset generation scenarios, including multi-stage processing, +quality control pipelines, and large-scale production deployments. These examples showcase techniques that go beyond basic +configuration to leverage the full capabilities of the system. ## Multi-Provider Pipeline @@ -18,7 +20,7 @@ topic_tree: depth: 3 temperature: 0.7 provider: "openai" - model: "gpt-3.5-turbo" + model: "gpt-4-turbo" save_as: "engineering_topics.jsonl" # High-quality content generation @@ -26,7 +28,7 @@ data_engine: instructions: "Create detailed, practical explanations with real-world examples and code samples suitable for senior developers." generation_system_prompt: "You are creating comprehensive educational content for software engineering professionals." provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" temperature: 0.8 max_retries: 5 @@ -36,12 +38,12 @@ dataset: num_steps: 500 batch_size: 8 provider: "openai" - model: "gpt-4" + model: "gpt-5" sys_msg: true save_as: "engineering_dataset.jsonl" ``` -This approach optimizes cost and quality by using GPT-3.5-turbo for broad topic exploration, Claude-3-Opus for detailed content generation, and GPT-4 for final dataset creation. +This approach optimizes cost and quality by using GPT-3.5-turbo for broad topic exploration, claude-sonnet-4-5 for detailed content generation, and GPT-5 for final dataset creation. ## Topic Graph with Visualization @@ -58,7 +60,7 @@ topic_graph: depth: 4 temperature: 0.8 provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" save_as: "ml_research_graph.json" data_engine: @@ -125,7 +127,7 @@ data_engine: instructions: "Create technically accurate documentation with working code examples, best practices, and common pitfalls. Include version-specific information and real-world usage patterns." generation_system_prompt: "You are creating high-quality technical documentation with emphasis on accuracy, clarity, and practical utility." provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" temperature: 0.7 max_retries: 5 request_timeout: 60 # Extended timeout for quality @@ -221,136 +223,79 @@ huggingface: - "training-data" ``` -Production deployment script with monitoring and resource management: - -```python -# production_deployment.py -import asyncio -import time -import logging -from deepfabric import DeepFabricConfig, DataSetGenerator, Tree - -# Configure logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def deploy_large_scale_generation(config_path, checkpoint_interval=500): - """Deploy large-scale generation with checkpointing and monitoring.""" - - config = DeepFabricConfig.from_yaml(config_path) - - # Load or create topic tree - tree = Tree(**config.get_tree_args()) - - async def _build_tree() -> None: - async for _ in tree.build_async(): - pass - - asyncio.run(_build_tree()) - tree.save("production_topics.jsonl") - - # Create generator with production settings - generator = DataSetGenerator(**config.get_engine_args()) - - # Large-scale generation with checkpointing - dataset_config = config.get_dataset_config() - total_steps = dataset_config["creation"]["num_steps"] - batch_size = dataset_config["creation"]["batch_size"] - - completed = 0 - start_time = time.time() - - while completed < total_steps: - remaining = min(checkpoint_interval, total_steps - completed) - - logger.info(f"Generating batch {completed}-{completed + remaining}") - - batch_dataset = generator.create_data( - num_steps=remaining, - batch_size=batch_size, - topic_model=tree - ) - - # Save checkpoint - checkpoint_file = f"checkpoint_{completed}_{completed + remaining}.jsonl" - batch_dataset.save(checkpoint_file) - - completed += remaining - elapsed = time.time() - start_time - rate = completed / elapsed - - logger.info(f"Progress: {completed}/{total_steps} ({completed/total_steps:.1%})") - logger.info(f"Rate: {rate:.1f} examples/second") - logger.info(f"ETA: {(total_steps - completed) / rate / 60:.1f} minutes") - -if __name__ == "__main__": - deploy_large_scale_generation("production-scale-dataset.yaml") -``` +## Dataset Transformation Pipeline -## Domain-Specific Validation - -Custom validation pipeline for specialized domains: - -```python -# domain_validator.py -import json -import re -from typing import List, Dict, Tuple - -def validate_code_examples(dataset_path: str) -> Dict[str, int]: - """Validate code examples in generated dataset.""" - - validation_results = { - "total_examples": 0, - "valid_code_blocks": 0, - "syntax_errors": 0, - "missing_explanations": 0, - "quality_score": 0 - } - - with open(dataset_path, 'r') as f: - for line in f: - example = json.loads(line) - validation_results["total_examples"] += 1 - - # Extract code blocks - code_blocks = re.findall(r'```[\w]*\n(.*?)\n```', - example["messages"][-1]["content"], - re.DOTALL) - - if code_blocks: - validation_results["valid_code_blocks"] += 1 - - # Basic syntax validation (simplified) - for code in code_blocks: - try: - compile(code, '', 'exec') - except SyntaxError: - validation_results["syntax_errors"] += 1 - - # Check for explanations - content = example["messages"][-1]["content"] - if len(content) > 200 and any(word in content.lower() - for word in ["because", "this", "when", "why"]): - validation_results["quality_score"] += 1 - - # Calculate quality metrics - if validation_results["total_examples"] > 0: - quality_rate = validation_results["quality_score"] / validation_results["total_examples"] - validation_results["overall_quality"] = quality_rate - - return validation_results - -def main(): - results = validate_code_examples("webdev_documentation.jsonl") - print(f"Dataset Quality Report:") - print(f"Total Examples: {results['total_examples']}") - print(f"Code Block Coverage: {results['valid_code_blocks']}/{results['total_examples']}") - print(f"Syntax Error Rate: {results['syntax_errors']}/{results['valid_code_blocks']}") - print(f"Overall Quality Score: {results['overall_quality']:.2%}") - -if __name__ == "__main__": - main() -``` +Download existing datasets from Hugging Face Hub, transform them with multiple formatters, validate, and republish. This workflow is ideal for dataset curation and format standardization: -These advanced workflows demonstrate production-ready patterns for sophisticated dataset generation scenarios, including resource optimization, quality control, and comprehensive validation pipelines. +```bash +#!/bin/bash +# dataset-transformation-pipeline.sh + +set -e # Exit on error + +SOURCE_REPO="community/agent-reasoning-dataset" +TARGET_REPO="your-org/curated-reasoning-dataset" +TEMP_DIR="./pipeline_temp" + +echo "=== Dataset Transformation Pipeline ===" +echo "Source: $SOURCE_REPO" +echo "Target: $TARGET_REPO" + +# Create temporary working directory +mkdir -p $TEMP_DIR +cd $TEMP_DIR + +# Stage 1: Download and format from Hub +echo "" +echo "Stage 1: Downloading and formatting from Hub..." +deepfabric format --repo $SOURCE_REPO --formatter trl -o stage1_trl.jsonl + +# Stage 2: Apply secondary formatting for different training frameworks +echo "" +echo "Stage 2: Creating multiple format variants..." +deepfabric format stage1_trl.jsonl -f harmony -o stage2_harmony.jsonl +deepfabric format stage1_trl.jsonl -f unsloth -o stage2_unsloth.jsonl +deepfabric format stage1_trl.jsonl -f chatml -o stage2_chatml.jsonl + +# Stage 3: Validate all outputs +echo "" +echo "Stage 3: Validating transformed datasets..." +python ../validate_formats.py stage1_trl.jsonl stage2_harmony.jsonl stage2_unsloth.jsonl stage2_chatml.jsonl + +# Stage 4: Quality assessment +echo "" +echo "Stage 4: Running quality assessment..." +python ../assess_quality.py stage2_*.jsonl + +# Stage 5: Upload curated versions +echo "" +echo "Stage 5: Uploading curated datasets..." + +deepfabric upload stage1_trl.jsonl \ + --repo ${TARGET_REPO}-trl \ + --tags curated trl agent-tools training + +deepfabric upload stage2_harmony.jsonl \ + --repo ${TARGET_REPO}-harmony \ + --tags curated harmony gpt-oss training + +deepfabric upload stage2_unsloth.jsonl \ + --repo ${TARGET_REPO}-unsloth \ + --tags curated unsloth training + +deepfabric upload stage2_chatml.jsonl \ + --repo ${TARGET_REPO}-chatml \ + --tags curated chatml training + +echo "" +echo "=== Pipeline Complete ===" +echo "Curated datasets available at:" +echo " - https://huggingface.co/datasets/${TARGET_REPO}-trl" +echo " - https://huggingface.co/datasets/${TARGET_REPO}-harmony" +echo " - https://huggingface.co/datasets/${TARGET_REPO}-unsloth" +echo " - https://huggingface.co/datasets/${TARGET_REPO}-chatml" + +# Cleanup +cd .. +rm -rf $TEMP_DIR +``` diff --git a/docs/examples/basic-usage.md b/docs/examples/basic-usage.md index 75249c1a..02158a9b 100644 --- a/docs/examples/basic-usage.md +++ b/docs/examples/basic-usage.md @@ -19,14 +19,14 @@ topic_tree: depth: 2 temperature: 0.7 provider: "openai" - model: "gpt-3.5-turbo" + model: "gpt-4-turbo" save_as: "programming_topics.jsonl" data_engine: instructions: "Create a clear explanation with a simple code example that a beginner could understand and follow." generation_system_prompt: "You are a programming instructor creating educational content for beginners." provider: "openai" - model: "gpt-3.5-turbo" + model: "gpt-4-turbo" temperature: 0.8 max_retries: 3 @@ -35,7 +35,7 @@ dataset: num_steps: 25 batch_size: 3 provider: "openai" - model: "gpt-3.5-turbo" + model: "gpt-4-turbo" sys_msg: true save_as: "programming_examples.jsonl" ``` @@ -180,14 +180,14 @@ topic_tree: depth: 1 temperature: 0.5 provider: "openai" - model: "gpt-3.5-turbo" + model: "gpt-4-turbo" save_as: "test_topics.jsonl" data_engine: instructions: "Create a simple example for testing purposes." generation_system_prompt: "You are creating test examples for development." provider: "openai" - model: "gpt-3.5-turbo" + model: "gpt-4-turbo" temperature: 0.5 max_retries: 2 @@ -196,7 +196,7 @@ dataset: num_steps: 5 batch_size: 1 provider: "openai" - model: "gpt-3.5-turbo" + model: "gpt-4-turbo" sys_msg: false save_as: "test_dataset.jsonl" ``` diff --git a/docs/examples/huggingface-integration.md b/docs/examples/huggingface-integration.md index ac159de4..f4e863d0 100644 --- a/docs/examples/huggingface-integration.md +++ b/docs/examples/huggingface-integration.md @@ -17,7 +17,7 @@ topic_tree: depth: 2 temperature: 0.7 provider: "openai" - model: "gpt-3.5-turbo" + model: "gpt-4-turbo" save_as: "python_basics_topics.jsonl" data_engine: @@ -80,7 +80,7 @@ data_engine: instructions: "Create detailed explanations with mathematical foundations, practical examples, and real-world applications suitable for undergraduate and graduate students." generation_system_prompt: "You are creating a comprehensive machine learning curriculum with theoretical foundations and practical applications." provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" temperature: 0.8 max_retries: 3 @@ -158,7 +158,7 @@ data_engine: instructions: "Create realistic, professional customer service interactions demonstrating empathy, problem-solving skills, and industry-specific knowledge. Include complex scenarios, difficult customers, and exemplary resolution techniques." generation_system_prompt: "You are creating professional customer support training data that demonstrates excellence in customer service across various industries and scenarios." provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" temperature: 0.8 max_retries: 5 request_timeout: 60 @@ -319,7 +319,7 @@ topic_graph: depth: 3 temperature: 0.8 provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" save_as: "nlp_research_graph.json" data_engine: @@ -444,4 +444,165 @@ huggingface: - "open-source" ``` -The Hugging Face integration provides a complete pathway from synthetic data generation to community sharing, enabling researchers and practitioners to contribute high-quality synthetic datasets to the broader machine learning ecosystem. \ No newline at end of file +The Hugging Face integration provides a complete pathway from synthetic data generation to community sharing, enabling researchers and practitioners to contribute high-quality synthetic datasets to the broader machine learning ecosystem. + +## Downloading and Reformatting Hub Datasets + +DeepFabric can download datasets directly from Hugging Face Hub and transform them into different training formats without requiring local files. This bidirectional workflow enables dataset curation, format conversion, and preparation for specific training frameworks. + +### Basic Download and Format + +Download a dataset from the Hub and apply a formatter: + +```bash +# Download and format to TRL SFT Tools format +deepfabric format --repo lukehinds/smol-test-sample --formatter trl + +# Download and format to ChatML +deepfabric format --repo username/conversation-dataset --formatter im_format + +# Download and format to GRPO for reasoning training +deepfabric format --repo org/math-problems --formatter grpo -o grpo_math.jsonl +``` + +### Multi-Format Conversion Workflow + +Convert a single Hub dataset to multiple training formats: + +```bash +#!/bin/bash +# multi-format-conversion.sh + +REPO="community/agent-tool-dataset" +BASE_NAME="agent_training" + +echo "Downloading and converting dataset: $REPO" + +# Format for TRL SFTTrainer +deepfabric format --repo $REPO --formatter trl -o "${BASE_NAME}_trl.jsonl" + +# Format for Unsloth training +deepfabric format --repo $REPO --formatter unsloth -o "${BASE_NAME}_unsloth.jsonl" + +# Format for Harmony (gpt-oss) +deepfabric format --repo $REPO --formatter harmony -o "${BASE_NAME}_harmony.jsonl" + +# Format for single tool call training +deepfabric format --repo $REPO --formatter chatml -o "${BASE_NAME}_chatml.jsonl" + +echo "Conversion complete. Created 4 formatted versions." +``` + +### Dataset Curation Pipeline + +Download, format, validate, and re-upload a curated version: + +```yaml +# curation-config.yaml +# Configuration for post-format processing if needed +dataset: + formatters: + - name: "trl_curated" + template: "builtin://trl_sft_tools" + output: "curated_trl.jsonl" + config: + include_system_prompt: true + system_prompt_override: | + You are a function calling AI model. You are provided with function + signatures within XML tags. You may call one or more + functions to assist with the user query. + validate_tool_schemas: true + remove_available_tools_field: true +``` + +Complete curation workflow: + +```bash +#!/bin/bash +# dataset-curation.sh + +SOURCE_REPO="community/raw-agent-dataset" +TARGET_REPO="your-org/curated-agent-dataset" + +echo "=== Dataset Curation Pipeline ===" + +# Step 1: Download and format from Hub +echo "Step 1: Downloading and formatting dataset..." +deepfabric format --repo $SOURCE_REPO --formatter trl -o stage1_formatted.jsonl + +# Step 2: Apply custom formatting with config (if needed for advanced options) +echo "Step 2: Applying advanced formatting options..." +deepfabric format stage1_formatted.jsonl -c curation-config.yaml + +# Step 3: Validate the curated dataset +echo "Step 3: Validating curated dataset..." +python validate_curated.py curated_trl.jsonl + +# Step 4: Upload curated version to your organization +echo "Step 4: Uploading curated dataset..." +deepfabric upload curated_trl.jsonl \ + --repo $TARGET_REPO \ + --tags curated agent-tools trl-format + +echo "=== Curation complete ===" +echo "Source: https://huggingface.co/datasets/$SOURCE_REPO" +echo "Curated: https://huggingface.co/datasets/$TARGET_REPO" +``` + +### Split-Specific Processing + +Process different dataset splits with different formatters: + +```bash +# Process training split for TRL +deepfabric format --repo org/dataset --split train --formatter trl -o train_trl.jsonl + +# Process validation split for evaluation (different format) +deepfabric format --repo org/dataset --split validation --formatter chatml -o val_chatml.jsonl + +# Process test split +deepfabric format --repo org/dataset --split test --formatter chatml -o test_chatml.jsonl +``` + +### Real-World Example: Reformatting for Fine-Tuning + +Convert a public agent dataset for TRL SFTTrainer fine-tuning: + +```bash +#!/bin/bash +# prepare-for-finetuning.sh + +echo "Preparing dataset for fine-tuning with TRL SFTTrainer" + +# Download and format from Hub +deepfabric format \ + --repo lukehinds/smol-test-sample \ + --formatter trl \ + --split train \ + -o training_data.jsonl + +# Verify the format +python - <<'PY' +from datasets import load_dataset +import json + +# Load and inspect +with open("training_data.jsonl", "r") as f: + first_example = json.loads(f.readline()) + +print("Example structure:") +print(json.dumps(first_example, indent=2)) + +# Verify required fields +assert "messages" in first_example, "Missing 'messages' field" +assert "tools" in first_example, "Missing 'tools' field" + +print("\n✓ Format validated for TRL SFTTrainer") +print(f"✓ Sample has {len(first_example['tools'])} tools available") +PY + +echo "Dataset ready for training!" +echo "Next: Use with TRL SFTTrainer" +``` + +This bidirectional integration enables a complete ecosystem workflow: generate datasets with DeepFabric → upload to Hub → share with community → download and reformat for specific use cases → iterate and improve. \ No newline at end of file diff --git a/docs/formatters/built-in-reference.md b/docs/formatters/built-in-reference.md index ab04198b..a145fd76 100644 --- a/docs/formatters/built-in-reference.md +++ b/docs/formatters/built-in-reference.md @@ -698,6 +698,26 @@ config: validate_tool_schemas: false ``` +### Quick Start with HuggingFace Hub + +Download and format datasets directly from the HuggingFace Hub: + +```bash +# Download from Hub and format to TRL SFT Tools format +deepfabric format --repo lukehinds/smol-test-sample --formatter trl + +# Specify a different split (default is 'train') +deepfabric format --repo username/dataset-name --formatter trl --split validation + +# Custom output path +deepfabric format --repo org/agent-dataset --formatter trl -o trl_formatted.jsonl +``` + +This workflow is ideal for: +- Converting existing agent/tool datasets to TRL format +- Reformatting community datasets for your training pipeline +- Experimenting with different formatters on public datasets + ### Usage with TRL SFTTrainer After formatting your dataset, use it directly with TRL's SFTTrainer: diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 1f0e9c5e..c4778f9e 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -135,11 +135,11 @@ DeepFabric supports any multiple providers through consistent configuration patt ```yaml topic_tree: provider: "openai" - model: "gpt-3.5-turbo" # Fast, cost-effective for topic generation + model: "gpt-4-turbo" # Fast, cost-effective for topic generation data_engine: provider: "anthropic" - model: "claude-3-opus" # High-quality for content generation + model: "claude-sonnet-4-5" # High-quality for content generation ``` Provider authentication occurs through environment variables following the pattern `{PROVIDER}_API_KEY`. For example, OpenAI requires `OPENAI_API_KEY` while Anthropic requires `ANTHROPIC_API_KEY`. diff --git a/docs/guide/dataset-generation.md b/docs/guide/dataset-generation.md index 6755a135..c045a509 100644 --- a/docs/guide/dataset-generation.md +++ b/docs/guide/dataset-generation.md @@ -37,7 +37,7 @@ dataset: num_steps: 100 # Total examples to generate batch_size: 5 # Examples per API call provider: "anthropic" # Can differ from data_engine - model: "claude-3-opus" + model: "claude-sonnet-4-5" sys_msg: true # Include system messages save_as: "training_dataset.jsonl" ``` diff --git a/docs/guide/instruction-formats/chain-of-thought/configuration/yaml-config.md b/docs/guide/instruction-formats/chain-of-thought/configuration/yaml-config.md index 59143cb4..ed71156c 100644 --- a/docs/guide/instruction-formats/chain-of-thought/configuration/yaml-config.md +++ b/docs/guide/instruction-formats/chain-of-thought/configuration/yaml-config.md @@ -234,8 +234,8 @@ topic_prompt: "Advanced physics problems requiring multi-step problem solving" - **Required**: Yes - **Description**: LLM provider and model for topic generation - **Options**: - - `openai`: `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo`, `gpt-3.5-turbo` - - `anthropic`: `claude-3-opus`, `claude-3-sonnet`, `claude-3-haiku` + - `openai`: `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo` + - `anthropic`: `claude-sonnet-4-5`, `claude-3-sonnet`, `claude-3-haiku` - `gemini`: `gemini-pro`, `gemini-2.5-flash-lite` - `ollama`: `mistral:latest`, `llama3:latest`, etc. @@ -383,7 +383,7 @@ huggingface: #### For Topic Generation - **Simple domains**: `gpt-4o-mini`, `claude-3-haiku` - **Complex domains**: `gpt-4o`, `claude-3-sonnet` -- **Interdisciplinary**: `gpt-4o`, `claude-3-opus` +- **Interdisciplinary**: `gpt-4o`, `claude-sonnet-4-5` #### For Data Generation - **Free-text CoT**: `gpt-4o-mini` sufficient diff --git a/docs/guide/instruction-formats/chain-of-thought/reference/troubleshooting.md b/docs/guide/instruction-formats/chain-of-thought/reference/troubleshooting.md index e9ac7b56..876375a2 100644 --- a/docs/guide/instruction-formats/chain-of-thought/reference/troubleshooting.md +++ b/docs/guide/instruction-formats/chain-of-thought/reference/troubleshooting.md @@ -85,7 +85,7 @@ OpenAI does not support your schema: Invalid schema for response_format 'default data_engine: provider: "openai" model: "gpt-4o-mini" # ✅ Supports structured output - # model: "gpt-3.5-turbo" # ⚠️ Limited schema support + # model: "gpt-4-turbo" # ⚠️ Limited schema support conversation_type: "cot_hybrid" # Complex schema ``` @@ -147,7 +147,7 @@ openai.api_key = "your-key-here" try: response = openai.ChatCompletion.create( - model="gpt-3.5-turbo", + model="gpt-4-turbo", messages=[{"role": "user", "content": "Hello"}], max_tokens=10 ) @@ -427,7 +427,7 @@ generator = DataSetGenerator( # Use faster models for development/testing data_engine: model: "gpt-4o-mini" # Faster than gpt-4o - # model: "gpt-3.5-turbo" # Even faster for simple tasks + # model: "gpt-4-turbo" # Even faster for simple tasks ``` #### 2. Reduce complexity diff --git a/docs/guide/provider-integration.md b/docs/guide/provider-integration.md index c774d4f4..f934bfba 100644 --- a/docs/guide/provider-integration.md +++ b/docs/guide/provider-integration.md @@ -51,7 +51,7 @@ model: "gpt-4" # Anthropic Claude provider: "anthropic" -model: "claude-3-opus" +model: "claude-sonnet-4-5" # Local Ollama provider: "ollama" @@ -72,12 +72,12 @@ DeepFabric supports using different providers for different components, enabling # Fast, cost-effective topic generation topic_tree: provider: "openai" - model: "gpt-3.5-turbo" + model: "gpt-4-turbo" # High-quality dataset generation data_engine: provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" # Different model for final dataset creation dataset: diff --git a/docs/guide/topic-graphs.md b/docs/guide/topic-graphs.md index 8dcf461b..4499a6ce 100644 --- a/docs/guide/topic-graphs.md +++ b/docs/guide/topic-graphs.md @@ -23,7 +23,7 @@ topic_graph: depth: 3 # Maximum distance from root temperature: 0.8 # Higher creativity for connections provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" save_as: "ai_research_graph.json" ``` diff --git a/examples/advanced.yaml b/examples/advanced.yaml index 1c359629..20a010be 100644 --- a/examples/advanced.yaml +++ b/examples/advanced.yaml @@ -53,7 +53,7 @@ dataset: # Can use different provider for final assembly provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" sys_msg: true save_as: "software_architecture_dataset.jsonl" diff --git a/examples/integrations.py b/examples/integrations.py index 0577849e..dd788609 100644 --- a/examples/integrations.py +++ b/examples/integrations.py @@ -110,7 +110,7 @@ def example_multiple_providers(): }, "anthropic_config": { "provider": "anthropic", - "model": "claude-3-opus", + "model": "claude-sonnet-4-5", "use_case": "Complex reasoning tasks" }, "ollama_config": { @@ -143,7 +143,7 @@ def example_multiple_providers(): "data_engine": { "instructions": "Create detailed explanations with code examples", "provider": "anthropic", # High quality for content - "model_name": "claude-3-opus", + "model_name": "claude-sonnet-4-5", "temperature": 0.3, }, "dataset": { diff --git a/examples/specialized.yaml b/examples/specialized.yaml index 580523e5..62ebf90c 100644 --- a/examples/specialized.yaml +++ b/examples/specialized.yaml @@ -11,7 +11,7 @@ topic_tree: topic_prompt: "Clinical diagnosis and patient care protocols" provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" temperature: 0.5 # Conservative for medical accuracy degree: 4 @@ -75,7 +75,7 @@ dataset: # High-quality provider for final dataset assembly provider: "anthropic" - model: "claude-3-opus" + model: "claude-sonnet-4-5" sys_msg: true # Include system prompts for training context save_as: "clinical_communication_dataset.jsonl" diff --git a/uv.lock b/uv.lock index 64b07fdb..c6ffedf6 100644 --- a/uv.lock +++ b/uv.lock @@ -397,7 +397,7 @@ wheels = [ [[package]] name = "deepfabric" -version = "2.11.0" +version = "2.11.1" source = { editable = "." } dependencies = [ { name = "anthropic" }, @@ -444,7 +444,7 @@ requires-dist = [ { name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9.0.0" }, { name = "mkdocstrings", extras = ["python"], marker = "extra == 'docs'", specifier = ">=0.30.0" }, { name = "openai", specifier = ">=1.107.2" }, - { name = "outlines", specifier = "==1.2.5" }, + { name = "outlines", specifier = "==1.2.7" }, { name = "posthog", specifier = ">=3.0.0" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, @@ -1283,7 +1283,7 @@ wheels = [ [[package]] name = "outlines" -version = "1.2.5" +version = "1.2.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cloudpickle" }, @@ -1297,9 +1297,9 @@ dependencies = [ { name = "pydantic" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b5/2a/dc8a08972e87974e0adae13910ee9b2f5298b1ad6756f335c844bd143979/outlines-1.2.5.tar.gz", hash = "sha256:24a2ef40dffd6529bf5f913f30a8997802a9c49903339421b1712f03c05aa788", size = 2825333, upload-time = "2025-09-15T20:08:46.519Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/72/70cbde9680d810fa140486512adccedfc7369fe993a963150835699900a7/outlines-1.2.7.tar.gz", hash = "sha256:1b588e7a6c789deae29dc212037089f4fc9f954b1d7d23f223d5451db45bb5b7", size = 2833655, upload-time = "2025-10-14T16:27:27.577Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/6d/31ad4cb24590a36fab776e413f67a56f4c97109bac1db4d3e9aed54fe834/outlines-1.2.5-py3-none-any.whl", hash = "sha256:60bd9dfaa4b7c5895a67695f0966bc06c6f0b8d98cda8fef2fb76ad5bc1bdef8", size = 93854, upload-time = "2025-09-15T20:08:44.775Z" }, + { url = "https://files.pythonhosted.org/packages/1d/c4/e9e0f12c04b2c12132b806ba8a1aefb382f162804182e03548af2ac78860/outlines-1.2.7-py3-none-any.whl", hash = "sha256:5d1cb695cb14213e64e632b742090880094877440bc292c5fd4ebb4a912d8c02", size = 98114, upload-time = "2025-10-14T16:27:26.396Z" }, ] [[package]]