Skip to content

Commit a66a5e6

Browse files
Merge branch 'main' into agent-traces
2 parents 9347590 + b806e10 commit a66a5e6

61 files changed

Lines changed: 4005 additions & 1001 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

Makefile

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,11 @@ check_dirs := src tests
88

99
# dev dependencies
1010
install:
11-
uv venv openr1 --python 3.11 && . openr1/bin/activate && uv pip install --upgrade pip
12-
uv pip install vllm==0.7.2
13-
uv pip install setuptools
14-
uv pip install flash-attn --no-build-isolation
11+
uv venv openr1 --python 3.11
12+
. openr1/bin/activate && uv pip install --upgrade pip && \
13+
uv pip install vllm==0.8.5.post1 && \
14+
uv pip install setuptools && \
15+
uv pip install flash-attn --no-build-isolation && \
1516
GIT_LFS_SKIP_SMUDGE=1 uv pip install -e ".[dev]"
1617

1718
style:
@@ -46,8 +47,7 @@ evaluate:
4647
--use-chat-template \
4748
--output-dir data/evals/$(MODEL); \
4849
else \
49-
lighteval vllm $$MODEL_ARGS "custom|$(TASK)|0|0" \
50-
--custom-tasks src/open_r1/evaluate.py \
50+
lighteval vllm $$MODEL_ARGS "lighteval|$(TASK)|0|0" \
5151
--use-chat-template \
5252
--output-dir data/evals/$(MODEL); \
5353
fi

README.md

Lines changed: 308 additions & 131 deletions
Large diffs are not rendered by default.

recipes/DeepSeek-R1-Distill-Qwen-1.5B/grpo/config_demo.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,5 @@ save_strategy: "epoch"
5454
save_total_limit: 1
5555
seed: 42
5656
temperature: 0.7
57+
use_liger_kernel: true
5758
warmup_ratio: 0.1

recipes/Mistral-Small-24B-Instruct-2501/sft/config_openr1_math.yaml

Lines changed: 0 additions & 42 deletions
This file was deleted.

recipes/OlympicCoder-32B/sft/config_v00.00.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,5 +45,5 @@ save_only_model: true # needed to bypass FSDP errors with saving paged optimizer
4545
save_strategy: epoch
4646
save_total_limit: 1
4747
seed: 42
48-
use_liger: false # fails on multi-node
48+
use_liger_kernel: false # fails on multi-node
4949
warmup_ratio: 0.03

recipes/OlympicCoder-7B/sft/config_v00.00.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,5 +42,5 @@ report_to:
4242
save_strategy: epoch
4343
save_total_limit: 1
4444
seed: 42
45-
use_liger: true
45+
use_liger_kernel: true
4646
warmup_ratio: 0.03
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Config for 1 node of 8 x H100s (80GB)
2+
# Model arguments
3+
model_name_or_path: open-r1/Qwen2.5-Math-7B-RoPE-300k
4+
model_revision: main
5+
torch_dtype: bfloat16
6+
attn_implementation: flash_attention_2
7+
8+
# Data training arguments
9+
chat_template: "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are Open-R1, a language model trained by Hugging Face to help users. Your role as an assistant involves thoroughly exploring questions through a systematic thinking process before providing the final precise and accurate solutions. This requires engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, backtracing, and iteration to develop well-considered thinking process. Please structure your response into two main sections: Thought and Solution using the specified format: <think> Thought section </think> Solution section. In the Thought section, detail your reasoning process in steps. Each step should include detailed considerations such as analysing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, and revisiting previous steps. In the Solution section, based on various attempts, explorations, and reflections from the Thought section, systematically present the final solution that you deem correct. The Solution section should be logical, accurate, and concise and detail necessary steps needed to reach the conclusion. Now, try to solve the following question through the above guidelines.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are Open-R1, a language model trained by Hugging Face to help users. Your role as an assistant involves thoroughly exploring questions through a systematic thinking process before providing the final precise and accurate solutions. This requires engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, backtracing, and iteration to develop well-considered thinking process. Please structure your response into two main sections: Thought and Solution using the specified format: <think> Thought section </think> Solution section. In the Thought section, detail your reasoning process in steps. Each step should include detailed considerations such as analysing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, and revisiting previous steps. In the Solution section, based on various attempts, explorations, and reflections from the Thought section, systematically present the final solution that you deem correct. The Solution section should be logical, accurate, and concise and detail necessary steps needed to reach the conclusion. Now, try to solve the following question through the above guidelines.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n"
10+
dataset_name: open-r1/Mixture-of-Thoughts
11+
dataset_config: all
12+
dataset_num_proc: 12
13+
eos_token: <|im_end|>
14+
15+
# SFT trainer config
16+
bf16: true
17+
do_eval: false
18+
eval_strategy: 'no'
19+
gradient_accumulation_steps: 8
20+
gradient_checkpointing: true
21+
gradient_checkpointing_kwargs:
22+
use_reentrant: false
23+
hub_model_id: OpenR1-Distill-7B
24+
hub_strategy: every_save
25+
learning_rate: 4.0e-05
26+
log_level: info
27+
logging_steps: 1
28+
logging_strategy: steps
29+
lr_scheduler_type: cosine_with_min_lr
30+
lr_scheduler_kwargs:
31+
min_lr_rate: 0.1
32+
packing: false
33+
max_grad_norm: 0.2
34+
max_length: 32768
35+
max_steps: -1
36+
num_train_epochs: 5
37+
output_dir: data/OpenR1-Distill-7B
38+
overwrite_output_dir: true
39+
per_device_eval_batch_size: 1
40+
per_device_train_batch_size: 2
41+
push_to_hub: true
42+
report_to:
43+
- wandb
44+
save_strategy: epoch
45+
save_total_limit: 1
46+
seed: 42
47+
use_liger_kernel: true
48+
warmup_ratio: 0.03

recipes/OpenR1-Qwen-7B/sft/config.yaml

Lines changed: 0 additions & 48 deletions
This file was deleted.

recipes/Qwen2.5-1.5B-Instruct/sft/config_demo.yaml

Lines changed: 0 additions & 44 deletions
This file was deleted.

recipes/Qwen2.5-7B-Instruct/grpo/config_demo.yaml

Lines changed: 0 additions & 55 deletions
This file was deleted.

0 commit comments

Comments
 (0)