flagos-ai · aoyulong · May 6, 2026 · Apr 17, 2026 · Apr 17, 2026 · Apr 17, 2026
diff --git a/docs/getting-started.md b/docs/getting-started.md
@@ -49,7 +49,7 @@ folder to get started.
     pip install -U scikit-build-core==0.11 pybind11 ninja cmake
     git clone https://github.com/flagos-ai/FlagGems
     cd FlagGems
-    pip install --no-build-isolation . 
+    pip install --no-build-isolation .
     ```
     See more details in [FlagGems](https://github.com/flagos-ai/FlagGems)
 

diff --git a/examples/aquila/conf/train/1_8b.yaml b/examples/aquila/conf/train/1_8b.yaml
@@ -60,6 +60,7 @@ model:
   clip_grad: 1.0
   train_samples: 8192  #8192 #29297664 #120B tokens 100step
   eval_iters: 0
+  eval_interval: 1000
   micro_batch_size: 8
   global_batch_size: 1024
   seed: 42

diff --git a/examples/aquila/conf/train/3b.yaml b/examples/aquila/conf/train/3b.yaml
@@ -68,6 +68,7 @@ model:
   clip_grad: 1.0
   train_samples: 244141056 #1TB tokens #29297664 #120B tokens
   eval_iters: 0
+  eval_interval: 1000
   micro_batch_size: 2
   global_batch_size: 2048
   seed: 42

diff --git a/examples/aquila/conf/train/7b.yaml b/examples/aquila/conf/train/7b.yaml
@@ -39,6 +39,7 @@ model:
   clip_grad: 1.0
   train_samples: 1002539063
   eval_iters: 0
+  eval_interval: 1000
   micro_batch_size: 2
   global_batch_size: 1728
   seed: 1234
@@ -57,7 +58,6 @@ data:
   data_path: ${data_path:??}
   split: 1
   tokenizer:
-    legacy_tokenizer: true
     tokenizer_type: AquilaTokenizerFS
     vocab_file: ./examples/aquila/tokenizer/vocab.json
     merge_file: ./examples/aquila/tokenizer/merges.txt

diff --git a/examples/deepseek_v3/conf/train/11b_a2b.yaml b/examples/deepseek_v3/conf/train/11b_a2b.yaml
@@ -90,6 +90,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 256
   eval_iters: 0
+  eval_interval: 1000
   train_samples: 244141056 #1T #29297664 #120B tokens
 
   optimizer:

diff --git a/examples/deepseek_v3/conf/train/16b_a3b.yaml b/examples/deepseek_v3/conf/train/16b_a3b.yaml
@@ -89,6 +89,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 128 #2048
   eval_iters: 0
+  eval_interval: 1000
   train_iters: 102400
 
   optimizer:

diff --git a/examples/deepseek_v3/conf/train/16b_a3b_finetune.yaml b/examples/deepseek_v3/conf/train/16b_a3b_finetune.yaml
@@ -88,6 +88,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 128 #2048
   eval_iters: 0
+  eval_interval: 1000
   train_iters: 102400
 
   # lora finetune

diff --git a/examples/deepseek_v3/conf/train/engram.yaml b/examples/deepseek_v3/conf/train/engram.yaml
@@ -120,6 +120,7 @@ model:
   micro_batch_size: 2
   global_batch_size: 2048
   eval_iters: 0
+  eval_interval: 1000
   train_iters: 20
 
   optimizer:
@@ -142,7 +143,6 @@ data:
   split: 1
   no_mmap_bin_files: true
   tokenizer:
-    legacy_tokenizer: true
     tokenizer_type: QwenTokenizerFS
     tokenizer_path: /workspace/qwentokenizer
     vocab_size: 151851

diff --git a/examples/llava_onevision/conf/train/1_5b.yaml b/examples/llava_onevision/conf/train/1_5b.yaml
@@ -50,6 +50,7 @@ model:
   profile_ranks: 7
   use_pytorch_profiler: True
   eval_iters: 0
+  eval_interval: 1000
   micro_batch_size: 2
   global_batch_size: 512
   allow_missing_vision_projection_checkpoint: True

diff --git a/examples/llava_onevision/conf/train/7b.yaml b/examples/llava_onevision/conf/train/7b.yaml
@@ -44,6 +44,7 @@ model:
   clip_grad: 1.0
   train_iters: 625
   eval_iters: 0
+  eval_interval: 1000
   micro_batch_size: 1
   global_batch_size: 320
   allow_missing_vision_projection_checkpoint: True

diff --git a/examples/qwen2_5/conf/train/10b.yaml b/examples/qwen2_5/conf/train/10b.yaml
@@ -62,6 +62,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 16
   eval_iters: 0
+  eval_interval: 1000
   train_samples: 25600000 #1T #29297664 #120B tokens
 
   optimizer:
@@ -80,7 +81,6 @@ data:
   split: 1
   no_mmap_bin_files: true
   tokenizer:
-    legacy_tokenizer: true
     tokenizer_type: QwenTokenizerFS
     tokenizer_path: xxx
     vocab_size: 151643

diff --git a/examples/qwen2_5_vl/conf/train/32b.yaml b/examples/qwen2_5_vl/conf/train/32b.yaml
@@ -60,6 +60,7 @@ model:
   clip_grad: 1.0
   train_iters: 15846
   eval_iters: 0
+  eval_interval: 1000
   micro_batch_size: 1
   global_batch_size: 256
   allow_missing_vision_projection_checkpoint: False

diff --git a/examples/qwen2_5_vl/conf/train/7b.yaml b/examples/qwen2_5_vl/conf/train/7b.yaml
@@ -60,6 +60,7 @@ model:
   clip_grad: 1.0
   train_iters: 62
   eval_iters: 0 # no valid
+  eval_interval: 1000
   micro_batch_size: 1
   global_batch_size: 16
   allow_missing_vision_projection_checkpoint: False

diff --git a/examples/qwen3/conf/train/0_6b.yaml b/examples/qwen3/conf/train/0_6b.yaml
@@ -64,6 +64,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 2048
   eval_iters: 0
+  eval_interval: 1000
   train_samples: 244141056 #1T #29297664 #120B tokens
 
   optimizer:
@@ -82,7 +83,6 @@ data:
   split: 1
   no_mmap_bin_files: true
   tokenizer:
-    legacy_tokenizer: true
     tokenizer_type: QwenTokenizerFS
     tokenizer_path: Qwen3-0.6B
     vocab_size: 151936

diff --git a/examples/qwen3/conf/train/10b.yaml b/examples/qwen3/conf/train/10b.yaml
@@ -64,6 +64,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 16
   eval_iters: 0
+  eval_interval: 1000
   train_samples: 25600000 #1T #29297664 #120B tokens
 
   optimizer:
@@ -82,7 +83,6 @@ data:
   split: 1
   no_mmap_bin_files: true
   tokenizer:
-    legacy_tokenizer: true
     tokenizer_type: QwenTokenizerFS
     tokenizer_path: xxx
     vocab_size: 151669

diff --git a/examples/qwen3/conf/train/10b_muon.yaml b/examples/qwen3/conf/train/10b_muon.yaml
@@ -62,6 +62,7 @@ model:
   micro_batch_size: 2
   global_batch_size: 2048
   eval_iters: 0
+  eval_interval: 1000
   train_samples: 29297664 #120B tokens
 
   optimizer:
@@ -85,7 +86,6 @@ data:
   split: 1
   no_mmap_bin_files: true
   tokenizer:
-    legacy_tokenizer: true
     tokenizer_type: QwenTokenizerFS
     tokenizer_path: <tokenizer_path>
     vocab_size: 151669

diff --git a/examples/qwen3/conf/train/14b.yaml b/examples/qwen3/conf/train/14b.yaml
@@ -64,6 +64,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 2048
   eval_iters: 0
+  eval_interval: 1000
   train_samples: 244141056 #1T #29297664 #120B tokens
 
   optimizer:

diff --git a/examples/qwen3/conf/train/235b_a22b.yaml b/examples/qwen3/conf/train/235b_a22b.yaml
@@ -73,6 +73,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 2048
   eval_iters: 0
+  eval_interval: 1000
   train_samples: 244141056 #1T #29297664 #120B tokens
 
   optimizer:

diff --git a/examples/qwen3/conf/train/30b_a3b.yaml b/examples/qwen3/conf/train/30b_a3b.yaml
@@ -69,6 +69,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 128 #2048
   eval_iters: 0
+  eval_interval: 1000
   train_iters: 102400
 
   optimizer:

diff --git a/examples/qwen3/conf/train/32b.yaml b/examples/qwen3/conf/train/32b.yaml
@@ -60,6 +60,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 128 #2048
   eval_iters: 0
+  eval_interval: 1000
   train_iters: 102400
 
   optimizer:

diff --git a/examples/qwen3/conf/train/engram.yaml b/examples/qwen3/conf/train/engram.yaml
@@ -82,6 +82,7 @@ model:
   # hf: max_steps: 24000
   train_iters: 102400
   eval_iters: 0
+  eval_interval: 1000
 
 
   optimizer:
@@ -105,7 +106,6 @@ data:
   split: 1
   no_mmap_bin_files: true
   tokenizer:
-    legacy_tokenizer: true
     tokenizer_type: Qwen2TokenizerFS
     tokenizer_path: xxx
     vocab_size: 151936

diff --git a/examples/qwen3/conf/train/te_fl.yaml b/examples/qwen3/conf/train/te_fl.yaml
@@ -70,6 +70,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 8 #2048
   eval_iters: 0
+  eval_interval: 1000
   train_iters: 102400
 
   optimizer:
@@ -92,7 +93,6 @@ data:
   split: 1
   no_mmap_bin_files: true
   tokenizer:
-    legacy_tokenizer: true
     tokenizer_type: QwenTokenizerFS
     tokenizer_path: xxx
     vocab_size: 151851

diff --git a/examples/qwen3_vl/conf/train/32b.yaml b/examples/qwen3_vl/conf/train/32b.yaml
@@ -61,6 +61,7 @@ model:
   clip_grad: 1.0
   train_iters: 62
   eval_iters: 0 # no valid
+  eval_interval: 1000
   micro_batch_size: 1
   global_batch_size: 16
   allow_missing_vision_projection_checkpoint: False

diff --git a/examples/qwen3_vl/conf/train/4b.yaml b/examples/qwen3_vl/conf/train/4b.yaml
@@ -60,6 +60,7 @@ model:
   clip_grad: 1.0
   train_iters: 62
   eval_iters: 0 # no valid
+  eval_interval: 1000
   micro_batch_size: 1
   global_batch_size: 16
   allow_missing_vision_projection_checkpoint: False

diff --git a/examples/qwen3_vl/conf/train/8b.yaml b/examples/qwen3_vl/conf/train/8b.yaml
@@ -63,6 +63,7 @@ model:
   clip_grad: 1.0
   train_iters: 5000
   eval_iters: 0 # no valid
+  eval_interval: 1000
   micro_batch_size: 1
   global_batch_size: 4
   allow_missing_vision_projection_checkpoint: False

diff --git a/examples/qwq/conf/train/32b.yaml b/examples/qwq/conf/train/32b.yaml
@@ -58,6 +58,7 @@ model:
   micro_batch_size: 1
   global_batch_size: 128
   eval_iters: 0
+  eval_interval: 1000
   train_iters: 102400
 
   optimizer:

diff --git a/examples/robobrain_x0/conf/train/robobrain_x0.yaml b/examples/robobrain_x0/conf/train/robobrain_x0.yaml
@@ -63,6 +63,7 @@ model:
   #######################
   train_iters: 330
   eval_iters: 0 # no valid
+  eval_interval: 1000
   micro_batch_size: 1
   global_batch_size: 256
   allow_missing_vision_projection_checkpoint: False

diff --git a/examples/robobrain_x0/conf/train/robobrain_x0_lerobot.yaml b/examples/robobrain_x0/conf/train/robobrain_x0_lerobot.yaml
@@ -66,6 +66,7 @@ model:
   # 1000
   train_iters: 50
   eval_iters: 0
+  eval_interval: 1000
   micro_batch_size: 1
   global_batch_size: 64
   allow_missing_vision_projection_checkpoint: False

@@ -6,6 +6,8 @@
 import torch
 import torch.nn as nn
 
+from megatron.plugin.platform import get_platform
+
 from .engram_config import EngramConfig
 from .multi_head_embedding import MultiHeadEmbedding
 
@@ -17,6 +19,8 @@
 ## Megatron
 from megatron.core.transformer.utils import sharded_state_dict_default
 
+cur_platform = get_platform()
+
 class Engram(nn.Module):
     def __init__(self, engram_cfg: EngramConfig, layer_id):
         super().__init__()
@@ -46,8 +50,8 @@ def __init__(self, engram_cfg: EngramConfig, layer_id):
         )
         self.embedding_cache = None  # Cache for pre-computed embeddings
         self.embedding_stream = None  # Stream for pre-computing embeddings
-        if torch.cuda.is_available():
-            self.embedding_stream = torch.cuda.Stream()
+        if cur_platform.is_available():
+            self.embedding_stream = cur_platform.Stream()
         self.short_conv = ShortConv(
             hidden_size=self.backbone_config.hidden_size,
             kernel_size=engram_cfg.engram_kernel_size,
@@ -91,7 +95,7 @@ def forward(self, hidden_states, hash_input_ids):
         if self.embedding_cache is not None:
             embeddings, embedding_event = self.embedding_cache
             if embedding_event is not None:
-                torch.cuda.current_stream().wait_event(embedding_event)  # Ensure pre-computed embeddings are ready
+                cur_platform.current_stream().wait_event(embedding_event)  # Ensure pre-computed embeddings are ready
             self.embedding_cache = None  # Clear cache after use
             del embedding_event  # Free the event
         else:
@@ -141,9 +145,9 @@ def pre_compute_embedding(self, input_ids: torch.Tensor):
         """
         assert input_ids is not None, "Input ids can not be None for EngramModel"
         self.embedding_stream.synchronize()  # Ensure previous computations on the stream are finished
-        with torch.cuda.stream(self.embedding_stream):
+        with cur_platform.stream(self.embedding_stream):
             embedding_result = self.memory(input_ids).flatten(start_dim=-2)
-        embedding_event = torch.cuda.Event()
+        embedding_event = cur_platform.Event()
         embedding_event.record(self.embedding_stream)
         self.embedding_cache = (embedding_result, embedding_event)