sglang/test/registered/moe/test_cutedsl_a2a.py at 5e4852bcb24219cfac838c706379b6d7307c9a5c · sgl-project/sglang · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
"""Test CuteDSL FP4 MoE + FlashInfer alltoall on B200 with DP attention.

Config: Qwen3.5-397B-A17B-NVFP4, B200x4, EP=4 DP=4, cutedsl + flashinfer a2a.
"""

import unittest
from types import SimpleNamespace

import torch

from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
    DEFAULT_URL_FOR_TEST,
    CustomTestCase,
    popen_launch_server,
)

register_cuda_ci(est_time=600, suite="stage-c-test-4-gpu-b200")

MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"

SKIP_TEST = torch.cuda.get_device_capability() < (10, 0)
SKIP_REASON = "Requires Blackwell (B200, sm_100a) or above."


@unittest.skipIf(SKIP_TEST, SKIP_REASON)
class TestCuteDslFlashinferA2A(CustomTestCase):
    """CuteDSL FP4 MoE + FlashInfer one-sided alltoall + DP4 EP4 on B200."""

    @classmethod
    def setUpClass(cls):
        cls.model = MODEL
        cls.base_url = DEFAULT_URL_FOR_TEST
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3,
            other_args=[
                "--trust-remote-code",
                "--quantization",
                "modelopt_fp4",
                "--tp",
                "4",
                "--ep-size",
                "4",
                "--dp",
                "4",
                "--enable-dp-attention",
                "--enable-dp-lm-head",
                "--moe-runner-backend",
                "flashinfer_cutedsl",
                "--moe-a2a-backend",
                "flashinfer",
                "--disable-radix-cache",
                "--disable-flashinfer-autotune",
                "--watchdog-timeout",
                "900",
            ],
            env={
                "FLASHINFER_DISABLE_VERSION_CHECK": "1",
                "SGLANG_MOE_NVFP4_DISPATCH": "0",
            },
        )

    @classmethod
    def tearDownClass(cls):
        kill_process_tree(cls.process.pid)

    def test_gsm8k(self):
        args = SimpleNamespace(
            base_url=self.base_url,
            eval_name="gsm8k",
            num_examples=1319,
            max_tokens=10240,
            repeat=1,
            num_threads=1319,
            num_shots=8,
            temperature=0.6,
            top_p=0.95,
            top_k=20,
        )
        metrics = run_eval(args)
        print(metrics)
        self.assertGreater(metrics["score"], 0.90)


if __name__ == "__main__":
    unittest.main()