Skip to content

Fsdp dsv3 proxy #3844

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ko3n1g merged 24 commits into from
Mar 16, 2026
Merged

Fsdp dsv3 proxy #3844

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
6f50c24
Add TP2 FSDP test
gautham-kollu Mar 6, 2026
ecda27d
add sharding
gautham-kollu Mar 6, 2026
9c96c29
Add test to github scope
gautham-kollu Mar 6, 2026
d109f3d
format
gautham-kollu Mar 6, 2026
6aceafd
With offloading and recompute
gautham-kollu Mar 6, 2026
aa021aa
Merge branch 'NVIDIA:main' into main
gautham-kollu Mar 12, 2026
8581027
First draft
gautham-kollu Mar 12, 2026
aa8bacb
remove old
gautham-kollu Mar 12, 2026
b0296c1
remove
gautham-kollu Mar 12, 2026
508c0ee
move tp=2 -> ep=2
gautham-kollu Mar 13, 2026
d607e1f
push
gautham-kollu Mar 13, 2026
8b6363a
Merge branch 'main' into fsdp_dsv3_proxy
gautham-kollu Mar 13, 2026
351bd36
Merge branch 'main' into fsdp_dsv3_proxy
ko3n1g Mar 13, 2026
d1999f9
Add missing typo
gautham-kollu Mar 13, 2026
2fdb142
FSDP requires >1
gautham-kollu Mar 13, 2026
64a6208
disable ckpt loading
gautham-kollu Mar 13, 2026
2a4e62d
Adding more flags
gautham-kollu Mar 13, 2026
070af04
fix config. using fp8
gautham-kollu Mar 14, 2026
3187355
1
gautham-kollu Mar 14, 2026
619134d
1
gautham-kollu Mar 14, 2026
4716b72
udpating scope
gautham-kollu Mar 14, 2026
dba6a8c
update golden values
gautham-kollu Mar 14, 2026
b70c8c4
fix lint
gautham-kollu Mar 16, 2026
c304356
Remove h100 configs
gautham-kollu Mar 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
287 changes: 287 additions & 0 deletions ...ional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,287 @@
{
"lm loss": {
"start_step": 1,
"end_step": 50,
"step_interval": 1,
"values": {
"1": 10.94944,
"2": 10.95158,
"3": 10.50143,
"4": 9.9637,
"5": 9.9402,
"6": 9.6731,
"7": 10.2345,
"8": 9.49643,
"9": 9.54137,
"10": 9.7923,
"11": 9.29954,
"12": 9.40392,
"13": 9.39508,
"14": 8.85071,
"15": 9.02369,
"16": 9.07021,
"17": 9.04484,
"18": 8.75671,
"19": 9.17766,
"20": 8.86116,
"21": 8.53586,
"22": 8.54907,
"23": 8.42586,
"24": 8.37914,
"25": 8.63571,
"26": 7.96589,
"27": 8.57436,
"28": 8.19058,
"29": 8.39383,
"30": 8.6699,
"31": 8.28275,
"32": 8.43083,
"33": 8.55346,
"34": 8.65736,
"35": 8.07845,
"36": 7.94562,
"37": 8.29186,
"38": 7.97668,
"39": 8.38836,
"40": 8.35237,
"41": 8.31549,
"42": 8.05591,
"43": 8.03009,
"44": 8.23739,
"45": 8.09515,
"46": 7.61452,
"47": 8.14972,
"48": 8.00299,
"49": 8.38216,
"50": 7.81157
}
},
"num-zeros": {
"start_step": 1,
"end_step": 50,
"step_interval": 1,
"values": {
"1": 19403658.0,
"2": 19274108.0,
"3": 19374004.0,
"4": 86537864.0,
"5": 137554544.0,
"6": 131043136.0,
"7": 167191584.0,
"8": 187932592.0,
"9": 167271824.0,
"10": 163003344.0,
"11": 222662128.0,
"12": 206727744.0,
"13": 231576672.0,
"14": 229976992.0,
"15": 248932672.0,
"16": 234972816.0,
"17": 252131904.0,
"18": 176733312.0,
"19": 175326720.0,
"20": 197382592.0,
"21": 225766720.0,
"22": 217633664.0,
"23": 196029024.0,
"24": 210323328.0,
"25": 221997792.0,
"26": 239705040.0,
"27": 246196976.0,
"28": 278753024.0,
"29": 272254432.0,
"30": 228998896.0,
"31": 252338576.0,
"32": 205052992.0,
"33": 250756576.0,
"34": 205128928.0,
"35": 192742864.0,
"36": 244582560.0,
"37": 180947680.0,
"38": 231918688.0,
"39": 220600064.0,
"40": 212460240.0,
"41": 215821280.0,
"42": 176641872.0,
"43": 203473536.0,
"44": 151341744.0,
"45": 167786640.0,
"46": 105920200.0,
"47": 173317104.0,
"48": 164021296.0,
"49": 100857144.0,
"50": 164130128.0
}
},
"mem-allocated-bytes": {
"start_step": 1,
"end_step": 50,
"step_interval": 1,
"values": {
"1": 4876392448.0,
"2": 4875814400.0,
"3": 4875040256.0,
"4": 4876553728.0,
"5": 4876546560.0,
"6": 4875578880.0,
"7": 4877725184.0,
"8": 4876062208.0,
"9": 4875521536.0,
"10": 4875812352.0,
"11": 4877753856.0,
"12": 4875833856.0,
"13": 4875491840.0,
"14": 4876834304.0,
"15": 4874819072.0,
"16": 4875979264.0,
"17": 4876512768.0,
"18": 4876787200.0,
"19": 4874727936.0,
"20": 4875113984.0,
"21": 4875528704.0,
"22": 4876432896.0,
"23": 4877065728.0,
"24": 4875671040.0,
"25": 4875840000.0,
"26": 4875620864.0,
"27": 4876904960.0,
"28": 4875815424.0,
"29": 4877359616.0,
"30": 4875890176.0,
"31": 4875692544.0,
"32": 4874448384.0,
"33": 4876354048.0,
"34": 4876618240.0,
"35": 4874722816.0,
"36": 4875591168.0,
"37": 4876935680.0,
"38": 4877427200.0,
"39": 4876846592.0,
"40": 4876000768.0,
"41": 4876271104.0,
"42": 4876566016.0,
"43": 4875017728.0,
"44": 4875452928.0,
"45": 4875992576.0,
"46": 4874968576.0,
"47": 4874319360.0,
"48": 4877893120.0,
"49": 4875783680.0,
"50": 4876252672.0
}
},
"mem-max-allocated-bytes": {
"start_step": 1,
"end_step": 50,
"step_interval": 1,
"values": {
"1": 41201033216.0,
"2": 41201033216.0,
"3": 41201033216.0,
"4": 41201033216.0,
"5": 41201033216.0,
"6": 41201033216.0,
"7": 41201033216.0,
"8": 41201033216.0,
"9": 41201033216.0,
"10": 41201033216.0,
"11": 41201033216.0,
"12": 41201033216.0,
"13": 41201033216.0,
"14": 41201033216.0,
"15": 41201033216.0,
"16": 41201033216.0,
"17": 41201033216.0,
"18": 41201033216.0,
"19": 41201033216.0,
"20": 41201033216.0,
"21": 41201033216.0,
"22": 41201033216.0,
"23": 41201033216.0,
"24": 41201033216.0,
"25": 41201033216.0,
"26": 41201033216.0,
"27": 41201033216.0,
"28": 41201033216.0,
"29": 41201033216.0,
"30": 41201033216.0,
"31": 41201033216.0,
"32": 41201033216.0,
"33": 41201033216.0,
"34": 41201033216.0,
"35": 41201033216.0,
"36": 41201033216.0,
"37": 41201033216.0,
"38": 41201033216.0,
"39": 41201033216.0,
"40": 41201033216.0,
"41": 41201033216.0,
"42": 41201033216.0,
"43": 41201033216.0,
"44": 41201033216.0,
"45": 41201033216.0,
"46": 41201033216.0,
"47": 41201033216.0,
"48": 41201033216.0,
"49": 41201033216.0,
"50": 41201033216.0
}
},
"iteration-time": {
"start_step": 1,
"end_step": 50,
"step_interval": 1,
"values": {
"1": 73.81742,
"2": 1.08519,
"3": 0.9475,
"4": 0.8839,
"5": 1.11345,
"6": 0.85209,
"7": 1.03653,
"8": 1.16512,
"9": 0.8689,
"10": 0.85758,
"11": 0.85766,
"12": 0.8648,
"13": 0.85582,
"14": 0.85912,
"15": 0.85612,
"16": 0.85625,
"17": 0.84689,
"18": 0.85414,
"19": 0.85342,
"20": 0.85913,
"21": 0.84294,
"22": 0.84528,
"23": 0.8484,
"24": 0.84952,
"25": 0.84758,
"26": 0.84799,
"27": 0.84573,
"28": 0.85082,
"29": 0.85369,
"30": 0.85037,
"31": 0.85238,
"32": 0.84846,
"33": 0.85245,
"34": 0.86084,
"35": 0.85495,
"36": 0.85092,
"37": 0.85315,
"38": 0.85318,
"39": 0.85153,
"40": 0.84991,
"41": 0.84921,
"42": 0.84843,
"43": 0.84456,
"44": 0.85002,
"45": 0.84683,
"46": 0.84268,
"47": 0.849,
"48": 0.8467,
"49": 0.84356,
"50": 0.84122
}
}
}
Loading
Loading