Skip to content

Commit 8ea3ca7

Browse files
committed
Fix triton fall back to CPU when GPU is unavailable in checkpoint.py. Solves running tox test inference_CI
1 parent 132de7b commit 8ea3ca7

1 file changed

Lines changed: 40 additions & 0 deletions

File tree

bris/checkpoint.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,48 @@ def _load_model(self) -> torch.nn.Module:
132132
"check module versions."
133133
) from e
134134
raise e
135+
if not torch.cuda.is_available():
136+
self._apply_triton_cpu_fallback(inst)
135137
return inst
136138

139+
def _apply_triton_cpu_fallback(self, model: torch.nn.Module) -> None:
140+
"""Replace Triton graph attention with the PyG backend when running on CPU.
141+
142+
anemoi-models checks is_triton_available() at model construction time and falls
143+
back to the PyG backend automatically. However, when a model is loaded from a
144+
checkpoint via torch.load() (weights_only=False), __init__ is not called —
145+
pickle restores __dict__ directly — so the Triton function reference is
146+
preserved even when no GPU is available. This method applies the same fallback
147+
after loading.
148+
149+
GraphTransformerConv has no trainable parameters, so the swap is safe.
150+
"""
151+
try:
152+
from anemoi.models.layers.block import GraphTransformerBaseBlock
153+
from anemoi.models.layers.conv import GraphTransformerConv
154+
except ImportError:
155+
LOGGER.warning(
156+
"Could not import anemoi.models layers to apply Triton->PyG CPU fallback."
157+
)
158+
return
159+
160+
patched = 0
161+
for module in model.modules():
162+
if (
163+
isinstance(module, GraphTransformerBaseBlock)
164+
and module.graph_attention_backend == "triton"
165+
):
166+
module.graph_attention_backend = "pyg"
167+
module.conv = GraphTransformerConv(out_channels=module.out_channels_conv)
168+
patched += 1
169+
170+
if patched:
171+
LOGGER.warning(
172+
"Checkpoint was saved with the Triton graph attention backend but no GPU "
173+
"is available. Fell back to the PyG backend for %d block(s).",
174+
patched,
175+
)
176+
137177
@property
138178
def graph(self) -> HeteroData:
139179
"""

0 commit comments

Comments
 (0)