remove the unnecessary memory free

Author: lmxyy

nunchaku/models/transformers/transformer_qwenimage.py

@@ -555,7 +555,8 @@ def forward(
         hidden_states = self.norm_out(hidden_states, temb)
         output = self.proj_out(hidden_states)
 
-        torch.cuda.empty_cache()
+        if self.offload:
+            torch.cuda.empty_cache()
 
         if not return_dict:
             return (output,)