fix:txt2_img pipe and controlnet pipe share components

Author: Bluear7878

nunchaku/caching/diffusers_adapters/flux.py

@@ -41,13 +41,19 @@ def apply_cache_on_transformer(
 
     @functools.wraps(original_forward)
     def new_forward(self, *args, **kwargs):
-        with (
-            unittest.mock.patch.object(self, "transformer_blocks", cached_transformer_blocks),
-            unittest.mock.patch.object(self, "single_transformer_blocks", dummy_single_transformer_blocks),
-        ):
+        cache_context = utils.get_current_cache_context()
+        if cache_context is not None:
+            with (
+                unittest.mock.patch.object(self, "transformer_blocks", cached_transformer_blocks),
+                unittest.mock.patch.object(self, "single_transformer_blocks", dummy_single_transformer_blocks),
+            ):
+                return original_forward(*args, **kwargs)
+        else:
             return original_forward(*args, **kwargs)
 
     transformer.forward = new_forward.__get__(transformer)
+    transformer.cached_transformer_blocks = cached_transformer_blocks
+    transformer.single_transformer_blocks = dummy_single_transformer_blocks
     transformer._is_cached = True
 
     return transformer

nunchaku/caching/diffusers_adapters/sana.py

@@ -23,7 +23,11 @@ def apply_cache_on_transformer(transformer: SanaTransformer2DModel, *, residual_
 
     @functools.wraps(original_forward)
     def new_forward(self, *args, **kwargs):
-        with unittest.mock.patch.object(self, "transformer_blocks", cached_transformer_blocks):
+        cache_context = utils.get_current_cache_context()
+        if cache_context is not None:
+            with unittest.mock.patch.object(self, "transformer_blocks", cached_transformer_blocks):
+                return original_forward(*args, **kwargs)
+        else:
             return original_forward(*args, **kwargs)
 
     transformer.forward = new_forward.__get__(transformer)