Support ControlNet v1.1 shuffle properly (huggingface#3340)

* add inferring_controlnet_cond_batch

* Revert "add inferring_controlnet_cond_batch"

This reverts commit abe8d63.

* set guess_mode to True
whenever global_pool_conditions is True

Co-authored-by: Patrick von Platen <[email protected]>

* nit

* add integration test

---------

Co-authored-by: Patrick von Platen <[email protected]>

Author: takuma104

models/controlnet.py

@@ -558,7 +558,7 @@ def forward(
         mid_block_res_sample = self.controlnet_mid_block(sample)
 
         # 6. scaling
-        if guess_mode:
+        if guess_mode and not self.config.global_pool_conditions:
             scales = torch.logspace(-1, 0, len(down_block_res_samples) + 1, device=sample.device)  # 0.1 to 1.0
 
             scales = scales * conditioning_scale

pipelines/stable_diffusion/pipeline_stable_diffusion_controlnet.py

@@ -930,6 +930,13 @@ def __call__(
         if isinstance(self.controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float):
             controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(self.controlnet.nets)
 
+        global_pool_conditions = (
+            self.controlnet.config.global_pool_conditions
+            if isinstance(self.controlnet, ControlNetModel)
+            else self.controlnet.nets[0].config.global_pool_conditions
+        )
+        guess_mode = guess_mode or global_pool_conditions
+
         # 3. Encode input prompt
         prompt_embeds = self._encode_prompt(
             prompt,