
    wig                        d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZmZmZmZmZ d dlZd dlZd dlmc mZ d dlmZ d dlmZ d	d
lmZmZmZmZmZmZ d	dlm Z m!Z! d	dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5 ddl"m6Z6  e)            r
d dl7m8Z8m9Z9m:Z:  e,j;        e<          Z=dZ>dZ? G d d          Z@dS )    N)defaultdict)nullcontext)partial)Path)CallableDictListOptionalUnion)validate_hf_hub_args)nn   )ImageProjectionIPAdapterFaceIDImageProjection"IPAdapterFaceIDPlusImageProjectionIPAdapterFullImageProjectionIPAdapterPlusImageProjectionMultiIPAdapterImageProjection)load_model_dict_into_metaload_state_dict)USE_PEFT_BACKEND_get_model_fileconvert_unet_state_dict_to_peftdelete_adapter_layersget_adapter_nameget_peft_kwargsis_accelerate_availableis_peft_versionis_torch_versionloggingset_adapter_layers!set_weights_and_activate_adapters   )LORA_WEIGHT_NAMELORA_WEIGHT_NAME_SAFETEXT_ENCODER_NAME	UNET_NAME)_maybe_expand_lora_scales)AttnProcsLayers)AlignDevicesHook
CpuOffloadremove_hook_from_modulez$pytorch_custom_diffusion_weights.binz,pytorch_custom_diffusion_weights.safetensorsc                      e Zd ZdZeZeZede	e
ee
ej        f         f         fd            Zd Zd Zed             Z	 	 	 	 d!d	e	e
ej        f         d
ede
dedef
dZd Zd"dZd#dZd Zd Zd Z	 d#de	ee
         e
f         dee	e eee          ee         ed         f                  fdZ!d Z"d Z#de	ee
         e
f         fdZ$d$dZ%d$dZ&d$dZ'd  Z(dS )%UNet2DConditionLoadersMixinz:
    Load LoRA layers into a [`UNet2DCondtionModel`].
    %pretrained_model_name_or_path_or_dictc                 B   |                     dd          }|                     dd          }|                     dd          }|                     dd          }|                     dd          }|                     dd          }|                     d	d          }	|                     d
d          }
|                     dd          }|                     dd          }|                     dd          }|                     dd          }|                     dd          }d}|d}d}ddd}d}t          |t                    s|r||t|                    d          r_	 t	          ||pt
          |||||||	|
|          }t          j                            |d          }n# t          $ r}|s|Y d}~nd}~ww xY w|0t	          ||pt          |||||||	|
|          }t          |          }n|}t          d |                                D                       }t          d |                                D                       }d}d}|r|                     |          }n7|r#|                     || j        |||          \  }}nt%          | d          |rQ|O|                     |          \  }}|                     |           |                     | j        | j                   |r|                                 dS |r|                                 dS dS )a  
        Load pretrained attention processor layers into [`UNet2DConditionModel`]. Attention processor layers have to be
        defined in
        [`attention_processor.py`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py)
        and be a `torch.nn.Module` class. Currently supported: LoRA, Custom Diffusion. For LoRA, one must install
        `peft`: `pip install -U peft`.

        Parameters:
            pretrained_model_name_or_path_or_dict (`str` or `os.PathLike` or `dict`):
                Can be either:

                    - A string, the model id (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on
                      the Hub.
                    - A path to a directory (for example `./my_model_directory`) containing the model weights saved
                      with [`ModelMixin.save_pretrained`].
                    - A [torch state
                      dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict).

            cache_dir (`Union[str, os.PathLike]`, *optional*):
                Path to a directory where a downloaded pretrained model configuration is cached if the standard cache
                is not used.
            force_download (`bool`, *optional*, defaults to `False`):
                Whether or not to force the (re-)download of the model weights and configuration files, overriding the
                cached versions if they exist.
            resume_download:
                Deprecated and ignored. All downloads are now resumed by default when possible. Will be removed in v1
                of Diffusers.
            proxies (`Dict[str, str]`, *optional*):
                A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
            local_files_only (`bool`, *optional*, defaults to `False`):
                Whether to only load local model weights and configuration files or not. If set to `True`, the model
                won't be downloaded from the Hub.
            token (`str` or *bool*, *optional*):
                The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from
                `diffusers-cli login` (stored in `~/.huggingface`) is used.
            revision (`str`, *optional*, defaults to `"main"`):
                The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier
                allowed by Git.
            subfolder (`str`, *optional*, defaults to `""`):
                The subfolder location of a model file within a larger model repository on the Hub or locally.
            network_alphas (`Dict[str, float]`):
                The value of the network alpha used for stable learning and preventing underflow. This value has the
                same meaning as the `--network_alpha` option in the kohya-ss trainer script. Refer to [this
                link](https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning).
            adapter_name (`str`, *optional*, defaults to None):
                Adapter name to be used for referencing the loaded adapter model. If not specified, it will use
                `default_{i}` where i is the total number of adapters being loaded.
            weight_name (`str`, *optional*, defaults to None):
                Name of the serialized state dict file.

        Example:

        ```py
        from diffusers import AutoPipelineForText2Image
        import torch

        pipeline = AutoPipelineForText2Image.from_pretrained(
            "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16
        ).to("cuda")
        pipeline.unet.load_attn_procs(
            "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic"
        )
        ```
        	cache_dirNforce_downloadFresume_downloadproxieslocal_files_onlytokenrevision	subfolderweight_nameuse_safetensorsadapter_name	_pipelinenetwork_alphasTattn_procs_weightspytorch)	file_type	frameworkz.safetensors)
weights_namer1   r2   r3   r4   r5   r6   r7   r8   
user_agentcpu)devicec              3      K   | ]}d |v V  	dS )custom_diffusionN .0ks     f/root/.openclaw/workspace/chatterbox_venv_py311/lib/python3.11/site-packages/diffusers/loaders/unet.py	<genexpr>z>UNet2DConditionLoadersMixin.load_attn_procs.<locals>.<genexpr>   s(      !U!Ua"4"9!U!U!U!U!U!U    c              3   H   K   | ]}d |v p|                     d          V  dS )loraz.alphaN)endswithrI   s     rL   rM   z>UNet2DConditionLoadersMixin.load_attn_procs.<locals>.<genexpr>   s7      WWv{:ajj&:&:WWWWWWrN   )
state_dict)rR   unet_identifier_keyr=   r;   r<   zQ does not seem to be in the correct format expected by Custom Diffusion training.)r<   dtyperE   )pop
isinstancedictrQ   r   r%   safetensorstorch	load_fileIOErrorr$   r   anykeysall_process_custom_diffusion_process_lora	unet_name
ValueError_optionally_disable_offloadingset_attn_processortorU   rE   enable_model_cpu_offloadenable_sequential_cpu_offload)selfr/   kwargsr1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   allow_picklerC   
model_filerR   eis_custom_diffusionis_lorais_model_cpu_offloadis_sequential_cpu_offloadattn_processorss                             rL   load_attn_procsz+UNet2DConditionLoadersMixin.load_attn_procsJ   s   F JJ{D11	$4e<< **%6==**Y--!::&8$??

7D))::j$//JJ{D11	jj55 **%6==zz.$77JJ{D11	$4d;;""OL ."
 


 
?FF )	? K$7'K,@,@,P,P'!0=%0%I4I"+'5(7 ')9#!)"+#-" " "J "-!2!<!<ZPU!<!V!VJJ   '  DDDD	
 !,9!,!@0@'#1$3#%5%')  
 -Z88
>J!!U!U:??CTCT!U!U!UUUWWZ__EVEVWWWWW$$)! 	"<<
<SSOO 	>B>P>P%$(N-)# ?Q ? ?; ";"; ppp    	:9#8>B>a>alu>a>v>v; "; ##O444GG$*T[G999   	6..00000& 	63355555	6 	6s   AF" "
F:,F55F:c                 z   ddl m} i }t          t                    }|                                D ]\  }}t          |          dk    ri ||<   d|v rad                    |                    d          d d                   d                    |                    d          dd                    }}n`d                    |                    d          d d                   d                    |                    d          dd                    }}|||         |<   |                                D ]\  }}	t          |	          dk    r |ddd d 	          ||<   +|	d
         j        d         }
|	d
         j        d         }d|	v rdnd} |d|||
	          ||<   ||         	                    |	           |S )Nr   )CustomDiffusionAttnProcessorr   to_out.F)train_kvtrain_q_outhidden_sizecross_attention_dimzto_k_custom_diffusion.weightr#   zto_q_custom_diffusion.weightT)
models.attention_processorru   r   rX   itemslenjoinsplitshaper   )ri   rR   ru   rr   custom_diffusion_grouped_dictkeyvalueattn_processor_keysub_key
value_dictr}   r|   r{   s                rL   r`   z5UNet2DConditionLoadersMixin._process_custom_diffusion   s   MMMMMM(3D(9(9%$**,, 	S 	SJC5zzQ57-c22s??25((399S>>#2#;N2O2OQTQYQYZ]ZcZcdgZhZhikililZmQnQn&&25((399S>>#2#;N2O2OQTQYQYZ]ZcZcdgZhZhikililZmQnQn&MR-.@A'JJ<BBDD 	A 	AOC:!##'C'C"4]a( ( ($$ '11O&P&VWX&Y#()GHNqQ&D
&R&RddX]'C'C! + +(;	( ( ($  $44Z@@@@rN   c                    t           st          d          ddlm}m}m} t          |                                          }	fd|	D             fd|                                D             }
|Afd|                                D             fd|                                D             }d}d}t          |
          dk    r|
n|}t          |          dk    rY|t          | d	i           v rt          d
| d          t          |          }|t          |          }i }|                                D ]\  }}d|v r|j        d         ||<   t          |||d          }d|v rM|d         r t          dd          rt          d          n%t          dd          r|                    d            |di |}|t!          |           }|                     |          \  }} ||| |            || ||          }|1t          |dd           }|rt$                              d| d           ||fS )N)PEFT backend is required for this method.r   )
LoraConfiginject_adapter_in_modelset_peft_model_state_dictc                 >    g | ]}|                               |S rH   
startswithrJ   rK   rS   s     rL   
<listcomp>z=UNet2DConditionLoadersMixin._process_lora.<locals>.<listcomp>)  s+    JJJ15H(I(IJQJJJrN   c                 R    i | ]#\  }}|v 	|                      d d          |$S rw    replace)rJ   rK   vrS   	unet_keyss      rL   
<dictcomp>z=UNet2DConditionLoadersMixin._process_lora.<locals>.<dictcomp>*  sJ     
 
 
<@AqZ[_hZhZhAII,///44aZhZhZhrN   c                 >    g | ]}|                               |S rH   r   r   s     rL   r   z=UNet2DConditionLoadersMixin._process_lora.<locals>.<listcomp>/  s+    ```allK^>_>_`!```rN   c                 R    i | ]#\  }}|v 	|                      d d          |$S r   r   )rJ   rK   r   
alpha_keysrS   s      rL   r   z=UNet2DConditionLoadersMixin._process_lora.<locals>.<dictcomp>0  sJ       @D1bcgqbqbq		0333R88!bqbqbqrN   Fpeft_configzAdapter name z? already in use in the Unet - please select a new adapter name.lora_Br#   T)is_unetuse_dora<z0.9.0zeYou need `peft` 0.9.0 at least to use DoRA-enabled LoRAs. Please upgrade your installation of `peft`.)r;   unexpected_keyszXLoading adapter weights from state_dict led to unexpected keys not found in the model:  z. rH   )r   rc   peftr   r   r   listr^   r   r   getattrr   r   r   r   rV   r   rd   loggerwarning)ri   rR   rS   r=   r;   r<   r   r   r   r^   unet_state_dictrp   rq   state_dict_to_be_usedrankr   vallora_config_kwargslora_configincompatible_keysr   r   r   s     `                  @@rL   ra   z)UNet2DConditionLoadersMixin._process_lora  s@      	JHIIIWWWWWWWWWWJOO%%&&JJJJJJJ	
 
 
 
 
DNDTDTDVDV
 
 
 %````^%8%8%:%:```J    HVH\H\H^H^  N  %$)!363G3G!3K3KQ[$%%))wt]B???? qLqqq   99NOOJ) "A!P!PD&,,.. - -Ss?? #	!DI!0~z[_!`!`!`///%j1 ;&sG44 ( D  
 'sG44 ;*..z:::$*::'9::K #/55 ?C>a>abk>l>l; ";##KLQQQQ 9 9$
L Y Y ,")*;=NPT"U"U" NN0+0 0 0  
 $%>>>rN   c                    d}d}||j         |j                                        D ]\  }}t          |t          j                  rt          |d          r|st          |j        t                    }|sTt          |j        t                    p9t          |j        d          o$t          |j        j
        d         t                    }t                              d           t          ||           ||fS )ar  
        Optionally removes offloading in case the pipeline has been already sequentially offloaded to CPU.

        Args:
            _pipeline (`DiffusionPipeline`):
                The pipeline to disable offloading for.

        Returns:
            tuple:
                A tuple indicating if `is_model_cpu_offload` or `is_sequential_cpu_offload` is True.
        FN_hf_hookhooksr   zAccelerate hooks detected. Since you have called `load_lora_weights()`, the previous hooks will be first removed. Then the LoRA parameters will be loaded and the hooks will be applied again.)recurse)hf_device_map
componentsr   rW   r   Modulehasattrr   r+   r*   r   r   infor,   )clsr<   rp   rq   _	components         rL   rd   z:UNet2DConditionLoadersMixin._optionally_disable_offloadingl  s     %$)! Y%<%D ) 4 : : < < Z Z9i33 Z	:8V8V Z/ Z/9):Lj/Y/Y,4 &y'9;KLL Z&y'97CC  Z *9+=+CA+FHX Y Y 2 KK Y   ,I?XYYYY$&?@@rN   TNsave_directoryis_main_processr9   save_functionsafe_serializationc                    ddl mmm t          j                            |          r t                              d| d           dS t          fd| j
                                        D                       }|r|                                 }n't          st          d          dd	lm}	  |	|           }||rd
 }nt"          j        }t	          j        |d           |#|r|rt(          nt*          }n|rt,          nt.          }t1          ||                                          }
 |||
           t                              d|
            dS )az  
        Save attention processor layers to a directory so that it can be reloaded with the
        [`~loaders.UNet2DConditionLoadersMixin.load_attn_procs`] method.

        Arguments:
            save_directory (`str` or `os.PathLike`):
                Directory to save an attention processor to (will be created if it doesn't exist).
            is_main_process (`bool`, *optional*, defaults to `True`):
                Whether the process calling this is the main process or not. Useful during distributed training and you
                need to call this function on all processes. In this case, set `is_main_process=True` only on the main
                process to avoid race conditions.
            save_function (`Callable`):
                The function to use to save the state dictionary. Useful during distributed training when you need to
                replace `torch.save` with another method. Can be configured with the environment variable
                `DIFFUSERS_SAVE_MODE`.
            safe_serialization (`bool`, *optional*, defaults to `True`):
                Whether to save the model using `safetensors` or with `pickle`.

        Example:

        ```py
        import torch
        from diffusers import DiffusionPipeline

        pipeline = DiffusionPipeline.from_pretrained(
            "CompVis/stable-diffusion-v1-4",
            torch_dtype=torch.float16,
        ).to("cuda")
        pipeline.unet.load_attn_procs("path-to-save-model", weight_name="pytorch_custom_diffusion_weights.bin")
        pipeline.unet.save_attn_procs("path-to-save-model", weight_name="pytorch_custom_diffusion_weights.bin")
        ```
        r   ru   CustomDiffusionAttnProcessor2_0$CustomDiffusionXFormersAttnProcessorzProvided path (z#) should be a directory, not a fileNc              3   D   K   | ]\  }}t          |f          V  d S NrW   )rJ   r   xru   r   r   s      rL   rM   z>UNet2DConditionLoadersMixin.save_attn_procs.<locals>.<genexpr>  sV       "
 "

 A	 -/NPtu "
 "
 "
 "
 "
 "
rN   zOPEFT backend is required for saving LoRAs using the `save_attn_procs()` method.r   )get_peft_model_state_dictc                 J    t           j                            | |ddi          S )Nformatpt)metadata)rY   rZ   	save_file)weightsfilenames     rL   r   zBUNet2DConditionLoadersMixin.save_attn_procs.<locals>.save_function  s'    &,66wT\^bSc6dddrN   T)exist_okzModel weights saved in )r~   ru   r   r   ospathisfiler   errorr]   rr   r    _get_custom_diffusion_state_dictr   rc   
peft.utilsr   rZ   savemakedirs!CUSTOM_DIFFUSION_WEIGHT_NAME_SAFEr%   CUSTOM_DIFFUSION_WEIGHT_NAMEr$   r   as_posixr   )ri   r   r   r9   r   r   rj   rn   rR   r   	save_pathru   r   r   s              @@@rL   save_attn_procsz+UNet2DConditionLoadersMixin.save_attn_procs  s   R	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 7>>.)) 	LL^>^^^___F! "
 "
 "
 "
 "
 "

 .4466"
 "
 "
 
 
  	9>>@@JJ# t !rsss<<<<<<22488J ! +e e e e !&

NT2222! hCVq??\q>Qg::Wg 55>>@@	j),,,9i99:::::rN   c                 B   ddl mmm t	          fd| j                                        D                       }|                                }| j                                        D ]/\  }}t          |                                          dk    ri ||<   0|S )Nr   r   c                 B    i | ]\  }}t          |f          ||S rH   r   )rJ   yr   ru   r   r   s      rL   r   zPUNet2DConditionLoadersMixin._get_custom_diffusion_state_dict.<locals>.<dictcomp>  sP       Q47< 1  rN   r   )	r~   ru   r   r   r)   rr   r   rR   r   )ri   model_to_saverR   nameattnru   r   r   s        @@@rL   r   z<UNet2DConditionLoadersMixin._get_custom_diffusion_state_dict  s    	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 (     "288::  
 
 #--//
.4466 	& 	&JD$4??$$%%**#%
4 rN         ?Fc                     t           st          d          || _        || _        |                     t          | j        |                     d S )Nz+PEFT backend is required for `fuse_lora()`.)adapter_names)r   rc   
lora_scale_safe_fusingapplyr   _fuse_lora_apply)ri   r   safe_fusingr   s       rL   	fuse_loraz%UNet2DConditionLoadersMixin.fuse_lora  sQ     	LJKKK$'

740NNNOOOOOrN   c                 @   ddl m} d| j        i}t          ||          r~| j        dk    r|                    | j                   t          t          j        |j	                  j
                  }d|v r||d<   nd|vr|t          d           |j	        di | d S d S )Nr   BaseTunerLayer
safe_merger   r   zThe `adapter_names` argument is not supported with your PEFT version. Please upgrade to the latest version of PEFT. `pip install -U peft`rH   )peft.tuners.tuners_utilsr   r   rW   r   scale_layerr   inspect	signaturemerge
parametersrc   )ri   moduler   r   merge_kwargssupported_merge_kwargss         rL   r   z,UNet2DConditionLoadersMixin._fuse_lora_apply  s    ;;;;;;$d&78fn-- 	)#%%""4?333 &*'*;FL*I*I*T%U%U""8880=_-- (>>>=C\ L  
 FL((<(((((	) 	)rN   c                 f    t           st          d          |                     | j                   d S )Nz-PEFT backend is required for `unfuse_lora()`.)r   rc   r   _unfuse_lora_applyri   s    rL   unfuse_loraz'UNet2DConditionLoadersMixin.unfuse_lora$  s5     	NLMMM

4*+++++rN   c                 ^    ddl m} t          ||          r|                                 d S d S )Nr   r   )r   r   rW   unmerge)ri   r   r   s      rL   r   z.UNet2DConditionLoadersMixin._unfuse_lora_apply)  sF    ;;;;;;fn-- 	NN	 	rN   c                 |    t           st          d          ddlm}  ||            t	          | d          r| `d S d S )Nz-PEFT backend is required for `unload_lora()`.r   )recurse_remove_peft_layersr   )r   rc   utilsr  r   r   )ri   r  s     rL   unload_loraz'UNet2DConditionLoadersMixin.unload_lora/  se     	NLMMM666666""4(((4'' 	!   	! 	!rN   r   r   c                    t           st          d          t          |t                    r|gn|}t          |t                    s|gt          |          z  }t          |          t          |          k    r0t          dt          |           dt          |           d          d |D             }t          | |          }t          | ||           dS )a  
        Set the currently active adapters for use in the UNet.

        Args:
            adapter_names (`List[str]` or `str`):
                The names of the adapters to use.
            adapter_weights (`Union[List[float], float]`, *optional*):
                The adapter(s) weights to use with the UNet. If `None`, the weights are set to `1.0` for all the
                adapters.

        Example:

        ```py
        from diffusers import AutoPipelineForText2Image
        import torch

        pipeline = AutoPipelineForText2Image.from_pretrained(
            "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16
        ).to("cuda")
        pipeline.load_lora_weights(
            "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic"
        )
        pipeline.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors", adapter_name="pixel")
        pipeline.set_adapters(["cinematic", "pixel"], adapter_weights=[0.5, 0.5])
        ```
        z.PEFT backend is required for `set_adapters()`.zLength of adapter names z- is not equal to the length of their weights rw   c                     g | ]}||nd	S )Nr   rH   )rJ   ws     rL   r   z<UNet2DConditionLoadersMixin.set_adapters.<locals>.<listcomp>i  s     @@@1113@@@rN   N)r   rc   rW   strr   r   r(   r"   )ri   r   r   s      rL   set_adaptersz(UNet2DConditionLoadersMixin.set_adapters9  s    >   	OMNNN+5mS+I+I\} '4(( 	5i#m"4"44G}W--{3}+=+={{lopwlxlx{{{   A@@@@ ,D'::)$wGGGGGrN   c                 T    t           st          d          t          | d           dS )a  
        Disable the UNet's active LoRA layers.

        Example:

        ```py
        from diffusers import AutoPipelineForText2Image
        import torch

        pipeline = AutoPipelineForText2Image.from_pretrained(
            "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16
        ).to("cuda")
        pipeline.load_lora_weights(
            "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic"
        )
        pipeline.disable_lora()
        ```
        r   FenabledNr   rc   r!   r   s    rL   disable_loraz(UNet2DConditionLoadersMixin.disable_lorap  s5    &   	JHIII4//////rN   c                 T    t           st          d          t          | d           dS )a  
        Enable the UNet's active LoRA layers.

        Example:

        ```py
        from diffusers import AutoPipelineForText2Image
        import torch

        pipeline = AutoPipelineForText2Image.from_pretrained(
            "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16
        ).to("cuda")
        pipeline.load_lora_weights(
            "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic"
        )
        pipeline.enable_lora()
        ```
        r   Tr  Nr  r   s    rL   enable_loraz'UNet2DConditionLoadersMixin.enable_lora  s5    &   	JHIII4......rN   c                     t           st          d          t          |t                    r|g}|D ]=}t	          | |           t          | d          r| j                            |d           >dS )a  
        Delete an adapter's LoRA layers from the UNet.

        Args:
            adapter_names (`Union[List[str], str]`):
                The names (single string or list of strings) of the adapter to delete.

        Example:

        ```py
        from diffusers import AutoPipelineForText2Image
        import torch

        pipeline = AutoPipelineForText2Image.from_pretrained(
            "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16
        ).to("cuda")
        pipeline.load_lora_weights(
            "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_names="cinematic"
        )
        pipeline.delete_adapters("cinematic")
        ```
        r   r   N)r   rc   rW   r  r   r   r   rV   )ri   r   r;   s      rL   delete_adaptersz+UNet2DConditionLoadersMixin.delete_adapters  s    .   	JHIIImS)) 	,*OM) 	9 	9L!$555 t]++ 9 $$\4888	9 	9rN   c                 ^   |r1t                      rddlm} nd}t                              d           |du rt          dd          st          d          i }d }|r|nt          }d	|v rd
}|d	         j        d         }|d	         j        d         d
z  }	 |            5  t          |	||          }d d d            n# 1 swxY w Y   |
                                D ] \  }
}|
                    dd          }|||<   !nd|v r|d         j        d         }|d         j        d         }	 |            5  t          |	|          }d d d            n# 1 swxY w Y   |
                                D ]L\  }
}|
                    dd          }|                    dd          }|                    dd          }|||<   Mn6d|v r|d         j        d         }|d         j        d         }|d         j        d         }|d         j        d         }|d         j        d         dz  } |            5  t          |||||          }d d d            n# 1 swxY w Y   |
                                D ]\  }
}|
                    dd          }|                    d d!          }|                    d"d#          }|                    d$d%          }|                    d&d'          }|                    d(d)          }|                    d*d+          }|                    d,d-          }|                    d.d/          }|                    d0d1          }|                    d2d3          }|                    d4d5          }|                    d6d7          }|                    d8d9          }|                    d:d;          }|                    d<d=          }|                    d>d?          }|                    d@dA          }|                    dBdC          }|                    dDdE          }|                    dFdG          }|                    dHdI          }dJ|v r|||                    dKdL          <   	dM|v r|||                    dNdO          <   (dP|v rW|                    dQdR          }|d         ||                    dPdS          <   |d         ||                    dPdT          <   dU|v r|||                    dUdV          <   d|k    r||dW<   dX|k    r||dY<   dZ|k    r||d[<   d\|k    r||d]<   |||<   ݐnd^|v r|d         j        d         }|d         j        d         }||z  }d^}||         j        d         }	|dZ         j        d         |	z  } |            5  t          |	|||_          }d d d            n# 1 swxY w Y   |
                                D ]6\  }
}|
                    dd          }|                    dd          }|||<   7n|d`         j        d         }|da         j        d         }|db         j        d         }|d`         j        dQ         }t!          dc |D                       }|r|dd         j        d         dz  n|de         j        d         dz  } |            5  t#          |||||f          }d d d            n# 1 swxY w Y   |
                                D ]\  }
}|
                    d dg          }|                    dhdi          }|                    djdk          }|                    dldm          }|                    dndo          }|                    dpdq          }|                    drds          }|                    dtdu          }|                    dvdw          }dP|v r|                    dx          }dy|dQ<   dx                    |          }|                    dQdR          }|d         ||                    dPdS          <   |d         ||                    dPdT          <   Vdz|v r6|                    dx          }dy|dQ<   dx                    |          }|||<   dU|v rJ|                    dx          }dy|dQ<   dx                    |          }|||                    dUdV          <   |                    d{d|          }|                    d}d~          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|||<   |s|                    |d           nt+          ||| j        | j                   |S )Nr   init_empty_weightsF,  Cannot initialize model with low cpu memory usage because `accelerate` was not found in the environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install `accelerate` for faster and less memory-intense model loading. You can do so with: 
```
pip install accelerate
```
.T>=1.9.0~Low memory initialization requires torch >= 1.9.0. Please either update your PyTorch version or set `low_cpu_mem_usage=False`.proj.weight   )r}   image_embed_dimnum_image_text_embedsprojimage_embedsproj.3.weightzproj.0.weight)r}   r  zproj.0zff.net.0.projzproj.2zff.net.2zproj.3norm"perceiver_resampler.proj_in.weightr#   z#perceiver_resampler.proj_out.weightz*perceiver_resampler.layers.0.0.to_q.weight@   )
embed_dimsoutput_dimshidden_dimsheadsid_embeddings_dimzperceiver_resampler.r   z0.tozattn.toz0.1.0.z0.ff.0.z0.1.1.weightz0.ff.1.net.0.proj.weightz0.1.3.weightz0.ff.1.net.2.weightz1.1.0.z1.ff.0.z1.1.1.weightz1.ff.1.net.0.proj.weightz1.1.3.weightz1.ff.1.net.2.weightz2.1.0.z2.ff.0.z2.1.1.weightz2.ff.1.net.0.proj.weightz2.1.3.weightz2.ff.1.net.2.weightz3.1.0.z3.ff.0.z3.1.1.weightz3.ff.1.net.0.proj.weightz3.1.3.weightz3.ff.1.net.2.weightz
layers.0.0zlayers.0.ln0z
layers.0.1zlayers.0.ln1z
layers.1.0zlayers.1.ln0z
layers.1.1zlayers.1.ln1z
layers.2.0zlayers.2.ln0z
layers.2.1zlayers.2.ln1z
layers.3.0zlayers.3.ln0z
layers.3.1zlayers.3.ln1norm1z0.norm10norm2z0.norm21to_kvr   )dimto_kto_vrv   zto_out.0zproj.net.0.proj.weightzproj.0.biaszproj.net.0.proj.biaszproj.2.weightzproj.net.2.weightzproj.2.biaszproj.net.2.biasnorm.weight)r}   r  mult
num_tokenslatentszproj_in.weightzproj_out.weightc              3      K   | ]}d |v V  	dS )r   NrH   rI   s     rL   rM   zZUNet2DConditionLoadersMixin._convert_ip_adapter_image_proj_to_diffusers.<locals>.<genexpr>U  s&      "C"C16Q;"C"C"C"C"C"CrN   zlayers.0.attn.to_q.weightzlayers.0.0.to_q.weight)r%  r&  r'  r(  num_queriesz2.toz	0.0.norm1z0.ln0z	0.0.norm2z0.ln1z	1.0.norm1z1.ln0z	1.0.norm2z1.ln1z	2.0.norm1z2.ln0z	2.0.norm2z2.ln1z	3.0.norm1z3.ln0z	3.0.norm2z3.ln1rw   r   to_qz0.1.0z0.ff.0z0.1.1z0.ff.1.net.0.projz0.1.3z0.ff.1.net.2z1.1.0z1.ff.0z1.1.1z1.ff.1.net.0.projz1.1.3z1.ff.1.net.2z2.1.0z2.ff.0z2.1.1z2.ff.1.net.0.projz2.1.3z2.ff.1.net.2z3.1.0z3.ff.0z3.1.1z3.ff.1.net.0.projz3.1.3z3.ff.1.net.2)strictrE   rU   )r   
accelerater  r   r   r   NotImplementedErrorr   r   r   r   r   r   r   chunkr   r]   r   r   r   r   r   rE   rU   )ri   rR   low_cpu_mem_usager  updated_state_dictimage_projectioninit_contextr  clip_embeddings_dimr}   r   r   diffusers_namer)  r%  r'  r&  r(  v_chunkid_embeddings_dim_inid_embeddings_dim_out
multiplier
norm_layerr4  attn_key_presentpartss                             rL   +_convert_ip_adapter_image_proj_to_diffuserszGUNet2DConditionLoadersMixin._convert_ip_adapter_image_proj_to_diffusers  s    	&(( 
9999999 %*!2   $$-=dG-L-L$%.  
  ->O))KJ&&$%!",]";"A""E",]";"A!"D"I  #2(;$7*?$ $ $                )..00 ; ;
U!$V^!D!D5:">22; 
**",_"="CA"F",_"="CA"F  #?(;M`$ $ $               
 )..00 ; ;
U!$X!G!G!/!7!7*!M!M!/!7!7&!I!I5:">22	; 2Z?? *? ; A! D#$HIOPQRJ$%IJPQRSK$%JKQRSTKKLRSTUY[[E  #E) + +&7$ $ $                )..00 +? +?
U!$-CR!H!H!/!7!7	!J!J!/!7!7)!L!L!/!7!7Hb!c!c!/!7!7H]!^!^!/!7!7)!L!L!/!7!7Hb!c!c!/!7!7H]!^!^!/!7!7)!L!L!/!7!7Hb!c!c!/!7!7H]!^!^!/!7!7)!L!L!/!7!7Hb!c!c!/!7!7H]!^!^!/!7!7n!U!U!/!7!7n!U!U!/!7!7n!U!U!/!7!7n!U!U!/!7!7n!U!U!/!7!7n!U!U!/!7!7n!U!U!/!7!7n!U!Un,,QV&~'='=i'M'MNN..QV&~'='=i'M'MNN..#kk!k33GRYZ[R\&~'='=gv'N'NORYZ[R\&~'='=gv'N'NOO//W\&~'='=h
'S'STT$66CH&'?@@"n44AF&'=>>$66>C&':;;"n44<A&'8999>&~66W+?Z j((#-o#>#DQ#G $.$?$Ea$H!.2FFJ&J",Z"8">q"A#O4:1=ATTJ  #A(;$8#)	$ $ $                )..00 ; ;
U!$X!G!G!/!7!7*!M!M5:">22; %/y$9$?$B!#$45;A>J$%67=a@K$Y/5a8K""C"C
"C"C"CCC $I
67=a@BFF 89?BbH    #?) + + 5$ $ $                )..00 -? -?
U!$VV!<!<!/!7!7W!M!M!/!7!7W!M!M!/!7!7W!M!M!/!7!7W!M!M!/!7!7W!M!M!/!7!7W!M!M!/!7!7W!M!M!/!7!7W!M!Mn,,*0055E%E!H%(XXe__N#kk!k33GRYZ[R\&~'='=gv'N'NORYZ[R\&~'='=gv'N'NOO~--*0055E%E!H%(XXe__N9>&~66//*0055E%E!H%(XXe__NW\&~'='=h
'S'STT%3%;%;GX%N%NN%3%;%;GEX%Y%YN%3%;%;G^%T%TN%3%;%;GX%N%NN%3%;%;GEX%Y%YN%3%;%;G^%T%TN%3%;%;GX%N%NN%3%;%;GEX%Y%YN%3%;%;G^%T%TN%3%;%;GX%N%NN%3%;%;GEX%Y%YN%3%;%;G^%T%TN9>&~66  	r,,-?,MMMM%&68JSWS^fjfpqqqqsZ   B>>CC4EEE1III4WWW	[**[.1[.c                    ddl m}m}m}m} |r1t                      rddlm} nd}t          	                    d           |du rt          dd	          st          d
          i }d}	|r|nt          }
| j                                        D ]}|                    d          rd n| j        j        }|                    d          r| j        j        d         }n|                    d          rOt)          |t+          d                             }t-          t/          | j        j                            |         }nI|                    d          r4t)          |t+          d                             }| j        j        |         }|d|v r(t1          t2          d          r|n|} |            ||<   )t1          t2          d          r|n|}g }|D ]c}d|d         v r|dgz  }d|d         v r|dgz  }$d|d         v r|dgz  }5d|d         v r|dgz  }F||d         d         j        d         gz  }d |
            5   |||d|          ||<   d d d            n# 1 swxY w Y   i }t7          |          D ]Y\  }}|                    d| d |d!         |	 d"         i           |                    d#| d |d!         |	 d$         i           Z|s||                             |           nt=          t?          |                                                     j!        }t=          t?          |                                                     j"        }tG          ||         |||%           |	dz  }	|S )&Nr   )AttnProcessorAttnProcessor2_0IPAdapterAttnProcessorIPAdapterAttnProcessor2_0r   r  Fr  Tr  r  r  r#   zattn1.processor	mid_blockr  	up_blocksz
up_blocks.down_blockszdown_blocks.motion_modulesscaled_dot_product_attentionr  
image_projr  r!  i  r#  r2  r5  r   )r|   r}   scaler4  zto_k_ip.z.weight
ip_adapterz.to_k_ip.weightzto_v_ip.z.to_v_ip.weightr:  )$r~   rM  rN  rO  rP  r   r;  r  r   r   r   r<  r   rr   r^   rQ   configr}   r   block_out_channelsintr   r   reversedr   Fr   	enumerateupdater   nextitervaluesrE   rU   r   )ri   state_dictsr>  rM  rN  rO  rP  r  
attn_procskey_idrA  r   r}   r|   block_idattn_processor_classr  rR   r   irE   rU   s                         rL   %_convert_ip_adapter_attn_to_diffuserszAUNet2DConditionLoadersMixin._convert_ip_adapter_attn_to_diffusers  s   	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
  	&(( 
9999999 %*!2   $$-=dG-L-L$%.   
->O))K(--// ;	 ;	D*.--8I*J*J"o$$PTP[Po{++ G"k<R@-- GtC$5$5677"8DK,J#K#KLLXV// GtC$7$7899"k<XF"*.>$.F.F(/3Q(R(Re$$Xe % $8#7#9#9
4   29<Z1[1[w--aw % )+%"- ` `J$
<(@@@-!4--(J|,DDD-#6--=LAYYY-!4--&*\*BBB-!4-- .*\2J92U2[\]2^1__--!\^^  ';';$/,?!#8	( ( (Jt$                
%.{%;%; u uMAz%%'<!'<'<'<j>VZ`WqWqWq>r&sttt%%'<!'<'<'<j>VZ`WqWqWq>r&stttt( ht$44Z@@@@!$z'8'8':':";";<<CF j&7&7&9&9!:!:;;AE-j.>
SYafgggg!s   ?II"	%I"	c                    t          |t                    s|g}d | _        |                     ||          }|                     |           g }|D ]4}|                     |d         |          }|                    |           5t          |          | _        d| j        _	        | 
                    | j        | j                   d S )N)r>  rV  ip_image_projrT   )rW   r   encoder_hid_projri  re   rK  appendr   rY  encoder_hid_dim_typerf   rU   rE   )ri   rc  r>  rd  image_projection_layersrR   image_projection_layers          rL   _load_ip_adapter_weightsz4UNet2DConditionLoadersMixin._load_ip_adapter_weights  s    +t,, 	(&-K !%??_p?qq

+++ #%% 	C 	CJ%)%U%U<(<M &V & &" $**+ABBBB =>U V V+:(dj55555rN   c                    i }t          | j                                                  D ]\  }}t          |          D ]\  }}| d|d         v r||vri ||<   ||                             d| d|d         | d         i           ||                             d| d|d         | d         i           ||                             d| d|d         | d         i           ||                             d| d|d         | d         i           ||                             d| d|d         | d         i           ||                             d| d|d         | d         i           ||                             d| d	|d         | d	         i           ||                             d| d
|d         | d
         i           |S )Nz.to_k_lora.down.weightrX  zunet.z.to_q_lora.down.weightz.to_v_lora.down.weightz.to_out_lora.down.weightz.to_k_lora.up.weightz.to_q_lora.up.weightz.to_v_lora.up.weightz.to_out_lora.up.weight)r^  rr   r^   r_  )ri   rc  
lora_dictsre  r   rh  rR   s          rL   _load_ip_adapter_lorasz2UNet2DConditionLoadersMixin._load_ip_adapter_loras  s   
%d&:&?&?&A&ABB 0	 0	LFD!*;!7!7 / /:444
<8PPP
**(*
1qM((@D@@@*\BZ#) A A AC   qM((@D@@@*\BZ#) A A AC   qM((@D@@@*\BZ#) A A AC   qM((BDBBBJ|D\#) C C CE   qM((;;;;Z=UY_VuVuVu=vw   qM((;;;;Z=UY_VuVuVu=vw   qM((;;;;Z=UY_VuVuVu=vw   qM((@D@@@*\BZ#) A A AC  S/` rN   )TNNT)r   FNr   )F))__name__
__module____qualname____doc__r&   text_encoder_namer'   rb   r   r   r  r   rZ   Tensorrs   r`   ra   classmethodrd   r   PathLikeboolr   r   r   r   r   r   r   r  r	   r
   floatr	  r  r  r  rK  ri  rq  rt  rH   rN   rL   r.   r.   B   s         *Ij6U3PTUXZ_ZfUfPgKgEh j6 j6 j6 j6Z     DP? P? P?d  A  A [ AJ !%"&#'X; X;c2;./X; X; 	X;
  X; !X; X; X; X;t  8P P P P) ) ) ),, , ,
  ! ! ! VZ5H 5HT#Y^,5H %tT%[$t*d4j PQR5H 5H 5H 5Hn0 0 0./ / /."9U49c>-B "9 "9 "9 "9HW  W  W  W r\ \ \ \|6 6 6 6.3 3 3 3 3rN   r.   )Ar   r   collectionsr   
contextlibr   	functoolsr   pathlibr   typingr   r   r	   r
   r   rY   rZ   torch.nn.functionalr   
functionalr]  huggingface_hub.utilsr   models.embeddingsr   r   r   r   r   r   models.modeling_utilsr   r   r  r   r   r   r   r   r   r   r   r   r    r!   r"   rP   r$   r%   r&   r'   unet_loader_utilsr(   r)   accelerate.hooksr*   r+   r,   
get_loggerru  r   r   r   r.   rH   rN   rL   <module>r     s    				 # # # # # # " " " " " "             8 8 8 8 8 8 8 8 8 8 8 8 8 8               6 6 6 6 6 6                      O N N N N N N N                            X W W W W W W W W W W W 8 8 8 8 8 8 " " " " " "  WVVVVVVVVVV		H	%	%  F $R !A A A A A A A A A ArN   