wi1MddlmZddlmZddlmZmZmZmZm Z m Z ddl Z ddl Z ddl m Z mZddlmZmZddlmZmZmZdd lmZd d lmZmZmZmZmZmZd d lm Z d d l!m"Z"m#Z#d dl$m%Z%d dl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-d dl.m/Z/ej0e1Z2eGddeZ3Gddej4Z5Gddej4Z6Gddej4Z7 d=de8de8de8d e8d!e8d"ee8d#ee e8e e8fd$ee8d%ee8d&e9d'ee9fd(Z: d>d)e8d*e8d!ee8d"ee8d#e8d$ee8d%ee8d'e9fd+Z;d,e8d-e8d.ee8fd/Z<Gd0d1e%eZ=Gd2d3e%eZ>Gd4d5ej4Z?Gd6d7ej4Z@Gd8d9ej4ZAd?d:ZBd;ZCd<ZDdS)@) dataclass)gcd)AnyDictListOptionalTupleUnionN)Tensornn) ConfigMixinregister_to_config) BaseOutputis_torch_versionlogging) apply_freeu)ADDED_KV_ATTENTION_PROCESSORSCROSS_ATTENTION_PROCESSORS AttentionAttentionProcessorAttnAddedKVProcessor AttnProcessor)ControlNetConditioningEmbedding)TimestepEmbedding Timesteps) ModelMixin)CrossAttnDownBlock2DCrossAttnUpBlock2D Downsample2D ResnetBlock2DTransformer2DModelUNetMidBlock2DCrossAttn Upsample2D)UNet2DConditionModelc"eZdZUdZdZeed<dS)ControlNetXSOutputa= The output of [`UNetControlNetXSModel`]. Args: sample (`Tensor` of shape `(batch_size, num_channels, height, width)`): The output of the `UNetControlNetXSModel`. Unlike `ControlNetOutput` this is NOT to be added to the base model output, but is already the final output. Nsample)__name__ __module__ __qualname____doc__r)r __annotations__n/root/.openclaw/workspace/chatterbox_venv_py311/lib/python3.11/site-packages/diffusers/models/controlnet_xs.pyr(r(3s.FFr0r(c eZdZdZ d dejdejdejdeejdeejf fd ZxZ S) DownBlockControlNetXSAdapterz}Components that together with corresponding components from the base model will form a `ControlNetXSCrossAttnDownBlock2D`Nresnets base_to_ctrl ctrl_to_base attentions downsamplerct||_||_||_||_||_dSN)super__init__r4r5r6r7 downsamplers)selfr4r5r6r7r8 __class__s r1r<z%DownBlockControlNetXSAdapter.__init__EsG  (($'r0)NN) r*r+r,r-r ModuleListrConv2dr< __classcell__r?s@r1r3r3As**/3+/ ( ( (m (m ( R]+ ( bi( ( ( ( ( ( ( ( ( ( (r0r3cDeZdZdZdedejdejffd ZxZS)MidBlockControlNetXSAdapterz|Components that together with corresponding components from the base model will form a `ControlNetXSCrossAttnMidBlock2D`midblockr5r6crt||_||_||_dSr:)r;r<rFr5r6)r>rFr5r6r?s r1r<z$MidBlockControlNetXSAdapter.__init__Ys6   ((r0) r*r+r,r-r$r r@r<rBrCs@r1rErEUsc)))!8) )eger))))))))))r0rEc2eZdZdZdejffd ZxZS)UpBlockControlNetXSAdapterzwComponents that together with corresponding components from the base model will form a `ControlNetXSCrossAttnUpBlock2D`r6cVt||_dSr:)r;r<r6)r>r6r?s r1r<z#UpBlockControlNetXSAdapter.__init__cs' (r0)r*r+r,r-r r@r<rBrCs@r1rIrI`sRBB)R]))))))))))r0rI TFbase_in_channelsbase_out_channelsctrl_in_channelsctrl_out_channels temb_channelsmax_norm_num_groupstransformer_layers_per_blocknum_attention_headscross_attention_dimadd_downsampleupcast_attentionc d} g} g}g}g}t|tr|g| z}t| D]}|dkr|n|}|dkr|n|}|t ||| t ||z||t ||z|t ||d|rB|t|||z|||| d| t |||t ||| r]|t ||t||zd|d }|t ||nd}ttj | tj |tj | }|rtj ||_ |||_ |S) Nr r max_factorh㈵> in_channels out_channelsrQgroups groups_outepsTr] num_layersrUuse_linear_projectionrWnorm_num_groupsopuse_convr^name)r4r5r6) isinstanceintrangeappendmake_zero_convr"find_largest_factorr#r!r3r r@r7r=)rMrNrOrPrQrR has_crossattnrSrTrUrVrWrcr4r7r6r5ir=down_block_componentss r1get_down_block_adapterrshs{JGJLL.44S(D'E 'R$ :  #R#R/0Avv++;L/0Avv++;L N+;=MNNOOO ,/??.+*+;>N+N[nooo./@M`aaa          "'%)<< 1;A>(;*.%5$78IVi$j$j$j       N+<>OPPQQQQ  N+<>OPPQQQ#  1 1DO`gk   N+<>OPPQQQQ 8 g&&]<00]<00 E+-=+D+D(-9*  r0 base_channels ctrl_channelsc t||}t|||z||tt|||z|||d| } t||} t || | S)NT rSr]r^rQ resnet_groupsrUrTrdrW)r5rFr6)rnr$rorrE) rtrurQrRrSrTrUrWr5rFr6s r1get_mid_block_adapterrys"-??L&%A!M1"#)#m]]=Z*[*[]pqq//")   H"-??L &L8bn o o oor0r^prev_output_channelctrl_skip_channelscg}d}t|D]5}|dkr|n|}|t|||6tt j|S)Nr)r6)rlrmrnrIr r@)r^rzr{r6rcrqresnet_in_channelss r1get_up_block_adapterrs~ LJ :  WW45FF00 N+=a+@BTUUVVVV %2=3N3N O O OOr0ceZdZdZe d$dededeedede de eeefdeedeededeede ede eeefde deffd Z e d%d ed!e ede eede eede dedededeefd"Zd#ZxZS)&ControlNetXSAdaptera A `ControlNetXSAdapter` model. To use it, pass it into a `UNetControlNetXSModel` (together with a `UNet2DConditionModel` base model). This model inherits from [`ModelMixin`] and [`ConfigMixin`]. Check the superclass documentation for it's generic methods implemented for all models (such as downloading or saving). Like `UNetControlNetXSModel`, `ControlNetXSAdapter` is compatible with StableDiffusion and StableDiffusion-XL. It's default parameters are compatible with StableDiffusion. Parameters: conditioning_channels (`int`, defaults to 3): Number of channels of conditioning input (e.g. an image) conditioning_channel_order (`str`, defaults to `"rgb"`): The channel order of conditional image. Will convert to `rgb` if it's `bgr`. conditioning_embedding_out_channels (`tuple[int]`, defaults to `(16, 32, 96, 256)`): The tuple of output channels for each block in the `controlnet_cond_embedding` layer. time_embedding_mix (`float`, defaults to 1.0): If 0, then only the control adapters's time embedding is used. If 1, then only the base unet's time embedding is used. Otherwise, both are combined. learn_time_embedding (`bool`, defaults to `False`): Whether a time embedding should be learned. If yes, `UNetControlNetXSModel` will combine the time embeddings of the base model and the control adapter. If no, `UNetControlNetXSModel` will use the base model's time embedding. num_attention_heads (`list[int]`, defaults to `[4]`): The number of attention heads. block_out_channels (`list[int]`, defaults to `[4, 8, 16, 16]`): The tuple of output channels for each block. base_block_out_channels (`list[int]`, defaults to `[320, 640, 1280, 1280]`): The tuple of output channels for each block in the base unet. cross_attention_dim (`int`, defaults to 1024): The dimension of the cross attention features. down_block_types (`list[str]`, defaults to `["CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D"]`): The tuple of downsample blocks to use. sample_size (`int`, defaults to 96): Height and width of input/output sample. transformer_layers_per_block (`Union[int, Tuple[int]]`, defaults to 1): The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. upcast_attention (`bool`, defaults to `True`): Whether the attention computation should always be upcasted. max_norm_num_groups (`int`, defaults to 32): Maximum number of groups in group normal. The actual number will the the largest divisor of the respective channels, that is <= max_norm_num_groups. r}rgbrK`?Frrri@irrLrrr DownBlock2DrrTrKconditioning_channelsconditioning_channel_order#conditioning_embedding_out_channelstime_embedding_mixlearn_time_embeddingrTblock_out_channelsbase_block_out_channelsrUdown_block_types sample_sizerSrWrRc\t|d}|ddz}|dvrtd|t|t| krtd|d| dt | t t fs| gt| z} t | t t fs| gt| z} t |t t fs|gt| z}t|t| krtd|d| dt|d|| |_|rt|||_ nd|_ tj g|_ tj g|_tjd|dd d |_t#|d|d|_|d}|d}t'| D]y\}}|}||}|}||}d |v}|t| d z k}|j t+|||||||| |||| || |  zt-|d|d|| d|d| d| |_|dgt'|D]8\}}|t|d z krd nd}|g|z9t t3|}|d}t5t| D]Q}|}||}fdt5d D}|jt7|||RdS)Nrr)rbgrz&unknown `conditioning_channel_order`: zbMust provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: z. `down_block_types`: .zdMust provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: conditioning_embedding_channelsrrr}r kernel_sizepadding CrossAttn) rMrNrOrPrQrRrprSrTrUrVrW)rtrurQrSrTrUrWr c8g|]}Sr/pop.0_r{s r1 z0ControlNetXSAdapter.__init__..&"N"N"N#5#9#9#;#;"N"N"Nr0)r^rzr{)r;r< ValueErrorlenrjlisttuplercontrolnet_cond_embeddingrtime_embeddingr r@ down_blocksup_connectionsrAconv_inrncontrol_to_base_for_conv_in enumeratermrsry mid_blockextendreversedrlr) r>rrrrrrTrrrUrrrSrWrRtime_embedding_input_dimtime_embedding_dimrNrPrqdown_block_typerMrOrpis_final_blockr^number_of_subblocks reversed_base_block_out_channelsprev_base_output_channelctrl_skip_channels_r{r?s @r1r<zControlNetXSAdapter.__init__!s. #:1#= 4Q7!; &^ ; ;bF`bbcc c ! " "c*:&;&; ; ;svHss`psss 6u FF b,H+ICP`LaLa+a (-e}== P#6"7#>N:O:O"O -e}== P#6"7#>N:O:O"O  " # #s+;'<'< < <vxKvvcsvvv  *I,>q,AB"7* * * &  '"34LN`"a"aD  "&D =,, mB//y$6q$9qRSTTT +9:LQ:OQhijQk+l+l(4A6.q1"+,<"="=   A0  7 : 0  21 5 '?:M#&6"7"7!";;N   # #&%5&7%5&7"4(;"/1Ma1P(;A(>(;A(>'5#5%5       $/1"5,R0,)Eb)I 3B 7 3B 7-   134();<< L LOA|/001444!   % %|n7J&J K K K K+/9P0Q0Q+R+R(.s%3p3p3pACJ4G4G3p3p3pr0)rrrrrrTrrrUrrrSrWrR) rconfigrattention_head_dimrUrrrSrWretodtype) clsrrrrTrrrrr fixed_size relative_sizemodels ` r1 from_unetzControlNetXSAdapter.from_unetsR(t3 "$. ]* w  0p3p3p3p3pQUQ\Qo3p3p3p  &"&+"@ "7'A0S1!5 31$(K$B $ ?![9 /)-)Q![9 $ ;   $  r0c td)NzA ControlNetXSAdapter cannot be run by itself. Use it together with a UNet2DConditionModel to instantiate a UNetControlNetXSModel.)r)r>argskwargss r1forwardzControlNetXSAdapter.forwards Q   r0)r}rrrFrrrrLrrrTrK)NNNFrr}rr)r*r+r,r-rrkstrr floatboolr rr< classmethodr&rrrrBrCs@r1rrs',,\&'*/:K$'%*67)7.D#'( &(?@!%#%)CC"C%(C.33Z C " C # C#3c ?3C"#JC"'sC!C *C"c]#C$',CsO&<%C&'C(!)CCCCCCJ'+2637%*"%%&*/:KII"IUOI%T#Y/ I &d3i0 I # I I #I%(I.33ZIII[IV       r0rc-eZdZdZdZe dJdeedee dee deedeede eeefde eeefde eeefdee deede deedeed e d!ed"eed#e d$e d%eed&e eeefd'ef*fd( Z e dKd)ed*eed+ee d%eee d ee d,eef d-ZdLd/ZdMd0Zed.ee effd1Zd2e eee efffd3Zd4Zd5e d6e d7e d8e fd9Zd:Zd;Zd<Z dNd=ed>e e je efd?e jd@ee jdAee dBee jdCee jdDee jdEeee e!fdFeee e jfdGe dHe d.e e"effdIZ#xZ$S)OUNetControlNetXSModela9 A UNet fused with a ControlNet-XS adapter model This model inherits from [`ModelMixin`] and [`ConfigMixin`]. Check the superclass documentation for it's generic methods implemented for all models (such as downloading or saving). `UNetControlNetXSModel` is compatible with StableDiffusion and StableDiffusion-XL. It's default parameters are compatible with StableDiffusion. It's parameters are either passed to the underlying `UNet2DConditionModel` or used exactly like in `ControlNetXSAdapter` . See their documentation for details. Trr UpBlock2Dr r r rrKrLrrNrr}rrFrrrrup_block_typesrrerUrSrTaddition_embed_typeaddition_time_embed_dimrWtime_cond_proj_dim%projection_class_embeddings_input_dimrctrl_conditioning_channels(ctrl_conditioning_embedding_out_channelsctrl_conditioning_channel_orderctrl_learn_time_embeddingctrl_block_out_channelsctrl_num_attention_headsctrl_max_norm_num_groupsc -t|dks|dkrtd|dkr|std| | dkrtdt|tt fs|gt |z}t|tt fs|gt |z}t|tt fs|gt |z}t|tt fs|gt |z}|}d|_tj d|ddd |_ t|d|| |_ tj d|ddd |_ t|d|d|_|d}|ddz}t!|dd d |_t%||| |_t%|||_| d|_d|_n,t!| d d |_t%| ||_g}|d}|d}t/|D]|\}}|}||}|}||}d|v} |t |dz k}!|t3|||||||| |||||||||! | }t5|d|d||||d|d|d|d|  |_g}"t t9|}#t t9|}$t t9|}%|dg-t/|D]8\}}&|t |dz krdnd}'-|&g|'z9t t9|}(|(d}&t/|D]\}})|&}*|(|}&|(t=|dzt |dz }+-fdt?dD},d|)v} |t |dz k}!|"tA|+|&|*|,||| |#||$||%||! | | tj!||_"tj!|"|_#tj$|d||_%tj&|_'tj |dddd |_(dS)Nrrz1`time_embedding_mix` needs to be between 0 and 1.zKTo use `time_embedding_mix` < 1, `ctrl_learn_time_embedding` must be `True` text_timezAs `UNetControlNetXSModel` currently only supports StableDiffusion and StableDiffusion-XL, `addition_embed_type` must be `None` or `'text_time'`.rr}rrT)flip_sin_to_cosdownscale_freq_shift) cond_proj_dim)r]time_embed_dimrrMrNrOrPrQrerrprSbase_num_attention_headsrrUrVrWr rtrurQrerrSrrrUrWr c8g|]}Sr/rrs r1rz2UNetControlNetXSModel.__init__..rr0) r]r^rzr{rQresolution_idxrprSrTrU add_upsamplerWre) num_channels num_groups))r;r<rrjrrrr]r rA base_conv_inrr ctrl_conv_inrnrrbase_time_projrbase_time_embeddingctrl_time_embeddingbase_add_time_projbase_add_embeddingrrm ControlNetXSCrossAttnDownBlock2DControlNetXSCrossAttnMidBlock2DrrrminrlControlNetXSCrossAttnUpBlock2Dr@r up_blocks GroupNormbase_conv_norm_outSiLU base_conv_act base_conv_out)/r>rrrrrerUrSrTrrrWrrrrrrrrrrrtime_embed_input_dimrrrNrPrqrrMrOrprr rev_transformer_layers_per_blockrev_num_attention_headsrev_cross_attention_dimr^rreversed_block_out_channels up_block_typerzr]rr{r?s/ @r1r<zUNetControlNetXSModel.__init__ s8@   ! !%7!%;%;PQQ Q  ! !*C !jkk k  */Bk/Q/Qd 6u FF b,H+ICP`LaLa+a (-e}== P#6"7#>N:O:O"O -e}== P#6"7#>N:O:O"O 2T5MBB Z(@'ACHXDYDY'Y $#6 Ia);A)>AWXYYY)H,CA,FG"<* * * & Ia)@)CQR\]^^^+9:QRS:TVhijVk+l+l( 2!4+A.2'(:1(=tjklll#4 ,$ $ $  $5AUft#u#u#u  &&*D #&*D # #&/0GY]tu&v&v&vD #&78]_m&n&nD # .q13A6"+,<"="=   A0  21 5 0  7 : '?:M#&6"7"7!";;N   0%5&7%5&7"0$3-E"/1Ma1P-Ea-H-Ea-H(;A(>'5#5%5    (9,R01"5(+%=)Eb)I%=b%A%=b%A 3B 7-     +/9U0V0V+W+W("&x0H'I'I"J"J"&x0C'D'D"E"E6a89()@AA L LOA|455999q   % %|n7J&J K K K K&*84F+G+G&H&H#215 ). 9 9   A}". 6q9L5c!a%EWAXAX[\A\6]6]^K"N"N"N"NU1XX"N"N"N '=8M#&8"9"9A"==N   . +!-(;':"0#$"/1QRS1T(?(B(?(B%3!3%5$3    $=55y11"$,z2UNetControlNetXSModel.from_unet..s7"# r0zWhen a controlnet is passed, none of these parameters should be passed: size_ratio, ctrl_block_out_channels, time_embedding_mix, ctrl_optional_kwargs.) rrrrrerUrSrrrWrrc$i|] \}}|v || Sr/r/)rkvparams_for_unets r1 z3UNetControlNetXSModel.from_unet..s)XXXDAq1CWCW1aCWCWCWr0rT)rrrrrrTrRc*i|]\}}|v d|z|S)ctrl_r/)rrrparams_for_controlnets r1rz3UNetControlNetXSModel.from_unet..s/ t t tDAqYZ^sYsYs1aYsYsYsr0r)rr conv_norm_outconv_outbase_ add_time_proj add_embeddingc3RK|]"\}}t||V#dSr:)r from_modulesrrcs r1rz2UNetControlNetXSModel.from_unet..&sJ* * 1 - 9 9!Q ? ?* * * * * * r0c3RK|]"\}}t||V#dSr:)rrr s r1rz2UNetControlNetXSModel.from_unet..+sJ( ( 1 + 7 71 = =( ( ( ( ( ( r0)rranyrritemsrr from_configgetattrload_state_dict state_dicthasattrrrrrrrr r@ziprrrrrrrr) rrr rrrr rmodules_from_unetmoptional_modules_from_unetrrs @@r1rzUNetControlNetXSModel.from_unets<  ,6j"9=QJJ(24KM_au'v !m    YXXXDK,=,=,?,?XXX151O-.! ! ! !u t t tJs r1rz.Gs3ddddGDRVDWDWDcgdD))DcDcDcr0FN) parameters requires_gradrfreeze_base_paramsrr)r>param base_partsr2dus` r1freeze_unet_paramsz(UNetControlNetXSModel.freeze_unet_params5s__&& ' 'E"&E      edddjddd  , ,D** , ,&+## ,! # #A " " " " ))+++ # #A " " " " # #r0c8t|dr ||_dSdS)Ngradient_checkpointing)r)r<)r>modulevalues r1_set_gradient_checkpointingz1UNetControlNetXSModel._set_gradient_checkpointingRs- 63 4 4 2,1F ) ) ) 2 2r0ci}dtdtjjdttt fffd |D]\}}||||S)z Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with indexed by its weight name. rir= processorsct|dr|d||d<|D]\}}|d||||S)N get_processorT)return_deprecated_lora .processorr)r)rCnamed_children)rir=rAsub_namechildfn_recursive_add_processorss r1rIzJUNetControlNetXSModel.attn_processors..fn_recursive_add_processorsasv// d282F2F^b2F2c2c d.../#)#8#8#:#: U U%++t,@,@h,@,@%TTTT r0)rtorchr ModulerrrF)r>rArir=rIs @r1attn_processorsz%UNetControlNetXSModel.attn_processorsVs  c 58? X\]`bt]tXu      !//11 B BLD& ' 'fj A A A Ar0 processorc zt|j}t|tr9t||kr&t dt|d|d|ddt dtjj ffd | D]\}}|||dS) a4 Sets the attention processor to use to compute attention. Parameters: processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): The instantiated processor class or a dictionary of processor classes that will be set as the processor for **all** `Attention` layers. If `processor` is a dict, the key needs to define the path to the corresponding cross attention processor. This is strongly recommended when setting trainable attention processors. z>A dict of processors was passed, but the number of processors z0 does not match the number of attention layers: z. Please make sure to pass z processor classes.rir=c,t|drVt|ts||n+|||d|D]\}}|d|||dS)N set_processorrEr)r)rjdictrPrrF)rir=rMrGrHfn_recursive_attn_processors r1rRzMUNetControlNetXSModel.set_attn_processor..fn_recursive_attn_processorsv// M!)T22M((3333(($7J7J7J)K)KLLL#)#8#8#:#: T T%++t,@,@h,@,@%SSSS T Tr0N) rrLkeysrjrQrrrJr rKrF)r>rMcountrir=rRs @r1set_attn_processorz(UNetControlNetXSModel.set_attn_processorpsD(--//00 i & & 3y>>U+B+BmQTU^Q_Q_mm05mmRWmmm   Tc T58? T T T T T T!//11 A ALD& ' 'fi @ @ @ @ A Ar0c td|jDrt}ntd|jDrt }nCt dt t|j||dS)ze Disables custom attention processors and sets the default attention implementation. c32K|]}|jtvVdSr:)r?rrprocs r1rzCUNetControlNetXSModel.set_default_attn_processor..s*ii4t~!>>iiiiiir0c32K|]}|jtvVdSr:)r?rrXs r1rzCUNetControlNetXSModel.set_default_attn_processor..s*hh$#==hhhhhhr0zOCannot call `set_default_attn_processor` when attention processors are of type N) allrLvaluesrrrnextiterrU)r>rMs r1set_default_attn_processorz0UNetControlNetXSModel.set_default_attn_processors ii4K_KfKfKhKhiii i i ,..II hh$J^JeJeJgJghhh h h %IINbfgklpmAmHmHmJmJhKhKcLcLNN   *****r0s1s2b1b2ct|jD]I\}}t|d|t|d|t|d|t|d|JdS)a>Enables the FreeU mechanism from https://arxiv.org/abs/2309.11497. The suffixes after the scaling factors represent the stage blocks where they are being applied. Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. Args: s1 (`float`): Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to mitigate the "oversmoothing effect" in the enhanced denoising process. s2 (`float`): Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to mitigate the "oversmoothing effect" in the enhanced denoising process. b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. r`rarbrcN)rrsetattr)r>r`rarbrcrqupsample_blocks r1 enable_freeuz"UNetControlNetXSModel.enable_freeusw$"+4>!:!: . . A~ ND" - - - ND" - - - ND" - - - ND" - - - -  . .r0chd}t|jD]<\}}|D]4}t||st||dt ||d5=dS)zDisables the FreeU mechanism.>rbrcr`raN)rrr)r&re)r> freeu_keysrqrfrs r1 disable_freeuz#UNetControlNetXSModel.disable_freeus--- !*4>!:!: 5 5 A~ 5 5>1--5D1Q1Q1]NAt444 5 5 5r0cBd|_|jD]/\}}dt|jjvrt d0|j|_|D]-}t|tr| d.dS)u1 Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, key, value) are fused. For cross-attention modules, key and value projection matrices are fused. This API is 🧪 experimental. NAddedzQ`fuse_qkv_projections()` is not supported for models having added KV projections.T)fuse) original_attn_processorsrLr$rr?r*rmodulesrjrfuse_projections)r>rattn_processorr=s r1fuse_qkv_projectionsz*UNetControlNetXSModel.fuse_qkv_projectionss)-%!%!5!;!;!=!= v v A~#n6?@@@@ !tuuuA)-(<%llnn 3 3F&),, 3''T'222 3 3r0cL|j||jdSdS)uDisables the fused QKV projection if enabled. This API is 🧪 experimental. N)rnrU)r>s r1unfuse_qkv_projectionsz,UNetControlNetXSModel.unfuse_qkv_projectionss2  ( 4  # #D$A B B B B B 5 4r0r)timestepencoder_hidden_statescontrolnet_condconditioning_scale class_labels timestep_condattention_maskcross_attention_kwargsadded_cond_kwargs return_dict apply_controlc  |jjdkrtj|dg}|5d||jz dz}|d}|} tj| sx|jj dk}t|tr|r tj n tj }n|r tjn tj}tj| g||j} n8t#| jdkr | d|j} | |jd} || }||j }|jjrL| rJ|||}|||}|jjd z}||z|d|z zz}n||}d}|jjn|jjd krd | vrt5|jd | d }d| vrt5|jd| d}||}||jddf}tj ||gd}||j}|!|}nt5d|jjd|||zn|}|}|x}}gg}}|"|}|#|}|$|}|||z }| r||%||zz}|&||&||j'D]D} | |||||| || \}}}!}"|(|!|(|"E|)|||||| || \}}|j*D]V}#t#|#j+}$||$ d}%||$ d}&|d|$ }|d|$ }|#||%|&|||| ||  }W|,|}|-|}|.|}| s|fSt_|S)ay The [`ControlNetXSModel`] forward method. Args: sample (`Tensor`): The noisy input tensor. timestep (`Union[torch.Tensor, float, int]`): The number of timesteps to denoise an input. encoder_hidden_states (`torch.Tensor`): The encoder hidden states. controlnet_cond (`Tensor`): The conditional input tensor of shape `(batch_size, sequence_length, hidden_size)`. conditioning_scale (`float`, defaults to `1.0`): How much the control model affects the base model outputs. class_labels (`torch.Tensor`, *optional*, defaults to `None`): Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings. timestep_cond (`torch.Tensor`, *optional*, defaults to `None`): Additional conditional embeddings for timestep. If provided, the embeddings will be summed with the timestep_embedding passed through the `self.time_embedding` layer to obtain the final timestep embeddings. attention_mask (`torch.Tensor`, *optional*, defaults to `None`): An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large negative values to the attention scores corresponding to "discard" tokens. cross_attention_kwargs (`dict[str]`, *optional*, defaults to `None`): A kwargs dictionary that if specified is passed along to the `AttnProcessor`. added_cond_kwargs (`dict`): Additional conditions for the Stable Diffusion XL UNet. return_dict (`bool`, defaults to `True`): Whether or not to return a [`~models.controlnet.ControlNetOutput`] instead of a plain tuple. apply_control (`bool`, defaults to `True`): If `False`, the input is run only through the base model. Returns: [`~models.controlnetxs.ControlNetXSOutput`] **or** `tuple`: If `return_dict` is `True`, a [`~models.controlnetxs.ControlNetXSOutput`] is returned, otherwise a tuple is returned where the first element is the sample tensor. rr)dimsNgmps)rdevicer)rg333333?r text_embedsz has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`time_idsz has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`rdimzgControlNet-XS currently only supports StableDiffusion and StableDiffusion-XL, so addition_embed_type = z is currently not supported.)hidden_states_basehidden_states_ctrltembrvrxr|r{r) hidden_statesres_hidden_states_tuple_baseres_hidden_states_tuple_ctrlrrvrxr|r{r)r))0rrrJfliprr unsqueeze is_tensorrtyperjrfloat32float64int32int64tensorrshapeexpandrrrrrrrr?getrflattenreshapeconcatrrrrrrmrrrrr4rrrr()'r>r)rurvrwrxryrzr{r|r}r~r timestepsis_mpsrt_emb ctrl_temb base_tembinterpolation_paramraug_embrr time_embeds add_embedscembh_ctrlh_basehs_basehs_ctrl guided_hintdown residual_hb residual_hcup n_resnetsskips_hbskips_hcs' r1rzUNetControlNetXSModel.forwardsn ; 6% ? ?#jsCCCO  %."3"3FL"A"AAXMN+55a88N y)) :]'50F(E** ?)/B U]'-> 5; i[fmTTTII  ! !Q & &!$**6=99I$$V\!_55 ##I.. v|,, ; 0 3] 300 FFI00 FFI"&+"@#"E 22Y!FYBY5ZZDD++E22D ; * 2  [ , ; ;$555 ~|||,// >>K!222 ~yyy),,Z88H11(2B2B2D2DEEK%--{/@/CR.HIIK{K&@bIIIJ#tz22J--j99GGxz~{F{Zxxx ")!4tg~~$%! r44_EE ""6**""6**  " k !F  \d>>vFFI[[[Fvv$ ( (D7;t#)#)&*#5'=-+ 8 8 8 4FFK NN; ' ' ' NN; ' ' ' '%%"&1#9)'(   .  BBJI z{{+H z{{+Hk zk*Gk zk*GR$-5-5&*#5'=-+   FF((00##F++##F++ 9 !0000r0)rrrrrKrLrrNNTNNrr}rrFrrrK)NNNNNr.N)F) NrNNNNNTT)%r*r+r,r- _supports_gradient_checkpointingrrrkr rr rrr<rr&rrrrr:r?propertyrrLrUr_rgrjrrrtr rJrr(rrBrCs@r1rrs  (,$&(( &u)?)+6:?@67-115!%,0?C$'*+?P/4*/.<;<(*;s[s[c]s[ * s[c s["#Js["#s[#3c ?3s[',CsO&<s[#3c ?3s[ &c]!s[""*##s[$%s[&%SM's[(08})s[,"-s[.%(/s[038*1s[2*-3s[4$(5s[6"'s7s[8#(U3Z"89s[:#&;s[s[s[s[s[s[j59&*9=.2/3ss"s01sUO s "*$u+!6 s %UO s'tnsss[sj####:2222c+=&=!>X0 AE2Dd3PbKbFc2c,d A A A AF + + + .u.%.U.....25553330 C C C$37.1/30415;??C "H1H1H1 eS01H1 %| H1 "%,/ H1 %UO H1u|,H1  -H1!.H1!)c3h 8H1$Del):$;<H1H1H1 !5( )H1H1H1H1H1H1H1H1r0rceZdZ d&dededed ed ed ed ed eeeeefdeedeedeededeeffd Ze de de fdZ d'dZ d(dededeedeedeed eed!eeeefd"eed#edeeeeed$feed$fffd%ZxZS))rrKTrrLFrMrNrOrPrQrerrSrrrUrVrWctg}g}g}g}g}g}d}t| tr| g|z} t |D]F}|dkr|n|}|dkr|n|}|t |||t|||||t||z||t||z|t||d|ru|t| || z|| || d|||t| || z|| || d|t|||t ||H| rz|t ||t|d|d |_ t||zd|d |_ |t ||nd|_ d|_ tj||_tj||_|rtj|ndg|z|_|rtj|ndg|z|_tj||_tj||_d |_dS) Nr rr]r^rQr_rYr[r\TrbrfrgF)r;r<rjrkrlrmrnr"ror#r!base_downsamplersctrl_downsamplersr r@ base_resnets ctrl_resnetsbase_attentionsctrl_attentionsr5r6r<)r>rMrNrOrPrQrerrprSrrrUrVrWrrrrr6r5rcrqr?s r1r<z)ControlNetXSCrossAttnDownBlock2D.__init__s"      2C 8 8 W,H+IJ+V (z""9 V9 VA3466//?P 3466//?P    /?AQ R R S S S    0!2"/*        03C C!2"/.(+;;H` 33DQijjj       &&&0)-EE$5#?#B,?.2)9(7       &&&0)-EE$5#?#B,?.2)9(;.get_first_cross_attention.#A&9!<B Br0rr7TFr)r4r]r^ time_emb_proj in_featuresnorm1rr)rr7rheadsrUrWr=rr'r(rrrrrr5r6)rrrrrMrNrOrPrQrctrl_num_groupsrprSrrrUrWrVrs r1rz-ControlNetXSCrossAttnDownBlock2D.from_modules+s C C C*1!4@*215B  "1 % 14D D +215B&.q1?K #+A.4? (039D >< 0 0 $ M+.~/H/K/^+_+_ ('@'@'P'P'V $'@'@'P'P'V $";";N"K"K"_ 88HHY  !M+/ ('+ $'+ $"& # '4D@-/-/'&%4')E%=%= 3)-   $ **>+A+L+L+N+NOOO **>+A+L+L+N+NOOO  Z  ! 1 1.2K2V2V2X2X Y Y Y  ! 1 1.2K2V2V2X2X Y Y Y  ^  # 3 3N4OPQ4R4]4]4_4_ ` ` `  # 3 3N4O4Z4Z4\4\ ] ] ] **>+F+Q+Q+S+STTT **>+F+Q+Q+S+STTT r0r.NcL|D] }d|_ |jg}t|jt jr||j|j||j|D] }|D] }d|_ !dSr0TNF) r3r4rrjrr r@rmrr>r6r7r2s r1r5z3ControlNetXSCrossAttnDownBlock2D.freeze_base_paramsjs__&& ' 'E"&E  '( d*BM : : 4   d2 3 3 3  ! -   d4 5 5 5 , ,D** , ,&+## , , ,r0rrrrvrrxr{r|encoder_attention_maskr.c |0|ddtd|} |} d} d} tt |j|j}tt |j|j}dd}t |||j |j D]4\\}}\}}}}| r!tj | || gd} |j rE|jr>tddrd d ini}tjjj||| |fi|} n || |} ||| ||||d d } | rq|j rE|jr>tddrd d ini}tjjj||| |fi|} n || |} ||| ||||d d } | r| || |zz} | | fz} | | fz} 6|j|j d }|j d }| r!tj | || gd} || } | r|| } | r| || |zz} | | fz} | | fz} | | | | fS)NscaleSPassing `scale` to `cross_attention_kwargs` is deprecated. `scale` will be ignored.r/cfd}|S)Nc"|diS|SNr~r/inputsr=r~s r1custom_forwardz_ControlNetXSCrossAttnDownBlock2D.forward..create_custom_forward..custom_forward**!66C{CCC!66?*r0r/r=r~rs`` r1create_custom_forwardzGControlNetXSCrossAttnDownBlock2D.forward..create_custom_forward* + + + + + + " !r0rr>=1.11.0 use_reentrantFrvr|r{rr~rrr:)rloggerwarningrr*rrrrr5r6rJcattrainingr<rutils checkpointrr)r>rrrvrrxr{r|rrrrbase_output_statesctrl_output_states base_blocks ctrl_blocksrb_resb_attnc_resc_attnb2cc2b ckpt_kwargss r1rz(ControlNetXSCrossAttnDownBlock2D.forward{s " -%))'488Dtuuu##3t0$2FGGHH 3t0$2FGGHH  " " " ";> d&79J; ; 8 @8 @ 6OUF_eVc3 AFCCKK#8a@@@} -!< -JZ[_aiJjJj.r.F.Fpr /:))%00" vt,,!*?+A#1+A %  = 1T%@ 1N^_cemNnNn2v?E2J2JtvK"[3>--e44& FF#U6400F%#V.C/E'5/E$) F C##f++0B"BB!3vi!? !3vi!?    ! -#B'C#B'C AFCCKK#8a@@@++F33F 8//77 C##f++0B"BB!3vi!? !3vi!? v13EEEr0) rKrKTrrrrLTFr)NNrNNNT)r*r+r,rkrr r rr<rrr3rr5r rrrrrrBrCs@r1rrsX "(*IJ2323-1#+0s,s,s,s, s,  s,  s,s,#&s,'/uS%*_/E&Fs,#+3-s,#+3-s,&c]s,s,#4.s,s,s,s,s,s,j<*><Pl<<<[<|,,,,*37/3.1+/;?37"oFoF"oFoF (/ oF %V, oF %UO oF!(oF!)c3h 8oF!) 0oFoF vvuVS[153EE FoFoFoFoFoFoFoFoFr0rcLeZdZ d"dededeed ed ed ed eed eedeedeffd Zedede fdZ d#dZ d$de de de dee dee deeeefdee dee d edee e ffd!ZxZS)%rNrKrrLFrtrurQrerrSrrrUrWc `tt|||_t ||||| |d| |_t |||z||t t|||z|| |d|  |_t|||_ d|_ dS)NT)rSr]rQrxrUrTrdrWrwF) r;r<rnr5r$ base_midblockror ctrl_midblockr6r<) r>rtrurQrerrSrrrUrWr?s r1r<z(ControlNetXSCrossAttnMidBlock2D.__init__s +=-HH4)E%') 3 8"&-    5)E% 5&'-M==#@AAC[!4 8"&-    "+=-HH&+###r0rrc $|j}|j}|j}d}|j}|j}t |jdj}|jdj j } |jdj j } |jdj j } ||j } ||j } ||j}||j}|||| | | || | || }|j||j||j||j||S)Nc<|jdjdjSrr)rFs r1rzOControlNetXSCrossAttnMidBlock2D.from_modules..get_first_cross_attention,s&q)J "*1-3> '/28C#<#<]#K#K#Q #<#<]#K#K#Q 77 FFZ44]CCT'''&%4)E%=%= 3-     **<+B+B+D+DEEE ++M,D,D,F,FGGG ++M,D,D,F,FGGG **<+B+B+D+DEEE r0r.c|D] }d|_ |jD] }d|_ dS)r0TFN)r3r4r)r>r6s r1r5z2ControlNetXSCrossAttnMidBlock2D.freeze_base_paramsPs]__&& ' 'E"&E  '2244 ( (E"'E   ( (r0rTrrrvrrxr|r{rrc R|0|ddtd|} |} |||||d} | r+tj| || gd} |j| fi| } | r)|j| fi| } | || |zz} | | fS)Nrr)rrvr{r|rrr) rrrrJrr5rrr6) r>rrrvrrxr|r{rrrr joint_argss r1rz'ControlNetXSCrossAttnMidBlock2D.forward[s " -%))'488Dtuuu##%:,&<&<     KY(9(9&(A(ABJJJF##F99j99  M'T'==*==Fd//77:LLLFv~r0)NrKrKrrrrLFr)NrNNNT)r*r+r,rkrrr<rr$rErr5r rrrrr rrBrCs@r1rrs (,!(*,-2323-1!&1,1,1,1, } 1,  1, #& 1,'*1,#+3-1,#+3-1,&c]1,1,1,1,1,1,1,f,.,3,,,[,\ ( ( ( ( 04.1;?+/37"""""" & " %V, " %UO "!)c3h 8"!("!) 0"" vv~ """"""""r0rcbeZdZ d'deded ed eed ed ed eedededededeffd Zede de fdZ d(dZ d)de dee dfdee dfde dee d eed!eeeefd"ee d#eed$ee d%ede fd&ZxZS)*rrKNTrrLFr]r^rzr{rQrerrSrTrUrrWc tg}g}g}d}||_| |_t | t r| g|z} t |D]}||dz kr|n|}|dkr|n|}|t||||t||z||||r3|t| || z|| || d| |tj ||_ |rtj |ndg|z|_tj ||_| rt!|d||_nd|_d|_||_dS) Nr}rrrTrb)rhr^F)r;r<has_cross_attentionrTrjrkrlrmrnr"r#r r@r4r7r6r% upsamplersr<r)r>r]r^rzr{rQrerrprSrTrUrrWr4r7r6rcrqres_skip_channelsr~r?s r1r<z'ControlNetXSCrossAttnUpBlock2D.__init__s    #0 #6 2C 8 8 W,H+IJ+V (z""  A01Z!^0C0C , 89Q!4!4L    /A!/DFX Y Y Z Z Z NN 25F F!-"/*     !!&+$(;;$0#?#B,?.2)9(7      }W-- 7D]"- 3334&S]J]M,77  #(S_```DOO"DO&+#,r0 base_upblock ctrl_upblockc|j}d}|jdj}|jdj|z }|jdj|z }d|D}|jdjj} |jdjj} |j} t|drRd} t|j dj } ||j }||j}||j}n d} d} d}d}d}|jdu}|||||| | | | | |||| }|j|j| r1|j |j |r7|j|jd|j||S) Nc<|jdjdjSrrrs r1rzNControlNetXSCrossAttnUpBlock2D.from_modules..get_first_cross_attentionrr0rrcg|] }|j Sr/)r])rr!s r1rz?ControlNetXSCrossAttnUpBlock2D.from_modules..sTTTq}TTTr0r7TF) r]r^rzr{rQrerrprSrTrUrrW)r6r4r^r]rrrrrr)rr7rrrUrWrr'r()rrrctrl_to_base_skip_connectionsrr^r]prev_output_channelsctrl_skip_channelssrQrrrprSrTrUrWrrs r1rz+ControlNetXSCrossAttnUpBlock2D.from_moduless((4(A% C C C$+A.; "*2.:\I +3A6B\QTT6STTT$,Q/=I !)!,2= %4 < . . $ M+.|/Fq/I/\+]+] (";";L"I"I"O ";";L"I"I"] 88FFW  !M+/ ("& "& # #.d: #% 42'&)')E 3 3%-   "  %%l&:&E&E&G&GHHH  S   , ,\-D-O-O-Q-Q R R R  V   , ,\-DQ-G-R-R-T-T U U U **+H+S+S+U+UVVV r0r.cL|D] }d|_ |jg}t|jt jr||j|j||j|D] }|D] }d|_ !dSr) r3r4r4rjr7r r@rmrrs r1r5z1ControlNetXSCrossAttnUpBlock2D.freeze_base_paramss__&& ' 'E"&E  l^ dor} 5 5 /   do . . . ? &   do . . . , ,D** , ,&+## , , ,r0rrr.rrrvrxr|r{ upsample_sizerrc $|0|ddtdtddo2tddo!tddotdddd} fd} t jjjt|t|D]\}}}}}| r||||zz }| ||\}}tj ||gd }j rEj r>td d rd dini}tjjj| |||fi|}n |||}||||||| dd}j|| }|S)Nrrr`rarbrccfd}|S)Nc"|diS|Srr/rs r1rz]ControlNetXSCrossAttnUpBlock2D.forward..create_custom_forward..custom_forward*rr0r/rs`` r1rzEControlNetXSCrossAttnUpBlock2D.forward..create_custom_forward)rr0c nr/tj||jjjjS||fS)N)r`rarbrc)rrr`rarbrc)r res_h_baseis_freeu_enabledr>s r1maybe_apply_freeu_to_subblockzMControlNetXSCrossAttnUpBlock2D.forward..maybe_apply_freeu_to_subblock2sO 1"'!wwww%j00r0rrrrrFrrr:)rrrr&r*r4r7r6rrJrrr<rrrr)r>rrrrrvrxr|r{r rrrrresnetattnrr res_h_ctrlrrs` @r1rz&ControlNetXSCrossAttnUpBlock2D.forwards= " -%))'488Dtuuu D$ % % *dD)) *dD)) *dD))  " " " " 1 1 1 1 1 1:= L O   1 2 2 1 2 2 : :   5FD#z: FZ3E!EE (E(EmU_(`(` %M:!I}j&AqIIIM} <!< r.s"!!!!!:::::::::::::::: AAAAAAAA9999999999++++++87777744444444&&&&&&:99999  H % %          (((((29(((()))))"))))))))))))*,EF)*)-',W!W!W!W!W! W!  W! "# W!#+5eCj+A"BW!"#W!"#W!W!tnW!W!W!W!z$()+())*)-"ppppC=p"# p #& p "# p"#pppppD P P PS  P P P PD D D D D *kD D D Ny 1y 1y 1y 1y 1J y 1y 1y 1xuFuFuFuFuFryuFuFuFp PPPPPbiPPPffffffRYfffRKKKK r0