From 19ce16db0597979db4568b014908112ef89c81f1 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Thu, 15 Aug 2024 21:06:19 -0400 Subject: [PATCH 01/20] first attempt at updating to photomaker v2 --- pmid.hpp | 241 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) diff --git a/pmid.hpp b/pmid.hpp index d1d8c3192..330402529 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -42,6 +42,247 @@ struct FuseBlock : public GGMLBlock { } }; +/* +class QFormerPerceiver(nn.Module): + def __init__(self, id_embeddings_dim, cross_attention_dim, num_tokens, embedding_dim=1024, use_residual=True, ratio=4): + super().__init__() + + self.num_tokens = num_tokens + self.cross_attention_dim = cross_attention_dim + self.use_residual = use_residual + print(cross_attention_dim*num_tokens) + self.token_proj = nn.Sequential( + nn.Linear(id_embeddings_dim, id_embeddings_dim*ratio), + nn.GELU(), + nn.Linear(id_embeddings_dim*ratio, cross_attention_dim*num_tokens), + ) + self.token_norm = nn.LayerNorm(cross_attention_dim) + self.perceiver_resampler = FacePerceiverResampler( + dim=cross_attention_dim, + depth=4, + dim_head=128, + heads=cross_attention_dim // 128, + embedding_dim=embedding_dim, + output_dim=cross_attention_dim, + ff_mult=4, + ) + + def forward(self, x, last_hidden_state): + x = self.token_proj(x) + x = x.reshape(-1, self.num_tokens, self.cross_attention_dim) + x = self.token_norm(x) # cls token + out = self.perceiver_resampler(x, last_hidden_state) # retrieve from patch tokens + if self.use_residual: # TODO: if use_residual is not true + out = x + 1.0 * out + return out +*/ + +struct QFormerPerceiver : public GGMLBlock { + // network hparams + int num_tokens; + int cross_attention_dim; + bool use_residul; + + +public: + QFormerPerceiver(int id_embeddings_dim, int cross_attention_d, int num_t, int embedding_dim=1024, + bool use_r=true, int ratio=4) + : cross_attention_dim(cross_attention_d), num_tokens(num_t), use_residul(use_r) { + // blocks["mlp1"] = std::shared_ptr(new FuseBlock(imb_d * 2, imb_d, imb_d, false)); + // blocks["mlp2"] = std::shared_ptr(new FuseBlock(imb_d, imb_d, imb_d, true)); + // blocks["layer_norm"] = std::shared_ptr(new LayerNorm(embed_dim)); + blocks["token_proj"] = std::shared_ptr(new Mlp(id_embeddings_dim, + id_embeddings_dim*ratio, + cross_attention_dim*num_tokens, + true)); + blocks["token_norm"] = std::shared_ptr(new LayerNorm(cross_attention_d)); + } + + + + struct ggml_tensor* forward(struct ggml_context* ctx, + struct ggml_tensor* x, + struct ggml_tensor* last_hidden_state){ + // x: [N, channels, h, w] + auto mlp1 = std::dynamic_pointer_cast(blocks["mlp1"]); + auto mlp2 = std::dynamic_pointer_cast(blocks["mlp2"]); + auto token_norm = std::dynamic_pointer_cast(blocks["token_norm"]); + + + } +}; + +/* +class FacePerceiverResampler(torch.nn.Module): + def __init__( + self, + *, + dim=768, + depth=4, + dim_head=64, + heads=16, + embedding_dim=1280, + output_dim=768, + ff_mult=4, + ): + super().__init__() + + self.proj_in = torch.nn.Linear(embedding_dim, dim) + self.proj_out = torch.nn.Linear(dim, output_dim) + self.norm_out = torch.nn.LayerNorm(output_dim) + self.layers = torch.nn.ModuleList([]) + for _ in range(depth): + self.layers.append( + torch.nn.ModuleList( + [ + PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads), + FeedForward(dim=dim, mult=ff_mult), + ] + ) + ) + + def forward(self, latents, x): + x = self.proj_in(x) + for attn, ff in self.layers: + latents = attn(x, latents) + latents + latents = ff(latents) + latents + latents = self.proj_out(latents) + return self.norm_out(latents) +*/ + +struct FacePerceiverResampler : public GGMLBlock { + // network hparams +public: + FacePerceiverResampler( int dim=768, + int depth=4, + int dim_head=64, + int heads=16, + int embedding_dim=1280, + int output_dim=768, + int ff_mult=4) { + blocks["proj_in"] = std::shared_ptr(new Linear(embedding_dim, dim, true)); + blocks["proj_out"] = std::shared_ptr(new Linear(dim, output_dim, true)); + blocks["norm_out"] = std::shared_ptr(new LayerNorm(output_dim)); + } + + + + struct ggml_tensor* forward(struct ggml_context* ctx, + struct ggml_tensor* x, + struct ggml_tensor* last_hidden_state){ + // x: [N, channels, h, w] + auto mlp1 = std::dynamic_pointer_cast(blocks["mlp1"]); + auto mlp2 = std::dynamic_pointer_cast(blocks["mlp2"]); + auto token_norm = std::dynamic_pointer_cast(blocks["token_norm"]); + + + } +}; + +/* + +def reshape_tensor(x, heads): + bs, length, width = x.shape + # (bs, length, width) --> (bs, length, n_heads, dim_per_head) + x = x.view(bs, length, heads, -1) + # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head) + x = x.transpose(1, 2) + # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head) + x = x.reshape(bs, heads, length, -1) + return x + +class PerceiverAttention(nn.Module): + def __init__(self, *, dim, dim_head=64, heads=8): + super().__init__() + self.scale = dim_head**-0.5 + self.dim_head = dim_head + self.heads = heads + inner_dim = dim_head * heads + + self.norm1 = nn.LayerNorm(dim) + self.norm2 = nn.LayerNorm(dim) + + self.to_q = nn.Linear(dim, inner_dim, bias=False) + self.to_kv = nn.Linear(dim, inner_dim * 2, bias=False) + self.to_out = nn.Linear(inner_dim, dim, bias=False) + + def forward(self, x, latents): + """ + Args: + x (torch.Tensor): image features + shape (b, n1, D) + latent (torch.Tensor): latent features + shape (b, n2, D) + """ + x = self.norm1(x) + latents = self.norm2(latents) + + b, l, _ = latents.shape + + q = self.to_q(latents) + kv_input = torch.cat((x, latents), dim=-2) + k, v = self.to_kv(kv_input).chunk(2, dim=-1) + + q = reshape_tensor(q, self.heads) + k = reshape_tensor(k, self.heads) + v = reshape_tensor(v, self.heads) + + # attention + scale = 1 / math.sqrt(math.sqrt(self.dim_head)) + weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards + weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype) + out = weight @ v + + out = out.permute(0, 2, 1, 3).reshape(b, l, -1) + + return self.to_out(out) + +*/ + +struct PerceiverAttention : public GGMLBlock { + // network hparams + float scale; // = dim_head**-0.5 + int dim_head; // = dim_head + int heads; // = heads +public: + PerceiverAttention(int dim, int dim_h=64, int h=8) + : scale(powf(dim_h, -0.5)), dim_head(dim_h), heads(h) { + + int inner_dim = dim_head * heads; + blocks["norm1"] = std::shared_ptr(new LayerNorm(dim)); + blocks["norm2"] = std::shared_ptr(new LayerNorm(dim)); + blocks["to_q"] = std::shared_ptr(new Linear(dim, inner_dim, false)); + blocks["to_kv"] = std::shared_ptr(new Linear(dim, inner_dim*2, false)); + blocks["to_out"] = std::shared_ptr(new Linear(inner_dim, dim, false)); + } + + struct ggml_tensor* forward(struct ggml_context* ctx, + struct ggml_tensor* x, + struct ggml_tensor* latents){ + + // x (torch.Tensor): image features + // shape (b, n1, D) + // latent (torch.Tensor): latent features + // shape (b, n2, D) + auto norm1 = std::dynamic_pointer_cast(blocks["norm1"]); + auto norm2 = std::dynamic_pointer_cast(blocks["norm2"]); + x = norm1->forward(ctx, x); + latents = norm2->forward(ctx, latents); + auto to_q = std::dynamic_pointer_cast(blocks["to_q"]); + auto q = to_q->forward(ctx, latents); + + auto kv_input = ggml_concat(ctx, x, latents, 1); + auto to_kv = std::dynamic_pointer_cast(blocks["to_kv"]); + auto kv = to_kv->forward(ctx, kv_input); + int64_t n = kv->ne[2] / 2; + int64_t offset = kv->nb[2] * n; + auto k = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*0); + auto v = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*1); + + } +}; + + struct FuseModule : public GGMLBlock { // network hparams int embed_dim; From ccbf073b0b6a932a4d2914751d8ed1058fdcad4f Mon Sep 17 00:00:00 2001 From: bssrdf Date: Fri, 16 Aug 2024 17:28:36 -0400 Subject: [PATCH 02/20] continue adding photomaker v2 modules --- pmid.hpp | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 148 insertions(+), 10 deletions(-) diff --git a/pmid.hpp b/pmid.hpp index 330402529..5d4ce5447 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -6,6 +6,11 @@ #include "clip.hpp" #include "lora.hpp" + +enum PMVersion { + VERSION_1, + VERSION_2, +}; struct FuseBlock : public GGMLBlock { // network hparams int in_dim; @@ -181,6 +186,15 @@ struct FacePerceiverResampler : public GGMLBlock { /* +def FeedForward(dim, mult=4): + inner_dim = int(dim * mult) + return nn.Sequential( + nn.LayerNorm(dim), + nn.Linear(dim, inner_dim, bias=False), + nn.GELU(), + nn.Linear(inner_dim, dim, bias=False), + ) + def reshape_tensor(x, heads): bs, length, width = x.shape # (bs, length, width) --> (bs, length, n_heads, dim_per_head) @@ -239,6 +253,34 @@ class PerceiverAttention(nn.Module): */ +struct FeedForward : public GGMLBlock { + // network hparams + int dim; + +public: + FeedForward(int d, int multi=4) + : dim(d) { + int inner_dim = dim * multi; + blocks["norm"] = std::shared_ptr(new LayerNorm(dim)); + blocks["ff"] = std::shared_ptr(new Mlp(dim, inner_dim, dim, false)); + } + + struct ggml_tensor* forward(struct ggml_context* ctx, + struct ggml_tensor* x){ + + auto norm = std::dynamic_pointer_cast(blocks["norm"]); + auto ff = std::dynamic_pointer_cast(blocks["ff"]); + + x = norm->forward(ctx, x); + x = ff->forward(ctx, x); + return x; + } + +}; + + + + struct PerceiverAttention : public GGMLBlock { // network hparams float scale; // = dim_head**-0.5 @@ -256,6 +298,20 @@ struct PerceiverAttention : public GGMLBlock { blocks["to_out"] = std::shared_ptr(new Linear(inner_dim, dim, false)); } + struct ggml_tensor* reshape_tensor(struct ggml_context* ctx, + struct ggml_tensor* x, + int heads) { + int64_t ne[4]; + for(int i = 0; i < 4; ++i) + ne[i] = x->ne[i]; + x = ggml_view_4d(ctx, x, x->ne[0], x->ne[1], heads, x->ne[2]/heads, + x->nb[1], x->nb[2], x->nb[3], 0); + // x = ggml_transpose(ctx, x); + x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); + x = ggml_reshape_4d(ctx, x, ne[0], heads, ne[1], ne[2]/heads); + return x; + } + struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x, struct ggml_tensor* latents){ @@ -264,6 +320,10 @@ struct PerceiverAttention : public GGMLBlock { // shape (b, n1, D) // latent (torch.Tensor): latent features // shape (b, n2, D) + int64_t ne[4]; + for(int i = 0; i < 4; ++i) + ne[i] = latents->ne[i]; + auto norm1 = std::dynamic_pointer_cast(blocks["norm1"]); auto norm2 = std::dynamic_pointer_cast(blocks["norm2"]); x = norm1->forward(ctx, x); @@ -278,7 +338,23 @@ struct PerceiverAttention : public GGMLBlock { int64_t offset = kv->nb[2] * n; auto k = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*0); auto v = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*1); - + q = reshape_tensor(ctx, q, heads); + k = reshape_tensor(ctx, k, heads); + v = reshape_tensor(ctx, v, heads); + + scale = 1.f / sqrt(sqrt((float)dim_head)); + k = ggml_scale_inplace(ctx, k, scale); + k = ggml_cont(ctx, ggml_permute(ctx, k, 0, 1, 3, 2)); + q = ggml_scale_inplace(ctx, q, scale); + auto weight = ggml_mul_mat(ctx, q, k); + weight = ggml_soft_max(ctx, weight); + auto out = ggml_mul_mat(ctx, weight, v); + + out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); + out = ggml_reshape_3d(ctx, out, ne[0], ne[1], ggml_nelements(out)/(ne[0]*ne[1])); + auto to_out = std::dynamic_pointer_cast(blocks["to_out"]); + out = to_out->forward(ctx, out); + return out; } }; @@ -400,10 +476,52 @@ struct PhotoMakerIDEncoderBlock : public CLIPVisionModelProjection { } }; +struct PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock : public CLIPVisionModelProjection { + PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock() + : CLIPVisionModelProjection(OPENAI_CLIP_VIT_L_14) { + blocks["visual_projection_2"] = std::shared_ptr(new Linear(1024, 1280, false)); + blocks["fuse_module"] = std::shared_ptr(new FuseModule(2048)); + } + + struct ggml_tensor* forward(struct ggml_context* ctx, + struct ggml_tensor* id_pixel_values, + struct ggml_tensor* prompt_embeds, + struct ggml_tensor* class_tokens_mask, + struct ggml_tensor* class_tokens_mask_pos, + struct ggml_tensor* left, + struct ggml_tensor* right) { + // x: [N, channels, h, w] + auto vision_model = std::dynamic_pointer_cast(blocks["vision_model"]); + auto visual_projection = std::dynamic_pointer_cast(blocks["visual_projection"]); + auto visual_projection_2 = std::dynamic_pointer_cast(blocks["visual_projection_2"]); + auto fuse_module = std::dynamic_pointer_cast(blocks["fuse_module"]); + + struct ggml_tensor* shared_id_embeds = vision_model->forward(ctx, id_pixel_values); // [N, hidden_size] + struct ggml_tensor* id_embeds = visual_projection->forward(ctx, shared_id_embeds); // [N, proj_dim(768)] + struct ggml_tensor* id_embeds_2 = visual_projection_2->forward(ctx, shared_id_embeds); // [N, 1280] + + id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3)); + id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3)); + + id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2, 2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right + id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3)); + + struct ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx, + prompt_embeds, + id_embeds, + class_tokens_mask, + class_tokens_mask_pos, + left, right); + return updated_prompt_embeds; + } +}; + struct PhotoMakerIDEncoder : public GGMLRunner { public: SDVersion version = VERSION_XL; + PMVersion pm_version = VERSION_1; PhotoMakerIDEncoderBlock id_encoder; + PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock id_encoder2; float style_strength; std::vector ctm; @@ -416,11 +534,17 @@ struct PhotoMakerIDEncoder : public GGMLRunner { std::vector zeros_right; public: - PhotoMakerIDEncoder(ggml_backend_t backend, ggml_type wtype, SDVersion version = VERSION_XL, float sty = 20.f) + PhotoMakerIDEncoder(ggml_backend_t backend, ggml_type wtype, SDVersion version = VERSION_XL, + PMVersion pm_v = VERSION_1, float sty = 20.f) : GGMLRunner(backend, wtype), version(version), + pm_version(pm_v), style_strength(sty) { - id_encoder.init(params_ctx, wtype); + if(pm_version == VERSION_1){ + id_encoder.init(params_ctx, wtype); + }else if(pm_version == VERSION_2){ + id_encoder2.init(params_ctx, wtype); + } } std::string get_desc() { @@ -428,7 +552,11 @@ struct PhotoMakerIDEncoder : public GGMLRunner { } void get_param_tensors(std::map& tensors, const std::string prefix) { - id_encoder.get_param_tensors(tensors, prefix); + if(pm_version == VERSION_1) + id_encoder.get_param_tensors(tensors, prefix); + else if(pm_version == VERSION_2) + id_encoder2.get_param_tensors(tensors, prefix); + } struct ggml_cgraph* build_graph( // struct ggml_allocr* allocr, @@ -506,12 +634,22 @@ struct PhotoMakerIDEncoder : public GGMLRunner { } } } - struct ggml_tensor* updated_prompt_embeds = id_encoder.forward(ctx0, - id_pixel_values_d, - prompt_embeds_d, - class_tokens_mask_d, - class_tokens_mask_pos, - left, right); + struct ggml_tensor* updated_prompt_embeds = NULL; + if(pm_version == VERSION_1) + updated_prompt_embeds = id_encoder.forward(ctx0, + id_pixel_values_d, + prompt_embeds_d, + class_tokens_mask_d, + class_tokens_mask_pos, + left, right); + else if(pm_version == VERSION_2) + updated_prompt_embeds = id_encoder2.forward(ctx0, + id_pixel_values_d, + prompt_embeds_d, + class_tokens_mask_d, + class_tokens_mask_pos, + left, right); + ggml_build_forward_expand(gf, updated_prompt_embeds); return gf; From a5357cbcd6269bd9f79982013c71c0ff76441b2f Mon Sep 17 00:00:00 2001 From: bssrdf Date: Sun, 18 Aug 2024 17:04:55 -0400 Subject: [PATCH 03/20] finishing the last few pieces for photomaker v2; id_embeds need to be done by a manual step and pass as an input file --- pmid.hpp | 115 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 88 insertions(+), 27 deletions(-) diff --git a/pmid.hpp b/pmid.hpp index 5d4ce5447..9d6dfe0af 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -157,30 +157,52 @@ class FacePerceiverResampler(torch.nn.Module): struct FacePerceiverResampler : public GGMLBlock { // network hparams + int depth; public: FacePerceiverResampler( int dim=768, - int depth=4, + int d=4, int dim_head=64, int heads=16, int embedding_dim=1280, int output_dim=768, - int ff_mult=4) { + int ff_mult=4) + : depth(d) { blocks["proj_in"] = std::shared_ptr(new Linear(embedding_dim, dim, true)); blocks["proj_out"] = std::shared_ptr(new Linear(dim, output_dim, true)); blocks["norm_out"] = std::shared_ptr(new LayerNorm(output_dim)); + + for (int i = 0; i < depth; i++) { + std::string name = "layers." + std::to_string(i) + ".0"; + blocks[name] = std::shared_ptr(new PerceiverAttention(dim, dim_head, heads)); + name = "layers." + std::to_string(i) + ".1"; + blocks[name] = std::shared_ptr(new FeedForward(dim, ff_mult)); + } } struct ggml_tensor* forward(struct ggml_context* ctx, - struct ggml_tensor* x, - struct ggml_tensor* last_hidden_state){ + struct ggml_tensor* latents, + struct ggml_tensor* x){ // x: [N, channels, h, w] - auto mlp1 = std::dynamic_pointer_cast(blocks["mlp1"]); - auto mlp2 = std::dynamic_pointer_cast(blocks["mlp2"]); - auto token_norm = std::dynamic_pointer_cast(blocks["token_norm"]); - - + auto proj_in = std::dynamic_pointer_cast(blocks["proj_in"]); + auto proj_out = std::dynamic_pointer_cast(blocks["proj_out"]); + auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); + + x = proj_in->forward(ctx, x); + for (int i = 0; i < depth; i++) { + std::string name = "layers." + std::to_string(i) + ".0"; + auto attn = std::dynamic_pointer_cast(blocks[name]); + name = "layers." + std::to_string(i) + ".1"; + auto ff = std::dynamic_pointer_cast(blocks[name]); + auto t = attn->forward(ctx, x, latents); + latents = ggml_add(ctx, t, latents); + t = ff->forward(ctx, latents); + latents = ggml_add(ctx, t, latents); + } + latents = proj_out->forward(ctx, latents); + latents = norm_out->forward(ctx, latents); + return latents; } }; @@ -261,15 +283,15 @@ struct FeedForward : public GGMLBlock { FeedForward(int d, int multi=4) : dim(d) { int inner_dim = dim * multi; - blocks["norm"] = std::shared_ptr(new LayerNorm(dim)); - blocks["ff"] = std::shared_ptr(new Mlp(dim, inner_dim, dim, false)); + blocks["0"] = std::shared_ptr(new LayerNorm(dim)); + blocks["1"] = std::shared_ptr(new Mlp(dim, inner_dim, dim, false)); } struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x){ - auto norm = std::dynamic_pointer_cast(blocks["norm"]); - auto ff = std::dynamic_pointer_cast(blocks["ff"]); + auto norm = std::dynamic_pointer_cast(blocks["0"]); + auto ff = std::dynamic_pointer_cast(blocks["1"]); x = norm->forward(ctx, x); x = ff->forward(ctx, x); @@ -347,6 +369,11 @@ struct PerceiverAttention : public GGMLBlock { k = ggml_cont(ctx, ggml_permute(ctx, k, 0, 1, 3, 2)); q = ggml_scale_inplace(ctx, q, scale); auto weight = ggml_mul_mat(ctx, q, k); + + // GGML's softmax() is equivalent to pytorch's softmax(x, dim=-1) + // in this case, dimension along which Softmax will be computed is the last dim + // in torch and the first dim in GGML, consistent with the convention that pytorch's + // last dimension (varying most rapidly) corresponds to GGML's first (varying most rapidly). weight = ggml_soft_max(ctx, weight); auto out = ggml_mul_mat(ctx, weight, v); @@ -477,34 +504,65 @@ struct PhotoMakerIDEncoderBlock : public CLIPVisionModelProjection { }; struct PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock : public CLIPVisionModelProjection { - PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock() - : CLIPVisionModelProjection(OPENAI_CLIP_VIT_L_14) { + + int cross_attention_dim; + int num_tokens; + + PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock(int id_embeddings_dim=512) + : CLIPVisionModelProjection(OPENAI_CLIP_VIT_L_14), + cross_attention_dim (2048), + num_tokens(2) { blocks["visual_projection_2"] = std::shared_ptr(new Linear(1024, 1280, false)); blocks["fuse_module"] = std::shared_ptr(new FuseModule(2048)); + /* + cross_attention_dim = 2048 + # projection + self.num_tokens = 2 + self.cross_attention_dim = cross_attention_dim + self.qformer_perceiver = QFormerPerceiver( + id_embeddings_dim, + cross_attention_dim, + self.num_tokens, + )*/ + blocks["qformer_perceiver"] = std::shared_ptr(new QFormerPerceiver(id_embeddings_dim, + cross_attention_dim, + num_tokens)); + } + /* + def forward(self, id_pixel_values, prompt_embeds, class_tokens_mask, id_embeds): + b, num_inputs, c, h, w = id_pixel_values.shape + id_pixel_values = id_pixel_values.view(b * num_inputs, c, h, w) + + last_hidden_state = self.vision_model(id_pixel_values)[0] + id_embeds = id_embeds.view(b * num_inputs, -1) + + id_embeds = self.qformer_perceiver(id_embeds, last_hidden_state) + id_embeds = id_embeds.view(b, num_inputs, self.num_tokens, -1) + updated_prompt_embeds = self.fuse_module(prompt_embeds, id_embeds, class_tokens_mask) + */ + struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* id_pixel_values, struct ggml_tensor* prompt_embeds, struct ggml_tensor* class_tokens_mask, struct ggml_tensor* class_tokens_mask_pos, + struct ggml_tensor* id_embeds, struct ggml_tensor* left, struct ggml_tensor* right) { // x: [N, channels, h, w] - auto vision_model = std::dynamic_pointer_cast(blocks["vision_model"]); - auto visual_projection = std::dynamic_pointer_cast(blocks["visual_projection"]); - auto visual_projection_2 = std::dynamic_pointer_cast(blocks["visual_projection_2"]); + auto vision_model = std::dynamic_pointer_cast(blocks["vision_model"]); auto fuse_module = std::dynamic_pointer_cast(blocks["fuse_module"]); + auto qformer_perceiver = std::dynamic_pointer_cast(blocks["qformer_perceiver"]); - struct ggml_tensor* shared_id_embeds = vision_model->forward(ctx, id_pixel_values); // [N, hidden_size] - struct ggml_tensor* id_embeds = visual_projection->forward(ctx, shared_id_embeds); // [N, proj_dim(768)] - struct ggml_tensor* id_embeds_2 = visual_projection_2->forward(ctx, shared_id_embeds); // [N, 1280] + struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values); // [N, hidden_size] - id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3)); - id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3)); + id_embeds = qformer_perceiver->forward(ctx, id_embeds, last_hidden_state); + // id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3)); - id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2, 2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right - id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3)); + // id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2, 2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right + // id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3)); struct ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx, prompt_embeds, @@ -562,7 +620,8 @@ struct PhotoMakerIDEncoder : public GGMLRunner { struct ggml_cgraph* build_graph( // struct ggml_allocr* allocr, struct ggml_tensor* id_pixel_values, struct ggml_tensor* prompt_embeds, - std::vector& class_tokens_mask) { + std::vector& class_tokens_mask, + struct ggml_tensor* id_embeds) { ctm.clear(); ctmf16.clear(); ctmpos.clear(); @@ -648,6 +707,7 @@ struct PhotoMakerIDEncoder : public GGMLRunner { prompt_embeds_d, class_tokens_mask_d, class_tokens_mask_pos, + id_embeds, left, right); ggml_build_forward_expand(gf, updated_prompt_embeds); @@ -658,12 +718,13 @@ struct PhotoMakerIDEncoder : public GGMLRunner { void compute(const int n_threads, struct ggml_tensor* id_pixel_values, struct ggml_tensor* prompt_embeds, + struct ggml_tensor* id_embeds, std::vector& class_tokens_mask, struct ggml_tensor** updated_prompt_embeds, ggml_context* output_ctx) { auto get_graph = [&]() -> struct ggml_cgraph* { // return build_graph(compute_allocr, id_pixel_values, prompt_embeds, class_tokens_mask); - return build_graph(id_pixel_values, prompt_embeds, class_tokens_mask); + return build_graph(id_pixel_values, prompt_embeds, class_tokens_mask, id_embeds); }; // GGMLRunner::compute(get_graph, n_threads, updated_prompt_embeds); From 8559ae718c28414f971072137f9bde3449b48496 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Mon, 19 Aug 2024 11:24:32 -0400 Subject: [PATCH 04/20] added a name converter for Photomaker V2; build ok --- model.cpp | 36 +++++ pmid.hpp | 348 +++++++++++++++++++++++-------------------- stable-diffusion.cpp | 9 +- 3 files changed, 229 insertions(+), 164 deletions(-) diff --git a/model.cpp b/model.cpp index 7ab2287bc..b931f03a5 100644 --- a/model.cpp +++ b/model.cpp @@ -142,6 +142,33 @@ std::unordered_map vae_decoder_name_map = { {"first_stage_model.decoder.mid.attn_1.to_v.weight", "first_stage_model.decoder.mid.attn_1.v.weight"}, }; +std::unordered_map pmid_v2_name_map = { + {"pmid.qformer_perceiver.perceiver_resampler.layers.0.1.1.weight", + "pmid.qformer_perceiver.perceiver_resampler.layers.0.1.1.fc1.weight"}, + {"pmid.qformer_perceiver.perceiver_resampler.layers.0.1.3.weight", + "pmid.qformer_perceiver.perceiver_resampler.layers.0.1.1.fc2.weight"}, + {"pmid.qformer_perceiver.perceiver_resampler.layers.1.1.1.weight", + "pmid.qformer_perceiver.perceiver_resampler.layers.1.1.1.fc1.weight"}, + {"pmid.qformer_perceiver.perceiver_resampler.layers.1.1.3.weight", + "pmid.qformer_perceiver.perceiver_resampler.layers.1.1.1.fc2.weight"}, + {"pmid.qformer_perceiver.perceiver_resampler.layers.2.1.1.weight", + "pmid.qformer_perceiver.perceiver_resampler.layers.2.1.1.fc1.weight"}, + {"pmid.qformer_perceiver.perceiver_resampler.layers.2.1.3.weight", + "pmid.qformer_perceiver.perceiver_resampler.layers.2.1.1.fc2.weight"}, + {"pmid.qformer_perceiver.perceiver_resampler.layers.3.1.1.weight", + "pmid.qformer_perceiver.perceiver_resampler.layers.3.1.1.fc1.weight"}, + {"pmid.qformer_perceiver.perceiver_resampler.layers.3.1.3.weight", + "pmid.qformer_perceiver.perceiver_resampler.layers.3.1.1.fc2.weight"}, + {"pmid.qformer_perceiver.token_proj.0.bias", + "pmid.qformer_perceiver.token_proj.fc1.bias"}, + {"pmid.qformer_perceiver.token_proj.2.bias", + "pmid.qformer_perceiver.token_proj.fc2.bias"}, + {"pmid.qformer_perceiver.token_proj.0.weight", + "pmid.qformer_perceiver.token_proj.fc1.weight"}, + {"pmid.qformer_perceiver.token_proj.2.weight", + "pmid.qformer_perceiver.token_proj.fc2.weight"}, +}; + std::string convert_open_clip_to_hf_clip(const std::string& name) { std::string new_name = name; std::string prefix; @@ -208,6 +235,13 @@ std::string convert_vae_decoder_name(const std::string& name) { return name; } +std::string convert_pmid_v2_name(const std::string& name) { + if (pmid_v2_name_map.find(name) != pmid_v2_name_map.end()) { + return pmid_v2_name_map[name]; + } + return name; +} + /* If not a SDXL LoRA the unet" prefix will have already been replaced by this * point and "te2" and "te1" don't seem to appear in non-SDXL only "te_" */ std::string convert_sdxl_lora_name(std::string tensor_name) { @@ -428,6 +462,8 @@ std::string convert_tensor_name(const std::string& name) { new_name = convert_open_clip_to_hf_clip(name); } else if (starts_with(name, "first_stage_model.decoder")) { new_name = convert_vae_decoder_name(name); + } else if (starts_with(name, "pmid.qformer_perceiver")) { + new_name = convert_pmid_v2_name(name); } else if (starts_with(name, "control_model.")) { // for controlnet pth models size_t pos = name.find('.'); if (pos != std::string::npos) { diff --git a/pmid.hpp b/pmid.hpp index 9d6dfe0af..92a0238d0 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -82,6 +82,163 @@ class QFormerPerceiver(nn.Module): return out */ + +struct PMFeedForward : public GGMLBlock { + // network hparams + int dim; + +public: + PMFeedForward(int d, int multi=4) + : dim(d) { + int inner_dim = dim * multi; + blocks["0"] = std::shared_ptr(new LayerNorm(dim)); + blocks["1"] = std::shared_ptr(new Mlp(dim, inner_dim, dim, false)); + } + + struct ggml_tensor* forward(struct ggml_context* ctx, + struct ggml_tensor* x){ + + auto norm = std::dynamic_pointer_cast(blocks["0"]); + auto ff = std::dynamic_pointer_cast(blocks["1"]); + + x = norm->forward(ctx, x); + x = ff->forward(ctx, x); + return x; + } + +}; + +struct PerceiverAttention : public GGMLBlock { + // network hparams + float scale; // = dim_head**-0.5 + int dim_head; // = dim_head + int heads; // = heads +public: + PerceiverAttention(int dim, int dim_h=64, int h=8) + : scale(powf(dim_h, -0.5)), dim_head(dim_h), heads(h) { + + int inner_dim = dim_head * heads; + blocks["norm1"] = std::shared_ptr(new LayerNorm(dim)); + blocks["norm2"] = std::shared_ptr(new LayerNorm(dim)); + blocks["to_q"] = std::shared_ptr(new Linear(dim, inner_dim, false)); + blocks["to_kv"] = std::shared_ptr(new Linear(dim, inner_dim*2, false)); + blocks["to_out"] = std::shared_ptr(new Linear(inner_dim, dim, false)); + } + + struct ggml_tensor* reshape_tensor(struct ggml_context* ctx, + struct ggml_tensor* x, + int heads) { + int64_t ne[4]; + for(int i = 0; i < 4; ++i) + ne[i] = x->ne[i]; + x = ggml_view_4d(ctx, x, x->ne[0], x->ne[1], heads, x->ne[2]/heads, + x->nb[1], x->nb[2], x->nb[3], 0); + // x = ggml_transpose(ctx, x); + x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); + x = ggml_reshape_4d(ctx, x, ne[0], heads, ne[1], ne[2]/heads); + return x; + } + + struct ggml_tensor* forward(struct ggml_context* ctx, + struct ggml_tensor* x, + struct ggml_tensor* latents){ + + // x (torch.Tensor): image features + // shape (b, n1, D) + // latent (torch.Tensor): latent features + // shape (b, n2, D) + int64_t ne[4]; + for(int i = 0; i < 4; ++i) + ne[i] = latents->ne[i]; + + auto norm1 = std::dynamic_pointer_cast(blocks["norm1"]); + auto norm2 = std::dynamic_pointer_cast(blocks["norm2"]); + x = norm1->forward(ctx, x); + latents = norm2->forward(ctx, latents); + auto to_q = std::dynamic_pointer_cast(blocks["to_q"]); + auto q = to_q->forward(ctx, latents); + + auto kv_input = ggml_concat(ctx, x, latents, 1); + auto to_kv = std::dynamic_pointer_cast(blocks["to_kv"]); + auto kv = to_kv->forward(ctx, kv_input); + int64_t n = kv->ne[2] / 2; + int64_t offset = kv->nb[2] * n; + auto k = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*0); + auto v = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*1); + q = reshape_tensor(ctx, q, heads); + k = reshape_tensor(ctx, k, heads); + v = reshape_tensor(ctx, v, heads); + + scale = 1.f / sqrt(sqrt((float)dim_head)); + k = ggml_scale_inplace(ctx, k, scale); + k = ggml_cont(ctx, ggml_permute(ctx, k, 0, 1, 3, 2)); + q = ggml_scale_inplace(ctx, q, scale); + auto weight = ggml_mul_mat(ctx, q, k); + + // GGML's softmax() is equivalent to pytorch's softmax(x, dim=-1) + // in this case, dimension along which Softmax will be computed is the last dim + // in torch and the first dim in GGML, consistent with the convention that pytorch's + // last dimension (varying most rapidly) corresponds to GGML's first (varying most rapidly). + weight = ggml_soft_max(ctx, weight); + auto out = ggml_mul_mat(ctx, weight, v); + + out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); + out = ggml_reshape_3d(ctx, out, ne[0], ne[1], ggml_nelements(out)/(ne[0]*ne[1])); + auto to_out = std::dynamic_pointer_cast(blocks["to_out"]); + out = to_out->forward(ctx, out); + return out; + } +}; + +struct FacePerceiverResampler : public GGMLBlock { + // network hparams + int depth; +public: + FacePerceiverResampler( int dim=768, + int d=4, + int dim_head=64, + int heads=16, + int embedding_dim=1280, + int output_dim=768, + int ff_mult=4) + : depth(d) { + blocks["proj_in"] = std::shared_ptr(new Linear(embedding_dim, dim, true)); + blocks["proj_out"] = std::shared_ptr(new Linear(dim, output_dim, true)); + blocks["norm_out"] = std::shared_ptr(new LayerNorm(output_dim)); + + for (int i = 0; i < depth; i++) { + std::string name = "layers." + std::to_string(i) + ".0"; + blocks[name] = std::shared_ptr(new PerceiverAttention(dim, dim_head, heads)); + name = "layers." + std::to_string(i) + ".1"; + blocks[name] = std::shared_ptr(new PMFeedForward(dim, ff_mult)); + } + } + + struct ggml_tensor* forward(struct ggml_context* ctx, + struct ggml_tensor* latents, + struct ggml_tensor* x){ + // x: [N, channels, h, w] + auto proj_in = std::dynamic_pointer_cast(blocks["proj_in"]); + auto proj_out = std::dynamic_pointer_cast(blocks["proj_out"]); + auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); + + x = proj_in->forward(ctx, x); + for (int i = 0; i < depth; i++) { + std::string name = "layers." + std::to_string(i) + ".0"; + auto attn = std::dynamic_pointer_cast(blocks[name]); + name = "layers." + std::to_string(i) + ".1"; + auto ff = std::dynamic_pointer_cast(blocks[name]); + auto t = attn->forward(ctx, x, latents); + latents = ggml_add(ctx, t, latents); + t = ff->forward(ctx, latents); + latents = ggml_add(ctx, t, latents); + } + latents = proj_out->forward(ctx, latents); + latents = norm_out->forward(ctx, latents); + return latents; + } +}; + struct QFormerPerceiver : public GGMLBlock { // network hparams int num_tokens; @@ -101,19 +258,43 @@ struct QFormerPerceiver : public GGMLBlock { cross_attention_dim*num_tokens, true)); blocks["token_norm"] = std::shared_ptr(new LayerNorm(cross_attention_d)); + blocks["perceiver_resampler"] = std::shared_ptr(new FacePerceiverResampler( + cross_attention_dim, + 4, + 128, + cross_attention_dim / 128, + embedding_dim, + cross_attention_dim, + 4)); } - + /* + def forward(self, x, last_hidden_state): + x = self.token_proj(x) + x = x.reshape(-1, self.num_tokens, self.cross_attention_dim) + x = self.token_norm(x) # cls token + out = self.perceiver_resampler(x, last_hidden_state) # retrieve from patch tokens + if self.use_residual: # TODO: if use_residual is not true + out = x + 1.0 * out + return out + */ struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x, struct ggml_tensor* last_hidden_state){ // x: [N, channels, h, w] - auto mlp1 = std::dynamic_pointer_cast(blocks["mlp1"]); - auto mlp2 = std::dynamic_pointer_cast(blocks["mlp2"]); + auto token_proj = std::dynamic_pointer_cast(blocks["token_proj"]); auto token_norm = std::dynamic_pointer_cast(blocks["token_norm"]); - - + auto perceiver_resampler = std::dynamic_pointer_cast(blocks["perceiver_resampler"]); + + x = token_proj->forward(ctx, x); + int64_t nel = ggml_nelements(x); + x = ggml_reshape_3d(ctx, x, cross_attention_dim, num_tokens, nel/(cross_attention_dim*num_tokens)); + x = token_norm->forward(ctx, x); + struct ggml_tensor* out = perceiver_resampler->forward(ctx, x, last_hidden_state); + if(use_residul) + out = ggml_add(ctx, x, out); + return out; } }; @@ -155,56 +336,7 @@ class FacePerceiverResampler(torch.nn.Module): return self.norm_out(latents) */ -struct FacePerceiverResampler : public GGMLBlock { - // network hparams - int depth; -public: - FacePerceiverResampler( int dim=768, - int d=4, - int dim_head=64, - int heads=16, - int embedding_dim=1280, - int output_dim=768, - int ff_mult=4) - : depth(d) { - blocks["proj_in"] = std::shared_ptr(new Linear(embedding_dim, dim, true)); - blocks["proj_out"] = std::shared_ptr(new Linear(dim, output_dim, true)); - blocks["norm_out"] = std::shared_ptr(new LayerNorm(output_dim)); - - for (int i = 0; i < depth; i++) { - std::string name = "layers." + std::to_string(i) + ".0"; - blocks[name] = std::shared_ptr(new PerceiverAttention(dim, dim_head, heads)); - name = "layers." + std::to_string(i) + ".1"; - blocks[name] = std::shared_ptr(new FeedForward(dim, ff_mult)); - } - } - - - struct ggml_tensor* forward(struct ggml_context* ctx, - struct ggml_tensor* latents, - struct ggml_tensor* x){ - // x: [N, channels, h, w] - auto proj_in = std::dynamic_pointer_cast(blocks["proj_in"]); - auto proj_out = std::dynamic_pointer_cast(blocks["proj_out"]); - auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); - - x = proj_in->forward(ctx, x); - for (int i = 0; i < depth; i++) { - std::string name = "layers." + std::to_string(i) + ".0"; - auto attn = std::dynamic_pointer_cast(blocks[name]); - name = "layers." + std::to_string(i) + ".1"; - auto ff = std::dynamic_pointer_cast(blocks[name]); - auto t = attn->forward(ctx, x, latents); - latents = ggml_add(ctx, t, latents); - t = ff->forward(ctx, latents); - latents = ggml_add(ctx, t, latents); - } - latents = proj_out->forward(ctx, latents); - latents = norm_out->forward(ctx, latents); - return latents; - } -}; /* @@ -275,115 +407,7 @@ class PerceiverAttention(nn.Module): */ -struct FeedForward : public GGMLBlock { - // network hparams - int dim; - -public: - FeedForward(int d, int multi=4) - : dim(d) { - int inner_dim = dim * multi; - blocks["0"] = std::shared_ptr(new LayerNorm(dim)); - blocks["1"] = std::shared_ptr(new Mlp(dim, inner_dim, dim, false)); - } - - struct ggml_tensor* forward(struct ggml_context* ctx, - struct ggml_tensor* x){ - - auto norm = std::dynamic_pointer_cast(blocks["0"]); - auto ff = std::dynamic_pointer_cast(blocks["1"]); - - x = norm->forward(ctx, x); - x = ff->forward(ctx, x); - return x; - } - -}; - - - - -struct PerceiverAttention : public GGMLBlock { - // network hparams - float scale; // = dim_head**-0.5 - int dim_head; // = dim_head - int heads; // = heads -public: - PerceiverAttention(int dim, int dim_h=64, int h=8) - : scale(powf(dim_h, -0.5)), dim_head(dim_h), heads(h) { - - int inner_dim = dim_head * heads; - blocks["norm1"] = std::shared_ptr(new LayerNorm(dim)); - blocks["norm2"] = std::shared_ptr(new LayerNorm(dim)); - blocks["to_q"] = std::shared_ptr(new Linear(dim, inner_dim, false)); - blocks["to_kv"] = std::shared_ptr(new Linear(dim, inner_dim*2, false)); - blocks["to_out"] = std::shared_ptr(new Linear(inner_dim, dim, false)); - } - - struct ggml_tensor* reshape_tensor(struct ggml_context* ctx, - struct ggml_tensor* x, - int heads) { - int64_t ne[4]; - for(int i = 0; i < 4; ++i) - ne[i] = x->ne[i]; - x = ggml_view_4d(ctx, x, x->ne[0], x->ne[1], heads, x->ne[2]/heads, - x->nb[1], x->nb[2], x->nb[3], 0); - // x = ggml_transpose(ctx, x); - x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); - x = ggml_reshape_4d(ctx, x, ne[0], heads, ne[1], ne[2]/heads); - return x; - } - - struct ggml_tensor* forward(struct ggml_context* ctx, - struct ggml_tensor* x, - struct ggml_tensor* latents){ - // x (torch.Tensor): image features - // shape (b, n1, D) - // latent (torch.Tensor): latent features - // shape (b, n2, D) - int64_t ne[4]; - for(int i = 0; i < 4; ++i) - ne[i] = latents->ne[i]; - - auto norm1 = std::dynamic_pointer_cast(blocks["norm1"]); - auto norm2 = std::dynamic_pointer_cast(blocks["norm2"]); - x = norm1->forward(ctx, x); - latents = norm2->forward(ctx, latents); - auto to_q = std::dynamic_pointer_cast(blocks["to_q"]); - auto q = to_q->forward(ctx, latents); - - auto kv_input = ggml_concat(ctx, x, latents, 1); - auto to_kv = std::dynamic_pointer_cast(blocks["to_kv"]); - auto kv = to_kv->forward(ctx, kv_input); - int64_t n = kv->ne[2] / 2; - int64_t offset = kv->nb[2] * n; - auto k = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*0); - auto v = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*1); - q = reshape_tensor(ctx, q, heads); - k = reshape_tensor(ctx, k, heads); - v = reshape_tensor(ctx, v, heads); - - scale = 1.f / sqrt(sqrt((float)dim_head)); - k = ggml_scale_inplace(ctx, k, scale); - k = ggml_cont(ctx, ggml_permute(ctx, k, 0, 1, 3, 2)); - q = ggml_scale_inplace(ctx, q, scale); - auto weight = ggml_mul_mat(ctx, q, k); - - // GGML's softmax() is equivalent to pytorch's softmax(x, dim=-1) - // in this case, dimension along which Softmax will be computed is the last dim - // in torch and the first dim in GGML, consistent with the convention that pytorch's - // last dimension (varying most rapidly) corresponds to GGML's first (varying most rapidly). - weight = ggml_soft_max(ctx, weight); - auto out = ggml_mul_mat(ctx, weight, v); - - out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); - out = ggml_reshape_3d(ctx, out, ne[0], ne[1], ggml_nelements(out)/(ne[0]*ne[1])); - auto to_out = std::dynamic_pointer_cast(blocks["to_out"]); - out = to_out->forward(ctx, out); - return out; - } -}; struct FuseModule : public GGMLBlock { diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index c4705db15..d1e3293f9 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -280,7 +280,12 @@ class StableDiffusionGGML { control_net = std::make_shared(controlnet_backend, model_data_type, version); } - pmid_model = std::make_shared(clip_backend, model_data_type, version); + //printf("Photomaker model file is %s \n", id_embeddings_path.c_str()); + if(id_embeddings_path.find("v2") != std::string::npos) { + pmid_model = std::make_shared(clip_backend, model_data_type, version, VERSION_2); + } else { + pmid_model = std::make_shared(clip_backend, model_data_type, version); + } if (id_embeddings_path.size() > 0) { pmid_lora = std::make_shared(backend, model_data_type, id_embeddings_path, ""); if (!pmid_lora->load_from_file(true)) { @@ -575,7 +580,7 @@ class StableDiffusionGGML { ggml_tensor* prompts_embeds, std::vector& class_tokens_mask) { ggml_tensor* res = NULL; - pmid_model->compute(n_threads, init_img, prompts_embeds, class_tokens_mask, &res, work_ctx); + pmid_model->compute(n_threads, init_img, prompts_embeds, NULL, class_tokens_mask, &res, work_ctx); return res; } From 2775180419adb3a2d5204830cd86e99afc61b253 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Mon, 19 Aug 2024 22:00:13 -0400 Subject: [PATCH 05/20] more debugging underway --- clip.hpp | 10 ++- examples/cli/main.cpp | 1 + model.cpp | 8 ++- model.h | 2 + pmid.hpp | 142 ++++++++++++++++++++++++++++++++++++++---- stable-diffusion.cpp | 33 ++++++++-- stable-diffusion.h | 1 + util.cpp | 18 ++++++ util.h | 2 +- 9 files changed, 197 insertions(+), 20 deletions(-) diff --git a/clip.hpp b/clip.hpp index 664da58c6..d35c95aba 100644 --- a/clip.hpp +++ b/clip.hpp @@ -751,7 +751,8 @@ class CLIPVisionModel : public GGMLBlock { blocks["post_layernorm"] = std::shared_ptr(new LayerNorm(hidden_size)); } - struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* pixel_values, bool return_pooled = true) { + struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* pixel_values, + bool return_pooled = true) { // pixel_values: [N, num_channels, image_size, image_size] auto embeddings = std::dynamic_pointer_cast(blocks["embeddings"]); auto pre_layernorm = std::dynamic_pointer_cast(blocks["pre_layernorm"]); @@ -761,14 +762,17 @@ class CLIPVisionModel : public GGMLBlock { auto x = embeddings->forward(ctx, pixel_values); // [N, num_positions, embed_dim] x = pre_layernorm->forward(ctx, x); x = encoder->forward(ctx, x, -1, false); + // print_ggml_tensor(x, true, "ClipVisionModel x: "); + auto last_hidden_state = x; x = post_layernorm->forward(ctx, x); // [N, n_token, hidden_size] - GGML_ASSERT(x->ne[3] == 1); + GGML_ASSERT(x->ne[3] == 1); if (return_pooled) { ggml_tensor* pooled = ggml_cont(ctx, ggml_view_2d(ctx, x, x->ne[0], x->ne[2], x->nb[2], 0)); return pooled; // [N, hidden_size] } else { - return x; // [N, n_token, hidden_size] + // return x; // [N, n_token, hidden_size] + return last_hidden_state; // [N, n_token, hidden_size] } } }; diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 6675095b5..34aaf5830 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -723,6 +723,7 @@ int main(int argc, const char* argv[]) { params.lora_model_dir.c_str(), params.embeddings_path.c_str(), params.stacked_id_embeddings_path.c_str(), + params.input_id_images_path.c_str(), vae_decode_only, params.vae_tiling, true, diff --git a/model.cpp b/model.cpp index b931f03a5..73c47ac55 100644 --- a/model.cpp +++ b/model.cpp @@ -1271,7 +1271,7 @@ bool ModelLoader::parse_data_pkl(uint8_t* buffer, reader.tensor_storage.reverse_ne(); reader.tensor_storage.file_index = file_index; // if(strcmp(prefix.c_str(), "scarlett") == 0) - // printf(" got tensor %s \n ", reader.tensor_storage.name.c_str()); + // printf(" ZIP got tensor %s \n ", reader.tensor_storage.name.c_str()); reader.tensor_storage.name = prefix + reader.tensor_storage.name; tensor_storages.push_back(reader.tensor_storage); // LOG_DEBUG("%s", reader.tensor_storage.name.c_str()); @@ -1307,7 +1307,9 @@ bool ModelLoader::init_from_ckpt_file(const std::string& file_path, const std::s std::string name = zip_entry_name(zip); size_t pos = name.find("data.pkl"); if (pos != std::string::npos) { + std::string dir = name.substr(0, pos); + printf("ZIP %d, name = %s, dir = %s \n", i, name.c_str(), dir.c_str()); void* pkl_data = NULL; size_t pkl_size; zip_entry_read(zip, &pkl_data, &pkl_size); @@ -1405,6 +1407,10 @@ std::vector remove_duplicates(const std::vector& v return res; } +bool ModelLoader::load_tensor(std::string tensor_name, ggml_backend_t backend){ + +} + bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, ggml_backend_t backend) { std::vector processed_tensor_storages; for (auto& tensor_storage : tensor_storages) { diff --git a/model.h b/model.h index 5bfce308f..fe6c45c90 100644 --- a/model.h +++ b/model.h @@ -148,6 +148,8 @@ class ModelLoader { bool load_tensors(std::map& tensors, ggml_backend_t backend, std::set ignore_tensors = {}); + bool load_tensor(std::string tensor_name, ggml_backend_t backend); + bool save_to_gguf_file(const std::string& file_path, ggml_type type); int64_t get_params_mem_size(ggml_backend_t backend, ggml_type type = GGML_TYPE_COUNT); ~ModelLoader() = default; diff --git a/pmid.hpp b/pmid.hpp index 92a0238d0..2c0f63076 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -131,14 +131,30 @@ struct PerceiverAttention : public GGMLBlock { int64_t ne[4]; for(int i = 0; i < 4; ++i) ne[i] = x->ne[i]; - x = ggml_view_4d(ctx, x, x->ne[0], x->ne[1], heads, x->ne[2]/heads, + // print_ggml_tensor(x, true, "PerceiverAttention reshape x 0: "); + // printf("heads = %d \n", heads); + // x = ggml_view_4d(ctx, x, x->ne[0], x->ne[1], heads, x->ne[2]/heads, + // x->nb[1], x->nb[2], x->nb[3], 0); + x = ggml_view_4d(ctx, x, x->ne[0]/heads, heads, x->ne[1], x->ne[2], x->nb[1], x->nb[2], x->nb[3], 0); // x = ggml_transpose(ctx, x); x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); - x = ggml_reshape_4d(ctx, x, ne[0], heads, ne[1], ne[2]/heads); + // print_ggml_tensor(x, true, "PerceiverAttention reshape x 1: "); + // x = ggml_reshape_4d(ctx, x, ne[0], heads, ne[1], ne[2]/heads); return x; } + std::vector chunk_half(struct ggml_context* ctx, + struct ggml_tensor* x){ + auto t = ggml_cont(ctx, ggml_permute(ctx, x, 2, 0, 1, 3)); + int64_t n = t->ne[2] / 2; + int64_t offset = t->nb[2] * n; + auto k = ggml_view_3d(ctx, t, t->ne[0], t->ne[1], n, t->nb[1], t->nb[2], offset*0); + auto v = ggml_view_3d(ctx, t, t->ne[0], t->ne[1], n, t->nb[1], t->nb[2], offset*1); + return {ggml_cont(ctx, ggml_permute(ctx, k, 1, 2, 0, 3)), + ggml_cont(ctx, ggml_permute(ctx, v, 1, 2, 0, 3))}; + } + struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x, struct ggml_tensor* latents){ @@ -150,6 +166,8 @@ struct PerceiverAttention : public GGMLBlock { int64_t ne[4]; for(int i = 0; i < 4; ++i) ne[i] = latents->ne[i]; + print_ggml_tensor(x, true, "PerceiverAttention x 0: "); + print_ggml_tensor(latents, true, "PerceiverAttention latents 0: "); auto norm1 = std::dynamic_pointer_cast(blocks["norm1"]); auto norm2 = std::dynamic_pointer_cast(blocks["norm2"]); @@ -158,28 +176,46 @@ struct PerceiverAttention : public GGMLBlock { auto to_q = std::dynamic_pointer_cast(blocks["to_q"]); auto q = to_q->forward(ctx, latents); + print_ggml_tensor(x, true, "PerceiverAttention x: "); + print_ggml_tensor(latents, true, "PerceiverAttention latents: "); + print_ggml_tensor(q, true, "PerceiverAttention q: "); auto kv_input = ggml_concat(ctx, x, latents, 1); auto to_kv = std::dynamic_pointer_cast(blocks["to_kv"]); auto kv = to_kv->forward(ctx, kv_input); - int64_t n = kv->ne[2] / 2; - int64_t offset = kv->nb[2] * n; - auto k = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*0); - auto v = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*1); + // int64_t n = kv->ne[0] / 2; + // int64_t offset = kv->nb[2] * n; + print_ggml_tensor(kv, true, "PerceiverAttention kv: "); + // auto k = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*0); + // auto v = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*1); + std::vector chunks = chunk_half(ctx, kv); + auto k = chunks[0]; + auto v = chunks[1]; + print_ggml_tensor(q, true, "PerceiverAttention bef reshape q: "); + print_ggml_tensor(k, true, "PerceiverAttention bef reshape k: "); + print_ggml_tensor(v, true, "PerceiverAttention bef reshape v: "); q = reshape_tensor(ctx, q, heads); k = reshape_tensor(ctx, k, heads); v = reshape_tensor(ctx, v, heads); - + print_ggml_tensor(q, true, "PerceiverAttention aft reshape q: "); + print_ggml_tensor(k, true, "PerceiverAttention aft reshape k: "); + print_ggml_tensor(v, true, "PerceiverAttention aft reshape v: "); scale = 1.f / sqrt(sqrt((float)dim_head)); k = ggml_scale_inplace(ctx, k, scale); - k = ggml_cont(ctx, ggml_permute(ctx, k, 0, 1, 3, 2)); + // k = ggml_cont(ctx, ggml_permute(ctx, k, 1, 0, 2, 3)); q = ggml_scale_inplace(ctx, q, scale); + print_ggml_tensor(q, true, "PerceiverAttention mul q: "); + print_ggml_tensor(k, true, "PerceiverAttention mul k: "); auto weight = ggml_mul_mat(ctx, q, k); // GGML's softmax() is equivalent to pytorch's softmax(x, dim=-1) // in this case, dimension along which Softmax will be computed is the last dim // in torch and the first dim in GGML, consistent with the convention that pytorch's // last dimension (varying most rapidly) corresponds to GGML's first (varying most rapidly). - weight = ggml_soft_max(ctx, weight); + weight = ggml_soft_max(ctx, weight); + weight = ggml_cont(ctx, ggml_transpose(ctx, weight)); + v = ggml_cont(ctx, ggml_transpose(ctx, v)); + print_ggml_tensor(weight, true, "PerceiverAttention mul weight: "); + print_ggml_tensor(v, true, "PerceiverAttention mul v: "); auto out = ggml_mul_mat(ctx, weight, v); out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); @@ -429,8 +465,13 @@ struct FuseModule : public GGMLBlock { auto mlp2 = std::dynamic_pointer_cast(blocks["mlp2"]); auto layer_norm = std::dynamic_pointer_cast(blocks["layer_norm"]); + print_ggml_tensor(id_embeds, true, "Fuseblock id_embeds: "); + print_ggml_tensor(prompt_embeds, true, "Fuseblock prompt_embeds: "); + auto prompt_embeds0 = ggml_cont(ctx, ggml_permute(ctx, prompt_embeds, 2, 0, 1, 3)); auto id_embeds0 = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3)); + print_ggml_tensor(id_embeds0, true, "Fuseblock id_embeds0: "); + print_ggml_tensor(prompt_embeds0, true, "Fuseblock prompt_embeds0: "); // concat is along dim 2 auto stacked_id_embeds = ggml_concat(ctx, prompt_embeds0, id_embeds0, 2); stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 1, 2, 0, 3)); @@ -580,10 +621,13 @@ struct PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock : public CLIPVisionMo auto fuse_module = std::dynamic_pointer_cast(blocks["fuse_module"]); auto qformer_perceiver = std::dynamic_pointer_cast(blocks["qformer_perceiver"]); - struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values); // [N, hidden_size] - + // struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values); // [N, hidden_size] + struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values, false); // [N, hidden_size] + print_ggml_tensor(id_pixel_values, true, "PhotoMakerIDEncoder2 id_pixel_values: "); + print_ggml_tensor(last_hidden_state, true, "PhotoMakerIDEncoder2 last_hidden_state: "); id_embeds = qformer_perceiver->forward(ctx, id_embeds, last_hidden_state); // id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3)); + print_ggml_tensor(id_embeds, true, "PhotoMakerIDEncoder2 id_embeds: "); // id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2, 2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right // id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3)); @@ -756,4 +800,80 @@ struct PhotoMakerIDEncoder : public GGMLRunner { } }; + +struct PhotoMakerIDEmbed : public GGMLRunner { + + std::map tensors; + std::string file_path; + ModelLoader *model_loader; + bool load_failed = false; + bool applied = false; + + PhotoMakerIDEmbed(ggml_backend_t backend, + ggml_type wtype, + ModelLoader *ml, + const std::string& file_path = "", + const std::string& prefix = "") + : file_path(file_path), GGMLRunner(backend, wtype), + model_loader(ml) { + if (!model_loader->init_from_file(file_path, prefix)) { + load_failed = true; + } + } + + std::string get_desc() { + return "id_embeds"; + } + + bool load_from_file(bool filter_tensor = false) { + LOG_INFO("loading PhotoMaker ID Embeds from '%s'", file_path.c_str()); + + if (load_failed) { + LOG_ERROR("init photomaker id embed from file failed: '%s'", file_path.c_str()); + return false; + } + + bool dry_run = true; + auto on_new_tensor_cb = [&](const TensorStorage& tensor_storage, ggml_tensor** dst_tensor) -> bool { + const std::string& name = tensor_storage.name; + + if (filter_tensor && !contains(name, "pmid.id_embeds")) { + // LOG_INFO("skipping LoRA tesnor '%s'", name.c_str()); + return true; + } + + if (dry_run) { + struct ggml_tensor* real = ggml_new_tensor(params_ctx, + tensor_storage.type, + tensor_storage.n_dims, + tensor_storage.ne); + tensors[name] = real; + } else { + auto real = tensors[name]; + *dst_tensor = real; + } + + return true; + }; + + model_loader->load_tensors(on_new_tensor_cb, backend); + alloc_params_buffer(); + + dry_run = false; + model_loader->load_tensors(on_new_tensor_cb, backend); + + LOG_DEBUG("finished loading PhotoMaker ID Embeds "); + return true; + } + + + struct ggml_tensor* get(){ + std::map::iterator pos; + pos = tensors.find("pmid.id_embeds"); + if(pos != tensors.end()) + return pos->second; + return NULL; + } +}; + #endif // __PMI_HPP__ diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index d1e3293f9..780222a70 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -85,6 +85,7 @@ class StableDiffusionGGML { std::shared_ptr control_net; std::shared_ptr pmid_model; std::shared_ptr pmid_lora; + std::shared_ptr pmid_id_embeds; std::string taesd_path; bool use_tiny_autoencoder = false; @@ -135,6 +136,7 @@ class StableDiffusionGGML { const std::string control_net_path, const std::string embeddings_path, const std::string id_embeddings_path, + const std::string input_id_images_path, const std::string& taesd_path, bool vae_tiling_, ggml_type wtype, @@ -298,6 +300,17 @@ class StableDiffusionGGML { } else { stacked_id = true; } + if(stacked_id){ + if(id_embeddings_path.find("v2") != std::string::npos) { + std::string id_embed_path = path_join(input_id_images_path, "id_embeds.safetensors"); + pmid_id_embeds = std::make_shared(backend, model_data_type, + &model_loader, id_embed_path, "pmid."); + if (!pmid_id_embeds->load_from_file(true)) { + LOG_ERROR("load photomaker id embed tensors from %s failed", id_embed_path.c_str()); + return false; + } + } + } } if (stacked_id) { if (!pmid_model->alloc_params_buffer()) { @@ -578,10 +591,10 @@ class StableDiffusionGGML { ggml_tensor* id_encoder(ggml_context* work_ctx, ggml_tensor* init_img, ggml_tensor* prompts_embeds, + ggml_tensor* id_embeds, std::vector& class_tokens_mask) { ggml_tensor* res = NULL; - pmid_model->compute(n_threads, init_img, prompts_embeds, NULL, class_tokens_mask, &res, work_ctx); - + pmid_model->compute(n_threads, init_img, prompts_embeds, id_embeds, class_tokens_mask, &res, work_ctx); return res; } @@ -919,6 +932,7 @@ sd_ctx_t* new_sd_ctx(const char* model_path_c_str, const char* lora_model_dir_c_str, const char* embed_dir_c_str, const char* id_embed_dir_c_str, + const char* input_id_images_dir_c_str, bool vae_decode_only, bool vae_tiling, bool free_params_immediately, @@ -939,6 +953,7 @@ sd_ctx_t* new_sd_ctx(const char* model_path_c_str, std::string control_net_path(control_net_path_c_str); std::string embd_path(embed_dir_c_str); std::string id_embd_path(id_embed_dir_c_str); + std::string input_id_images_path(input_id_images_dir_c_str); std::string lora_model_dir(lora_model_dir_c_str); sd_ctx->sd = new StableDiffusionGGML(n_threads, @@ -955,6 +970,7 @@ sd_ctx_t* new_sd_ctx(const char* model_path_c_str, control_net_path, embd_path, id_embd_path, + input_id_images_path, taesd_path, vae_tiling, (ggml_type)wtype, @@ -1045,11 +1061,16 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx, } // preprocess input id images std::vector input_id_images; + bool pmv2 = false; if (sd_ctx->sd->pmid_model && input_id_images_path.size() > 0) { std::vector img_files = get_files_from_dir(input_id_images_path); for (std::string img_file : img_files) { int c = 0; int width, height; + if(ends_with(img_file, "id_embeds.safetensors")){ + pmv2 = true; + continue; + } uint8_t* input_image_buffer = stbi_load(img_file.c_str(), &width, &height, &c, 3); if (input_image_buffer == NULL) { LOG_ERROR("PhotoMaker load image from '%s' failed", img_file.c_str()); @@ -1097,8 +1118,12 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx, sd_ctx->sd->diffusion_model->get_adm_in_channels()); id_cond = std::get<0>(cond_tup); class_tokens_mask = std::get<1>(cond_tup); // - - id_cond.c_crossattn = sd_ctx->sd->id_encoder(work_ctx, init_img, id_cond.c_crossattn, class_tokens_mask); + struct ggml_tensor * id_embeds = NULL; + if(pmv2){ + id_embeds = sd_ctx->sd->pmid_id_embeds->get(); + // print_ggml_tensor(id_embeds, true, "id_embeds:"); + } + id_cond.c_crossattn = sd_ctx->sd->id_encoder(work_ctx, init_img, id_cond.c_crossattn, id_embeds, class_tokens_mask); t1 = ggml_time_ms(); LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0); if (sd_ctx->sd->free_params_immediately) { diff --git a/stable-diffusion.h b/stable-diffusion.h index f78748faf..62103ab14 100644 --- a/stable-diffusion.h +++ b/stable-diffusion.h @@ -125,6 +125,7 @@ SD_API sd_ctx_t* new_sd_ctx(const char* model_path, const char* lora_model_dir, const char* embed_dir_c_str, const char* stacked_id_embed_dir_c_str, + const char* input_id_images_dir_c_str, bool vae_decode_only, bool vae_tiling, bool free_params_immediately, diff --git a/util.cpp b/util.cpp index 5de5ce26e..cd058bb07 100644 --- a/util.cpp +++ b/util.cpp @@ -276,6 +276,24 @@ std::string path_join(const std::string& p1, const std::string& p2) { return p1 + "/" + p2; } +std::vector splitString(const std::string& str, char delimiter) { + std::vector result; + size_t start = 0; + size_t end = str.find(delimiter); + + while (end != std::string::npos) { + result.push_back(str.substr(start, end - start)); + start = end + 1; + end = str.find(delimiter, start); + } + + // Add the last segment after the last delimiter + result.push_back(str.substr(start)); + + return result; +} + + sd_image_t* preprocess_id_image(sd_image_t* img) { int shortest_edge = 224; int size = shortest_edge; diff --git a/util.h b/util.h index 9b1e6734f..14fa812e5 100644 --- a/util.h +++ b/util.h @@ -45,7 +45,7 @@ sd_image_f32_t resize_sd_image_f32_t(sd_image_f32_t image, int target_width, int sd_image_f32_t clip_preprocess(sd_image_f32_t image, int size); std::string path_join(const std::string& p1, const std::string& p2); - +std::vector splitString(const std::string& str, char delimiter); void pretty_progress(int step, int steps, float time); void log_printf(sd_log_level_t level, const char* file, int line, const char* format, ...); From e0fbab865ba4ea6639e5cdeebb3c8cb5762cd3da Mon Sep 17 00:00:00 2001 From: bssrdf Date: Tue, 20 Aug 2024 21:07:46 -0400 Subject: [PATCH 06/20] failing at cuda mat_mul --- conditioner.hpp | 3 ++- ggml_extend.hpp | 10 +++++-- model.cpp | 6 +---- model.h | 1 - pmid.hpp | 64 +++++++++++++++++++++++++++++++++++--------- stable-diffusion.cpp | 12 ++++++--- 6 files changed, 71 insertions(+), 25 deletions(-) diff --git a/conditioner.hpp b/conditioner.hpp index e01be2b21..8cb5518ae 100644 --- a/conditioner.hpp +++ b/conditioner.hpp @@ -285,7 +285,8 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { tokenizer.pad_tokens(tokens, weights, max_length, padding); for (uint32_t i = 0; i < tokens.size(); i++) { - if (class_idx + 1 <= i && i < class_idx + 1 + num_input_imgs) + if (class_idx + 1 <= i && i < class_idx + 1 + 2*num_input_imgs) // photomaker V2 has num_tokens(=2)*num_input_imgs + // hardcode for now class_token_mask.push_back(true); else class_token_mask.push_back(false); diff --git a/ggml_extend.hpp b/ggml_extend.hpp index 47fd3a148..3e66edc01 100644 --- a/ggml_extend.hpp +++ b/ggml_extend.hpp @@ -988,6 +988,11 @@ struct GGMLRunner { params_buffer_size / (1024.0 * 1024.0), ggml_backend_is_cpu(backend) ? "RAM" : "VRAM", num_tensors); + // printf("%s params backend buffer size = % 6.2f MB(%s) (%i tensors)\n", + // get_desc().c_str(), + // params_buffer_size / (1024.0 * 1024.0), + // ggml_backend_is_cpu(backend) ? "RAM" : "VRAM", + // num_tensors); return true; } @@ -1149,11 +1154,12 @@ class Linear : public UnaryBlock { int64_t out_features; bool bias; - void init_params(struct ggml_context* ctx, ggml_type wtype) { + void init_params(struct ggml_context* ctx, ggml_type wtype) { params["weight"] = ggml_new_tensor_2d(ctx, wtype, in_features, out_features); if (bias) { params["bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, out_features); - } + } + } public: diff --git a/model.cpp b/model.cpp index 73c47ac55..e775abd45 100644 --- a/model.cpp +++ b/model.cpp @@ -932,7 +932,7 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const } TensorStorage tensor_storage(prefix + name, type, ne, n_dims, file_index, ST_HEADER_SIZE_LEN + header_size_ + begin); - tensor_storage.reverse_ne(); + tensor_storage.reverse_ne(); size_t tensor_data_size = end - begin; @@ -1407,10 +1407,6 @@ std::vector remove_duplicates(const std::vector& v return res; } -bool ModelLoader::load_tensor(std::string tensor_name, ggml_backend_t backend){ - -} - bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, ggml_backend_t backend) { std::vector processed_tensor_storages; for (auto& tensor_storage : tensor_storages) { diff --git a/model.h b/model.h index fe6c45c90..3e1a50223 100644 --- a/model.h +++ b/model.h @@ -148,7 +148,6 @@ class ModelLoader { bool load_tensors(std::map& tensors, ggml_backend_t backend, std::set ignore_tensors = {}); - bool load_tensor(std::string tensor_name, ggml_backend_t backend); bool save_to_gguf_file(const std::string& file_path, ggml_type type); int64_t get_params_mem_size(ggml_backend_t backend, ggml_type type = GGML_TYPE_COUNT); diff --git a/pmid.hpp b/pmid.hpp index 2c0f63076..655183268 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -91,7 +91,7 @@ struct PMFeedForward : public GGMLBlock { PMFeedForward(int d, int multi=4) : dim(d) { int inner_dim = dim * multi; - blocks["0"] = std::shared_ptr(new LayerNorm(dim)); + blocks["0"] = std::shared_ptr(new LayerNorm(dim)); blocks["1"] = std::shared_ptr(new Mlp(dim, inner_dim, dim, false)); } @@ -204,7 +204,9 @@ struct PerceiverAttention : public GGMLBlock { // k = ggml_cont(ctx, ggml_permute(ctx, k, 1, 0, 2, 3)); q = ggml_scale_inplace(ctx, q, scale); print_ggml_tensor(q, true, "PerceiverAttention mul q: "); - print_ggml_tensor(k, true, "PerceiverAttention mul k: "); + print_ggml_tensor(k, true, "PerceiverAttention mul k: "); + ggml_set_name(q, "PerceiverAttention q"); + ggml_set_name(k, "PerceiverAttention k"); auto weight = ggml_mul_mat(ctx, q, k); // GGML's softmax() is equivalent to pytorch's softmax(x, dim=-1) @@ -216,6 +218,8 @@ struct PerceiverAttention : public GGMLBlock { v = ggml_cont(ctx, ggml_transpose(ctx, v)); print_ggml_tensor(weight, true, "PerceiverAttention mul weight: "); print_ggml_tensor(v, true, "PerceiverAttention mul v: "); + ggml_set_name(weight, "PerceiverAttention w"); + ggml_set_name(v, "PerceiverAttention v"); auto out = ggml_mul_mat(ctx, weight, v); out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); @@ -258,6 +262,14 @@ struct FacePerceiverResampler : public GGMLBlock { auto proj_out = std::dynamic_pointer_cast(blocks["proj_out"]); auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); + // std::map tensors; + // get_param_tensors(tensors, ""); + // for(auto& pair : tensors) { + // struct ggml_tensor* t = pair.second; + // std::string name = pair.first; + // ggml_set_name(t, name.c_str()); + // } + x = proj_in->forward(ctx, x); for (int i = 0; i < depth; i++) { std::string name = "layers." + std::to_string(i) + ".0"; @@ -468,14 +480,16 @@ struct FuseModule : public GGMLBlock { print_ggml_tensor(id_embeds, true, "Fuseblock id_embeds: "); print_ggml_tensor(prompt_embeds, true, "Fuseblock prompt_embeds: "); - auto prompt_embeds0 = ggml_cont(ctx, ggml_permute(ctx, prompt_embeds, 2, 0, 1, 3)); - auto id_embeds0 = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3)); - print_ggml_tensor(id_embeds0, true, "Fuseblock id_embeds0: "); - print_ggml_tensor(prompt_embeds0, true, "Fuseblock prompt_embeds0: "); + // auto prompt_embeds0 = ggml_cont(ctx, ggml_permute(ctx, prompt_embeds, 2, 0, 1, 3)); + // auto id_embeds0 = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3)); + // print_ggml_tensor(id_embeds0, true, "Fuseblock id_embeds0: "); + // print_ggml_tensor(prompt_embeds0, true, "Fuseblock prompt_embeds0: "); // concat is along dim 2 - auto stacked_id_embeds = ggml_concat(ctx, prompt_embeds0, id_embeds0, 2); - stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 1, 2, 0, 3)); - + // auto stacked_id_embeds = ggml_concat(ctx, prompt_embeds0, id_embeds0, 2); + auto stacked_id_embeds = ggml_concat(ctx, prompt_embeds, id_embeds, 0); + print_ggml_tensor(stacked_id_embeds, true, "Fuseblock stacked_id_embeds 0: "); + // stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 1, 2, 0, 3)); + // print_ggml_tensor(stacked_id_embeds, true, "Fuseblock stacked_id_embeds 1: "); // stacked_id_embeds = mlp1.forward(ctx, stacked_id_embeds); // stacked_id_embeds = ggml_add(ctx, stacked_id_embeds, prompt_embeds); // stacked_id_embeds = mlp2.forward(ctx, stacked_id_embeds); @@ -507,6 +521,8 @@ struct FuseModule : public GGMLBlock { // print_ggml_tensor(class_tokens_mask_pos, true, "class_tokens_mask_pos"); struct ggml_tensor* image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos); ggml_set_name(image_token_embeds, "image_token_embeds"); + valid_id_embeds = ggml_reshape_2d(ctx, valid_id_embeds, valid_id_embeds->ne[0], + ggml_nelements(valid_id_embeds)/valid_id_embeds->ne[0]); struct ggml_tensor* stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds); stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); @@ -625,10 +641,29 @@ struct PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock : public CLIPVisionMo struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values, false); // [N, hidden_size] print_ggml_tensor(id_pixel_values, true, "PhotoMakerIDEncoder2 id_pixel_values: "); print_ggml_tensor(last_hidden_state, true, "PhotoMakerIDEncoder2 last_hidden_state: "); + print_ggml_tensor(id_embeds, true, "PhotoMakerIDEncoder2 id_embeds0: "); id_embeds = qformer_perceiver->forward(ctx, id_embeds, last_hidden_state); // id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3)); print_ggml_tensor(id_embeds, true, "PhotoMakerIDEncoder2 id_embeds: "); - + print_ggml_tensor(prompt_embeds, true, "PhotoMakerIDEncoder2 prompt_embeds: "); + // if(id_embeds->backend == GGML_BACKEND_TYPE_GPU){ + // if(!ggml_backend_buffer_is_host(id_embeds->buffer)){ + // printf("id_embeds on GPU\n"); + // } + // else{ + // printf("id_embeds on CPU\n"); + // } + // if(prompt_embeds->buffer != NULL && !ggml_backend_buffer_is_host(prompt_embeds->buffer)){ + // if(prompt_embeds->backend == GGML_BACKEND_TYPE_GPU){ + // printf("prompt_embeds on GPU\n"); + // } + // else{ + // if(prompt_embeds->buffer == NULL){ + // printf("prompt_embeds buffer is NULL\n"); + // } + // printf("prompt_embeds on CPU\n"); + // } + // id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2, 2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right // id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3)); @@ -710,19 +745,23 @@ struct PhotoMakerIDEncoder : public GGMLRunner { struct ggml_tensor* id_pixel_values_d = to_backend(id_pixel_values); struct ggml_tensor* prompt_embeds_d = to_backend(prompt_embeds); + struct ggml_tensor* id_embeds_d = to_backend(id_embeds); struct ggml_tensor* left = NULL; struct ggml_tensor* right = NULL; for (int i = 0; i < class_tokens_mask.size(); i++) { if (class_tokens_mask[i]) { + // printf(" 1,"); ctm.push_back(0.f); // here use 0.f instead of 1.f to make a scale mask ctmf16.push_back(ggml_fp32_to_fp16(0.f)); // here use 0.f instead of 1.f to make a scale mask ctmpos.push_back(i); } else { + // printf(" 0,"); ctm.push_back(1.f); // here use 1.f instead of 0.f to make a scale mask ctmf16.push_back(ggml_fp32_to_fp16(1.f)); // here use 0.f instead of 1.f to make a scale mask } } + // printf("\n"); if (ctmpos[0] > 0) { left = ggml_new_tensor_3d(ctx0, type, hidden_size, 1, ctmpos[0]); } @@ -775,7 +814,7 @@ struct PhotoMakerIDEncoder : public GGMLRunner { prompt_embeds_d, class_tokens_mask_d, class_tokens_mask_pos, - id_embeds, + id_embeds_d, left, right); ggml_build_forward_expand(gf, updated_prompt_embeds); @@ -840,8 +879,7 @@ struct PhotoMakerIDEmbed : public GGMLRunner { if (filter_tensor && !contains(name, "pmid.id_embeds")) { // LOG_INFO("skipping LoRA tesnor '%s'", name.c_str()); return true; - } - + } if (dry_run) { struct ggml_tensor* real = ggml_new_tensor(params_ctx, tensor_storage.type, diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 780222a70..993fb3a29 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -202,6 +202,11 @@ class StableDiffusionGGML { LOG_INFO("Stable Diffusion weight type: %s", ggml_type_name(model_data_type)); LOG_DEBUG("ggml tensor size = %d bytes", (int)sizeof(ggml_tensor)); + if(ggml_backend_is_cpu(backend)) + LOG_INFO("Stable Diffusion backend is CPU"); + else + LOG_INFO("Stable Diffusion backend is GPU"); + if (version == VERSION_XL) { scale_factor = 0.13025f; if (vae_path.size() == 0 && taesd_path.size() == 0) { @@ -284,7 +289,8 @@ class StableDiffusionGGML { //printf("Photomaker model file is %s \n", id_embeddings_path.c_str()); if(id_embeddings_path.find("v2") != std::string::npos) { - pmid_model = std::make_shared(clip_backend, model_data_type, version, VERSION_2); + // pmid_model = std::make_shared(clip_backend, model_data_type, version, VERSION_2); + pmid_model = std::make_shared(backend, model_data_type, version, VERSION_2); } else { pmid_model = std::make_shared(clip_backend, model_data_type, version); } @@ -1118,9 +1124,9 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx, sd_ctx->sd->diffusion_model->get_adm_in_channels()); id_cond = std::get<0>(cond_tup); class_tokens_mask = std::get<1>(cond_tup); // - struct ggml_tensor * id_embeds = NULL; + struct ggml_tensor* id_embeds = NULL; if(pmv2){ - id_embeds = sd_ctx->sd->pmid_id_embeds->get(); + id_embeds = sd_ctx->sd->pmid_id_embeds->get(); // print_ggml_tensor(id_embeds, true, "id_embeds:"); } id_cond.c_crossattn = sd_ctx->sd->id_encoder(work_ctx, init_img, id_cond.c_crossattn, id_embeds, class_tokens_mask); From c17223edb2749af6a04e68d0155ee139715783a0 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Wed, 21 Aug 2024 14:40:42 -0400 Subject: [PATCH 07/20] updated chunk_half to be more efficient; redo feedforward --- pmid.hpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pmid.hpp b/pmid.hpp index 655183268..8d709f365 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -146,13 +146,12 @@ struct PerceiverAttention : public GGMLBlock { std::vector chunk_half(struct ggml_context* ctx, struct ggml_tensor* x){ - auto t = ggml_cont(ctx, ggml_permute(ctx, x, 2, 0, 1, 3)); - int64_t n = t->ne[2] / 2; - int64_t offset = t->nb[2] * n; - auto k = ggml_view_3d(ctx, t, t->ne[0], t->ne[1], n, t->nb[1], t->nb[2], offset*0); - auto v = ggml_view_3d(ctx, t, t->ne[0], t->ne[1], n, t->nb[1], t->nb[2], offset*1); - return {ggml_cont(ctx, ggml_permute(ctx, k, 1, 2, 0, 3)), - ggml_cont(ctx, ggml_permute(ctx, v, 1, 2, 0, 3))}; + + auto tlo = ggml_view_4d(ctx, x, x->ne[0]/2, x->ne[1], x->ne[2], x->ne[3], x->nb[1], x->nb[2], x->nb[3], 0); + auto tli = ggml_view_4d(ctx, x, x->ne[0]/2, x->ne[1], x->ne[2], x->ne[3], x->nb[1], x->nb[2], x->nb[3], x->nb[0]*x->ne[0]/2); + return {ggml_cont(ctx, tlo), + ggml_cont(ctx, tli)}; + } struct ggml_tensor* forward(struct ggml_context* ctx, @@ -278,7 +277,10 @@ struct FacePerceiverResampler : public GGMLBlock { auto ff = std::dynamic_pointer_cast(blocks[name]); auto t = attn->forward(ctx, x, latents); latents = ggml_add(ctx, t, latents); - t = ff->forward(ctx, latents); + auto latents2 = ggml_reshape_2d(ctx, latents, latents->ne[0], latents->ne[1]*latents->ne[2]); + // t = ff->forward(ctx, latents); + t = ff->forward(ctx, latents2); + t = ggml_reshape_3d(ctx, t, latents->ne[0], latents->ne[1], latents->ne[2]); latents = ggml_add(ctx, t, latents); } latents = proj_out->forward(ctx, latents); From 71b89571b06c51064eb0e1fbdc47624e1c187ef0 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Thu, 22 Aug 2024 20:56:14 -0400 Subject: [PATCH 08/20] fixed a bug: carefully using ggml_view_4d to get chunks of a tensor; strides need to be recalculated or set properly; still failing at soft_max cuda op --- pmid.hpp | 71 ++++++++++++++++++++++++++++++-------------- stable-diffusion.cpp | 1 + 2 files changed, 49 insertions(+), 23 deletions(-) diff --git a/pmid.hpp b/pmid.hpp index 8d709f365..77d182abb 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -135,10 +135,11 @@ struct PerceiverAttention : public GGMLBlock { // printf("heads = %d \n", heads); // x = ggml_view_4d(ctx, x, x->ne[0], x->ne[1], heads, x->ne[2]/heads, // x->nb[1], x->nb[2], x->nb[3], 0); - x = ggml_view_4d(ctx, x, x->ne[0]/heads, heads, x->ne[1], x->ne[2], - x->nb[1], x->nb[2], x->nb[3], 0); - // x = ggml_transpose(ctx, x); - x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); + x = ggml_reshape_4d(ctx, x, x->ne[0]/heads, heads, x->ne[1], x->ne[2]); + // x = ggml_view_4d(ctx, x, x->ne[0]/heads, heads, x->ne[1], x->ne[2], + // x->nb[1], x->nb[2], x->nb[3], 0); + // x = ggml_cont(ctx, x); + x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); // print_ggml_tensor(x, true, "PerceiverAttention reshape x 1: "); // x = ggml_reshape_4d(ctx, x, ne[0], heads, ne[1], ne[2]/heads); return x; @@ -181,14 +182,25 @@ struct PerceiverAttention : public GGMLBlock { auto kv_input = ggml_concat(ctx, x, latents, 1); auto to_kv = std::dynamic_pointer_cast(blocks["to_kv"]); auto kv = to_kv->forward(ctx, kv_input); - // int64_t n = kv->ne[0] / 2; - // int64_t offset = kv->nb[2] * n; print_ggml_tensor(kv, true, "PerceiverAttention kv: "); // auto k = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*0); // auto v = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*1); - std::vector chunks = chunk_half(ctx, kv); - auto k = chunks[0]; - auto v = chunks[1]; + // std::vector chunks = chunk_half(ctx, kv); + // auto k = chunks[0]; + // auto v = chunks[1]; + // printf("KV1: %ld, %ld, %ld, %ld \n", kv->ne[0], kv->ne[1], kv->ne[2], kv->ne[3]); + // printf("KV2: %ld, %ld, %ld, %ld \n", kv->nb[0], kv->nb[1], kv->nb[2], kv->nb[3]); + // printf("KV3: %ld, %ld, %ld, %ld \n", kv->nb[0]*kv->ne[0], kv->nb[1]*kv->ne[1], kv->nb[2]*kv->ne[2], kv->nb[3]*kv->ne[3]); + auto k = ggml_view_4d(ctx, kv, kv->ne[0]/2, kv->ne[1], kv->ne[2], kv->ne[3], kv->nb[1]/2, kv->nb[2]/2, kv->nb[3]/2, 0); + auto v = ggml_view_4d(ctx, kv, kv->ne[0]/2, kv->ne[1], kv->ne[2], kv->ne[3], kv->nb[1]/2, kv->nb[2]/2, kv->nb[3]/2, kv->nb[0]*(kv->ne[0]/2)); + // printf("K1: %ld, %ld, %ld, %ld \n", k->ne[0], k->ne[1], k->ne[2], k->ne[3]); + // printf("K2: %ld, %ld, %ld, %ld \n", k->nb[0], k->nb[1], k->nb[2], k->nb[3]); + // printf("K3: %ld, %ld, %ld, %ld \n", k->nb[0]*k->ne[0], k->nb[1]*k->ne[1], k->nb[2]*k->ne[2], k->nb[3]*k->ne[3]); + // printf("V1: %ld, %ld, %ld, %ld \n", v->ne[0], v->ne[1], v->ne[2], v->ne[3]); + // printf("V2: %ld, %ld, %ld, %ld \n", v->nb[0], v->nb[1], v->nb[2], v->nb[3]); + // printf("V3: %ld, %ld, %ld, %ld \n", v->nb[0]*v->ne[0], v->nb[1]*v->ne[1], v->nb[2]*v->ne[2], v->nb[3]*v->ne[3]); + k = ggml_cont(ctx, k); + v = ggml_cont(ctx, v); print_ggml_tensor(q, true, "PerceiverAttention bef reshape q: "); print_ggml_tensor(k, true, "PerceiverAttention bef reshape k: "); print_ggml_tensor(v, true, "PerceiverAttention bef reshape v: "); @@ -207,12 +219,14 @@ struct PerceiverAttention : public GGMLBlock { ggml_set_name(q, "PerceiverAttention q"); ggml_set_name(k, "PerceiverAttention k"); auto weight = ggml_mul_mat(ctx, q, k); + print_ggml_tensor(weight, true, "PerceiverAttention softmax weight: "); // GGML's softmax() is equivalent to pytorch's softmax(x, dim=-1) // in this case, dimension along which Softmax will be computed is the last dim // in torch and the first dim in GGML, consistent with the convention that pytorch's // last dimension (varying most rapidly) corresponds to GGML's first (varying most rapidly). - weight = ggml_soft_max(ctx, weight); + // weight = ggml_soft_max(ctx, weight); + weight = ggml_soft_max_inplace(ctx, weight); weight = ggml_cont(ctx, ggml_transpose(ctx, weight)); v = ggml_cont(ctx, ggml_transpose(ctx, v)); print_ggml_tensor(weight, true, "PerceiverAttention mul weight: "); @@ -277,10 +291,10 @@ struct FacePerceiverResampler : public GGMLBlock { auto ff = std::dynamic_pointer_cast(blocks[name]); auto t = attn->forward(ctx, x, latents); latents = ggml_add(ctx, t, latents); - auto latents2 = ggml_reshape_2d(ctx, latents, latents->ne[0], latents->ne[1]*latents->ne[2]); - // t = ff->forward(ctx, latents); - t = ff->forward(ctx, latents2); - t = ggml_reshape_3d(ctx, t, latents->ne[0], latents->ne[1], latents->ne[2]); + // auto latents2 = ggml_reshape_2d(ctx, latents, latents->ne[0], latents->ne[1]*latents->ne[2]); + t = ff->forward(ctx, latents); + // t = ff->forward(ctx, latents2); + // t = ggml_reshape_3d(ctx, t, latents->ne[0], latents->ne[1], latents->ne[2]); latents = ggml_add(ctx, t, latents); } latents = proj_out->forward(ctx, latents); @@ -502,6 +516,8 @@ struct FuseModule : public GGMLBlock { stacked_id_embeds = mlp2->forward(ctx, stacked_id_embeds); stacked_id_embeds = layer_norm->forward(ctx, stacked_id_embeds); + print_ggml_tensor(stacked_id_embeds, true, "Fuseblock stacked_id_embeds 1: "); + return stacked_id_embeds; } @@ -525,23 +541,29 @@ struct FuseModule : public GGMLBlock { ggml_set_name(image_token_embeds, "image_token_embeds"); valid_id_embeds = ggml_reshape_2d(ctx, valid_id_embeds, valid_id_embeds->ne[0], ggml_nelements(valid_id_embeds)/valid_id_embeds->ne[0]); - struct ggml_tensor* stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds); + struct ggml_tensor* stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds); - stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); + // stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); + print_ggml_tensor(stacked_id_embeds, true, "AA stacked_id_embeds"); + print_ggml_tensor(left, true, "AA left"); + print_ggml_tensor(right, true, "AA right"); if (left && right) { - stacked_id_embeds = ggml_concat(ctx, left, stacked_id_embeds, 2); - stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right, 2); + stacked_id_embeds = ggml_concat(ctx, left, stacked_id_embeds, 1); + stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right, 1); } else if (left) { - stacked_id_embeds = ggml_concat(ctx, left, stacked_id_embeds, 2); + stacked_id_embeds = ggml_concat(ctx, left, stacked_id_embeds, 1); } else if (right) { - stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right, 2); + stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right, 1); } - stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); + print_ggml_tensor(stacked_id_embeds, true, "BB stacked_id_embeds"); + // stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); + // print_ggml_tensor(stacked_id_embeds, true, "CC stacked_id_embeds"); class_tokens_mask = ggml_cont(ctx, ggml_transpose(ctx, class_tokens_mask)); class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds); prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask); struct ggml_tensor* updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds); ggml_set_name(updated_prompt_embeds, "updated_prompt_embeds"); + print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds: "); return updated_prompt_embeds; } }; @@ -765,11 +787,14 @@ struct PhotoMakerIDEncoder : public GGMLRunner { } // printf("\n"); if (ctmpos[0] > 0) { - left = ggml_new_tensor_3d(ctx0, type, hidden_size, 1, ctmpos[0]); + // left = ggml_new_tensor_3d(ctx0, type, hidden_size, 1, ctmpos[0]); + left = ggml_new_tensor_3d(ctx0, type, hidden_size, ctmpos[0], 1); } if (ctmpos[ctmpos.size() - 1] < seq_length - 1) { + // right = ggml_new_tensor_3d(ctx0, type, + // hidden_size, 1, seq_length - ctmpos[ctmpos.size() - 1] - 1); right = ggml_new_tensor_3d(ctx0, type, - hidden_size, 1, seq_length - ctmpos[ctmpos.size() - 1] - 1); + hidden_size, seq_length - ctmpos[ctmpos.size() - 1] - 1, 1); } struct ggml_tensor* class_tokens_mask_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ctmpos.size()); diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 993fb3a29..c6ce1a467 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -291,6 +291,7 @@ class StableDiffusionGGML { if(id_embeddings_path.find("v2") != std::string::npos) { // pmid_model = std::make_shared(clip_backend, model_data_type, version, VERSION_2); pmid_model = std::make_shared(backend, model_data_type, version, VERSION_2); + LOG_INFO("using PhotoMaker Version 2"); } else { pmid_model = std::make_shared(clip_backend, model_data_type, version); } From 707e345ec4d567f5b05b730b34d9d03438d9461a Mon Sep 17 00:00:00 2001 From: bssrdf Date: Thu, 22 Aug 2024 21:49:18 -0400 Subject: [PATCH 09/20] redo weight calculation and weight*v --- pmid.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pmid.hpp b/pmid.hpp index 77d182abb..1567c4568 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -218,7 +218,8 @@ struct PerceiverAttention : public GGMLBlock { print_ggml_tensor(k, true, "PerceiverAttention mul k: "); ggml_set_name(q, "PerceiverAttention q"); ggml_set_name(k, "PerceiverAttention k"); - auto weight = ggml_mul_mat(ctx, q, k); + // auto weight = ggml_mul_mat(ctx, q, k); + auto weight = ggml_mul_mat(ctx, k, q); print_ggml_tensor(weight, true, "PerceiverAttention softmax weight: "); // GGML's softmax() is equivalent to pytorch's softmax(x, dim=-1) @@ -227,7 +228,7 @@ struct PerceiverAttention : public GGMLBlock { // last dimension (varying most rapidly) corresponds to GGML's first (varying most rapidly). // weight = ggml_soft_max(ctx, weight); weight = ggml_soft_max_inplace(ctx, weight); - weight = ggml_cont(ctx, ggml_transpose(ctx, weight)); + // weight = ggml_cont(ctx, ggml_transpose(ctx, weight)); v = ggml_cont(ctx, ggml_transpose(ctx, v)); print_ggml_tensor(weight, true, "PerceiverAttention mul weight: "); print_ggml_tensor(v, true, "PerceiverAttention mul v: "); From 89d71d709fc44f8c64beda6fa6a0422581196157 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Fri, 23 Aug 2024 10:54:23 -0400 Subject: [PATCH 10/20] fixed a bug now Photomaker V2 kinds of working --- pmid.hpp | 101 +++++++------------------------------------------------ 1 file changed, 12 insertions(+), 89 deletions(-) diff --git a/pmid.hpp b/pmid.hpp index 1567c4568..e61a800bd 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -166,8 +166,6 @@ struct PerceiverAttention : public GGMLBlock { int64_t ne[4]; for(int i = 0; i < 4; ++i) ne[i] = latents->ne[i]; - print_ggml_tensor(x, true, "PerceiverAttention x 0: "); - print_ggml_tensor(latents, true, "PerceiverAttention latents 0: "); auto norm1 = std::dynamic_pointer_cast(blocks["norm1"]); auto norm2 = std::dynamic_pointer_cast(blocks["norm2"]); @@ -176,51 +174,21 @@ struct PerceiverAttention : public GGMLBlock { auto to_q = std::dynamic_pointer_cast(blocks["to_q"]); auto q = to_q->forward(ctx, latents); - print_ggml_tensor(x, true, "PerceiverAttention x: "); - print_ggml_tensor(latents, true, "PerceiverAttention latents: "); - print_ggml_tensor(q, true, "PerceiverAttention q: "); auto kv_input = ggml_concat(ctx, x, latents, 1); auto to_kv = std::dynamic_pointer_cast(blocks["to_kv"]); auto kv = to_kv->forward(ctx, kv_input); - print_ggml_tensor(kv, true, "PerceiverAttention kv: "); - // auto k = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*0); - // auto v = ggml_view_3d(ctx, kv, kv->ne[0], kv->ne[1], n, kv->nb[1], kv->nb[2], offset*1); - // std::vector chunks = chunk_half(ctx, kv); - // auto k = chunks[0]; - // auto v = chunks[1]; - // printf("KV1: %ld, %ld, %ld, %ld \n", kv->ne[0], kv->ne[1], kv->ne[2], kv->ne[3]); - // printf("KV2: %ld, %ld, %ld, %ld \n", kv->nb[0], kv->nb[1], kv->nb[2], kv->nb[3]); - // printf("KV3: %ld, %ld, %ld, %ld \n", kv->nb[0]*kv->ne[0], kv->nb[1]*kv->ne[1], kv->nb[2]*kv->ne[2], kv->nb[3]*kv->ne[3]); auto k = ggml_view_4d(ctx, kv, kv->ne[0]/2, kv->ne[1], kv->ne[2], kv->ne[3], kv->nb[1]/2, kv->nb[2]/2, kv->nb[3]/2, 0); auto v = ggml_view_4d(ctx, kv, kv->ne[0]/2, kv->ne[1], kv->ne[2], kv->ne[3], kv->nb[1]/2, kv->nb[2]/2, kv->nb[3]/2, kv->nb[0]*(kv->ne[0]/2)); - // printf("K1: %ld, %ld, %ld, %ld \n", k->ne[0], k->ne[1], k->ne[2], k->ne[3]); - // printf("K2: %ld, %ld, %ld, %ld \n", k->nb[0], k->nb[1], k->nb[2], k->nb[3]); - // printf("K3: %ld, %ld, %ld, %ld \n", k->nb[0]*k->ne[0], k->nb[1]*k->ne[1], k->nb[2]*k->ne[2], k->nb[3]*k->ne[3]); - // printf("V1: %ld, %ld, %ld, %ld \n", v->ne[0], v->ne[1], v->ne[2], v->ne[3]); - // printf("V2: %ld, %ld, %ld, %ld \n", v->nb[0], v->nb[1], v->nb[2], v->nb[3]); - // printf("V3: %ld, %ld, %ld, %ld \n", v->nb[0]*v->ne[0], v->nb[1]*v->ne[1], v->nb[2]*v->ne[2], v->nb[3]*v->ne[3]); k = ggml_cont(ctx, k); v = ggml_cont(ctx, v); - print_ggml_tensor(q, true, "PerceiverAttention bef reshape q: "); - print_ggml_tensor(k, true, "PerceiverAttention bef reshape k: "); - print_ggml_tensor(v, true, "PerceiverAttention bef reshape v: "); q = reshape_tensor(ctx, q, heads); k = reshape_tensor(ctx, k, heads); v = reshape_tensor(ctx, v, heads); - print_ggml_tensor(q, true, "PerceiverAttention aft reshape q: "); - print_ggml_tensor(k, true, "PerceiverAttention aft reshape k: "); - print_ggml_tensor(v, true, "PerceiverAttention aft reshape v: "); scale = 1.f / sqrt(sqrt((float)dim_head)); k = ggml_scale_inplace(ctx, k, scale); - // k = ggml_cont(ctx, ggml_permute(ctx, k, 1, 0, 2, 3)); q = ggml_scale_inplace(ctx, q, scale); - print_ggml_tensor(q, true, "PerceiverAttention mul q: "); - print_ggml_tensor(k, true, "PerceiverAttention mul k: "); - ggml_set_name(q, "PerceiverAttention q"); - ggml_set_name(k, "PerceiverAttention k"); // auto weight = ggml_mul_mat(ctx, q, k); - auto weight = ggml_mul_mat(ctx, k, q); - print_ggml_tensor(weight, true, "PerceiverAttention softmax weight: "); + auto weight = ggml_mul_mat(ctx, k, q); // NOTE order of mul is opposite to pytorch // GGML's softmax() is equivalent to pytorch's softmax(x, dim=-1) // in this case, dimension along which Softmax will be computed is the last dim @@ -228,14 +196,9 @@ struct PerceiverAttention : public GGMLBlock { // last dimension (varying most rapidly) corresponds to GGML's first (varying most rapidly). // weight = ggml_soft_max(ctx, weight); weight = ggml_soft_max_inplace(ctx, weight); - // weight = ggml_cont(ctx, ggml_transpose(ctx, weight)); v = ggml_cont(ctx, ggml_transpose(ctx, v)); - print_ggml_tensor(weight, true, "PerceiverAttention mul weight: "); - print_ggml_tensor(v, true, "PerceiverAttention mul v: "); - ggml_set_name(weight, "PerceiverAttention w"); - ggml_set_name(v, "PerceiverAttention v"); - auto out = ggml_mul_mat(ctx, weight, v); - + // auto out = ggml_mul_mat(ctx, weight, v); + auto out = ggml_mul_mat(ctx, v, weight); // NOTE order of mul is opposite to pytorch out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); out = ggml_reshape_3d(ctx, out, ne[0], ne[1], ggml_nelements(out)/(ne[0]*ne[1])); auto to_out = std::dynamic_pointer_cast(blocks["to_out"]); @@ -276,14 +239,6 @@ struct FacePerceiverResampler : public GGMLBlock { auto proj_out = std::dynamic_pointer_cast(blocks["proj_out"]); auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); - // std::map tensors; - // get_param_tensors(tensors, ""); - // for(auto& pair : tensors) { - // struct ggml_tensor* t = pair.second; - // std::string name = pair.first; - // ggml_set_name(t, name.c_str()); - // } - x = proj_in->forward(ctx, x); for (int i = 0; i < depth; i++) { std::string name = "layers." + std::to_string(i) + ".0"; @@ -292,10 +247,7 @@ struct FacePerceiverResampler : public GGMLBlock { auto ff = std::dynamic_pointer_cast(blocks[name]); auto t = attn->forward(ctx, x, latents); latents = ggml_add(ctx, t, latents); - // auto latents2 = ggml_reshape_2d(ctx, latents, latents->ne[0], latents->ne[1]*latents->ne[2]); t = ff->forward(ctx, latents); - // t = ff->forward(ctx, latents2); - // t = ggml_reshape_3d(ctx, t, latents->ne[0], latents->ne[1], latents->ne[2]); latents = ggml_add(ctx, t, latents); } latents = proj_out->forward(ctx, latents); @@ -315,9 +267,6 @@ struct QFormerPerceiver : public GGMLBlock { QFormerPerceiver(int id_embeddings_dim, int cross_attention_d, int num_t, int embedding_dim=1024, bool use_r=true, int ratio=4) : cross_attention_dim(cross_attention_d), num_tokens(num_t), use_residul(use_r) { - // blocks["mlp1"] = std::shared_ptr(new FuseBlock(imb_d * 2, imb_d, imb_d, false)); - // blocks["mlp2"] = std::shared_ptr(new FuseBlock(imb_d, imb_d, imb_d, true)); - // blocks["layer_norm"] = std::shared_ptr(new LayerNorm(embed_dim)); blocks["token_proj"] = std::shared_ptr(new Mlp(id_embeddings_dim, id_embeddings_dim*ratio, cross_attention_dim*num_tokens, @@ -494,8 +443,8 @@ struct FuseModule : public GGMLBlock { auto mlp2 = std::dynamic_pointer_cast(blocks["mlp2"]); auto layer_norm = std::dynamic_pointer_cast(blocks["layer_norm"]); - print_ggml_tensor(id_embeds, true, "Fuseblock id_embeds: "); - print_ggml_tensor(prompt_embeds, true, "Fuseblock prompt_embeds: "); + // print_ggml_tensor(id_embeds, true, "Fuseblock id_embeds: "); + // print_ggml_tensor(prompt_embeds, true, "Fuseblock prompt_embeds: "); // auto prompt_embeds0 = ggml_cont(ctx, ggml_permute(ctx, prompt_embeds, 2, 0, 1, 3)); // auto id_embeds0 = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3)); @@ -504,7 +453,7 @@ struct FuseModule : public GGMLBlock { // concat is along dim 2 // auto stacked_id_embeds = ggml_concat(ctx, prompt_embeds0, id_embeds0, 2); auto stacked_id_embeds = ggml_concat(ctx, prompt_embeds, id_embeds, 0); - print_ggml_tensor(stacked_id_embeds, true, "Fuseblock stacked_id_embeds 0: "); + // print_ggml_tensor(stacked_id_embeds, true, "Fuseblock stacked_id_embeds 0: "); // stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 1, 2, 0, 3)); // print_ggml_tensor(stacked_id_embeds, true, "Fuseblock stacked_id_embeds 1: "); // stacked_id_embeds = mlp1.forward(ctx, stacked_id_embeds); @@ -517,7 +466,7 @@ struct FuseModule : public GGMLBlock { stacked_id_embeds = mlp2->forward(ctx, stacked_id_embeds); stacked_id_embeds = layer_norm->forward(ctx, stacked_id_embeds); - print_ggml_tensor(stacked_id_embeds, true, "Fuseblock stacked_id_embeds 1: "); + // print_ggml_tensor(stacked_id_embeds, true, "Fuseblock stacked_id_embeds 1: "); return stacked_id_embeds; } @@ -545,9 +494,9 @@ struct FuseModule : public GGMLBlock { struct ggml_tensor* stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds); // stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); - print_ggml_tensor(stacked_id_embeds, true, "AA stacked_id_embeds"); - print_ggml_tensor(left, true, "AA left"); - print_ggml_tensor(right, true, "AA right"); + // print_ggml_tensor(stacked_id_embeds, true, "AA stacked_id_embeds"); + // print_ggml_tensor(left, true, "AA left"); + // print_ggml_tensor(right, true, "AA right"); if (left && right) { stacked_id_embeds = ggml_concat(ctx, left, stacked_id_embeds, 1); stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right, 1); @@ -556,7 +505,7 @@ struct FuseModule : public GGMLBlock { } else if (right) { stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right, 1); } - print_ggml_tensor(stacked_id_embeds, true, "BB stacked_id_embeds"); + // print_ggml_tensor(stacked_id_embeds, true, "BB stacked_id_embeds"); // stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); // print_ggml_tensor(stacked_id_embeds, true, "CC stacked_id_embeds"); class_tokens_mask = ggml_cont(ctx, ggml_transpose(ctx, class_tokens_mask)); @@ -564,7 +513,7 @@ struct FuseModule : public GGMLBlock { prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask); struct ggml_tensor* updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds); ggml_set_name(updated_prompt_embeds, "updated_prompt_embeds"); - print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds: "); + // print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds: "); return updated_prompt_embeds; } }; @@ -664,34 +613,8 @@ struct PhotoMakerIDEncoder_CLIPInsightfaceExtendtokenBlock : public CLIPVisionMo // struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values); // [N, hidden_size] struct ggml_tensor* last_hidden_state = vision_model->forward(ctx, id_pixel_values, false); // [N, hidden_size] - print_ggml_tensor(id_pixel_values, true, "PhotoMakerIDEncoder2 id_pixel_values: "); - print_ggml_tensor(last_hidden_state, true, "PhotoMakerIDEncoder2 last_hidden_state: "); - print_ggml_tensor(id_embeds, true, "PhotoMakerIDEncoder2 id_embeds0: "); id_embeds = qformer_perceiver->forward(ctx, id_embeds, last_hidden_state); - // id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3)); - print_ggml_tensor(id_embeds, true, "PhotoMakerIDEncoder2 id_embeds: "); - print_ggml_tensor(prompt_embeds, true, "PhotoMakerIDEncoder2 prompt_embeds: "); - // if(id_embeds->backend == GGML_BACKEND_TYPE_GPU){ - // if(!ggml_backend_buffer_is_host(id_embeds->buffer)){ - // printf("id_embeds on GPU\n"); - // } - // else{ - // printf("id_embeds on CPU\n"); - // } - // if(prompt_embeds->buffer != NULL && !ggml_backend_buffer_is_host(prompt_embeds->buffer)){ - // if(prompt_embeds->backend == GGML_BACKEND_TYPE_GPU){ - // printf("prompt_embeds on GPU\n"); - // } - // else{ - // if(prompt_embeds->buffer == NULL){ - // printf("prompt_embeds buffer is NULL\n"); - // } - // printf("prompt_embeds on CPU\n"); - // } - // id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2, 2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right - // id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3)); - struct ggml_tensor* updated_prompt_embeds = fuse_module->forward(ctx, prompt_embeds, id_embeds, From 161afa8725a2d4a07c7fde70cab87663745ca08d Mon Sep 17 00:00:00 2001 From: bssrdf Date: Fri, 23 Aug 2024 10:59:14 -0400 Subject: [PATCH 11/20] add python script for face detection (Photomaker V2 needs) --- face_detect.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 face_detect.py diff --git a/face_detect.py b/face_detect.py new file mode 100644 index 000000000..46fd9462d --- /dev/null +++ b/face_detect.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Aug 18 15:52:46 2024 + +@author: merli +""" +import os +import sys + +import numpy as np +import torch +from diffusers.utils import load_image +# pip install insightface==0.7.3 +from insightface.app import FaceAnalysis +from insightface.data import get_image as ins_get_image +from safetensors.torch import save_file + +### +# https://github.com/cubiq/ComfyUI_IPAdapter_plus/issues/165#issue-2055829543 +### +class FaceAnalysis2(FaceAnalysis): + # NOTE: allows setting det_size for each detection call. + # the model allows it but the wrapping code from insightface + # doesn't show it, and people end up loading duplicate models + # for different sizes where there is absolutely no need to + def get(self, img, max_num=0, det_size=(640, 640)): + if det_size is not None: + self.det_model.input_size = det_size + + return super().get(img, max_num) + +def analyze_faces(face_analysis: FaceAnalysis, img_data: np.ndarray, det_size=(640, 640)): + # NOTE: try detect faces, if no faces detected, lower det_size until it does + detection_sizes = [None] + [(size, size) for size in range(640, 256, -64)] + [(256, 256)] + + for size in detection_sizes: + faces = face_analysis.get(img_data, det_size=size) + if len(faces) > 0: + return faces + + return [] + +if __name__ == "__main__": + #face_detector = FaceAnalysis2(providers=['CUDAExecutionProvider'], allowed_modules=['detection', 'recognition']) + face_detector = FaceAnalysis2(providers=['CPUExecutionProvider'], allowed_modules=['detection', 'recognition']) + face_detector.prepare(ctx_id=0, det_size=(640, 640)) + #input_folder_name = './scarletthead_woman' + #input_folder_name = './Elizabeth' + input_folder_name = sys.argv[1] + image_basename_list = os.listdir(input_folder_name) + image_path_list = sorted([os.path.join(input_folder_name, basename) for basename in image_basename_list]) + + input_id_images = [] + for image_path in image_path_list: + input_id_images.append(load_image(image_path)) + + id_embed_list = [] + + for img in input_id_images: + img = np.array(img) + img = img[:, :, ::-1] + faces = analyze_faces(face_detector, img) + if len(faces) > 0: + id_embed_list.append(torch.from_numpy((faces[0]['embedding']))) + + if len(id_embed_list) == 0: + raise ValueError(f"No face detected in input image pool") + + id_embeds = torch.stack(id_embed_list) + + for r in id_embeds: + print(r) + #torch.save(id_embeds, input_folder_name+'/id_embeds.pt'); + weights = dict() + weights["id_embeds"] = id_embeds + save_file(weights, input_folder_name+'/id_embeds.safetensors') + + \ No newline at end of file From 1afffa42b23e003b47acf624118153ef254e0d5f Mon Sep 17 00:00:00 2001 From: bssrdf Date: Fri, 23 Aug 2024 13:46:05 -0400 Subject: [PATCH 12/20] updated readme for photomaker --- docs/photo_maker.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/photo_maker.md b/docs/photo_maker.md index b69ad97d9..4d279e30e 100644 --- a/docs/photo_maker.md +++ b/docs/photo_maker.md @@ -29,4 +29,24 @@ Example: ```bash bin/sd -m ../models/sdxlUnstableDiffusers_v11.safetensors --vae ../models/sdxl_vae.safetensors --stacked-id-embd-dir ../models/photomaker-v1.safetensors --input-id-images-dir ../assets/photomaker_examples/scarletthead_woman -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" --cfg-scale 5.0 --sampling-method euler -H 1024 -W 1024 --style-ratio 10 --vae-on-cpu -o output.png -``` \ No newline at end of file +``` + +## PhotoMaker Version 2 + +[PhotoMaker Version 2 (PMV2)](https://github.com/TencentARC/PhotoMaker/blob/main/README_pmv2.md) has some key improvements. Unfortunately it has a very heavy dependency which makes running it a bit involved in ```SD.cpp```. + +Running PMV2 is now a two-step process: + +- Run a python script ```face_detect.py``` to obtain **id_embeds** for the given input images +``` +python face_detect.py input_image_dir +``` +An ```id_embeds.safetensors``` file will be generated in ```input_images_dir``` + +- Run the same command as in version 1 but replacing ```photomaker-v1.safetensors``` with ```photomaker-v2.safetensors```. + + You can download ```photomaker-v2.safetensors``` from [here](https://huggingface.co/bssrdf/PhotoMakerV2) + +- All the command line parameters from Version 1 remain the same for Version 2 + + From da576c4af8cbe17156a863cc055f38f91424ffc9 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Fri, 23 Aug 2024 18:16:37 -0400 Subject: [PATCH 13/20] fixed a bug causing PMV1 crashing; both V1 and V2 work --- conditioner.hpp | 16 ++++++++++------ model.h | 5 +++++ pmid.hpp | 9 +++++---- stable-diffusion.cpp | 12 ++++++++---- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/conditioner.hpp b/conditioner.hpp index 8cb5518ae..9a8c634b1 100644 --- a/conditioner.hpp +++ b/conditioner.hpp @@ -4,6 +4,7 @@ #include "clip.hpp" #include "t5.hpp" + struct SDCondition { struct ggml_tensor* c_crossattn = NULL; // aka context struct ggml_tensor* c_vector = NULL; // aka y @@ -44,6 +45,7 @@ struct Conditioner { // Ref: https://github.com/AUTOMATIC1111/stable-diffusion-webui/blob/cad87bf4e3e0b0a759afa94e933527c3123d59bc/modules/sd_hijack_clip.py#L283 struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { SDVersion version = VERSION_1_x; + PMVersion pm_version = VERSION_1; CLIPTokenizer tokenizer; ggml_type wtype; std::shared_ptr text_model; @@ -59,8 +61,9 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { ggml_type wtype, const std::string& embd_dir, SDVersion version = VERSION_1_x, + PMVersion pv = VERSION_1, int clip_skip = -1) - : version(version), tokenizer(version == VERSION_2_x ? 0 : 49407), embd_dir(embd_dir), wtype(wtype) { + : version(version), pm_version(pv), tokenizer(version == VERSION_2_x ? 0 : 49407), embd_dir(embd_dir), wtype(wtype) { if (clip_skip <= 0) { clip_skip = 1; if (version == VERSION_2_x || version == VERSION_XL) { @@ -159,7 +162,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { tokenize_with_trigger_token(std::string text, int num_input_imgs, int32_t image_token, - bool padding = false) { + bool padding = false){ return tokenize_with_trigger_token(text, num_input_imgs, image_token, text_model->model.n_token, padding); } @@ -283,9 +286,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { weights.insert(weights.begin(), 1.0); tokenizer.pad_tokens(tokens, weights, max_length, padding); - + int offset = pm_version == VERSION_2 ? 2*num_input_imgs: num_input_imgs; for (uint32_t i = 0; i < tokens.size(); i++) { - if (class_idx + 1 <= i && i < class_idx + 1 + 2*num_input_imgs) // photomaker V2 has num_tokens(=2)*num_input_imgs + // if (class_idx + 1 <= i && i < class_idx + 1 + 2*num_input_imgs) // photomaker V2 has num_tokens(=2)*num_input_imgs + if (class_idx + 1 <= i && i < class_idx + 1 + offset) // photomaker V2 has num_tokens(=2)*num_input_imgs // hardcode for now class_token_mask.push_back(true); else @@ -531,7 +535,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { int height, int num_input_imgs, int adm_in_channels = -1, - bool force_zero_embeddings = false) { + bool force_zero_embeddings = false){ auto image_tokens = convert_token_to_id(trigger_word); // if(image_tokens.size() == 1){ // printf(" image token id is: %d \n", image_tokens[0]); @@ -969,7 +973,7 @@ struct SD3CLIPEmbedder : public Conditioner { int height, int num_input_imgs, int adm_in_channels = -1, - bool force_zero_embeddings = false) { + bool force_zero_embeddings = false){ GGML_ASSERT(0 && "Not implemented yet!"); } diff --git a/model.h b/model.h index 3e1a50223..8f6f0298f 100644 --- a/model.h +++ b/model.h @@ -26,6 +26,11 @@ enum SDVersion { VERSION_COUNT, }; +enum PMVersion { + VERSION_1, + VERSION_2, +}; + struct TensorStorage { std::string name; ggml_type type = GGML_TYPE_F32; diff --git a/pmid.hpp b/pmid.hpp index e61a800bd..6187b117d 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -7,10 +7,6 @@ #include "lora.hpp" -enum PMVersion { - VERSION_1, - VERSION_2, -}; struct FuseBlock : public GGMLBlock { // network hparams int in_dim; @@ -660,6 +656,11 @@ struct PhotoMakerIDEncoder : public GGMLRunner { return "pmid"; } + PMVersion get_version() const{ + return pm_version; + } + + void get_param_tensors(std::map& tensors, const std::string prefix) { if(pm_version == VERSION_1) id_encoder.get_param_tensors(tensors, prefix); diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index c6ce1a467..337dd7e24 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -247,7 +247,11 @@ class StableDiffusionGGML { cond_stage_model = std::make_shared(clip_backend, model_data_type); diffusion_model = std::make_shared(backend, model_data_type, version); } else { - cond_stage_model = std::make_shared(clip_backend, model_data_type, embeddings_path, version); + if(id_embeddings_path.find("v2") != std::string::npos) { + cond_stage_model = std::make_shared(clip_backend, model_data_type, embeddings_path, version, VERSION_2); + }else{ + cond_stage_model = std::make_shared(clip_backend, model_data_type, embeddings_path, version); + } diffusion_model = std::make_shared(backend, model_data_type, version); } cond_stage_model->alloc_params_buffer(); @@ -316,6 +320,7 @@ class StableDiffusionGGML { LOG_ERROR("load photomaker id embed tensors from %s failed", id_embed_path.c_str()); return false; } + } } } @@ -1068,14 +1073,13 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx, } // preprocess input id images std::vector input_id_images; - bool pmv2 = false; + bool pmv2 = sd_ctx->sd->pmid_model->get_version() == VERSION_2; if (sd_ctx->sd->pmid_model && input_id_images_path.size() > 0) { std::vector img_files = get_files_from_dir(input_id_images_path); for (std::string img_file : img_files) { int c = 0; int width, height; - if(ends_with(img_file, "id_embeds.safetensors")){ - pmv2 = true; + if(ends_with(img_file, "safetensors")){ continue; } uint8_t* input_image_buffer = stbi_load(img_file.c_str(), &width, &height, &c, 3); From ac8acac90c05abe1fc6c3640289ead6aaac955e3 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Mon, 26 Aug 2024 11:43:05 -0400 Subject: [PATCH 14/20] fixed clean_input_ids for PMV2 --- conditioner.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conditioner.hpp b/conditioner.hpp index 9a8c634b1..6831802d3 100644 --- a/conditioner.hpp +++ b/conditioner.hpp @@ -271,7 +271,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { std::vector clean_input_ids_tmp; for (uint32_t i = 0; i < class_token_index[0]; i++) clean_input_ids_tmp.push_back(clean_input_ids[i]); - for (uint32_t i = 0; i < num_input_imgs; i++) + for (uint32_t i = 0; i < (pm_version == VERSION_2 ? 2*num_input_imgs: num_input_imgs); i++) clean_input_ids_tmp.push_back(class_token); for (uint32_t i = class_token_index[0] + 1; i < clean_input_ids.size(); i++) clean_input_ids_tmp.push_back(clean_input_ids[i]); From 8dda9185387643c26116cd3013f616caf8e2454a Mon Sep 17 00:00:00 2001 From: bssrdf Date: Mon, 26 Aug 2024 21:04:36 -0400 Subject: [PATCH 15/20] fixed a double counting bug in tokenize_with_trigger_token --- conditioner.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conditioner.hpp b/conditioner.hpp index 6831802d3..5eee0ca2b 100644 --- a/conditioner.hpp +++ b/conditioner.hpp @@ -282,8 +282,9 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { tokens.insert(tokens.end(), clean_input_ids.begin(), clean_input_ids.end()); weights.insert(weights.end(), clean_input_ids.size(), curr_weight); } - tokens.insert(tokens.begin(), tokenizer.BOS_TOKEN_ID); - weights.insert(weights.begin(), 1.0); + // BUG!! double couting, pad_tokens will add BOS at the beginning + // tokens.insert(tokens.begin(), tokenizer.BOS_TOKEN_ID); + // weights.insert(weights.begin(), 1.0); tokenizer.pad_tokens(tokens, weights, max_length, padding); int offset = pm_version == VERSION_2 ? 2*num_input_imgs: num_input_imgs; From 66f48d1ef6ed299ff99752bbb98118e28ebf2c3d Mon Sep 17 00:00:00 2001 From: bssrdf Date: Tue, 27 Aug 2024 10:09:06 -0400 Subject: [PATCH 16/20] updated photomaker readme --- docs/photo_maker.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/photo_maker.md b/docs/photo_maker.md index 4d279e30e..8305a33bd 100644 --- a/docs/photo_maker.md +++ b/docs/photo_maker.md @@ -43,6 +43,8 @@ python face_detect.py input_image_dir ``` An ```id_embeds.safetensors``` file will be generated in ```input_images_dir``` +**Note: this step is only needed to run once; the same ```id_embeds``` can be reused** + - Run the same command as in version 1 but replacing ```photomaker-v1.safetensors``` with ```photomaker-v2.safetensors```. You can download ```photomaker-v2.safetensors``` from [here](https://huggingface.co/bssrdf/PhotoMakerV2) From cbf3108772b0079c0ee34b7b281bcace175b5b4a Mon Sep 17 00:00:00 2001 From: bssrdf Date: Wed, 28 Aug 2024 11:48:07 -0400 Subject: [PATCH 17/20] removed some commented code --- stable-diffusion.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 1fd5c1ba4..782f994eb 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -388,7 +388,6 @@ class StableDiffusionGGML { LOG_ERROR("load photomaker id embed tensors from %s failed", id_embed_path.c_str()); return false; } - } } } @@ -397,14 +396,8 @@ class StableDiffusionGGML { LOG_ERROR(" pmid model params buffer allocation failed"); return false; } - // LOG_INFO("pmid param memory buffer size = %.2fMB ", - // pmid_model->params_buffer_size / 1024.0 / 1024.0); pmid_model->get_param_tensors(tensors, "pmid"); } - // if(stacked_id){ - // pmid_model.init_params(GGML_TYPE_F32); - // pmid_model.map_by_name(tensors, "pmid."); - // } } struct ggml_init_params params; From 19d5939b3d8e38f7728d39df2409c5af06b22ac8 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Fri, 30 Aug 2024 10:42:01 -0400 Subject: [PATCH 18/20] improved reconstructing class word free prompt --- clip.hpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/clip.hpp b/clip.hpp index 402b50355..eebe4b3c1 100644 --- a/clip.hpp +++ b/clip.hpp @@ -343,6 +343,14 @@ class CLIPTokenizer { } } + std::string clean_up_tokenization(std::string &text){ + + std::regex pattern(R"( ,)"); + // Replace " ," with "," + std::string result = std::regex_replace(text, pattern, ","); + return result; + } + std::string decode(const std::vector& tokens) { std::string text = ""; for (int t : tokens) { @@ -351,8 +359,12 @@ class CLIPTokenizer { std::u32string ts = decoder[t]; // printf("%d, %s \n", t, utf32_to_utf8(ts).c_str()); std::string s = utf32_to_utf8(ts); - if (s.length() >= 4 && ends_with(s, "")) { - text += " " + s.replace(s.length() - 4, s.length() - 1, ""); + if (s.length() >= 4 ){ + if(ends_with(s, "")) { + text += s.replace(s.length() - 4, s.length() - 1, "") + " "; + }else{ + text += s; + } } else { text += " " + s; } @@ -364,6 +376,7 @@ class CLIPTokenizer { // std::string s((char *)bytes.data()); // std::string s = ""; + text = clean_up_tokenization(text); return trim(text); } From 6157a7649a1230ce82d8e0002e199343c53413a8 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Fri, 25 Oct 2024 12:46:32 -0400 Subject: [PATCH 19/20] changed reading id_embed to raw binary using existing load tensor function; this is more efficient than using model load and also makes it easier to work with sd server --- examples/cli/main.cpp | 1 - face_detect.py | 29 +++++++++++++++++++++++------ stable-diffusion.cpp | 18 ++---------------- stable-diffusion.h | 1 - 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 19ba9cf1b..ceae27b83 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -773,7 +773,6 @@ int main(int argc, const char* argv[]) { params.lora_model_dir.c_str(), params.embeddings_path.c_str(), params.stacked_id_embeddings_path.c_str(), - params.input_id_images_path.c_str(), vae_decode_only, params.vae_tiling, true, diff --git a/face_detect.py b/face_detect.py index 46fd9462d..3d04a74b1 100644 --- a/face_detect.py +++ b/face_detect.py @@ -68,11 +68,28 @@ def analyze_faces(face_analysis: FaceAnalysis, img_data: np.ndarray, det_size=(6 id_embeds = torch.stack(id_embed_list) - for r in id_embeds: - print(r) - #torch.save(id_embeds, input_folder_name+'/id_embeds.pt'); - weights = dict() - weights["id_embeds"] = id_embeds - save_file(weights, input_folder_name+'/id_embeds.safetensors') + # for r in id_embeds: + # print(r) + # #torch.save(id_embeds, input_folder_name+'/id_embeds.pt'); + # weights = dict() + # weights["id_embeds"] = id_embeds + # save_file(weights, input_folder_name+'/id_embeds.safetensors') + + binary_data = id_embeds.numpy().tobytes() + two = 4 + zero = 0 + one = 1 + tensor_name = "id_embeds" +# Write binary data to a file + with open(input_folder_name+'/id_embeds.bin', "wb") as f: + f.write(two.to_bytes(4, byteorder='little')) + f.write((len(tensor_name)).to_bytes(4, byteorder='little')) + f.write(zero.to_bytes(4, byteorder='little')) + f.write((id_embeds.shape[1]).to_bytes(4, byteorder='little')) + f.write((id_embeds.shape[0]).to_bytes(4, byteorder='little')) + f.write(one.to_bytes(4, byteorder='little')) + f.write(one.to_bytes(4, byteorder='little')) + f.write(tensor_name.encode('ascii')) + f.write(binary_data) \ No newline at end of file diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 782f994eb..d642f2219 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -146,7 +146,6 @@ class StableDiffusionGGML { const std::string control_net_path, const std::string embeddings_path, const std::string id_embeddings_path, - const std::string input_id_images_path, const std::string& taesd_path, bool vae_tiling_, ggml_type wtype, @@ -379,17 +378,6 @@ class StableDiffusionGGML { } else { stacked_id = true; } - if(stacked_id){ - if(id_embeddings_path.find("v2") != std::string::npos) { - std::string id_embed_path = path_join(input_id_images_path, "id_embeds.safetensors"); - pmid_id_embeds = std::make_shared(backend, model_wtype, - &model_loader, id_embed_path, "pmid."); - if (!pmid_id_embeds->load_from_file(true)) { - LOG_ERROR("load photomaker id embed tensors from %s failed", id_embed_path.c_str()); - return false; - } - } - } } if (stacked_id) { if (!pmid_model->alloc_params_buffer()) { @@ -1032,7 +1020,6 @@ sd_ctx_t* new_sd_ctx(const char* model_path_c_str, const char* lora_model_dir_c_str, const char* embed_dir_c_str, const char* id_embed_dir_c_str, - const char* input_id_images_dir_c_str, bool vae_decode_only, bool vae_tiling, bool free_params_immediately, @@ -1056,7 +1043,6 @@ sd_ctx_t* new_sd_ctx(const char* model_path_c_str, std::string control_net_path(control_net_path_c_str); std::string embd_path(embed_dir_c_str); std::string id_embd_path(id_embed_dir_c_str); - std::string input_id_images_path(input_id_images_dir_c_str); std::string lora_model_dir(lora_model_dir_c_str); sd_ctx->sd = new StableDiffusionGGML(n_threads, @@ -1076,7 +1062,6 @@ sd_ctx_t* new_sd_ctx(const char* model_path_c_str, control_net_path, embd_path, id_embd_path, - input_id_images_path, taesd_path, vae_tiling, (ggml_type)wtype, @@ -1226,7 +1211,8 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx, class_tokens_mask = std::get<1>(cond_tup); // struct ggml_tensor* id_embeds = NULL; if(pmv2){ - id_embeds = sd_ctx->sd->pmid_id_embeds->get(); + // id_embeds = sd_ctx->sd->pmid_id_embeds->get(); + id_embeds = load_tensor_from_file(work_ctx, path_join(input_id_images_path, "id_embeds.bin")); // print_ggml_tensor(id_embeds, true, "id_embeds:"); } id_cond.c_crossattn = sd_ctx->sd->id_encoder(work_ctx, init_img, id_cond.c_crossattn, id_embeds, class_tokens_mask); diff --git a/stable-diffusion.h b/stable-diffusion.h index 3de5b1f57..0d4cc1fda 100644 --- a/stable-diffusion.h +++ b/stable-diffusion.h @@ -132,7 +132,6 @@ SD_API sd_ctx_t* new_sd_ctx(const char* model_path, const char* lora_model_dir, const char* embed_dir_c_str, const char* stacked_id_embed_dir_c_str, - const char* input_id_images_dir_c_str, bool vae_decode_only, bool vae_tiling, bool free_params_immediately, From 5d5b9d384d2efc68f45f32395474fd9b1abe5337 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Fri, 25 Oct 2024 13:02:16 -0400 Subject: [PATCH 20/20] minor clean up --- face_detect.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/face_detect.py b/face_detect.py index 3d04a74b1..7131af31f 100644 --- a/face_detect.py +++ b/face_detect.py @@ -1,9 +1,3 @@ -# -*- coding: utf-8 -*- -""" -Created on Sun Aug 18 15:52:46 2024 - -@author: merli -""" import os import sys @@ -45,7 +39,6 @@ def analyze_faces(face_analysis: FaceAnalysis, img_data: np.ndarray, det_size=(6 face_detector = FaceAnalysis2(providers=['CPUExecutionProvider'], allowed_modules=['detection', 'recognition']) face_detector.prepare(ctx_id=0, det_size=(640, 640)) #input_folder_name = './scarletthead_woman' - #input_folder_name = './Elizabeth' input_folder_name = sys.argv[1] image_basename_list = os.listdir(input_folder_name) image_path_list = sorted([os.path.join(input_folder_name, basename) for basename in image_basename_list])