From f04aa6e65c8d558651e4f3c4656c3406df781c0a Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 30 Dec 2021 17:51:09 +0800 Subject: [PATCH 1/3] Fix missing LFS objects --- models/lfs.go | 37 +++++++++++++++++------ routers/web/repo/lfs.go | 28 +++++++++++------ templates/repo/settings/lfs_pointers.tmpl | 2 +- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index 56924ffcf2d56..7b5e4ca6f1ba5 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -7,6 +7,7 @@ package models import ( "context" "errors" + "fmt" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" @@ -145,6 +146,11 @@ func LFSObjectAccessible(user *user_model.User, oid string) (bool, error) { return count > 0, err } +// LFSObjectIsAssociated checks if a provided Oid is associated +func LFSObjectIsAssociated(oid string) (bool, error) { + return x.Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) +} + // LFSAutoAssociate auto associates accessible LFSMetaObjects func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int64) error { ctx, committer, err := db.TxContext() @@ -162,18 +168,29 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6 oidMap[meta.Oid] = meta } - cond := builder.NewCond() + newMetas := make([]*LFSMetaObject, 0, len(metas)) + if !user.IsAdmin { - cond = builder.In("`lfs_meta_object`.repository_id", + cond := builder.In("`lfs_meta_object`.repository_id", builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user))) - } - newMetas := make([]*LFSMetaObject, 0, len(metas)) - if err := sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil { - return err - } - for i := range newMetas { - newMetas[i].Size = oidMap[newMetas[i].Oid].Size - newMetas[i].RepositoryID = repoID + if err := sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil { + return err + } + if len(newMetas) != len(oidMap) { + return fmt.Errorf("can not collect all LFS objects from database, expected %d, actually %d", len(oidMap), len(newMetas)) + } + for i := range newMetas { + newMetas[i].Size = oidMap[newMetas[i].Oid].Size + newMetas[i].RepositoryID = repoID + } + } else { + // admin can associate any LFS object to any repository + for i := range metas { + newMetas = append(newMetas, &LFSMetaObject{ + Pointer: lfs.Pointer{Oid: metas[i].Oid, Size: metas[i].Size}, + RepositoryID: repoID, + }) + } } if err := db.Insert(ctx, newMetas); err != nil { return err diff --git a/routers/web/repo/lfs.go b/routers/web/repo/lfs.go index 28d6b12860af1..6cc05430dde0d 100644 --- a/routers/web/repo/lfs.go +++ b/routers/web/repo/lfs.go @@ -421,12 +421,13 @@ func LFSPointerFiles(ctx *context.Context) { var numAssociated, numNoExist, numAssociatable int type pointerResult struct { - SHA string - Oid string - Size int64 - InRepo bool - Exists bool - Accessible bool + SHA string + Oid string + Size int64 + InRepo bool + Exists bool + Accessible bool + Associatable bool } results := []pointerResult{} @@ -461,22 +462,29 @@ func LFSPointerFiles(ctx *context.Context) { // Can we fix? // OK well that's "simple" // - we need to check whether current user has access to a repo that has access to the file - result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid) + result.Associatable, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid) if err != nil { return err } - } else { - result.Accessible = true + if !result.Associatable { + associated, err := models.LFSObjectIsAssociated(pointerBlob.Oid) + if err != nil { + return err + } + result.Associatable = !associated + } } } + result.Accessible = result.InRepo || result.Associatable + if result.InRepo { numAssociated++ } if !result.Exists { numNoExist++ } - if !result.InRepo && result.Accessible { + if result.Associatable { numAssociatable++ } diff --git a/templates/repo/settings/lfs_pointers.tmpl b/templates/repo/settings/lfs_pointers.tmpl index bf23062ae3cc7..440e544232558 100644 --- a/templates/repo/settings/lfs_pointers.tmpl +++ b/templates/repo/settings/lfs_pointers.tmpl @@ -11,7 +11,7 @@
{{.CsrfTokenHtml}} {{range .Pointers}} - {{if and (not .InRepo) .Exists .Accessible}} + {{if .Associatable}} {{end}} {{end}} From eaec30e88285dedc5ce3251f5dd8ca4c7960a71d Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 31 Dec 2021 14:47:03 +0800 Subject: [PATCH 2/3] ignore duplicated error # Conflicts: # models/lfs.go --- models/lfs.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index 7b5e4ca6f1ba5..0117a596310c2 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -183,20 +183,19 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6 newMetas[i].Size = oidMap[newMetas[i].Oid].Size newMetas[i].RepositoryID = repoID } + if _, err := sess.InsertMulti(newMetas); err != nil { + return err + } } else { - // admin can associate any LFS object to any repository + // admin can associate any LFS object to any repository, and we do not care about duplicated errors for i := range metas { - newMetas = append(newMetas, &LFSMetaObject{ + _, _ = sess.Insert(&LFSMetaObject{ Pointer: lfs.Pointer{Oid: metas[i].Oid, Size: metas[i].Size}, RepositoryID: repoID, }) } } - if err := db.Insert(ctx, newMetas); err != nil { - return err - } - - return committer.Commit() + return sess.Commit() } // IterateLFS iterates lfs object From 79ba0357d445ae96f3f42ba65b9f7198f5907886 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 1 Jan 2022 00:34:33 +0800 Subject: [PATCH 3/3] Fix for 1.16 --- models/lfs.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index 0117a596310c2..cf596f5468382 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -13,6 +13,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" @@ -148,7 +149,7 @@ func LFSObjectAccessible(user *user_model.User, oid string) (bool, error) { // LFSObjectIsAssociated checks if a provided Oid is associated func LFSObjectIsAssociated(oid string) (bool, error) { - return x.Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) + return db.GetEngine(db.DefaultContext).Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) } // LFSAutoAssociate auto associates accessible LFSMetaObjects @@ -168,34 +169,40 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6 oidMap[meta.Oid] = meta } - newMetas := make([]*LFSMetaObject, 0, len(metas)) - if !user.IsAdmin { - cond := builder.In("`lfs_meta_object`.repository_id", - builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user))) - if err := sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil { + newMetas := make([]*LFSMetaObject, 0, len(metas)) + cond := builder.In( + "`lfs_meta_object`.repository_id", + builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user)), + ) + err = sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas) + if err != nil { return err } if len(newMetas) != len(oidMap) { - return fmt.Errorf("can not collect all LFS objects from database, expected %d, actually %d", len(oidMap), len(newMetas)) + return fmt.Errorf("unable collect all LFS objects from database, expected %d, actually %d", len(oidMap), len(newMetas)) } for i := range newMetas { newMetas[i].Size = oidMap[newMetas[i].Oid].Size newMetas[i].RepositoryID = repoID } - if _, err := sess.InsertMulti(newMetas); err != nil { + if err = db.Insert(ctx, newMetas); err != nil { return err } } else { - // admin can associate any LFS object to any repository, and we do not care about duplicated errors + // admin can associate any LFS object to any repository, and we do not care about errors (eg: duplicated unique key), + // even if error occurs, it won't hurt users and won't make things worse for i := range metas { - _, _ = sess.Insert(&LFSMetaObject{ + _, err = sess.Insert(&LFSMetaObject{ Pointer: lfs.Pointer{Oid: metas[i].Oid, Size: metas[i].Size}, RepositoryID: repoID, }) + if err != nil { + log.Warn("failed to insert LFS meta object into database, err=%v", err) + } } } - return sess.Commit() + return committer.Commit() } // IterateLFS iterates lfs object