From c60fe1f5accae635730c35a47ecf6ff653deb2b6 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 27 May 2020 20:53:41 +0100 Subject: [PATCH 1/5] Add logging to long migrations Also fix v136 to not use models Signed-off-by: Andrew Thornton --- models/migrations/v115.go | 43 +++++++++++++++++++++++++- models/migrations/v128.go | 20 ++++++++++-- models/migrations/v134.go | 19 +++++++++++- models/migrations/v136.go | 64 +++++++++++++++++++++++++++++++++++---- 4 files changed, 136 insertions(+), 10 deletions(-) diff --git a/models/migrations/v115.go b/models/migrations/v115.go index 67d37536c09e2..eb6eb27e014be 100644 --- a/models/migrations/v115.go +++ b/models/migrations/v115.go @@ -8,8 +8,10 @@ import ( "crypto/md5" "fmt" "io/ioutil" + "math" "os" "path/filepath" + "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -26,6 +28,16 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { LowerName string `xorm:"UNIQUE NOT NULL"` Avatar string } + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + count, err := x.Count(new(User)) + if err != nil { + return err + } + log.Info("%d User Avatar(s) to migrate ...", count) + deleteList := make(map[string]struct{}) start := 0 for { @@ -42,7 +54,6 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { } log.Info("select users [%d - %d]", start, start+len(users)) - start += 50 for _, user := range users { oldAvatar := user.Avatar @@ -73,6 +84,19 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { } deleteList[filepath.Join(setting.AvatarUploadPath, oldAvatar)] = struct{}{} + start++ + select { + case <-ticker.C: + log.Info( + "%d/%d (%2.0f%%) User Avatar(s) migrated (%d old avatars to be deleted) in %d batches. %d Remaining ...", + start, + count, + float64(start)/float64(count)*100, + len(deleteList), + int(math.Ceil(float64(start)/float64(50))), + count-int64(start)) + default: + } } if err := sess.Commit(); err != nil { _ = sess.Rollback() @@ -80,11 +104,28 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { } } + deleteCount := len(deleteList) + log.Info("Deleting %d old avatars ...", deleteCount) + i := 0 for file := range deleteList { if err := os.Remove(file); err != nil { log.Warn("os.Remove: %v", err) } + i++ + select { + case <-ticker.C: + log.Info( + "%d/%d (%2.0f%%) Old User Avatar(s) deleted. %d Remaining ...", + i, + deleteCount, + float64(i)/float64(deleteCount)*100, + deleteCount-i) + default: + } } + + log.Info("Completed migrating %d User Avatar(s) and deleting %d Old Avatars", count, deleteCount) + return nil } diff --git a/models/migrations/v128.go b/models/migrations/v128.go index 1f4bc20527295..228c310186d5d 100644 --- a/models/migrations/v128.go +++ b/models/migrations/v128.go @@ -6,8 +6,10 @@ package migrations import ( "fmt" + "math" "path/filepath" "strings" + "time" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" @@ -43,6 +45,15 @@ func fixMergeBase(x *xorm.Engine) error { limit = 50 } + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + count, err := x.Count(new(PullRequest)) + if err != nil { + return err + } + log.Info("%d Pull Request(s) to migrate ...", count) + i := 0 for { prs := make([]PullRequest, 0, 50) @@ -53,7 +64,6 @@ func fixMergeBase(x *xorm.Engine) error { break } - i += len(prs) for _, pr := range prs { baseRepo := &Repository{ID: pr.BaseRepoID} has, err := x.Table("repository").Get(baseRepo) @@ -102,8 +112,14 @@ func fixMergeBase(x *xorm.Engine) error { } pr.MergeBase = strings.TrimSpace(pr.MergeBase) x.ID(pr.ID).Cols("merge_base").Update(pr) + i++ + select { + case <-ticker.C: + log.Info("%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", i, count, float64(i)/float64(count)*100, int(math.Ceil(float64(i)/float64(limit))), count-int64(i)) + default: + } } } - + log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(i)/float64(limit)))) return nil } diff --git a/models/migrations/v134.go b/models/migrations/v134.go index 527cbafe07ae4..ad7c24e0639bc 100644 --- a/models/migrations/v134.go +++ b/models/migrations/v134.go @@ -6,8 +6,10 @@ package migrations import ( "fmt" + "math" "path/filepath" "strings" + "time" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" @@ -43,6 +45,14 @@ func refixMergeBase(x *xorm.Engine) error { limit = 50 } + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + count, err := x.Where("has_merged = ?", true).Count(new(PullRequest)) + if err != nil { + return err + } + log.Info("%d Merged Pull Request(s) to migrate ...", count) + i := 0 for { prs := make([]PullRequest, 0, 50) @@ -53,7 +63,6 @@ func refixMergeBase(x *xorm.Engine) error { break } - i += len(prs) for _, pr := range prs { baseRepo := &Repository{ID: pr.BaseRepoID} has, err := x.Table("repository").Get(baseRepo) @@ -90,7 +99,15 @@ func refixMergeBase(x *xorm.Engine) error { } pr.MergeBase = strings.TrimSpace(pr.MergeBase) x.ID(pr.ID).Cols("merge_base").Update(pr) + i++ + select { + case <-ticker.C: + log.Info("%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", i, count, float64(i)/float64(count)*100, int(math.Ceil(float64(i)/float64(limit))), count-int64(i)) + default: + } } } + + log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(i)/float64(limit)))) return nil } diff --git a/models/migrations/v136.go b/models/migrations/v136.go index da5706258eb67..1d06441706080 100644 --- a/models/migrations/v136.go +++ b/models/migrations/v136.go @@ -6,16 +6,40 @@ package migrations import ( "fmt" + "math" + "path/filepath" + "strings" + "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - pull_service "code.gitea.io/gitea/services/pull" "xorm.io/xorm" ) func addCommitDivergenceToPulls(x *xorm.Engine) error { + type Repository struct { + ID int64 `xorm:"pk autoincr"` + OwnerID int64 `xorm:"UNIQUE(s) index"` + OwnerName string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string `xorm:"INDEX NOT NULL"` + } + + type PullRequest struct { + ID int64 `xorm:"pk autoincr"` + + CommitsAhead int + CommitsBehind int + + BaseRepoID int64 `xorm:"INDEX"` + BaseBranch string + + HasMerged bool `xorm:"INDEX"` + MergedCommitID string `xorm:"VARCHAR(40)"` + } if err := x.Sync2(new(models.PullRequest)); err != nil { return fmt.Errorf("Sync2: %v", err) @@ -25,6 +49,15 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { batchSize := setting.Database.IterateBufferSize sess := x.NewSession() defer sess.Close() + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + count, err := sess.Where("has_merged = ?", false).Count(new(PullRequest)) + if err != nil { + return err + } + log.Info("%d Unmerged Pull Request(s) to migrate ...", count) + for { if err := sess.Begin(); err != nil { return err @@ -40,16 +73,29 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { last += len(results) for _, pr := range results { - divergence, err := pull_service.GetDiverging(pr) + baseRepo := &Repository{ID: pr.BaseRepoID} + has, err := x.Table("repository").Get(baseRepo) + if err != nil { + return fmt.Errorf("Unable to get base repo %d %v", pr.BaseRepoID, err) + } + if !has { + log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID) + continue + } + userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName)) + repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git") + + gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index) + + divergence, err := git.GetDivergingCommits(repoPath, pr.BaseBranch, gitRefName) if err != nil { log.Warn("Could not recalculate Divergence for pull: %d", pr.ID) pr.CommitsAhead = 0 pr.CommitsBehind = 0 } - if divergence != nil { - pr.CommitsAhead = divergence.Ahead - pr.CommitsBehind = divergence.Behind - } + pr.CommitsAhead = divergence.Ahead + pr.CommitsBehind = divergence.Behind + if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil { return fmt.Errorf("Update Cols: %v", err) } @@ -58,6 +104,12 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { if err := sess.Commit(); err != nil { return err } + select { + case <-ticker.C: + log.Info("%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", last, count, float64(last)/float64(count)*100, int(math.Ceil(float64(last)/float64(batchSize))), count-int64(last)) + default: + } } + log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(last)/float64(batchSize)))) return nil } From babd8f1e134600a00cda1fcb02e58fce44f74a7a Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 28 May 2020 09:25:53 +0100 Subject: [PATCH 2/5] fix tests Signed-off-by: Andrew Thornton --- integrations/migration-test/migration_test.go | 1 - models/migrations/v115.go | 12 +++++++----- models/migrations/v128.go | 4 +++- models/migrations/v134.go | 4 +++- models/migrations/v136.go | 17 +++++++++++++---- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index 4ee045db38348..dad14c7ca18ea 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -250,7 +250,6 @@ func doMigrationTest(t *testing.T, version string) { return } - setting.NewXORMLogService(false) err := models.SetEngine() assert.NoError(t, err) diff --git a/models/migrations/v115.go b/models/migrations/v115.go index eb6eb27e014be..28c8afc1ef6b9 100644 --- a/models/migrations/v115.go +++ b/models/migrations/v115.go @@ -40,6 +40,7 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { deleteList := make(map[string]struct{}) start := 0 + migrated := 0 for { if err := sess.Begin(); err != nil { return fmt.Errorf("session.Begin: %v", err) @@ -52,6 +53,7 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { _ = sess.Rollback() break } + start += 50 log.Info("select users [%d - %d]", start, start+len(users)) @@ -84,17 +86,17 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { } deleteList[filepath.Join(setting.AvatarUploadPath, oldAvatar)] = struct{}{} - start++ + migrated++ select { case <-ticker.C: log.Info( "%d/%d (%2.0f%%) User Avatar(s) migrated (%d old avatars to be deleted) in %d batches. %d Remaining ...", - start, + migrated, count, - float64(start)/float64(count)*100, + float64(migrated)/float64(count)*100, len(deleteList), - int(math.Ceil(float64(start)/float64(50))), - count-int64(start)) + int(math.Ceil(float64(migrated)/float64(50))), + count-int64(migrated)) default: } } diff --git a/models/migrations/v128.go b/models/migrations/v128.go index 228c310186d5d..9f698e3e57c6b 100644 --- a/models/migrations/v128.go +++ b/models/migrations/v128.go @@ -55,15 +55,17 @@ func fixMergeBase(x *xorm.Engine) error { log.Info("%d Pull Request(s) to migrate ...", count) i := 0 + start := 0 for { prs := make([]PullRequest, 0, 50) - if err := x.Limit(limit, i).Asc("id").Find(&prs); err != nil { + if err := x.Limit(limit, start).Asc("id").Find(&prs); err != nil { return fmt.Errorf("Find: %v", err) } if len(prs) == 0 { break } + start += 50 for _, pr := range prs { baseRepo := &Repository{ID: pr.BaseRepoID} has, err := x.Table("repository").Get(baseRepo) diff --git a/models/migrations/v134.go b/models/migrations/v134.go index ad7c24e0639bc..3ad94e7368836 100644 --- a/models/migrations/v134.go +++ b/models/migrations/v134.go @@ -54,15 +54,17 @@ func refixMergeBase(x *xorm.Engine) error { log.Info("%d Merged Pull Request(s) to migrate ...", count) i := 0 + start := 0 for { prs := make([]PullRequest, 0, 50) - if err := x.Limit(limit, i).Asc("id").Where("has_merged = ?", true).Find(&prs); err != nil { + if err := x.Limit(limit, start).Asc("id").Where("has_merged = ?", true).Find(&prs); err != nil { return fmt.Errorf("Find: %v", err) } if len(prs) == 0 { break } + start += 50 for _, pr := range prs { baseRepo := &Repository{ID: pr.BaseRepoID} has, err := x.Table("repository").Get(baseRepo) diff --git a/models/migrations/v136.go b/models/migrations/v136.go index 1d06441706080..3f718798ba87f 100644 --- a/models/migrations/v136.go +++ b/models/migrations/v136.go @@ -45,7 +45,9 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { return fmt.Errorf("Sync2: %v", err) } - var last int + last := 0 + migrated := 0 + batchSize := setting.Database.IterateBufferSize sess := x.NewSession() defer sess.Close() @@ -70,7 +72,7 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { if len(results) == 0 { break } - last += len(results) + last += batchSize for _, pr := range results { baseRepo := &Repository{ID: pr.BaseRepoID} @@ -99,6 +101,7 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil { return fmt.Errorf("Update Cols: %v", err) } + migrated++ } if err := sess.Commit(); err != nil { @@ -106,10 +109,16 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error { } select { case <-ticker.C: - log.Info("%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", last, count, float64(last)/float64(count)*100, int(math.Ceil(float64(last)/float64(batchSize))), count-int64(last)) + log.Info( + "%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", + migrated, + count, + float64(migrated)/float64(count)*100, + int(math.Ceil(float64(migrated)/float64(batchSize))), + count-int64(migrated)) default: } } - log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(last)/float64(batchSize)))) + log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(migrated)/float64(batchSize)))) return nil } From 9f4d340d9d8d9e513f8b380593c24f673717df63 Mon Sep 17 00:00:00 2001 From: zeripath Date: Fri, 29 May 2020 19:46:58 +0100 Subject: [PATCH 3/5] Apply suggestions from code review --- models/migrations/v115.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/v115.go b/models/migrations/v115.go index 28c8afc1ef6b9..9b5c70a7ac2cb 100644 --- a/models/migrations/v115.go +++ b/models/migrations/v115.go @@ -53,9 +53,9 @@ func renameExistingUserAvatarName(x *xorm.Engine) error { _ = sess.Rollback() break } - start += 50 log.Info("select users [%d - %d]", start, start+len(users)) + start += 50 for _, user := range users { oldAvatar := user.Avatar From 9b8475caea12e6b507961ce2c288c719e8cea7c2 Mon Sep 17 00:00:00 2001 From: zeripath Date: Fri, 29 May 2020 19:48:56 +0100 Subject: [PATCH 4/5] Update integrations/migration-test/migration_test.go --- integrations/migration-test/migration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index dad14c7ca18ea..f45a3dd552c69 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -249,7 +249,7 @@ func doMigrationTest(t *testing.T, version string) { if !restoreOldDB(t, version) { return } - + setting.NewXORMLogService(false) err := models.SetEngine() assert.NoError(t, err) From 881436678ef56d7bd23dd446d4419f7261645296 Mon Sep 17 00:00:00 2001 From: zeripath Date: Fri, 29 May 2020 20:26:30 +0100 Subject: [PATCH 5/5] Update integrations/migration-test/migration_test.go --- integrations/migration-test/migration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index f45a3dd552c69..4ee045db38348 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -249,6 +249,7 @@ func doMigrationTest(t *testing.T, version string) { if !restoreOldDB(t, version) { return } + setting.NewXORMLogService(false) err := models.SetEngine() assert.NoError(t, err)