Skip to content

Commit 495354c

Browse files
zeripathzeripath
authored
Add logging to long migrations (#11647)
* Add logging to long migrations Also fix v136 to not use models Signed-off-by: Andrew Thornton <[email protected]>
1 parent 6e44808 commit 495354c

File tree

4 files changed

+154
-13
lines changed

4 files changed

+154
-13
lines changed

‎models/migrations/v115.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ import (
88
"crypto/md5"
99
"fmt"
1010
"io/ioutil"
11+
"math"
1112
"os"
1213
"path/filepath"
14+
"time"
1315

1416
"code.gitea.io/gitea/modules/log"
1517
"code.gitea.io/gitea/modules/setting"
@@ -26,8 +28,19 @@ func renameExistingUserAvatarName(x *xorm.Engine) error {
2628
LowerName string `xorm:"UNIQUE NOT NULL"`
2729
Avatar string
2830
}
31+
32+
ticker := time.NewTicker(5 * time.Second)
33+
defer ticker.Stop()
34+
35+
count, err := x.Count(new(User))
36+
if err != nil {
37+
return err
38+
}
39+
log.Info("%d User Avatar(s) to migrate ...", count)
40+
2941
deleteList := make(map[string]struct{})
3042
start := 0
43+
migrated := 0
3144
for {
3245
if err := sess.Begin(); err != nil {
3346
return fmt.Errorf("session.Begin: %v", err)
@@ -73,18 +86,48 @@ func renameExistingUserAvatarName(x *xorm.Engine) error {
7386
}
7487

7588
deleteList[filepath.Join(setting.AvatarUploadPath, oldAvatar)] = struct{}{}
89+
migrated++
90+
select {
91+
case <-ticker.C:
92+
log.Info(
93+
"%d/%d (%2.0f%%) User Avatar(s) migrated (%d old avatars to be deleted) in %d batches. %d Remaining ...",
94+
migrated,
95+
count,
96+
float64(migrated)/float64(count)*100,
97+
len(deleteList),
98+
int(math.Ceil(float64(migrated)/float64(50))),
99+
count-int64(migrated))
100+
default:
101+
}
76102
}
77103
if err := sess.Commit(); err != nil {
78104
_ = sess.Rollback()
79105
return fmt.Errorf("commit session: %v", err)
80106
}
81107
}
82108

109+
deleteCount := len(deleteList)
110+
log.Info("Deleting %d old avatars ...", deleteCount)
111+
i := 0
83112
for file := range deleteList {
84113
if err := os.Remove(file); err != nil {
85114
log.Warn("os.Remove: %v", err)
86115
}
116+
i++
117+
select {
118+
case <-ticker.C:
119+
log.Info(
120+
"%d/%d (%2.0f%%) Old User Avatar(s) deleted. %d Remaining ...",
121+
i,
122+
deleteCount,
123+
float64(i)/float64(deleteCount)*100,
124+
deleteCount-i)
125+
default:
126+
}
87127
}
128+
129+
log.Info("Completed migrating %d User Avatar(s) and deleting %d Old Avatars", count, deleteCount)
130+
88131
return nil
89132
}
90133

‎models/migrations/v128.go‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ package migrations
66

77
import (
88
"fmt"
9+
"math"
910
"path/filepath"
1011
"strings"
12+
"time"
1113

1214
"code.gitea.io/gitea/modules/git"
1315
"code.gitea.io/gitea/modules/log"
@@ -43,17 +45,27 @@ func fixMergeBase(x *xorm.Engine) error {
4345
limit = 50
4446
}
4547

48+
ticker := time.NewTicker(5 * time.Second)
49+
defer ticker.Stop()
50+
51+
count, err := x.Count(new(PullRequest))
52+
if err != nil {
53+
return err
54+
}
55+
log.Info("%d Pull Request(s) to migrate ...", count)
56+
4657
i := 0
58+
start := 0
4759
for {
4860
prs := make([]PullRequest, 0, 50)
49-
if err := x.Limit(limit, i).Asc("id").Find(&prs); err != nil {
61+
if err := x.Limit(limit, start).Asc("id").Find(&prs); err != nil {
5062
return fmt.Errorf("Find: %v", err)
5163
}
5264
if len(prs) == 0 {
5365
break
5466
}
5567

56-
i += len(prs)
68+
start += 50
5769
for _, pr := range prs {
5870
baseRepo := &Repository{ID: pr.BaseRepoID}
5971
has, err := x.Table("repository").Get(baseRepo)
@@ -102,8 +114,14 @@ func fixMergeBase(x *xorm.Engine) error {
102114
}
103115
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
104116
x.ID(pr.ID).Cols("merge_base").Update(pr)
117+
i++
118+
select {
119+
case <-ticker.C:
120+
log.Info("%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", i, count, float64(i)/float64(count)*100, int(math.Ceil(float64(i)/float64(limit))), count-int64(i))
121+
default:
122+
}
105123
}
106124
}
107-
125+
log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(i)/float64(limit))))
108126
return nil
109127
}

‎models/migrations/v134.go‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ package migrations
66

77
import (
88
"fmt"
9+
"math"
910
"path/filepath"
1011
"strings"
12+
"time"
1113

1214
"code.gitea.io/gitea/modules/git"
1315
"code.gitea.io/gitea/modules/log"
@@ -43,17 +45,26 @@ func refixMergeBase(x *xorm.Engine) error {
4345
limit = 50
4446
}
4547

48+
ticker := time.NewTicker(5 * time.Second)
49+
defer ticker.Stop()
50+
count, err := x.Where("has_merged = ?", true).Count(new(PullRequest))
51+
if err != nil {
52+
return err
53+
}
54+
log.Info("%d Merged Pull Request(s) to migrate ...", count)
55+
4656
i := 0
57+
start := 0
4758
for {
4859
prs := make([]PullRequest, 0, 50)
49-
if err := x.Limit(limit, i).Asc("id").Where("has_merged = ?", true).Find(&prs); err != nil {
60+
if err := x.Limit(limit, start).Asc("id").Where("has_merged = ?", true).Find(&prs); err != nil {
5061
return fmt.Errorf("Find: %v", err)
5162
}
5263
if len(prs) == 0 {
5364
break
5465
}
5566

56-
i += len(prs)
67+
start += 50
5768
for _, pr := range prs {
5869
baseRepo := &Repository{ID: pr.BaseRepoID}
5970
has, err := x.Table("repository").Get(baseRepo)
@@ -90,7 +101,15 @@ func refixMergeBase(x *xorm.Engine) error {
90101
}
91102
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
92103
x.ID(pr.ID).Cols("merge_base").Update(pr)
104+
i++
105+
select {
106+
case <-ticker.C:
107+
log.Info("%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", i, count, float64(i)/float64(count)*100, int(math.Ceil(float64(i)/float64(limit))), count-int64(i))
108+
default:
109+
}
93110
}
94111
}
112+
113+
log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(i)/float64(limit))))
95114
return nil
96115
}

‎models/migrations/v136.go‎

Lines changed: 69 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,60 @@ package migrations
66

77
import (
88
"fmt"
9+
"math"
10+
"path/filepath"
11+
"strings"
12+
"time"
913

1014
"code.gitea.io/gitea/models"
15+
"code.gitea.io/gitea/modules/git"
1116
"code.gitea.io/gitea/modules/log"
1217
"code.gitea.io/gitea/modules/setting"
13-
pull_service "code.gitea.io/gitea/services/pull"
1418

1519
"xorm.io/xorm"
1620
)
1721

1822
func addCommitDivergenceToPulls(x *xorm.Engine) error {
23+
type Repository struct {
24+
ID int64 `xorm:"pk autoincr"`
25+
OwnerID int64 `xorm:"UNIQUE(s) index"`
26+
OwnerName string
27+
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
28+
Name string `xorm:"INDEX NOT NULL"`
29+
}
30+
31+
type PullRequest struct {
32+
ID int64 `xorm:"pk autoincr"`
33+
34+
CommitsAhead int
35+
CommitsBehind int
36+
37+
BaseRepoID int64 `xorm:"INDEX"`
38+
BaseBranch string
39+
40+
HasMerged bool `xorm:"INDEX"`
41+
MergedCommitID string `xorm:"VARCHAR(40)"`
42+
}
1943

2044
if err := x.Sync2(new(models.PullRequest)); err != nil {
2145
return fmt.Errorf("Sync2: %v", err)
2246
}
2347

24-
var last int
48+
last := 0
49+
migrated := 0
50+
2551
batchSize := setting.Database.IterateBufferSize
2652
sess := x.NewSession()
2753
defer sess.Close()
54+
55+
ticker := time.NewTicker(5 * time.Second)
56+
defer ticker.Stop()
57+
count, err := sess.Where("has_merged = ?", false).Count(new(PullRequest))
58+
if err != nil {
59+
return err
60+
}
61+
log.Info("%d Unmerged Pull Request(s) to migrate ...", count)
62+
2863
for {
2964
if err := sess.Begin(); err != nil {
3065
return err
@@ -37,27 +72,53 @@ func addCommitDivergenceToPulls(x *xorm.Engine) error {
3772
if len(results) == 0 {
3873
break
3974
}
40-
last += len(results)
75+
last += batchSize
4176

4277
for _, pr := range results {
43-
divergence, err := pull_service.GetDiverging(pr)
78+
baseRepo := &Repository{ID: pr.BaseRepoID}
79+
has, err := x.Table("repository").Get(baseRepo)
80+
if err != nil {
81+
return fmt.Errorf("Unable to get base repo %d %v", pr.BaseRepoID, err)
82+
}
83+
if !has {
84+
log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID)
85+
continue
86+
}
87+
userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName))
88+
repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git")
89+
90+
gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index)
91+
92+
divergence, err := git.GetDivergingCommits(repoPath, pr.BaseBranch, gitRefName)
4493
if err != nil {
4594
log.Warn("Could not recalculate Divergence for pull: %d", pr.ID)
4695
pr.CommitsAhead = 0
4796
pr.CommitsBehind = 0
4897
}
49-
if divergence != nil {
50-
pr.CommitsAhead = divergence.Ahead
51-
pr.CommitsBehind = divergence.Behind
52-
}
98+
pr.CommitsAhead = divergence.Ahead
99+
pr.CommitsBehind = divergence.Behind
100+
53101
if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil {
54102
return fmt.Errorf("Update Cols: %v", err)
55103
}
104+
migrated++
56105
}
57106

58107
if err := sess.Commit(); err != nil {
59108
return err
60109
}
110+
select {
111+
case <-ticker.C:
112+
log.Info(
113+
"%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...",
114+
migrated,
115+
count,
116+
float64(migrated)/float64(count)*100,
117+
int(math.Ceil(float64(migrated)/float64(batchSize))),
118+
count-int64(migrated))
119+
default:
120+
}
61121
}
122+
log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(migrated)/float64(batchSize))))
62123
return nil
63124
}

0 commit comments

Comments
 (0)