Skip to content

Commit 938026f

Browse files
zeripathzeripath
committed
Move migration clone to goroutine
The current code places the git clone in the POST goroutine, blocking that goroutine until it is finished. This PR asynchronises this, placing the clone within its own goroutine. Fix #3770
1 parent 5bd594c commit 938026f

File tree

8 files changed

+221
-109
lines changed

8 files changed

+221
-109
lines changed

‎models/action.go‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -30,26 +30,29 @@ type ActionType int
3030

3131
// Possible action types.
3232
const (
33-
ActionCreateRepo ActionType = iota + 1 // 1
34-
ActionRenameRepo // 2
35-
ActionStarRepo // 3
36-
ActionWatchRepo // 4
37-
ActionCommitRepo // 5
38-
ActionCreateIssue // 6
39-
ActionCreatePullRequest // 7
40-
ActionTransferRepo // 8
41-
ActionPushTag // 9
42-
ActionCommentIssue // 10
43-
ActionMergePullRequest // 11
44-
ActionCloseIssue // 12
45-
ActionReopenIssue // 13
46-
ActionClosePullRequest // 14
47-
ActionReopenPullRequest // 15
48-
ActionDeleteTag // 16
49-
ActionDeleteBranch // 17
50-
ActionMirrorSyncPush // 18
51-
ActionMirrorSyncCreate // 19
52-
ActionMirrorSyncDelete // 20
33+
ActionCreateRepo ActionType = iota + 1 // 1
34+
ActionRenameRepo // 2
35+
ActionStarRepo // 3
36+
ActionWatchRepo // 4
37+
ActionCommitRepo // 5
38+
ActionCreateIssue // 6
39+
ActionCreatePullRequest // 7
40+
ActionTransferRepo // 8
41+
ActionPushTag // 9
42+
ActionCommentIssue // 10
43+
ActionMergePullRequest // 11
44+
ActionCloseIssue // 12
45+
ActionReopenIssue // 13
46+
ActionClosePullRequest // 14
47+
ActionReopenPullRequest // 15
48+
ActionDeleteTag // 16
49+
ActionDeleteBranch // 17
50+
ActionMirrorSyncPush // 18
51+
ActionMirrorSyncCreate // 19
52+
ActionMirrorSyncDelete // 20
53+
ActionMigrationStarted // 21
54+
ActionMigrationSuccessful // 22
55+
ActionMigrationFailure // 23
5356
)
5457

5558
var (

‎models/release_test.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,9 @@ func TestRelease_MirrorDelete(t *testing.T) {
108108
IsMirror: true,
109109
RemoteAddr: repoPath,
110110
}
111-
mirror, err := MigrateRepository(user, user, migrationOptions)
111+
mirror, err := MigrateRepository(user, user, migrationOptions, func(err error) string {
112+
return err.Error()
113+
})
112114
assert.NoError(t, err)
113115

114116
gitRepo, err := git.OpenRepository(repoPath)

‎models/repo.go‎

Lines changed: 164 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -896,115 +896,198 @@ func wikiRemoteURL(remote string) string {
896896
}
897897

898898
// MigrateRepository migrates a existing repository from other project hosting.
899-
func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) {
899+
func MigrateRepository(doer, u *User, opts MigrateRepoOptions, messageConverter func(error) string) (*Repository, error) {
900900
repo, err := CreateRepository(doer, u, CreateRepoOptions{
901901
Name: opts.Name,
902902
Description: opts.Description,
903903
IsPrivate: opts.IsPrivate,
904904
IsMirror: opts.IsMirror,
905+
NoWatchers: true,
905906
})
906907
if err != nil {
907908
return nil, err
908909
}
909910

910-
repoPath := RepoPath(u.Name, opts.Name)
911-
wikiPath := WikiPath(u.Name, opts.Name)
912-
913-
if u.IsOrganization() {
914-
t, err := u.GetOwnerTeam()
915-
if err != nil {
916-
return nil, err
911+
env, ok := os.LookupEnv("GIT_TERMINAL_PROMPT=0")
912+
os.Setenv("GIT_TERMINAL_PROMPT", "0")
913+
if _, err = git.NewCommand("ls-remote", "-h", opts.RemoteAddr).RunTimeout(1 * time.Minute); err != nil {
914+
if ok {
915+
os.Setenv("GIT_TERMINAL_PROMPT", env)
916+
} else {
917+
os.Unsetenv("GIT_TERMINAL_PROMPT")
917918
}
918-
repo.NumWatches = t.NumMembers
919-
} else {
920-
repo.NumWatches = 1
919+
return repo, fmt.Errorf("Clone: %v", err)
921920
}
921+
if ok {
922+
os.Setenv("GIT_TERMINAL_PROMPT", env)
923+
} else {
924+
os.Unsetenv("GIT_TERMINAL_PROMPT")
925+
}
926+
927+
// OK if we succeeded above then we know that the clone should start...
928+
go func() {
929+
repoPath := RepoPath(u.Name, opts.Name)
930+
wikiPath := WikiPath(u.Name, opts.Name)
931+
932+
failedMigration := func(err error) {
933+
NotifyWatchers(&Action{
934+
ActUserID: doer.ID,
935+
ActUser: doer,
936+
OpType: ActionMigrationFailure,
937+
RepoID: repo.ID,
938+
Repo: repo,
939+
IsPrivate: repo.IsPrivate,
940+
Content: messageConverter(err),
941+
})
922942

923-
migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second
943+
if repo != nil {
944+
if errDelete := DeleteRepository(doer, u.ID, repo.ID); errDelete != nil {
945+
log.Error(4, "DeleteRepository: %v", errDelete)
946+
}
947+
}
924948

925-
if err := os.RemoveAll(repoPath); err != nil {
926-
return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err)
927-
}
949+
}
950+
NotifyWatchers(&Action{
951+
ActUserID: doer.ID,
952+
ActUser: doer,
953+
OpType: ActionMigrationStarted,
954+
RepoID: repo.ID,
955+
Repo: repo,
956+
Content: util.SanitizeURLCredentials(opts.RemoteAddr, true),
957+
IsPrivate: repo.IsPrivate,
958+
})
959+
repo.IsArchived = true
960+
if _, err := x.ID(repo.ID).AllCols().Update(repo); err != nil {
961+
failedMigration(err)
962+
return
963+
}
928964

929-
if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{
930-
Mirror: true,
931-
Quiet: true,
932-
Timeout: migrateTimeout,
933-
}); err != nil {
934-
return repo, fmt.Errorf("Clone: %v", err)
935-
}
965+
if u.IsOrganization() {
966+
t, err := u.GetOwnerTeam()
967+
if err != nil {
968+
failedMigration(err)
969+
return
970+
}
971+
repo.NumWatches = t.NumMembers
972+
} else {
973+
repo.NumWatches = 1
974+
}
936975

937-
wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
938-
if len(wikiRemotePath) > 0 {
939-
if err := os.RemoveAll(wikiPath); err != nil {
940-
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
976+
migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second
977+
978+
if err := os.RemoveAll(repoPath); err != nil {
979+
failedMigration(fmt.Errorf("Failed to remove %s: %v", repoPath, err))
980+
return
941981
}
942982

943-
if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
983+
if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{
944984
Mirror: true,
945985
Quiet: true,
946986
Timeout: migrateTimeout,
947-
Branch: "master",
948987
}); err != nil {
949-
log.Warn("Clone wiki: %v", err)
988+
failedMigration(fmt.Errorf("Clone: %v", err))
989+
return
990+
991+
}
992+
993+
wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
994+
if len(wikiRemotePath) > 0 {
950995
if err := os.RemoveAll(wikiPath); err != nil {
951-
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
996+
failedMigration(fmt.Errorf("Failed to remove %s: %v", wikiPath, err))
997+
return
952998
}
953-
}
954-
}
955999

956-
// Check if repository is empty.
957-
_, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1")
958-
if err != nil {
959-
if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") {
960-
repo.IsEmpty = true
961-
} else {
962-
return repo, fmt.Errorf("check empty: %v - %s", err, stderr)
1000+
if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
1001+
Mirror: true,
1002+
Quiet: true,
1003+
Timeout: migrateTimeout,
1004+
Branch: "master",
1005+
}); err != nil {
1006+
log.Warn("Clone wiki: %v", err)
1007+
if err := os.RemoveAll(wikiPath); err != nil {
1008+
failedMigration(fmt.Errorf("Failed to remove %s: %v", wikiPath, err))
1009+
return
1010+
}
1011+
}
9631012
}
964-
}
9651013

966-
if !repo.IsEmpty {
967-
// Try to get HEAD branch and set it as default branch.
968-
gitRepo, err := git.OpenRepository(repoPath)
1014+
// Check if repository is empty.
1015+
_, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1")
9691016
if err != nil {
970-
return repo, fmt.Errorf("OpenRepository: %v", err)
971-
}
972-
headBranch, err := gitRepo.GetHEADBranch()
973-
if err != nil {
974-
return repo, fmt.Errorf("GetHEADBranch: %v", err)
1017+
if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") {
1018+
repo.IsEmpty = true
1019+
} else {
1020+
failedMigration(fmt.Errorf("check empty: %v - %s", err, stderr))
1021+
return
1022+
}
9751023
}
976-
if headBranch != nil {
977-
repo.DefaultBranch = headBranch.Name
1024+
1025+
if !repo.IsEmpty {
1026+
// Try to get HEAD branch and set it as default branch.
1027+
gitRepo, err := git.OpenRepository(repoPath)
1028+
if err != nil {
1029+
failedMigration(fmt.Errorf("OpenRepository: %v", err))
1030+
return
1031+
}
1032+
headBranch, err := gitRepo.GetHEADBranch()
1033+
if err != nil {
1034+
failedMigration(fmt.Errorf("GetHEADBranch: %v", err))
1035+
return
1036+
}
1037+
if headBranch != nil {
1038+
repo.DefaultBranch = headBranch.Name
1039+
}
1040+
1041+
if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
1042+
log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
1043+
}
9781044
}
9791045

980-
if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
981-
log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
1046+
if err = repo.UpdateSize(); err != nil {
1047+
log.Error(4, "Failed to update size for repository: %v", err)
9821048
}
983-
}
9841049

985-
if err = repo.UpdateSize(); err != nil {
986-
log.Error(4, "Failed to update size for repository: %v", err)
987-
}
1050+
if opts.IsMirror {
1051+
if _, err = x.InsertOne(&Mirror{
1052+
RepoID: repo.ID,
1053+
Interval: setting.Mirror.DefaultInterval,
1054+
EnablePrune: true,
1055+
NextUpdateUnix: util.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval),
1056+
}); err != nil {
1057+
failedMigration(fmt.Errorf("InsertOne: %v", err))
1058+
return
1059+
}
9881060

989-
if opts.IsMirror {
990-
if _, err = x.InsertOne(&Mirror{
991-
RepoID: repo.ID,
992-
Interval: setting.Mirror.DefaultInterval,
993-
EnablePrune: true,
994-
NextUpdateUnix: util.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval),
995-
}); err != nil {
996-
return repo, fmt.Errorf("InsertOne: %v", err)
1061+
repo.IsMirror = true
1062+
err = UpdateRepository(repo, false)
1063+
} else {
1064+
repo, err = CleanUpMigrateInfo(repo)
9971065
}
9981066

999-
repo.IsMirror = true
1000-
err = UpdateRepository(repo, false)
1001-
} else {
1002-
repo, err = CleanUpMigrateInfo(repo)
1003-
}
1067+
if err != nil {
1068+
if !repo.IsEmpty {
1069+
UpdateRepoIndexer(repo)
1070+
}
1071+
failedMigration(err)
1072+
return
1073+
}
10041074

1005-
if err != nil && !repo.IsEmpty {
1006-
UpdateRepoIndexer(repo)
1007-
}
1075+
repo.IsArchived = false
1076+
if _, err := x.ID(repo.ID).AllCols().Update(repo); err != nil {
1077+
failedMigration(err)
1078+
return
1079+
}
1080+
1081+
NotifyWatchers(&Action{
1082+
ActUserID: doer.ID,
1083+
ActUser: doer,
1084+
OpType: ActionMigrationSuccessful,
1085+
RepoID: repo.ID,
1086+
Repo: repo,
1087+
IsPrivate: repo.IsPrivate,
1088+
Content: util.SanitizeURLCredentials(opts.RemoteAddr, true),
1089+
})
1090+
}()
10081091

10091092
return repo, err
10101093
}
@@ -1120,6 +1203,7 @@ type CreateRepoOptions struct {
11201203
IsPrivate bool
11211204
IsMirror bool
11221205
AutoInit bool
1206+
NoWatchers bool
11231207
}
11241208

11251209
func getRepoInitFile(tp, name string) ([]byte, error) {
@@ -1273,7 +1357,7 @@ func IsUsableRepoName(name string) error {
12731357
return isUsableName(reservedRepoNames, reservedRepoPatterns, name)
12741358
}
12751359

1276-
func createRepository(e *xorm.Session, doer, u *User, repo *Repository) (err error) {
1360+
func createRepository(e *xorm.Session, doer, u *User, repo *Repository, noWatchers bool) (err error) {
12771361
if err = IsUsableRepoName(repo.Name); err != nil {
12781362
return err
12791363
}
@@ -1359,10 +1443,12 @@ func createRepository(e *xorm.Session, doer, u *User, repo *Repository) (err err
13591443
return fmt.Errorf("watchRepo: %v", err)
13601444
}
13611445
}
1362-
if err = newRepoAction(e, doer, repo); err != nil {
1363-
return fmt.Errorf("newRepoAction: %v", err)
1364-
}
13651446

1447+
if !noWatchers {
1448+
if err = newRepoAction(e, doer, repo); err != nil {
1449+
return fmt.Errorf("newRepoAction: %v", err)
1450+
}
1451+
}
13661452
return nil
13671453
}
13681454

@@ -1388,7 +1474,7 @@ func CreateRepository(doer, u *User, opts CreateRepoOptions) (_ *Repository, err
13881474
return nil, err
13891475
}
13901476

1391-
if err = createRepository(sess, doer, u, repo); err != nil {
1477+
if err = createRepository(sess, doer, u, repo, opts.NoWatchers); err != nil {
13921478
return nil, err
13931479
}
13941480

@@ -2420,7 +2506,7 @@ func ForkRepository(doer, u *User, oldRepo *Repository, name, desc string) (_ *R
24202506
return nil, err
24212507
}
24222508

2423-
if err = createRepository(sess, doer, u, repo); err != nil {
2509+
if err = createRepository(sess, doer, u, repo, true); err != nil {
24242510
return nil, err
24252511
}
24262512

0 commit comments

Comments
 (0)