WIP: Move migration git clone to its own goroutine

models/action.go

@@ -30,26 +30,29 @@ type ActionType int
 
 // Possible action types.
 const (
-	ActionCreateRepo        ActionType = iota + 1 // 1
-	ActionRenameRepo                              // 2
-	ActionStarRepo                                // 3
-	ActionWatchRepo                               // 4
-	ActionCommitRepo                              // 5
-	ActionCreateIssue                             // 6
-	ActionCreatePullRequest                       // 7
-	ActionTransferRepo                            // 8
-	ActionPushTag                                 // 9
-	ActionCommentIssue                            // 10
-	ActionMergePullRequest                        // 11
-	ActionCloseIssue                              // 12
-	ActionReopenIssue                             // 13
-	ActionClosePullRequest                        // 14
-	ActionReopenPullRequest                       // 15
-	ActionDeleteTag                               // 16
-	ActionDeleteBranch                            // 17
-	ActionMirrorSyncPush                          // 18
-	ActionMirrorSyncCreate                        // 19
-	ActionMirrorSyncDelete                        // 20
+	ActionCreateRepo          ActionType = iota + 1 // 1
+	ActionRenameRepo                                // 2
+	ActionStarRepo                                  // 3
+	ActionWatchRepo                                 // 4
+	ActionCommitRepo                                // 5
+	ActionCreateIssue                               // 6
+	ActionCreatePullRequest                         // 7
+	ActionTransferRepo                              // 8
+	ActionPushTag                                   // 9
+	ActionCommentIssue                              // 10
+	ActionMergePullRequest                          // 11
+	ActionCloseIssue                                // 12
+	ActionReopenIssue                               // 13
+	ActionClosePullRequest                          // 14
+	ActionReopenPullRequest                         // 15
+	ActionDeleteTag                                 // 16
+	ActionDeleteBranch                              // 17
+	ActionMirrorSyncPush                            // 18
+	ActionMirrorSyncCreate                          // 19
+	ActionMirrorSyncDelete                          // 20
+	ActionMigrationStarted                          // 21
+	ActionMigrationSuccessful                       // 22
+	ActionMigrationFailure                          // 23
 )
 
 var (

models/release_test.go

@@ -108,9 +108,25 @@ func TestRelease_MirrorDelete(t *testing.T) {
 		IsMirror:    true,
 		RemoteAddr:  repoPath,
 	}
-	mirror, err := MigrateRepository(user, user, migrationOptions)
+
+	out := make(chan bool)
+	defer close(out)
+
+	mirror, err := MigrateRepository(user, user, migrationOptions, func(err error) string {
+		if err != nil {
+			out <- false
+			return err.Error()
+		}
+		out <- true
+		return repoPath
+	})
+
 	assert.NoError(t, err)
 
+	// Now we have to wait til the migration is reported.
+	success := <-out
+	assert.True(t, success)
+
 	gitRepo, err := git.OpenRepository(repoPath)
 	assert.NoError(t, err)
 

models/repo.go

@@ -36,7 +36,6 @@ import (
 	"github.com/go-xorm/builder"
 	"github.com/go-xorm/xorm"
 	version "github.com/mcuadros/go-version"
-	ini "gopkg.in/ini.v1"
 )
 
 var repoWorkingPool = sync.NewExclusivePool()
@@ -868,162 +867,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) {
 	return repo.cloneLink(x, false)
 }
 
-// MigrateRepoOptions contains the repository migrate options
-type MigrateRepoOptions struct {
-	Name        string
-	Description string
-	IsPrivate   bool
-	IsMirror    bool
-	RemoteAddr  string
-}
-
-/*
-	GitHub, GitLab, Gogs: *.wiki.git
-	BitBucket: *.git/wiki
-*/
-var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"}
-
-// wikiRemoteURL returns accessible repository URL for wiki if exists.
-// Otherwise, it returns an empty string.
-func wikiRemoteURL(remote string) string {
-	remote = strings.TrimSuffix(remote, ".git")
-	for _, suffix := range commonWikiURLSuffixes {
-		wikiURL := remote + suffix
-		if git.IsRepoURLAccessible(wikiURL) {
-			return wikiURL
-		}
-	}
-	return ""
-}
-
-// MigrateRepository migrates a existing repository from other project hosting.
-func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) {
-	repo, err := CreateRepository(doer, u, CreateRepoOptions{
-		Name:        opts.Name,
-		Description: opts.Description,
-		IsPrivate:   opts.IsPrivate,
-		IsMirror:    opts.IsMirror,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	repoPath := RepoPath(u.Name, opts.Name)
-	wikiPath := WikiPath(u.Name, opts.Name)
-
-	if u.IsOrganization() {
-		t, err := u.GetOwnerTeam()
-		if err != nil {
-			return nil, err
-		}
-		repo.NumWatches = t.NumMembers
-	} else {
-		repo.NumWatches = 1
-	}
-
-	migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second
-
-	if err := os.RemoveAll(repoPath); err != nil {
-		return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err)
-	}
-
-	if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{
-		Mirror:  true,
-		Quiet:   true,
-		Timeout: migrateTimeout,
-	}); err != nil {
-		return repo, fmt.Errorf("Clone: %v", err)
-	}
-
-	wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
-	if len(wikiRemotePath) > 0 {
-		if err := os.RemoveAll(wikiPath); err != nil {
-			return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
-		}
-
-		if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
-			Mirror:  true,
-			Quiet:   true,
-			Timeout: migrateTimeout,
-			Branch:  "master",
-		}); err != nil {
-			log.Warn("Clone wiki: %v", err)
-			if err := os.RemoveAll(wikiPath); err != nil {
-				return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
-			}
-		}
-	}
-
-	// Check if repository is empty.
-	_, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1")
-	if err != nil {
-		if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") {
-			repo.IsEmpty = true
-		} else {
-			return repo, fmt.Errorf("check empty: %v - %s", err, stderr)
-		}
-	}
-
-	if !repo.IsEmpty {
-		// Try to get HEAD branch and set it as default branch.
-		gitRepo, err := git.OpenRepository(repoPath)
-		if err != nil {
-			return repo, fmt.Errorf("OpenRepository: %v", err)
-		}
-		headBranch, err := gitRepo.GetHEADBranch()
-		if err != nil {
-			return repo, fmt.Errorf("GetHEADBranch: %v", err)
-		}
-		if headBranch != nil {
-			repo.DefaultBranch = headBranch.Name
-		}
-
-		if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
-			log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
-		}
-	}
-
-	if err = repo.UpdateSize(); err != nil {
-		log.Error(4, "Failed to update size for repository: %v", err)
-	}
-
-	if opts.IsMirror {
-		if _, err = x.InsertOne(&Mirror{
-			RepoID:         repo.ID,
-			Interval:       setting.Mirror.DefaultInterval,
-			EnablePrune:    true,
-			NextUpdateUnix: util.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval),
-		}); err != nil {
-			return repo, fmt.Errorf("InsertOne: %v", err)
-		}
-
-		repo.IsMirror = true
-		err = UpdateRepository(repo, false)
-	} else {
-		repo, err = CleanUpMigrateInfo(repo)
-	}
-
-	if err != nil && !repo.IsEmpty {
-		UpdateRepoIndexer(repo)
-	}
-
-	return repo, err
-}
-
-// cleanUpMigrateGitConfig removes mirror info which prevents "push --all".
-// This also removes possible user credentials.
-func cleanUpMigrateGitConfig(configPath string) error {
-	cfg, err := ini.Load(configPath)
-	if err != nil {
-		return fmt.Errorf("open config file: %v", err)
-	}
-	cfg.DeleteSection("remote \"origin\"")
-	if err = cfg.SaveToIndent(configPath, "\t"); err != nil {
-		return fmt.Errorf("save config file: %v", err)
-	}
-	return nil
-}
-
 // createDelegateHooks creates all the hooks scripts for the repo
 func createDelegateHooks(repoPath string) (err error) {
 	var (
@@ -1063,30 +906,6 @@ func createDelegateHooks(repoPath string) (err error) {
 	return nil
 }
 
-// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
-func CleanUpMigrateInfo(repo *Repository) (*Repository, error) {
-	repoPath := repo.RepoPath()
-	if err := createDelegateHooks(repoPath); err != nil {
-		return repo, fmt.Errorf("createDelegateHooks: %v", err)
-	}
-	if repo.HasWiki() {
-		if err := createDelegateHooks(repo.WikiPath()); err != nil {
-			return repo, fmt.Errorf("createDelegateHooks.(wiki): %v", err)
-		}
-	}
-
-	if err := cleanUpMigrateGitConfig(repo.GitConfigPath()); err != nil {
-		return repo, fmt.Errorf("cleanUpMigrateGitConfig: %v", err)
-	}
-	if repo.HasWiki() {
-		if err := cleanUpMigrateGitConfig(path.Join(repo.WikiPath(), "config")); err != nil {
-			return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %v", err)
-		}
-	}
-
-	return repo, UpdateRepository(repo, false)
-}
-
 // initRepoCommit temporarily changes with work directory.
 func initRepoCommit(tmpPath string, sig *git.Signature) (err error) {
 	var stderr string
@@ -1121,6 +940,7 @@ type CreateRepoOptions struct {
 	IsPrivate   bool
 	IsMirror    bool
 	AutoInit    bool
+	NoWatchers  bool
 }
 
 func getRepoInitFile(tp, name string) ([]byte, error) {
@@ -1274,7 +1094,7 @@ func IsUsableRepoName(name string) error {
 	return isUsableName(reservedRepoNames, reservedRepoPatterns, name)
 }
 
-func createRepository(e *xorm.Session, doer, u *User, repo *Repository) (err error) {
+func createRepository(e *xorm.Session, doer, u *User, repo *Repository, noWatchers bool) (err error) {
 	if err = IsUsableRepoName(repo.Name); err != nil {
 		return err
 	}
@@ -1360,10 +1180,12 @@ func createRepository(e *xorm.Session, doer, u *User, repo *Repository) (err err
 			return fmt.Errorf("watchRepo: %v", err)
 		}
 	}
-	if err = newRepoAction(e, doer, repo); err != nil {
-		return fmt.Errorf("newRepoAction: %v", err)
-	}
 
+	if !noWatchers {
+		if err = newRepoAction(e, doer, repo); err != nil {
+			return fmt.Errorf("newRepoAction: %v", err)
+		}
+	}
 	return nil
 }
 
@@ -1390,7 +1212,7 @@ func CreateRepository(doer, u *User, opts CreateRepoOptions) (_ *Repository, err
 		return nil, err
 	}
 
-	if err = createRepository(sess, doer, u, repo); err != nil {
+	if err = createRepository(sess, doer, u, repo, opts.NoWatchers); err != nil {
 		return nil, err
 	}
 
@@ -2422,7 +2244,7 @@ func ForkRepository(doer, u *User, oldRepo *Repository, name, desc string) (_ *R
 		return nil, err
 	}
 
-	if err = createRepository(sess, doer, u, repo); err != nil {
+	if err = createRepository(sess, doer, u, repo, true); err != nil {
 		return nil, err
 	}