From a0ca05e6d6b82161e8d9277f6e7cae8e544eaa23 Mon Sep 17 00:00:00 2001 From: jez Date: Sat, 21 May 2011 10:20:29 +0000 Subject: [PATCH 1/6] Add --progress flag. Recent versions of git do not output progress without this flag when not connected to a tty. --- git/remote.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/remote.py b/git/remote.py index d3639f7bf..b81c357f0 100644 --- a/git/remote.py +++ b/git/remote.py @@ -546,7 +546,7 @@ def fetch(self, refspec=None, progress=None, **kwargs): :note: As fetch does not provide progress information to non-ttys, we cannot make it available here unfortunately as in the 'push' method.""" - proc = self.repo.git.fetch(self, refspec, with_extended_output=True, as_process=True, v=True, **kwargs) + proc = self.repo.git.fetch(self, refspec, with_extended_output=True, as_process=True, v=True, progress=True, **kwargs) return self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) def pull(self, refspec=None, progress=None, **kwargs): @@ -557,7 +557,7 @@ def pull(self, refspec=None, progress=None, **kwargs): :param progress: see 'push' method :param kwargs: Additional arguments to be passed to git-pull :return: Please see 'fetch' method """ - proc = self.repo.git.pull(self, refspec, with_extended_output=True, as_process=True, v=True, **kwargs) + proc = self.repo.git.pull(self, refspec, with_extended_output=True, as_process=True, v=True, progress=True, **kwargs) return self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) def push(self, refspec=None, progress=None, **kwargs): @@ -578,7 +578,7 @@ def push(self, refspec=None, progress=None, **kwargs): in their flags. If the operation fails completely, the length of the returned IterableList will be null.""" - proc = self.repo.git.push(self, refspec, porcelain=True, as_process=True, **kwargs) + proc = self.repo.git.push(self, refspec, porcelain=True, as_process=True, progress=True, **kwargs) return self._get_push_info(proc, progress or RemoteProgress()) @property From 1f8cb16e6d88626507f60cd584b54e9d9987576a Mon Sep 17 00:00:00 2001 From: jez Date: Sat, 21 May 2011 10:17:08 +0000 Subject: [PATCH 2/6] Parse more git-fetch operation log codes. --- git/util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/git/util.py b/git/util.py index 7cbef07fb..0492b63c6 100644 --- a/git/util.py +++ b/git/util.py @@ -109,8 +109,8 @@ class RemoteProgress(object): Handler providing an interface to parse progress information emitted by git-push and git-fetch and to dispatch callbacks allowing subclasses to react to the progress. """ - _num_op_codes = 5 - BEGIN, END, COUNTING, COMPRESSING, WRITING = [1 << x for x in range(_num_op_codes)] + _num_op_codes = 7 + BEGIN, END, COUNTING, COMPRESSING, WRITING, RECEIVING, RESOLVING = [1 << x for x in range(_num_op_codes)] STAGE_MASK = BEGIN|END OP_MASK = ~STAGE_MASK @@ -168,6 +168,10 @@ def _parse_progress_line(self, line): op_code |= self.COMPRESSING elif op_name == "Writing objects": op_code |= self.WRITING + elif op_name == 'Receiving objects': + op_code |= self.RECEIVING + elif op_name == 'Resolving deltas': + op_code |= self.RESOLVING else: raise ValueError("Operation name %r unknown" % op_name) From f0f02bbb99b5a888eaedb92ae0df8bb95174e53d Mon Sep 17 00:00:00 2001 From: jez Date: Sat, 21 May 2011 10:13:45 +0000 Subject: [PATCH 3/6] Add progress tracking for git-clone. --- git/remote.py | 50 +++++++++--------------------------------------- git/repo/base.py | 23 ++++++++++++++++------ git/util.py | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 47 deletions(-) diff --git a/git/remote.py b/git/remote.py index b81c357f0..0d82ac317 100644 --- a/git/remote.py +++ b/git/remote.py @@ -24,7 +24,11 @@ TagReference ) -from git.util import join_path +from git.util import ( + join_path, + _digest_process_messages, + _finalize_proc + ) from gitdb.util import join import re @@ -432,42 +436,6 @@ def update(self, **kwargs): self.repo.git.remote("update", self.name) return self - def _digest_process_messages(self, fh, progress): - """Read progress messages from file-like object fh, supplying the respective - progress messages to the progress instance. - - :return: list(line, ...) list of lines without linebreaks that did - not contain progress information""" - line_so_far = '' - dropped_lines = list() - while True: - char = fh.read(1) - if not char: - break - - if char in ('\r', '\n'): - dropped_lines.extend(progress._parse_progress_line(line_so_far)) - line_so_far = '' - else: - line_so_far += char - # END process parsed line - # END while file is not done reading - return dropped_lines - - - def _finalize_proc(self, proc): - """Wait for the process (fetch, pull or push) and handle its errors accordingly""" - try: - proc.wait() - except GitCommandError,e: - # if a push has rejected items, the command has non-zero return status - # a return status of 128 indicates a connection error - reraise the previous one - if proc.poll() == 128: - raise - pass - # END exception handling - - def _get_fetch_info_from_stderr(self, proc, progress): # skip first line as it is some remote info we are not interested in output = IterableList('name') @@ -477,7 +445,7 @@ def _get_fetch_info_from_stderr(self, proc, progress): # this also waits for the command to finish # Skip some progress lines that don't provide relevant information fetch_info_lines = list() - for line in self._digest_process_messages(proc.stderr, progress): + for line in _digest_process_messages(proc.stderr, progress): if line.startswith('From') or line.startswith('remote: Total'): continue elif line.startswith('warning:'): @@ -499,7 +467,7 @@ def _get_fetch_info_from_stderr(self, proc, progress): output.extend(FetchInfo._from_line(self.repo, err_line, fetch_line) for err_line,fetch_line in zip(fetch_info_lines, fetch_head_info)) - self._finalize_proc(proc) + _finalize_proc(proc) return output def _get_push_info(self, proc, progress): @@ -507,7 +475,7 @@ def _get_push_info(self, proc, progress): # we hope stdout can hold all the data, it should ... # read the lines manually as it will use carriage returns between the messages # to override the previous one. This is why we read the bytes manually - self._digest_process_messages(proc.stderr, progress) + _digest_process_messages(proc.stderr, progress) output = IterableList('name') for line in proc.stdout.readlines(): @@ -519,7 +487,7 @@ def _get_push_info(self, proc, progress): # END exception handling # END for each line - self._finalize_proc(proc) + _finalize_proc(proc) return output diff --git a/git/repo/base.py b/git/repo/base.py index 0405a5f9c..55aec05b0 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -18,6 +18,11 @@ ) +from git.util import ( + _digest_process_messages, + _finalize_proc + ) + from gitdb.util import ( join, isfile, @@ -652,7 +657,7 @@ def init(cls, path=None, mkdir=True, **kwargs): return Repo(path) @classmethod - def _clone(cls, git, url, path, odb_default_type, **kwargs): + def _clone(cls, git, url, path, odb_default_type, progress, **kwargs): # special handling for windows for path at which the clone should be # created. # tilde '~' will be expanded to the HOME no matter where the ~ occours. Hence @@ -679,7 +684,10 @@ def _clone(cls, git, url, path, odb_default_type, **kwargs): # END windows handling try: - git.clone(url, path, **kwargs) + proc = git.clone(url, path, with_extended_output=True, as_process=True, v=True, progress=True, **kwargs) + if progress: + _digest_process_messages(proc.stderr, progress) + _finalize_proc(proc) finally: if prev_cwd is not None: os.chdir(prev_cwd) @@ -703,11 +711,13 @@ def _clone(cls, git, url, path, odb_default_type, **kwargs): # END handle remote repo return repo - def clone(self, path, **kwargs): + def clone(self, path, progress=None, **kwargs): """Create a clone from this repository. :param path: is the full path of the new repo (traditionally ends with ./.git). + :param progress: See 'git.remote.Remote.push'. + :param kwargs: odbt = ObjectDatabase Type, allowing to determine the object database implementation used by the returned Repo instance @@ -715,16 +725,17 @@ def clone(self, path, **kwargs): All remaining keyword arguments are given to the git-clone command :return: ``git.Repo`` (the newly cloned repo)""" - return self._clone(self.git, self.git_dir, path, type(self.odb), **kwargs) + return self._clone(self.git, self.git_dir, path, type(self.odb), progress, **kwargs) @classmethod - def clone_from(cls, url, to_path, **kwargs): + def clone_from(cls, url, to_path, progress=None, **kwargs): """Create a clone from the given URL :param url: valid git url, see http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS :param to_path: Path to which the repository should be cloned to + :param progress: See 'git.remote.Remote.push'. :param kwargs: see the ``clone`` method :return: Repo instance pointing to the cloned directory""" - return cls._clone(Git(os.getcwd()), url, to_path, GitCmdObjectDB, **kwargs) + return cls._clone(Git(os.getcwd()), url, to_path, GitCmdObjectDB, progress, **kwargs) def archive(self, ostream, treeish=None, prefix=None, **kwargs): """Archive the tree at the given revision. diff --git a/git/util.py b/git/util.py index 0492b63c6..fcae36890 100644 --- a/git/util.py +++ b/git/util.py @@ -100,6 +100,40 @@ def get_user_id(): # END get username from login return "%s@%s" % (username, platform.node()) +def _digest_process_messages(fh, progress): + """Read progress messages from file-like object fh, supplying the respective + progress messages to the progress instance. + + :return: list(line, ...) list of lines without linebreaks that did + not contain progress information""" + line_so_far = '' + dropped_lines = list() + while True: + char = fh.read(1) + if not char: + break + + if char in ('\r', '\n'): + dropped_lines.extend(progress._parse_progress_line(line_so_far)) + line_so_far = '' + else: + line_so_far += char + # END process parsed line + # END while file is not done reading + return dropped_lines + +def _finalize_proc(proc): + """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly""" + try: + proc.wait() + except GitCommandError,e: + # if a push has rejected items, the command has non-zero return status + # a return status of 128 indicates a connection error - reraise the previous one + if proc.poll() == 128: + raise + pass + # END exception handling + #} END utilities #{ Classes From 6b655beb8865b5ea3d19bb42e45fedbfc25e6624 Mon Sep 17 00:00:00 2001 From: jez Date: Tue, 24 May 2011 18:44:43 +0000 Subject: [PATCH 4/6] Add version-specific behavior. --progress is not a valid flag for earlier versions of git, so we check for the version before using it. --- git/cmd.py | 13 ++++++++++++- git/remote.py | 9 ++++++--- git/repo/base.py | 3 ++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 60887f5da..a6968bc33 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -41,7 +41,7 @@ class Git(object): of the command to stdout. Set its value to 'full' to see details about the returned values. """ - __slots__ = ("_working_dir", "cat_file_all", "cat_file_header") + __slots__ = ("_working_dir", "cat_file_all", "cat_file_header", "_version") # CONFIGURATION # The size in bytes read from stdout when copying git's output to another stream @@ -205,6 +205,7 @@ def __init__(self, working_dir=None): .git directory in case of bare repositories.""" super(Git, self).__init__() self._working_dir = working_dir + self._version = self._get_version_as_tuple() # cached command slots self.cat_file_header = None @@ -513,3 +514,13 @@ def clear_cache(self): self.cat_file_all = None self.cat_file_header = None return self + + def _get_version_as_tuple(self): + """ Get and parse git version string. """ + version_str = self._call_process('version') + version_numbers = version_str.rpartition(' ')[2] + return tuple(version_numbers.split('.')) + + @property + def version(self): + return self._version diff --git a/git/remote.py b/git/remote.py index 0d82ac317..440be6b11 100644 --- a/git/remote.py +++ b/git/remote.py @@ -514,7 +514,8 @@ def fetch(self, refspec=None, progress=None, **kwargs): :note: As fetch does not provide progress information to non-ttys, we cannot make it available here unfortunately as in the 'push' method.""" - proc = self.repo.git.fetch(self, refspec, with_extended_output=True, as_process=True, v=True, progress=True, **kwargs) + if self.repo.git.version >= (1, 7, 0, 0): kwargs['progress'] = True + proc = self.repo.git.fetch(self, refspec, with_extended_output=True, as_process=True, v=True, **kwargs) return self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) def pull(self, refspec=None, progress=None, **kwargs): @@ -525,7 +526,8 @@ def pull(self, refspec=None, progress=None, **kwargs): :param progress: see 'push' method :param kwargs: Additional arguments to be passed to git-pull :return: Please see 'fetch' method """ - proc = self.repo.git.pull(self, refspec, with_extended_output=True, as_process=True, v=True, progress=True, **kwargs) + if self.repo.git.version >= (1, 7, 0, 0): kwargs['progress'] = True + proc = self.repo.git.pull(self, refspec, with_extended_output=True, as_process=True, v=True, **kwargs) return self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) def push(self, refspec=None, progress=None, **kwargs): @@ -546,7 +548,8 @@ def push(self, refspec=None, progress=None, **kwargs): in their flags. If the operation fails completely, the length of the returned IterableList will be null.""" - proc = self.repo.git.push(self, refspec, porcelain=True, as_process=True, progress=True, **kwargs) + if self.repo.git.version >= (1, 7, 0, 0): kwargs['progress'] = True + proc = self.repo.git.push(self, refspec, porcelain=True, as_process=True, progress=True, v=True, **kwargs) return self._get_push_info(proc, progress or RemoteProgress()) @property diff --git a/git/repo/base.py b/git/repo/base.py index 55aec05b0..dfb959b58 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -684,7 +684,8 @@ def _clone(cls, git, url, path, odb_default_type, progress, **kwargs): # END windows handling try: - proc = git.clone(url, path, with_extended_output=True, as_process=True, v=True, progress=True, **kwargs) + if git.version >= (1, 7, 0, 0): kwargs['progress'] = True + proc = git.clone(url, path, with_extended_output=True, as_process=True, v=True, **kwargs) if progress: _digest_process_messages(proc.stderr, progress) _finalize_proc(proc) From 12c93f564b81603f537425b61d8c57b83d9f24c7 Mon Sep 17 00:00:00 2001 From: jez Date: Wed, 25 May 2011 10:11:52 +0000 Subject: [PATCH 5/6] Rename version to version_info. --- git/cmd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index a6968bc33..1b455b757 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -41,7 +41,7 @@ class Git(object): of the command to stdout. Set its value to 'full' to see details about the returned values. """ - __slots__ = ("_working_dir", "cat_file_all", "cat_file_header", "_version") + __slots__ = ("_working_dir", "cat_file_all", "cat_file_header", "_version_info") # CONFIGURATION # The size in bytes read from stdout when copying git's output to another stream @@ -205,7 +205,7 @@ def __init__(self, working_dir=None): .git directory in case of bare repositories.""" super(Git, self).__init__() self._working_dir = working_dir - self._version = self._get_version_as_tuple() + self._version_info = self._get_version_as_tuple() # cached command slots self.cat_file_header = None @@ -522,5 +522,5 @@ def _get_version_as_tuple(self): return tuple(version_numbers.split('.')) @property - def version(self): - return self._version + def version_info(self): + return self._version_info From c296c6f51aa86cb45cc7e731296a3d01905df38d Mon Sep 17 00:00:00 2001 From: jez Date: Thu, 2 Jun 2011 00:29:35 +0000 Subject: [PATCH 6/6] Fix up the renaming. --- git/remote.py | 6 +++--- git/repo/base.py | 2 +- git/util.py | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/git/remote.py b/git/remote.py index 440be6b11..885443529 100644 --- a/git/remote.py +++ b/git/remote.py @@ -514,7 +514,7 @@ def fetch(self, refspec=None, progress=None, **kwargs): :note: As fetch does not provide progress information to non-ttys, we cannot make it available here unfortunately as in the 'push' method.""" - if self.repo.git.version >= (1, 7, 0, 0): kwargs['progress'] = True + if self.repo.git.version_info >= (1, 7, 0, 0): kwargs['progress'] = True proc = self.repo.git.fetch(self, refspec, with_extended_output=True, as_process=True, v=True, **kwargs) return self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) @@ -526,7 +526,7 @@ def pull(self, refspec=None, progress=None, **kwargs): :param progress: see 'push' method :param kwargs: Additional arguments to be passed to git-pull :return: Please see 'fetch' method """ - if self.repo.git.version >= (1, 7, 0, 0): kwargs['progress'] = True + if self.repo.git.version_info >= (1, 7, 0, 0): kwargs['progress'] = True proc = self.repo.git.pull(self, refspec, with_extended_output=True, as_process=True, v=True, **kwargs) return self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) @@ -548,7 +548,7 @@ def push(self, refspec=None, progress=None, **kwargs): in their flags. If the operation fails completely, the length of the returned IterableList will be null.""" - if self.repo.git.version >= (1, 7, 0, 0): kwargs['progress'] = True + if self.repo.git.version_info >= (1, 7, 0, 0): kwargs['progress'] = True proc = self.repo.git.push(self, refspec, porcelain=True, as_process=True, progress=True, v=True, **kwargs) return self._get_push_info(proc, progress or RemoteProgress()) diff --git a/git/repo/base.py b/git/repo/base.py index dfb959b58..07dcb2c83 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -684,7 +684,7 @@ def _clone(cls, git, url, path, odb_default_type, progress, **kwargs): # END windows handling try: - if git.version >= (1, 7, 0, 0): kwargs['progress'] = True + if git.version_info >= (1, 7, 0, 0): kwargs['progress'] = True proc = git.clone(url, path, with_extended_output=True, as_process=True, v=True, **kwargs) if progress: _digest_process_messages(proc.stderr, progress) diff --git a/git/util.py b/git/util.py index fcae36890..e7099ff73 100644 --- a/git/util.py +++ b/git/util.py @@ -10,6 +10,7 @@ import time import tempfile import platform +from exc import GitCommandError from gitdb.util import ( make_sha,