feat: partial cloning (#188)

This commit introduces the `partial_clone_repo` function, which performs a sparse clone
of a repository (`git clone --filter=blob:none --sparse`) based on query parameters
from a `ParsedQuery` object.

- Add a new method (extact_clone_config) in ParsedQuery to encapsulate the creation
  of a CloneConfig from query parameters.
- Replace repeated CloneConfig instantiation in repository_ingest.py and
  query_processor.py with calls to the new method.
- Simplify code and improve maintainability by centralizing CloneConfig logic.

* Refactor cloning logic to support subpath-based partial clones

- Add `repo_name` and `subpath` fields to `CloneConfig` for flexible cloning.
- Split out `partial_clone_repo` and `full_clone_repo` to handle subpath vs. full clones.
- Update `CloneConfig` to include `repo_name` and `subpath`.
- Simplify query processing to always call `clone_repo`, which now delegates to partial or full clone.
- Improve docstrings to reflect new parameters and return types.

---------

Co-authored-by: cyclotruc <[email protected]>

Author: filipchristiansen

src/gitingest/query_parser.py

@@ -13,7 +13,7 @@
 from gitingest.config import MAX_FILE_SIZE, TMP_BASE_PATH
 from gitingest.exceptions import InvalidPatternError
 from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
-from gitingest.repository_clone import _check_repo_exists, fetch_remote_branch_list
+from gitingest.repository_clone import CloneConfig, _check_repo_exists, fetch_remote_branch_list
 
 HEX_DIGITS: Set[str] = set(string.hexdigits)
 
@@ -35,11 +35,11 @@ class ParsedQuery:  # pylint: disable=too-many-instance-attributes
 
     user_name: Optional[str]
     repo_name: Optional[str]
-    subpath: str
     local_path: Path
     url: Optional[str]
     slug: str
     id: str
+    subpath: str = "/"
     type: Optional[str] = None
     branch: Optional[str] = None
     commit: Optional[str] = None
@@ -48,6 +48,31 @@ class ParsedQuery:  # pylint: disable=too-many-instance-attributes
     include_patterns: Optional[Set[str]] = None
     pattern_type: Optional[str] = None
 
+    def extact_clone_config(self) -> CloneConfig:
+        """
+        Extract the relevant fields for the CloneConfig object.
+
+        Returns
+        -------
+        CloneConfig
+            A CloneConfig object containing the relevant fields.
+
+        Raises
+        ------
+        ValueError
+            If the 'url' parameter is not provided.
+        """
+        if not self.url:
+            raise ValueError("The 'url' parameter is required.")
+
+        return CloneConfig(
+            url=self.url,
+            local_path=str(self.local_path),
+            commit=self.commit,
+            branch=self.branch,
+            subpath=self.subpath,
+        )
+
 
 async def parse_query(
     source: str,
@@ -171,7 +196,6 @@ async def _parse_repo_source(source: str) -> ParsedQuery:
         user_name=user_name,
         repo_name=repo_name,
         url=url,
-        subpath="/",
         local_path=local_path,
         slug=slug,
         id=_id,
@@ -363,7 +387,6 @@ def _parse_path(path_str: str) -> ParsedQuery:
         user_name=None,
         repo_name=None,
         url=None,
-        subpath="/",
         local_path=path_obj,
         slug=f"{path_obj.parent.name}/{path_obj.name}",
         id=str(uuid.uuid4()),

src/gitingest/repository_clone.py

@@ -29,16 +29,19 @@ class CloneConfig:
         The specific commit hash to check out after cloning (default is None).
     branch : str, optional
         The branch to clone (default is None).
+    subpath : str
+        The subpath to clone from the repository (default is "/").
     """
 
     url: str
     local_path: str
     commit: Optional[str] = None
     branch: Optional[str] = None
+    subpath: str = "/"
 
 
 @async_timeout(TIMEOUT)
-async def clone_repo(config: CloneConfig) -> Tuple[bytes, bytes]:
+async def clone_repo(config: CloneConfig) -> None:
     """
     Clone a repository to a local path based on the provided configuration.
 
@@ -49,35 +52,21 @@ async def clone_repo(config: CloneConfig) -> Tuple[bytes, bytes]:
     Parameters
     ----------
     config : CloneConfig
-        A dictionary containing the following keys:
-            - url (str): The URL of the repository.
-            - local_path (str): The local path to clone the repository to.
-            - commit (str, optional): The specific commit hash to checkout.
-            - branch (str, optional): The branch to clone. Defaults to 'main' or 'master' if not provided.
-
-    Returns
-    -------
-    Tuple[bytes, bytes]
-        A tuple containing the stdout and stderr of the Git commands executed.
+        The configuration for cloning the repository.
 
     Raises
     ------
     ValueError
-        If the 'url' or 'local_path' parameters are missing, or if the repository is not found.
+        If the repository is not found or if the provided URL is invalid.
     OSError
-        If there is an error creating the parent directory structure.
+        If an error occurs while creating the parent directory for the repository.
     """
     # Extract and validate query parameters
     url: str = config.url
     local_path: str = config.local_path
     commit: Optional[str] = config.commit
     branch: Optional[str] = config.branch
-
-    if not url:
-        raise ValueError("The 'url' parameter is required.")
-
-    if not local_path:
-        raise ValueError("The 'local_path' parameter is required.")
+    partial_clone: bool = config.subpath != "/"
 
     # Create parent directory if it doesn't exist
     parent_dir = Path(local_path).parent
@@ -90,34 +79,32 @@ async def clone_repo(config: CloneConfig) -> Tuple[bytes, bytes]:
     if not await _check_repo_exists(url):
         raise ValueError("Repository not found, make sure it is public")
 
-    if commit:
-        # Scenario 1: Clone and checkout a specific commit
-        # Clone the repository without depth to ensure full history for checkout
-        clone_cmd = ["git", "clone", "--recurse-submodules", "--single-branch", url, local_path]
-        await _run_git_command(*clone_cmd)
+    clone_cmd = ["git", "clone", "--recurse-submodules", "--single-branch"]
 
-        # Checkout the specific commit
-        checkout_cmd = ["git", "-C", local_path, "checkout", commit]
-        return await _run_git_command(*checkout_cmd)
+    if partial_clone:
+        clone_cmd += ["--filter=blob:none", "--sparse"]
 
-    if branch and branch.lower() not in ("main", "master"):
-        # Scenario 2: Clone a specific branch with shallow depth
-        clone_cmd = [
-            "git",
-            "clone",
-            "--recurse-submodules",
-            "--depth=1",
-            "--single-branch",
-            "--branch",
-            branch,
-            url,
-            local_path,
-        ]
-        return await _run_git_command(*clone_cmd)
-
-    # Scenario 3: Clone the default branch with shallow depth
-    clone_cmd = ["git", "clone", "--recurse-submodules", "--depth=1", "--single-branch", url, local_path]
-    return await _run_git_command(*clone_cmd)
+    if not commit:
+        clone_cmd += ["--depth=1"]
+        if branch and branch.lower() not in ("main", "master"):
+            clone_cmd += ["--branch", branch]
+
+    clone_cmd += [url, local_path]
+
+    # Clone the repository
+    await _run_command(*clone_cmd)
+
+    if commit or partial_clone:
+        checkout_cmd = ["git", "-C", local_path]
+
+        if partial_clone:
+            checkout_cmd += ["sparse-checkout", "set", config.subpath.lstrip("/")]
+
+        if commit:
+            checkout_cmd += ["checkout", commit]
+
+        # Check out the specific commit and/or subpath
+        await _run_command(*checkout_cmd)
 
 
 async def _check_repo_exists(url: str) -> bool:
@@ -176,7 +163,7 @@ async def fetch_remote_branch_list(url: str) -> List[str]:
         A list of branch names available in the remote repository.
     """
     fetch_branches_command = ["git", "ls-remote", "--heads", url]
-    stdout, _ = await _run_git_command(*fetch_branches_command)
+    stdout, _ = await _run_command(*fetch_branches_command)
     stdout_decoded = stdout.decode()
 
     return [
@@ -186,41 +173,28 @@ async def fetch_remote_branch_list(url: str) -> List[str]:
     ]
 
 
-async def _run_git_command(*args: str) -> Tuple[bytes, bytes]:
+async def _run_command(*args: str) -> Tuple[bytes, bytes]:
     """
-    Execute a Git command asynchronously and captures its output.
+    Execute a command asynchronously and captures its output.
 
     Parameters
     ----------
     *args : str
-        The Git command and its arguments to execute.
+        The command and its arguments to execute.
 
     Returns
     -------
     Tuple[bytes, bytes]
-        A tuple containing the stdout and stderr of the Git command.
+        A tuple containing the stdout and stderr of the command.
 
     Raises
     ------
     RuntimeError
-        If Git is not installed or if the Git command exits with a non-zero status.
+        If command exits with a non-zero status.
     """
-    # Check if Git is installed
-    try:
-        version_proc = await asyncio.create_subprocess_exec(
-            "git",
-            "--version",
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
-        _, stderr = await version_proc.communicate()
-        if version_proc.returncode != 0:
-            error_message = stderr.decode().strip() if stderr else "Git command not found"
-            raise RuntimeError(f"Git is not installed or not accessible: {error_message}")
-    except FileNotFoundError as exc:
-        raise RuntimeError("Git is not installed. Please install Git before proceeding.") from exc
+    await check_git_installed()
 
-    # Execute the requested Git command
+    # Execute the requested command
     proc = await asyncio.create_subprocess_exec(
         *args,
         stdout=asyncio.subprocess.PIPE,
@@ -229,11 +203,36 @@ async def _run_git_command(*args: str) -> Tuple[bytes, bytes]:
     stdout, stderr = await proc.communicate()
     if proc.returncode != 0:
         error_message = stderr.decode().strip()
-        raise RuntimeError(f"Git command failed: {' '.join(args)}\nError: {error_message}")
+        raise RuntimeError(f"Command failed: {' '.join(args)}\nError: {error_message}")
 
     return stdout, stderr
 
 
+async def check_git_installed() -> None:
+    """
+    Check if Git is installed and accessible on the system.
+
+    Raises
+    ------
+    RuntimeError
+        If Git is not installed or if the Git command exits with a non-zero status.
+    """
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "git",
+            "--version",
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        _, stderr = await proc.communicate()
+        if proc.returncode != 0:
+            error_message = stderr.decode().strip() if stderr else "Git command not found"
+            raise RuntimeError(f"Git is not installed or not accessible: {error_message}")
+
+    except FileNotFoundError as exc:
+        raise RuntimeError("Git is not installed. Please install Git before proceeding.") from exc
+
+
 def _get_status_code(response: str) -> int:
     """
     Extract the status code from an HTTP response.

src/gitingest/repository_ingest.py

@@ -8,7 +8,7 @@
 from gitingest.config import TMP_BASE_PATH
 from gitingest.query_ingestion import run_ingest_query
 from gitingest.query_parser import ParsedQuery, parse_query
-from gitingest.repository_clone import CloneConfig, clone_repo
+from gitingest.repository_clone import clone_repo
 
 
 async def ingest_async(
@@ -70,13 +70,7 @@ async def ingest_async(
             selected_branch = branch if branch else parsed_query.branch  # prioritize branch argument
             parsed_query.branch = selected_branch
 
-            # Extract relevant fields for CloneConfig
-            clone_config = CloneConfig(
-                url=parsed_query.url,
-                local_path=str(parsed_query.local_path),
-                commit=parsed_query.commit,
-                branch=selected_branch,
-            )
+            clone_config = parsed_query.extact_clone_config()
             clone_coroutine = clone_repo(clone_config)
 
             if inspect.iscoroutine(clone_coroutine):

src/server/query_processor.py

@@ -7,7 +7,7 @@
 
 from gitingest.query_ingestion import run_ingest_query
 from gitingest.query_parser import ParsedQuery, parse_query
-from gitingest.repository_clone import CloneConfig, clone_repo
+from gitingest.repository_clone import clone_repo
 from server.server_config import EXAMPLE_REPOS, MAX_DISPLAY_SIZE, templates
 from server.server_utils import Colors, log_slider_to_size
 
@@ -84,15 +84,11 @@ async def process_query(
         if not parsed_query.url:
             raise ValueError("The 'url' parameter is required.")
 
-        clone_config = CloneConfig(
-            url=parsed_query.url,
-            local_path=str(parsed_query.local_path),
-            commit=parsed_query.commit,
-            branch=parsed_query.branch,
-        )
+        clone_config = parsed_query.extact_clone_config()
         await clone_repo(clone_config)
+
         summary, tree, content = run_ingest_query(parsed_query)
-        with open(f"{clone_config.local_path}.txt", "w", encoding="utf-8") as f:
+        with open(f"{parsed_query.local_path}.txt", "w", encoding="utf-8") as f:
             f.write(tree + "\n" + content)
     except Exception as e:
         # hack to print error message when query is not defined

tests/query_parser/test_query_parser.py

@@ -153,7 +153,7 @@ async def test_parse_url_with_subpaths() -> None:
     Then user, repo, branch, and subpath should be identified correctly.
     """
     url = "https://github.com/user/repo/tree/main/subdir/file"
-    with patch("gitingest.repository_clone._run_git_command", new_callable=AsyncMock) as mock_run_git_command:
+    with patch("gitingest.repository_clone._run_command", new_callable=AsyncMock) as mock_run_git_command:
         mock_run_git_command.return_value = (b"refs/heads/main\nrefs/heads/dev\nrefs/heads/feature-branch\n", b"")
         with patch(
             "gitingest.repository_clone.fetch_remote_branch_list", new_callable=AsyncMock
@@ -332,7 +332,7 @@ async def test_parse_url_branch_and_commit_distinction(url: str, expected_branch
     When `_parse_repo_source` is called with branch fetching,
     Then the function should correctly set `branch` or `commit` based on the URL content.
     """
-    with patch("gitingest.repository_clone._run_git_command", new_callable=AsyncMock) as mock_run_git_command:
+    with patch("gitingest.repository_clone._run_command", new_callable=AsyncMock) as mock_run_git_command:
         # Mocking the return value to include 'main' and some additional branches
         mock_run_git_command.return_value = (b"refs/heads/main\nrefs/heads/dev\nrefs/heads/feature-branch\n", b"")
         with patch(
@@ -439,7 +439,7 @@ async def test_parse_repo_source_with_failed_git_command(url, expected_branch, e
 
         with pytest.warns(
             RuntimeWarning,
-            match="Warning: Failed to fetch branch list: Git command failed: "
+            match="Warning: Failed to fetch branch list: Command failed: "
             "git ls-remote --heads https://github.com/user/repo",
         ):
 
@@ -469,7 +469,7 @@ async def test_parse_repo_source_with_various_url_patterns(url, expected_branch,
     When `_parse_repo_source` is called with remote branch fetching,
     Then the correct branch/subpath should be set or None if unmatched.
     """
-    with patch("gitingest.repository_clone._run_git_command", new_callable=AsyncMock) as mock_run_git_command:
+    with patch("gitingest.repository_clone._run_command", new_callable=AsyncMock) as mock_run_git_command:
         with patch(
             "gitingest.repository_clone.fetch_remote_branch_list", new_callable=AsyncMock
         ) as mock_fetch_branches: