refactor: centralize PAT validation, streamline repo checks & misc cleanup (#349)

* refactor: centralize PAT validation, streamline repo checks & housekeeping

* `.venv*` to `.gitignore`
* `# type: ignore[attr-defined]` hints in `compat_typing.py` for IDE-agnostic imports
* Helpful PAT string in `InvalidGitHubTokenError` for easier debugging

* Bump **ruff-pre-commit** hook → `v0.12.1`
* CONTRIBUTING:
  * Require **Python 3.9+**
  * Recommend signed (`-S`) commits
* PAT validation now happens **only** in entry points
  (`utils.auth.resolve_token` for CLI/lib, `server.process_query` for Web UI)
* Unified `_check_github_repo_exists` into `check_repo_exists`, replacing
  `curl -I` with `curl --silent --location --write-out %{http_code} -o /dev/null`
* Broaden `_GITHUB_PAT_PATTERN`
* `create_git_auth_header` raises `ValueError` when hostname is missing
* Tests updated to expect raw HTTP-code output

* Superfluous “token can be set via `GITHUB_TOKEN`” notes in docstrings
* `.gitingestignore` & `.terraform` from `DEFAULT_IGNORE_PATTERNS`
* Token validation inside `create_git_command`
* Obsolete `test_create_git_command_invalid_token`

* Adjust `test_clone.py` and `test_git_utils.py` for new status-code handling
* Consolidate mocks after token-validation relocation

BREAKING CHANGE:
`create_git_command` no longer validates GitHub tokens; callers must ensure
tokens are valid (via `validate_github_token`) before invoking lower-level
git helpers.


---------

Co-authored-by: Copilot <[email protected]>

Author: filipchristiansen

.gitignore

@@ -126,6 +126,7 @@ celerybeat.pid
 # Environments
 .env
 .venv
+.venv*
 env/
 venv/
 ENV/

.pre-commit-config.yaml

@@ -75,7 +75,7 @@ repos:
         args: ["--disable=line-length"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.0
+    rev: v0.12.1
     hooks:
       - id: ruff-check
       - id: ruff-format

CONTRIBUTING.md

@@ -19,6 +19,8 @@ Thanks for your interest in contributing to Gitingest! 🚀 Gitingest aims to be
    cd gitingest
    ```
 
+   **Note**: To contribute, ensure you have **Python 3.9 or newer** installed, as some of the `pre-commit` hooks (e.g. `pyupgrade`) require Python 3.9+.
+
 3. Set up the development environment and install dependencies:
 
    ```bash
@@ -31,7 +33,7 @@ Thanks for your interest in contributing to Gitingest! 🚀 Gitingest aims to be
 4. Create a new branch for your changes:
 
     ```bash
-    git checkout -b your-branch
+    git checkout -S -b your-branch
     ```
 
 5. Make your changes. Make sure to add corresponding tests for your changes.
@@ -66,10 +68,18 @@ Thanks for your interest in contributing to Gitingest! 🚀 Gitingest aims to be
 
 9. Confirm that everything is working as expected. If you encounter any issues, fix them and repeat steps 6 to 8.
 
-10. Commit your changes:
+10. Commit your changes (signed):
+
+    All commits to Gitingest must be [GPG-signed](https://docs.github.com/en/authentication/managing-commit-signature-verification) so that the project can verify the authorship of every contribution. You can either configure Git globally with:
+
+    ```bash
+    git config --global commit.gpgSign true
+    ```
+
+    or pass the `-S` flag as shown below.
 
     ```bash
-    git commit -m "Your commit message"
+    git commit -S -m "Your commit message"
     ```
 
     If `pre-commit` raises any issues, fix them and repeat steps 6 to 9.

src/gitingest/clone.py

@@ -13,7 +13,6 @@
     ensure_git_installed,
     is_github_host,
     run_command,
-    validate_github_token,
 )
 from gitingest.utils.os_utils import ensure_directory
 from gitingest.utils.timeout_wrapper import async_timeout
@@ -23,7 +22,7 @@
 
 
 @async_timeout(DEFAULT_TIMEOUT)
-async def clone_repo(config: CloneConfig, token: str | None = None) -> None:
+async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
     """Clone a repository to a local path based on the provided configuration.
 
     This function handles the process of cloning a Git repository to the local file system.
@@ -36,7 +35,6 @@ async def clone_repo(config: CloneConfig, token: str | None = None) -> None:
         The configuration for cloning the repository.
     token : str | None
         GitHub personal access token (PAT) for accessing private repositories.
-        Can also be set via the ``GITHUB_TOKEN`` environment variable.
 
     Raises
     ------
@@ -51,10 +49,6 @@ async def clone_repo(config: CloneConfig, token: str | None = None) -> None:
     branch: str | None = config.branch
     partial_clone: bool = config.subpath != "/"
 
-    # Validate token if provided
-    if token and is_github_host(url):
-        validate_github_token(token)
-
     # Create parent directory if it doesn't exist
     await ensure_directory(Path(local_path).parent)
 

src/gitingest/query_parser.py

@@ -49,7 +49,6 @@ async def parse_query(
         Patterns to ignore. Can be a set of strings or a single string.
     token : str | None
         GitHub personal access token (PAT) for accessing private repositories.
-        Can also be set via the ``GITHUB_TOKEN`` environment variable.
 
     Returns
     -------
@@ -109,7 +108,6 @@ async def _parse_remote_repo(source: str, token: str | None = None) -> Ingestion
         The URL or domain-less slug to parse.
     token : str | None
         GitHub personal access token (PAT) for accessing private repositories.
-        Can also be set via the ``GITHUB_TOKEN`` environment variable.
 
     Returns
     -------
@@ -301,7 +299,6 @@ async def try_domains_for_user_and_repo(user_name: str, repo_name: str, token: s
         The name of the repository.
     token : str | None
         GitHub personal access token (PAT) for accessing private repositories.
-        Can also be set via the ``GITHUB_TOKEN`` environment variable.
 
     Returns
     -------
@@ -316,7 +313,7 @@ async def try_domains_for_user_and_repo(user_name: str, repo_name: str, token: s
     """
     for domain in KNOWN_GIT_HOSTS:
         candidate = f"https://{domain}/{user_name}/{repo_name}"
-        if await check_repo_exists(candidate, token=token if domain == "github.com" else None):
+        if await check_repo_exists(candidate, token=token if domain.startswith("github.") else None):
             return domain
 
     msg = f"Could not find a valid repository host for '{user_name}/{repo_name}'."

src/gitingest/utils/auth.py

@@ -4,6 +4,8 @@
 
 import os
 
+from gitingest.utils.git_utils import validate_github_token
+
 
 def resolve_token(token: str | None) -> str | None:
     """Resolve the token to use for the query.
@@ -19,4 +21,7 @@ def resolve_token(token: str | None) -> str | None:
         The resolved token.
 
     """
-    return token or os.getenv("GITHUB_TOKEN")
+    token = token or os.getenv("GITHUB_TOKEN")
+    if token:
+        validate_github_token(token)
+    return token

src/gitingest/utils/compat_typing.py

@@ -1,13 +1,13 @@
 """Compatibility layer for typing."""
 
 try:
-    from typing import ParamSpec, TypeAlias  # Py ≥ 3.10
+    from typing import ParamSpec, TypeAlias  # type: ignore[attr-defined]  # Py ≥ 3.10
 except ImportError:
-    from typing_extensions import ParamSpec, TypeAlias  # Py 3.8 / 3.9
+    from typing_extensions import ParamSpec, TypeAlias  # type: ignore[attr-defined]  # Py 3.8 / 3.9
 
 try:
-    from typing import Annotated  # Py ≥ 3.9
+    from typing import Annotated  # type: ignore[attr-defined]  # Py ≥ 3.9
 except ImportError:
-    from typing_extensions import Annotated  # Py 3.8
+    from typing_extensions import Annotated  # type: ignore[attr-defined]  # Py 3.8
 
 __all__ = ["Annotated", "ParamSpec", "TypeAlias"]

src/gitingest/utils/exceptions.py

@@ -42,7 +42,8 @@ class InvalidGitHubTokenError(ValueError):
     """Exception raised when a GitHub Personal Access Token is malformed."""
 
     def __init__(self) -> None:
-        super().__init__(
-            "Invalid GitHub token format. Token should start with 'github_pat_' or 'ghp_' "
-            "followed by at least 36 characters of letters, numbers, and underscores.",
+        msg = (
+            "Invalid GitHub token format. To generate a token, go to "
+            "https://github.com/settings/tokens/new?description=gitingest&scopes=repo."
         )
+        super().__init__(msg)