Skip to content
1 change: 1 addition & 0 deletions src/gitingest/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ def main(

It calls the async main function to run the command.

\b
Parameters
----------
source : str
Expand Down
14 changes: 7 additions & 7 deletions src/gitingest/ingestion.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def ingest_query(query: IngestionQuery) -> Tuple[str, str, str]:

relative_path = path.relative_to(query.local_path)

file_node = FileSystemNode(
query.root_node = FileSystemNode(
name=path.name,
type=FileSystemNodeType.FILE,
size=path.stat().st_size,
Expand All @@ -63,12 +63,12 @@ def ingest_query(query: IngestionQuery) -> Tuple[str, str, str]:
path=path,
)

if not file_node.content:
raise ValueError(f"File {file_node.name} has no content")
if not query.root_node.content:
raise ValueError(f"File {query.root_node.name} has no content")

return format_node(file_node, query)
return format_node(query.root_node, query)

root_node = FileSystemNode(
query.root_node = FileSystemNode(
name=path.name,
type=FileSystemNodeType.DIRECTORY,
path_str=str(path.relative_to(query.local_path)),
Expand All @@ -78,12 +78,12 @@ def ingest_query(query: IngestionQuery) -> Tuple[str, str, str]:
stats = FileSystemStats()

_process_node(
node=root_node,
node=query.root_node,
query=query,
stats=stats,
)

return format_node(root_node, query)
return format_node(query.root_node, query)


def apply_gitingest_file(path: Path, query: IngestionQuery) -> None:
Expand Down
23 changes: 13 additions & 10 deletions src/gitingest/output_formatters.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Functions to ingest and analyze a codebase directory or single file."""

from typing import Optional, Tuple
from typing import List, Optional, Tuple

import tiktoken

Expand Down Expand Up @@ -35,8 +35,8 @@ def format_node(node: FileSystemNode, query: IngestionQuery) -> Tuple[str, str,
summary += f"File: {node.name}\n"
summary += f"Lines: {len(node.content.splitlines()):,}\n"

tree = "Directory structure:\n" + _create_tree_structure(query, node)
_create_tree_structure(query, node)
tree_patterns = [("Directory structure:\n", "")] + create_tree_structure(query, node)
tree = "".join([x[0] for x in tree_patterns])

content = _gather_file_contents(node)

Expand Down Expand Up @@ -108,7 +108,9 @@ def _gather_file_contents(node: FileSystemNode) -> str:
return "\n".join(_gather_file_contents(child) for child in node.children)


def _create_tree_structure(query: IngestionQuery, node: FileSystemNode, prefix: str = "", is_last: bool = True) -> str:
def create_tree_structure(
query: IngestionQuery, node: FileSystemNode, prefix: str = "", is_last: bool = True
) -> List[Tuple[str, str]]:
"""
Generate a tree-like string representation of the file structure.

Expand All @@ -128,14 +130,15 @@ def _create_tree_structure(query: IngestionQuery, node: FileSystemNode, prefix:

Returns
-------
str
A string representing the directory structure formatted as a tree.
List[Tuple[str, str]]
A list pairs of strings, the first a line representing the directory structure formatted as a tree
and the second is the corresponding filename with path.
"""
if not node.name:
# If no name is present, use the slug as the top-level directory name
node.name = query.slug

tree_str = ""
tree_items: List[Tuple[str, str]] = []
current_prefix = "└── " if is_last else "├── "

# Indicate directories with a trailing slash
Expand All @@ -145,13 +148,13 @@ def _create_tree_structure(query: IngestionQuery, node: FileSystemNode, prefix:
elif node.type == FileSystemNodeType.SYMLINK:
display_name += " -> " + node.path.readlink().name

tree_str += f"{prefix}{current_prefix}{display_name}\n"
tree_items.append((f"{prefix}{current_prefix}{display_name}\n", node.path_str))

if node.type == FileSystemNodeType.DIRECTORY and node.children:
prefix += " " if is_last else "│ "
for i, child in enumerate(node.children):
tree_str += _create_tree_structure(query, node=child, prefix=prefix, is_last=i == len(node.children) - 1)
return tree_str
tree_items += create_tree_structure(query, node=child, prefix=prefix, is_last=i == len(node.children) - 1)
return tree_items


def _format_token_count(text: str) -> Optional[str]:
Expand Down
2 changes: 2 additions & 0 deletions src/gitingest/schemas/ingestion_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from pydantic import BaseModel, ConfigDict, Field

from gitingest.config import MAX_FILE_SIZE
from gitingest.schemas.filesystem_schema import FileSystemNode


@dataclass
Expand Down Expand Up @@ -59,6 +60,7 @@ class IngestionQuery(BaseModel): # pylint: disable=too-many-instance-attributes
max_file_size: int = Field(default=MAX_FILE_SIZE)
ignore_patterns: Optional[Set[str]] = None
include_patterns: Optional[Set[str]] = None
root_node: Optional[FileSystemNode] = None

model_config = ConfigDict(arbitrary_types_allowed=True)

Expand Down
10 changes: 7 additions & 3 deletions src/server/query_processor.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
"""Process a query by parsing input, cloning a repository, and generating a summary."""

from functools import partial
from typing import Optional
from typing import Any, Dict, Optional

from fastapi import Request
from starlette.templating import _TemplateResponse

from gitingest.cloning import clone_repo
from gitingest.ingestion import ingest_query
from gitingest.output_formatters import create_tree_structure
from gitingest.query_parsing import IngestionQuery, parse_query
from server.server_config import EXAMPLE_REPOS, MAX_DISPLAY_SIZE, templates
from server.server_utils import Colors, log_slider_to_size
Expand Down Expand Up @@ -69,7 +70,7 @@ async def process_query(
template_response = partial(templates.TemplateResponse, name=template)
max_file_size = log_slider_to_size(slider_position)

context = {
context: Dict[str, Any] = {
"request": request,
"repo_url": input_text,
"examples": EXAMPLE_REPOS if is_index else [],
Expand All @@ -94,6 +95,9 @@ async def process_query(
clone_config = query.extract_clone_config()
await clone_repo(clone_config, token=token)
summary, tree, content = ingest_query(query)

tree_with_filenames = ((query.root_node is not None) and create_tree_structure(query, query.root_node)) or []

with open(f"{clone_config.local_path}.txt", "w", encoding="utf-8") as f:
f.write(tree + "\n" + content)
except Exception as exc:
Expand Down Expand Up @@ -129,7 +133,7 @@ async def process_query(
{
"result": True,
"summary": summary,
"tree": tree,
"tree": tree_with_filenames,
"content": content,
"ingest_id": query.id,
}
Expand Down
17 changes: 5 additions & 12 deletions src/server/templates/components/result.jinja
Original file line number Diff line number Diff line change
@@ -1,11 +1,5 @@
<script>
function getFileName(line) {
// Skips "|", "└", "├" found in file tree
const index = line.search(/[a-zA-Z0-9]/);
return line.substring(index).trim();
}

function toggleFile(element) {
function toggleFile(element, filePath) {
const patternInput = document.getElementById("pattern");
const patternFiles = patternInput.value ? patternInput.value.split(",").map(item => item.trim()) : [];

Expand All @@ -16,13 +10,12 @@
element.classList.toggle('line-through');
element.classList.toggle('text-gray-500');

const fileName = getFileName(element.textContent);
const fileIndex = patternFiles.indexOf(fileName);
const fileIndex = patternFiles.indexOf(filePath);

if (fileIndex !== -1) {
patternFiles.splice(fileIndex, 1);
} else {
patternFiles.push(fileName);
patternFiles.push(filePath);
}

patternInput.value = patternFiles.join(", ");
Expand Down Expand Up @@ -95,10 +88,10 @@
id="directory-structure-container"
readonly>
<input type="hidden" id="directory-structure-content" value="{{ tree }}" />
{% for line in tree.splitlines() %}
{% for line, filepath in tree %}
<pre name="tree-line"
class="cursor-pointer hover:line-through hover:text-gray-500"
onclick="toggleFile(this)">{{ line }}</pre>
onclick="toggleFile(this, '{{ filepath }}')">{{ line }}</pre>
{% endfor %}
</div>
</div>
Expand Down
Loading