refactor: Refactor output formatting

.pre-commit-config.yaml

@@ -84,6 +84,7 @@ repos:
     rev: v1.36.4
     hooks:
       - id: djlint-reformat-jinja
+        exclude: ^src/gitingest/format/
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.45.0

src/gitingest/entrypoint.py

@@ -15,6 +15,7 @@
 from gitingest.clone import clone_repo
 from gitingest.config import MAX_FILE_SIZE
 from gitingest.ingestion import ingest_query
+from gitingest.output_formatter import DefaultFormatter
 from gitingest.query_parser import parse_local_dir_path, parse_remote_repo
 from gitingest.utils.auth import resolve_token
 from gitingest.utils.compat_func import removesuffix
@@ -44,12 +45,13 @@ async def ingest_async(
     include_submodules: bool = False,
     token: str | None = None,
     output: str | None = None,
-) -> tuple[str, str, str]:
+) -> str:
     """Ingest a source and process its contents.
 
     This function analyzes a source (URL or local path), clones the corresponding repository (if applicable),
-    and processes its files according to the specified query parameters. It returns a summary, a tree-like
-    structure of the files, and the content of the files. The results can optionally be written to an output file.
+    and processes its files according to the specified query parameters. It returns a single digest string.
+
+    The output is generated lazily using a ContextV1 object and the DefaultFormatter class.
 
     Parameters
     ----------
@@ -79,11 +81,8 @@ async def ingest_async(
 
     Returns
     -------
-    tuple[str, str, str]
-        A tuple containing:
-        - A summary string of the analyzed repository or directory.
-        - A tree-like string representation of the file structure.
-        - The content of the files in the repository or directory.
+    str
+        The full digest string.
 
     """
     logger.info("Starting ingestion process", extra={"source": source})
@@ -138,14 +137,15 @@ async def ingest_async(
             _apply_gitignores(query)
 
         logger.info("Processing files and generating output")
-        summary, tree, content = ingest_query(query)
 
         if output:
             logger.debug("Writing output to file", extra={"output_path": output})
-        await _write_output(tree, content=content, target=output)
-
+        context = ingest_query(query)
+        formatter = DefaultFormatter()
+        digest = formatter.format(context, context.query)
+        await _write_output(digest, content=None, target=output)
         logger.info("Ingestion completed successfully")
-        return summary, tree, content
+        return digest
 
 
 def ingest(
@@ -160,12 +160,13 @@ def ingest(
     include_submodules: bool = False,
     token: str | None = None,
     output: str | None = None,
-) -> tuple[str, str, str]:
+) -> str:
     """Provide a synchronous wrapper around ``ingest_async``.
 
     This function analyzes a source (URL or local path), clones the corresponding repository (if applicable),
-    and processes its files according to the specified query parameters. It returns a summary, a tree-like
-    structure of the files, and the content of the files. The results can optionally be written to an output file.
+    and processes its files according to the specified query parameters. It returns a single digest string.
+
+    The output is generated lazily using a ContextV1 object and the DefaultFormatter class.
 
     Parameters
     ----------
@@ -195,11 +196,8 @@ def ingest(
 
     Returns
     -------
-    tuple[str, str, str]
-        A tuple containing:
-        - A summary string of the analyzed repository or directory.
-        - A tree-like string representation of the file structure.
-        - The content of the files in the repository or directory.
+    str
+        The full digest string.
 
     See Also
     --------
@@ -208,7 +206,7 @@ def ingest(
     """
     return asyncio.run(
         ingest_async(
-            source=source,
+            source,
             max_file_size=max_file_size,
             include_patterns=include_patterns,
             exclude_patterns=exclude_patterns,

src/gitingest/format/DebugFormatter/Source.j2

@@ -0,0 +1,4 @@
+{{ SEPARATOR }}
+DEBUG: {{ class_name }}
+Fields: {{ fields_str }}
+{{ SEPARATOR }}

src/gitingest/format/DefaultFormatter/ContextV1.j2

@@ -0,0 +1,11 @@
+# Generated using https://gitingest.com/{{ source.query.user_name }}/{{ source.query.repo_name }}{{ source.query.subpath }}
+
+Sources used:
+{%- for src in source %}
+- {{ src.name }}: {{ src.__class__.__name__ }}
+{% endfor %}
+
+{%- for src in source.sources %}
+{{ formatter.format(src, source.query) }}
+{%- endfor %}
+# End of https://gitingest.com/{{ source.query.user_name }}/{{ source.query.repo_name }}{{ source.query.subpath }}

src/gitingest/format/DefaultFormatter/FileSystemDirectory.j2

@@ -0,0 +1,7 @@
+{%- if source.depth == 0 %}{{ source.name }}:
+{{ source.tree }}
+
+{% endif -%}
+{%- for child in source.children -%}
+{{ formatter.format(child, query) }}
+{%- endfor -%}

src/gitingest/format/DefaultFormatter/FileSystemFile.j2

@@ -0,0 +1,4 @@
+{{ SEPARATOR }}
+{{ source.name }}
+{{ SEPARATOR }}
+{{ source.content }}

src/gitingest/format/DefaultFormatter/FileSystemSymlink.j2

@@ -0,0 +1,3 @@
+{{ SEPARATOR }}
+{{ source.name }}{% if source.target %} -> {{ source.target }}{% endif %}
+{{ SEPARATOR }}

src/gitingest/format/DefaultFormatter/GitRepository.j2

@@ -0,0 +1,7 @@
+{%- if source.depth == 0 %}🔗 Git Repository: {{ source.name }}
+{{ source.tree }}
+
+{% endif -%}
+{%- for child in source.children -%}
+{{ formatter.format(child, query) }}
+{%- endfor -%}

src/gitingest/format/SummaryFormatter/ContextV1.j2

@@ -0,0 +1,5 @@
+Repository: {{ source.query.user_name }}/{{ source.query.repo_name }}
+Commit: {{ source.query.commit }}
+Files analyzed: {{ source.file_count }}
+
+Estimated tokens: {{ source.token_count }}

src/gitingest/format/SummaryFormatter/FileSystemDirectory.j2

@@ -0,0 +1,2 @@
+Directory structure:
+{{ source.tree }}

src/gitingest/ingestion.py

@@ -6,8 +6,8 @@
 from typing import TYPE_CHECKING
 
 from gitingest.config import MAX_DIRECTORY_DEPTH, MAX_FILES, MAX_TOTAL_SIZE_BYTES
-from gitingest.output_formatter import format_node
-from gitingest.schemas import FileSystemNode, FileSystemNodeType, FileSystemStats
+from gitingest.schemas import ContextV1, FileSystemNode, FileSystemStats
+from gitingest.schemas.filesystem import FileSystemDirectory, FileSystemFile, FileSystemSymlink, GitRepository
 from gitingest.utils.ingestion_utils import _should_exclude, _should_include
 from gitingest.utils.logging_config import get_logger
 
@@ -18,12 +18,18 @@
 logger = get_logger(__name__)
 
 
-def ingest_query(query: IngestionQuery) -> tuple[str, str, str]:
+def _is_git_repository(path: Path) -> bool:
+    """Check if a directory contains a .git folder."""
+    return (path / ".git").exists()
+
+
+def ingest_query(query: IngestionQuery) -> ContextV1:
     """Run the ingestion process for a parsed query.
 
-    This is the main entry point for analyzing a codebase directory or single file. It processes the query
-    parameters, reads the file or directory content, and generates a summary, directory structure, and file content,
-    along with token estimations.
+    This is the main entry point for analyzing a codebase directory or single file.
+
+    It processes the query parameters, reads the file or directory content, and returns
+    a ContextV1 object that can generate the final output digest on demand.
 
     Parameters
     ----------
@@ -32,8 +38,10 @@ def ingest_query(query: IngestionQuery) -> tuple[str, str, str]:
 
     Returns
     -------
-    tuple[str, str, str]
-        A tuple containing the summary, directory structure, and file contents.
+    ContextV1
+        A ContextV1 object representing the ingested file system nodes.
+        Use str(DefaultFormatter(context)) to get the summary, directory structure,
+        and file contents.
 
     Raises
     ------
@@ -70,11 +78,8 @@ def ingest_query(query: IngestionQuery) -> tuple[str, str, str]:
 
         relative_path = path.relative_to(query.local_path)
 
-        file_node = FileSystemNode(
+        file_node = FileSystemFile(
             name=path.name,
-            type=FileSystemNodeType.FILE,
-            size=path.stat().st_size,
-            file_count=1,
             path_str=str(relative_path),
             path=path,
         )
@@ -91,16 +96,21 @@ def ingest_query(query: IngestionQuery) -> tuple[str, str, str]:
                 "file_size": file_node.size,
             },
         )
-        return format_node(file_node, query=query)
-
-    logger.info("Processing directory", extra={"directory_path": str(path)})
+        return ContextV1(sources=[file_node], query=query)
 
-    root_node = FileSystemNode(
-        name=path.name,
-        type=FileSystemNodeType.DIRECTORY,
-        path_str=str(path.relative_to(query.local_path)),
-        path=path,
-    )
+    # Check if this is a git repository and create appropriate node type
+    if _is_git_repository(path):
+        root_node = GitRepository(
+            name=path.name,
+            path_str=str(path.relative_to(query.local_path)),
+            path=path,
+        )
+    else:
+        root_node = FileSystemDirectory(
+            name=path.name,
+            path_str=str(path.relative_to(query.local_path)),
+            path=path,
+        )
 
     stats = FileSystemStats()
 
@@ -117,10 +127,10 @@ def ingest_query(query: IngestionQuery) -> tuple[str, str, str]:
         },
     )
 
-    return format_node(root_node, query=query)
+    return ContextV1(sources=[root_node], query=query)
 
 
-def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystemStats) -> None:
+def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystemStats) -> None:  # noqa: C901
     """Process a file or directory item within a directory.
 
     This function handles each file or directory item, checking if it should be included or excluded based on the
@@ -161,13 +171,21 @@ def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystem
                 continue
             _process_file(path=sub_path, parent_node=node, stats=stats, local_path=query.local_path)
         elif sub_path.is_dir():
-            child_directory_node = FileSystemNode(
-                name=sub_path.name,
-                type=FileSystemNodeType.DIRECTORY,
-                path_str=str(sub_path.relative_to(query.local_path)),
-                path=sub_path,
-                depth=node.depth + 1,
-            )
+            # Check if this subdirectory is a git repository
+            if _is_git_repository(sub_path):
+                child_directory_node = GitRepository(
+                    name=sub_path.name,
+                    path_str=str(sub_path.relative_to(query.local_path)),
+                    path=sub_path,
+                    depth=node.depth + 1,
+                )
+            else:
+                child_directory_node = FileSystemDirectory(
+                    name=sub_path.name,
+                    path_str=str(sub_path.relative_to(query.local_path)),
+                    path=sub_path,
+                    depth=node.depth + 1,
+                )
 
             _process_node(node=child_directory_node, query=query, stats=stats)
 
@@ -201,9 +219,8 @@ def _process_symlink(path: Path, parent_node: FileSystemNode, stats: FileSystemS
         The base path of the repository or directory being processed.
 
     """
-    child = FileSystemNode(
+    child = FileSystemSymlink(
         name=path.name,
-        type=FileSystemNodeType.SYMLINK,
         path_str=str(path.relative_to(local_path)),
         path=path,
         depth=parent_node.depth + 1,
@@ -213,7 +230,7 @@ def _process_symlink(path: Path, parent_node: FileSystemNode, stats: FileSystemS
     parent_node.file_count += 1
 
 
-def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStats, local_path: Path) -> None:
+def _process_file(path: Path, parent_node: FileSystemDirectory, stats: FileSystemStats, local_path: Path) -> None:
     """Process a file in the file system.
 
     This function checks the file's size, increments the statistics, and reads its content.
@@ -223,7 +240,7 @@ def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStat
     ----------
     path : Path
         The full path of the file.
-    parent_node : FileSystemNode
+    parent_node : FileSystemDirectory
         The dictionary to accumulate the results.
     stats : FileSystemStats
         Statistics tracking object for the total file count and size.
@@ -258,11 +275,8 @@ def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStat
     stats.total_files += 1
     stats.total_size += file_size
 
-    child = FileSystemNode(
+    child = FileSystemFile(
         name=path.name,
-        type=FileSystemNodeType.FILE,
-        size=file_size,
-        file_count=1,
         path_str=str(path.relative_to(local_path)),
         path=path,
         depth=parent_node.depth + 1,