[Feature] Incremental analysis caching, concurrent cronjob, and token-aware repo context

config_example.yaml

@@ -10,6 +10,12 @@ analyzer:
   exclude_api_analysis: false
   # Worker pool configuration
   max_workers: 0  # Maximum concurrent analyzer agents (0=auto-detect CPU count)
+  # Repository context (what gets injected into analyzer prompts)
+  respect_gitignore: true    # Honor the repo's .gitignore when building the analysis context
+  max_context_files: 1000    # Max files listed in the injected repo structure (0=unlimited)
+  # Incremental-analysis cache (skip re-analysis when the repository is unchanged)
+  cache_enabled: true        # Enable the incremental cache (.ai/docs/.manifest.json)
+  force_reanalysis: false    # Force re-analysis even on a cache hit (CLI: --force-reanalysis)
 
 # Generator configuration options
 generate:
@@ -47,3 +53,7 @@ cronjob:
     max_days_since_last_commit: 30
     # Path to clone projects for cronjob execution
     working_path: "/tmp/cronjob/projects"
+    # Maximum projects analyzed concurrently (0=auto-detect CPU count).
+    # Effective LLM concurrency is roughly this value * analyzer.max_workers,
+    # so tune both together to stay within provider rate limits.
+    max_project_workers: 4

src/agents/analyzer.py

@@ -13,7 +13,17 @@
 from pydantic_ai.settings import ModelSettings
 
 import config
-from utils import Logger, PromptManager, WorkerPool, create_retrying_client
+from utils import (
+    Logger,
+    PromptManager,
+    WorkerPool,
+    build_repo_context,
+    create_retrying_client,
+    is_up_to_date,
+    load_manifest,
+    write_manifest,
+)
+from utils.cache import manifest_path
 
 from .tools import FileReadTool, ListFilesTool
 
@@ -26,11 +36,28 @@ class AnalyzerAgentConfig(BaseModel):
     exclude_request_flow: bool = Field(default=False, description="Exclude request flow analysis")
     exclude_api_analysis: bool = Field(default=False, description="Exclude api analysis")
     max_workers: int = Field(default=0, description="Maximum concurrent workers (0=auto-detect CPU count)")
+    respect_gitignore: bool = Field(
+        default=True,
+        description="Respect the repository's .gitignore when building the analysis context",
+    )
+    max_context_files: int = Field(
+        default=1000,
+        description="Max files listed in the repo structure injected into prompts (0=unlimited)",
+    )
+    cache_enabled: bool = Field(
+        default=True,
+        description="Enable incremental-analysis caching (skip re-analysis when the repository is unchanged)",
+    )
+    force_reanalysis: bool = Field(
+        default=False,
+        description="Force re-analysis even when the incremental cache reports the repository is unchanged",
+    )
 
 
 class AnalyzerAgent:
     def __init__(self, cfg: AnalyzerAgentConfig) -> None:
         self._config = cfg
+        self._repo_context = None
 
         self._prompt_manager = PromptManager(file_path=Path(__file__).parent / "prompts" / "analyzer.yaml")
 
@@ -48,6 +75,21 @@ def __init__(self, cfg: AnalyzerAgentConfig) -> None:
     async def run(self):
         Logger.info("Starting analyzer agent")
 
+        self._repo_context = build_repo_context(
+            self._config.repo_path,
+            max_files=self._config.max_context_files,
+            respect_gitignore=self._config.respect_gitignore,
+        )
+        Logger.info(
+            "Repository context built",
+            data={
+                "file_count": self._repo_context.file_count,
+                "shown_file_count": self._repo_context.shown_file_count,
+                "truncated": self._repo_context.truncated,
+                "languages": self._repo_context.languages,
+            },
+        )
+
         analysis_files = []
         agent_tasks = {}  # Dict preserves insertion order in Python 3.7+
 
@@ -106,6 +148,18 @@ async def run(self):
                 file_path=file_path,
             )
 
+        docs_dir = self._config.repo_path / ".ai" / "docs"
+
+        if self._is_cache_valid(docs_dir, analysis_files):
+            Logger.info(
+                "Repository unchanged since last analysis; skipping (incremental cache hit)",
+                data={
+                    "manifest": str(manifest_path(docs_dir)),
+                    "fingerprint": self._repo_context.fingerprint,
+                },
+            )
+            return
+
         Logger.debug(f"Running {len(agent_tasks)} agents with worker pool")
 
         # Run all agents concurrently using worker pool
@@ -122,6 +176,35 @@ async def run(self):
                 Logger.info(f"Agent {agent_name} completed successfully")
 
         self.validate_succession(analysis_files)
+        self._update_cache(docs_dir, analysis_files)
+
+    def _is_cache_valid(self, docs_dir: Path, analysis_files: List[Path]) -> bool:
+        if not self._config.cache_enabled or self._config.force_reanalysis:
+            return False
+
+        manifest = load_manifest(docs_dir)
+        return is_up_to_date(
+            manifest,
+            repo_fingerprint=self._repo_context.fingerprint,
+            analyzer_version=config.VERSION,
+            expected_files=analysis_files,
+        )
+
+    def _update_cache(self, docs_dir: Path, analysis_files: List[Path]) -> None:
+        if not self._config.cache_enabled:
+            return
+
+        if not all(file.exists() for file in analysis_files):
+            Logger.info("Skipping cache manifest write (analysis incomplete)")
+            return
+
+        write_manifest(
+            docs_dir,
+            analyzer_version=config.VERSION,
+            repo_fingerprint=self._repo_context.fingerprint,
+            analysis_files=analysis_files,
+        )
+        Logger.info("Wrote incremental-analysis manifest", data={"path": str(manifest_path(docs_dir))})
 
     def validate_succession(self, analysis_files: List[Path]):
         missing_files = []
@@ -300,9 +383,17 @@ def _api_analyzer_agent(self) -> Agent:
         )
 
     def _render_prompt(self, prompt_name: str) -> str:
+        if self._repo_context is None:
+            self._repo_context = build_repo_context(
+                self._config.repo_path,
+                max_files=self._config.max_context_files,
+                respect_gitignore=self._config.respect_gitignore,
+            )
+
         template_vars = {
             "repo_path": str(self._config.repo_path),
-            "repo_structure": ListFilesTool()._run(str(self._config.repo_path)),
+            "repo_structure": self._repo_context.structure,
+            "detected_languages": ", ".join(self._repo_context.languages),
         }
 
         return self._prompt_manager.render_prompt(prompt_name, **template_vars)

src/agents/tools/dir_tool/list_files.py

@@ -8,215 +8,7 @@
 
 import config
 from utils import Logger
-
-DEFAULT_IGNORED_DIRS = [
-    # Version Control
-    ".git",  # Git repository metadata
-    ".svn",  # Subversion metadata
-    ".hg",  # Mercurial metadata
-    # Development Environment
-    ".venv",  # Python virtual environment
-    ".old-venv",  # Old Python virtual environment backup
-    "venv",  # Python virtual environment (common name)
-    "env",  # Python virtual environment (common name)
-    ".idea",  # JetBrains IDE configuration files
-    ".vscode",  # Visual Studio Code settings
-    ".eclipse",  # Eclipse IDE files
-    # Runtime/Build Artifacts
-    "__pycache__",  # Python bytecode cache files
-    "node_modules",  # Node.js dependencies
-    "target",  # Maven/Gradle build output (Java)
-    "build",  # Generic build output
-    "dist",  # Distribution files
-    "out",  # Output directory
-    ".next",  # Next.js build output
-    ".nuxt",  # Nuxt.js build output
-    ".output",  # Nitro output
-    # Language/Framework Specific
-    # Python
-    ".tox",  # Tox environments
-    ".nox",  # Nox environments
-    ".pytest_cache",  # Pytest cache
-    ".mypy_cache",  # MyPy cache
-    ".pyre",  # Pyre type checker
-    ".pytype",  # Pytype static analyzer
-    "site-packages",  # Python packages
-    ".eggs",  # Python eggs
-    "wheels",  # Python wheels directory
-    # Go
-    "vendor",  # Go vendor dependencies
-    ".mod",  # Go module cache
-    "go.work.sum",  # Go workspace sum
-    # Java/JVM
-    ".gradle",  # Gradle cache
-    ".m2",  # Maven local repository
-    ".metadata",  # Eclipse metadata
-    ".recommenders",  # Eclipse recommenders
-    "bin",  # Java compiled classes
-    "gen",  # Generated sources
-    # Node.js/JavaScript
-    ".npm",  # NPM cache
-    ".yarn",  # Yarn cache
-    "yarn-error.log",  # Yarn error logs
-    ".pnpm-store",  # PNPM store
-    ".turbo",  # Turborepo cache
-    ".rush",  # Rush.js cache
-    "lerna-debug.log*",  # Lerna debug logs
-    ".eslintcache",  # ESLint cache
-    ".parcel-cache",  # Parcel cache
-    ".cache",  # General cache
-    "coverage",  # Coverage reports
-    # PHP
-    ".phpunit.result.cache",  # PHPUnit cache
-    "composer.phar",  # Composer executable
-    ".phplint-cache",  # PHP Lint cache
-    # Framework Specific
-    "bower_components",  # Bower components
-    ".bundle",  # Ruby bundle
-    "Pods",  # iOS CocoaPods
-    "DerivedData",  # Xcode derived data
-    ".cargo",  # Rust cargo
-    ".stack-work",  # Haskell Stack
-    "elm-stuff",  # Elm packages
-    "_site",  # Jekyll/Static site generators
-    # Infrastructure/Deployment
-    "k8s",  # Kubernetes configuration files
-    ".terraform",  # Terraform state
-    ".docker",  # Docker build context
-    # Documentation/Generated
-    "docs/_build",  # Sphinx documentation build
-    "site",  # MkDocs site
-    # Logging/Output
-    "logs",  # Application log files
-    ".logs",  # Hidden log directory
-    "log",  # Log directory
-    # Static Assets
-    "assets",  # Static files (images, fonts, etc.)
-    "public",  # Public web assets (when not source)
-    "static",  # Static files
-    # Testing
-    ".coverage",  # Coverage data
-    "htmlcov",  # Coverage HTML reports
-    ".nyc_output",  # NYC coverage output
-    "jest-coverage",  # Jest coverage
-    # OS/System
-    ".DS_Store",  # macOS metadata
-    "Thumbs.db",  # Windows thumbnails
-]
-
-DEFAULT_IGNORED_EXTENSIONS = [
-    # Compiled/Binary Files
-    ".pyc",  # Python bytecode
-    ".pyo",  # Python optimized bytecode
-    ".pyd",  # Python extension module (Windows)
-    ".class",  # Java bytecode
-    ".o",  # Object files
-    ".so",  # Shared libraries (Linux)
-    ".dll",  # Dynamic libraries (Windows)
-    ".dylib",  # Dynamic libraries (macOS)
-    ".exe",  # Executable files
-    ".bin",  # Binary files
-    ".a",  # Static libraries
-    ".lib",  # Library files (Windows)
-    # Other Languages
-    ".beam",  # Erlang/Elixir compiled
-    ".hi",  # Haskell interface files
-    ".cmi",  # OCaml compiled interface
-    ".cmo",  # OCaml compiled object
-    ".cmx",  # OCaml optimized compiled
-    ".rlib",  # Rust library
-    ".pdb",  # Program database (debugging)
-    # Java/JVM Archives
-    ".jar",  # Java archive
-    ".war",  # Web application archive
-    ".ear",  # Enterprise application archive
-    ".aar",  # Android archive
-    # .NET
-    ".mdb",  # Mono debug database
-    # Compressed Archives
-    ".zip",  # ZIP archive
-    ".tar",  # Tar archive
-    ".tar.gz",  # Compressed tar archive
-    ".tgz",  # Compressed tar archive (short)
-    ".tar.bz2",  # Bzip2 compressed tar
-    ".tbz2",  # Bzip2 compressed tar (short)
-    ".tar.xz",  # XZ compressed tar
-    ".rar",  # RAR archive
-    ".7z",  # 7-Zip archive
-    ".gz",  # Gzip compressed
-    ".bz2",  # Bzip2 compressed
-    ".xz",  # XZ compressed
-    # Package Manager Files
-    ".whl",  # Python wheel
-    ".egg",  # Python egg (deprecated)
-    ".phar",  # PHP Archive
-    ".deb",  # Debian package
-    ".rpm",  # RPM package
-    ".msi",  # Windows installer
-    ".dmg",  # macOS disk image
-    ".pkg",  # Package files
-    ".gem",  # Ruby gem
-    ".nupkg",  # NuGet package
-    # Runtime/Cache Files
-    ".log",  # Log files
-    ".tmp",  # Temporary files
-    ".temp",  # Temporary files
-    ".swp",  # Vim swap files
-    ".swo",  # Vim swap files
-    "~",  # Backup files
-    ".bak",  # Backup files
-    ".orig",  # Original files
-    ".cache",  # Cache files
-    ".pid",  # Process ID files
-    # Database Files
-    ".dat",  # Data files
-    ".db",  # Database files
-    ".sqlite",  # SQLite database
-    ".sqlite3",  # SQLite 3 database
-    ".accdb",  # Microsoft Access database (newer)
-    # Configuration/Environment
-    ".env",  # Environment variable files
-    ".env.local",  # Local environment variables
-    ".env.production",  # Production environment variables
-    ".env.development",  # Development environment variables
-    # IDE/Editor Files
-    ".iml",  # IntelliJ module files
-    ".ipr",  # IntelliJ project files
-    ".iws",  # IntelliJ workspace files
-    ".sublime-project",  # Sublime Text project
-    ".sublime-workspace",  # Sublime Text workspace
-    ".vscode",  # VS Code settings (file)
-    # OS/System Files
-    ".DS_Store",  # macOS metadata
-    "Thumbs.db",  # Windows thumbnails
-    "desktop.ini",  # Windows desktop settings
-    ".localized",  # macOS localization
-    # Media Files (often not needed for code analysis)
-    ".jpg",  # JPEG image
-    ".jpeg",  # JPEG image
-    ".png",  # PNG image
-    ".gif",  # GIF image
-    ".bmp",  # Bitmap image
-    ".svg",  # SVG image (keeping minimal, might be needed)
-    ".ico",  # Icon files
-    ".mp3",  # Audio files
-    ".mp4",  # Video files
-    ".avi",  # Video files
-    ".mov",  # Video files
-    ".pdf",  # PDF files
-    ".doc",  # Word documents
-    ".docx",  # Word documents
-    ".xls",  # Excel files
-    ".xlsx",  # Excel files
-    ".ppt",  # PowerPoint files
-    ".pptx",  # PowerPoint files
-    # Font Files
-    ".ttf",  # TrueType fonts
-    ".otf",  # OpenType fonts
-    ".woff",  # Web fonts
-    ".woff2",  # Web fonts
-    ".eot",  # Embedded fonts
-]
+from utils.ignore_patterns import DEFAULT_IGNORED_DIRS, DEFAULT_IGNORED_EXTENSIONS
 
 
 class ListFilesTool:

src/handlers/analyze.py

@@ -30,6 +30,10 @@ async def handle(self):
                     "exclude_dependencies": self.config.exclude_dependencies,
                     "exclude_request_flow": self.config.exclude_request_flow,
                     "max_workers": self.config.max_workers,
+                    "respect_gitignore": self.config.respect_gitignore,
+                    "max_context_files": self.config.max_context_files,
+                    "cache_enabled": self.config.cache_enabled,
+                    "force_reanalysis": self.config.force_reanalysis,
                     "input": str(self.config.repo_path),
                 }
             )