Merge pull request #80 from FSoft-AI4Code/feat/gitignore

automatically apply gitignore rules

Author: anhnh2002

.gitignore

@@ -38,7 +38,8 @@ MANIFEST
 htmlcov/
 .tox/
 .hypothesis/
-tests/
+tests/*
+!tests/test_gitignore_filtering.py
 
 # Jupyter
 *.ipynb

README.md

@@ -221,6 +221,9 @@ codewiki generate --focus "src/core,src/api" --doc-type architecture
 
 # Add custom instructions for the AI agent
 codewiki generate --instructions "Focus on public APIs and include usage examples"
+
+# Analyze files ignored by Git (Git ignore filtering is enabled by default)
+codewiki generate --no-gitignore
 ```
 
 #### Pattern Behavior (Important!)
@@ -238,6 +241,12 @@ codewiki generate --instructions "Focus on public APIs and include usage example
     - Glob patterns: `*.test.js`, `*_test.py`, `*.min.*`
     - Directory patterns: `build/`, `dist/`, `coverage/`
 
+- **`--use-gitignore/--no-gitignore`**: Git ignore rules are applied by default
+  - Root and nested `.gitignore` files are respected before call-graph analysis
+  - Tracked files remain included, matching Git behavior
+  - Built-in and explicit `--exclude` patterns still apply when Git includes a path
+  - Use `codewiki config set --no-gitignore` to persistently disable this behavior
+
 #### Setting Persistent Defaults
 
 Save your preferred settings as defaults:
@@ -266,6 +275,7 @@ codewiki config agent --clear
 |--------|-------------|----------|---------|
 | `--include` | File patterns to include | **Replaces** defaults | `*.cs`, `*.py`, `src/**/*.ts` |
 | `--exclude` | Patterns to exclude | **Merges** with defaults | `Tests,Specs`, `*.test.js`, `build/` |
+| `--use-gitignore/--no-gitignore` | Apply Git ignore rules | Enabled by default | `--no-gitignore` |
 | `--focus` | Modules to document in detail | Standalone option | `src/core,src/api` |
 | `--doc-type` | Documentation style | Standalone option | `api`, `architecture`, `user-guide`, `developer` |
 | `--instructions` | Custom agent instructions | Standalone option | Free-form text |

codewiki/cli/adapters/doc_generator.py

@@ -146,7 +146,8 @@ def generate(self) -> DocumentationJob:
                 max_token_per_module=self.config.get('max_token_per_module', 36369),
                 max_token_per_leaf_module=self.config.get('max_token_per_leaf_module', 16000),
                 max_depth=self.config.get('max_depth', 2),
-                agent_instructions=self.config.get('agent_instructions')
+                agent_instructions=self.config.get('agent_instructions'),
+                use_gitignore=self.config.get('use_gitignore', True),
             )
 
             # Run backend documentation generation

codewiki/cli/commands/config.py

@@ -111,6 +111,11 @@ def config_group():
     type=str,
     help="Azure OpenAI deployment name"
 )
+@click.option(
+    "--use-gitignore/--no-gitignore",
+    default=None,
+    help="Apply Git ignore rules during repository analysis (default: enabled)",
+)
 def config_set(
     api_key: Optional[str],
     base_url: Optional[str],
@@ -124,7 +129,8 @@ def config_set(
     provider: Optional[str] = None,
     aws_region: Optional[str] = None,
     api_version: Optional[str] = None,
-    azure_deployment: Optional[str] = None
+    azure_deployment: Optional[str] = None,
+    use_gitignore: Optional[bool] = None,
 ):
     """
     Set configuration values for CodeWiki.
@@ -170,10 +176,14 @@ def config_set(
     \b
     # Set max depth for hierarchical decomposition
     $ codewiki config set --max-depth 3
+
+    \b
+    # Persistently disable Git ignore filtering
+    $ codewiki config set --no-gitignore
     """
     try:
         # Check if at least one option is provided
-        if not any([api_key, base_url, main_model, cluster_model, fallback_model, max_tokens, max_token_per_module, max_token_per_leaf_module, max_depth, provider, aws_region, api_version, azure_deployment]):
+        if not any([api_key, base_url, main_model, cluster_model, fallback_model, max_tokens, max_token_per_module, max_token_per_leaf_module, max_depth, provider, aws_region, api_version, azure_deployment, use_gitignore is not None]):
             click.echo("No options provided. Use --help for usage information.")
             sys.exit(EXIT_CONFIG_ERROR)
 
@@ -237,6 +247,9 @@ def config_set(
         if azure_deployment is not None:
             validated_data['azure_deployment'] = azure_deployment
 
+        if use_gitignore is not None:
+            validated_data['use_gitignore'] = use_gitignore
+
         # Create config manager and save
         manager = ConfigManager()
         manager.load()  # Load existing config if present
@@ -254,7 +267,8 @@ def config_set(
             provider=validated_data.get('provider'),
             aws_region=validated_data.get('aws_region'),
             api_version=validated_data.get('api_version'),
-            azure_deployment=validated_data.get('azure_deployment')
+            azure_deployment=validated_data.get('azure_deployment'),
+            use_gitignore=validated_data.get('use_gitignore'),
         )
 
         # Display success messages
@@ -315,6 +329,9 @@ def config_set(
         if azure_deployment:
             click.secho(f"✓ Azure Deployment: {azure_deployment}", fg="green")
 
+        if use_gitignore is not None:
+            click.secho(f"✓ Use gitignore: {use_gitignore}", fg="green")
+
         click.echo("\n" + click.style("Configuration updated successfully.", fg="green", bold=True))
 
     except ConfigurationError as e:
@@ -375,6 +392,7 @@ def config_show(output_json: bool):
                 "max_token_per_module": config.max_token_per_module if config else 36369,
                 "max_token_per_leaf_module": config.max_token_per_leaf_module if config else 16000,
                 "max_depth": config.max_depth if config else 2,
+                "use_gitignore": config.use_gitignore if config else True,
                 "agent_instructions": config.agent_instructions.to_dict() if config and config.agent_instructions else {},
                 "config_file": str(manager.config_file_path)
             }
@@ -435,6 +453,7 @@ def config_show(output_json: bool):
             click.secho("Decomposition Settings", fg="cyan", bold=True)
             if config:
                 click.echo(f"  Max Depth:               {config.max_depth}")
+                click.echo(f"  Use Gitignore:           {config.use_gitignore}")
 
             click.echo()
             click.secho("Agent Instructions", fg="cyan", bold=True)
@@ -859,4 +878,3 @@ def config_agent(
         sys.exit(e.exit_code)
     except Exception as e:
         sys.exit(handle_error(e))
-

codewiki/cli/commands/generate.py

@@ -266,6 +266,11 @@ def _find_affected(tree, parent_names=None):
     default=None,
     help="Custom instructions for the documentation agent",
 )
+@click.option(
+    "--use-gitignore/--no-gitignore",
+    default=None,
+    help="Apply Git ignore rules during analysis (default: enabled)",
+)
 @click.option(
     "--verbose",
     "-v",
@@ -319,6 +324,7 @@ def generate_command(
     focus: Optional[str],
     doc_type: Optional[str],
     instructions: Optional[str],
+    use_gitignore: Optional[bool],
     verbose: bool,
     max_tokens: Optional[int],
     max_token_per_module: Optional[int],
@@ -346,6 +352,10 @@ def generate_command(
     \b
     # Force full regeneration
     $ codewiki generate --no-cache
+
+    \b
+    # Analyze ignored files as well
+    $ codewiki generate --no-gitignore
     
     \b
     # C# project: only .cs files, exclude tests
@@ -521,10 +531,12 @@ def generate_command(
             effective_max_token_per_module = max_token_per_module if max_token_per_module is not None else config.max_token_per_module
             effective_max_token_per_leaf = max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module
             effective_max_depth = max_depth if max_depth is not None else config.max_depth
+            effective_use_gitignore = use_gitignore if use_gitignore is not None else config.use_gitignore
             logger.debug(f"Max tokens: {effective_max_tokens}")
             logger.debug(f"Max token/module: {effective_max_token_per_module}")
             logger.debug(f"Max token/leaf module: {effective_max_token_per_leaf}")
             logger.debug(f"Max depth: {effective_max_depth}")
+            logger.debug(f"Use gitignore: {effective_use_gitignore}")
 
         # Get agent instructions (merge runtime with persistent)
         agent_instructions_dict = None
@@ -562,6 +574,8 @@ def generate_command(
                 'max_token_per_leaf_module': max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module,
                 # Max depth setting (runtime override takes precedence)
                 'max_depth': max_depth if max_depth is not None else config.max_depth,
+                # Gitignore setting (runtime override takes precedence)
+                'use_gitignore': use_gitignore if use_gitignore is not None else config.use_gitignore,
             },
             verbose=verbose,
             generate_html=github_pages,
@@ -629,4 +643,3 @@ def generate_command(
         sys.exit(130)
     except Exception as e:
         sys.exit(handle_error(e, verbose=verbose))
-

codewiki/cli/config_manager.py

@@ -135,7 +135,8 @@ def save(
         provider: Optional[str] = None,
         aws_region: Optional[str] = None,
         api_version: Optional[str] = None,
-        azure_deployment: Optional[str] = None
+        azure_deployment: Optional[str] = None,
+        use_gitignore: Optional[bool] = None,
     ):
         """
         Save configuration to file and keyring.
@@ -155,6 +156,7 @@ def save(
             aws_region: AWS region for Bedrock provider
             api_version: Azure OpenAI API version
             azure_deployment: Azure OpenAI deployment name
+            use_gitignore: Apply Git ignore rules during repository analysis
         """
         # Ensure config directory exists
         try:
@@ -204,6 +206,8 @@ def save(
             self._config.api_version = api_version
         if azure_deployment is not None:
             self._config.azure_deployment = azure_deployment
+        if use_gitignore is not None:
+            self._config.use_gitignore = use_gitignore
 
         # Validate configuration whenever the minimum required fields are set.
         # Caw providers only need main_model; API providers need base_url +
@@ -330,4 +334,3 @@ def keyring_available(self) -> bool:
     def config_file_path(self) -> Path:
         """Get configuration file path."""
         return CONFIG_FILE
-

codewiki/cli/models/config.py

@@ -121,6 +121,7 @@ class Configuration:
         max_token_per_module: Maximum tokens per module for clustering (default: 36369)
         max_token_per_leaf_module: Maximum tokens per leaf module (default: 16000)
         max_depth: Maximum depth for hierarchical decomposition (default: 2)
+        use_gitignore: Apply Git ignore rules during repository analysis
         agent_instructions: Custom agent instructions for documentation generation
     """
     base_url: str
@@ -136,6 +137,7 @@ class Configuration:
     max_token_per_module: int = 36369
     max_token_per_leaf_module: int = 16000
     max_depth: int = 2
+    use_gitignore: bool = True
     agent_instructions: AgentInstructions = field(default_factory=AgentInstructions)
 
     def validate(self):
@@ -172,6 +174,7 @@ def to_dict(self) -> dict:
             'max_token_per_module': self.max_token_per_module,
             'max_token_per_leaf_module': self.max_token_per_leaf_module,
             'max_depth': self.max_depth,
+            'use_gitignore': self.use_gitignore,
             'fallback_model': self.fallback_model,
         }
         if self.agent_instructions and not self.agent_instructions.is_empty():
@@ -207,6 +210,7 @@ def from_dict(cls, data: dict) -> 'Configuration':
             max_token_per_module=data.get('max_token_per_module', 36369),
             max_token_per_leaf_module=data.get('max_token_per_leaf_module', 16000),
             max_depth=data.get('max_depth', 2),
+            use_gitignore=data.get('use_gitignore', True),
             agent_instructions=agent_instructions,
         )
 
@@ -273,6 +277,6 @@ def to_backend_config(self, repo_path: str, output_dir: str, api_key: str, runti
             max_token_per_module=self.max_token_per_module,
             max_token_per_leaf_module=self.max_token_per_leaf_module,
             max_depth=self.max_depth,
-            agent_instructions=final_instructions.to_dict() if final_instructions else None
+            agent_instructions=final_instructions.to_dict() if final_instructions else None,
+            use_gitignore=self.use_gitignore,
         )
-

codewiki/mcp/server.py

@@ -102,6 +102,11 @@ def _fine_grained_tools() -> list[Tool]:
                         "type": "string",
                         "description": "Comma-separated patterns to exclude (e.g., '*test*,*spec*')",
                     },
+                    "use_gitignore": {
+                        "type": "boolean",
+                        "description": "Apply Git ignore rules before analysis (default: true)",
+                        "default": True,
+                    },
                 },
                 "required": ["repo_path"],
             },
@@ -320,6 +325,11 @@ def _legacy_tools() -> list[Tool]:
                         "type": "string",
                         "description": "Comma-separated patterns to exclude",
                     },
+                    "use_gitignore": {
+                        "type": "boolean",
+                        "description": "Apply Git ignore rules before analysis (default: true)",
+                        "default": True,
+                    },
                 },
                 "required": ["repo_path"],
             },
@@ -489,6 +499,7 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]:
         aws_region=getattr(config, "aws_region", "us-east-1"),
         max_tokens=config.max_tokens,
         agent_instructions=agent_instructions or None,
+        use_gitignore=arguments.get("use_gitignore", True),
     )
 
     from codewiki.cli.utils.repo_validator import get_git_commit_hash

codewiki/mcp/tools/analysis.py

@@ -304,6 +304,7 @@ def handle_analyze_repo(
         llm_api_key="not-needed",
         main_model="unused",
         cluster_model="unused",
+        use_gitignore=arguments.get("use_gitignore", True),
     )
 
     # Apply optional include/exclude patterns