docs: add feature sections

Document redaction, RAG, skeleton mode, few-shot examples, and CI usage. Add --redact/--no-redact flag to match documentation.

Made-with: Cursor

Author: Sayeem3051

README.md

@@ -199,6 +199,58 @@ patterns = parse_ctxengignore(Path("."))
 # → list of pattern strings, or [] if no file
 ```
 
+### Secrets & PII Redaction
+
+ctxeng automatically redacts common secrets and PII from file contents before sending to any LLM:
+
+- API keys and tokens
+- Passwords and credentials
+- Email addresses
+- Private keys
+
+Redaction happens before token counting, tracing, and output — your secrets never leave your machine.
+
+To disable:
+
+```bash
+ctxeng build "Your query" --no-redact
+```
+
+### RAG (Intelligent Chunk Retrieval)
+
+For large repositories, `--rag` switches from whole-file inclusion to **chunk-level retrieval**. ctxeng splits top-ranked files into overlapping chunks, then selects the most relevant chunks for your query:
+
+- Uses **embeddings** when `sentence-transformers` is installed
+- Falls back to **lexical retrieval** when embeddings aren’t available
+
+```bash
+ctxeng build "Explain the login flow" --rag
+```
+
+### AST Skeleton (Python)
+
+`--skeleton` replaces Python file bodies with an AST-derived outline (imports, classes, methods, function signatures). This is useful when you want a high-level overview within a tight token budget:
+
+```bash
+ctxeng build "Give me a high-level architecture overview" --skeleton
+```
+
+### Few-shot Examples
+
+You can inject a small “style guide” / best-practice library into the context with `--fewshot`. Put markdown/text files under `.ctxeng/examples/` and enable:
+
+```bash
+ctxeng build "Refactor this module" --fewshot
+```
+
+### CI subcommand
+
+Use `ctxeng ci` for pipeline-friendly context generation. It always writes to a file and is designed for non-interactive runs:
+
+```bash
+ctxeng ci "Generate release notes" --output context.md --fmt markdown --trace
+```
+
 ### Import graph (Python)
 
 After files are scored, **ctxeng** parses static ``import`` / ``from … import`` statements in each discovered ``.py`` file, resolves **relative imports** from the file’s location, and can **pull in imported modules** from the same collection set before the token budget is applied.

ctxeng/builder.py

@@ -56,6 +56,7 @@ def __init__(self, root: str | Path = ".") -> None:
         self._rag_chunk_overlap = 20
         self._rag_embedding_model = "all-MiniLM-L6-v2"
         self._skeleton = False
+        self._redact = True
         self._fewshot = False
         self._fewshot_dir: str | Path = ".ctxeng/examples"
         self._fewshot_max_files = 5
@@ -167,6 +168,11 @@ def skeleton(self, enabled: bool = True) -> ContextBuilder:
         self._skeleton = enabled
         return self
 
+    def redact(self, enabled: bool = True) -> ContextBuilder:
+        """Enable/disable secrets & PII redaction before output."""
+        self._redact = enabled
+        return self
+
     def fewshot(
         self,
         enabled: bool = True,
@@ -225,6 +231,7 @@ def _build_engine(self) -> ContextEngine:
             rag_chunk_overlap=self._rag_chunk_overlap,
             rag_embedding_model=self._rag_embedding_model,
             skeleton=self._skeleton,
+            redact=self._redact,
             fewshot=self._fewshot,
             fewshot_dir=self._fewshot_dir,
             fewshot_max_files=self._fewshot_max_files,

ctxeng/cli.py

@@ -37,6 +37,8 @@ def _cmd_ci(args: argparse.Namespace) -> None:
         builder = builder.no_gitignore()
     if args.trace:
         builder = builder.trace(True, trace_dir=args.trace_dir, trace_id=args.trace_id)
+    if not args.redact:
+        builder = builder.redact(False)
     if args.rag:
         builder = builder.rag(
             True,
@@ -95,6 +97,8 @@ def cmd_build(args: argparse.Namespace) -> None:
         builder = builder.deny(*args.deny)
     if args.trace:
         builder = builder.trace(True, trace_dir=args.trace_dir, trace_id=args.trace_id)
+    if not args.redact:
+        builder = builder.redact(False)
     if args.rag:
         builder = builder.rag(
             True,
@@ -249,6 +253,8 @@ def cmd_watch(args: argparse.Namespace) -> None:
         builder = builder.use_semantic(model=args.semantic_model)
     if args.budget:
         builder = builder.with_budget(args.budget)
+    if not args.redact:
+        builder = builder.redact(False)
 
     query = " ".join(args.query) if args.query else ""
 
@@ -334,6 +340,12 @@ def main() -> None:
         "--trace-id",
         help="Provide a custom trace id (default: random)",
     )
+    build_p.add_argument(
+        "--redact",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Redact secrets and PII before output (default: on)",
+    )
     build_p.add_argument(
         "--rag",
         action="store_true",
@@ -471,6 +483,12 @@ def main() -> None:
         "--trace-id",
         help="Provide a custom trace id (default: random)",
     )
+    watch_p.add_argument(
+        "--redact",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Redact secrets and PII before output (default: on)",
+    )
     watch_p.add_argument(
         "--rag",
         action="store_true",
@@ -576,6 +594,12 @@ def main() -> None:
     ci_p.add_argument("--trace", action="store_true", help="Write JSONL trace under .ctxeng/traces/")
     ci_p.add_argument("--trace-dir", help="Override trace output directory")
     ci_p.add_argument("--trace-id", help="Provide a custom trace id")
+    ci_p.add_argument(
+        "--redact",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Redact secrets and PII before output (default: on)",
+    )
     ci_p.add_argument("--rag", action="store_true", help="Enable chunk-level retrieval (RAG)")
     ci_p.add_argument("--rag-max-chunks", type=int, default=20)
     ci_p.add_argument("--rag-chunk-max-lines", type=int, default=120)

ctxeng/core.py

@@ -69,6 +69,7 @@ def __init__(
         rag_chunk_overlap: int = 20,
         rag_embedding_model: str = "all-MiniLM-L6-v2",
         skeleton: bool = False,
+        redact: bool = True,
         fewshot: bool = False,
         fewshot_dir: str | Path = ".ctxeng/examples",
         fewshot_max_files: int = 5,
@@ -98,6 +99,7 @@ def __init__(
         self.rag_chunk_overlap = rag_chunk_overlap
         self.rag_embedding_model = rag_embedding_model
         self.skeleton = skeleton
+        self.redact = redact
         self.fewshot = fewshot
         self.fewshot_dir = fewshot_dir
         self.fewshot_max_files = fewshot_max_files
@@ -286,22 +288,25 @@ def build(
             )
 
         # 4. Redact sensitive info before budgeting/output
-        redacted_files = 0
-        redacted_total = 0
-        for f in context_files:
-            r = redact_text(f.content, redact_secrets=True, redact_pii=True)
-            if r.total:
-                f.content = r.text
-                f.redaction_count = r.total
-                redacted_files += 1
-                redacted_total += r.total
+        if self.redact:
+            redacted_files = 0
+            redacted_total = 0
+            for f in context_files:
+                r = redact_text(f.content, redact_secrets=True, redact_pii=True)
+                if r.total:
+                    f.content = r.text
+                    f.redaction_count = r.total
+                    redacted_files += 1
+                    redacted_total += r.total
 
-        if trace_writer:
-            trace_writer.emit(
-                "redaction_summary",
-                files_with_redactions=redacted_files,
-                total_redactions=redacted_total,
-            )
+            if trace_writer:
+                trace_writer.emit(
+                    "redaction_summary",
+                    files_with_redactions=redacted_files,
+                    total_redactions=redacted_total,
+                )
+        elif trace_writer:
+            trace_writer.emit("redaction_summary", disabled=True)
 
         # 5. Optimize for token budget
         query_tokens = count_tokens(query, self.model) if query else 0