feat: make max file size configurable via SEMBLE_MAX_FILE_BYTES

Files larger than 1 MB are skipped during indexing without any indication, silently leaving gaps in search results (#250).

Resolve the limit per call from the SEMBLE_MAX_FILE_BYTES environment variable (following SEMBLE_CACHE_LOCATION / SEMBLE_CLONE_TIMEOUT / SEMBLE_MODEL_NAME), falling back to the unchanged 1 MB default; malformed or nonpositive values warn and fall back instead of crashing indexing. Warn at index time naming files skipped for size, with the CLI surfacing warnings on stderr via an idempotent, CLI-owned handler.

Author: TommyC81

README.md

@@ -174,6 +174,8 @@ Savings are calculated as follows: for each call, semble records the total chara
 
 By default, your Semble savings statistics and any saved indexes are stored in the OS cache folder (`~/Library/Caches/semble/` on macOS, `~/.cache/semble/` on Linux, `%LOCALAPPDATA%\semble\Cache\` on Windows). To override this location you can supply an environment variable `SEMBLE_CACHE_LOCATION` which should be the full path to the target cache location e.g. `~/my-folder/my-caches/semble`.
 
+Files larger than 1 MB are skipped during indexing to keep index builds lean. Skipped files are reported as a warning at index time. If you work with large generated or ingested documents, you can raise (or lower) this limit with the `SEMBLE_MAX_FILE_BYTES` environment variable (in bytes).
+
 On first use, Semble also downloads the embedding model from Hugging Face and caches it in the standard Hugging Face cache (`~/.cache/huggingface/` by default, or `$HF_HOME` if set); this only happens once and requires network access.
 
 Use `semble clear` to remove cached data: `semble clear index` (saved indexes), `semble clear savings` (usage stats), `semble clear orphans` (indexes for repos no longer present on disk), or `semble clear all` (everything).

src/semble/cli.py

@@ -2,6 +2,7 @@
 import asyncio
 import io
 import json
+import logging
 import re
 import sys
 import warnings
@@ -207,7 +208,24 @@ def _run_clear(clear_type: _CLEAR_CHOICE) -> None:
         _clear_orphans(cache_folder)
 
 
+class _CliLogHandler(logging.StreamHandler):
+    """stderr handler owned by the CLI; setup is idempotent on this type, not on foreign handlers."""
+
+
+def _configure_cli_logging() -> None:
+    """Surface semble warnings (e.g. skipped oversized files) on stderr without touching the root logger."""
+    package_logger = logging.getLogger("semble")
+    if any(isinstance(handler, _CliLogHandler) for handler in package_logger.handlers):
+        return
+    handler = _CliLogHandler(sys.stderr)
+    handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
+    package_logger.addHandler(handler)
+    if package_logger.level == logging.NOTSET:
+        package_logger.setLevel(logging.WARNING)
+
+
 def _cli_main() -> None:
+    _configure_cli_logging()
     parser = argparse.ArgumentParser(prog="semble")
     parser.add_argument("-V", "--version", action="version", version=__version__)
     sub = parser.add_subparsers(dest="command")

src/semble/index/create.py

@@ -1,4 +1,5 @@
 import contextlib
+import logging
 from collections.abc import Sequence
 from pathlib import Path
 
@@ -15,13 +16,29 @@
     detect_language,
     get_extensions,
     get_file_status,
+    get_max_file_bytes,
     read_file_text,
 )
 from semble.index.sparse import enrich_for_bm25
 from semble.index.types import FileManifestEntry, PreviousIndex, make_chunk_id
 from semble.tokens import tokenize
 from semble.types import Chunk, ContentType, EmbeddingMatrix
 
+logger = logging.getLogger(__name__)
+
+
+def _warn_skipped_large(skipped_large: list[str]) -> None:
+    """Warn about files skipped for exceeding the maximum indexable file size."""
+    if skipped_large:
+        logger.warning(
+            "Skipped %d file(s) exceeding the maximum file size of %d bytes "
+            "(raise SEMBLE_MAX_FILE_BYTES to include them): %s%s",
+            len(skipped_large),
+            get_max_file_bytes(),
+            ", ".join(skipped_large[:5]),
+            " ..." if len(skipped_large) > 5 else "",
+        )
+
 
 def _reindex_file(
     bm25_index: BM25,
@@ -79,10 +96,14 @@ def create_index_from_path(
     manifest: dict[str, FileManifestEntry] = {}
     embedding_parts: list[tuple[int, int, int]] = []
 
+    skipped_large: list[str] = []
+
     for file_path in walk_files(path, resolved_extensions):
         language = detect_language(file_path)
         with contextlib.suppress(OSError):
             file_status = get_file_status(file_path, None)
+            if file_status is FileStatus.TOO_LARGE:
+                skipped_large.append(str(file_path))
             if file_status != FileStatus.VALID:
                 continue
 
@@ -109,6 +130,8 @@ def create_index_from_path(
     for indexed_path in previous_manifest.keys() - manifest.keys():
         _reindex_file(bm25_index, indexed_path, [], previous_manifest[indexed_path])
 
+    _warn_skipped_large(skipped_large)
+
     if not chunks:
         raise ValueError(f"No supported files found under {path}.")
 

src/semble/index/files.py

@@ -1,12 +1,16 @@
+import logging
+import os
 from collections import defaultdict
 from collections.abc import Sequence
 from enum import Enum
 from pathlib import Path
 
 from semble.types import ContentType
 
-_MAX_FILE_BYTES = 1_000_000  # 1 MB max file size to read and index
+_DEFAULT_MAX_FILE_BYTES = 1_000_000  # Default 1 MB max file size to read and index
 _EMPTY_FILE_BYTES = 128
+
+logger = logging.getLogger(__name__)
 _EXTENSION_TO_LANGUAGE = {
     ".4th": "forth",
     ".ada": "ada",
@@ -488,14 +492,33 @@ def read_file_text(file_path: Path) -> str:
     return file_path.read_text(encoding="utf-8", errors="replace")
 
 
+def get_max_file_bytes() -> int:
+    """Resolve the maximum file size to index from SEMBLE_MAX_FILE_BYTES, falling back to the default.
+
+    Malformed or nonpositive values warn and fall back to the default rather than crash indexing.
+    """
+    raw = os.environ.get("SEMBLE_MAX_FILE_BYTES")
+    if raw is None:
+        return _DEFAULT_MAX_FILE_BYTES
+    try:
+        value = int(raw)
+    except ValueError:
+        logger.warning("Invalid SEMBLE_MAX_FILE_BYTES %r, using the default of %d bytes", raw, _DEFAULT_MAX_FILE_BYTES)
+        return _DEFAULT_MAX_FILE_BYTES
+    if value <= 0:
+        logger.warning("SEMBLE_MAX_FILE_BYTES must be positive, using the default of %d bytes", _DEFAULT_MAX_FILE_BYTES)
+        return _DEFAULT_MAX_FILE_BYTES
+    return value
+
+
 def get_file_status(file_path: Path, write_time: float | None) -> FileStatus:
     """Checks if a file should be indexed based on its size and modification time."""
     stat = file_path.stat()
     if write_time is not None and stat.st_mtime > write_time:
         # Index invalid, file invalid
         return FileStatus.NEWER
     size = stat.st_size
-    if size > _MAX_FILE_BYTES:
+    if size > get_max_file_bytes():
         # index valid, file invalid
         return FileStatus.TOO_LARGE
     if size < _EMPTY_FILE_BYTES and not read_file_text(file_path).strip():

tests/index/test_index.py

@@ -1,3 +1,4 @@
+import logging
 from pathlib import Path
 from typing import Any
 from unittest.mock import MagicMock, patch
@@ -8,7 +9,7 @@
 
 from semble import SembleIndex
 from semble.index.create import create_index_from_path
-from semble.index.files import _MAX_FILE_BYTES, FileStatus, get_file_status
+from semble.index.files import _DEFAULT_MAX_FILE_BYTES, FileStatus, get_file_status, get_max_file_bytes
 from semble.types import ContentType
 from tests.conftest import make_chunk
 
@@ -69,11 +70,45 @@ def test_index_empty_returns_zero_chunks(mock_model: StaticModel, tmp_path: Path
         create_index_from_path(tmp_path, mock_model)
 
 
-def test_oversized_file_is_skipped(mock_model: StaticModel, tmp_path: Path) -> None:
-    """Files exceeding _MAX_FILE_BYTES are silently skipped during indexing."""
-    (tmp_path / "big.py").write_bytes(b"x" * (_MAX_FILE_BYTES + 1))
-    with pytest.raises(ValueError):  # no indexable content remains
-        create_index_from_path(tmp_path, mock_model)
+def test_max_file_bytes_resolution(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The limit resolves from SEMBLE_MAX_FILE_BYTES, falling back to the 1 MB default."""
+    monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False)
+    assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES
+    monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", str(_DEFAULT_MAX_FILE_BYTES + 5))
+    assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES + 5
+
+
+def test_max_file_bytes_invalid_values_fall_back(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Malformed or nonpositive SEMBLE_MAX_FILE_BYTES values fall back to the default."""
+    monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False)
+    assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES
+    for bad in ("not-a-number", "0", "-5"):
+        monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", bad)
+        assert get_max_file_bytes() == _DEFAULT_MAX_FILE_BYTES
+
+
+def test_oversized_file_is_skipped_with_warning(
+    mock_model: StaticModel, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    """Files exceeding the limit are skipped during indexing, with a warning naming them."""
+    monkeypatch.delenv("SEMBLE_MAX_FILE_BYTES", raising=False)
+    (tmp_path / "big.py").write_bytes(b"x" * (_DEFAULT_MAX_FILE_BYTES + 1))
+    with caplog.at_level(logging.WARNING, logger="semble.index.create"):
+        with pytest.raises(ValueError):  # no indexable content remains
+            create_index_from_path(tmp_path, mock_model)
+    assert "big.py" in caplog.text
+    assert str(_DEFAULT_MAX_FILE_BYTES) in caplog.text
+    assert "SEMBLE_MAX_FILE_BYTES" in caplog.text
+
+
+def test_oversized_file_indexed_when_limit_raised(
+    mock_model: StaticModel, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Raising SEMBLE_MAX_FILE_BYTES lets oversized files into the index."""
+    monkeypatch.setenv("SEMBLE_MAX_FILE_BYTES", str(_DEFAULT_MAX_FILE_BYTES + 1024))
+    (tmp_path / "big.py").write_bytes(b"x = 1\n" + b"#" * _DEFAULT_MAX_FILE_BYTES)
+    _, _, chunks, _ = create_index_from_path(tmp_path, mock_model)
+    assert any(chunk.file_path.endswith("big.py") for chunk in chunks)
 
 
 def test_tiny_invalid_utf8_file_status_does_not_crash(tmp_path: Path) -> None: