feat: add moss-chunking - pluggable chunking strategies over a shared chunk contract

.github/workflows/publish-moss-chunking.yml

@@ -0,0 +1,157 @@
+name: Publish moss-chunking
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: moss-chunking-release
+  cancel-in-progress: false
+
+jobs:
+  determine-version:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.compute.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Read version from pyproject
+        id: compute
+        shell: python
+        run: |
+          import os, pathlib, sys, tomllib
+
+          raw = pathlib.Path("packages/moss-chunking/pyproject.toml").read_bytes()
+          version = tomllib.loads(raw.decode("utf-8")).get("project", {}).get("version")
+          if not version:
+              print("Could not find project.version in pyproject", file=sys.stderr)
+              sys.exit(1)
+
+          out = pathlib.Path(os.environ["GITHUB_OUTPUT"])
+          with out.open("a", encoding="utf-8") as fh:
+              fh.write(f"version={version}\n")
+
+          print(f"Publishing version: {version}")
+
+  # Build + smoke-test on every supported Python version. Publishing waits for
+  # the whole matrix (see the publish job's `needs`), so a failure on any
+  # version blocks the upload/tag instead of racing it.
+  build-test:
+    needs: determine-version
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      # `pytest-asyncio` is not optional here: the suite's async tests rely on
+      # the `asyncio_mode = "auto"` setting in pyproject.toml, and plain pytest
+      # fails them outright with "async def functions are not natively
+      # supported", which would block every release rather than skip a test.
+      - name: Install build tooling
+        run: |
+          python -m pip install --upgrade pip
+          pip install build pytest pytest-asyncio
+
+      - name: Build distributions
+        working-directory: packages/moss-chunking
+        run: |
+          rm -rf dist
+          python -m build
+
+      - name: Install the built wheel
+        shell: python
+        run: |
+          import glob, subprocess, sys
+
+          wheels = glob.glob("packages/moss-chunking/dist/*.whl")
+          if not wheels:
+              raise SystemExit("No wheel found in dist/")
+
+          subprocess.check_call(
+              [sys.executable, "-m", "pip", "install", "--force-reinstall", wheels[0]]
+          )
+
+      - name: Smoke test the public surface
+        shell: python
+        run: |
+          import importlib
+
+          module = importlib.import_module("moss_chunking")
+          missing = [name for name in module.__all__ if not hasattr(module, name)]
+          if missing:
+              raise SystemExit(f"__all__ names missing from the built wheel: {missing}")
+          print(f"import ok; {len(module.__all__)} public names resolve")
+
+      # Run the suite against the installed wheel, not the source tree. A wheel
+      # that imports cleanly can still ship broken offsets or metadata, and that
+      # is precisely what these tests assert.
+      - name: Test the installed wheel
+        run: python -m pytest -q packages/moss-chunking/tests
+
+  # Only runs once every build-test matrix leg has passed.
+  publish:
+    needs: [determine-version, build-test]
+    # This job holds PYPI_API_TOKEN and pushes a release tag. `workflow_dispatch`
+    # accepts any ref, so without this guard an unmerged branch could be
+    # published under the package name and tagged as a release.
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      VERSION: ${{ needs.determine-version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Fail if this version was already released
+        run: |
+          git fetch --tags --force
+          # Match the tag ref exactly: `git rev-parse` alone resolves anything
+          # that happens to parse as a revision, not just a tag by this name.
+          if git rev-parse --verify "refs/tags/moss-chunking-v${VERSION}" >/dev/null 2>&1; then
+            echo "::error::moss-chunking v${VERSION} is already tagged/released; bump the version in pyproject.toml before releasing."
+            exit 1
+          fi
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tooling
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+
+      - name: Build distributions
+        working-directory: packages/moss-chunking
+        run: |
+          rm -rf dist
+          python -m build
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          twine upload packages/moss-chunking/dist/*
+
+      - name: Tag release
+        run: |
+          git config user.name "github-actions"
+          git config user.email "github-actions@users.noreply.github.com"
+          git tag "moss-chunking-v${VERSION}"
+          git push origin "moss-chunking-v${VERSION}"

packages/moss-chunking/.gitignore

@@ -0,0 +1,10 @@
+build/
+dist/
+*.egg-info/
+__pycache__/
+*.py[cod]
+.venv/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.env

packages/moss-chunking/README.md

@@ -0,0 +1,165 @@
+# moss-chunking
+
+Pluggable text chunking for Moss. Splitters are swappable; the shape of what they
+emit is not.
+
+## Why this exists
+
+Chunking isn't in any Moss SDK, so every app rolls its own. Two of them, in this
+repo, both already wrap chunks in the SDK's `DocumentInfo` — and still can't be
+treated uniformly, because they agree on nothing inside it:
+
+| | `examples/moss-pikachu` | `apps/moss-llamaindex` |
+| --- | --- | --- |
+| id | `{path}#chunk-0001` | `{filename}-p{page}-c{idx}` |
+| metadata | `path`, `filename`, `chunk`, `extension`, `modified_at` | `source`, `page` |
+| split | 1800 chars / 300 overlap | 400 words / 2-sentence overlap |
+
+Zero metadata keys in common. The envelope is shared; the contract is missing.
+
+Splitting strategy is genuinely contested and content-dependent — code, markdown
+and transcripts all want different cuts — which is why it stays pluggable, and
+why this is a standalone package rather than something frozen into five language
+runtimes. But the *output* isn't contested. Nobody wants a bespoke ID scheme;
+they wrote one because none was written down.
+
+So this package pins the output and leaves the cutting open.
+
+## Install
+
+```bash
+uv pip install -e ".[dev]"   # from packages/moss-chunking
+```
+
+Depends only on `moss`. Sentence detection is regex-based rather than nltk-backed
+so there is no model download or corpus to provision.
+
+## Use
+
+```python
+from moss_chunking import SentenceSplitter, chunk_document, ingest
+
+docs = chunk_document(text, source="notes.md", strategy=SentenceSplitter())
+await ingest(docs, project_id, project_key, "my-index")
+```
+
+`docs` are ordinary `DocumentInfo`s — they go anywhere the SDK takes documents;
+`ingest` is just the connector template's one-call shortcut into a fresh index.
+
+To re-chunk one document inside an index that already exists, use
+`refresh_source` rather than `add_docs`:
+
+```python
+await refresh_source(client, "my-index", "notes.md", docs)
+```
+
+`add_docs` upserts, which replaces the chunks the new cut still produces — but a
+document that shrinks from 21 chunks to 6 leaves `#chunk-0006` through
+`#chunk-0020` in the index, still searchable, holding text the document no longer
+contains. `refresh_source` deletes that tail first. Passing no documents removes
+the source entirely, which is how a deleted file leaves the index.
+
+## The contract
+
+Every chunk, from every strategy, carries:
+
+| key | meaning |
+| --- | --- |
+| `source` | what was chunked — path, URL, document name |
+| `chunk_index` | position in the sequence, from `0` |
+| `locator_type` | `char`, `line` or `page` |
+| `locator_start` / `locator_end` | position, in that unit |
+
+IDs are `{source}#chunk-{index:04d}` — zero-padded so they sort in cut order, and
+stable across runs so re-chunking an unchanged document replaces its chunks
+rather than duplicating them. Indices run `0, 1, 2, …`; `chunk_document` rejects a
+strategy that skips or repeats one, because a repeat renders the same ID twice and
+the second chunk silently overwrites the first. Values are all strings, because
+Moss types metadata as `Dict[str, str]`.
+
+The sort only holds while the padding is fixed width, so `chunk_id` rejects an
+index above `MAX_CHUNK_INDEX` (9999) rather than emitting `chunk-10000`, which
+sorts before `chunk-9999`. A document that cuts into more than 10,000 chunks
+wants a coarser strategy or a per-section `source`.
+
+That stability is only worth something if you keep it on the way in, which is why
+`ingest` drops the one option the connector template it mirrors does offer:
+`auto_id`. Random UUIDs defeat the contract — re-indexing an unchanged document
+appends a second copy of every chunk instead of replacing what is already there.
+
+**Position is not a fixed field list.** Plain text is located by character offset,
+PDFs by page, code by line; character offsets are meaningless for a PDF and page
+numbers are meaningless for a source file. So a chunk declares its unit instead of
+assuming one. That's the one real design call in the package.
+
+Pass source-level facts a splitter can't know via `extra`:
+
+```python
+chunk_document(text, "notes.md", CharSplitter(), extra={"extension": "md", "page": 3})
+```
+
+Values are stringified on the way out, so passing an `int` is fine — Moss types
+metadata as `Dict[str, str]`, and coercing here beats failing at the SDK
+boundary. `extra` cannot shadow the reserved keys above: that's an error, not a
+silent overwrite, and the reserved keys win at render even if one is added to
+`extra` afterwards.
+
+## Strategies
+
+| | cuts on | ceiling | notes |
+| --- | --- | --- | --- |
+| `CharSplitter` | fixed character windows | hard | pikachu's 1800/300 are the defaults |
+| `SentenceSplitter` | sentence boundaries | soft | llamaindex's 400 words / 2 sentences are the defaults |
+| `ParagraphSplitter` | blank lines | soft | never breaks a paragraph, even an oversized one |
+| `RecursiveSplitter` | paragraphs → lines → sentences → words | hard | falls back to a hard cut if nothing fits |
+
+Soft ceiling means a single unit larger than the budget is emitted whole rather
+than cut. `RecursiveSplitter` is the one to reach for when the ceiling must hold.
+
+Write your own by implementing `split(text) -> Iterable[Chunk]`:
+
+```python
+from collections.abc import Iterator
+
+from moss_chunking import Chunk
+
+
+class MyStrategy:
+    def split(self, text: str) -> Iterator[Chunk]:
+        yield Chunk(text=..., index=..., locator_type="line", locator_start=..., locator_end=...)
+```
+
+A strategy never builds a `DocumentInfo` — that's the contract's job, and handing
+it to callers is exactly how pikachu and llamaindex drifted apart.
+
+The invariant the tests enforce for every strategy:
+`text[chunk.locator_start:chunk.locator_end] == chunk.text`. Offsets point into
+the original string, never a normalized copy. Break it and position metadata
+becomes decorative — you can address a chunk but not find it again.
+
+Semantic chunking is not here yet: it needs embeddings, which makes it a
+different shape of dependency. It's the obvious next strategy.
+
+## Enrichment
+
+Moss scores BM25 over chunk text, so facts that live only in metadata — the
+filename, the folder — are invisible to the keyword half of a hybrid query.
+Restating them in the text makes them matchable. Pikachu already does this by
+hand; here it's a composable post-step that works with any strategy:
+
+```python
+from moss_chunking import prepend_source_context
+
+doc = prepend_source_context(doc, filename="notes.md", path="/docs/notes.md")
+```
+
+ID and metadata are untouched, so an enriched chunk stays addressable. Any
+embedding is dropped: it was computed from the text this rewrites, and a vector
+that no longer describes its chunk skews the dense half of every hybrid query
+without ever announcing itself. Enrich first, embed after.
+
+## Tests
+
+```bash
+.venv/bin/python -m pytest -q
+```

packages/moss-chunking/pyproject.toml

@@ -0,0 +1,60 @@
+[project]
+name = "moss-chunking"
+version = "0.0.1"
+description = "Pluggable text chunking strategies that emit Moss DocumentInfo under a shared ID and metadata contract."
+readme = "README.md"
+requires-python = ">=3.10,<3.15"
+license = { text = "BSD-2-Clause" }
+authors = [{ name = "InferEdge Inc.", email = "contact@moss.dev" }]
+keywords = ["moss", "chunking", "splitter", "retrieval", "rag"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: BSD License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Text Processing :: Linguistic",
+]
+# Most packages here declare `moss>=1.1.1`, but that floor cannot resolve: every
+# moss release before 1.7.2 pins an `inferedge-moss-core` that is no longer on
+# PyPI. 1.7.2 is the earliest version that actually installs, and the earliest
+# whose `DocumentInfo` is verified to accept `payload`.
+dependencies = [
+    "moss>=1.7.2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.23.0",
+    "python-dotenv>=1.0.0",
+    "ruff>=0.5.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/usemoss/moss"
+Repository = "https://github.com/usemoss/moss"
+Source = "https://github.com/usemoss/moss/tree/main/packages/moss-chunking"
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+# Flat layout: src/ itself IS the package.
+[tool.setuptools]
+packages = ["moss_chunking"]
+package-dir = { "moss_chunking" = "src" }
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "W", "F", "I", "B", "UP"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"