feat(visualizer): add HTTP content negotiation for visualizer API backward compatibility (#34)

* feat(visualizer): add HTTP content negotiation for visualizer API backward compatibility

- Default to JSON responses for v2.2.0+ backward compatibility
- Support MessagePack via Accept: application/msgpack header
- Browser visualizer explicitly requests MessagePack for performance
- Add test for JSON backward compatibility
- Update docs with JSON and MessagePack client examples

* fix(visualizer): improve dependency handling and multilingual JSON support

- Add Request to ImportError fallback block for dependency-free imports
- Place Request before default params (FastAPI auto-injects Request)
- Add Request to ImportError fallback block for dependency-free imports
- Replace Request hint to Request | None union type
- Add ensure_ascii=False to json.dumps() to preserve non-ASCII characters

Author: speedyk-005

CHANGELOG.md

@@ -22,12 +22,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Installation**: Added `viz` as alias for `visualization` extra
 
 ### Changed
+- **Visualizer API Content Negotiation**:
+  - Added HTTP Accept header content negotiation to `/api/chunk` endpoint
+  - Clients can request MessagePack via `Accept: application/msgpack` header
 - **Core Performance Overhaul**:
   - Replaced regex-based `_find_span` with deterministic finder (~2x faster span detection, avoiding regex backtracking)
   - Switched from `regex` library to stdlib `re` - ~2x faster for simple patterns
   - Replaced `box` library with `dotdict3` - 12x faster (0.467s vs 0.039s per 10k accesses)
   - Switched from JSON to MessagePack encoding in visualizer (~30-50% smaller payloads, faster encoding)
   - Added `@lru_cache(maxsize=52)` on `_get_special_lang_handler` for caching handler lookups
+  - Added explicitly MessagePack requests for visualizer when triggered via cli (~30-50% smaller payloads, faster encoding)
 - **Fallback Splitter**: Renamed `FallbackSplitter` to `_clean_sentences` (works for 50+ languages)
 - **SentenceSplitter Rename**: Renamed `_filter_sentences` method to `split_text`
 - **Lazy Imports**:

docs/getting-started/programmatic/visualizer.md

@@ -201,64 +201,108 @@ python server.py
 
 Use this Python client to chunk files programmatically:
 
-``` py linenums="1"
-import requests
+=== "JSON (Default - Backward Compatible)"
+
+    ``` py linenums="1"
+    import requests
+
+    # Connect to your running server
+    base_url = "http://127.0.0.1:8000"
+
+    # Check if token counter is available
+    response = requests.get(f"{base_url}/api/token_counter_status")
+    print(response.json())  # {"token_counter_available": false}
+
+    # Chunk a file
+    with open("my_document.txt", "rb") as f:
+        files = {"file": ("my_document.txt", f, "text/plain")}
+        data = {
+            "mode": "document",  # or "code"
+            "params": '{"max_sentences": 3, "overlap_percent": 20}'
+        }
+
+        response = requests.post(f"{base_url}/api/chunk", files=files, data=data)
+
+    if response.status_code == 200:
+        result = response.json()
+        print(f"Created {result['stats']['chunk_count']} chunks")
+
+        # Access chunks
+        for chunk in result["chunks"]:
+            print(f"Chunk content: {chunk['content']}")
+            print(f"Metadata: {chunk['metadata']}")
+    else:
+        print(f"Error: {response.status_code} - {response.text}")
+    ```
 
-# Connect to your running server
-base_url = "http://127.0.0.1:8000"
+=== "MessagePack (Faster - ~30-50% smaller payloads)"
 
-# Check if token counter is available
-response = requests.get(f"{base_url}/api/token_counter_status")
-print(response.json())  # {"token_counter_available": false}
+    ``` py linenums="1"
+    import requests
+    import msgpack
 
-# Chunk a file
-with open("my_document.txt", "rb") as f:
-    files = {"file": ("my_document.txt", f, "text/plain")}
-    data = {
-        "mode": "document",  # or "code"
-        "params": '{"max_sentences": 3, "overlap_percent": 20}'
-    }
+    # Connect to your running server
+    base_url = "http://127.0.0.1:8000"
 
-    response = requests.post(f"{base_url}/api/chunk", files=files, data=data)
+    # Request MessagePack response using Accept header
+    headers = {"Accept": "application/msgpack"}
 
-if response.status_code == 200:
-    result = response.json()
-    print(f"Created {result['stats']['chunk_count']} chunks")
+    with open("my_document.txt", "rb") as f:
+        files = {"file": ("my_document.txt", f, "text/plain")}
+        data = {
+            "mode": "document",
+            "params": '{"max_sentences": 3, "overlap_percent": 20}'
+        }
 
-    # Access chunks
-    for chunk in result["chunks"]:
-        print(f"Chunk content: {chunk['content']}")
-        print(f"Metadata: {chunk['metadata']}")
-else:
-    print(f"Error: {response.status_code} - {response.text}")
-```
+        response = requests.post(f"{base_url}/api/chunk", files=files, data=data, headers=headers)
+
+    if response.status_code == 200:
+        result = msgpack.unpackb(response.content, raw=False)
+        print(f"Created {result['stats']['chunk_count']} chunks")
+
+        for chunk in result["chunks"]:
+            print(f"Chunk content: {chunk['content']}")
+    else:
+        print(f"Error: {response.status_code} - {response.text}")
+    ```
 
 
 
 #### Response Format
 
-The `/api/chunk` endpoint returns:
+The `/api/chunk` endpoint supports content negotiation:
+
+- **JSON (default)**: Returns JSON response - backward compatible with v2.2.0 and earlier
+- **MessagePack**: Add `Accept: application/msgpack` header for ~30-50% smaller payloads
 
-```json
-{
-  "text": "Original file content...",
-  "chunks": [
+=== "JSON Response"
+
+    ```json
     {
-      "content": "Chunk text content...",
-      "metadata": {
-        "source": "filename.txt",
-        "chunk_num": 1,
-        "span": [0, 150],
-        // ... additional metadata
+      "text": "Original file content...",
+      "chunks": [
+        {
+          "content": "Chunk text content...",
+          "metadata": {
+            "source": "filename.txt",
+            "chunk_num": 1,
+            "span": [0, 150],
+            // ... additional metadata
+          }
+        }
+      ],
+      "stats": {
+        "text_length": 696,
+        "chunk_count": 3,
+        "mode": "document"
       }
     }
-  ],
-  "stats": {
-    "text_length": 696,
-    "chunk_count": 3,
-    "mode": "document"
-  }
-}
+    ```
+
+=== "MessagePack Response"
+
+    Same structure as JSON, but encoded in MessagePack binary format for smaller payloads.
+    Decode using: `msgpack.unpackb(response.content, raw=False)`
 ```
 
 !!! tip "Perfect for Integration"

src/chunklet/visualizer/static/js/app.js

@@ -365,6 +365,9 @@ async function processUploadedFile() {
         // Send POST with FormData
         const response = await fetch('/api/chunk', {
             method: 'POST',
+            headers: {
+                'Accept': 'application/msgpack'
+            },
             body: formData
         });
 

src/chunklet/visualizer/visualizer.py

@@ -10,7 +10,7 @@
     import msgpack
     import uvicorn
     from charset_normalizer import detect
-    from fastapi import FastAPI, File, Form, HTTPException, UploadFile
+    from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile
     from fastapi.responses import HTMLResponse, Response
     from fastapi.staticfiles import StaticFiles
 except ImportError:  # pragma: no cover
@@ -24,6 +24,7 @@
     File = lambda x: x  # noqa: E731
     Form = lambda x: x  # noqa: E731
     HTTPException = None
+    Request = None
     HTMLResponse = lambda x: x  # noqa: E731
     Response = lambda x: x  # noqa: E731
     StaticFiles = None
@@ -124,6 +125,7 @@ async def _get_index(self):
     @validate_input
     async def _chunk_file(
         self,
+        request: Request,
         file: UploadFile = File(...),
         mode: str = Form("document"),
         params: str = Form("{}"),
@@ -135,9 +137,13 @@ async def _chunk_file(
             file: File uploaded by the client.
             mode: Determines which chunker to use ("document" or "code").
             params: JSON string containing chunking parameters.
+            request: Optional Request object for content negotiation.
 
         Returns:
-            MessagePack-encoded response with original text, chunks, and stats.
+            MessagePack or JSON response with original text, chunks, and stats.
+            Uses content negotiation: client can request MessagePack via
+            Accept: application/msgpack header. Defaults to JSON for backward
+            compatibility.
 
         Raises:
             HTTPException: If chunking fails.
@@ -183,9 +189,16 @@ async def _chunk_file(
                 },
             }
 
+            accept = request.headers.get("Accept", "") if request else ""
+            if "application/msgpack" in accept:
+                return Response(
+                    content=msgpack.packb(response_data, use_bin_type=True),
+                    media_type="application/msgpack",
+                )
+
             return Response(
-                content=msgpack.packb(response_data, use_bin_type=True),
-                media_type="application/msgpack",
+                content=json.dumps(response_data, ensure_ascii=False),
+                media_type="application/json",
             )
 
         except Exception as e:

tests/test_visualization.py

@@ -101,7 +101,7 @@ def test_chunk_file(visualizer_server):
     sample_file_path = Path(__file__).parent.parent / "samples" / "sample_text.txt"
     assert sample_file_path.exists(), f"Sample file not found: {sample_file_path}"
 
-    # Test
+    # Test with MessagePack format (explicit request)
     with open(sample_file_path, "rb") as f:
         files = {"file": ("sample_text.txt", f, "text/plain")}
         data = {
@@ -110,8 +110,9 @@ def test_chunk_file(visualizer_server):
                 {"max_sentences": 3, "overlap_percent": 20}  # Chunk by 3 sentences
             ),
         }
+        headers = {"Accept": "application/msgpack"}
 
-        response = requests.post(url, files=files, data=data)
+        response = requests.post(url, files=files, data=data, headers=headers)
         assert response.status_code == 200
 
         result = msgpack.unpackb(response.content, raw=False)
@@ -129,6 +130,32 @@ def test_chunk_file(visualizer_server):
             assert "metadata" in chunk
 
 
+def test_chunk_file_json_backward_compatible(visualizer_server):
+    """Test that JSON response works for backward compatibility."""
+    url = f"{visualizer_server['url']}/api/chunk"
+
+    sample_file_path = Path(__file__).parent.parent / "samples" / "sample_text.txt"
+    assert sample_file_path.exists(), f"Sample file not found: {sample_file_path}"
+
+    # Test JSON (default - backward compatible)
+    with open(sample_file_path, "rb") as f:
+        files = {"file": ("sample_text.txt", f, "text/plain")}
+        data = {
+            "mode": "document",
+            "params": json.dumps({"max_sentences": 2}),
+        }
+        # No Accept header = JSON default
+
+        response = requests.post(url, files=files, data=data)
+        assert response.status_code == 200
+
+        result = response.json()
+        assert "text" in result
+        assert "chunks" in result
+        assert "stats" in result
+        assert result["stats"]["chunk_count"] > 1
+
+
 def test_chunk_file_invalid_format(visualizer_server):
     """Test uploading invalid file format."""
     url = f"{visualizer_server['url']}/api/chunk"