diff --git a/.cursor/rules/api-models.mdc b/.cursor/rules/api-models.mdc
new file mode 100644
index 0000000..031d35f
--- /dev/null
+++ b/.cursor/rules/api-models.mdc
@@ -0,0 +1,85 @@
+---
+description: Rules for Pydantic models and request/response validation
+globs: ["src/data/data_type.py"]
+alwaysApply: true
+---
+
+# API Models Guidelines
+
+Pydantic models validate request bodies and ensure type safety. Models are defined in `src/data/data_type.py`.
+
+## Model Definition
+
+**Basic structure:**
+```python
+from typing import Dict, List, Any
+from pydantic import BaseModel
+
+class EmbeddingRequest(BaseModel):
+    project_id: str
+    embedding_id: str
+
+class EmbeddingRebuildRequest(BaseModel):
+    # example request structure:
+    # {"<embedding_id>":[{"record_id":"<record_id>","attribute_name":"<attribute_name>","sub_key":<sub_key>}]}
+    # note that sub_key is optional and only for embedding lists relevant
+    # also sub_key is an int but converted to string in the request
+    changes: Dict[str, List[Dict[str, Any]]]
+
+class EmbeddingCalcTensorByPkl(BaseModel):
+    texts: List[str]
+```
+
+## Naming Conventions
+
+- Request models: `EmbeddingRequest`, `EmbeddingRebuildRequest`, `EmbeddingCalcTensorByPkl`
+- Use descriptive names that indicate the operation and data type
+- Use `Request` suffix for request body models
+
+## Usage in Routes
+
+```python
+from src.data import data_type
+
+@app.post("/embed")
+def embed(request: data_type.EmbeddingRequest) -> responses.PlainTextResponse:
+    status_code = controller.manage_encoding_thread(
+        request.project_id, request.embedding_id
+    )
+    return responses.PlainTextResponse(status_code=status_code)
+
+@app.post("/re_embed_records/{project_id}")
+def re_embed_record(
+    project_id: str, 
+    request: data_type.EmbeddingRebuildRequest
+) -> responses.PlainTextResponse:
+    controller.re_embed_records(project_id, request.changes)
+    return responses.PlainTextResponse(status_code=status.HTTP_200_OK)
+```
+
+## Field Validation
+
+```python
+from pydantic import field_validator, Field
+
+class EmbeddingRequest(BaseModel):
+    project_id: str = Field(min_length=1)
+    embedding_id: str = Field(min_length=1)
+    
+    @field_validator('project_id', 'embedding_id')
+    @classmethod
+    def validate_ids(cls, v):
+        if not v or not v.strip():
+            raise ValueError('ID cannot be empty')
+        return v.strip()
+```
+
+## Best Practices
+
+1. Use standard Python types (`str`, `int`, `List`, `Dict`) - Pydantic handles validation
+2. Provide defaults for optional fields using `Optional[Type] = None`
+3. Use descriptive model names that indicate purpose
+4. Document complex nested structures with comments
+5. Use proper type hints for all fields
+6. Keep models focused on request/response data structure
+7. Use `Dict[str, Any]` for flexible nested structures when needed
diff --git a/.cursor/rules/controllers.mdc b/.cursor/rules/controllers.mdc
new file mode 100644
index 0000000..3b1db78
--- /dev/null
+++ b/.cursor/rules/controllers.mdc
@@ -0,0 +1,155 @@
+---
+description: Rules for controller module and business logic
+globs: ["controller.py"]
+alwaysApply: true
+---
+
+# Controllers Guidelines
+
+The controller module (`controller.py`) contains business logic for embedding operations and orchestrates interactions between routes, submodules, embedders, and external services.
+
+## Import Patterns
+
+```python
+# Submodules
+from submodules.model.business_objects import (
+    attribute,
+    embedding,
+    general,
+    project,
+    record,
+    tokenization,
+    notification,
+    organization,
+)
+from submodules.model import enums, daemon
+from submodules.s3 import controller as s3
+
+# Embedders
+from src.embedders import Transformer, util
+from src.embedders.classification.contextual import (
+    OpenAISentenceEmbedder,
+    HuggingFaceSentenceEmbedder,
+)
+from src.util import request_util
+from src.util.decorator import param_throttle
+from src.util.embedders import get_embedder
+from src.util.notification import send_project_update
+```
+
+## Function Patterns
+
+**Async embedding operations:**
+```python
+from submodules.model import daemon
+from fastapi import status
+
+def manage_encoding_thread(project_id: str, embedding_id: str) -> int:
+    daemon.run_without_db_token(prepare_run, project_id, embedding_id)
+    return status.HTTP_200_OK
+```
+
+**Embedding lifecycle:**
+```python
+def delete_embedding(project_id: str, embedding_id: str) -> int:
+    object_name = f"embedding_tensors_{embedding_id}.csv.bz2"
+    org_id = organization.get_id_by_project_id(project_id)
+    s3.delete_object(org_id, f"{project_id}/{object_name}")
+    request_util.delete_embedding_from_neural_search(embedding_id)
+    json_path = util.INFERENCE_DIR / project_id / f"embedder-{embedding_id}.json"
+    json_path.unlink(missing_ok=True)
+    return status.HTTP_200_OK
+```
+
+**Embedding state management:**
+```python
+def run_encoding(project_id: str, user_id: str, embedding_id: str, ...) -> int:
+    session_token = general.get_ctx_token()
+    try:
+        # Update embedding state
+        embedding.update_embedding_state_encoding(project_id, embedding_id, with_commit=True)
+        send_project_update(project_id, f"embedding:{embedding_id}:state:{enums.EmbeddingState.ENCODING.value}")
+        
+        # Process batches
+        for pair in generate_batches(...):
+            embedding.create_tensors(project_id, embedding_id, record_ids_batched, tensors, with_commit=True)
+            send_progress_update_throttle(project_id, embedding_id, state, initial_count)
+        
+        # Finalize
+        embedding.update_embedding_state_finished(project_id, embedding_id, with_commit=True)
+    finally:
+        general.remove_and_refresh_session(session_token)
+    return status.HTTP_200_OK
+```
+
+## Business Logic Patterns
+
+**Batch processing:**
+```python
+def generate_batches(
+    project_id: str,
+    record_ids: List[str],
+    embedding_type: str,
+    attribute_values_raw: List[str],
+    embedder: Transformer,
+    attribute_name: str,
+    for_delta: bool = False,
+) -> Iterator[Dict[List[str], List[Any]]]:
+    # Process records in batches using embedder.batch_size
+    # Yield batches of record_ids and embeddings
+    pass
+```
+
+**Session management:**
+```python
+def prepare_run(project_id: str, embedding_id: str) -> None:
+    session_token = general.get_ctx_token()
+    try:
+        t = __prepare_encoding(project_id, embedding_id)
+    finally:
+        general.remove_and_refresh_session(session_token)
+    if t:
+        run_encoding(*t)
+```
+
+**Error handling with notifications:**
+```python
+try:
+    # Embedding operation
+    pass
+except Exception as e:
+    embedding.update_embedding_state_failed(project_id, embedding_id, with_commit=True)
+    send_project_update(project_id, f"embedding:{embedding_id}:state:{enums.EmbeddingState.FAILED.value}")
+    notification.create(
+        project_id,
+        user_id,
+        str(e),
+        enums.Notification.ERROR.value,
+        enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
+        True,
+    )
+    return status.HTTP_500_INTERNAL_SERVER_ERROR
+```
+
+**Throttled progress updates:**
+```python
+@param_throttle(seconds=5)
+def send_progress_update_throttle(
+    project_id: str, embedding_id: str, state: str, initial_count: int
+) -> None:
+    progress = resolve_progress(embedding_id, state, initial_count)
+    send_project_update(project_id, f"embedding:{embedding_id}:progress:{progress}")
+```
+
+## Best Practices
+
+1. Single responsibility per function
+2. Always validate inputs and check embedding existence
+3. Use type hints for all parameters
+4. Use `with_commit=True` when modifying database state
+5. Use submodule business objects, never SQLAlchemy directly
+6. Manage database sessions with `general.get_ctx_token()` and `general.remove_and_refresh_session()`
+7. Use `daemon.run_without_db_token()` for background operations
+8. Update embedding state and send project updates for progress tracking
+9. Clean up resources (delete embedders, call gc.collect()) after operations
+10. Handle errors gracefully with appropriate notifications and state updates
diff --git a/.cursor/rules/exceptions.mdc b/.cursor/rules/exceptions.mdc
new file mode 100644
index 0000000..6740763
--- /dev/null
+++ b/.cursor/rules/exceptions.mdc
@@ -0,0 +1,84 @@
+---
+description: Rules for exception handling and custom exceptions
+globs: ["**/*.py"]
+alwaysApply: true
+---
+
+# Exceptions Guidelines
+
+## Exception Locations
+
+**Submodule exceptions:**
+```python
+from submodules.model.exceptions import EntityNotFoundException, EntityAlreadyExistsException
+```
+
+**Standard Python exceptions:**
+- `ValueError` - Invalid input values
+- `Exception` - General errors (with specific messages)
+
+## Usage Patterns
+
+**Raising exceptions:**
+```python
+# Validation
+if not embedding.get(project_id, embedding_id):
+    raise ValueError(f"Embedding {embedding_id} not found in project {project_id}")
+
+# Not found (from submodules)
+embedding_item = embedding.get(project_id, embedding_id)
+if not embedding_item:
+    # Handle gracefully - return early or raise
+    return
+
+# Business logic errors
+if not embedder:
+    raise Exception(
+        f"couldn't find matching embedder for requested embedding with type {embedding_type} model {model} and platform {platform}"
+    )
+```
+
+**Handling in controllers:**
+```python
+try:
+    embedder = get_embedder(...)
+    if not embedder:
+        raise Exception("Could not initialize embedder")
+except Exception as e:
+    print(traceback.format_exc(), flush=True)
+    embedding.update_embedding_state_failed(project_id, embedding_id, with_commit=True)
+    send_project_update(project_id, f"embedding:{embedding_id}:state:{enums.EmbeddingState.FAILED.value}")
+    notification.create(...)
+    return status.HTTP_422_UNPROCESSABLE_ENTITY
+```
+
+**Handling in routes:**
+```python
+@app.post("/calc-tensor-by-pkl/{project_id}/{embedding_id}")
+def calc_tensor(...):
+    if tensor := controller.calc_tensors(project_id, embedding_id, request.texts):
+        return responses.JSONResponse(status_code=status.HTTP_200_OK, content={"tensor": tensor})
+    return responses.PlainTextResponse(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        content="Error while calculating tensor",
+    )
+```
+
+## HTTP Status Code Mapping
+
+- `200`: Successful operations
+- `422`: `UnprocessableEntity` - Invalid input or model initialization failures
+- `500`: `InternalServerError` - Runtime errors, API connection errors, general exceptions
+
+## Error Handling Best Practices
+
+1. Use specific exception types when available from submodules
+2. Provide clear error messages with context (project_id, embedding_id, etc.)
+3. Log exceptions with `print(traceback.format_exc(), flush=True)` for debugging
+4. Update embedding state to `FAILED` when errors occur
+5. Send project updates to notify users of failures
+6. Create notifications for user-facing errors
+7. Return appropriate HTTP status codes from routes
+8. Clean up resources (sessions, embedders) in `finally` blocks
+9. Don't swallow exceptions silently - always handle or propagate
+10. Use early returns for validation failures to avoid deep nesting
diff --git a/.cursor/rules/fastapi-routes.mdc b/.cursor/rules/fastapi-routes.mdc
new file mode 100644
index 0000000..1cec03c
--- /dev/null
+++ b/.cursor/rules/fastapi-routes.mdc
@@ -0,0 +1,120 @@
+---
+description: Rules for FastAPI route definitions and HTTP handling
+globs: ["app.py"]
+alwaysApply: true
+---
+
+# FastAPI Routes Guidelines
+
+Routes handle HTTP request/response logic. Keep routes thin - validate input, call controllers, format responses.
+
+## Route Structure
+
+Routes are defined directly in `app.py`:
+
+```python
+from fastapi import FastAPI, responses, status, Request
+from src.data import data_type
+import controller
+
+app = FastAPI(title="refinery-embedder")
+
+@app.post("/embed")
+def embed(request: data_type.EmbeddingRequest) -> responses.PlainTextResponse:
+    status_code = controller.manage_encoding_thread(
+        request.project_id, request.embedding_id
+    )
+    return responses.PlainTextResponse(status_code=status_code)
+```
+
+## Response Patterns
+
+```python
+# JSON response with data
+return responses.JSONResponse(
+    status_code=status.HTTP_200_OK, 
+    content={"tensor": tensor}
+)
+
+# Plain text response
+return responses.PlainTextResponse(status_code=status.HTTP_200_OK)
+
+# Error responses
+return responses.PlainTextResponse(
+    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+    content="Error message"
+)
+```
+
+## Request Parameters
+
+```python
+# Path parameters
+@app.delete("/delete/{project_id}/{embedding_id}")
+def delete_embedding(
+    project_id: str, 
+    embedding_id: str
+) -> responses.PlainTextResponse:
+    pass
+
+# Request body with Pydantic models
+from src.data import data_type
+
+@app.post("/re_embed_records/{project_id}")
+def re_embed_record(
+    project_id: str, 
+    request: data_type.EmbeddingRebuildRequest
+) -> responses.PlainTextResponse:
+    controller.re_embed_records(project_id, request.changes)
+    return responses.PlainTextResponse(status_code=status.HTTP_200_OK)
+```
+
+## Health Check Pattern
+
+```python
+@app.get("/healthcheck")
+def healthcheck() -> responses.PlainTextResponse:
+    text = ""
+    status_code = status.HTTP_200_OK
+    database_test = general.test_database_connection()
+    if not database_test.get("success"):
+        error_name = database_test.get("error")
+        text += f"database_error:{error_name}:"
+        status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
+    if not text:
+        text = "OK"
+    return responses.PlainTextResponse(text, status_code=status_code)
+```
+
+## Error Handling
+
+```python
+from fastapi import status, responses
+
+@app.post("/calc-tensor-by-pkl/{project_id}/{embedding_id}")
+def calc_tensor(
+    project_id: str, 
+    embedding_id: str, 
+    request: data_type.EmbeddingCalcTensorByPkl
+) -> Union[responses.JSONResponse, responses.PlainTextResponse]:
+    if tensor := controller.calc_tensors(project_id, embedding_id, request.texts):
+        return responses.JSONResponse(
+            status_code=status.HTTP_200_OK, 
+            content={"tensor": tensor}
+        )
+    return responses.PlainTextResponse(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        content="Error while calculating tensor",
+    )
+```
+
+## Best Practices
+
+1. Keep routes thin - delegate to controllers
+2. Use Pydantic models from `src.data.data_type` for request validation
+3. Return appropriate `responses.JSONResponse` or `responses.PlainTextResponse`
+4. Use proper HTTP status codes from `fastapi.status`
+5. Use type hints for all parameters and return types
+6. Handle errors gracefully with informative error messages
+7. Use kebab-case for route paths: `/calc-tensor-by-pkl`, `/re_embed_records`
+8. For async operations, use `daemon.run_without_db_token()` to run in background
diff --git a/.cursor/rules/guidelines.mdc b/.cursor/rules/guidelines.mdc
new file mode 100644
index 0000000..f3a2496
--- /dev/null
+++ b/.cursor/rules/guidelines.mdc
@@ -0,0 +1,77 @@
+---
+description: Main guidelines and architecture overview for refinery-embedder
+alwaysApply: true
+---
+
+# Refinery Embedder Guidelines
+
+Embedder service guidelines ensuring consistency across API routes, controllers, middleware, and related code.
+
+## Overview
+
+The `refinery-embedder` is a FastAPI-based microservice that manages the creation of document- and token-level embeddings for the Refinery platform. It handles embedding generation using various transformer models (HuggingFace, OpenAI, Azure, etc.) and manages embedding lifecycle operations including creation, updates, deletion, and tensor calculations.
+
+## Architecture
+
+The embedder follows a simplified layered architecture:
+
+- **FastAPI Routes** (`app.py`): HTTP endpoint definitions and request/response handling
+- **Controller** (`controller.py`): Business logic for embedding operations and orchestration
+- **Middleware** (`app.py`): Inline database session management middleware
+- **Submodules** (`submodules/`): Shared modules for models and S3 operations
+- **Utilities** (`src/util/`): Helper functions for embedders, notifications, request handling, and decorators
+- **Embedders** (`src/embedders/`): Embedding model implementations (classification, extraction, reduction)
+- **Data Models** (`src/data/data_type.py`): Pydantic models for request/response validation
+
+## Guideline Files
+
+- **[submodules.mdc](./submodules.mdc)** - Rules for working with submodules
+- **[fastapi-routes.mdc](./fastapi-routes.mdc)** - Rules for FastAPI route definitions
+- **[controllers.mdc](./controllers.mdc)** - Rules for controller modules and business logic
+- **[middleware.mdc](./middleware.mdc)** - Rules for middleware and request processing
+- **[exceptions.mdc](./exceptions.mdc)** - Rules for exception handling and custom exceptions
+- **[api-models.mdc](./api-models.mdc)** - Rules for Pydantic models and request/response validation
+- **[utilities.mdc](./utilities.mdc)** - Rules for utility functions and helpers
+
+## General Principles
+
+1. **Separation of Concerns**: Routes handle HTTP, controllers contain business logic, submodules handle data access
+2. **Type Safety**: Use type hints consistently, leverage Pydantic for validation
+3. **Consistency**: Follow established naming conventions and patterns
+4. **Security**: Always validate authentication and authorization, sanitize inputs
+5. **Error Handling**: Use appropriate HTTP status codes and consistent error responses
+6. **Submodule Integration**: Always check submodule rules before making changes to submodule code
+
+## Directory Structure
+
+```
+refinery-embedder/
+├── app.py                 # FastAPI application with routes and middleware
+├── controller.py           # Business logic for embedding operations
+├── src/
+│   ├── data/
+│   │   └── data_type.py   # Pydantic models for request validation
+│   ├── embedders/         # Embedding model implementations
+│   │   ├── classification/ # Attribute-level embedders
+│   │   ├── extraction/     # Token-level embedders
+│   │   └── enums.py        # Embedding enums
+│   └── util/               # Utility functions
+│       ├── embedders.py   # Embedder factory functions
+│       ├── request_util.py # External API communication
+│       ├── notification.py # Project update notifications
+│       └── decorator.py   # Throttling decorators
+├── submodules/            # Git submodules
+│   ├── model/             # Database models and business objects
+│   └── s3/                # S3 storage operations
+└── requirements.txt       # Python dependencies
+```
+
+## Quick Reference
+
+**Response Patterns**: Use `responses.JSONResponse` for data responses, `responses.PlainTextResponse` for status responses. Return appropriate HTTP status codes from `fastapi.status`.
+
+**Database**: Sessions managed by inline middleware in `app.py`. Use submodule business objects (`submodules.model.business_objects`), never create SQLAlchemy sessions directly.
+
+**Embedding Operations**: Use `daemon.run_without_db_token()` for async embedding operations. Manage embedding state through `embedding.update_embedding_state_*` functions.
+
+**Submodules**: Always check `submodules/{name}/.cursor/rules/` before modifying submodule code.
diff --git a/.cursor/rules/middleware.mdc b/.cursor/rules/middleware.mdc
new file mode 100644
index 0000000..ecc96e5
--- /dev/null
+++ b/.cursor/rules/middleware.mdc
@@ -0,0 +1,86 @@
+---
+description: Rules for middleware and request processing
+globs: ["app.py"]
+alwaysApply: true
+---
+
+# Middleware Guidelines
+
+Middleware handles cross-cutting concerns: database session management. Middleware is defined inline in `app.py`.
+
+## Database Session Management
+
+Sessions are managed automatically by inline middleware in `app.py`. Never create SQLAlchemy sessions directly.
+
+**Middleware pattern:**
+```python
+from submodules.model.business_objects import general
+from fastapi import Request
+
+@app.middleware("http")
+async def handle_db_session(request: Request, call_next):
+    session_token = general.get_ctx_token()
+    request.state.session_token = session_token
+    try:
+        response = await call_next(request)
+    finally:
+        general.remove_and_refresh_session(session_token)
+    return response
+```
+
+## Request State
+
+Middleware sets `request.state` with session token:
+
+```python
+from fastapi import Request
+
+@app.post("/endpoint")
+def my_endpoint(request: Request):
+    session_token = request.state.session_token  # Set by middleware
+    # Use session_token if needed for operations
+```
+
+## Session Management in Controllers
+
+For background operations, manage sessions explicitly:
+
+```python
+from submodules.model.business_objects import general
+
+def prepare_run(project_id: str, embedding_id: str) -> None:
+    session_token = general.get_ctx_token()
+    try:
+        # Operations that need database access
+        t = __prepare_encoding(project_id, embedding_id)
+    finally:
+        general.remove_and_refresh_session(session_token)
+    if t:
+        run_encoding(*t)
+```
+
+## Error Handling
+
+Middleware should not catch exceptions - let them propagate to FastAPI's error handling:
+
+```python
+@app.middleware("http")
+async def handle_db_session(request: Request, call_next):
+    session_token = general.get_ctx_token()
+    request.state.session_token = session_token
+    try:
+        response = await call_next(request)
+    finally:
+        # Always cleanup session, even if exception occurs
+        general.remove_and_refresh_session(session_token)
+    return response
+```
+
+## Best Practices
+
+1. Never create SQLAlchemy sessions directly - use `general.get_ctx_token()` and `general.remove_and_refresh_session()`
+2. Always cleanup resources in `finally` blocks
+3. Use `request.state` to pass session token between middleware and routes if needed
+4. Keep middleware lightweight - only handle session management
+5. For background operations, use `daemon.run_without_db_token()` and manage sessions explicitly
+6. Session cleanup is critical - always use `finally` blocks
diff --git a/.cursor/rules/submodules.mdc b/.cursor/rules/submodules.mdc
new file mode 100644
index 0000000..ae46793
--- /dev/null
+++ b/.cursor/rules/submodules.mdc
@@ -0,0 +1,66 @@
+---
+description: Rules for working with Git submodules (model and s3)
+alwaysApply: true
+---
+
+# Submodules Guidelines
+
+## Critical Rule
+
+**BEFORE modifying code in `submodules/`, check the submodule's `.cursor/rules/` directory first.**
+
+- `submodules/model/.cursor/rules/` - Database models, business objects, enums, database operations
+- `submodules/s3/.cursor/rules/` - S3 storage operations
+
+Do NOT duplicate submodule rules in parent repository rules.
+
+## Import Patterns
+
+**From model submodule:**
+```python
+from submodules.model.business_objects import project, record
+from submodules.model import enums, Project, Record
+from submodules.model.util import sql_alchemy_to_dict
+from submodules.model.exceptions import EntityNotFoundException
+from submodules.model.session_wrapper import run_db_async_with_session
+```
+
+**From S3 submodule:**
+```python
+from submodules.s3 import controller as s3
+s3.put_object(bucket, object_name, data)
+s3.get_object(bucket, object_name)
+```
+
+## When to Modify Submodules
+
+**Modify when**: Adding data layer logic, new database operations, new S3 operations.
+
+**Don't modify when**: Change belongs in embedder code (routes/controllers), just using existing functionality.
+
+## Database Schema Changes
+
+**When adding new tables or columns** in `submodules/model`:
+
+1. Make changes in `submodules/model/models.py` (add entity class or modify columns)
+2. Update `submodules/model/enums.py` if adding a new table (add to `Tablenames` enum)
+3. Create migration: `./db commit "Description of changes"` (must be run from `refinery-gateway` root)
+4. Apply migration: `./db migrate` (must be run from `refinery-gateway` root)
+
+**Important**: Database migrations MUST be done only from `refinery-gateway` root directory, not from within the submodule.
+
+**This is mandatory** - schema changes require both `commit` and `migrate` steps. See `submodules/model/.cursor/rules/entities.mdc` for detailed migration guidelines.
+
+## Common Patterns
+
+```python
+# Business objects
+project = project.get(project_id)
+new_project = project.create(name="...", organization_id=org_id, with_commit=True)
+project.update(project_id, {"name": "..."}, with_commit=True)
+
+# S3 operations
+if s3.bucket_exists(bucket_name):
+    s3.put_object(bucket_name, object_name, json_data)
+    url = s3.create_access_link(bucket_name, object_name)
+```
diff --git a/.cursor/rules/utilities.mdc b/.cursor/rules/utilities.mdc
new file mode 100644
index 0000000..5177693
--- /dev/null
+++ b/.cursor/rules/utilities.mdc
@@ -0,0 +1,79 @@
+---
+description: Rules for utility functions and helper modules
+globs: ["src/util/**/*.py"]
+alwaysApply: true
+---
+
+# Utilities Guidelines
+
+Utilities contain reusable, stateless helper functions organized in `src/util/`.
+
+## Common Utilities
+
+**Embedder Factory:**
+```python
+from src.util.embedders import get_embedder
+from submodules.model import enums
+
+embedder = get_embedder(
+    project_id=project_id,
+    embedding_type=enums.EmbeddingType.ON_ATTRIBUTE.value,
+    language_code=iso2_code,
+    platform=platform,
+    model=model,
+    api_token=api_token,
+    additional_data=additional_data,
+)
+```
+
+**Request Utilities (External API Communication):**
+```python
+from src.util import request_util
+
+# Neural search integration
+request_util.post_embedding_to_neural_search(project_id, embedding_id)
+request_util.delete_embedding_from_neural_search(embedding_id)
+
+# Model provider integration
+config_string = request_util.get_model_path(model_name)
+```
+
+**Notifications:**
+```python
+from src.util.notification import send_project_update
+
+# Send project update via websocket
+send_project_update(project_id, f"embedding:{embedding_id}:state:{state}")
+send_project_update(project_id, f"embedding:{embedding_id}:progress:{progress}")
+
+# Global updates
+send_project_update(project_id, message, is_global=True)
+```
+
+**Decorators:**
+```python
+from src.util.decorator import param_throttle
+
+@param_throttle(seconds=5)
+def send_progress_update_throttle(
+    project_id: str, embedding_id: str, state: str, initial_count: int
+) -> None:
+    # Function will only execute once per project_id every 5 seconds
+    pass
+```
+
+## Best Practices
+
+1. Keep utilities stateless and reusable
+2. Use type hints for all functions
+3. Document with docstrings
+4. Single responsibility per function
+5. Avoid side effects when possible (except for external API calls)
+6. Make functions easily testable
+7. Use decorators for cross-cutting concerns like throttling
+
+## Utility vs Controller vs Route
+
+- **Utilities** (`src/util/`): Pure transformations, external API communication, embedder factories, decorators, notification helpers
+- **Controllers** (`controller.py`): Business logic requiring database access, embedding orchestration, state management
+- **Routes** (`app.py`): HTTP handling, input validation, response formatting
diff --git a/submodules/model b/submodules/model
index c996e22..eefe59f 160000
--- a/submodules/model
+++ b/submodules/model
@@ -1 +1 @@
-Subproject commit c996e22c1ec38ef43a46b495aa1e450a8383df50
+Subproject commit eefe59f48ba8a92c82ace65427e141f1dad8586a
diff --git a/submodules/s3 b/submodules/s3
index f8ac7e0..45ffb4e 160000
--- a/submodules/s3
+++ b/submodules/s3
@@ -1 +1 @@
-Subproject commit f8ac7e0d76d5b019e5a9ff8dcbe50747fc9dee15
+Subproject commit 45ffb4ec747860c913715cbd436b8b4ab3ad0bf0