diff --git a/DashAI/alembic/versions/k4l5m6n7o8p9_scope_documents_to_session.py b/DashAI/alembic/versions/k4l5m6n7o8p9_scope_documents_to_session.py new file mode 100644 index 000000000..bca1fdc3c --- /dev/null +++ b/DashAI/alembic/versions/k4l5m6n7o8p9_scope_documents_to_session.py @@ -0,0 +1,272 @@ +"""scope RAG documents to a single session + +Gives ``document`` a ``session_id`` foreign key, so a document belongs to +exactly one RAG session instead of being a globally deduplicated library +entry. ``UNIQUE(file_hash)`` becomes ``UNIQUE(session_id, file_hash)``: the +same file uploaded into two sessions is now two rows, each free to pick its +own extractor without disturbing the other. + +Also drops ``rag_document_pipeline_session_link``, which nothing in +production ever wrote to. + +This migration performs **no filesystem I/O**. Existing rows keep their +current ``file_path``, and cloned rows deliberately share the path of the +original; only uploads made after this migration use the content-addressed +``blobs/`` layout. Deletion is reference-counted by ``file_path``, so a shared +path is only unlinked once the last row pointing at it is gone. + +Revision ID: k4l5m6n7o8p9 +Revises: b7c1d4e9f206 +Create Date: 2026-09-03 +""" + +import json +import logging +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "k4l5m6n7o8p9" +down_revision: Union[str, None] = "b7c1d4e9f206" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +log = logging.getLogger("alembic.runtime.migration") + +#: Tables holding rows that reference a document, in deletion order. +_DEPENDENT_TABLES = ( + "chunk", + "rag_embedding_matrix", + "rag_chunk_set_document", + "processed_document_content", + "rag_document_pipeline_session_link", +) + + +def _load_parameters(raw) -> dict: + """Return a session's ``parameters`` as a dict, whatever the driver gave us.""" + if isinstance(raw, dict): + return raw + if not raw: + return {} + try: + parsed = json.loads(raw) + except (TypeError, ValueError): + return {} + return parsed if isinstance(parsed, dict) else {} + + +def _document_owners(conn) -> dict: + """Map each document id to the RAG sessions claiming it, lowest id first. + + The pre-migration link between a document and a session is the JSON list + ``generative_session.parameters["documents"]``. + """ + owners: dict = {} + rows = conn.execute( + sa.text( + "SELECT id, parameters FROM generative_session " + "WHERE task_name = 'RAGTask' ORDER BY id" + ) + ).fetchall() + for session_id, raw in rows: + for doc_id in _load_parameters(raw).get("documents") or []: + if isinstance(doc_id, bool) or not isinstance(doc_id, int): + continue + owners.setdefault(doc_id, []).append(session_id) + return owners + + +def _delete_documents(conn, doc_ids) -> None: + """Delete documents and every row referencing them.""" + if not doc_ids: + return + present = set(sa.inspect(conn).get_table_names()) + for doc_id in doc_ids: + for table in _DEPENDENT_TABLES: + if table in present: + conn.execute( + sa.text("DELETE FROM " + table + " WHERE document_id = :did"), + {"did": doc_id}, + ) + conn.execute(sa.text("DELETE FROM document WHERE id = :did"), {"did": doc_id}) + + +def _clone_document(conn, doc_id: int, session_id: int) -> int: + """Copy a document row (and its cached text) over to another session. + + The clone reuses the original's ``extractor_id`` -- ``rag_extractor`` rows + are immutable value objects -- and its ``file_path``, since the bytes on + disk are identical. Chunks, embeddings and chunk-set membership are + deliberately *not* copied: changing a session's document list changes its + chunk-set signature, so the session re-indexes by itself on its next + message. + """ + conn.execute( + sa.text( + "INSERT INTO document (session_id, file_name, file_type, file_path, " + "file_hash, optional_metadata, extractor_id, created, last_modified) " + "SELECT :sid, file_name, file_type, file_path, file_hash, " + "optional_metadata, extractor_id, created, last_modified " + "FROM document WHERE id = :did" + ), + {"sid": session_id, "did": doc_id}, + ) + new_id = conn.execute(sa.text("SELECT last_insert_rowid()")).scalar() + conn.execute( + sa.text( + "INSERT INTO processed_document_content " + "(document_id, content, signature, char_count) " + "SELECT :new_id, content, signature, char_count " + "FROM processed_document_content WHERE document_id = :did" + ), + {"new_id": new_id, "did": doc_id}, + ) + return new_id + + +def _replace_in_session_documents( + conn, session_id: int, old_id: int, new_id: int +) -> None: + """Point a session's ``documents`` list at its own clone.""" + raw = conn.execute( + sa.text("SELECT parameters FROM generative_session WHERE id = :sid"), + {"sid": session_id}, + ).scalar() + parameters = _load_parameters(raw) + parameters["documents"] = [ + new_id if doc_id == old_id else doc_id + for doc_id in parameters.get("documents") or [] + ] + conn.execute( + sa.text("UPDATE generative_session SET parameters = :params WHERE id = :sid"), + {"params": json.dumps(parameters), "sid": session_id}, + ) + + +def upgrade() -> None: + conn = op.get_bind() + + # The global UNIQUE(file_hash) has to go before the backfill, not after: + # splitting a shared document into one row per session inserts rows that + # deliberately repeat a hash. + with op.batch_alter_table("document", schema=None) as batch_op: + batch_op.add_column(sa.Column("session_id", sa.Integer(), nullable=True)) + batch_op.drop_constraint("uq_document_file_hash", type_="unique") + + owners = _document_owners(conn) + all_doc_ids = { + row[0] for row in conn.execute(sa.text("SELECT id FROM document")).fetchall() + } + + # Orphans: with the global documents page gone these are unreachable + # forever, and a nullable session_id would defeat the whole invariant. + orphans = sorted(doc_id for doc_id in all_doc_ids if doc_id not in owners) + if orphans: + abandoned = conn.execute( + sa.text("SELECT id, file_path FROM document WHERE session_id IS NULL") + ).fetchall() + log.info( + "Deleting %d RAG document(s) that no session references. Their files " + "are left on disk for manual cleanup: %s", + len(orphans), + ", ".join( + "#%s %s" % (doc_id, path) + for doc_id, path in abandoned + if doc_id in set(orphans) + ), + ) + _delete_documents(conn, orphans) + + for doc_id, session_ids in sorted(owners.items()): + if doc_id not in all_doc_ids: + continue # stale id left behind in a session's parameters + conn.execute( + sa.text("UPDATE document SET session_id = :sid WHERE id = :did"), + {"sid": session_ids[0], "did": doc_id}, + ) + for extra_session_id in session_ids[1:]: + new_id = _clone_document(conn, doc_id, extra_session_id) + _replace_in_session_documents(conn, extra_session_id, doc_id, new_id) + + # Anything still unclaimed (e.g. a document whose session was deleted + # without its parameters being cleaned up) has nothing left to belong to. + _delete_documents( + conn, + [ + row[0] + for row in conn.execute( + sa.text("SELECT id FROM document WHERE session_id IS NULL") + ).fetchall() + ], + ) + + # upload() wrote params={} while update_extractor() wrote NULL, which is one + # reason extractor rows could never be deduplicated. Settle on {}. + conn.execute(sa.text("UPDATE rag_extractor SET params = '{}' WHERE params IS NULL")) + + with op.batch_alter_table("document", schema=None) as batch_op: + batch_op.alter_column("session_id", existing_type=sa.Integer(), nullable=False) + batch_op.create_foreign_key( + "fk_document_session_id_generative_session", + "generative_session", + ["session_id"], + ["id"], + ondelete="CASCADE", + ) + batch_op.create_unique_constraint( + "uq_document_session_file_hash", ["session_id", "file_hash"] + ) + + if "rag_document_pipeline_session_link" in sa.inspect(conn).get_table_names(): + op.drop_table("rag_document_pipeline_session_link") + + +def downgrade() -> None: + """Restore the global document library. + + Lossy: ``UNIQUE(file_hash)`` cannot be restored while per-session copies of + the same file exist, so every copy but the lowest-id one is deleted. + ``rag_document_pipeline_session_link`` comes back empty, which is the only + state it was ever in. + """ + conn = op.get_bind() + + duplicates = [ + row[0] + for row in conn.execute( + sa.text( + "SELECT id FROM document WHERE id NOT IN " + "(SELECT MIN(id) FROM document GROUP BY file_hash)" + ) + ).fetchall() + ] + _delete_documents(conn, duplicates) + + with op.batch_alter_table("document", schema=None) as batch_op: + batch_op.drop_constraint("uq_document_session_file_hash", type_="unique") + batch_op.drop_constraint( + "fk_document_session_id_generative_session", type_="foreignkey" + ) + batch_op.drop_column("session_id") + batch_op.create_unique_constraint("uq_document_file_hash", ["file_hash"]) + + op.create_table( + "rag_document_pipeline_session_link", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("document_id", sa.Integer(), nullable=False), + sa.Column("session_id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["document_id"], ["document.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["session_id"], ["generative_session.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["pipeline_id"], ["rag_pipeline.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("document_id", "session_id", name="uix_document_session"), + sa.UniqueConstraint("session_id", "pipeline_id", name="uix_session_pipeline"), + sa.UniqueConstraint("document_id", "pipeline_id", name="uix_document_pipeline"), + ) diff --git a/DashAI/alembic/versions/l5m6n7o8p9q0_add_index_job_id_to_generative_session.py b/DashAI/alembic/versions/l5m6n7o8p9q0_add_index_job_id_to_generative_session.py new file mode 100644 index 000000000..85bb03b33 --- /dev/null +++ b/DashAI/alembic/versions/l5m6n7o8p9q0_add_index_job_id_to_generative_session.py @@ -0,0 +1,36 @@ +"""track a RAG session's in-flight indexing job + +Indexing used to happen inside the chat job, so there was nothing to track: the +chat process *was* the index run. Now that a session can be indexed eagerly by +a job of its own, the session needs to point at that job so the API can tell +"already indexing" from "not indexed yet", coalesce duplicate requests, and +cancel the run before invalidating what it is writing. + +``index_job_id`` is a pointer, never the truth. The job queue's ``task_copy`` +table stays authoritative for whether the job is alive; a stale pointer simply +resolves to nothing and is overwritten by the next indexing request. + +Revision ID: l5m6n7o8p9q0 +Revises: k4l5m6n7o8p9 +Create Date: 2026-09-09 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "l5m6n7o8p9q0" +down_revision: Union[str, None] = "k4l5m6n7o8p9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table("generative_session") as batch_op: + batch_op.add_column(sa.Column("index_job_id", sa.String(), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table("generative_session") as batch_op: + batch_op.drop_column("index_job_id") diff --git a/DashAI/alembic/versions/m6n7o8p9q0r1_merge_rag_index_and_develop_heads.py b/DashAI/alembic/versions/m6n7o8p9q0r1_merge_rag_index_and_develop_heads.py new file mode 100644 index 000000000..4fc934d92 --- /dev/null +++ b/DashAI/alembic/versions/m6n7o8p9q0r1_merge_rag_index_and_develop_heads.py @@ -0,0 +1,27 @@ +"""merge the RAG indexing head with develop + +``feat/rag-uiv2`` and ``develop`` both branch from ``b7c1d4e9f206``: the RAG +line scoped documents to a session and added the indexing job pointer, while +develop added the report table and the prediction split. Neither side touches +the other's tables, so this is an empty merge point whose only job is to give +Alembic a single head again. + +Revision ID: m6n7o8p9q0r1 +Revises: l5m6n7o8p9q0, b7e4d2a19c63 +Create Date: 2026-09-09 +""" + +from typing import Sequence, Union + +revision: str = "m6n7o8p9q0r1" +down_revision: Union[str, Sequence[str], None] = ("l5m6n7o8p9q0", "b7e4d2a19c63") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/DashAI/back/api/api_v1/endpoints/documents.py b/DashAI/back/api/api_v1/endpoints/documents.py index 8b2ffa5da..668efaacf 100644 --- a/DashAI/back/api/api_v1/endpoints/documents.py +++ b/DashAI/back/api/api_v1/endpoints/documents.py @@ -18,18 +18,38 @@ from kink import di from sqlalchemy.orm import sessionmaker -from DashAI.back.api.api_v1.schemas import DocumentResponse +from DashAI.back.api.api_v1.schemas import ( + DocumentResponse, + UpdateExtractorRequest, +) +from DashAI.back.dependencies.database.models import Document, GenerativeSession from DashAI.back.models.RAG.documents import DocumentFileType from DashAI.back.models.RAG.exceptions import ( RAGDocumentExtractionError, RAGDocumentFileTypeError, ) from DashAI.back.services.RAG.document_service import DocumentService +from DashAI.back.services.RAG.index_job_service import cancel_live_index_job router = APIRouter() log = logging.getLogger(__name__) +def _cancel_index_for_document(db, document_id: int) -> None: + """Stop a running index before changing what it is indexing. + + Dropping a document or re-extracting its text deletes the very chunk, + retriever and embedding rows a running job is writing, so the two must not + overlap. The caller's transaction commits the cleared pointer. + """ + document = db.get(Document, document_id) + if document is None or document.session_id is None: + return + session = db.get(GenerativeSession, document.session_id) + if session is not None: + cancel_live_index_job(session, di["job_queue"]) + + base_url = "/api/v1/document" DISPOSITION_ATTACHMENT = "attachment" @@ -101,17 +121,6 @@ def _serve_document( ) from e -@router.get("/", response_model=List[DocumentResponse]) -async def get_all_documents( - request: Request, - session_factory: sessionmaker = Depends(lambda: di["session_factory"]), -): - """Get all documents with file_url included.""" - with session_factory() as db: - base = str(request.base_url).rstrip("/") - return DocumentService(db).get_all(base_url=base) - - @router.get("/{document_id}", response_model=DocumentResponse) async def get_document( document_id: int, @@ -146,22 +155,24 @@ async def view_document( return _serve_document(document_id, DISPOSITION_INLINE, session_factory) -@router.post("/", response_model=DocumentResponse, status_code=status.HTTP_201_CREATED) +@router.post( + "/session/{session_id}", + response_model=DocumentResponse, + status_code=status.HTTP_201_CREATED, +) async def upload_document( + session_id: int, file: UploadFile = File(...), metadata: str = Form(...), - force: bool = False, - response: Response = None, config: Dict[str, Any] = Depends(lambda: di["config"]), session_factory: sessionmaker = Depends(lambda: di["session_factory"]), ): - """Upload a new document to the RAG system with file content and metadata. + """Upload a document into one RAG session. - If a document with the same content hash already exists and ``force`` is - ``False``, returns ``409 Conflict`` with the existing document and the - affected sessions so the client can ask for confirmation. With - ``force=True`` the existing document is overwritten and its RAG artifacts - are invalidated. Extraction failures are surfaced as ``500``. + Documents belong to exactly one session, so the same file can be uploaded + into several sessions independently. Uploading it twice into the *same* + session changes nothing and returns ``409 Conflict`` with the existing + document. Extraction failures are surfaced as ``500``. """ from DashAI.back.dependencies.registry.component_registry import ComponentRegistry @@ -213,9 +224,9 @@ async def upload_document( file_name, file_type, str(docs_folder_path), + session_id, optional_metadata, registry=registry, - force=force, ) except RAGDocumentExtractionError as e: raise HTTPException( @@ -230,14 +241,11 @@ async def upload_document( raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail={ - "detail": "Document already exists", + "detail": "This session already has this document", "existing_document": result.document.model_dump(mode="json"), - "affected_sessions": result.affected_sessions, }, ) - if result.updated and response is not None: - response.status_code = status.HTTP_200_OK return result.document @@ -258,21 +266,6 @@ async def get_documents_by_session( ) from e -@router.get("/related-sessions/{document_id}", response_model=List[int]) -async def get_related_sessions( - document_id: int, - session_factory: sessionmaker = Depends(lambda: di["session_factory"]), -): - """Get all generative session IDs related to a specific document.""" - with session_factory() as db: - try: - return DocumentService(db).get_related_sessions(document_id) - except ValueError as e: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail=str(e) - ) from e - - @router.delete("/{document_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_document( document_id: int, @@ -282,6 +275,7 @@ async def delete_document( with session_factory() as db: try: + _cancel_index_for_document(db, document_id) DocumentService(db).delete(document_id) return Response(status_code=status.HTTP_204_NO_CONTENT) except ValueError as e: @@ -375,78 +369,36 @@ async def extract_document_text( ) from e -@router.put("/{document_id}/extractor") +@router.put("/{document_id}/extractor", response_model=DocumentResponse) async def update_document_extractor( document_id: int, - request: Request, - config: Dict[str, Any] = Depends(lambda: di["config"]), + body: UpdateExtractorRequest, session_factory: sessionmaker = Depends(lambda: di["session_factory"]), ): """Commit an extractor choice for a document. - Request body: - {"extractor": {"component": "PyMuPDFExtractor", "params": {}}, "force": false} - - If the document is linked to RAG pipelines and force=false, returns 409 - Conflict with affected session info. With force=true, artifacts are - invalidated. + Re-extracts the text and drops the chunks, retrievers and embeddings + fitted over the previous extraction, as one transaction: if extraction + fails, nothing changes and the error is reported as ``422``. """ from DashAI.back.dependencies.registry.component_registry import ComponentRegistry registry: ComponentRegistry = di["component_registry"] - try: - body = await request.json() - except Exception as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Invalid JSON body", - ) from e - - extractor_ref = body.get("extractor") - force = body.get("force", False) - - if not extractor_ref or not isinstance(extractor_ref, dict): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Missing or invalid 'extractor' in request body. " - "Expected {component: str, params: dict}.", - ) - with session_factory() as db: try: - result = DocumentService(db, registry).update_extractor( + _cancel_index_for_document(db, document_id) + return DocumentService(db, registry).update_extractor( document_id, - extractor_ref=extractor_ref, - force=force, + extractor_ref=body.extractor.model_dump(), ) - return result + except RAGDocumentExtractionError as e: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e) + ) from e except ValueError as e: msg = str(e) - if "linked to" in msg and "RAG pipeline" in msg: - linked_ids = DocumentService(db).get_related_sessions(document_id) - from DashAI.back.dependencies.database.models import GenerativeSession - - affected_sessions = [] - if linked_ids: - sessions = ( - db.query(GenerativeSession) - .filter(GenerativeSession.id.in_(linked_ids)) - .all() - ) - affected_sessions = [{"id": s.id, "name": s.name} for s in sessions] - raise HTTPException( - status_code=409, - detail={ - "detail": msg, - "affected_sessions": affected_sessions, - }, - ) from e - if "does not support" in msg: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, detail=msg - ) from e - if "not found in registry" in msg: + if "does not support" in msg or "not found in registry" in msg: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=msg ) from e diff --git a/DashAI/back/api/api_v1/endpoints/generative_process.py b/DashAI/back/api/api_v1/endpoints/generative_process.py index 58cc3cf1a..94aaa8f03 100644 --- a/DashAI/back/api/api_v1/endpoints/generative_process.py +++ b/DashAI/back/api/api_v1/endpoints/generative_process.py @@ -1,332 +1,350 @@ -import logging -from typing import TYPE_CHECKING, Any, Dict - -from fastapi import APIRouter, Depends, Form, HTTPException, Request, status -from fastapi.responses import FileResponse -from kink import di -from sqlalchemy import exc -from starlette.datastructures import UploadFile -from typing_extensions import Annotated - -from DashAI.back.dependencies.database.models import ( - GenerativeProcess, - GenerativeSession, - ProcessData, -) - -if TYPE_CHECKING: - from sqlalchemy.orm import sessionmaker - - from DashAI.back.dependencies.registry import ComponentRegistry - from DashAI.back.tasks.base_generative_task import BaseGenerativeTask - -router = APIRouter() -log = logging.getLogger(__name__) - - -@router.post("/", status_code=status.HTTP_201_CREATED) -async def upload_generative_process( - request: Request, - session_id: Annotated[int, Form(...)], - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), - config: Dict[str, Any] = Depends(lambda: di["config"]), -): - """Create a new generative session. - - Parameters - ---------- - request : Request - The incoming HTTP request containing form data and files. - session_id : int - The ID of the generative session to which this process belongs. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - The generated session can be used to access and query the database. - config : Dict[str, Any] - A dictionary containing configuration settings, including the path for images. - - Returns - ------- - dict - A dictionary with the new generative session on the database - and the input/output data. - - Raises - ------ - HTTPException - If there's an internal database error or if the session ID does not exist. - """ - form = await request.form() - input_items = [] - - # Filter and sort only indexed keys like 'text_0', 'file_1' - indexed_keys = [key for key in form if "_" in key and key.split("_")[1].isdigit()] - for key in sorted(indexed_keys, key=lambda x: int(x.split("_")[1])): - value = form[key] - if isinstance(value, UploadFile): - content = await value.read() - input_items.append(content) # raw image bytes - else: - input_items.append(str(value)) # text string - - with session_factory() as db: - try: - session = db.query(GenerativeSession).filter_by(id=session_id).first() - if not session: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Session with ID {session_id} does not exist.", - ) - - task: "BaseGenerativeTask" = di["component_registry"][session.task_name][ - "class" - ]() - - process = GenerativeProcess( - session_id=session_id, - ) - db.add(process) - db.commit() - db.refresh(process) - - processed_input = task.prepare_input_for_database( - input_items, images_path=config["IMAGES_PATH"] - ) - - processed_data = [] - for data in processed_input: - input_data = ProcessData( - data=data[0], - data_type=data[1], - is_input=True, - process_id=process.id, - ) - processed_data.append(input_data) - db.add_all(processed_data) - db.commit() - db.refresh(process) - - process = process.__dict__ - - process["input"] = task.process_input_from_database(process["input"]) - - return process - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - - -@router.get("/{process_id}", status_code=status.HTTP_200_OK, response_model=None) -async def get_generative_process( - process_id: int, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), - component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), -): - """Get a generative process by its session ID. - - Parameters - ---------- - process_id : str - The ID of the generative process to retrieve. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - The generated session can be used to access and query the database. - - Returns - ------- - dict - A dictionary with the generative process data. - - Raises - ------ - HTTPException - If the generative process is not found or if there's an internal database error. - """ - with session_factory() as db: - try: - process = db.query(GenerativeProcess).filter_by(id=process_id).all() - if not process: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Generative process with ID {process_id} does not exist.", - ) - generative_session: GenerativeSession = db.get( - GenerativeSession, process[0].session_id - ) - - task: "BaseGenerativeTask" = component_registry[ - generative_session.task_name - ]["class"]() - - process = [p.__dict__ for p in process] - - process = [ - { - **p, - "input": task.process_input_from_database(p["input"]), - "output": task.process_output_from_database(p["output"]), - } - for p in process - ] - - return process[0] - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - - -@router.delete( - "/{process_id}", status_code=status.HTTP_204_NO_CONTENT, response_model=None -) -async def delete_generative_process( - process_id: int, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), -): - """Delete a generative process by its ID. - - Parameters - ---------- - process_id : str - The ID of the generative process to delete. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - The generated session can be used to access and query the database. - - Returns - ------- - None - - Raises - ------ - HTTPException - If the generative process is not found or if there's an internal database error. - """ - with session_factory() as db: - try: - process = db.query(GenerativeProcess).filter_by(id=process_id).first() - if not process: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Generative process with ID {process_id} does not exist.", - ) - # Delete all associated input and output data - db.query(ProcessData).filter_by(process_id=process.id).delete() - # Delete the generative process itself - db.delete(process) - # Commit the changes to the database - db.commit() - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - - -@router.get( - "/session/{session_id}", status_code=status.HTTP_200_OK, response_model=None -) -async def get_generative_process_by_session_id( - session_id: int, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), - component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), -): - """Get a generative process by its session ID. - - Parameters - ---------- - session_id : str - The ID of the generative process to retrieve. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - The generated session can be used to access and query the database. - - Returns - ------- - dict - A dictionary with the generative process data. - - Raises - ------ - HTTPException - If the generative process is not found or if there's an internal database error. - """ - - with session_factory() as db: - try: - process = db.query(GenerativeProcess).filter_by(session_id=session_id).all() - generative_session: GenerativeSession = db.get( - GenerativeSession, session_id - ) - - task: "BaseGenerativeTask" = component_registry[ - generative_session.task_name - ]["class"]() - - process = [p.__dict__ for p in process] - - process = [ - { - **p, - "input": task.process_input_from_database(p["input"]), - "output": task.process_output_from_database(p["output"]), - } - for p in process - ] - - return process - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - - -@router.get("/file/{filename}", status_code=200, response_model=None) -async def get_generative_file( - filename: str, - config: Dict[str, Any] = Depends(lambda: di["config"]), -): - """Serve a generated media file with an auto-detected mime type. - - Handles every modality produced by generative tasks (image, audio, - video) by resolving the file under ``IMAGES_PATH`` and returning it - with the mime type guessed from its extension. Falls back to - ``application/octet-stream`` when the extension is unknown. - - Parameters - ---------- - filename : str - The relative path or filename of the file to retrieve inside - ``IMAGES_PATH``. - config : Dict[str, Any] - Application configuration container; must expose - ``IMAGES_PATH`` (injected via ``kink``). - - Returns - ------- - FileResponse - The file served with a guessed mime type. - - Raises - ------ - HTTPException - 404 if no file exists at the resolved path. - """ - import mimetypes - import os - - file_path = os.path.join(config["IMAGES_PATH"], filename) - - if not os.path.exists(file_path): - raise HTTPException(status_code=404, detail="File not found") - - media_type, _ = mimetypes.guess_type(file_path) - return FileResponse(file_path, media_type=media_type or "application/octet-stream") +import logging +from typing import TYPE_CHECKING, Any, Dict + +from fastapi import APIRouter, Depends, Form, HTTPException, Request, status +from fastapi.responses import FileResponse +from kink import di +from sqlalchemy import exc +from starlette.datastructures import UploadFile +from typing_extensions import Annotated + +from DashAI.back.dependencies.database.models import ( + GenerativeProcess, + GenerativeSession, + ProcessData, +) +from DashAI.back.models.RAG.RAG_constants import RAG_PARAM_DOCUMENTS + +if TYPE_CHECKING: + from sqlalchemy.orm import sessionmaker + + from DashAI.back.dependencies.registry import ComponentRegistry + from DashAI.back.tasks.base_generative_task import BaseGenerativeTask + +router = APIRouter() +log = logging.getLogger(__name__) + +#: Only RAG sessions require documents before they can answer. +_RAG_TASK_NAME = "RAGTask" + + +@router.post("/", status_code=status.HTTP_201_CREATED) +async def upload_generative_process( + request: Request, + session_id: Annotated[int, Form(...)], + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), + config: Dict[str, Any] = Depends(lambda: di["config"]), +): + """Create a new generative session. + + Parameters + ---------- + request : Request + The incoming HTTP request containing form data and files. + session_id : int + The ID of the generative session to which this process belongs. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + config : Dict[str, Any] + A dictionary containing configuration settings, including the path for images. + + Returns + ------- + dict + A dictionary with the new generative session on the database + and the input/output data. + + Raises + ------ + HTTPException + If there's an internal database error or if the session ID does not exist. + """ + form = await request.form() + input_items = [] + + # Filter and sort only indexed keys like 'text_0', 'file_1' + indexed_keys = [key for key in form if "_" in key and key.split("_")[1].isdigit()] + for key in sorted(indexed_keys, key=lambda x: int(x.split("_")[1])): + value = form[key] + if isinstance(value, UploadFile): + content = await value.read() + input_items.append(content) # raw image bytes + else: + input_items.append(str(value)) # text string + + with session_factory() as db: + try: + session = db.query(GenerativeSession).filter_by(id=session_id).first() + if not session: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Session with ID {session_id} does not exist.", + ) + + # A RAG session is created empty and gains documents as they are + # uploaded. Answer synchronously here rather than letting the job + # fail deep inside the retriever, fitting an index over no text. + if session.task_name == _RAG_TASK_NAME and not ( + (session.parameters or {}).get(RAG_PARAM_DOCUMENTS) or [] + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Add at least one document to this session before " + "sending a message." + ), + ) + + task: "BaseGenerativeTask" = di["component_registry"][session.task_name][ + "class" + ]() + + process = GenerativeProcess( + session_id=session_id, + ) + db.add(process) + db.commit() + db.refresh(process) + + processed_input = task.prepare_input_for_database( + input_items, images_path=config["IMAGES_PATH"] + ) + + processed_data = [] + for data in processed_input: + input_data = ProcessData( + data=data[0], + data_type=data[1], + is_input=True, + process_id=process.id, + ) + processed_data.append(input_data) + db.add_all(processed_data) + db.commit() + db.refresh(process) + + process = process.__dict__ + + process["input"] = task.process_input_from_database(process["input"]) + + return process + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.get("/{process_id}", status_code=status.HTTP_200_OK, response_model=None) +async def get_generative_process( + process_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), +): + """Get a generative process by its session ID. + + Parameters + ---------- + process_id : str + The ID of the generative process to retrieve. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + + Returns + ------- + dict + A dictionary with the generative process data. + + Raises + ------ + HTTPException + If the generative process is not found or if there's an internal database error. + """ + with session_factory() as db: + try: + process = db.query(GenerativeProcess).filter_by(id=process_id).all() + if not process: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Generative process with ID {process_id} does not exist.", + ) + generative_session: GenerativeSession = db.get( + GenerativeSession, process[0].session_id + ) + + task: "BaseGenerativeTask" = component_registry[ + generative_session.task_name + ]["class"]() + + process = [p.__dict__ for p in process] + + process = [ + { + **p, + "input": task.process_input_from_database(p["input"]), + "output": task.process_output_from_database(p["output"]), + } + for p in process + ] + + return process[0] + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.delete( + "/{process_id}", status_code=status.HTTP_204_NO_CONTENT, response_model=None +) +async def delete_generative_process( + process_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Delete a generative process by its ID. + + Parameters + ---------- + process_id : str + The ID of the generative process to delete. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + + Returns + ------- + None + + Raises + ------ + HTTPException + If the generative process is not found or if there's an internal database error. + """ + with session_factory() as db: + try: + process = db.query(GenerativeProcess).filter_by(id=process_id).first() + if not process: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Generative process with ID {process_id} does not exist.", + ) + # Delete all associated input and output data + db.query(ProcessData).filter_by(process_id=process.id).delete() + # Delete the generative process itself + db.delete(process) + # Commit the changes to the database + db.commit() + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.get( + "/session/{session_id}", status_code=status.HTTP_200_OK, response_model=None +) +async def get_generative_process_by_session_id( + session_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), +): + """Get a generative process by its session ID. + + Parameters + ---------- + session_id : str + The ID of the generative process to retrieve. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + + Returns + ------- + dict + A dictionary with the generative process data. + + Raises + ------ + HTTPException + If the generative process is not found or if there's an internal database error. + """ + + with session_factory() as db: + try: + process = db.query(GenerativeProcess).filter_by(session_id=session_id).all() + generative_session: GenerativeSession = db.get( + GenerativeSession, session_id + ) + + task: "BaseGenerativeTask" = component_registry[ + generative_session.task_name + ]["class"]() + + process = [p.__dict__ for p in process] + + process = [ + { + **p, + "input": task.process_input_from_database(p["input"]), + "output": task.process_output_from_database(p["output"]), + } + for p in process + ] + + return process + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.get("/file/{filename}", status_code=200, response_model=None) +async def get_generative_file( + filename: str, + config: Dict[str, Any] = Depends(lambda: di["config"]), +): + """Serve a generated media file with an auto-detected mime type. + + Handles every modality produced by generative tasks (image, audio, + video) by resolving the file under ``IMAGES_PATH`` and returning it + with the mime type guessed from its extension. Falls back to + ``application/octet-stream`` when the extension is unknown. + + Parameters + ---------- + filename : str + The relative path or filename of the file to retrieve inside + ``IMAGES_PATH``. + config : Dict[str, Any] + Application configuration container; must expose + ``IMAGES_PATH`` (injected via ``kink``). + + Returns + ------- + FileResponse + The file served with a guessed mime type. + + Raises + ------ + HTTPException + 404 if no file exists at the resolved path. + """ + import mimetypes + import os + + file_path = os.path.join(config["IMAGES_PATH"], filename) + + if not os.path.exists(file_path): + raise HTTPException(status_code=404, detail="File not found") + + media_type, _ = mimetypes.guess_type(file_path) + return FileResponse(file_path, media_type=media_type or "application/octet-stream") diff --git a/DashAI/back/api/api_v1/endpoints/generative_session.py b/DashAI/back/api/api_v1/endpoints/generative_session.py index 2d0eb1873..79e95e43f 100644 --- a/DashAI/back/api/api_v1/endpoints/generative_session.py +++ b/DashAI/back/api/api_v1/endpoints/generative_session.py @@ -1,846 +1,867 @@ -import logging -from datetime import datetime -from typing import TYPE_CHECKING, Union - -from fastapi import APIRouter, Depends, Header, HTTPException, status -from kink import di -from sqlalchemy import exc, select - -from DashAI.back.api.api_v1.schemas.generative_session_params import ( - GenerativeSessionBulkDeleteParams, - GenerativeSessionParams, -) -from DashAI.back.core.utils import localize -from DashAI.back.dependencies.database.models import ( - GenerativeProcess, - GenerativeSession, - GenerativeSessionParameterHistory, - ProcessData, -) -from DashAI.back.dependencies.downloads.nested import missing_downloads -from DashAI.back.models.base_generative_model import BaseGenerativeModel -from DashAI.back.models.RAG.exceptions.base import RAGWorkflowError -from DashAI.back.services.RAG.cleanup_service import CleanupService -from DashAI.back.services.RAG.session_validation_service import ( - SessionValidationService, -) -from DashAI.back.tasks.base_generative_task import BaseGenerativeTask -from DashAI.back.tasks.RAG_task import RAGTask - -if TYPE_CHECKING: - from sqlalchemy.orm import sessionmaker - - from DashAI.back.dependencies.registry import ComponentRegistry - - -router = APIRouter() -log = logging.getLogger(__name__) - - -@router.post("/", status_code=status.HTTP_201_CREATED) -async def upload_generative_session( - params: GenerativeSessionParams, - accept_language: str | None = Header(default=None), - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), - component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), -): - """Create a new generative session and log the initial parameters in the history.""" - - with session_factory() as db: - try: - # Check if the model is registered - if params.model_name not in component_registry: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Model {params.model_name} is not registered.", - ) - model_class = component_registry[params.model_name]["class"] - - # Guard: model requires download but has not been downloaded -> 409. - # Reconcile against the filesystem so a model downloaded after startup - # (in the worker process) is recognised without an API restart. - if getattr( - model_class, "REQUIRES_DOWNLOAD", False - ) and not component_registry.refresh_download_status(params.model_name): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Model {params.model_name} must be downloaded before use." - ), - ) - - # Check if the model is a subclass of GenerativeModel - if not issubclass(model_class, BaseGenerativeModel): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Model {params.model_name} is not a valid " - f"generative model.", - ) - - # Check if the task is registered - if params.task_name not in component_registry: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Task {params.task_name} is not registered.", - ) - task_class = component_registry[params.task_name]["class"] - - # RAG: validate and normalise RAG-specific parameters, filling in - # the components the caller did not choose. - if task_class == RAGTask: - try: - params.parameters = SessionValidationService( - db, component_registry - ).prepare_RAG_params(params.parameters, accept_language) - except (ValueError, RAGWorkflowError) as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(e), - ) from e - - # A parameter may select another component that itself needs - # downloading; block until every nested one is present. Runs after - # the RAG defaults are applied so it sees the final configuration. - nested_missing = missing_downloads(params.parameters, component_registry) - if nested_missing: - names = ", ".join(m["name"] for m in nested_missing) - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"These components must be downloaded before use: {names}." - ), - ) - - # Validate schema - try: - model_class.SCHEMA.model_validate(params.parameters) - except ValueError as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid parameters for model {params.model_name}: {e}", - ) from e - - # Check if the task is a subclass of BaseGenerativeTask - if not issubclass(task_class, BaseGenerativeTask): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Task {params.task_name} is not a valid generative task.", - ) - - now = datetime.now() - session = GenerativeSession( - model_name=params.model_name, - task_name=params.task_name, - parameters=params.parameters, - name=params.name, - description=params.description, - created=now, - last_modified=now, - ) - db.add(session) - try: - db.commit() - except exc.IntegrityError as e: - db.rollback() - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=( - f"Generative session with name '{params.name}' already exists." - ), - ) from e - db.refresh(session) - - session_params_entry = GenerativeSessionParameterHistory( - session_id=session.id, - parameters=session.parameters, - model_name=session.model_name, - modified_at=datetime.now(), - ) - db.add(session_params_entry) - db.commit() - - return { - "id": session.id, - "model_name": session.model_name, - "task_name": session.task_name, - "parameters": session.parameters, - "name": session.name, - "description": session.description, - "created": session.created, - "last_modified": session.last_modified, - # Localized here so the client never receives a language object. - "display_name": localize( - component_registry[session.task_name]["display_name"], - accept_language, - ), - } - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - - -@router.get("/{session_id}", status_code=status.HTTP_200_OK) -async def get_generative_session( - session_id: int, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), -): - """Get a generative session by its ID. - - Parameters - ---------- - session_id : int - The ID of the generative session to retrieve. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - The generated session can be used to access and query the database. - - Returns - ------- - dict - A dictionary with the generative session on the database - - Raises - ------ - HTTPException - If the generative session does not exist or if there's an internal - database error. - """ - - with session_factory() as db: - try: - session = db.get(GenerativeSession, session_id) - if not session: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=(f"Generative session {session_id} does not exist in DB."), - ) - return session - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - - -@router.get("/", status_code=status.HTTP_200_OK) -async def get_all_generative_sessions( - task_name: Union[str, None] = None, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), -): - """Get all generative sessions ordered by creation date. - - Parameters - ---------- - task_name : str | None - If given, return only sessions of that generative task. A view scoped to - one task (the RAG entry point, say) uses this; the shared session list - asks for everything and groups the result by task. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - The generated session can be used to access and query the database. - - Returns - ------- - list - A list of dictionaries with the matching generative sessions, ordered by - creation date. - - Raises - ------ - HTTPException - If there's an internal database error. - """ - - with session_factory() as db: - try: - query = db.query(GenerativeSession) - if task_name is not None: - query = query.filter(GenerativeSession.task_name == task_name) - sessions = query.order_by(GenerativeSession.created.asc()).all() - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - - session_list = [] - for session in sessions: - session_list.append( - { - "id": session.id, - "task_name": session.task_name, - "model_name": session.model_name, - "parameters": session.parameters, - "name": session.name, - "description": session.description, - "created": session.created, - "last_modified": session.last_modified, - } - ) - return session_list - - -@router.delete("/", status_code=status.HTTP_204_NO_CONTENT) -async def delete_generative_sessions( - params: GenerativeSessionBulkDeleteParams, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), -): - """Delete multiple generative sessions, in a single transaction. - - Parameters - ---------- - params : GenerativeSessionBulkDeleteParams - The IDs of the generative sessions to delete. IDs that do not match - an existing session are silently skipped rather than failing the - whole request. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - The generated session can be used to access and query the database. - - Raises - ------ - HTTPException - If there's an internal database error. - """ - - with session_factory() as db: - try: - for session_id in params.ids: - session = db.get(GenerativeSession, session_id) - if not session: - continue - - # Delete all the processes associated with the session - processes = ( - db.query(GenerativeProcess) - .filter(GenerativeProcess.session_id == session_id) - .all() - ) - # Delete all the process data associated with the processes - for process in processes: - process_data = ( - db.query(ProcessData) - .filter(ProcessData.process_id == process.id) - .all() - ) - for data in process_data: - db.delete(data) - # Delete the processes - for process in processes: - db.delete(process) - - # Delete the session parameter history entries - parameters_history = ( - db.query(GenerativeSessionParameterHistory) - .filter(GenerativeSessionParameterHistory.session_id == session_id) - .all() - ) - for entry in parameters_history: - db.delete(entry) - - # Finally, delete the session itself - db.delete(session) - - db.commit() - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - except Exception as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal server error", - ) from e - finally: - db.rollback() - db.close() - - -@router.delete("/{session_id}", status_code=status.HTTP_204_NO_CONTENT) -async def delete_generative_session( - session_id: int, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), -): - """Delete a generative session by its ID. - - Parameters - ---------- - session_id : int - The ID of the generative session to delete. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - The generated session can be used to access and query the database. - - Raises - ------ - HTTPException - If the generative session does not exist or if there's an internal - database error. - """ - - with session_factory() as db: - try: - session = db.get(GenerativeSession, session_id) - if not session: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Generative session {session_id} does not exist in DB.", - ) - - old_parameters = dict(session.parameters or {}) - - # Delete all the processes associated with the session - processes = ( - db.query(GenerativeProcess) - .filter(GenerativeProcess.session_id == session_id) - .all() - ) - # Delete all the process data associated with the processes - for process in processes: - process_data = ( - db.query(ProcessData) - .filter(ProcessData.process_id == process.id) - .all() - ) - for data in process_data: - db.delete(data) - # Delete the processes - for process in processes: - db.delete(process) - - # Delete the session parameter history entries - parameters_history = ( - db.query(GenerativeSessionParameterHistory) - .filter(GenerativeSessionParameterHistory.session_id == session_id) - .all() - ) - for entry in parameters_history: - db.delete(entry) - # Finally, delete the session itself - db.delete(session) - - CleanupService(db).cleanup_orphaned_resources(session_id, old_parameters) - db.commit() - except HTTPException: - raise - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - except Exception as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal server error", - ) from e - finally: - db.close() - - -@router.patch("/{session_id}", status_code=status.HTTP_200_OK) -async def update_generative_session( - session_id: int, - name: Union[str, None] = None, - description: Union[str, None] = None, - model_name: Union[str, None] = None, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), - component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), -): - """Update the generative session associated with the provided ID. - - Parameters - ---------- - session_id : int - ID of the generative session to update. - name : Union[str, None], optional - New name for the session. - description : Union[str, None], optional - New description for the session. - model_name : Union[str, None], optional - New model (component name) for the session. Must be a registered - generative model; if it requires a download it must already be - downloaded. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - The generated session can be used to access and query the database. - component_registry : ComponentRegistry - The DashAI component registry, used to validate the new model. - - Returns - ------- - Dict - A dictionary containing the updated generative session record. - - Raises - ------ - HTTPException - If the session does not exist, the name is invalid or taken, or the new - model is unknown, not a generative model, or not yet downloaded. - """ - from DashAI.back.models.base_generative_model import BaseGenerativeModel - - with session_factory() as db: - try: - session = db.get(GenerativeSession, session_id) - if session is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Generative session not found", - ) - - # Validate name if provided - if name is not None: - if not name or not name.strip(): - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="Name cannot be empty", - ) - - new_name = name.strip() - - # Check if name is different from current name - if new_name != session.name: - # Check if name already exists - exists = db.execute( - select(GenerativeSession.id).where( - GenerativeSession.name == new_name, - GenerativeSession.id != session_id, - ) - ).scalar() - if exists: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Generative session name already exists", - ) - setattr(session, "name", new_name) - - if description is not None: - setattr(session, "description", description) - - # Validate and apply a model change if provided. A model may be - # selected even when it is not downloaded yet; the chat blocks input - # and offers a download until the weights become available. - if model_name is not None and model_name != session.model_name: - try: - model_class = component_registry[model_name]["class"] - except KeyError as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Model {model_name} is not registered.", - ) from e - if not issubclass(model_class, BaseGenerativeModel): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Model {model_name} is not a valid generative model.", - ) - - # Resolve the parameters for the new model: reuse the most - # recent parameters used for it in this session, else fall back - # to the model's schema defaults (its field placeholders). - last_used = ( - db.query(GenerativeSessionParameterHistory) - .filter( - GenerativeSessionParameterHistory.session_id == session_id, - GenerativeSessionParameterHistory.model_name == model_name, - ) - .order_by(GenerativeSessionParameterHistory.modified_at.desc()) - .first() - ) - if last_used is not None: - new_parameters = last_used.parameters - else: - properties = model_class.get_schema().get("properties", {}) - new_parameters = { - key: prop.get("placeholder") for key, prop in properties.items() - } - - session.model_name = model_name - session.parameters = new_parameters - db.add( - GenerativeSessionParameterHistory( - session_id=session.id, - parameters=new_parameters, - model_name=model_name, - modified_at=datetime.now(), - ) - ) - - if name is not None or description is not None or model_name is not None: - session.last_modified = datetime.now() - db.commit() - db.refresh(session) - return session - else: - raise HTTPException( - status_code=status.HTTP_304_NOT_MODIFIED, - detail="Record not modified", - ) - except HTTPException: - raise - except exc.IntegrityError as e: - db.rollback() - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Generative session name already exists", - ) from e - except exc.SQLAlchemyError as e: - db.rollback() - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - - -@router.put("/{session_id}/parameters", status_code=status.HTTP_200_OK) -async def update_generative_session_params( - session_id: int, - new_params: dict, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), - component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), -): - with session_factory() as db: - try: - session = db.get(GenerativeSession, session_id) - if not session: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Generative session {session_id} does not exist in DB.", - ) - - old_parameters = dict(session.parameters or {}) - try: - task_class = component_registry[session.task_name]["class"] - except KeyError as e: - raise HTTPException( - status_code=404, - detail=( - f"Task '{session.task_name}' is not registered" - " in the component registry." - ), - ) from e - - # ── RAG-specific validation of new_params ── - if task_class is not None and task_class == RAGTask: - try: - normalized = SessionValidationService( - db, component_registry - ).validate_update_payload(new_params) - except (ValueError, RAGWorkflowError) as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) - ) from e - - # Merge validated new params into old params - updated_parameters = {**old_parameters, **normalized} - - # Cleanup orphaned RAG resources - CleanupService(db).cleanup_orphaned_resources( - session_id, old_parameters, updated_parameters - ) - else: - # Non-RAG update: simple merge without RAG validation - updated_parameters = {**old_parameters, **new_params} - - # ── Persist ── - session_params_entry = GenerativeSessionParameterHistory( - session_id=session.id, - parameters=updated_parameters, - model_name=session.model_name, - modified_at=datetime.now(), - ) - db.add(session_params_entry) - - session.parameters = updated_parameters - session.last_modified = datetime.now() - db.commit() - db.refresh(session) - - return {"id": session.id, "parameters": session.parameters} - except HTTPException: - raise - except exc.SQLAlchemyError as e: - db.rollback() - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - - -@router.get("/{session_id}/parameters-history", status_code=status.HTTP_200_OK) -async def get_generative_session_parameters_history( - session_id: int, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), -): - """ - Get all parameter history entries for a generative session. - - Parameters - ---------- - session_id : int - The ID of the generative session to retrieve the parameter history for. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - - Returns - ------- - list - A list of dictionaries with all parameter history entries for the session. - - Raises - ------ - HTTPException - If the generative session does not exist or if there's an internal - database error. - """ - with session_factory() as db: - try: - # Check if the generative session exists - session = db.get(GenerativeSession, session_id) - if not session: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Generative session {session_id} does not exist in DB.", - ) - - # Get the session parameter history - parameters_history = ( - db.query(GenerativeSessionParameterHistory) - .filter(GenerativeSessionParameterHistory.session_id == session_id) - .order_by(GenerativeSessionParameterHistory.modified_at.asc()) - .all() - ) - - # Convert the objects to dictionaries (explicit loop for clarity) - history_list = [] - for entry in parameters_history: - history_list.append( - { - "id": entry.id, - "session_id": entry.session_id, - "parameters": entry.parameters, - "modified_at": entry.modified_at, - } - ) - return history_list - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e - - -@router.get("/parameters-history/{session_id}", status_code=status.HTTP_200_OK) -async def get_parameter_history_entry( - session_id: int, - session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), -): - """ - Get history entry for a generative session by its ID. - - Parameters - ---------- - session_id : int - The ID of the generative session to retrieve. - session_factory : Callable[..., ContextManager[Session]] - A factory that creates a context manager that handles a SQLAlchemy session. - The generated session can be used to access and query the database. - - Returns - ------- - list - A list of dictionaries with the parameter history entries for the session. - - Raises - ------ - HTTPException - If the generative session does not exist or if there's an internal - database error. - """ - - with session_factory() as db: - try: - # Check if the generative session exists - session = db.get(GenerativeSession, session_id) - if not session: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Generative session {session_id} does not exist in DB.", - ) - - # Get the parameter history entry for the session - parameters_history = ( - db.query(GenerativeSessionParameterHistory) - .filter(GenerativeSessionParameterHistory.session_id == session_id) - .order_by(GenerativeSessionParameterHistory.modified_at.asc()) - .all() - ) - - parameters_history = [p.__dict__ for p in parameters_history] - if not parameters_history: - return [] - - events = [] - prev_params = parameters_history[0]["parameters"] - prev_model = parameters_history[0].get("model_name") - - for i in range(1, len(parameters_history)): - curr = parameters_history[i] - curr_params = curr["parameters"] - curr_model = curr.get("model_name") - changes = [] - - # A model switch resets parameters to the new model's own - # values, so the raw parameter diff would be noise; report only - # the model change for that entry. - if curr_model and prev_model and curr_model != prev_model: - changes.append( - { - "parameter": "model", - "oldValue": prev_model, - "newValue": curr_model, - } - ) - else: - for key in curr_params: - old_val = prev_params.get(key) - new_val = curr_params[key] - if old_val != new_val: - changes.append( - { - "parameter": key, - "oldValue": old_val, - "newValue": new_val, - } - ) - - events.append( - { - "id": curr["id"], - "timestamp": curr["modified_at"], - "changes": changes, - } - ) - prev_params = curr_params - prev_model = curr_model - - return events - - except exc.SQLAlchemyError as e: - log.exception(e) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal database error", - ) from e +import logging +from datetime import datetime +from typing import TYPE_CHECKING, Union + +from fastapi import APIRouter, Depends, Header, HTTPException, status +from kink import di +from sqlalchemy import exc, select + +from DashAI.back.api.api_v1.schemas.generative_session_params import ( + GenerativeSessionBulkDeleteParams, + GenerativeSessionParams, +) +from DashAI.back.core.utils import localize +from DashAI.back.dependencies.database.models import ( + GenerativeProcess, + GenerativeSession, + GenerativeSessionParameterHistory, + ProcessData, +) +from DashAI.back.dependencies.downloads.nested import missing_downloads +from DashAI.back.models.base_generative_model import BaseGenerativeModel +from DashAI.back.models.RAG.exceptions.base import RAGWorkflowError +from DashAI.back.services.RAG.cleanup_service import CleanupService +from DashAI.back.services.RAG.document_service import DocumentService +from DashAI.back.services.RAG.index_job_service import cancel_live_index_job +from DashAI.back.services.RAG.session_validation_service import ( + SessionValidationService, +) +from DashAI.back.tasks.base_generative_task import BaseGenerativeTask +from DashAI.back.tasks.RAG_task import RAGTask + +if TYPE_CHECKING: + from sqlalchemy.orm import sessionmaker + + from DashAI.back.dependencies.registry import ComponentRegistry + + +router = APIRouter() +log = logging.getLogger(__name__) + + +@router.post("/", status_code=status.HTTP_201_CREATED) +async def upload_generative_session( + params: GenerativeSessionParams, + accept_language: str | None = Header(default=None), + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), +): + """Create a new generative session and log the initial parameters in the history.""" + + with session_factory() as db: + try: + # Check if the model is registered + if params.model_name not in component_registry: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Model {params.model_name} is not registered.", + ) + model_class = component_registry[params.model_name]["class"] + + # Guard: model requires download but has not been downloaded -> 409. + # Reconcile against the filesystem so a model downloaded after startup + # (in the worker process) is recognised without an API restart. + if getattr( + model_class, "REQUIRES_DOWNLOAD", False + ) and not component_registry.refresh_download_status(params.model_name): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Model {params.model_name} must be downloaded before use." + ), + ) + + # Check if the model is a subclass of GenerativeModel + if not issubclass(model_class, BaseGenerativeModel): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Model {params.model_name} is not a valid " + f"generative model.", + ) + + # Check if the task is registered + if params.task_name not in component_registry: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Task {params.task_name} is not registered.", + ) + task_class = component_registry[params.task_name]["class"] + + # RAG: validate and normalise RAG-specific parameters, filling in + # the components the caller did not choose. + if task_class == RAGTask: + try: + params.parameters = SessionValidationService( + db, component_registry + ).prepare_RAG_params(params.parameters, accept_language) + except (ValueError, RAGWorkflowError) as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e), + ) from e + + # A parameter may select another component that itself needs + # downloading; block until every nested one is present. Runs after + # the RAG defaults are applied so it sees the final configuration. + nested_missing = missing_downloads(params.parameters, component_registry) + if nested_missing: + names = ", ".join(m["name"] for m in nested_missing) + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"These components must be downloaded before use: {names}." + ), + ) + + # Validate schema + try: + model_class.SCHEMA.model_validate(params.parameters) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid parameters for model {params.model_name}: {e}", + ) from e + + # Check if the task is a subclass of BaseGenerativeTask + if not issubclass(task_class, BaseGenerativeTask): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Task {params.task_name} is not a valid generative task.", + ) + + now = datetime.now() + session = GenerativeSession( + model_name=params.model_name, + task_name=params.task_name, + parameters=params.parameters, + name=params.name, + description=params.description, + created=now, + last_modified=now, + ) + db.add(session) + try: + db.commit() + except exc.IntegrityError as e: + db.rollback() + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Generative session with name '{params.name}' already exists." + ), + ) from e + db.refresh(session) + + session_params_entry = GenerativeSessionParameterHistory( + session_id=session.id, + parameters=session.parameters, + model_name=session.model_name, + modified_at=datetime.now(), + ) + db.add(session_params_entry) + db.commit() + + return { + "id": session.id, + "model_name": session.model_name, + "task_name": session.task_name, + "parameters": session.parameters, + "name": session.name, + "description": session.description, + "created": session.created, + "last_modified": session.last_modified, + # Localized here so the client never receives a language object. + "display_name": localize( + component_registry[session.task_name]["display_name"], + accept_language, + ), + } + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.get("/{session_id}", status_code=status.HTTP_200_OK) +async def get_generative_session( + session_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Get a generative session by its ID. + + Parameters + ---------- + session_id : int + The ID of the generative session to retrieve. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + + Returns + ------- + dict + A dictionary with the generative session on the database + + Raises + ------ + HTTPException + If the generative session does not exist or if there's an internal + database error. + """ + + with session_factory() as db: + try: + session = db.get(GenerativeSession, session_id) + if not session: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=(f"Generative session {session_id} does not exist in DB."), + ) + return session + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.get("/", status_code=status.HTTP_200_OK) +async def get_all_generative_sessions( + task_name: Union[str, None] = None, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Get all generative sessions ordered by creation date. + + Parameters + ---------- + task_name : str | None + If given, return only sessions of that generative task. A view scoped to + one task (the RAG entry point, say) uses this; the shared session list + asks for everything and groups the result by task. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + + Returns + ------- + list + A list of dictionaries with the matching generative sessions, ordered by + creation date. + + Raises + ------ + HTTPException + If there's an internal database error. + """ + + with session_factory() as db: + try: + query = db.query(GenerativeSession) + if task_name is not None: + query = query.filter(GenerativeSession.task_name == task_name) + sessions = query.order_by(GenerativeSession.created.asc()).all() + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + session_list = [] + for session in sessions: + session_list.append( + { + "id": session.id, + "task_name": session.task_name, + "model_name": session.model_name, + "parameters": session.parameters, + "name": session.name, + "description": session.description, + "created": session.created, + "last_modified": session.last_modified, + } + ) + return session_list + + +@router.delete("/", status_code=status.HTTP_204_NO_CONTENT) +async def delete_generative_sessions( + params: GenerativeSessionBulkDeleteParams, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Delete multiple generative sessions, in a single transaction. + + Parameters + ---------- + params : GenerativeSessionBulkDeleteParams + The IDs of the generative sessions to delete. IDs that do not match + an existing session are silently skipped rather than failing the + whole request. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + + Raises + ------ + HTTPException + If there's an internal database error. + """ + + with session_factory() as db: + try: + for session_id in params.ids: + session = db.get(GenerativeSession, session_id) + if not session: + continue + + # Documents belong to the session. The ORM cascade drops the + # rows, but only this removes their files and fitted artifacts + # from disk. Deferred so the whole batch stays one transaction: + # committing per session would leave earlier sessions' files + # gone if a later one failed. + DocumentService(db).delete_by_session(session_id, commit=False) + + # Delete all the processes associated with the session + processes = ( + db.query(GenerativeProcess) + .filter(GenerativeProcess.session_id == session_id) + .all() + ) + # Delete all the process data associated with the processes + for process in processes: + process_data = ( + db.query(ProcessData) + .filter(ProcessData.process_id == process.id) + .all() + ) + for data in process_data: + db.delete(data) + # Delete the processes + for process in processes: + db.delete(process) + + # Delete the session parameter history entries + parameters_history = ( + db.query(GenerativeSessionParameterHistory) + .filter(GenerativeSessionParameterHistory.session_id == session_id) + .all() + ) + for entry in parameters_history: + db.delete(entry) + + # Finally, delete the session itself + db.delete(session) + + db.commit() + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + except Exception as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal server error", + ) from e + finally: + db.rollback() + db.close() + + +@router.delete("/{session_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_generative_session( + session_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """Delete a generative session by its ID. + + Parameters + ---------- + session_id : int + The ID of the generative session to delete. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + + Raises + ------ + HTTPException + If the generative session does not exist or if there's an internal + database error. + """ + + with session_factory() as db: + try: + session = db.get(GenerativeSession, session_id) + if not session: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Generative session {session_id} does not exist in DB.", + ) + + old_parameters = dict(session.parameters or {}) + + # Stop any index still running for a session that is going away. + cancel_live_index_job(session, di["job_queue"]) + + # Delete all the processes associated with the session + processes = ( + db.query(GenerativeProcess) + .filter(GenerativeProcess.session_id == session_id) + .all() + ) + # Delete all the process data associated with the processes + for process in processes: + process_data = ( + db.query(ProcessData) + .filter(ProcessData.process_id == process.id) + .all() + ) + for data in process_data: + db.delete(data) + # Delete the processes + for process in processes: + db.delete(process) + + # Documents belong to the session. The ORM cascade drops the rows, + # but only this removes their files and fitted artifacts from disk. + # Deferred so nothing is unlinked before the delete commits. + DocumentService(db).delete_by_session(session_id, commit=False) + + # Delete the session parameter history entries + parameters_history = ( + db.query(GenerativeSessionParameterHistory) + .filter(GenerativeSessionParameterHistory.session_id == session_id) + .all() + ) + for entry in parameters_history: + db.delete(entry) + # Finally, delete the session itself + db.delete(session) + + CleanupService(db).cleanup_orphaned_resources(session_id, old_parameters) + db.commit() + except HTTPException: + raise + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + except Exception as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal server error", + ) from e + finally: + db.close() + + +@router.patch("/{session_id}", status_code=status.HTTP_200_OK) +async def update_generative_session( + session_id: int, + name: Union[str, None] = None, + description: Union[str, None] = None, + model_name: Union[str, None] = None, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), +): + """Update the generative session associated with the provided ID. + + Parameters + ---------- + session_id : int + ID of the generative session to update. + name : Union[str, None], optional + New name for the session. + description : Union[str, None], optional + New description for the session. + model_name : Union[str, None], optional + New model (component name) for the session. Must be a registered + generative model; if it requires a download it must already be + downloaded. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + component_registry : ComponentRegistry + The DashAI component registry, used to validate the new model. + + Returns + ------- + Dict + A dictionary containing the updated generative session record. + + Raises + ------ + HTTPException + If the session does not exist, the name is invalid or taken, or the new + model is unknown, not a generative model, or not yet downloaded. + """ + from DashAI.back.models.base_generative_model import BaseGenerativeModel + + with session_factory() as db: + try: + session = db.get(GenerativeSession, session_id) + if session is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Generative session not found", + ) + + # Validate name if provided + if name is not None: + if not name or not name.strip(): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Name cannot be empty", + ) + + new_name = name.strip() + + # Check if name is different from current name + if new_name != session.name: + # Check if name already exists + exists = db.execute( + select(GenerativeSession.id).where( + GenerativeSession.name == new_name, + GenerativeSession.id != session_id, + ) + ).scalar() + if exists: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Generative session name already exists", + ) + setattr(session, "name", new_name) + + if description is not None: + setattr(session, "description", description) + + # Validate and apply a model change if provided. A model may be + # selected even when it is not downloaded yet; the chat blocks input + # and offers a download until the weights become available. + if model_name is not None and model_name != session.model_name: + try: + model_class = component_registry[model_name]["class"] + except KeyError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Model {model_name} is not registered.", + ) from e + if not issubclass(model_class, BaseGenerativeModel): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Model {model_name} is not a valid generative model.", + ) + + # Resolve the parameters for the new model: reuse the most + # recent parameters used for it in this session, else fall back + # to the model's schema defaults (its field placeholders). + last_used = ( + db.query(GenerativeSessionParameterHistory) + .filter( + GenerativeSessionParameterHistory.session_id == session_id, + GenerativeSessionParameterHistory.model_name == model_name, + ) + .order_by(GenerativeSessionParameterHistory.modified_at.desc()) + .first() + ) + if last_used is not None: + new_parameters = last_used.parameters + else: + properties = model_class.get_schema().get("properties", {}) + new_parameters = { + key: prop.get("placeholder") for key, prop in properties.items() + } + + session.model_name = model_name + session.parameters = new_parameters + db.add( + GenerativeSessionParameterHistory( + session_id=session.id, + parameters=new_parameters, + model_name=model_name, + modified_at=datetime.now(), + ) + ) + + if name is not None or description is not None or model_name is not None: + session.last_modified = datetime.now() + db.commit() + db.refresh(session) + return session + else: + raise HTTPException( + status_code=status.HTTP_304_NOT_MODIFIED, + detail="Record not modified", + ) + except HTTPException: + raise + except exc.IntegrityError as e: + db.rollback() + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Generative session name already exists", + ) from e + except exc.SQLAlchemyError as e: + db.rollback() + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.put("/{session_id}/parameters", status_code=status.HTTP_200_OK) +async def update_generative_session_params( + session_id: int, + new_params: dict, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), +): + with session_factory() as db: + try: + session = db.get(GenerativeSession, session_id) + if not session: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Generative session {session_id} does not exist in DB.", + ) + + old_parameters = dict(session.parameters or {}) + try: + task_class = component_registry[session.task_name]["class"] + except KeyError as e: + raise HTTPException( + status_code=404, + detail=( + f"Task '{session.task_name}' is not registered" + " in the component registry." + ), + ) from e + + # ── RAG-specific validation of new_params ── + if task_class is not None and task_class == RAGTask: + try: + normalized = SessionValidationService( + db, component_registry + ).validate_update_payload(new_params) + except (ValueError, RAGWorkflowError) as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) + ) from e + + # Merge validated new params into old params + updated_parameters = {**old_parameters, **normalized} + + # A running index writes the very rows the cleanup below + # deletes, so it has to be stopped before, not after. + cancel_live_index_job(session, di["job_queue"]) + + # Cleanup orphaned RAG resources + CleanupService(db).cleanup_orphaned_resources( + session_id, old_parameters, updated_parameters + ) + else: + # Non-RAG update: simple merge without RAG validation + updated_parameters = {**old_parameters, **new_params} + + # ── Persist ── + session_params_entry = GenerativeSessionParameterHistory( + session_id=session.id, + parameters=updated_parameters, + model_name=session.model_name, + modified_at=datetime.now(), + ) + db.add(session_params_entry) + + session.parameters = updated_parameters + session.last_modified = datetime.now() + db.commit() + db.refresh(session) + + return {"id": session.id, "parameters": session.parameters} + except HTTPException: + raise + except exc.SQLAlchemyError as e: + db.rollback() + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.get("/{session_id}/parameters-history", status_code=status.HTTP_200_OK) +async def get_generative_session_parameters_history( + session_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """ + Get all parameter history entries for a generative session. + + Parameters + ---------- + session_id : int + The ID of the generative session to retrieve the parameter history for. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + + Returns + ------- + list + A list of dictionaries with all parameter history entries for the session. + + Raises + ------ + HTTPException + If the generative session does not exist or if there's an internal + database error. + """ + with session_factory() as db: + try: + # Check if the generative session exists + session = db.get(GenerativeSession, session_id) + if not session: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Generative session {session_id} does not exist in DB.", + ) + + # Get the session parameter history + parameters_history = ( + db.query(GenerativeSessionParameterHistory) + .filter(GenerativeSessionParameterHistory.session_id == session_id) + .order_by(GenerativeSessionParameterHistory.modified_at.asc()) + .all() + ) + + # Convert the objects to dictionaries (explicit loop for clarity) + history_list = [] + for entry in parameters_history: + history_list.append( + { + "id": entry.id, + "session_id": entry.session_id, + "parameters": entry.parameters, + "modified_at": entry.modified_at, + } + ) + return history_list + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + +@router.get("/parameters-history/{session_id}", status_code=status.HTTP_200_OK) +async def get_parameter_history_entry( + session_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), +): + """ + Get history entry for a generative session by its ID. + + Parameters + ---------- + session_id : int + The ID of the generative session to retrieve. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + + Returns + ------- + list + A list of dictionaries with the parameter history entries for the session. + + Raises + ------ + HTTPException + If the generative session does not exist or if there's an internal + database error. + """ + + with session_factory() as db: + try: + # Check if the generative session exists + session = db.get(GenerativeSession, session_id) + if not session: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Generative session {session_id} does not exist in DB.", + ) + + # Get the parameter history entry for the session + parameters_history = ( + db.query(GenerativeSessionParameterHistory) + .filter(GenerativeSessionParameterHistory.session_id == session_id) + .order_by(GenerativeSessionParameterHistory.modified_at.asc()) + .all() + ) + + parameters_history = [p.__dict__ for p in parameters_history] + if not parameters_history: + return [] + + events = [] + prev_params = parameters_history[0]["parameters"] + prev_model = parameters_history[0].get("model_name") + + for i in range(1, len(parameters_history)): + curr = parameters_history[i] + curr_params = curr["parameters"] + curr_model = curr.get("model_name") + changes = [] + + # A model switch resets parameters to the new model's own + # values, so the raw parameter diff would be noise; report only + # the model change for that entry. + if curr_model and prev_model and curr_model != prev_model: + changes.append( + { + "parameter": "model", + "oldValue": prev_model, + "newValue": curr_model, + } + ) + else: + for key in curr_params: + old_val = prev_params.get(key) + new_val = curr_params[key] + if old_val != new_val: + changes.append( + { + "parameter": key, + "oldValue": old_val, + "newValue": new_val, + } + ) + + events.append( + { + "id": curr["id"], + "timestamp": curr["modified_at"], + "changes": changes, + } + ) + prev_params = curr_params + prev_model = curr_model + + return events + + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e diff --git a/DashAI/back/api/api_v1/endpoints/prompts.py b/DashAI/back/api/api_v1/endpoints/prompts.py index 2cfc3f2bf..a2f80b901 100644 --- a/DashAI/back/api/api_v1/endpoints/prompts.py +++ b/DashAI/back/api/api_v1/endpoints/prompts.py @@ -4,9 +4,7 @@ from DashAI.back.api.api_v1.schemas.RAG_prompt import ( RAGPromptSchema, - RAGPromptUpdateSchema, ) -from DashAI.back.dependencies.database.models import GenerativeSession from DashAI.back.dependencies.registry import ComponentRegistry from DashAI.back.models.RAG.exceptions import ( RAGDatabaseError, @@ -48,61 +46,6 @@ async def create_RAG_prompt( # noqa: N802 ) from e -@router.patch("/{prompt_id}", status_code=status.HTTP_200_OK) -async def update_RAG_prompt( # noqa: N802 - prompt_id: int, - prompt: RAGPromptUpdateSchema, - component_registry: ComponentRegistry = Depends(lambda: di["component_registry"]), - session_factory: sessionmaker = Depends(lambda: di["session_factory"]), -): - """Update an existing prompt in place.""" - - with session_factory() as db: - try: - service = PromptService(db, component_registry) - result = service.update( - prompt_id, - name=prompt.name, - parameters=prompt.parameters, - ) - return result - except ValueError as e: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail=str(e) - ) from e - - -@router.post("/{prompt_id}/sessions/{session_id}", status_code=status.HTTP_201_CREATED) -async def update_RAG_prompt_for_session( # noqa: N802 - prompt_id: int, - session_id: int, - prompt: RAGPromptUpdateSchema, - component_registry: ComponentRegistry = Depends(lambda: di["component_registry"]), - session_factory: sessionmaker = Depends(lambda: di["session_factory"]), -): - """Create a session-scoped copy of a prompt and attach it to the session.""" - - with session_factory() as db: - try: - service = PromptService(db, component_registry) - prompt_result = service.create_session_copy( - prompt_id, - session_id, - parameters=prompt.parameters, - name=prompt.name, - ) - session = db.get(GenerativeSession, session_id) - return { - "prompt": prompt_result, - "session_id": session_id, - "parameters": session.parameters if session else None, - } - except ValueError as e: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail=str(e) - ) from e - - @router.get("/", status_code=status.HTTP_200_OK) async def get_all_prompts( session_factory: sessionmaker = Depends(lambda: di["session_factory"]), diff --git a/DashAI/back/api/api_v1/endpoints/rag.py b/DashAI/back/api/api_v1/endpoints/rag.py index aaedfe4e3..f5e911744 100644 --- a/DashAI/back/api/api_v1/endpoints/rag.py +++ b/DashAI/back/api/api_v1/endpoints/rag.py @@ -12,10 +12,18 @@ from kink import di from DashAI.back.core.utils import localize +from DashAI.back.dependencies.database.models import GenerativeSession +from DashAI.back.job.RAG_index_job import RAGIndexJob +from DashAI.back.models.RAG.RAG_constants import RAG_PARAM_KEYS from DashAI.back.services.RAG.chunking_presets import ( get_chunking_presets as resolve_chunking_presets, ) -from DashAI.back.services.RAG.index_status_service import IndexStatusService +from DashAI.back.services.RAG.index_status_service import ( + STATUS_INDEXED, + STATUS_INDEXING, + STATUS_NO_DOCUMENTS, + IndexStatusService, +) from DashAI.back.services.RAG.retriever_presets import ( get_retriever_presets as resolve_retriever_presets, ) @@ -27,6 +35,7 @@ if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker + from DashAI.back.dependencies.job_queues.base_job_queue import BaseJobQueue from DashAI.back.dependencies.registry import ComponentRegistry router = APIRouter() @@ -166,11 +175,12 @@ def session_index_status( accept_language: str | None = Header(default=None), session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), + job_queue: "BaseJobQueue" = Depends(lambda: di["job_queue"]), ): """Report whether a RAG session's documents are already indexed. - Read-only: indexing itself still happens inside the chat job, so this never - triggers work, it only reports what the job would find. + Read-only: it never chunks, embeds or enqueues, so it is safe to poll while + an indexing job runs. Parameters ---------- @@ -182,12 +192,14 @@ def session_index_status( Factory for the SQLAlchemy session. component_registry : ComponentRegistry Registry used to resolve retriever kinds. + job_queue : BaseJobQueue + Queue consulted for a running indexing job. Returns ------- dict ``{status, chunk_set_id, total_chunks, retriever_ready, documents, - message}``. + message, job_id, job}``. Raises ------ @@ -196,9 +208,88 @@ def session_index_status( """ with session_factory() as db: try: - state = IndexStatusService(db, component_registry).get_status(session_id) + state = IndexStatusService(db, component_registry, job_queue).get_status( + session_id + ) except ValueError as e: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=str(e) ) from e return localize(state, accept_language) + + +@router.post("/sessions/{session_id}/index", status_code=status.HTTP_202_ACCEPTED) +def start_session_indexing( + session_id: int, + accept_language: str | None = Header(default=None), + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), + component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), + job_queue: "BaseJobQueue" = Depends(lambda: di["job_queue"]), +): + """Index a session's documents now, if they are not already. + + Idempotent and coalescing: nothing to index, already indexed, and already + indexing all return the current state without enqueueing anything. That is + what lets callers fire this after *every* save without deciding for + themselves which settings invalidate the index — the chunk-set signature + already owns that rule, and a second opinion could only drift from it. + + Returns the same payload as ``/index-status`` so no follow-up GET is needed. + + Parameters + ---------- + session_id : int + The RAG session to index. + accept_language : str | None + The 'Accept-Language' header, used to localize the status message. + session_factory : Callable[..., ContextManager[Session]] + Factory for the SQLAlchemy session. + component_registry : ComponentRegistry + Registry used to resolve retriever kinds. + job_queue : BaseJobQueue + Queue the indexing job is submitted to. + + Returns + ------- + dict + The session's index status, as ``/index-status`` reports it. + + Raises + ------ + HTTPException + 404 if the session does not exist, 409 if its parameters do not + describe a complete pipeline. + """ + with session_factory() as db: + service = IndexStatusService(db, component_registry, job_queue) + try: + state = service.get_status(session_id) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=str(e) + ) from e + + if state["status"] in (STATUS_NO_DOCUMENTS, STATUS_INDEXED, STATUS_INDEXING): + return localize(state, accept_language) + + session = db.get(GenerativeSession, session_id) + missing = RAG_PARAM_KEYS - set(session.parameters or {}) + if missing: + # Unreachable for sessions created through the API, which always + # get every key; a legacy row would otherwise fail deep inside the + # job with a far less useful message. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"Session {session_id} is missing configuration: {sorted(missing)}." + ), + ) + + # No set_status_as_delivered() here: that hook exists to move a job's + # own DB entity into "delivered", and indexing has none — the queue + # owns the lifecycle from here. + job = RAGIndexJob(session_id=session_id) + session.index_job_id = str(job_queue.put(job).id) + db.commit() + + return localize(service.get_status(session_id), accept_language) diff --git a/DashAI/back/api/api_v1/schemas/RAG_prompt.py b/DashAI/back/api/api_v1/schemas/RAG_prompt.py index 898f3441b..ff7bd5be7 100644 --- a/DashAI/back/api/api_v1/schemas/RAG_prompt.py +++ b/DashAI/back/api/api_v1/schemas/RAG_prompt.py @@ -15,15 +15,3 @@ class RAGPromptSchema(BaseModel): class_name: str name: str parameters: Optional[Dict[str, Any]] = None - - -class RAGPromptUpdateSchema(BaseModel): - """Schema for updating an existing RAG prompt. - - Attributes: - name: Optional new name for the prompt. - parameters: Optional new configuration dict. - """ - - name: Optional[str] = None - parameters: Optional[Dict[str, Any]] = None diff --git a/DashAI/back/api/api_v1/schemas/__init__.py b/DashAI/back/api/api_v1/schemas/__init__.py index 76f42ae53..12ce1b0f8 100644 --- a/DashAI/back/api/api_v1/schemas/__init__.py +++ b/DashAI/back/api/api_v1/schemas/__init__.py @@ -1 +1,11 @@ -from DashAI.back.api.api_v1.schemas.document import DocumentResponse +from DashAI.back.api.api_v1.schemas.document import ( + DocumentResponse, + ExtractorRef, + UpdateExtractorRequest, +) + +__all__ = [ + "DocumentResponse", + "ExtractorRef", + "UpdateExtractorRequest", +] diff --git a/DashAI/back/api/api_v1/schemas/document.py b/DashAI/back/api/api_v1/schemas/document.py index ebd8d9935..a4786a2c2 100644 --- a/DashAI/back/api/api_v1/schemas/document.py +++ b/DashAI/back/api/api_v1/schemas/document.py @@ -1,11 +1,12 @@ from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional from pydantic import BaseModel class DocumentResponse(BaseModel): id: int + session_id: int file_name: str file_type: str file_hash: str @@ -14,6 +15,18 @@ class DocumentResponse(BaseModel): optional_metadata: Optional[Dict[str, Any]] extractor: Optional[Dict[str, Any]] = None default_extractor: Optional[Dict[str, Any]] = None - related_sessions: List[int] | None file_url: str preview_url: str + + +class ExtractorRef(BaseModel): + """A ``{component, params}`` reference to an extractor configuration.""" + + component: str + params: Dict[str, Any] = {} + + +class UpdateExtractorRequest(BaseModel): + """Body of ``PUT /document/{id}/extractor``.""" + + extractor: ExtractorRef diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py index 1b41d8d5e..bf878f36d 100644 --- a/DashAI/back/dependencies/database/models.py +++ b/DashAI/back/dependencies/database/models.py @@ -583,6 +583,10 @@ class GenerativeSession(Base): # metadata name: Mapped[str] = mapped_column(String, unique=True, nullable=False) description: Mapped[str] = mapped_column(String, nullable=True) + # Huey id of the RAG indexing job most recently started for this session. + # A pointer, never the truth: the job queue stays authoritative for whether + # that job is still alive, so a stale id simply resolves to nothing. + index_job_id: Mapped[str] = mapped_column(String, nullable=True) # Relationship with GenerativeSessionParameterHistory parameters_history: Mapped[List["GenerativeSessionParameterHistory"]] = ( @@ -598,9 +602,11 @@ class GenerativeSession(Base): "GenerativeProcess", cascade="all, delete-orphan", back_populates="session" ) - # Relationship with RAGDocumentPipelineSessionLink - pipeline_links: Mapped[List["RAGDocumentPipelineSessionLink"]] = relationship( - back_populates="session" + # RAG documents belong to exactly one session. SQLite foreign keys are not + # enforced here, so this ORM cascade -- not the ondelete on Document -- is + # what deletes them when the session goes away. + documents: Mapped[List["Document"]] = relationship( + "Document", back_populates="session", cascade="all, delete-orphan" ) @@ -873,10 +879,15 @@ class Document(Base): Table to store all the information about a document. """ id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) + session_id: Mapped[int] = mapped_column( + ForeignKey("generative_session.id", ondelete="CASCADE"), nullable=False + ) file_name: Mapped[str] = mapped_column(String, nullable=False) file_type: Mapped[str] = mapped_column(String, nullable=False) file_path: Mapped[str] = mapped_column(String, nullable=False) - file_hash: Mapped[str] = mapped_column(String, nullable=False, unique=True) + # Unique per session, not globally: the same file uploaded into two + # sessions is two documents, each with its own extractor choice. + file_hash: Mapped[str] = mapped_column(String, nullable=False) optional_metadata: Mapped[Dict[str, Any]] = mapped_column(JSON, nullable=True) extractor_id: Mapped[int] = mapped_column( ForeignKey("rag_extractor.id", ondelete="RESTRICT"), nullable=False @@ -888,11 +899,8 @@ class Document(Base): onupdate=datetime.now, ) - # Create a relationship for the related sessions and related chunks - pipeline_links: Mapped[List["RAGDocumentPipelineSessionLink"]] = relationship( - "RAGDocumentPipelineSessionLink", - cascade="all, delete-orphan", - back_populates="document", + session: Mapped["GenerativeSession"] = relationship( + "GenerativeSession", back_populates="documents" ) chunks: Mapped[List["Chunk"]] = relationship( @@ -915,15 +923,20 @@ class Document(Base): cascade="all, delete-orphan", ) - @property - def get_related_sessions(self) -> List["GenerativeSession"]: - """Return a list of sessions related to the document.""" - return [link.session for link in self.pipeline_links] + # SQLite foreign keys are not enforced (no PRAGMA foreign_keys=ON), so the + # ondelete=CASCADE above is documentation only: this cascade is what + # actually removes the membership rows when a document is deleted. + chunk_set_links: Mapped[List["RAGChunkSetDocument"]] = relationship( + "RAGChunkSetDocument", + back_populates="document", + cascade="all, delete-orphan", + ) - @property - def get_related_pipelines(self) -> List["RAGPipeline"]: - """Return a list of pipelines related to the document.""" - return [link.pipeline for link in self.pipeline_links] + __table_args__ = ( + UniqueConstraint( + "session_id", "file_hash", name="uq_document_session_file_hash" + ), + ) def get_embedding_matrix( self, chunk_set_id: int, embedding_model_id: int @@ -1017,7 +1030,9 @@ class RAGChunkSetDocument(Base): "RAGChunkSet", back_populates="documents", ) - document: Mapped["Document"] = relationship("Document") + document: Mapped["Document"] = relationship( + "Document", back_populates="chunk_set_links" + ) __table_args__ = ( UniqueConstraint( @@ -1070,9 +1085,11 @@ class RAGPrompt(Base): onupdate=datetime.now, ) - # Relationship with RAGPipeline + # No cascade: rows here are deduplicated by parameters_hash, so one row is + # shared by every session that landed on the same template. Cascading would + # delete *other* sessions' pipelines along with the prompt. pipelines: Mapped[List["RAGPipeline"]] = relationship( - "RAGPipeline", back_populates="prompt", cascade="all, delete-orphan" + "RAGPipeline", back_populates="prompt", passive_deletes=True ) __table_args__ = ( @@ -1136,9 +1153,6 @@ class RAGPipeline(Base): generation_model: Mapped["RAGGenerationModel"] = relationship( "RAGGenerationModel", back_populates="pipelines" ) - pipeline_links: Mapped[List["RAGDocumentPipelineSessionLink"]] = relationship( - "RAGDocumentPipelineSessionLink", back_populates="pipeline" - ) class RAGChunkingModel(Base): @@ -1397,37 +1411,6 @@ def get_by_tuple( ) -""" -RAG relationship tables -""" - - -class RAGDocumentPipelineSessionLink(Base): - __tablename__ = "rag_document_pipeline_session_link" - - id: Mapped[int] = mapped_column(primary_key=True) - document_id: Mapped[int] = mapped_column( - ForeignKey("document.id", ondelete="CASCADE"), nullable=False - ) - session_id: Mapped[int] = mapped_column( - ForeignKey("generative_session.id", ondelete="CASCADE"), nullable=False - ) - pipeline_id: Mapped[int] = mapped_column( - ForeignKey("rag_pipeline.id", ondelete="CASCADE"), nullable=False - ) - - # Relationships - document = relationship("Document", back_populates="pipeline_links") - pipeline = relationship("RAGPipeline", back_populates="pipeline_links") - session = relationship("GenerativeSession", back_populates="pipeline_links") - - __table_args__ = ( - UniqueConstraint("document_id", "session_id", name="uix_document_session"), - UniqueConstraint("session_id", "pipeline_id", name="uix_session_pipeline"), - UniqueConstraint("document_id", "pipeline_id", name="uix_document_pipeline"), - ) - - class Datafile(Base): __tablename__ = "datafile" diff --git a/DashAI/back/job/RAG_index_job.py b/DashAI/back/job/RAG_index_job.py new file mode 100644 index 000000000..05df24c08 --- /dev/null +++ b/DashAI/back/job/RAG_index_job.py @@ -0,0 +1,143 @@ +"""Eager indexing of a RAG session's documents. + +The chat job indexes lazily, as a side effect of answering the first message, +which makes that message pay for the whole chunking and embedding run. This job +does the same work up front — when a document is uploaded, or when a +configuration change invalidates the index — so the chat stays fast and the +progress is something the user can actually watch. +""" + +import gc +import logging +from typing import Dict + +from kink import di, inject + +from DashAI.back.dependencies.database.models import GenerativeSession +from DashAI.back.job.base_job import BaseJob, JobError +from DashAI.back.models.RAG.RAG_constants import RAG_PARAM_KEYS as _RAG_PARAM_KEYS +from DashAI.back.models.RAG.RAG_pipeline import RAGPipelineConfig +from DashAI.back.services.RAG.setup_service import SetupService + +log = logging.getLogger(__name__) + + +class RAGIndexJob(BaseJob): + """Chunks, embeds and fits the retriever for one RAG session. + + Deliberately never builds the generation model: indexing must not load LLM + weights it will not use. See :meth:`SetupService.build_index`. + + ``ISOLATED`` stays True (the default): this loads embedding models, and the + subprocess boundary is what frees their memory once the job is done. + """ + + def set_status_as_delivered(self) -> None: + """Required by :class:`BaseJob`, but nothing calls it for this job. + + The hook exists to move a job's own DB entity into "delivered". + Indexing has no entity — the index *is* the chunk and retriever rows — + so there is nothing to move, and the endpoint does not call it. + """ + log.debug("Index job delivered for session %s", self.kwargs.get("session_id")) + + def set_status_as_error(self) -> None: + """Record failure. Unlike the delivered hook, the queue really does + call this (on cancel, kill and delete), so it must never raise. There + is no status column to write: the queue keeps the error message in + ``task_copy``, which is what the index-status endpoint reports back. + """ + log.debug("Index job failed for session %s", self.kwargs.get("session_id")) + + def get_job_name(self) -> str: + """Get a descriptive name for the job.""" + session_id = self.kwargs.get("session_id") + if not session_id: + return "Indexing documents" + + try: + with di["session_factory"]() as db: + session = db.get(GenerativeSession, session_id) + if session and session.name: + return f"Indexing: {session.name}" + except Exception as e: + log.exception(f"Error getting job name: {e}") + + return f"Indexing session #{session_id}" + + @inject + def run(self) -> Dict[str, int]: + """Build the session's index, reporting progress as it goes. + + Returns: + ``{"chunk_set_id": int, "total_chunks": int}``, which the queue + stores as the task result. Callers read the index through + ``IndexStatusService`` instead; this is for the job log. + + Raises: + JobError: If the session is missing, holds no documents, or its + parameters do not describe a complete pipeline. + """ + component_registry = di["component_registry"] + session_factory = di["session_factory"] + config = di["config"] + + if "session_id" not in self.kwargs: + raise JobError("RAGIndexJob requires 'session_id' in kwargs.") + + session_id: int = self.kwargs["session_id"] + + try: + with session_factory() as db: + session = db.get(GenerativeSession, session_id) + if not session: + raise JobError(f"Session {session_id} not found in DB.") + + # Whitelist-only, same as RAGJob: session parameters may carry + # keys the pipeline config would reject. + raw_params = dict(session.parameters or {}) + clean_params = { + k: v for k, v in raw_params.items() if k in _RAG_PARAM_KEYS + } + if not (clean_params.get("documents") or []): + raise JobError(f"Session {session_id} has no documents to index.") + + pipeline_config = RAGPipelineConfig.from_kwargs( + db=db, + component_registry=component_registry, + session_id=session_id, + env_RAG_path=config["RAG_PATH"], + **clean_params, + ) + setup_service = SetupService( + db, + component_registry, + config["RAG_PATH"], + ) + result = setup_service.build_index( + pipeline_config, + progress=self.report_progress, + ) + log.debug( + "Indexed session %d: chunk set %d, %d chunks", + session_id, + result.chunk_set_id, + result.total_chunks, + ) + return { + "chunk_set_id": result.chunk_set_id, + "total_chunks": result.total_chunks, + } + except JobError: + self.set_status_as_error() + raise + except Exception as e: + log.exception(e) + self.set_status_as_error() + raise JobError(f"Error indexing session {session_id}.") from e + finally: + import torch + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + gc.collect() diff --git a/DashAI/back/models/RAG/retrievers/dense/dense_retriever.py b/DashAI/back/models/RAG/retrievers/dense/dense_retriever.py index fece71279..caaebb7ac 100644 --- a/DashAI/back/models/RAG/retrievers/dense/dense_retriever.py +++ b/DashAI/back/models/RAG/retrievers/dense/dense_retriever.py @@ -4,6 +4,7 @@ import numpy as np from sklearn.metrics.pairwise import pairwise_distances +from DashAI.back.core.atomic import atomic_open from DashAI.back.core.schema_fields import ( BaseSchema, enum_field, @@ -117,6 +118,10 @@ def compute_missing_embeddings(self): Iterates over all documents; if an ``embeddings.npy`` file does not yet exist at the expected path, the embedding model is used to encode the chunk texts and the result is saved. + + The write is atomic because the presence of the file is what marks a + document as embedded: a half-written matrix left behind by a killed + indexing job would be skipped forever and break every later load. """ for doc_id, doc_chunks in self.chunks.items(): matrix_dir = self._persistence.matrix_dirs.get(doc_id) @@ -130,7 +135,10 @@ def compute_missing_embeddings(self): raise RAGRetrieverError(f"No chunks found for document ID {doc_id}.") embeddings = self.embedding_model.batch_encode(chunk_texts) os.makedirs(matrix_dir, exist_ok=True) - np.save(matrix_path, embeddings) + # np.save appends '.npy' to a path but not to a file object, which + # is what keeps the temp file and the final name in agreement. + with atomic_open(matrix_path, "wb") as f: + np.save(f, embeddings) def init_similarity_matrix(self): """Load all persisted embedding matrices into a single similarity matrix. diff --git a/DashAI/back/services/RAG/cleanup_service.py b/DashAI/back/services/RAG/cleanup_service.py index d94d08306..84a4e31aa 100644 --- a/DashAI/back/services/RAG/cleanup_service.py +++ b/DashAI/back/services/RAG/cleanup_service.py @@ -1,14 +1,10 @@ -import json import logging -import shutil -from pathlib import Path -from typing import Any from sqlalchemy.orm import Session from DashAI.back.dependencies.database.models import ( - GenerativeSession, RAGChunkingModel, + RAGDenseRetriever, RAGEmbeddingMatrix, RAGEmbeddingModel, RAGPipeline, @@ -16,6 +12,7 @@ RAGRetrieverChild, ) from DashAI.back.models.RAG.RAG_constants import COMPOSITE_RETRIEVER_NAMES +from DashAI.back.services.RAG.deferred_fs import remove_after_commit, remove_now from DashAI.back.services.RAG.retriever_db_service import RetrieverDBService log = logging.getLogger(__name__) @@ -67,14 +64,13 @@ def _component_changed(key: str) -> bool: retriever_model_params = old_parameters.get("retriever_model") or {} retriever_component_name = retriever_model_params.get("component", "") - should_cleanup_retriever = ( - bool(retriever_model_params) - and _component_changed("retriever_model") - and not self._other_sessions_with_same_config( - session_id, - old_parameters, - keys=("documents", "chunking_model", "retriever_model"), - ) + # The retriever rows deleted below hang off this session's chunk + # set, which no other session can share now that documents belong + # to one session. Rows keyed by configuration alone -- the chunking + # model here, the embedding model in _cleanup_dense_retriever -- + # *are* shared, and are guarded where they are deleted. + should_cleanup_retriever = bool(retriever_model_params) and ( + _component_changed("retriever_model") ) if should_cleanup_retriever: @@ -91,29 +87,17 @@ def _component_changed(key: str) -> bool: # ── Chunking model cleanup (AFTER retriever) ── chunking_model_params = old_parameters.get("chunking_model") or {} - should_cleanup_chunking = ( - bool(chunking_model_params) - and _component_changed("chunking_model") - and not self._other_sessions_with_same_config( - session_id, - old_parameters, - keys=("documents", "chunking_model"), - ) + should_cleanup_chunking = bool(chunking_model_params) and ( + _component_changed("chunking_model") ) if should_cleanup_chunking: - chunking_models = ( - self.db.query(RAGChunkingModel) - .filter( - RAGChunkingModel.class_name - == chunking_model_params.get("component"), - RAGChunkingModel.parameters - == chunking_model_params.get("params"), - ) - .all() - ) - for chunking_model in chunking_models: - self.db.delete(chunking_model) + # Ask this session's own pipeline which row it was using, rather + # than looking one up by class name and params. Chunking rows + # are stored with their params key-sorted while a session's + # parameters keep whatever order the client sent, so a JSON + # comparison silently misses the very row it means to match. + self._drop_chunking_model_if_unused(session_id) self.db.commit() except Exception: @@ -124,87 +108,83 @@ def _component_changed(key: str) -> bool: @staticmethod def _delete_path(path_value: str | None) -> None: - """Delete a filesystem path recursively if it exists. + """Delete a filesystem path immediately, if it exists. - Logs a warning if deletion fails (e.g. permission error, file in use). + Prefer queueing the path with + :func:`~DashAI.back.services.RAG.deferred_fs.remove_after_commit` when + it is tied to rows being deleted in a transaction. Args: path_value: Absolute path to delete. Silently skipped if ``None`` or the path does not exist. """ - if not path_value: - return - path = Path(path_value) - if path.exists(): - try: - shutil.rmtree(path) - except OSError as exc: - log.warning("Failed to remove %s: %s", path_value, exc) - - def _other_sessions_with_same_config( - self, - session_id: int, - expected_parameters: dict[str, Any], - *, - keys: tuple[str, ...], - ) -> bool: - """Return True if any other session matches all specified keys. + remove_now(path_value) - Used to avoid deleting shared resources that another session still - depends on. + def _drop_chunking_model_if_unused(self, session_id: int) -> None: + """Release the chunking model a session's pipeline points at. - Args: - session_id: Current session id (excluded from the check). - expected_parameters: Parameter dict to compare against. - keys: Subset of keys to compare for equality. + The row is shared: it is keyed by configuration, so every session that + settled on the same chunking uses one record -- the common case, since + a new session takes the backend defaults. It may only go once no other + pipeline references it. - Returns: - True if at least one other session shares the same config values - for all specified keys. + Parameters + ---------- + session_id : int """ + pipeline = ( + self.db.query(RAGPipeline).filter_by(session_id=session_id).one_or_none() + ) + if pipeline is None or pipeline.chunking_model_id is None: + return - def _sort_params(params: dict[str, Any]) -> dict[str, Any]: - """Return a recursively canonicalized copy for deterministic comparison. - - Sorts dict keys, recursively sorts list elements (by canonical JSON), - and normalizes dicts inside lists so configs equal regardless of - key or list ordering. - """ + chunking_model_id = pipeline.chunking_model_id + # Release this session's claim first, so the count below sees the truth. + pipeline.chunking_model_id = None + self.db.flush() - def _canonical(value: Any) -> Any: - if isinstance(value, dict): - return { - key: _canonical(item) - for key, item in sorted( - value.items(), key=lambda kv: str(kv[0]) - ) - } - if isinstance(value, list): - return sorted( - (_canonical(item) for item in value), - key=lambda item: json.dumps(item, sort_keys=True, default=str), - ) - return value - - return {key: _canonical(item) for key, item in params.items()} + still_used = ( + self.db.query(RAGPipeline) + .filter(RAGPipeline.chunking_model_id == chunking_model_id) + .count() + ) + if still_used: + return + record = self.db.get(RAGChunkingModel, chunking_model_id) + if record is not None: + self.db.delete(record) - expected_parameters = _sort_params(expected_parameters) - other_sessions = ( - self.db.query(GenerativeSession) - .filter( - GenerativeSession.id != session_id, - GenerativeSession.task_name == "RAGTask", + def _embedding_model_in_use( + self, embedding_model_id: int, *, exclude_dense_retriever_id: int | None = None + ) -> bool: + """Whether anything still references an embedding model. + + Parameters + ---------- + embedding_model_id : int + exclude_dense_retriever_id : int | None + A dense retriever being deleted in the same transaction, which + should not count as a live reference. + + Returns + ------- + bool + """ + retrievers = self.db.query(RAGDenseRetriever).filter( + RAGDenseRetriever.embedding_model_id == embedding_model_id + ) + if exclude_dense_retriever_id is not None: + retrievers = retrievers.filter( + RAGDenseRetriever.id != exclude_dense_retriever_id ) - .all() + if retrievers.count(): + return True + return bool( + self.db.query(RAGEmbeddingMatrix) + .filter(RAGEmbeddingMatrix.embedding_model_id == embedding_model_id) + .count() ) - for other_session in other_sessions: - other_params = other_session.parameters or {} - other_params = _sort_params(other_params) - if all(other_params.get(k) == expected_parameters.get(k) for k in keys): - return True - return False - def _find_pipeline_id(self, session_id: int) -> int | None: """Find pipeline DB record ID for a session. @@ -345,8 +325,12 @@ def _cleanup_dense_retriever( RAGEmbeddingMatrix.id.in_(matrix_ids) ).delete(synchronize_session="fetch") - embedding_model = self.db.query(RAGEmbeddingModel).get(embedding_model_id) - if embedding_model is not None: + # Embedding models are also keyed by configuration alone, so another + # session's dense retriever or embedding matrix may still need this row. + embedding_model = self.db.get(RAGEmbeddingModel, embedding_model_id) + if embedding_model is not None and not self._embedding_model_in_use( + embedding_model_id, exclude_dense_retriever_id=dense_retriever.id + ): self.db.delete(embedding_model) self.db.delete(dense_retriever) @@ -366,7 +350,12 @@ def _cleanup_sparse_retriever( self._delete_path(sparse_retriever.storage_folder) self.db.delete(sparse_retriever) - def invalidate_document_artifacts(self, document_id: int) -> None: + def invalidate_document_artifacts( + self, + document_id: int, + *, + commit: bool = True, + ) -> None: """Delete all RAG artifacts associated with a document. When a document's extractor changes, all chunk sets, retrievers, @@ -375,8 +364,21 @@ def invalidate_document_artifacts(self, document_id: int) -> None: Also closes the orphaned-artifacts gap on document deletion. + The whole chunk set is deleted, including the chunks of sibling + documents in the same set. Documents belong to exactly one session, so + a chunk set does too: re-chunking the set is precisely what has to + happen when one of its documents changes. + Args: document_id: Document ID whose artifacts should be removed. + commit: When ``False`` the caller owns the transaction and is + responsible for committing (or rolling back). Use it to make + the invalidation part of a larger unit of work. + + The artifact directories are queued with + :func:`~DashAI.back.services.RAG.deferred_fs.remove_after_commit`, so + they are removed when the transaction commits and left alone if it does + not -- whoever owns that transaction. """ from DashAI.back.dependencies.database.models import ( RAGChunkSet, @@ -413,7 +415,7 @@ def invalidate_document_artifacts(self, document_id: int) -> None: .all() ) for sparse_detail, bridge in sparse_detail_links: - self._delete_path(sparse_detail.storage_folder) + remove_after_commit(self.db, sparse_detail.storage_folder) self.db.delete(bridge) self.db.delete(sparse_detail) @@ -438,7 +440,7 @@ def invalidate_document_artifacts(self, document_id: int) -> None: .all() ) for matrix in matrices: - self._delete_path(matrix.storage_folder) + remove_after_commit(self.db, matrix.storage_folder) self.db.delete(matrix) remaining = ( @@ -488,4 +490,5 @@ def invalidate_document_artifacts(self, document_id: int) -> None: if chunk_set is not None: self.db.delete(chunk_set) - self.db.commit() + if commit: + self.db.commit() diff --git a/DashAI/back/services/RAG/deferred_fs.py b/DashAI/back/services/RAG/deferred_fs.py new file mode 100644 index 000000000..34d01619c --- /dev/null +++ b/DashAI/back/services/RAG/deferred_fs.py @@ -0,0 +1,120 @@ +"""Filesystem removals that wait for the database transaction to commit. + +Deleting a file or a directory cannot be rolled back, so doing it while a +transaction is still open couples two things that can disagree: if the +transaction then fails, the rows survive and point at something that is gone. +Doing it after the commit by hand works, but only for as long as every call +site remembers to -- and that has now been the source of the same bug three +times over. + +Registering the removal against the session instead makes the ordering a +property of the transaction rather than of the caller's discipline: the paths +are removed when (and only when) the session that queued them commits, and are +dropped on rollback. + + remove_after_commit(db, blob_path) + db.delete(document) + db.commit() # the file goes here, not before + +A session may commit many times over its life -- a job doing several +operations, say. Each commit takes only what was queued since the last one, +because the queue lives in ``Session.info`` and is drained as it is read. + +Savepoints need more care than they first appear to. ``after_commit`` fires +when a savepoint is *released*, and both ``after_rollback`` and +``after_soft_rollback`` fire when one is rolled back -- so none of the three +means "the work is durable" on its own. What does is the end of the +transaction with no parent: it happens exactly once per real transaction, and +is preceded by ``after_commit`` only if that transaction committed. +""" + +import logging +import os +import shutil +from typing import List, Optional + +from sqlalchemy import event +from sqlalchemy.orm import Session + +log = logging.getLogger(__name__) + +#: Key under which pending paths live in ``Session.info``. +_PENDING_KEY = "dashai_pending_path_removals" +#: Set between a commit and the end of the transaction it belonged to. +_COMMITTED_KEY = "dashai_transaction_committed" + + +def remove_after_commit(db: Session, path: Optional[str]) -> None: + """Queue a filesystem path for removal once ``db`` commits. + + Parameters + ---------- + db : Session + The session whose commit should trigger the removal. + path : str | None + A file or directory. ``None`` and empty paths are ignored, so callers + do not have to guard a nullable column. + """ + if not path: + return + db.info.setdefault(_PENDING_KEY, []).append(path) + + +def remove_now(path: Optional[str]) -> None: + """Remove a file or directory immediately, warning if it cannot be removed. + + Prefer :func:`remove_after_commit`. This exists for paths that are not tied + to a transaction at all. + + Parameters + ---------- + path : str | None + """ + if not path: + return + try: + if os.path.isdir(path): + shutil.rmtree(path) + elif os.path.exists(path): + os.remove(path) + except OSError as exc: + log.warning("Failed to remove %s: %s", path, exc) + + +def _take_pending(session: Session) -> List[str]: + """Remove and return the paths queued against a session.""" + return session.info.pop(_PENDING_KEY, []) + + +@event.listens_for(Session, "after_commit") +def _mark_committed(session: Session) -> None: + """Note that a commit happened; which commit is settled below. + + This fires for a savepoint release as well as for the real thing, so it + cannot act on its own. + """ + session.info[_COMMITTED_KEY] = True + + +@event.listens_for(Session, "after_transaction_end") +def _settle_pending_paths(session: Session, transaction: object) -> None: + """Remove or discard the queued paths once the real transaction ends. + + ``after_commit`` and ``after_rollback`` both fire for savepoints, so neither + can be trusted to mean "the work is durable". The transaction that has no + parent is the outermost one, and it ends exactly once -- preceded by + ``after_commit`` if it committed, and not if it rolled back. That is the + only moment at which removing a file is safe. + """ + if getattr(transaction, "parent", None) is not None: + # An inner transaction or a savepoint ended. Releasing a savepoint is + # not a commit, so drop the mark it may have just left. + session.info.pop(_COMMITTED_KEY, None) + return + + committed = session.info.pop(_COMMITTED_KEY, False) + pending = _take_pending(session) + if not committed: + return + for path in pending: + remove_now(path) diff --git a/DashAI/back/services/RAG/document_service.py b/DashAI/back/services/RAG/document_service.py index 38a496f1a..1652d4ed8 100644 --- a/DashAI/back/services/RAG/document_service.py +++ b/DashAI/back/services/RAG/document_service.py @@ -2,7 +2,7 @@ import logging import mimetypes import os -from dataclasses import dataclass, field +from dataclasses import dataclass from datetime import datetime from typing import Dict, List, Optional, Tuple @@ -15,6 +15,7 @@ ) from DashAI.back.dependencies.database.models import ( GenerativeSession, + GenerativeSessionParameterHistory, RAGExtractor, ) from DashAI.back.models.RAG.documents import ( @@ -29,9 +30,17 @@ ) from DashAI.back.models.RAG.extractors.base_extractor import BaseExtractor from DashAI.back.models.RAG.utils import hash_function +from DashAI.back.services.RAG.deferred_fs import remove_after_commit log = logging.getLogger(__name__) +#: Only RAG sessions own documents. +_RAG_TASK_NAME = "RAGTask" +#: Session parameter key mirroring the documents a session owns. +_DOCUMENTS_KEY = "documents" +#: Sub-directory holding content-addressed document blobs. +_BLOBS_DIRNAME = "blobs" + _DOCUMENT_CLASSES: dict[DocumentFileType, type[BaseDocument]] = { DocumentFileType.TXT: TxtDocument, DocumentFileType.PDF: PDFDocument, @@ -49,17 +58,14 @@ class DocumentUploadResult: """Outcome of a document upload attempt. - ``duplicate`` is set when the same file (by content hash) already exists - and ``force=False``: in that case the caller should surface a conflict and - let the user decide whether to overwrite. ``created`` / ``updated`` are set - for the success paths. + ``duplicate`` is set when the session already holds this exact file (by + content hash), in which case nothing is modified and the caller should + surface a conflict. ``created`` is set for the success path. """ document: DocumentResponse created: bool = False - updated: bool = False duplicate: bool = False - affected_sessions: List[dict] = field(default_factory=list) class DocumentService: @@ -78,33 +84,232 @@ def __init__(self, db: Session, registry=None): self.db = db self._registry = registry - def _resolve_extractor(self, db_doc) -> "Optional[BaseExtractor]": - """Resolve the extractor for a document from its extractor_record.""" + def _resolve_extractor_ref( + self, db_doc + ) -> "Tuple[Optional[BaseExtractor], Optional[str], dict]": + """Resolve a document's extractor together with how it is configured. + + Callers need the component name and params, not just the instance: the + extraction cache signature is built from them. Deriving them separately + (e.g. reading ``params`` as ``{}`` while instantiating with the stored + params) makes the signature disagree with itself, which never hits the + cache and re-invalidates the index on every call. + + Parameters + ---------- + db_doc : DocumentDBModel + + Returns + ------- + tuple + ``(extractor, component_name, params)``. The extractor is ``None`` + when no component applies or no registry is available; the name and + params still describe what *would* be used. + """ extractor_record = db_doc.extractor_record # RAGExtractor or None if extractor_record is not None: component_name = extractor_record.component_name - params = extractor_record.params or {} + params = dict(extractor_record.params or {}) else: # Default by file type component_name = self._DEFAULT_EXTRACTORS.get(db_doc.file_type) - if component_name is None: - return None params = {} - if self._registry is None: - return None + if component_name is None or self._registry is None: + return None, component_name, params try: extractor_cls = self._registry[component_name]["class"] - return extractor_cls(**params) except KeyError: - return None + return None, component_name, params + return extractor_cls(**params), component_name, params + + def _resolve_extractor(self, db_doc) -> "Optional[BaseExtractor]": + """Resolve the extractor instance for a document.""" + extractor, _, _ = self._resolve_extractor_ref(db_doc) + return extractor # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ + def _get_rag_session_or_raise(self, session_id: int) -> GenerativeSession: + """Return the RAG session that may own documents, or raise. + + Parameters + ---------- + session_id : int + + Returns + ------- + GenerativeSession + + Raises + ------ + ValueError + If the session does not exist or is not a RAG session. + """ + session = self.db.get(GenerativeSession, session_id) + if session is None: + raise ValueError(f"GenerativeSession with ID {session_id} not found.") + if session.task_name != _RAG_TASK_NAME: + raise ValueError( + f"Session {session_id} is a '{session.task_name}' session; " + "only RAG sessions hold documents." + ) + return session + + @staticmethod + def _write_blob(docs_path: str, file_hash: str, file_content: bytes) -> str: + """Store bytes content-addressed and return their path. + + Naming files after ``file_name`` alone let two different uploads with + the same name resolve to the same path, so the second silently + overwrote the first. Keying on the content hash removes that collision + and lets sessions holding identical files share one file on disk. + + Parameters + ---------- + docs_path : str + Root documents directory. + file_hash : str + SHA-256 of the file contents. + file_content : bytes + + Returns + ------- + str + Absolute path of the stored blob. + """ + blobs_dir = os.path.join(docs_path, _BLOBS_DIRNAME) + os.makedirs(blobs_dir, exist_ok=True) + file_path = os.path.join(blobs_dir, file_hash) + # Identical content yields an identical path; writing again is a no-op + # apart from the cost, so skip it. + if not os.path.exists(file_path): + with open(file_path, "wb") as f: + f.write(file_content) + return file_path + + def _get_or_create_extractor_row( + self, component_name: str, params: dict + ) -> RAGExtractor: + """Return the extractor record for a configuration, creating it once. + + ``rag_extractor`` rows are immutable value objects, so identical + configurations should share one row. Creating a fresh row per call + leaked one row on every extractor change, and none of them could be + removed while a document still pointed at any of them. + + Parameters + ---------- + component_name : str + params : dict + Canonicalised (key-sorted) extractor parameters. + + Returns + ------- + RAGExtractor + An existing or freshly flushed record. + """ + existing = ( + self.db.query(RAGExtractor) + .filter_by(component_name=component_name, params=params) + .first() + ) + if existing is not None: + return existing + record = RAGExtractor(component_name=component_name, params=params) + self.db.add(record) + self.db.flush() + return record + + def _drop_extractor_if_orphaned(self, extractor_id: int | None) -> None: + """Delete an extractor record once no document references it. + + Call this *after* flushing the deletion of the documents that used to + reference it: ``document.extractor_id`` is NOT NULL, so removing the + record while a row still points at it fails on the next flush. + + Parameters + ---------- + extractor_id : int | None + """ + if extractor_id is None: + return + still_used = ( + self.db.query(DocumentDBModel) + .filter(DocumentDBModel.extractor_id == extractor_id) + .count() + ) + if still_used: + return + record = self.db.get(RAGExtractor, extractor_id) + if record is not None: + self.db.delete(record) + + def _sync_session_documents(self, session: GenerativeSession) -> None: + """Mirror the session's documents into its ``parameters`` list. + + The foreign key is the authority on which documents a session owns, + but ``parameters["documents"]`` is what the pipeline, the chunk-set + signature and the parameter history all read, so the two must be kept + in step. Clients never send this key; only the document endpoints + change it. + + Parameters + ---------- + session : GenerativeSession + """ + self.db.flush() + document_ids = [ + row[0] + for row in self.db.query(DocumentDBModel.id) + .filter_by(session_id=session.id) + .order_by(DocumentDBModel.id) + .all() + ] + parameters = dict(session.parameters or {}) + if parameters.get(_DOCUMENTS_KEY) == document_ids: + return + parameters[_DOCUMENTS_KEY] = document_ids + session.parameters = parameters + self.db.add( + GenerativeSessionParameterHistory( + session_id=session.id, parameters=parameters + ) + ) + + def _is_last_reference(self, file_path: str, exclude_id: int) -> bool: + """Whether this document is the last one pointing at a stored file. + + Blobs are shared by every session holding the same bytes (and legacy + rows migrated from the global library may share a path too), so a file + may only be removed when no other row references it. + + Parameters + ---------- + file_path : str + exclude_id : int + The document being deleted, ignored when counting references. + + Returns + ------- + bool + """ + if not file_path: + return False + others = ( + self.db.query(DocumentDBModel) + .filter( + DocumentDBModel.file_path == file_path, + DocumentDBModel.id != exclude_id, + ) + .count() + ) + return not others + def _to_response( self, doc: DocumentDBModel, base_url: str = "" ) -> DocumentResponse: @@ -141,6 +346,7 @@ def _to_response( return DocumentResponse( id=doc.id, + session_id=doc.session_id, file_name=doc.file_name, file_type=doc.file_type, file_hash=doc.file_hash, @@ -149,9 +355,6 @@ def _to_response( optional_metadata=doc.optional_metadata, extractor=extractor_dict, default_extractor=default_extractor_dict, - related_sessions=[s.id for s in doc.get_related_sessions] - if doc.get_related_sessions - else None, file_url=f"{base_url}/api/v1/document/{doc.id}/download", preview_url=f"{base_url}/api/v1/document/{doc.id}/view", ) @@ -172,19 +375,21 @@ def upload( file_name: str, file_type: str | DocumentFileType, docs_path: str, + session_id: int, optional_metadata: dict = None, registry=None, - force: bool = False, ) -> DocumentUploadResult: - """Upload a document, deduplicating by content hash. + """Upload a document into one RAG session. - Handles hash deduplication, file storage, and DB record creation. + Documents belong to exactly one session, so deduplication is per + session: uploading the same bytes into a *different* session creates a + second, independent document, free to pick its own extractor. + Re-uploading into the same session changes nothing and is reported as + a duplicate. - If a document with the same file hash already exists and ``force`` is - ``False``, nothing is modified and the result reports the existing - document together with its related sessions so the caller can ask the - user whether to overwrite. With ``force=True`` the file is rewritten - and RAG artifacts are invalidated. + The bytes are stored content-addressed under ``/blobs``, so + two sessions holding the same file share one file on disk and two + different files with the same name cannot overwrite each other. After the record is committed, extraction runs immediately (when a component registry is available) to populate the extraction cache so a @@ -201,24 +406,24 @@ def upload( File extension / type, e.g. ``DocumentFileType.PDF`` or ``"pdf"``. docs_path : str Directory on disk where the file will be written. + session_id : int + The RAG session that will own the document. optional_metadata : dict, optional Arbitrary metadata attached to the document. registry : ComponentRegistry, optional Component registry used to resolve extractors. When provided, default extraction runs on upload to warm the cache. - force : bool - If ``True`` and the file already exists, overwrite it instead of - reporting a duplicate. Returns ------- DocumentUploadResult - Result describing the created/updated document or the duplicate. + Result describing the created document or the duplicate. Raises ------ ValueError - If ``docs_path`` does not exist or a database error occurs. + If ``docs_path`` does not exist, the session does not exist or is + not a RAG session, or a database error occurs. RAGDocumentExtractionError If pre-extraction fails during upload. """ @@ -227,51 +432,32 @@ def upload( if not os.path.isdir(docs_path): raise ValueError(f"Documents folder does not exist: {docs_path}") + session = self._get_rag_session_or_raise(session_id) optional_metadata = optional_metadata or {} file_content_hash = hash_function(file_content) - file_path = os.path.join(docs_path, file_name) try: existing = ( self.db.query(DocumentDBModel) - .filter_by(file_hash=file_content_hash) + .filter_by(session_id=session_id, file_hash=file_content_hash) .first() ) - - if existing is not None and not force: - affected_sessions = [ - {"id": s.id, "name": s.name} for s in existing.get_related_sessions - ] - return DocumentUploadResult( - document=self._to_response(existing), - duplicate=True, - affected_sessions=affected_sessions, - ) - if existing is not None: - return self._overwrite_existing( - existing, - file_content, - file_path, - file_name, - optional_metadata, - registry, + return DocumentUploadResult( + document=self._to_response(existing), duplicate=True ) - with open(file_path, "wb") as f: - f.write(file_content) + file_path = self._write_blob(docs_path, file_content_hash, file_content) default_component = self._DEFAULT_EXTRACTORS.get(file_type) extractor_record = None if default_component: - extractor_record = RAGExtractor( - component_name=default_component, - params={}, + extractor_record = self._get_or_create_extractor_row( + default_component, {} ) - self.db.add(extractor_record) - self.db.flush() doc = DocumentDBModel( + session_id=session_id, file_name=file_name, file_type=file_type, file_path=file_path, @@ -280,12 +466,21 @@ def upload( extractor_id=extractor_record.id if extractor_record else None, ) self.db.add(doc) + self.db.flush() + self._sync_session_documents(session) self.db.commit() self.db.refresh(doc) if registry is not None: self._registry = registry - self._pre_extract_or_raise(doc.id, file_name) + try: + self._pre_extract_or_raise(doc.id, file_name) + except Exception: + # The caller is told the upload failed, so the session must + # not keep a document with no text: it would be listed in + # the panel and dragged into the next indexing run. + self.delete(doc.id) + raise return DocumentUploadResult(document=self._to_response(doc), created=True) @@ -293,60 +488,6 @@ def upload( log.exception(e) raise ValueError("Database error during document upload.") from e - def _overwrite_existing( - self, - existing: DocumentDBModel, - file_content: bytes, - file_path: str, - file_name: str, - optional_metadata: dict, - registry, - ) -> DocumentUploadResult: - """Overwrite an existing document's file and metadata on force re-upload. - - Re-extracts text and invalidates any RAG artifacts tied to the - document so downstream fitted models are recomputed on the next run. - - Parameters - ---------- - existing : DocumentDBModel - The existing document row matched by content hash. - file_content : bytes - Raw file bytes to write to disk. - file_path : str - Absolute path of the uploaded file. - file_name : str - Original file name. - optional_metadata : dict - Metadata to attach to the document. - registry : ComponentRegistry, optional - Registry used to resolve extractors for re-extraction. - - Returns - ------- - DocumentUploadResult - Result marking the document as updated. - """ - from DashAI.back.services.RAG.cleanup_service import CleanupService - - with open(file_path, "wb") as f: - f.write(file_content) - - existing.file_name = file_name - existing.file_path = file_path - existing.optional_metadata = optional_metadata - existing.last_modified = datetime.now() - self.db.commit() - self.db.refresh(existing) - - CleanupService(self.db).invalidate_document_artifacts(existing.id) - - if registry is not None: - self._registry = registry - self._pre_extract_or_raise(existing.id, file_name) - - return DocumentUploadResult(document=self._to_response(existing), updated=True) - def _pre_extract_or_raise(self, document_id: int, file_name: str) -> None: """Extract text after upload, raising a typed error on failure. @@ -395,31 +536,10 @@ def get(self, document_id: int) -> DocumentResponse: log.exception(e) raise ValueError("Database error retrieving document.") from e - def get_all(self, base_url: str = "") -> List[DocumentResponse]: - """Get all documents with ``file_url`` included. - - Parameters - ---------- - base_url : str - Base URL prefix for download links. - - Returns - ------- - list[DocumentResponse] - """ - try: - docs: List[DocumentDBModel] = self.db.query(DocumentDBModel).all() - return [self._to_response(d, base_url) for d in docs] - except exc.SQLAlchemyError as e: - log.exception(e) - raise ValueError("Database error listing documents.") from e - def get_by_session( self, session_id: int, base_url: str = "" ) -> List[DocumentResponse]: - """Get documents linked to a generative session. - - Documents are identified from ``session.parameters["documents"]``. + """Get the documents owned by a generative session. Parameters ---------- @@ -435,13 +555,10 @@ def get_by_session( if session is None: raise ValueError(f"GenerativeSession with ID {session_id} not found.") - document_ids: List[int] = session.parameters.get("documents", []) - if not document_ids: - return [] - docs = ( self.db.query(DocumentDBModel) - .filter(DocumentDBModel.id.in_(document_ids)) + .filter_by(session_id=session_id) + .order_by(DocumentDBModel.id) .all() ) return [self._to_response(d, base_url) for d in docs] @@ -451,7 +568,11 @@ def get_by_session( raise ValueError("Database error retrieving session documents.") from e def delete(self, document_id: int) -> None: - """Delete a document file from disk and its DB record. + """Delete a document, its RAG artifacts, and its file. + + The chunks, retrievers and embedding matrices fitted over this document + are invalidated first: leaving them behind kept stale directories on + disk that nothing would ever reclaim. Parameters ---------- @@ -462,21 +583,80 @@ def delete(self, document_id: int) -> None: ValueError If the document does not exist. """ + from DashAI.back.services.RAG.cleanup_service import CleanupService + try: doc = self._get_document_or_raise(document_id) + session = self.db.get(GenerativeSession, doc.session_id) + file_path = doc.file_path - if os.path.exists(doc.file_path): - os.remove(doc.file_path) + CleanupService(self.db).invalidate_document_artifacts( + document_id, commit=False + ) + # Decided while the rows are still here; removed only if the commit + # below succeeds. + if self._is_last_reference(file_path, document_id): + remove_after_commit(self.db, file_path) + extractor_id = doc.extractor_id self.db.delete(doc) + self.db.flush() + self._drop_extractor_if_orphaned(extractor_id) + if session is not None: + self._sync_session_documents(session) self.db.commit() - except exc.SQLAlchemyError as e: + self.db.rollback() log.exception(e) raise ValueError("Database error deleting document.") from e - except OSError as e: + + def delete_by_session(self, session_id: int, *, commit: bool = True) -> None: + """Delete every document a session owns, with its files and artifacts. + + Called before a session is deleted. The ORM cascade would drop the rows + on its own, but nothing would remove the files or the fitted artifacts + from disk. + + Files are queued against the session and removed when it commits, so a + caller that owns the transaction gets the right ordering for free. + + Parameters + ---------- + session_id : int + commit : bool + When ``False`` the caller owns the transaction and commits itself. + The bulk session delete relies on this: it removes several sessions + under one transaction, and committing part-way would leave the + already-processed sessions gone if a later one failed. + """ + from DashAI.back.services.RAG.cleanup_service import CleanupService + + documents = ( + self.db.query(DocumentDBModel).filter_by(session_id=session_id).all() + ) + if not documents: + return + + try: + extractor_ids = set() + for doc in documents: + CleanupService(self.db).invalidate_document_artifacts( + doc.id, commit=False + ) + if self._is_last_reference(doc.file_path, doc.id): + remove_after_commit(self.db, doc.file_path) + extractor_ids.add(doc.extractor_id) + self.db.delete(doc) + self.db.flush() + for extractor_id in extractor_ids: + self._drop_extractor_if_orphaned(extractor_id) + if commit: + self.db.commit() + except exc.SQLAlchemyError as e: + if commit: + self.db.rollback() log.exception(e) - raise ValueError("Error deleting physical file.") from e + raise ValueError("Database error deleting session documents.") from e def update_metadata( self, @@ -675,11 +855,9 @@ def extract_text( ) from err extractor = extractor_cls(**params) else: - extractor = self._resolve_extractor(doc) + extractor, component_name, params = self._resolve_extractor_ref(doc) if extractor is None: raise ValueError(f"No extractor available for document {document_id}") - component_name = extractor.__class__.__name__ - params = {} # Check compatibility supported = getattr(extractor, "SUPPORTED_FILE_TYPES", []) @@ -725,12 +903,18 @@ def extract_text( char_count = len(text) if existing is not None: - CleanupService(self.db).invalidate_document_artifacts(document_id) + # A different extractor or different params produced different + # text, so everything fitted over the old text is stale. + CleanupService(self.db).invalidate_document_artifacts( + document_id, commit=False + ) existing.content = text existing.signature = signature existing.char_count = char_count created, updated = False, True else: + # Nothing has been chunked yet -- chunking needs extracted text -- + # so there are no artifacts to invalidate on a first extraction. cache_entry = ProcessedDocumentContent( document_id=document_id, content=text, @@ -751,84 +935,126 @@ def extract_text( } def update_extractor( - self, document_id: int, extractor_ref: dict, force: bool = False + self, document_id: int, extractor_ref: dict ) -> "DocumentResponse": - """Persist an extractor choice via rag_extractor table. + """Change a document's extractor, re-extract it, and drop stale artifacts. - When the extractor changes, the single ``processed_document_content`` - row is re-extracted with the new extractor. With ``force=True`` (or - when the document has no linked sessions) RAG artifacts are - invalidated via ``CleanupService.invalidate_document_artifacts``. + The whole operation is one transaction, and the extraction runs *before* + anything is mutated. Committing the new ``extractor_id`` first meant a + failing extractor (a malformed PDF under ``strict=True``, say) left the + document pointing at an extractor that had never produced its text, + still serving the previous extractor's chunks. + + Artifacts are invalidated unconditionally. The old ``force`` flag was + meant to make the user confirm a destructive re-index, but it asked + ``RAGDocumentPipelineSessionLink`` -- a table nothing ever wrote to -- + which sessions were affected, so the confirmation never triggered. + A document now belongs to exactly one session, so there is nobody else + to warn. Args: document_id: Document ID. - extractor_ref: {component, params} dict. - force: If True, skip confirmation and invalidate artifacts. + extractor_ref: ``{component, params}`` dict. Returns: - DocumentResponse with updated extractor. + DocumentResponse with the updated extractor. Raises: - ValueError if document not found or extractor invalid. + ValueError: If the document does not exist, or the extractor is + unknown or incompatible with the document's file type. + RAGDocumentExtractionError: If extraction with the new extractor + fails. Nothing is changed in that case. """ - from DashAI.back.dependencies.database.models import RAGExtractor + from DashAI.back.dependencies.database.models import ProcessedDocumentContent from DashAI.back.services.RAG.cleanup_service import CleanupService doc = self._get_document_or_raise(document_id) component_name = extractor_ref.get("component") - params = extractor_ref.get("params", {}) - if not component_name: raise ValueError("extractor_ref must include 'component' key") + # Canonical (key-sorted) form, so the same configuration always hashes + # and compares equal. + params = dict(sorted((extractor_ref.get("params") or {}).items())) - # Validate extractor exists and is compatible - if self._registry is not None: - try: - extractor_cls = self._registry[component_name]["class"] - except KeyError as err: - raise ValueError( - f"Extractor '{component_name}' not found in registry" - ) from err + if self._registry is None: + raise ValueError("No registry available to resolve extractor") + try: + extractor_cls = self._registry[component_name]["class"] + except KeyError as err: + raise ValueError( + f"Extractor '{component_name}' not found in registry" + ) from err - supported = getattr(extractor_cls, "SUPPORTED_FILE_TYPES", []) - if supported and doc.file_type not in supported: - raise ValueError( - f"Extractor '{component_name}' does not support file type " - f"'{doc.file_type}'. Supported types: {supported}" - ) + supported = getattr(extractor_cls, "SUPPORTED_FILE_TYPES", []) + if supported and doc.file_type not in supported: + raise ValueError( + f"Extractor '{component_name}' does not support file type " + f"'{doc.file_type}'. Supported types: {supported}" + ) - if not force: - linked_session_ids = self.get_related_sessions(document_id) - if linked_session_ids: - raise ValueError( - f"Document is linked to {len(linked_session_ids)} RAG " - f"pipeline(s). Changing the extractor will delete existing " - f"chunks and retrievers. Use force=true to proceed." - ) + signature = self._build_text_signature(doc.file_hash, component_name, params) + cached = ( + self.db.query(ProcessedDocumentContent) + .filter_by(document_id=document_id) + .first() + ) - # Create a new RAGExtractor record (no dedup for now — simple approach) - extractor_record = RAGExtractor( - component_name=component_name, - params=params if params else None, + # Saving the extractor it already has must not throw away a good index. + current = doc.extractor_record + unchanged = ( + current is not None + and current.component_name == component_name + and dict(current.params or {}) == params + and cached is not None + and cached.signature == signature ) - self.db.add(extractor_record) - self.db.flush() # Get the ID - doc.extractor_id = extractor_record.id - doc.last_modified = datetime.now() + if unchanged: + return self._to_response(doc) - if force: - CleanupService(self.db).invalidate_document_artifacts(document_id) + extractor = extractor_cls(**params) + try: + text = extractor.extract(doc.file_path) + except Exception as e: + log.exception(e) + raise RAGDocumentExtractionError( + f"Failed to extract text from '{doc.file_name}' with " + f"'{component_name}': {e}" + ) from e - self.db.commit() - self.db.refresh(doc) + try: + record = self._get_or_create_extractor_row(component_name, params) + previous_extractor_id = doc.extractor_id + doc.extractor_id = record.id + doc.last_modified = datetime.now() - # Re-extract with the new extractor so the single processed content row - # reflects the new extractor config (keeps the 1:1 invariant). - if self._registry is not None: - self.extract_text(document_id, extractor_ref=extractor_ref) - self.db.refresh(doc) + CleanupService(self.db).invalidate_document_artifacts( + document_id, commit=False + ) + + if cached is not None: + cached.content = text + cached.signature = signature + cached.char_count = len(text) + else: + self.db.add( + ProcessedDocumentContent( + document_id=document_id, + content=text, + signature=signature, + char_count=len(text), + ) + ) + + if previous_extractor_id != record.id: + self._drop_extractor_if_orphaned(previous_extractor_id) + + self.db.commit() + except Exception: + self.db.rollback() + raise + self.db.refresh(doc) return self._to_response(doc) def validate_exist(self, document_ids: List[int]) -> None: @@ -853,28 +1079,38 @@ def validate_exist(self, document_ids: List[int]) -> None: if missing: raise ValueError(f"Documents with IDs {', '.join(missing)} not found.") - def get_related_sessions(self, document_id: int) -> List[int]: - """Get session IDs linked to a document. + def validate_belong_to_session( + self, document_ids: List[int], session_id: int + ) -> None: + """Raise ``ValueError`` unless every document belongs to the session. + + A session must never reference another session's document: the two + would share chunks and an extractor choice. Parameters ---------- - document_id : int - - Returns - ------- - list[int] - Session IDs related to the document. + document_ids : list[int] + session_id : int Raises ------ ValueError - If the document is not found. + If a document is missing or owned by a different session. """ - try: - doc = self._get_document_or_raise(document_id) - if not doc.get_related_sessions: - return [] - return [s.id for s in doc.get_related_sessions] - except exc.SQLAlchemyError as e: - log.exception(e) - raise ValueError("Database error retrieving related sessions.") from e + if not document_ids: + return + rows = ( + self.db.query(DocumentDBModel.id, DocumentDBModel.session_id) + .filter(DocumentDBModel.id.in_(document_ids)) + .all() + ) + owner_by_id = {row.id: row.session_id for row in rows} + missing = [str(i) for i in document_ids if i not in owner_by_id] + if missing: + raise ValueError(f"Documents with IDs {', '.join(missing)} not found.") + foreign = [str(i) for i in document_ids if owner_by_id[i] != session_id] + if foreign: + raise ValueError( + f"Documents with IDs {', '.join(foreign)} belong to a different " + "session." + ) diff --git a/DashAI/back/services/RAG/embedding_storage_service.py b/DashAI/back/services/RAG/embedding_storage_service.py index 838f73179..6a04c1217 100644 --- a/DashAI/back/services/RAG/embedding_storage_service.py +++ b/DashAI/back/services/RAG/embedding_storage_service.py @@ -7,6 +7,7 @@ import numpy as np +from DashAI.back.core.atomic import atomic_open from DashAI.back.dependencies.database.models import RAGEmbeddingMatrix from DashAI.back.models.RAG.exceptions import RAGEmbeddingLoadError from DashAI.back.services.RAG.retriever_db_service import RetrieverDBService @@ -93,7 +94,8 @@ def save_embeddings( Steps ----- 1. Create the directory via :meth:`_matrix_dir`. - 2. Write the array to ``embeddings.npy`` with :func:`numpy.save`. + 2. Write the array to ``embeddings.npy`` atomically, so a killed + indexing job never leaves a truncated matrix behind. 3. Persist a matching ``RAGEmbeddingMatrix`` record through :meth:`RetrieverDBService.save_embedding_matrix`. @@ -117,7 +119,10 @@ def save_embeddings( os.makedirs(matrix_dir, exist_ok=True) matrix_path = self._matrix_path(doc_id, chunk_set_id, embedding_model_id) - np.save(matrix_path, embeddings) + # np.save appends '.npy' to a path but not to a file object, which is + # what keeps the temp file and the final name in agreement. + with atomic_open(matrix_path, "wb") as f: + np.save(f, embeddings) record = self._db_service.save_embedding_matrix( document_id=doc_id, diff --git a/DashAI/back/services/RAG/index_job_service.py b/DashAI/back/services/RAG/index_job_service.py new file mode 100644 index 000000000..b5f32b052 --- /dev/null +++ b/DashAI/back/services/RAG/index_job_service.py @@ -0,0 +1,99 @@ +"""Resolving and cancelling a RAG session's indexing job. + +``GenerativeSession.index_job_id`` is only a pointer. The job queue's +``task_copy`` table is what actually knows whether that job is alive, so every +question about indexing state is answered by asking the queue, never by +trusting the column. That is what makes a stale pointer harmless: a job whose +worker was killed is flipped to ``killed`` by the queue's watchdog within +seconds, and stops looking live on its own. +""" + +import logging +from typing import Any, Dict, Optional + +from DashAI.back.dependencies.database.models import GenerativeSession +from DashAI.back.dependencies.job_queues.base_job_queue import ( + BaseJobQueue, + JobQueueError, +) + +log = logging.getLogger(__name__) + +#: Queue states in which a job still has work left to do. +_LIVE_STATUSES = ("not_started", "started") + + +def get_index_job( + session: GenerativeSession, + job_queue: Optional[BaseJobQueue], +) -> Optional[Dict[str, Any]]: + """Return the queue state of a session's indexing job, alive or not. + + Includes finished and failed jobs on purpose: a failed index has to stay + visible after a page reload, and the queue row is the only place holding + the error message. + + Parameters + ---------- + session : GenerativeSession + The session whose ``index_job_id`` should be resolved. + job_queue : Optional[BaseJobQueue] + The queue to ask. ``None`` disables resolution entirely, which is what + callers that have no queue handy (most tests) want. + + Returns + ------- + Optional[dict] + The queue's status dict, or ``None`` when there is no job to resolve. + """ + if job_queue is None or not session.index_job_id: + return None + try: + return job_queue.status(session.index_job_id) + except JobQueueError: + # Dismissed from task_copy: the job is gone, and so is any record of it. + return None + except Exception: # pragma: no cover - status must never break a read + log.exception("Index job lookup failed for session %s", session.id) + return None + + +def get_live_index_job( + session: GenerativeSession, + job_queue: Optional[BaseJobQueue], +) -> Optional[Dict[str, Any]]: + """Return the session's indexing job only while it still has work to do.""" + state = get_index_job(session, job_queue) + if state is None or state.get("status") not in _LIVE_STATUSES: + return None + return state + + +def cancel_live_index_job( + session: GenerativeSession, + job_queue: Optional[BaseJobQueue], +) -> bool: + """Stop an in-flight index for a session whose inputs are about to change. + + Call this *before* mutating documents or parameters. Otherwise the request + handler and the indexing worker race over the very same chunk, retriever + and embedding rows — the cleanup deleting what the job is still writing. + + Clears ``index_job_id`` but does not commit; the caller's own transaction + is what makes the change durable. + + Returns + ------- + bool + Whether a live job was found and cancelled. + """ + if get_live_index_job(session, job_queue) is None: + return False + job_id = session.index_job_id + try: + job_queue.cancel(job_id, reason="cancelled") + except Exception: # pragma: no cover - a doomed job must not block the write + log.exception("Could not cancel index job %s", job_id) + session.index_job_id = None + log.debug("Cancelled index job %s for session %s", job_id, session.id) + return True diff --git a/DashAI/back/services/RAG/index_status_service.py b/DashAI/back/services/RAG/index_status_service.py index adef13137..fc15fc2a6 100644 --- a/DashAI/back/services/RAG/index_status_service.py +++ b/DashAI/back/services/RAG/index_status_service.py @@ -1,9 +1,13 @@ """Read-only view of whether a RAG session's documents are already indexed. -Indexing is not a job of its own: chunking, embedding and retriever fitting all -happen inside ``RAGJob`` while answering a chat message, and everything is -content-addressed, so "is this indexed?" is answered by looking for the rows the -pipeline would otherwise create. +Indexing normally runs as ``RAGIndexJob``, started when documents change or a +configuration change invalidates the index. Everything it builds is +content-addressed, so "is this indexed?" is answered by looking for the rows +that job would otherwise create — which also covers the fallback case where the +chat job did the indexing itself. + +Whether a run is currently *in flight* is the one thing the database cannot +answer, so that comes from the job queue via ``index_job_service``. This service only reads. It never chunks, embeds or writes, so it is safe to call on every page load. @@ -25,6 +29,7 @@ GenerativeSessionParameterHistory, RAGChunkSet, ) +from DashAI.back.dependencies.job_queues.base_job_queue import BaseJobQueue from DashAI.back.dependencies.registry.component_registry import ComponentRegistry from DashAI.back.models.RAG.RAG_constants import ( RAG_PARAM_CHUNKING_MODEL, @@ -34,43 +39,53 @@ from DashAI.back.models.RAG.retrievers.dense.dense_retriever import DenseRetriever from DashAI.back.models.RAG.retrievers.sparse.sparse_retriever import SparseRetriever from DashAI.back.services.RAG.chunking_service import ChunkingService +from DashAI.back.services.RAG.index_job_service import get_index_job, get_live_index_job from DashAI.back.services.RAG.retriever_db_service import RetrieverDBService log = logging.getLogger(__name__) +#: The session holds no documents, so there is nothing to index yet. +STATUS_NO_DOCUMENTS = "no_documents" #: Session has never been indexed under any configuration. STATUS_NOT_INDEXED = "not_indexed" #: A previous configuration was indexed, the current one is not. STATUS_STALE = "stale" +#: An indexing job is running right now. +STATUS_INDEXING = "indexing" #: Everything the pipeline needs is already on disk and in the database. STATUS_INDEXED = "indexed" _MESSAGES = { + STATUS_NO_DOCUMENTS: MultilingualString( + en="Add a document to this session to start asking questions.", + es="Agrega un documento a esta sesión para empezar a hacer preguntas.", + pt="Adicione um documento a esta sessão para começar a fazer perguntas.", + de="Fügen Sie dieser Sitzung ein Dokument hinzu, um Fragen zu stellen.", + zh="向此会话添加文档后即可开始提问。", + ), STATUS_NOT_INDEXED: MultilingualString( - en="The documents will be indexed when you send your first message.", - es="Los documentos se indexarán cuando envíes tu primer mensaje.", - pt="Os documentos serão indexados quando você enviar a primeira mensagem.", - de="Die Dokumente werden beim Senden der ersten Nachricht indexiert.", - zh="文档将在你发送第一条消息时建立索引。", + en="These documents are not indexed yet.", + es="Estos documentos aún no están indexados.", + pt="Estes documentos ainda não estão indexados.", + de="Diese Dokumente sind noch nicht indexiert.", + zh="这些文档尚未建立索引。", ), STATUS_STALE: MultilingualString( - en=( - "The configuration changed, so the documents will be re-indexed " - "with your next message." - ), - es=( - "La configuración cambió, así que los documentos se reindexarán " - "en tu próximo mensaje." - ), - pt=( - "A configuração mudou, então os documentos serão reindexados na " - "sua próxima mensagem." - ), + en="The configuration changed, so the documents need to be re-indexed.", + es=("La configuración cambió, así que los documentos se deben reindexar."), + pt=("A configuração mudou, então os documentos precisam ser reindexados."), de=( - "Die Konfiguration hat sich geändert, daher werden die Dokumente " - "mit der nächsten Nachricht neu indexiert." + "Die Konfiguration hat sich geändert, daher müssen die Dokumente " + "neu indexiert werden." ), - zh="配置已更改，文档将在你的下一条消息时重新建立索引。", + zh="配置已更改，文档需要重新建立索引。", + ), + STATUS_INDEXING: MultilingualString( + en="Indexing the documents…", + es="Indexando los documentos…", + pt="Indexando os documentos…", + de="Die Dokumente werden indexiert…", + zh="正在为文档建立索引…", ), STATUS_INDEXED: MultilingualString( en="The documents are indexed and ready to answer questions.", @@ -85,7 +100,12 @@ class IndexStatusService: """Reports the indexing state of a RAG session without mutating anything.""" - def __init__(self, db: Session, registry: ComponentRegistry): + def __init__( + self, + db: Session, + registry: ComponentRegistry, + job_queue: Optional[BaseJobQueue] = None, + ): """Initialise the service. Parameters @@ -94,9 +114,14 @@ def __init__(self, db: Session, registry: ComponentRegistry): SQLAlchemy session used for the read-only lookups. registry : ComponentRegistry Registry used to tell dense retrievers from sparse ones. + job_queue : Optional[BaseJobQueue] + Queue consulted for a running indexing job. Optional so callers + that only care about what is persisted can omit it; without it the + service simply never reports ``indexing``. """ self._db = db self._registry = registry + self._job_queue = job_queue # Reused so the signature matches the one the pipeline computes; a # second implementation would drift and misreport a stale index. self._chunking = ChunkingService(db, registry) @@ -129,9 +154,9 @@ def get_status(self, session_id: int) -> Dict[str, Any]: raise ValueError(f"Generative session {session_id} does not exist.") parameters = dict(session.parameters or {}) - # Documents come from the session parameters, never from - # RAGDocumentPipelineSessionLink: nothing in production writes that - # table, so it is always empty. + # Read from the parameters rather than the session's documents + # relationship: _was_indexed_before compares against historized + # parameters, and both sides have to come from the same source. document_ids = [ doc_id for doc_id in parameters.get(RAG_PARAM_DOCUMENTS) or [] if doc_id ] @@ -145,11 +170,16 @@ def get_status(self, session_id: int) -> Dict[str, Any]: parameters.get(RAG_PARAM_RETRIEVER_MODEL), chunk_set.id ) all_chunked = bool(document_ids) and all(doc["indexed"] for doc in documents) + # Resolved even when the job is already over: a failed run has to stay + # visible after a reload, and the queue row holds the error message. + job = get_index_job(session, self._job_queue) status = self._resolve_status( session_id=session_id, parameters=parameters, all_chunked=all_chunked, retriever_ready=retriever_ready, + has_documents=bool(document_ids), + is_indexing=get_live_index_job(session, self._job_queue) is not None, ) return { @@ -159,6 +189,15 @@ def get_status(self, session_id: int) -> Dict[str, Any]: "retriever_ready": retriever_ready, "documents": documents, "message": _MESSAGES[status], + "job_id": session.index_job_id if job else None, + "job": { + "status": job["status"], + "progress": job["progress"], + "progress_message": job["progress_message"], + "error": job["error"], + } + if job + else None, } # ── Private helpers ─────────────────────────────────────────────── @@ -169,8 +208,19 @@ def _resolve_status( parameters: Dict[str, Any], all_chunked: bool, retriever_ready: bool, + has_documents: bool, + is_indexing: bool = False, ) -> str: - """Classify the session into one of the three indexing states.""" + """Classify the session into one of the five indexing states.""" + if not has_documents: + # Reporting "not indexed" here would promise an indexing run that + # cannot happen, and hide the one thing the user has to do. + return STATUS_NO_DOCUMENTS + if is_indexing: + # Beats "indexed" on purpose: a re-index of a stale configuration + # finds the old rows still in place, and reporting it as done would + # invite a question the pipeline cannot yet answer. + return STATUS_INDEXING if all_chunked and retriever_ready: return STATUS_INDEXED if self._was_indexed_before(session_id, parameters): diff --git a/DashAI/back/services/RAG/prompt_service.py b/DashAI/back/services/RAG/prompt_service.py index 20421219a..edc6b12a8 100644 --- a/DashAI/back/services/RAG/prompt_service.py +++ b/DashAI/back/services/RAG/prompt_service.py @@ -3,12 +3,10 @@ from datetime import datetime from typing import Any -from sqlalchemy import exc, select +from sqlalchemy import exc from sqlalchemy.orm import Session from DashAI.back.dependencies.database.models import ( - GenerativeSession, - GenerativeSessionParameterHistory, RAGPrompt, ) from DashAI.back.dependencies.registry.component_registry import ComponentRegistry @@ -190,88 +188,6 @@ def _find_by_hash(self, params: dict[str, Any]) -> RAGPrompt | None: log.exception(e) raise RAGDatabaseError("Error looking up prompt by hash.") from e - def update( - self, - prompt_id: int, - name: str | None = None, - parameters: dict[str, Any] | None = None, - ) -> PromptResponse: - """Update an existing prompt in place. - - Validates the template if parameters change. - - Args: - prompt_id: Primary key of the prompt to update. - name: New name (optional). - parameters: New parameters including ``template`` or - ``templates`` (optional). - - Returns: - The updated prompt response. - - Raises: - RAGPromptValidationError: If not found or validation fails. - RAGDatabaseError: If a database error occurs. - """ - try: - prompt = self.db.get(RAGPrompt, prompt_id) - except exc.SQLAlchemyError as e: - log.exception(e) - raise RAGDatabaseError("Error retrieving prompt from database.") from e - - if prompt is None: - raise RAGPromptValidationError( - f"Prompt with ID {prompt_id} does not exist." - ) - - changed = False - - if name is not None: - name = name.strip() - if not name: - raise RAGPromptValidationError("Prompt name cannot be empty.") - if name != prompt.name: - prompt.name = name - changed = True - - if parameters is not None: - if prompt.class_name not in self._registry: - raise RAGPromptValidationError( - f"Component {prompt.class_name} is not registered in the registry." - ) - prompt_class = self._registry[prompt.class_name]["class"] - if not issubclass(prompt_class, Prompt): - raise RAGPromptValidationError( - f"Component {prompt.class_name} is not a valid Prompt subclass." - ) - - if "templates" in parameters: - for _lang, tmpl in parameters["templates"].items(): - self._validate_prompt_template(prompt.class_name, tmpl) - elif "template" in parameters: - self._validate_prompt_template( - prompt.class_name, parameters["template"] - ) - else: - raise RAGPromptValidationError( - "Prompt parameters must include 'template' or 'templates'." - ) - prompt.parameters = parameters - prompt.parameters_hash = build_parameters_hash(parameters) - changed = True - - if not changed: - return self._serialize_prompt(prompt) - - try: - self.db.commit() - self.db.refresh(prompt) - return self._serialize_prompt(prompt) - except exc.SQLAlchemyError as e: - self.db.rollback() - log.exception(e) - raise RAGDatabaseError("Error updating prompt in database.") from e - def get_all(self) -> list[PromptResponse]: """Get all prompts. @@ -319,109 +235,6 @@ def get_all(self) -> list[PromptResponse]: log.exception(e) raise RAGDatabaseError("Error listing prompts in database.") from e - def create_session_copy( - self, - prompt_id: int, - session_id: int, - parameters: dict[str, Any] | None = None, - name: str | None = None, - ) -> PromptResponse: - """Create a session-scoped copy of a prompt. - - Adds ``cloned_for_session`` to the parameters dict to avoid UNIQUE - constraint collisions. Generates a unique name and updates the - session's parameters dict with the new prompt_id. - - Args: - prompt_id: Primary key of the prompt to copy. - session_id: Target session id. - parameters: Override parameters for the copy (optional). - name: Override name for the copy (optional). - - Returns: - The newly created prompt response. - - Raises: - RAGPromptValidationError: If the prompt or session does not exist. - RAGDatabaseError: If a database error occurs. - """ - try: - existing_prompt = self.db.get(RAGPrompt, prompt_id) - except exc.SQLAlchemyError as e: - log.exception(e) - raise RAGDatabaseError("Error retrieving prompt from database.") from e - - if existing_prompt is None: - raise RAGPromptValidationError( - f"Prompt with ID {prompt_id} does not exist." - ) - - try: - session = self.db.get(GenerativeSession, session_id) - except exc.SQLAlchemyError as e: - log.exception(e) - raise RAGDatabaseError("Error retrieving session from database.") from e - - if session is None: - raise RAGPromptValidationError( - f"GenerativeSession with ID {session_id} not found." - ) - - if parameters is not None: - if "templates" in parameters: - for _lang, tmpl in parameters["templates"].items(): - self._validate_prompt_template(existing_prompt.class_name, tmpl) - elif "template" in parameters: - self._validate_prompt_template( - existing_prompt.class_name, parameters["template"] - ) - else: - raise RAGPromptValidationError( - "Prompt parameters must include 'template' or 'templates'." - ) - new_parameters = parameters - else: - new_parameters = existing_prompt.parameters - - new_parameters = dict(new_parameters or {}) - new_parameters["cloned_for_session"] = session_id - - base_name = (name or existing_prompt.name or existing_prompt.class_name).strip() - new_name = self._build_session_prompt_name(base_name, session_id) - - try: - params_hash = build_parameters_hash(new_parameters) - new_prompt = RAGPrompt( - class_name=existing_prompt.class_name, - name=new_name, - parameters=new_parameters, - parameters_hash=params_hash, - ) - self.db.add(new_prompt) - self.db.commit() - self.db.refresh(new_prompt) - - session_parameters = dict(session.parameters or {}) - session_parameters["prompt_id"] = new_prompt.id - session.parameters = session_parameters - session.last_modified = datetime.now() - self.db.add( - GenerativeSessionParameterHistory( - session_id=session.id, - parameters=session_parameters, - modified_at=datetime.now(), - ) - ) - self.db.commit() - self.db.refresh(session) - - return self._serialize_prompt(new_prompt) - - except exc.SQLAlchemyError as e: - self.db.rollback() - log.exception(e) - raise RAGDatabaseError("Error creating session copy in database.") from e - def validate_template(self, class_name: str, template: str) -> None: """Validate a template against the prompt class's required placeholders. @@ -583,14 +396,3 @@ def _serialize_prompt(self, prompt: RAGPrompt) -> PromptResponse: created=prompt.created, last_modified=prompt.last_modified, ) - - def _build_session_prompt_name(self, base_name: str, session_id: int) -> str: - """Generate a unique name for a session-scoped prompt copy.""" - candidate = f"{base_name} - session {session_id}" - suffix = 2 - while self.db.execute( - select(RAGPrompt.id).where(RAGPrompt.name == candidate) - ).scalar(): - candidate = f"{base_name} - session {session_id} ({suffix})" - suffix += 1 - return candidate diff --git a/DashAI/back/services/RAG/session_defaults_service.py b/DashAI/back/services/RAG/session_defaults_service.py index adde49ae1..18bdf0d29 100644 --- a/DashAI/back/services/RAG/session_defaults_service.py +++ b/DashAI/back/services/RAG/session_defaults_service.py @@ -3,7 +3,7 @@ ``schema_field`` never sets a pydantic default, so historically the client had to send a fully expanded configuration for all four RAG components. This module resolves that configuration server-side, which is what lets a session be created -from just a name, its documents and a generation model. +from just a name and a generation model. The defaults are deliberately expressed as *presets* rather than loose parameter bags: every default configuration therefore matches a named preset, so the @@ -56,8 +56,9 @@ def build_default_parameters( ) -> Dict[str, Any]: """Return the default configuration for the components the user need not pick. - ``documents`` and ``generation_model`` are intentionally absent: neither has - a sensible default, so both stay required at session creation. + ``generation_model`` is intentionally absent: it has no sensible default, + so it stays required at session creation. ``documents`` is absent because a + session starts empty and gains documents as they are uploaded into it. Parameters ---------- diff --git a/DashAI/back/services/RAG/session_validation_service.py b/DashAI/back/services/RAG/session_validation_service.py index aec9e2c71..245f12bd2 100644 --- a/DashAI/back/services/RAG/session_validation_service.py +++ b/DashAI/back/services/RAG/session_validation_service.py @@ -36,6 +36,7 @@ ) from DashAI.back.models.RAG.RAG_constants import ( RAG_MODEL_KEYS, + RAG_PARAM_DOCUMENTS, RAG_PARAM_GENERATION_MODEL, ) from DashAI.back.services.RAG.document_service import DocumentService @@ -47,6 +48,12 @@ log = logging.getLogger(__name__) +#: Rejection message for clients trying to set the document list directly. +_DOCUMENTS_NOT_SETTABLE = ( + "'documents' is managed by the document endpoints: upload a document into " + "the session instead of setting this list." +) + DEFAULT_PROMPT_NAMES = frozenset( { DefaultRAGGenerationPrompt.__name__, @@ -99,10 +106,11 @@ def prepare_RAG_params( # noqa: N802 ) -> dict[str, Any]: """Validate and normalise parameters for a *new* RAG session. - Only ``documents`` and ``generation_model`` are required: neither has a - sensible default. ``chunking_model``, ``retriever_model`` and ``prompt`` - are filled in from the backend defaults when the caller omits them, so a - session can be created from just a name, its documents and a model. + Only ``generation_model`` is required: it has no sensible default. + ``chunking_model``, ``retriever_model`` and ``prompt`` are filled in + from the backend defaults when the caller omits them, so a session can + be created from just a name and a model. ``documents`` always starts + empty -- documents are uploaded into the session afterwards. Parameters ---------- @@ -158,8 +166,12 @@ def prepare_RAG_params( # noqa: N802 if param_errors: raise ValueError("; ".join(param_errors)) - # ── 4. Validate documents ── - self._validate_documents(normalized) + # ── 4. Documents are owned by the document endpoints ── + # A session starts empty: there is no session id to attach documents to + # until it exists. Say so instead of silently dropping the list. + if normalized.get(RAG_PARAM_DOCUMENTS): + raise ValueError(_DOCUMENTS_NOT_SETTABLE) + normalized[RAG_PARAM_DOCUMENTS] = [] # ── 5. Validate prompt template placeholders ── if "prompt" in normalized: @@ -221,9 +233,12 @@ def validate_update_payload( if param_errors: raise ValueError("; ".join(param_errors)) - # ── 4. Validate documents if provided ── - if "documents" in normalized: - self._validate_documents(normalized) + # ── 4. Documents are not editable through this endpoint ── + # The foreign key on ``document`` is the authority on which documents a + # session owns, and the upload/delete endpoints keep this list in step + # with it. Accepting the list here would let the two disagree. + if RAG_PARAM_DOCUMENTS in normalized: + raise ValueError(_DOCUMENTS_NOT_SETTABLE) # ── 5. Validate prompt component ref (already resolved in step 0) ── if "prompt" in normalized: @@ -368,23 +383,33 @@ def _validate_component_params(self, normalized: dict[str, Any]) -> list[str]: errors.append(f"Invalid parameters for '{name}' at '{path}': {e}") return errors - def _validate_documents(self, normalized: dict[str, Any]) -> None: - """Check documents list is non-empty and all IDs exist in DB. + def _validate_documents( + self, normalized: dict[str, Any], session_id: int | None = None + ) -> None: + """Check every document id is an integer the session actually owns. + + An empty list is valid: a session starts with no documents and gains + them as they are uploaded. Parameters ---------- normalized : dict - Normalised parameters dict (must contain ``documents``). + Normalised parameters dict. + session_id : int | None + When given, every document must belong to this session. Raises ------ ValueError - If documents are empty, not all integers, or any ID is - missing from the database. + If any entry is not an integer, is missing from the database, or + belongs to another session. """ - docs = normalized.get("documents", []) - if not docs: - raise ValueError("Documents list must not be empty.") - if not all(isinstance(d, int) for d in docs): + docs = normalized.get(RAG_PARAM_DOCUMENTS) or [] + if not all(isinstance(d, int) and not isinstance(d, bool) for d in docs): raise ValueError("Documents must be a list of integers.") - self._document_service.validate_exist(docs) + if not docs: + return + if session_id is None: + self._document_service.validate_exist(docs) + else: + self._document_service.validate_belong_to_session(docs, session_id) diff --git a/DashAI/back/services/RAG/setup_service.py b/DashAI/back/services/RAG/setup_service.py index 6d8a89c36..4b40ec9cd 100644 --- a/DashAI/back/services/RAG/setup_service.py +++ b/DashAI/back/services/RAG/setup_service.py @@ -1,5 +1,6 @@ import logging -from typing import Dict +from dataclasses import dataclass +from typing import Callable, Dict, Optional from sqlalchemy.orm import Session @@ -8,6 +9,9 @@ from DashAI.back.models.RAG.chunking_models.base_chunking_model import ( BaseChunkingModel, ) +from DashAI.back.models.RAG.chunking_models.chunking_model_factory import ( + ChunkingFactoryResult, +) from DashAI.back.models.RAG.documents import BaseDocument, Chunk from DashAI.back.models.RAG.prompts.prompt import Prompt from DashAI.back.models.RAG.RAG_models_factory import RAGModelsFactory @@ -20,10 +24,34 @@ from DashAI.back.services.RAG.document_service import DocumentService from DashAI.back.services.RAG.llm_service import LLMService from DashAI.back.services.RAG.prompt_service import PromptService -from DashAI.back.services.RAG.retriever_setup_service import RetrieverSetupService +from DashAI.back.services.RAG.retriever_setup_service import ( + RetrieverSetupResult, + RetrieverSetupService, +) log = logging.getLogger(__name__) +#: Called at each indexing checkpoint with ``(fraction, message)``. ``fraction`` +#: is 0-1, or None when the remaining work is unknown. +ProgressFn = Callable[[Optional[float], Optional[str]], None] + + +@dataclass(frozen=True) +class IndexResult: + """Everything the indexing half of the pipeline produces. + + Deliberately holds no generation model: an indexing run must not load LLM + weights, so this is the widest result a caller can get without one. + """ + + pipeline_id: int + chunk_set_id: int + documents: Dict[int, BaseDocument] + chunking_model_id: int + chunking: ChunkingFactoryResult + retriever: RetrieverSetupResult + total_chunks: int + class SetupService: """Assembles RAG pipeline components into a ready-to-use RAGPipeline instance. @@ -58,8 +86,12 @@ def __init__( self._prompts = PromptService(db, registry) self._llm = LLMService(db, registry) - def build_pipeline(self, config: RAGPipelineConfig) -> RAGPipeline: - """Assemble a complete RAG pipeline from configuration. + def build_index( + self, + config: RAGPipelineConfig, + progress: Optional[ProgressFn] = None, + ) -> IndexResult: + """Build everything that makes a session's documents retrievable. Sequence -------- @@ -68,20 +100,26 @@ def build_pipeline(self, config: RAGPipelineConfig) -> RAGPipeline: 3. Get or create the chunk set (identity via SHA-256 signature) 4. Create the chunking model and persist chunks 5. Setup the retriever (dense embedding / sparse / composite) - 6. Get or create the LLM record - 7. Resolve the prompt component and persist it - 8. Update the pipeline DB record with FK component IDs - 9. Build and return the ``RAGPipeline`` instance + + This stops short of the generation model on purpose: + :meth:`LLMService.get_or_create` *instantiates* the LLM, and indexing + must never load model weights it will not use. + + Every step is content-addressed, so calling this when nothing changed + is a cheap no-op that reuses the cached chunk set and retriever. Parameters ---------- config : RAGPipelineConfig Typed pipeline configuration. + progress : Optional[ProgressFn] + Called at each checkpoint with ``(fraction, message)``. Kept as a + plain callable so this service stays independent of the job system. Returns ------- - RAGPipeline - Fully assembled pipeline ready for ``generate()``. + IndexResult + The persisted index and the in-memory models built along the way. Raises ------ @@ -90,8 +128,12 @@ def build_pipeline(self, config: RAGPipelineConfig) -> RAGPipeline: RuntimeError If a database error occurs. """ + report = progress or (lambda fraction, message: None) + + report(0.02, "Preparing the index") pipeline_id = self._ensure_db_record(config.session_id) + report(0.08, "Loading documents") documents = self._documents.load(config.documents) chunk_set = self._chunking.get_or_create_chunk_set( @@ -104,6 +146,7 @@ def build_pipeline(self, config: RAGPipelineConfig) -> RAGPipeline: }, ) + report(0.20, "Splitting documents into chunks") chunking_record_id, chunking_result = self._chunking.create( documents, chunk_set.id, @@ -111,6 +154,7 @@ def build_pipeline(self, config: RAGPipelineConfig) -> RAGPipeline: config.chunking_model.params, ) + report(0.45, "Building the retriever") retriever_service = RetrieverSetupService( self._db, self._registry, @@ -124,6 +168,47 @@ def build_pipeline(self, config: RAGPipelineConfig) -> RAGPipeline: config.retriever_model.params, ) + report(1.0, "Index ready") + return IndexResult( + pipeline_id=pipeline_id, + chunk_set_id=chunk_set.id, + documents=documents, + chunking_model_id=chunking_record_id, + chunking=chunking_result, + retriever=retriever_result, + total_chunks=sum(len(c) for c in chunking_result.chunks.values()), + ) + + def build_pipeline(self, config: RAGPipelineConfig) -> RAGPipeline: + """Assemble a complete RAG pipeline from configuration. + + Sequence + -------- + 1-5. Build the index (delegated to :meth:`build_index`) + 6. Get or create the LLM record + 7. Resolve the prompt component and persist it + 8. Update the pipeline DB record with FK component IDs + 9. Build and return the ``RAGPipeline`` instance + + Parameters + ---------- + config : RAGPipelineConfig + Typed pipeline configuration. + + Returns + ------- + RAGPipeline + Fully assembled pipeline ready for ``generate()``. + + Raises + ------ + ValueError + If any referenced document, component or parameter is invalid. + RuntimeError + If a database error occurs. + """ + index = self.build_index(config) + llm_result = self._llm.get_or_create( config.generation_model.component, config.generation_model.params, @@ -143,20 +228,20 @@ def build_pipeline(self, config: RAGPipelineConfig) -> RAGPipeline: self._update_db_record( config.session_id, - chunking_record_id, + index.chunking_model_id, prompt_response.id, llm_result.db_record_id, ) pipeline = self._assemble_pipeline_instance( config=config, - pipeline_id=pipeline_id, - documents=documents, + pipeline_id=index.pipeline_id, + documents=index.documents, prompt_model=prompt_model, - chunking_model_id=chunking_record_id, - chunking_model=chunking_result.model, - chunks=chunking_result.chunks, - retriever=retriever_result.model, + chunking_model_id=index.chunking_model_id, + chunking_model=index.chunking.model, + chunks=index.chunking.chunks, + retriever=index.retriever.model, llm_model=llm_result.model, ) return pipeline diff --git a/DashAI/front/src/App.jsx b/DashAI/front/src/App.jsx index 675d1fd66..6f02656fe 100644 --- a/DashAI/front/src/App.jsx +++ b/DashAI/front/src/App.jsx @@ -1,6 +1,12 @@ import React from "react"; -import { BrowserRouter, Outlet, Route, Routes } from "react-router-dom"; +import { + BrowserRouter, + Navigate, + Outlet, + Route, + Routes, +} from "react-router-dom"; import { TourRegistryProvider } from "./contexts/TourRegistryContext"; import ModuleThemeWrapper from "./components/ModuleThemeWrapper"; @@ -19,9 +25,6 @@ import HubContent from "./pages/hub/HubContent"; import HubImportPage from "./pages/hub/HubImportPage"; import JobQueueWidget from "./components/jobs/JobQueueWidget"; import RAGCreatePage from "./pages/generative/RAG/RAGCreatePage"; -import RAGDocumentsPage from "./pages/generative/RAG/RAGDocumentsPage"; -import RAGHomePage from "./pages/generative/RAG/RAGHomePage"; -import RAGPromptsPage from "./pages/generative/RAG/RAGPromptsPage"; import RAGSessionPage from "./pages/generative/RAGSession/RAGSessionPage"; import SessionRouter from "./pages/generative/SessionRouter"; import { DatasetsAndNotebooksProvider } from "./components/custom/contexts/DatasetsAndNotebooksContext"; @@ -105,25 +108,24 @@ function App() { /> } /> {/* RAG is an entry point of the Generative module, not a step - inside session creation. Its own provider scopes the session - list to RAG so the shared list stays separate. Route - matching is case-insensitive, so the previous + inside session creation, and the entry point *is* creating a + session: picking RAG used to land on a menu whose only card + was "new session", so starting one took two clicks. Existing + sessions are one click away in the left panel, which its own + provider scopes to RAG so the shared list stays separate. + Route matching is case-insensitive, so the previous /app/generative/RAG/... links keep working. */} - + } /> - - - } + element={} /> } /> + {/* Documents and prompts belong to a session now, so these + two pages are gone. There is no catch-all route, so keep the + paths redirecting for a release rather than serving a blank + page to anyone who bookmarked them. */} - - - } + element={} /> - - - } + element={} /> ; -}): Promise<{ id: number }> => { - const response = await api.post("/v1/prompt/", prompt); - if (response.status !== 201) { - throw new Error(`Failed to create RAG prompt: ${response.statusText}`); - } - return response.data; -}; - -/** Fetches the RAG sessions. Filtering happens server-side. @returns List of RAG sessions. */ -export const getRAGSessions = async (): Promise => { - const response = await api.get("/v1/generative-session/", { - params: { task_name: RAG_TASK_NAME }, - }); - if (response.status !== 200) { - throw new Error(`Failed to fetch RAG sessions: ${response.statusText}`); - } - - return response.data; -}; - /** Fetches a single RAG session by ID. @param sessionId - The session ID. @returns The session object. */ export const getRAGSession = async (sessionId: number): Promise => { const response = await api.get( @@ -90,30 +59,6 @@ export const createRAGSession = async ( return response.data; }; -/** Updates an existing RAG session. @param sessionId - The session ID. @param sessionData - Partial fields to update. @returns The updated session. */ -export const updateRAGSession = async ( - sessionId: number, - sessionData: Partial, -): Promise => { - const response = await api.put( - `/v1/generative-session/${sessionId}`, - sessionData, - ); - if (response.status !== 200) { - throw new Error(`Failed to update RAG session: ${response.statusText}`); - } - - return response.data; -}; - -/** Deletes a RAG session by ID. @param sessionId - The session ID. */ -export const deleteRAGSession = async (sessionId: number): Promise => { - const response = await api.delete(`/v1/generative-session/${sessionId}`); - if (response.status !== 204) { - throw new Error(`Failed to delete RAG session: ${response.statusText}`); - } -}; - /** Updates only the parameters of an existing RAG session. @param sessionId - The session ID. @param newParams - The new parameters payload. @returns The updated session. */ export const updateGenerativeSessionParams = async ( sessionId: number, @@ -185,22 +130,6 @@ export const getChunkingPresets = async (): Promise => { return response.data; }; -/** - * Fetches the configuration a new RAG session gets when the user picks none. - * This is the very same dict the backend applies on create, so showing it is - * an honest preview rather than a client-side guess. - * @returns The resolved defaults for chunking, retrieval and prompt. - */ -export const getSessionDefaults = async (): Promise => { - const response = await api.get( - "/v1/rag/session-defaults", - ); - if (response.status !== 200) { - throw new Error(`Failed to fetch session defaults: ${response.statusText}`); - } - return response.data; -}; - /** * Fetches a session's configuration already resolved into friendly labels. * @param sessionId - The RAG session ID. @@ -222,8 +151,8 @@ export const getSessionConfiguration = async ( /** * Fetches whether a session's documents are indexed for its current config. - * Read-only: it never triggers indexing, it only reports what the chat job - * would find. + * Read-only: it never triggers indexing, so it is safe to poll while an + * indexing job runs. * @param sessionId - The RAG session ID. * @returns The indexing status, with a localized message ready to render. */ @@ -239,6 +168,29 @@ export const getSessionIndexStatus = async ( return response.data; }; +/** + * Starts indexing a session's documents, unless there is nothing to do. + * + * Safe to call after any change: the backend already owns the rule for which + * settings invalidate the index, and short-circuits when the documents are + * already indexed or a job is running. Deciding that here too could only + * drift from it. + * + * @param sessionId - The RAG session ID. + * @returns The resulting index status, so no follow-up fetch is needed. + */ +export const startSessionIndexing = async ( + sessionId: number, +): Promise => { + const response = await api.post( + `/v1/rag/sessions/${sessionId}/index`, + ); + if (response.status !== 202 && response.status !== 200) { + throw new Error(`Failed to start indexing: ${response.statusText}`); + } + return response.data; +}; + /** Fetches generator components related to TextToTextGenerationTask. @returns List of generator components. */ export const getGeneratorComponents = async (): Promise => { const response = await api.get( @@ -263,15 +215,6 @@ export const getChunkingComponents = async (): Promise => { return response; }; -/** Fetches all uploaded documents. @returns List of document responses. */ -export const loadDocuments = async (): Promise => { - const response = await api.get("/v1/document/"); - if (response.status !== 200) { - throw new Error(`Failed to load documents: ${response.statusText}`); - } - return response.data; -}; - /** Fetches documents scoped to a specific RAG session. @param sessionId - The session ID. @returns List of document responses. */ export const getSessionDocuments = async ( sessionId: number, @@ -296,10 +239,10 @@ export const deleteDocument = async (documentId: number): Promise => { /** * Result of a document upload attempt. * - * When the uploaded file already exists (same content hash) and `force` was - * not used, the backend answers `409 Conflict`; the result is flagged as - * `duplicate` and carries the existing document plus the affected sessions so - * the UI can ask for confirmation before forcing the update. + * When the session already holds this exact file (same content hash), the + * backend answers `409 Conflict`; the result is flagged as `duplicate` and + * carries the document the session already has. There is nothing to confirm: + * the file is in the session either way. */ export type AddDocumentResult = | { @@ -309,25 +252,23 @@ export type AddDocumentResult = | { duplicate: true; existingDocument: IDocumentResponse; - affectedSessions: { id: number; name: string }[]; }; /** - * Uploads a document file with optional metadata via multipart/form-data. + * Uploads a document into a RAG session via multipart/form-data. + * @param sessionId - The session that will own the document. * @param file - The File object to upload. * @param optional_metadata - Optional metadata (name, source, etc.). - * @param force - If true, overwrite the existing document when a duplicate - * (same content hash) is detected. * @returns The upload result (created document or duplicate info). */ export const addDocument = async ({ + sessionId, file, optional_metadata, - force = false, }: { + sessionId: number | string; file: File; optional_metadata?: Record; - force?: boolean; }): Promise => { if (optional_metadata) { optional_metadata.last_modified = file.lastModified; @@ -344,12 +285,9 @@ export const addDocument = async ({ try { const response = await api.post( - "/v1/document/", + `/v1/document/session/${sessionId}`, formData, - { - headers: { "Content-Type": "multipart/form-data" }, - params: force ? { force: "true" } : undefined, - }, + { headers: { "Content-Type": "multipart/form-data" } }, ); if (response.status !== 201 && response.status !== 200) { @@ -373,73 +311,12 @@ export const addDocument = async ({ return { duplicate: true, existingDocument: detail?.existing_document, - affectedSessions: detail?.affected_sessions ?? [], }; } throw error; } }; -/** Class name prefix that identifies non-generation prompt types. */ -const AUGMENTATION_PROMPT_CLASS_PREFIX = "Augmentation"; - -/** - * Checks whether a prompt's class_name corresponds to a generation prompt - * (i.e. NOT an augmentation prompt). - * - * @param className - The prompt component class name to test. - * @returns `true` if the class is a generation prompt, `false` if it is an augmentation prompt. - */ -export function isGenerationPromptClass(className: string): boolean { - return !className.includes(AUGMENTATION_PROMPT_CLASS_PREFIX); -} - -/** Fetches default prompt components (children of RAGGenerationPrompt). @returns List of default prompt components. */ -export const getDefaultPrompts = async (): Promise => { - return getChildComponents("RAGGenerationPrompt", false); -}; - -/** Fetches all saved RAG prompts (user-created). @returns List of RAG prompts. */ -export const getRAGPrompts = async (): Promise => { - const response = await api.get("/v1/prompt/"); - if (response.status !== 200) { - throw new Error(`Failed to fetch RAG prompts: ${response.statusText}`); - } - return response.data; -}; - -/** - * Fetches custom (non-Default) prompt components for the given parent types. - * @param types - Array of parent component type names to fetch children from. - * @returns List of custom prompt components (excluding Default* classes). - */ -export const getCustomPrompts = async ( - types: string[] = ["RAGGenerationPrompt", "AugmentationPrompt"], -): Promise => { - let allChildren: IComponent[] = []; - for (const type of types) { - const response = await api.get( - `/v1/component/${type}/children`, - { params: { recursive: false } }, - ); - if (response.status !== 200) { - throw new Error( - `Failed to fetch ${type} children: ${response.statusText}`, - ); - } - const filtered = response.data.filter( - (child) => - !( - child.name && - typeof child.name === "string" && - child.name.includes("Default") - ), - ); - allChildren.push(...filtered); - } - return allChildren; -}; - /** Fetches all available extractor components (children of BaseExtractor). @returns List of extractor components. */ export const getExtractorOptions = async (): Promise => { const response = await getChildComponents("BaseExtractor", false); @@ -449,6 +326,11 @@ export const getExtractorOptions = async (): Promise => { return response; }; +/** Fetches default prompt components (children of RAGGenerationPrompt). @returns List of default prompt components. */ +export const getDefaultPrompts = async (): Promise => { + return getChildComponents("RAGGenerationPrompt", false); +}; + /** * Extracts text from a document using a specified extractor. * @param docId - The document ID. @@ -479,20 +361,22 @@ export const extractDocumentText = async ( }; /** - * Persists an extractor choice for a document and optionally invalidates RAG artifacts. + * Persists an extractor choice for a document, re-extracting its text. + * + * The chunks and retrievers fitted over the previous extraction are discarded + * server-side. The document belongs to one session, so nothing else is + * affected and there is nothing to confirm. + * * @param docId - The document ID. * @param extractorRef - The {component, params} for the extractor. - * @param force - If true, bypass confirmation and invalidate artifacts. * @returns The updated document response. */ export const updateDocumentExtractor = async ( docId: number, extractorRef: { component: string; params?: Record }, - force: boolean = false, ): Promise => { const response = await api.put(`/v1/document/${docId}/extractor`, { extractor: extractorRef, - force, }); if (response.status !== 200) { throw new Error( diff --git a/DashAI/front/src/components/custom/ComponentSelector.jsx b/DashAI/front/src/components/custom/ComponentSelector.jsx index 7479e152b..89d7c899b 100644 --- a/DashAI/front/src/components/custom/ComponentSelector.jsx +++ b/DashAI/front/src/components/custom/ComponentSelector.jsx @@ -47,6 +47,7 @@ function ComponentSelector({ emptyText, getIcon, flat = false, + showFooter = true, tourDataFor = null, tourDataMatchFn = null, onDownloadChange = null, @@ -410,29 +411,34 @@ function ComponentSelector({ )} - - - {t("componentsAvailable", { count: filtered.length })} - - {selected && ( - } - label={getLabel(selected)} - color="primary" - variant="outlined" - size="small" - /> - )} - + {/* A running count of the options is only worth the strip when the list + is long enough to be worth scanning; the selected card already shows + its own tick. */} + {showFooter && ( + + + {t("componentsAvailable", { count: filtered.length })} + + {selected && ( + } + label={getLabel(selected)} + color="primary" + variant="outlined" + size="small" + /> + )} + + )} { - const { t } = useTranslation("generative"); +/** + * The chunks one document contributed to an answer. + * + * Wears the module's dialog treatment -- titled row with a close affordance, + * divided body, actions along the bottom -- and lays each chunk out as a flat + * outlined card, the same card the configuration panel uses, so a fragment + * reads like the rest of the RAG views rather than like a bare list row. + * + * @param {object} props + * @param {boolean} props.open - Whether the dialog is visible. + * @param {Function} props.onClose - Closes the dialog. + * @param {object} [props.document] - The document the chunks came from. + * @param {Array} [props.chunks] - The chunks it provided. + * @returns {JSX.Element|null} The dialog, or nothing without a document. + */ +const DocumentReferencesModal = ({ open, onClose, document, chunks }) => { + const { t } = useTranslation(["generative", "common"]); if (!document || !chunks) return null; - const getDocumentTitle = (docId, chunks) => { - const firstChunk = chunks[0]; + const getDocumentTitle = (docId, docChunks) => { + const firstChunk = docChunks[0]; if (firstChunk.document_title) return firstChunk.document_title; if (firstChunk.document_name) return firstChunk.document_name; if (firstChunk.title) return firstChunk.title; if (firstChunk.name) return firstChunk.name; - return t("documentReferences.fallbackTitle", { + return t("generative:documentReferences.fallbackTitle", { id: docId, defaultValue: `Document ${docId}`, }); @@ -45,7 +53,6 @@ const DocumentReferencesModal = ({ try { const cleanText = chunkText.replace(/\\n/g, "\n"); await navigator.clipboard.writeText(cleanText); - // You could add a toast notification here if you have a notification system } catch (err) { console.error("Failed to copy text: ", err); } @@ -57,122 +64,113 @@ const DocumentReferencesModal = ({ onClose={onClose} maxWidth="md" fullWidth - PaperProps={{ - sx: { - borderRadius: 2, - maxHeight: "80vh", - }, - }} + PaperProps={{ sx: { maxHeight: "80vh" } }} > - - - + + + {getDocumentTitle(document.id, chunks)} - - + + - - - {t("documentReferences.chunksCount", { count: chunks.length })} + + + {t("generative:documentReferences.chunksCount", { + count: chunks.length, + })} - + {chunks.map((chunk, index) => ( - - - + + + - - - {chunk.document_position - ? t("documentReferences.chunkLabel", { - position: chunk.document_position, - }) - : t("documentReferences.chunkLabel", { - position: index + 1, - })} - - - handleCopyChunk(chunk.text)} - sx={{ - color: "text.secondary", - "&:hover": { - color: "primary.main", - backgroundColor: "action.hover", - }, - }} - > - - - + /> + + {t("generative:documentReferences.chunkLabel", { + position: chunk.document_position ?? index + 1, + })} + + - + handleCopyChunk(chunk.text)} + aria-label={t("common:copy", "Copy")} sx={{ - color: "text.primary", - lineHeight: 1.6, - whiteSpace: "pre-wrap", - backgroundColor: "background.default", - p: 1.5, - borderRadius: 1, - border: 1, - borderColor: "divider", + color: "text.secondary", + "&:hover": { + color: "primary.main", + backgroundColor: "action.hover", + }, }} > - {chunk.text.replace(/\\n/g, "\n")} - - - - {index < chunks.length - 1 && } - + + + + + + + {chunk.text.replace(/\\n/g, "\n")} + + ))} - + - - diff --git a/DashAI/front/src/components/generative/GenerativeChat.jsx b/DashAI/front/src/components/generative/GenerativeChat.jsx index 223b32583..2999bf738 100644 --- a/DashAI/front/src/components/generative/GenerativeChat.jsx +++ b/DashAI/front/src/components/generative/GenerativeChat.jsx @@ -30,12 +30,12 @@ import { } from "../credentials/credentialStatus"; import VpnKeyOutlinedIcon from "@mui/icons-material/VpnKeyOutlined"; import { useSnackbar } from "notistack"; +import { getApiErrorMessage } from "../../utils/apiError"; import { MediaInput } from "./MediaInput"; import JobQueueWidget from "../jobs/JobQueueWidget"; import { getRunStatus } from "../../utils/runStatus"; import TemplateModal from "../custom/TemplateModal"; import SourcesDisplay from "./SourcesDisplay"; -import RAGBreadcrumbs from "./RAG/RAGBreadcrumbs"; import { Trans, useTranslation } from "react-i18next"; import { useGenerative } from "./GenerativeContext"; import { useTourContext } from "../tour/TourProvider"; @@ -46,8 +46,8 @@ import { useTheme } from "@mui/material/styles"; * * @param {object} props * @param {object} [props.indexStatus] - For RAG sessions, the backend-reported - * indexing state. When documents are not indexed yet, the first answer also - * pays for indexing, so the waiting state says so instead of looking stuck. + * indexing state. The composer is disabled while a run is in flight: the + * retriever cannot answer over chunks that are still being written. * @returns {JSX.Element} The chat. */ export default function GenerativeChat({ indexStatus }) { @@ -71,6 +71,9 @@ export default function GenerativeChat({ indexStatus }) { const [messages, setMessages] = useState([]); const [messagesWithHistory, setMessagesWithHistory] = useState([]); const [isLoadingMessage, setIsLoadingMessage] = useState(false); + // A question asked mid-index would retrieve over chunks that are still being + // written, so the composer waits for the run to finish. + const isIndexing = indexStatus?.status === "indexing"; const chatContainerRef = useRef(null); const isAtBottomRef = useRef(true); const [showScrollButton, setShowScrollButton] = useState(false); @@ -210,24 +213,37 @@ export default function GenerativeChat({ indexStatus }) { setIsLoadingMessage(true); setShouldAutoScroll(true); // Enable auto-scroll when sending new message - postProcess(sessionId, input).then((response) => { - // Add the new message to the chat - setMessages((prevMessages) => [...prevMessages, response]); - - // Enqueue the generative process job - enqueueGenerativeProcessJob(response.id).then(() => { - startJobQueue(true).then(() => { - setIsLoadingMessage(false); - }); + postProcess(sessionId, input) + .then((response) => { + // Add the new message to the chat + setMessages((prevMessages) => [...prevMessages, response]); + + // Enqueue the generative process job + return enqueueGenerativeProcessJob(response.id) + .then(() => startJobQueue(true)) + .then(() => { + setIsLoadingMessage(false); + }); + }) + .then(() => { + // End tour if on final step + if (tourContext?.run && tourContext?.stepIndex === 8) { + setTimeout(() => { + tourContext.stopTour(); + }, 100); + } + }) + .catch((error) => { + // Without this the composer stays disabled forever and says nothing. + // A RAG session starts with no documents, so the backend refusing the + // first message is a routine outcome, not an exceptional one. + console.error("Failed to send message:", error); + setIsLoadingMessage(false); + enqueueSnackbar( + getApiErrorMessage(error, t("generative:error.failedToSendMessage")), + { variant: "error" }, + ); }); - - // End tour if on final step - if (tourContext?.run && tourContext?.stepIndex === 8) { - setTimeout(() => { - tourContext.stopTour(); - }, 100); - } - }); }; useEffect(() => { @@ -474,13 +490,6 @@ export default function GenerativeChat({ indexStatus }) { height={"100%"} sx={{ overflow: "hidden", minHeight: 0 }} > - {/* RAG Breadcrumbs - only show for RAG tasks */} - {taskName === "RAGTask" && ( - - - - )} - {/* Model display */} - {sessionInfo?.name ? sessionInfo.name : "Untitled Session"}{" "} + {sessionInfo?.name || t("generative:label.untitledSession")}{" "} {sessionInfo?.description ? ":" : null} {sessionInfo?.description} @@ -631,7 +640,11 @@ export default function GenerativeChat({ indexStatus }) { ) : ( <> - {indexStatus && indexStatus.status !== "indexed" && ( + {/* Only where the chat job still has to index: with + eager indexing this is the fallback path, and + "no_documents" never indexes at all. */} + {(indexStatus?.status === "not_indexed" || + indexStatus?.status === "stale") && ( ) : ( - { - handleSendMessage(input); - }} - isLoading={isLoadingMessage} - inputsCardinality={inputsCardinality} - /> + <> + { + handleSendMessage(input); + }} + isLoading={isLoadingMessage || isIndexing} + inputsCardinality={inputsCardinality} + /> + {/* Says why the composer is disabled, rather than leaving it looking + broken. The message is localized by the backend. */} + {isIndexing && ( + + {indexStatus.message} + + )} + )} {/* Session Info Modal */} diff --git a/DashAI/front/src/components/generative/GenerativeHubHeader.jsx b/DashAI/front/src/components/generative/GenerativeHubHeader.jsx new file mode 100644 index 000000000..fe0a9401a --- /dev/null +++ b/DashAI/front/src/components/generative/GenerativeHubHeader.jsx @@ -0,0 +1,56 @@ +import PropTypes from "prop-types"; +import { Box, Typography } from "@mui/material"; +import ViewModuleIcon from "@mui/icons-material/ViewModule"; +import { useTranslation } from "react-i18next"; +import NewItemButton from "../threeSectionLayout/NewItemButton"; + +/** + * The 64px header at the top of the generative module's left panel. + * + * Every view in the module starts with this row, so the way back to the hub + * sits in the same place on all of them. It is extracted from `SessionBar` + * because the RAG session view splits its left panel between documents and + * sessions, and the header has to stay above that split rather than travel + * with the session list. + * + * The row holds nothing but the way back: a second control beside it read as + * an action on the hub button rather than on the view below. + * + * @param {object} props + * @param {boolean} [props.showHubButton=false] - Whether to offer the way + * back to the hub. The hub itself has nowhere to go, so it shows its name. + * @param {Function} [props.onHubClick] - Navigates to the hub. + * @returns {JSX.Element} The header row. + */ +export default function GenerativeHubHeader({ + showHubButton = false, + onHubClick, +}) { + const { t } = useTranslation(["generative"]); + + return ( + + {showHubButton ? ( + + + + ) : ( + + {t("generative:label.generativeModule")} + + )} + + ); +} + +GenerativeHubHeader.propTypes = { + showHubButton: PropTypes.bool, + onHubClick: PropTypes.func, +}; diff --git a/DashAI/front/src/components/generative/MainGenerativeBox.jsx b/DashAI/front/src/components/generative/MainGenerativeBox.jsx deleted file mode 100644 index 2cc0250ed..000000000 --- a/DashAI/front/src/components/generative/MainGenerativeBox.jsx +++ /dev/null @@ -1,20 +0,0 @@ -import React from "react"; -import { Box } from "@mui/material"; - -export default function MainGenerativeBox({ children }) { - return ( - - {children} - - ); -} diff --git a/DashAI/front/src/components/generative/RAG/DocumentDetailPanel.jsx b/DashAI/front/src/components/generative/RAG/DocumentDetailPanel.jsx deleted file mode 100644 index 48ed8abb0..000000000 --- a/DashAI/front/src/components/generative/RAG/DocumentDetailPanel.jsx +++ /dev/null @@ -1,422 +0,0 @@ -import { useState, useEffect, useRef, useCallback } from "react"; -import { useTranslation } from "react-i18next"; -import { - Box, - Typography, - FormControl, - InputLabel, - Select, - MenuItem, - Button, - CircularProgress, - Divider, - Paper, - Dialog, - DialogTitle, - DialogContent, - DialogContentText, - DialogActions, - Alert, - Collapse, - IconButton, -} from "@mui/material"; -import PropTypes from "prop-types"; -import { ExpandMore, ExpandLess } from "@mui/icons-material"; -import { - getExtractorOptions, - extractDocumentText, - updateDocumentExtractor, -} from "../../../api/rag"; -import FormSchema from "../../shared/FormSchema"; -import FormSchemaContainer from "../../shared/FormSchemaContainer"; -import { resolveDefaults } from "../../../utils/schema"; - -/** - * Document detail panel showing document info, extractor selection, - * content preview, and extractor change flow. - * - * The extractor params are rendered through DashAI's schema-driven form - * (FormSchema), so each extractor shows fields generated from its SCHEMA - * (e.g. EasyOCRExtractor shows `languages` and `gpu`). - * - * @param {object} props - * @param {object} [props.selectedDocument] - The currently selected document, or null. - * @param {function} [props.onExtractorChanged] - Callback invoked when extractor is saved. - * @returns {JSX.Element} - */ -export default function DocumentDetailPanel({ - selectedDocument, - onExtractorChanged, -}) { - const { t } = useTranslation(["generative", "common"]); - const [extractors, setExtractors] = useState([]); - const [selectedExtractor, setSelectedExtractor] = useState(""); - const [params, setParams] = useState({}); - const [content, setContent] = useState(""); - const [contentLoading, setContentLoading] = useState(false); - const [saving, setSaving] = useState(false); - const [error, setError] = useState(""); - const [confirmOpen, setConfirmOpen] = useState(false); - const [affectedSessions, setAffectedSessions] = useState([]); - const [previewOpen, setPreviewOpen] = useState(false); - const formikRef = useRef(null); - - // Load extractor options on mount - useEffect(() => { - const loadExtractors = async () => { - try { - const options = await getExtractorOptions(); - setExtractors(options); - } catch (e) { - console.error("Failed to load extractors:", e); - } - }; - loadExtractors(); - }, []); - - // Reset when document changes - useEffect(() => { - if (selectedDocument) { - const currentName = selectedDocument.extractor?.component || ""; - const currentParams = selectedDocument.extractor?.params || {}; - setSelectedExtractor(currentName); - setContent(""); - setError(""); - if (currentName && Object.keys(currentParams).length === 0) { - resolveDefaults(currentName) - .then((defaults) => setParams(defaults)) - .catch((err) => { - console.error( - `Failed to resolve defaults for ${currentName}:`, - err, - ); - setParams({}); - }); - } else { - setParams(currentParams); - } - } - }, [selectedDocument]); - - /** - * Build the { component, params } ref to send to the backend, using the - * latest values from the schema form when available. - * @returns {{ component: string, params: object }} - */ - const buildExtractorRef = useCallback(() => { - const latestParams = - formikRef.current?.values && - Object.keys(formikRef.current.values).length > 0 - ? formikRef.current.values - : params; - return { component: selectedExtractor, params: latestParams }; - }, [selectedExtractor, params]); - - /** - * Handle extractor selection: resolve the schema defaults for the chosen - * component so the form starts with the right param values. - * @param {object} e - The select change event. - */ - const handleExtractorChange = async (e) => { - const name = e.target.value; - setSelectedExtractor(name); - setError(""); - try { - const defaults = await resolveDefaults(name); - setParams(defaults); - } catch (err) { - console.error(`Failed to resolve defaults for ${name}:`, err); - setParams({}); - } - }; - - /** - * Store the latest param values coming from the schema form. - * @param {object} values - The form parameter values. - */ - const handleParamsChange = useCallback((values) => { - setParams(values); - }, []); - - if (!selectedDocument) { - return ( - - - {t("generative:ragDocumentsPage.detailPanel.noDocumentSelected")} - - - ); - } - - const docExtractorName = selectedDocument.extractor?.component || ""; - const docExtractorParams = selectedDocument.extractor?.params || {}; - - // Filter extractors by file type compatibility - const compatibleExtractors = extractors.filter((ext) => { - const supportedTypes = ext.metadata?.supported_file_types || []; - return ( - supportedTypes.length === 0 || - supportedTypes.includes(selectedDocument.file_type) - ); - }); - - const hasChanged = - !!selectedExtractor && - (selectedExtractor !== docExtractorName || - JSON.stringify(params) !== JSON.stringify(docExtractorParams)); - - const handleProcessDocument = async () => { - setContentLoading(true); - setError(""); - setContent(""); - try { - const ref = buildExtractorRef(); - const result = await extractDocumentText( - Number(selectedDocument.id), - ref, - false, // Preview mode - ); - setContent(result.text); - setPreviewOpen(true); - } catch (e) { - setError(e.message || "Extraction failed"); - } finally { - setContentLoading(false); - } - }; - - const handleSaveExtractor = async () => { - if (!hasChanged) return; - setSaving(true); - setError(""); - try { - const ref = buildExtractorRef(); - await updateDocumentExtractor(Number(selectedDocument.id), ref, false); - if (onExtractorChanged) onExtractorChanged(); - } catch (e) { - const message = e.response?.data?.detail || e.message || ""; - if (e.response?.status === 409) { - // Need confirmation - const detail = - typeof e.response.data.detail === "object" - ? e.response.data.detail - : { detail: message, affected_sessions: [] }; - setAffectedSessions(detail.affected_sessions || []); - setConfirmOpen(true); - } else { - setError(message || "Failed to save extractor"); - } - } finally { - setSaving(false); - } - }; - - const handleConfirmForce = async () => { - setConfirmOpen(false); - setSaving(true); - try { - const ref = buildExtractorRef(); - await updateDocumentExtractor(Number(selectedDocument.id), ref, true); - if (onExtractorChanged) onExtractorChanged(); - } catch (e) { - const message = e.response?.data?.detail || e.message || ""; - setError(typeof message === "object" ? JSON.stringify(message) : message); - } finally { - setSaving(false); - } - }; - - return ( - - {/* Document Info */} - - {t("generative:ragDocumentsPage.detailPanel.documentInfo")} - - - - {t("generative:ragDocumentsPage.detailPanel.name")}:{" "} - {selectedDocument.file_name || selectedDocument.name} - - - {t("generative:ragDocumentsPage.detailPanel.type")}:{" "} - {selectedDocument.file_type} - - - - - - {/* Extractor Selector */} - - {t("generative:ragDocumentsPage.detailPanel.extractor")} - - - - {t("generative:ragDocumentsPage.detailPanel.extractor")} - - - - - {/* Schema-driven extractor params form */} - - - - - {/* Action buttons */} - - - {hasChanged && ( - - )} - - - {error && ( - setError("")}> - {error} - - )} - - - - {/* Content Preview */} - setPreviewOpen((prev) => !prev)} - > - - {t("generative:ragDocumentsPage.detailPanel.contentPreview")} - - - {previewOpen ? : } - - - - {contentLoading && ( - - - - {t("generative:ragDocumentsPage.detailPanel.extracting")} - - - )} - {!contentLoading && !content && ( - - {t("generative:ragDocumentsPage.detailPanel.noContent")} - - )} - {!contentLoading && content && ( - - {content} - - )} - - - {/* Confirmation Dialog */} - setConfirmOpen(false)}> - - {t( - "generative:ragDocumentsPage.detailPanel.changeExtractorConfirmTitle", - )} - - - - {t( - "generative:ragDocumentsPage.detailPanel.changeExtractorConfirmBody", - { - count: affectedSessions.length, - }, - )} - - {affectedSessions.length > 0 && ( - - {affectedSessions.map((s) => ( - - • {s.name} (ID: {s.id}) - - ))} - - )} - - - - - - - - ); -} - -DocumentDetailPanel.propTypes = { - selectedDocument: PropTypes.shape({ - id: PropTypes.oneOfType([PropTypes.string, PropTypes.number]), - file_name: PropTypes.string, - file_type: PropTypes.string, - name: PropTypes.string, - extractor: PropTypes.shape({ - component: PropTypes.string, - params: PropTypes.object, - }), - default_extractor: PropTypes.shape({ - component: PropTypes.string, - params: PropTypes.object, - }), - }), - onExtractorChanged: PropTypes.func, -}; diff --git a/DashAI/front/src/components/generative/RAG/DocumentExtractorModal.jsx b/DashAI/front/src/components/generative/RAG/DocumentInspectorModal.jsx similarity index 86% rename from DashAI/front/src/components/generative/RAG/DocumentExtractorModal.jsx rename to DashAI/front/src/components/generative/RAG/DocumentInspectorModal.jsx index 92dc5357e..107324620 100644 --- a/DashAI/front/src/components/generative/RAG/DocumentExtractorModal.jsx +++ b/DashAI/front/src/components/generative/RAG/DocumentInspectorModal.jsx @@ -34,13 +34,17 @@ import { resolveDefaults } from "../../../utils/schema"; import { normalizeUrl } from "../../../utils/urlUtils"; /** - * Modal for inspecting and configuring a document's text extractor. + * Modal for reading a document and choosing how its text is extracted. * * Shows a split view with the original file on the left and the extracted text * on the right. The extractor selector sits next to the explanation; a * "Settings" button opens a separate dialog with the schema-driven params form. + * + * This is where the old standalone documents page's capabilities live now that + * documents belong to a session: the session's left panel is too narrow to read + * a document in, and the centre column stays with the conversation. */ -export default function DocumentExtractorModal({ +export default function DocumentInspectorModal({ open, onClose, document, @@ -56,8 +60,6 @@ export default function DocumentExtractorModal({ const [contentLoading, setContentLoading] = useState(false); const [saving, setSaving] = useState(false); const [error, setError] = useState(""); - const [confirmOpen, setConfirmOpen] = useState(false); - const [affectedSessions, setAffectedSessions] = useState([]); const [settingsOpen, setSettingsOpen] = useState(false); const formikRef = useRef(null); @@ -101,7 +103,6 @@ export default function DocumentExtractorModal({ setError(""); setContent(""); setRawContent(""); - setConfirmOpen(false); setSettingsOpen(false); const currentName = document.extractor?.component || ""; @@ -142,8 +143,13 @@ export default function DocumentExtractorModal({ } setParams(initialParams); + // Extract on open rather than making the user ask for it: reading + // the text is the reason this modal exists. if (initialName) { - setParams(initialParams); + performExtract(document.id, { + component: initialName, + params: initialParams, + }); } } catch (e) { setError(e.message || "Failed to load extractor options"); @@ -218,41 +224,9 @@ export default function DocumentExtractorModal({ setError(""); try { const ref = buildExtractorRef(); - const updated = await updateDocumentExtractor( - Number(document.id), - ref, - false, - ); - if (onExtractorChanged) onExtractorChanged(updated); - } catch (e) { - const message = e.response?.data?.detail || e.message || ""; - if (e.response?.status === 409) { - const detail = - typeof e.response.data.detail === "object" - ? e.response.data.detail - : { detail: message, affected_sessions: [] }; - setAffectedSessions(detail.affected_sessions || []); - setConfirmOpen(true); - } else { - setError( - typeof message === "object" ? JSON.stringify(message) : message, - ); - } - } finally { - setSaving(false); - } - }; - - const handleConfirmForce = async () => { - setConfirmOpen(false); - setSaving(true); - try { - const ref = buildExtractorRef(); - const updated = await updateDocumentExtractor( - Number(document.id), - ref, - true, - ); + // No confirmation step: the document belongs to this session alone, so + // re-indexing it cannot disturb anybody else. + const updated = await updateDocumentExtractor(Number(document.id), ref); if (onExtractorChanged) onExtractorChanged(updated); } catch (e) { const message = e.response?.data?.detail || e.message || ""; @@ -543,52 +517,13 @@ export default function DocumentExtractorModal({ - - {/* ── Force-conflict confirmation dialog ── */} - setConfirmOpen(false)}> - - {t( - "generative:ragDocumentsPage.detailPanel.changeExtractorConfirmTitle", - )} - - - - {t( - "generative:ragDocumentsPage.detailPanel.changeExtractorConfirmBody", - { count: affectedSessions.length }, - )} - - {affectedSessions.length > 0 && ( - - {affectedSessions.map((s) => ( - - • {s.name} (ID: {s.id}) - - ))} - - )} - - - - - - ); return createPortal(dialogContent, globalThis.document.body); } -DocumentExtractorModal.propTypes = { +DocumentInspectorModal.propTypes = { open: PropTypes.bool.isRequired, onClose: PropTypes.func.isRequired, document: PropTypes.shape({ diff --git a/DashAI/front/src/components/generative/RAG/DocumentList.jsx b/DashAI/front/src/components/generative/RAG/DocumentList.jsx index f2d569d52..dc0db4dc6 100644 --- a/DashAI/front/src/components/generative/RAG/DocumentList.jsx +++ b/DashAI/front/src/components/generative/RAG/DocumentList.jsx @@ -1,76 +1,60 @@ -import React, { useState } from "react"; +import React from "react"; +import PropTypes from "prop-types"; import { Box } from "@mui/material"; import DocumentListItem from "./DocumentListItem"; -import DocumentPreviewModal from "./DocumentPreviewModal"; -import { normalizeUrl } from "../../../utils/urlUtils"; /** - * Renders a vertical list of document items with preview-on-click capability. + * Renders a vertical list of document rows. * - * @param {object} props - * @param {Array} props.documents - Array of document objects to display. - * @param {object} [props.indexStateByDocument] - Map of document id to its + * Purely presentational: the owner decides what a row click does and which + * controls it offers, so one place drives both the preview and the inspector + * instead of each list keeping its own copy of that state. + * + * @param {object} props + * @param {Array} props.documents - Document objects to display. + * @param {object} [props.indexStateByDocument] - Map of document id to its * indexing state, used to badge each row. - * @returns {JSX.Element} + * @param {Function} [props.onDocumentClick] - Called with the clicked document. + * @param {Function} [props.renderActions] - Called with a document, returning + * the controls to reveal on hover for that row. + * @returns {JSX.Element} The list. */ -export default function DocumentList({ documents, indexStateByDocument }) { - const [previewOpen, setPreviewOpen] = useState(false); - const [previewDoc, setPreviewDoc] = useState(null); - const [txtContent, setTxtContent] = useState(""); - - /** - * Opens the document preview modal, fetching TXT content if applicable. - * @param {object} doc - The document to preview. - */ - const handleOpenPreview = async (doc) => { - setPreviewDoc(doc); - if (doc.file_type === "txt" && doc.preview) { - try { - const res = await fetch(normalizeUrl(doc.preview)); - const text = await res.text(); - setTxtContent(text); - } catch (e) { - console.error("Error loading TXT:", e); - setTxtContent("Error loading TXT file"); - } - } - setPreviewOpen(true); - }; - - const handleClosePreview = () => { - setPreviewOpen(false); - setPreviewDoc(null); - setTxtContent(""); - }; - +export default function DocumentList({ + documents, + indexStateByDocument, + onDocumentClick, + renderActions, +}) { return ( - <> - - {documents.map((document) => ( - handleOpenPreview(document)} - /> - ))} - - - + + {documents.map((document) => ( + onDocumentClick(document) : undefined + } + actions={renderActions?.(document)} + /> + ))} + ); } + +DocumentList.propTypes = { + documents: PropTypes.array.isRequired, + indexStateByDocument: PropTypes.object, + onDocumentClick: PropTypes.func, + renderActions: PropTypes.func, +}; diff --git a/DashAI/front/src/components/generative/RAG/DocumentListItem.jsx b/DashAI/front/src/components/generative/RAG/DocumentListItem.jsx index 30d15cdb2..6d1f90888 100644 --- a/DashAI/front/src/components/generative/RAG/DocumentListItem.jsx +++ b/DashAI/front/src/components/generative/RAG/DocumentListItem.jsx @@ -32,6 +32,9 @@ const getDocumentIcon = (fileType) => { * @param {function} [props.onClick] - Click handler for the item. * @param {object} [props.indexState] - This document's indexing state within * the session (`{ chunks, indexed }`), as reported by the backend. + * @param {node} [props.actions] - Controls revealed on hover, at the end of + * the row. They stop their own clicks, so acting on a row does not also + * trigger the row itself. * @returns {JSX.Element} */ export default function DocumentListItem({ @@ -39,6 +42,7 @@ export default function DocumentListItem({ disabled = false, onClick, indexState, + actions, }) { const { t } = useTranslation(["generative"]); const [isHovered, setIsHovered] = useState(false); @@ -142,13 +146,24 @@ export default function DocumentListItem({ + + {actions && ( + event.stopPropagation()} + sx={{ + display: "flex", + alignItems: "center", + flexShrink: 0, + // Kept mounted so the row does not reflow when the mouse arrives, + // and faded rather than hidden: `visibility: hidden` would drop + // these controls out of the tab order, and with the documents page + // gone there is no other way to reach them without a mouse. + opacity: isHovered ? 1 : 0, + transition: "opacity 0.2s", + "&:focus-within": { opacity: 1 }, + }} + > + {actions} + + )} ); } diff --git a/DashAI/front/src/components/generative/RAG/DocumentSelector.jsx b/DashAI/front/src/components/generative/RAG/DocumentSelector.jsx deleted file mode 100644 index 96879c41b..000000000 --- a/DashAI/front/src/components/generative/RAG/DocumentSelector.jsx +++ /dev/null @@ -1,571 +0,0 @@ -import { - Box, - Button, - Dialog, - IconButton, - Tooltip, - Typography, -} from "@mui/material"; -import { useEffect, useState, useCallback, useRef, useMemo } from "react"; -import PropTypes from "prop-types"; -import AddIcon from "@mui/icons-material/AddCircleOutline"; -import { Visibility, Delete, Settings } from "@mui/icons-material"; -import { useTranslation } from "react-i18next"; -import { useSnackbar } from "notistack"; -import { getApiErrorMessage } from "../../../utils/apiError"; -import { useTheme } from "@mui/material/styles"; -import { - MaterialReactTable, - useMaterialReactTable, - MRT_GlobalFilterTextField, -} from "material-react-table"; -import { MRT_Localization_ES } from "material-react-table/locales/es"; -import { MRT_Localization_EN } from "material-react-table/locales/en"; -import Upload from "../../shared/Upload"; -import { loadDocuments, addDocument, deleteDocument } from "../../../api/rag"; -import DuplicateDocumentDialog from "./DuplicateDocumentDialog"; -import { formatDate } from "../../../utils"; -import { normalizeUrl } from "../../../utils/urlUtils"; -import DocumentPreviewModal from "./DocumentPreviewModal"; -import DocumentExtractorModal from "./DocumentExtractorModal"; -import RAGSectionColumn from "../../../pages/generative/RAGSession/components/RAGSectionColumn"; - -/** - * Document selection table used in the RAG setup wizard. Supports multi-select, - * upload, delete, search, and preview of documents. - * - * @param {object} props - * @param {Array} [props.selectedIds=[]] - Initially selected document IDs. - * @param {function} [props.onSelect] - Callback invoked with the array of selected document objects. - * @param {object|Array|function} [props.sx] - MUI sx prop forwarded to the container. - * @returns {JSX.Element} - */ -export default function DocumentSelector({ - selectedIds: initialSelectedIds = [], - onSelect, - sx, -}) { - const { t, i18n } = useTranslation(["generative"]); - const { enqueueSnackbar } = useSnackbar(); - const theme = useTheme(); - const [documents, setDocuments] = useState([]); - const [selectedIds, setSelectedIds] = useState(initialSelectedIds); - const [isLoading, setIsLoading] = useState(true); - const [uploadOpen, setUploadOpen] = useState(false); - - const [previewOpen, setPreviewOpen] = useState(false); - const [previewDoc, setPreviewDoc] = useState(null); - const [txtContent, setTxtContent] = useState(""); - - const [extractorModalOpen, setExtractorModalOpen] = useState(false); - const [extractorDoc, setExtractorDoc] = useState(null); - - const [duplicatePending, setDuplicatePending] = useState(null); - - const previousSelectedIdsRef = useRef( - JSON.stringify([...initialSelectedIds].map(String).sort()), - ); - - const localization = i18n.language.startsWith("es") - ? MRT_Localization_ES - : MRT_Localization_EN; - - const tableData = useMemo( - () => - documents.map((doc) => ({ - ...doc, - preview: doc.preview_url, - file_type: doc.file_name.split(".").pop().toLowerCase(), - })), - [documents], - ); - - const getNormalizedIdsKey = (ids) => - JSON.stringify([...ids].map(String).sort()); - - useEffect(() => { - const fetchDocuments = async () => { - setIsLoading(true); - try { - const docs = await loadDocuments(); - const sortedDocs = docs.sort((a, b) => { - const dateA = new Date(a.created); - const dateB = new Date(b.created); - return dateB - dateA; - }); - setDocuments(sortedDocs); - } catch (error) { - console.error("Failed to load documents:", error); - } finally { - setIsLoading(false); - } - }; - - fetchDocuments(); - }, []); - - useEffect(() => { - if ( - getNormalizedIdsKey(selectedIds) !== - getNormalizedIdsKey(initialSelectedIds) - ) { - setSelectedIds([...initialSelectedIds]); - } - }, [initialSelectedIds]); - - useEffect(() => { - const currentKey = getNormalizedIdsKey(selectedIds); - if (currentKey !== previousSelectedIdsRef.current) { - const selectedIdSet = new Set(selectedIds.map(String)); - const selectedDocs = documents.filter((doc) => - selectedIdSet.has(String(doc.id)), - ); - onSelect?.(selectedDocs); - previousSelectedIdsRef.current = currentKey; - } - }, [selectedIds, documents, onSelect]); - - const handleToggleSelection = useCallback((id) => { - setSelectedIds((prev) => { - const newSelected = prev.includes(id) - ? prev.filter((x) => x !== id) - : [...prev, id]; - return newSelected; - }); - }, []); - - /** - * Opens the document preview modal, fetching TXT content if applicable. - * @param {object} doc - The document to preview. - */ - const handleOpenPreview = async (doc) => { - setPreviewDoc(doc); - if (doc.file_type === "txt" && doc.preview) { - try { - const res = await fetch(normalizeUrl(doc.preview)); - const text = await res.text(); - setTxtContent(text); - } catch (e) { - setTxtContent("Error loading TXT file"); - } - } - setPreviewOpen(true); - }; - - const handleClosePreview = () => { - setPreviewOpen(false); - setPreviewDoc(null); - setTxtContent(""); - }; - - /** - * Merge an updated document (e.g. after extractor change) into the local - * documents state so the table reflects the saved extractor. - * @param {object} updatedDoc - The document returned by the update API. - */ - const handleExtractorChanged = useCallback((updatedDoc) => { - if (!updatedDoc) return; - setDocuments((prev) => - prev.map((doc) => - String(doc.id) === String(updatedDoc.id) - ? { ...doc, ...updatedDoc } - : doc, - ), - ); - }, []); - - const handleSelectAll = useCallback(() => { - setSelectedIds(documents.map((doc) => doc.id)); - }, [documents]); - - const handleDeselectAll = useCallback(() => { - setSelectedIds([]); - }, []); - - const handleAddDocument = useCallback(async (newDoc) => { - try { - return await addDocument(newDoc); - } catch (error) { - console.error("Failed to add document:", error); - // Re-throw the error so the caller can handle it appropriately - // Only 409 (duplicate) should be handled by the upload flow - throw error; - } - }, []); - - /** - * Merges uploaded documents into local state and selects them. - * @param {object[]} docs - Documents returned by the upload API. - */ - const applyUploadedDocuments = useCallback((docs) => { - if (docs.length === 0) return; - setDocuments((prev) => { - const nextDocs = [...docs, ...prev]; - return nextDocs.filter( - (doc, index, array) => - index === array.findIndex((candidate) => candidate.id === doc.id), - ); - }); - setSelectedIds((prev) => { - const nextSelected = new Set(prev.map(String)); - docs.forEach((doc) => nextSelected.add(String(doc.id))); - return Array.from(nextSelected); - }); - }, []); - - /** - * Deletes a document from the server and removes it from local state and selection. - * @param {number|string} id - The document ID to delete. - */ - const handleRemoveDocument = useCallback(async (id) => { - try { - await deleteDocument(id); - setDocuments((prev) => prev.filter((doc) => doc.id !== id)); - setSelectedIds((prev) => prev.filter((x) => x !== id)); - } catch (error) { - console.error("Failed to delete document:", error); - } - }, []); - - /** - * Handles multi-file upload, saving each document and updating local state. - * If a file already exists (409), pauses and asks the user for confirmation. - * @param {File|File[]} files - File(s) to upload. - * @param {string} [url] - Optional source URL. - */ - const handleFileUpload = useCallback( - async (files, url) => { - if (!files) return; - - const fileList = Array.isArray(files) ? files : [files]; - const uploadedDocuments = []; - - for (const file of fileList) { - try { - const result = await handleAddDocument({ - file, - optional_metadata: { - name: file.name, - source: url || "local_upload", - }, - }); - if (result.duplicate) { - applyUploadedDocuments(uploadedDocuments); - setDuplicatePending({ - file, - url: url || "local_upload", - affectedSessions: result.affectedSessions || [], - }); - return; - } - if (result.document) { - uploadedDocuments.push(result.document); - } - } catch (error) { - // Anything other than a duplicate: tell the user which file failed - // and why, then keep going with the rest of the selection. - console.error("Upload failed:", error); - enqueueSnackbar( - t("generative:rag.documents.uploadFailedReason", { - file: file.name, - reason: getApiErrorMessage( - error, - t("generative:rag.documents.uploadFailed"), - ), - }), - { variant: "error" }, - ); - } - } - - applyUploadedDocuments(uploadedDocuments); - setUploadOpen(false); - }, - [handleAddDocument, applyUploadedDocuments, enqueueSnackbar, t], - ); - - /** - * Re-uploads the pending duplicate file with force=true after user confirmation. - */ - const handleConfirmDuplicate = useCallback(async () => { - if (!duplicatePending) return; - const { file, url } = duplicatePending; - setDuplicatePending(null); - try { - const result = await addDocument({ - file, - optional_metadata: { name: file.name, source: url }, - force: true, - }); - if (!result.duplicate && result.document) { - applyUploadedDocuments([result.document]); - } - } catch (error) { - console.error("Forced upload failed:", error); - enqueueSnackbar( - t("generative:rag.documents.uploadFailedReason", { - file: file.name, - reason: getApiErrorMessage( - error, - t("generative:rag.documents.uploadFailed"), - ), - }), - { variant: "error" }, - ); - } - setUploadOpen(false); - }, [duplicatePending, applyUploadedDocuments, enqueueSnackbar, t]); - - const selectedIdSet = useMemo( - () => new Set(selectedIds.map((id) => String(id))), - [selectedIds], - ); - - const columns = useMemo( - () => [ - { - accessorKey: "file_name", - header: t("generative:rag.documents.table.name"), - size: 250, - Cell: ({ row }) => row.original.file_name, - }, - { - accessorKey: "file_type", - header: t("generative:rag.documents.table.type"), - size: 80, - Cell: ({ row }) => row.original.file_type?.toUpperCase() || "-", - }, - { - accessorKey: "created", - header: t("generative:rag.documents.table.created"), - size: 150, - Cell: ({ row }) => formatDate(row.original.created) || "-", - }, - { - id: "actions", - header: t("generative:rag.documents.table.actions"), - size: 100, - enableSorting: false, - enableColumnFilter: false, - muiTableHeadCellProps: { - align: "center", - }, - muiTableBodyCellProps: { - align: "center", - }, - Cell: ({ row }) => ( - - - { - setExtractorDoc(row.original); - setExtractorModalOpen(true); - }} - > - - - - - handleOpenPreview(row.original)} - > - - - - - handleRemoveDocument(row.original.id)} - color="error" - > - - - - - ), - }, - ], - [handleRemoveDocument, t], - ); - - const rowSelection = useMemo( - () => - tableData.reduce((acc, doc) => { - acc[String(doc.id)] = selectedIdSet.has(String(doc.id)); - return acc; - }, {}), - [tableData, selectedIdSet], - ); - - const table = useMaterialReactTable({ - columns, - data: tableData, - enableSelectAll: true, - enableRowSelection: true, - selectAllMode: "all", - enableColumnOrdering: false, - enableColumnActions: false, - enableColumnHiding: true, - enableDensityToggle: false, - enableFullScreenToggle: false, - enablePagination: true, - enableBottomToolbar: true, - enableTopToolbar: true, - enableGlobalFilter: true, - initialState: { - columnVisibility: { - file_type: false, - created: false, - }, - pagination: { - pageIndex: 0, - pageSize: 5, - }, - }, - muiPaginationProps: { - rowsPerPageOptions: [5, 10, 25, 50], - showFirstButton: false, - showLastButton: false, - }, - muiTablePaperProps: { - sx: { - boxShadow: "none", - borderRadius: 1, - display: "flex", - flexDirection: "column", - }, - }, - muiTableContainerProps: { - sx: { - maxHeight: "none", - flex: 1, - }, - }, - muiTableHeadCellProps: { - sx: { - backgroundColor: theme.palette.action.hover, - }, - }, - muiTableBodyCellProps: { - sx: { - padding: "8px 16px", - }, - }, - muiTableBodyRowProps: ({ row }) => ({ - sx: { - backgroundColor: selectedIdSet.has(String(row.original.id)) - ? theme.palette.action.selected - : "inherit", - "&:hover": { - backgroundColor: theme.palette.action.hover, - }, - }, - }), - rowNumberDisplayMode: "hidden", - renderTopToolbarCustomActions: () => ( - - - - ), - state: { - rowSelection, - isLoading, - }, - onRowSelectionChange: (updater) => { - const nextRowSelection = - typeof updater === "function" ? updater(rowSelection) : updater; - - const nextSelectedIdSet = new Set( - Object.entries(nextRowSelection) - .filter(([, isSelected]) => Boolean(isSelected)) - .map(([rowId]) => String(rowId)), - ); - - tableData.forEach((doc) => { - const idKey = String(doc.id); - const wasSelected = selectedIdSet.has(idKey); - const isSelectedNow = nextSelectedIdSet.has(idKey); - if (wasSelected !== isSelectedNow) { - handleToggleSelection(doc.id); - } - }); - }, - getRowId: (row) => String(row.id), - localization, - }); - - return ( - - - {t("generative:rag.setup.selectDocumentsDescription")} - - - - - - - setUploadOpen(false)} - maxWidth="sm" - fullWidth - paperProps={{ - sx: { - maxHeight: "80vh", - minHeight: "300px", - display: "flex", - flexDirection: "column", - }, - }} - > - - - setDuplicatePending(null)} - onConfirm={handleConfirmDuplicate} - /> - - { - setExtractorModalOpen(false); - setExtractorDoc(null); - }} - document={extractorDoc} - onExtractorChanged={handleExtractorChanged} - /> - - ); -} - -DocumentSelector.propTypes = { - selectedIds: PropTypes.arrayOf( - PropTypes.oneOfType([PropTypes.string, PropTypes.number]), - ), - onSelect: PropTypes.func, - sx: PropTypes.oneOfType([PropTypes.array, PropTypes.object, PropTypes.func]), -}; diff --git a/DashAI/front/src/components/generative/RAG/DocumentTable.jsx b/DashAI/front/src/components/generative/RAG/DocumentTable.jsx deleted file mode 100644 index 2e86d26d2..000000000 --- a/DashAI/front/src/components/generative/RAG/DocumentTable.jsx +++ /dev/null @@ -1,379 +0,0 @@ -import React, { useState } from "react"; -import Dialog from "@mui/material/Dialog"; -import Upload from "../../shared/Upload"; -import { addDocument } from "../../../api/rag"; -import DuplicateDocumentDialog from "./DuplicateDocumentDialog"; -import AddIcon from "@mui/icons-material/AddCircleOutline"; -import { - Paper, - Typography, - IconButton, - Tooltip, - LinearProgress, - Button, - Grid, -} from "@mui/material"; -import PropTypes from "prop-types"; -import { DataGrid } from "@mui/x-data-grid"; -import VisibilityIcon from "@mui/icons-material/Visibility"; -import SettingsIcon from "@mui/icons-material/Settings"; -import { formatDate } from "../../../utils"; -import DeleteItemModal from "../../custom/DeleteItemModal"; -import DocumentPreviewModal from "./DocumentPreviewModal"; -import DocumentExtractorModal from "./DocumentExtractorModal"; -import { normalizeUrl } from "../../../utils/urlUtils"; -import { useTranslation } from "react-i18next"; -import { useSnackbar } from "notistack"; -import { getApiErrorMessage } from "../../../utils/apiError"; - -/** - * DataGrid table listing documents with preview, deletion, and upload actions. - * - * @param {object} props - * @param {Array} props.documents - Array of document objects. - * @param {function} props.onRemove - Callback invoked with document ID when deleting. - * @param {function} [props.onAddDocument] - Callback invoked with the saved document after upload. - * @param {function} [props.onSelectDocument] - Callback invoked with the selected row when a row is clicked. - * @param {boolean} [props.isLoading=false] - Whether the data is still loading. - * @param {string} [props.tableTitle=null] - Custom table title (shown when showTableTitle is true). - * @param {boolean} [props.showTableTitle=false] - Whether to show the table title header. - * @returns {JSX.Element} - */ -export default function DocumentTable({ - documents, - onRemove, - onAddDocument, - onSelectDocument = null, - onExtractorChanged = null, - isLoading = false, - tableTitle = null, - showTableTitle = false, -}) { - const { t } = useTranslation(["generative", "common"]); - const { enqueueSnackbar } = useSnackbar(); - const [previewOpen, setPreviewOpen] = useState(false); - const [previewDoc, setPreviewDoc] = useState(null); - const [txtContent, setTxtContent] = useState(""); - const [uploadOpen, setUploadOpen] = useState(false); - const [extractorModalOpen, setExtractorModalOpen] = useState(false); - const [extractorDoc, setExtractorDoc] = useState(null); - const [duplicatePending, setDuplicatePending] = useState(null); - - /** - * Opens the document preview modal, fetching TXT content if applicable. - * @param {object} doc - The document to preview. - */ - const handleOpenPreview = async (doc) => { - setPreviewDoc(doc); - if (doc.file_type === "txt" && doc.preview) { - try { - const res = await fetch(normalizeUrl(doc.preview)); - const text = await res.text(); - setTxtContent(text); - } catch (e) { - setTxtContent("Error loading TXT file"); - } - } - setPreviewOpen(true); - }; - - const handleRemoveDocument = (id) => { - if (onRemove) onRemove(id); - }; - - /** - * Handles file upload from the Upload component, saving each file via the API. - * If a file already exists (409), pauses and asks the user for confirmation. - * @param {File|File[]} files - File(s) to upload. - * @param {string} [url] - Optional source URL. - */ - const handleFileUpload = async (files, url) => { - if (!files) return; - const fileList = Array.isArray(files) ? files : [files]; - for (const file of fileList) { - const docToAdd = { - file, - optional_metadata: { - name: file.name, - source: url || "local_upload", - }, - }; - try { - const result = await addDocument(docToAdd); - if (result.duplicate) { - setDuplicatePending({ - file, - url: url || "local_upload", - affectedSessions: result.affectedSessions || [], - }); - return; - } - if (onAddDocument) onAddDocument(result.document); - } catch (error) { - // Anything other than a duplicate: tell the user which file failed and - // why, then keep going with the rest of the selection. - console.error("Upload failed:", error); - enqueueSnackbar( - t("generative:rag.documents.uploadFailedReason", { - file: file.name, - reason: getApiErrorMessage( - error, - t("generative:rag.documents.uploadFailed"), - ), - }), - { variant: "error" }, - ); - } - } - setUploadOpen(false); - }; - - /** - * Re-uploads the pending duplicate file with force=true after user confirmation. - */ - const handleConfirmDuplicate = async () => { - if (!duplicatePending) return; - const { file, url } = duplicatePending; - setDuplicatePending(null); - try { - const result = await addDocument({ - file, - optional_metadata: { name: file.name, source: url }, - force: true, - }); - if (!result.duplicate && onAddDocument) { - onAddDocument(result.document); - } - } catch (error) { - console.error("Forced upload failed:", error); - enqueueSnackbar( - t("generative:rag.documents.uploadFailedReason", { - file: file.name, - reason: getApiErrorMessage( - error, - t("generative:rag.documents.uploadFailed"), - ), - }), - { variant: "error" }, - ); - } - setUploadOpen(false); - }; - - const handleClosePreview = () => { - setPreviewOpen(false); - setPreviewDoc(null); - setTxtContent(""); - }; - - const columns = [ - { - field: "id", - headerName: t("generative:rag.documents.table.id"), - flex: 0.1, - editable: false, - }, - { - field: "file_name", - headerName: t("generative:rag.documents.table.name"), - flex: 0.6, - editable: false, - }, - { - field: "created", - headerName: t("generative:rag.documents.table.created"), - flex: 0.4, - editable: false, - valueGetter: (value) => formatDate(value), - }, - { - field: "last_modified", - headerName: t("generative:rag.documents.table.lastModified"), - flex: 0.4, - editable: false, - valueGetter: (value, row) => { - return row?.optional_metadata?.last_modified - ? formatDate(row.optional_metadata.last_modified) - : t("common:na"); - }, - }, - { - field: "extractor", - headerName: t("generative:rag.documents.table.extractor"), - flex: 0.3, - editable: false, - valueGetter: (value, row) => row?.extractor?.component, - }, - { - field: "actions", - type: "actions", - headerName: t("generative:rag.documents.table.actions"), - flex: 0.3, - getActions: (params) => [ - - { - setExtractorDoc(params.row); - setExtractorModalOpen(true); - }} - > - - - , - - handleOpenPreview(params.row)} - > - - - , - handleRemoveDocument(params.row.id)} - item="document" - />, - ], - }, - ]; - - return ( - - {showTableTitle ? ( - - - {tableTitle || t("generative:rag.documents.table.currentDocuments")} - - - - ) : ( - - - - )} - {documents.length === 0 && !isLoading ? ( - - {t("generative:rag.documents.table.noDocumentsAvailable")} - - ) : ( - { - if (onSelectDocument) { - onSelectDocument(params.row); - } - }} - autoHeight - loading={isLoading} - slots={{ - loadingOverlay: LinearProgress, - }} - getRowId={(row) => row.id} - sx={{ - "& .MuiDataGrid-cell:focus": { outline: "none" }, - minHeight: 300, - }} - /> - )} - - {extractorDoc && ( - { - setExtractorModalOpen(false); - setExtractorDoc(null); - }} - document={extractorDoc} - onExtractorChanged={(updatedDoc) => { - if (onExtractorChanged) onExtractorChanged(); - setExtractorModalOpen(false); - setExtractorDoc(null); - }} - /> - )} - setUploadOpen(false)} - maxWidth="sm" - fullWidth - > - - - setDuplicatePending(null)} - onConfirm={handleConfirmDuplicate} - /> - - ); -} - -DocumentTable.propTypes = { - documents: PropTypes.arrayOf( - PropTypes.shape({ - id: PropTypes.string.isRequired, - name: PropTypes.string.isRequired, - createdAt: PropTypes.string.isRequired, - preview: PropTypes.string, - }), - ).isRequired, - onRemove: PropTypes.func.isRequired, - onSelectDocument: PropTypes.func, - onExtractorChanged: PropTypes.func, - isLoading: PropTypes.bool, - tableTitle: PropTypes.string, - showTableTitle: PropTypes.bool, -}; diff --git a/DashAI/front/src/components/generative/RAG/DocumentsBar.jsx b/DashAI/front/src/components/generative/RAG/DocumentsBar.jsx index cdc6c9460..8c4fa41ef 100644 --- a/DashAI/front/src/components/generative/RAG/DocumentsBar.jsx +++ b/DashAI/front/src/components/generative/RAG/DocumentsBar.jsx @@ -1,166 +1,154 @@ -import { useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import PropTypes from "prop-types"; import { useTranslation } from "react-i18next"; import { Box, - Typography, Button, Dialog, IconButton, Tooltip, + Typography, } from "@mui/material"; import AddIcon from "@mui/icons-material/AddCircleOutline"; -import ViewListIcon from "@mui/icons-material/ViewList"; -import { useNavigate } from "react-router-dom"; +import DeleteOutlineIcon from "@mui/icons-material/DeleteOutline"; +import TuneIcon from "@mui/icons-material/Tune"; +import { useSnackbar } from "notistack"; import SearchBar from "../../threeSectionLayout/SearchBar"; -import DocumentList from "./DocumentList"; +import DeleteConfirmationModal from "../../threeSectionLayout/DeleteConfirmationModal"; import Upload from "../../shared/Upload"; -import DuplicateDocumentDialog from "./DuplicateDocumentDialog"; -import { useSnackbar } from "notistack"; +import DocumentList from "./DocumentList"; +import DocumentPreviewModal from "./DocumentPreviewModal"; +import DocumentInspectorModal from "./DocumentInspectorModal"; import { getApiErrorMessage } from "../../../utils/apiError"; +import { normalizeUrl } from "../../../utils/urlUtils"; import { - getSessionDocuments, addDocument, - loadDocuments, + deleteDocument, + getSessionDocuments, } from "../../../api/rag"; /** - * Documents sidebar showing a searchable list of documents for the current RAG session. - * Supports upload and navigation to the full document table view. + * Shapes a document response for the list rows. + * @param {object} doc - A document as returned by the API. + * @returns {object} The row model. + */ +function toRow(doc) { + return { + id: doc.id, + name: doc.file_name, + type: doc.file_type, + uploadedAt: doc.created, + file_name: doc.file_name, + file_type: doc.file_type, + preview: doc.preview_url, + created: doc.created, + optional_metadata: doc.optional_metadata, + // Carried so the inspector opens on the document's own extractor rather + // than having to fetch it again. + extractor: doc.extractor, + default_extractor: doc.default_extractor, + }; +} + +/** + * The documents of one RAG session: add, inspect, and remove them. + * + * Everything the old standalone documents page could do lives here, because a + * document now belongs to exactly one session and there is nowhere else to + * manage it from. Reading the extracted text and choosing an extractor happen + * in a modal: the panel is too narrow to read a document in, and the centre + * column is deliberately reserved for the conversation. * * @param {object} props - * @param {number|string} [props.selectedSessionId] - Session ID to scope documents to. - * @param {string} props.taskName - Task name for context (e.g. "RAGTask"). - * @param {function} [props.onDocumentChange] - Callback fired after document upload. + * @param {number} props.sessionId - The session whose documents these are. + * @param {object} [props.indexStatus] - Index state, used to badge each row + * with its chunk count. + * @param {Function} [props.onDocumentChange] - Called after the set of + * documents changes, so the caller can re-poll the index status. * @param {boolean} [props.showSearch=true] - Whether to show the search bar. - * @returns {JSX.Element} + * @returns {JSX.Element} The documents panel. */ export default function DocumentsBar({ - selectedSessionId, - taskName, + sessionId, + indexStatus, onDocumentChange, showSearch = true, - indexStatus, }) { const { t } = useTranslation("generative"); - const [searchQuery, setSearchQuery] = useState(""); - const [documents, setDocuments] = useState([]); - const [filteredDocuments, setFilteredDocuments] = useState([]); - const [uploadOpen, setUploadOpen] = useState(false); - const [duplicatePending, setDuplicatePending] = useState(null); const { enqueueSnackbar } = useSnackbar(); - const navigate = useNavigate(); - const goToDocumentsDetail = () => { - navigate("/app/generative/rag/documents"); - }; + const [documents, setDocuments] = useState([]); + const [searchQuery, setSearchQuery] = useState(""); + const [uploadOpen, setUploadOpen] = useState(false); + const [previewDoc, setPreviewDoc] = useState(null); + const [previewText, setPreviewText] = useState(""); + const [inspectDoc, setInspectDoc] = useState(null); + const [pendingDelete, setPendingDelete] = useState(null); // Per-document chunk counts, so each row can say whether it is indexed. + // Indexing is a property of the session, not of one file, so every row shows + // it while a run is in flight. const indexStateByDocument = useMemo(() => { + const indexing = indexStatus?.status === "indexing"; const map = {}; (indexStatus?.documents ?? []).forEach((entry) => { - map[entry.document_id] = entry; + map[entry.document_id] = { ...entry, indexing }; }); return map; }, [indexStatus]); - useEffect(() => { - const fetchDocuments = async () => { - try { - let data; - if (selectedSessionId) { - data = await getSessionDocuments(selectedSessionId); - } else { - data = await loadDocuments(); - } - - const transformedDocuments = data.map((doc) => ({ - id: doc.id, - name: doc.file_name, - type: doc.file_type, - uploadedAt: doc.created, - file_name: doc.file_name, - file_type: doc.file_type, - preview: doc.preview_url, - created: doc.created, - optional_metadata: doc.optional_metadata, - })); - - setDocuments(transformedDocuments); - setFilteredDocuments(transformedDocuments); - } catch (error) { - enqueueSnackbar(t("documentsBar.failedFetch"), { - variant: "error", - }); - console.error("Failed to fetch documents:", error); - } - }; - - fetchDocuments(); - }, [selectedSessionId, enqueueSnackbar, t]); + const fetchDocuments = useCallback(async () => { + if (!sessionId) return; + try { + const data = await getSessionDocuments(sessionId); + setDocuments(data.map(toRow)); + } catch (error) { + console.error("Failed to fetch documents:", error); + enqueueSnackbar(t("documentsBar.failedFetch"), { variant: "error" }); + } + }, [sessionId, enqueueSnackbar, t]); useEffect(() => { - if (!searchQuery.trim()) { - setFilteredDocuments(documents); - return; - } + fetchDocuments(); + }, [fetchDocuments]); - const filtered = documents.filter((doc) => - doc.name.toLowerCase().includes(searchQuery.toLowerCase()), - ); - setFilteredDocuments(filtered); - }, [searchQuery, documents]); + const filteredDocuments = useMemo(() => { + const query = searchQuery.trim().toLowerCase(); + if (!query) return documents; + return documents.filter((doc) => doc.name.toLowerCase().includes(query)); + }, [documents, searchQuery]); /** - * Handles file upload, saving each file and updating local document state immediately. - * If a file already exists (409), pauses and asks the user for confirmation. + * Uploads the selected files into this session. * @param {File|File[]} files - File(s) to upload. * @param {string} [url] - Optional source URL. */ const handleFileUpload = async (files, url) => { if (!files) return; - const fileList = Array.isArray(files) ? files : [files]; - let uploadedCount = 0; + let uploaded = 0; for (const file of fileList) { try { const result = await addDocument({ + sessionId, file, - optional_metadata: { - name: file.name, - source: url || "local_upload", - }, + optional_metadata: { name: file.name, source: url || "local_upload" }, }); - if (result.duplicate) { - setDuplicatePending({ - file, - url: url || "local_upload", - affectedSessions: result.affectedSessions || [], - }); - return; + // The session already holds these exact bytes, so there is nothing + // to do and nothing to confirm. + enqueueSnackbar( + t("documentsBar.alreadyInSession", { file: file.name }), + { variant: "info" }, + ); + continue; } - - const savedDoc = result.document; - - // Add to local state immediately for UI feedback - const transformedDoc = { - id: savedDoc.id, - name: savedDoc.file_name, - type: savedDoc.file_type, - uploadedAt: savedDoc.created, - file_name: savedDoc.file_name, - file_type: savedDoc.file_type, - preview: savedDoc.preview_url, - created: savedDoc.created, - optional_metadata: savedDoc.optional_metadata, - }; - - setDocuments((prevDocs) => [transformedDoc, ...prevDocs]); - uploadedCount += 1; + setDocuments((previous) => [toRow(result.document), ...previous]); + uploaded += 1; } catch (error) { - // Anything other than a duplicate: tell the user which file failed and - // why, then keep going with the rest of the selection. + // Tell the user which file failed and why, then keep going with the + // rest of the selection. console.error("Upload failed:", error); enqueueSnackbar( t("documentsBar.uploadFailedReason", { @@ -172,64 +160,54 @@ export default function DocumentsBar({ } } - if (uploadedCount > 0) { - enqueueSnackbar( - t("documentsBar.successUpload", { count: uploadedCount }), - { variant: "success" }, - ); - if (onDocumentChange) { - onDocumentChange(); - } + if (uploaded > 0) { + enqueueSnackbar(t("documentsBar.successUpload", { count: uploaded }), { + variant: "success", + }); + onDocumentChange?.(); } setUploadOpen(false); }; - /** - * Re-uploads the pending duplicate file with force=true after user confirmation. - */ - const handleConfirmDuplicate = async () => { - if (!duplicatePending) return; - const { file, url } = duplicatePending; - setDuplicatePending(null); + /** Opens the plain preview, fetching the text for a txt document. */ + const handlePreview = async (doc) => { + setPreviewText(""); + setPreviewDoc(doc); + if (doc.file_type === "txt" && doc.preview) { + try { + const response = await fetch(normalizeUrl(doc.preview)); + setPreviewText(await response.text()); + } catch (error) { + console.error("Error loading TXT:", error); + setPreviewText(t("documentsBar.failedPreview")); + } + } + }; + + const handleConfirmDelete = async () => { + const doc = pendingDelete; + setPendingDelete(null); + if (!doc) return; try { - const result = await addDocument({ - file, - optional_metadata: { name: file.name, source: url }, - force: true, + await deleteDocument(doc.id); + setDocuments((previous) => previous.filter((row) => row.id !== doc.id)); + enqueueSnackbar(t("documentsBar.deleted", { file: doc.name }), { + variant: "success", }); - if (!result.duplicate) { - const savedDoc = result.document; - const transformedDoc = { - id: savedDoc.id, - name: savedDoc.file_name, - type: savedDoc.file_type, - uploadedAt: savedDoc.created, - file_name: savedDoc.file_name, - file_type: savedDoc.file_type, - preview: savedDoc.preview_url, - created: savedDoc.created, - optional_metadata: savedDoc.optional_metadata, - }; - setDocuments((prevDocs) => [transformedDoc, ...prevDocs]); - enqueueSnackbar(t("documentsBar.successUpload", { count: 1 }), { - variant: "success", - }); - if (onDocumentChange) { - onDocumentChange(); - } - } + onDocumentChange?.(); } catch (error) { - enqueueSnackbar(t("documentsBar.failedUpload"), { - variant: "error", - }); - console.error("Failed to upload document:", error); - } finally { - setUploadOpen(false); + console.error("Failed to delete document:", error); + enqueueSnackbar( + getApiErrorMessage(error, t("documentsBar.failedDelete")), + { variant: "error" }, + ); } }; - const handleDetailedView = () => { - navigate("/app/generative/rag/documents"); + /** Re-reads the list after an extractor change, and re-polls the index. */ + const handleExtractorChanged = async () => { + await fetchDocuments(); + onDocumentChange?.(); }; return ( @@ -240,93 +218,89 @@ export default function DocumentsBar({ overflow: "hidden", height: "100%", width: "100%", - minWidth: 0, // Prevent flex shrinking issues - maxWidth: "100%", // Ensure consistent width + minWidth: 0, + maxWidth: "100%", }} > - - {t("documentsBar.title")} - - - - - - + {t("documentsBar.title")} - {t("documentsBar.documentCount", { count: filteredDocuments.length })} - {selectedSessionId - ? t("documentsBar.inCurrentSession") - : t("documentsBar.available")} + {t("documentsBar.documentCount", { + count: filteredDocuments.length, + })} + {t("documentsBar.inCurrentSession")} - {/* Add documents button - only show when no session is selected */} - {!selectedSessionId && ( - - - - )} - {showSearch && documents.length >= 1 && ( + + + + + + {showSearch && documents.length > 1 && ( - - - setSearchQuery(e.target.value)} - onClear={() => setSearchQuery("")} - placeholder={t("documentsBar.searchPlaceholder")} - /> - - {!selectedSessionId && ( - - - - - - )} - + setSearchQuery(event.target.value)} + onClear={() => setSearchQuery("")} + placeholder={t("documentsBar.searchPlaceholder")} + /> )} {filteredDocuments.length > 0 ? ( ( + <> + + setInspectDoc(doc)} + aria-label={t("documentsBar.inspect")} + > + + + + + setPendingDelete(doc)} + aria-label={t("documentsBar.delete")} + > + + + + + )} /> ) : ( {searchQuery ? t("documentsBar.noDocumentsFound") - : selectedSessionId - ? t("documentsBar.noDocumentsInSession") - : t("documentsBar.noDocumentsAvailable")} + : t("documentsBar.noDocumentsInSession")} )} @@ -376,18 +348,45 @@ export default function DocumentsBar({ > - setDuplicatePending(null)} - onConfirm={handleConfirmDuplicate} + + { + setPreviewDoc(null); + setPreviewText(""); + }} + document={previewDoc} + txtContent={previewText} + /> + + setInspectDoc(null)} + document={inspectDoc} + onExtractorChanged={handleExtractorChanged} + /> + + setPendingDelete(null)} + onConfirm={handleConfirmDelete} + content={pendingDelete?.name} + warning={t("documentsBar.deleteWarning")} /> ); } + +DocumentsBar.propTypes = { + sessionId: PropTypes.oneOfType([PropTypes.string, PropTypes.number]) + .isRequired, + indexStatus: PropTypes.object, + onDocumentChange: PropTypes.func, + showSearch: PropTypes.bool, +}; diff --git a/DashAI/front/src/components/generative/RAG/DuplicateDocumentDialog.jsx b/DashAI/front/src/components/generative/RAG/DuplicateDocumentDialog.jsx deleted file mode 100644 index 74a51bc9d..000000000 --- a/DashAI/front/src/components/generative/RAG/DuplicateDocumentDialog.jsx +++ /dev/null @@ -1,88 +0,0 @@ -import { - Box, - Button, - Dialog, - DialogActions, - DialogContent, - DialogContentText, - DialogTitle, - Typography, -} from "@mui/material"; -import PropTypes from "prop-types"; -import { useTranslation } from "react-i18next"; - -/** - * Confirmation dialog shown when the user uploads a file that already exists. - * Lists the affected sessions and warns that fitted models will be deleted. - * - * @param {object} props - * @param {boolean} props.open - Whether the dialog is visible. - * @param {Array} props.affectedSessions - [{id, name}] sessions using the document. - * @param {function} props.onCancel - Close without forcing the update. - * @param {function} props.onConfirm - Force the update (re-upload with force=true). - * @returns {JSX.Element} - */ -export default function DuplicateDocumentDialog({ - open, - affectedSessions = [], - onCancel, - onConfirm, -}) { - const { t } = useTranslation("generative"); - - return ( - - {t("rag.documents.duplicate.title")} - - - {t("rag.documents.duplicate.message")} - - {affectedSessions.length > 0 ? ( - <> - - {t("rag.documents.duplicate.affectedSessions")} - - - {affectedSessions.map((session) => ( - - {session.name} - - ))} - - - ) : ( - - {t("rag.documents.duplicate.noAffectedSessions")} - - )} - - {t("rag.documents.duplicate.warning")} - - - - - - - - ); -} - -DuplicateDocumentDialog.propTypes = { - open: PropTypes.bool.isRequired, - affectedSessions: PropTypes.arrayOf( - PropTypes.shape({ - id: PropTypes.oneOfType([PropTypes.string, PropTypes.number]), - name: PropTypes.string, - }), - ), - onCancel: PropTypes.func.isRequired, - onConfirm: PropTypes.func.isRequired, -}; diff --git a/DashAI/front/src/components/generative/RAG/PresetCardList.jsx b/DashAI/front/src/components/generative/RAG/PresetCardList.jsx new file mode 100644 index 000000000..791db0eb6 --- /dev/null +++ b/DashAI/front/src/components/generative/RAG/PresetCardList.jsx @@ -0,0 +1,79 @@ +import PropTypes from "prop-types"; +import { Box, Stack, Typography } from "@mui/material"; +import CheckIcon from "@mui/icons-material/Check"; +import Paper from "@mui/material/Paper"; + +/** + * A vertical list of selectable preset cards. + * + * Wears the same card treatment as `ComponentSelector` -- flat `Paper`, a + * border that turns primary when active, a tick in the corner -- so a preset + * reads like the component choices elsewhere in the module. It does not reuse + * that component: `ComponentSelector` also brings a search field, category + * chips, download controls and a viewport-breakpoint grid, none of which apply + * to a recipe, and a two-column grid is unreadable in a panel this narrow. + * + * @param {object} props + * @param {Array} props.presets - Presets, each `{key, display_name, + * description, component, params}`. + * @param {string} [props.activeKey] - The preset the draft currently matches, + * or null when the configuration is custom. + * @param {Function} props.onSelect - Called with the chosen preset. + * @returns {JSX.Element} The card list. + */ +export default function PresetCardList({ presets, activeKey, onSelect }) { + return ( + + {presets.map((preset) => { + const isSelected = activeKey === preset.key; + return ( + onSelect(preset)} + sx={{ + p: 2, + cursor: "pointer", + border: 1, + borderColor: isSelected ? "primary.main" : "divider", + bgcolor: isSelected ? "action.selected" : "background.paper", + transition: "all 0.2s", + "&:hover": { borderColor: "secondary.main" }, + }} + > + + + + {preset.display_name} + + {preset.description && ( + + {preset.description} + + )} + + {isSelected && ( + + )} + + + ); + })} + + ); +} + +PresetCardList.propTypes = { + presets: PropTypes.array.isRequired, + activeKey: PropTypes.string, + onSelect: PropTypes.func.isRequired, +}; diff --git a/DashAI/front/src/components/generative/RAG/PromptEditor.jsx b/DashAI/front/src/components/generative/RAG/PromptEditor.jsx new file mode 100644 index 000000000..11c92ca8a --- /dev/null +++ b/DashAI/front/src/components/generative/RAG/PromptEditor.jsx @@ -0,0 +1,318 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import PropTypes from "prop-types"; +import { + Alert, + Box, + Button, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + IconButton, + MenuItem, + Stack, + TextField, + Tooltip, + Typography, +} from "@mui/material"; +import CloseIcon from "@mui/icons-material/Close"; +import OpenInFullIcon from "@mui/icons-material/OpenInFull"; +import { useTranslation } from "react-i18next"; +import HighlightedTextarea from "./HighlightedTextarea"; +import PlaceholdersList from "./PlaceholdersList"; +import { getDefaultPrompts } from "../../../api/rag"; +import { LANGUAGE_CODES } from "../../../constants/languages"; + +/** The component a hand-written template is stored under. */ +const CUSTOM_PROMPT_COMPONENT = "CustomRAGGenerationPrompt"; + +/** + * Placeholders every RAG prompt has to contain, used until the registry's own + * answer arrives (and if that request fails). + */ +const FALLBACK_REQUIRED = ["{chunks}", "{input}"]; + +/** + * The prompt of one RAG session, edited in place. + * + * This replaced a picker over a shared prompt library. Prompt rows are + * deduplicated by a hash of their parameters, so two sessions that chose the + * same template shared one row: editing it rewrote the other session's prompt. + * The template now lives in the session's own parameters, and the registry's + * built-in templates are only ever used to seed it. + * + * Nothing is sent while typing. The panel's Save writes the whole draft, so a + * half-finished template never reaches the pipeline. + * + * @param {object} props + * @param {object} props.promptModel - The draft `{component, params}` ref. + * @param {Function} props.setPromptModel - Replaces the draft ref. + * @param {Function} [props.onValidityChange] - Called with whether the + * template has every required placeholder. + * @returns {JSX.Element} The editor. + */ +export default function PromptEditor({ + promptModel, + setPromptModel, + onValidityChange, +}) { + const { t } = useTranslation(["generative", "common"]); + const [seeds, setSeeds] = useState([]); + const [placeholderSpec, setPlaceholderSpec] = useState({ + required: FALLBACK_REQUIRED, + descriptions: {}, + }); + const [expanded, setExpanded] = useState(false); + const textareaRef = useRef(null); + const dialogTextareaRef = useRef(null); + + const template = promptModel?.params?.template ?? ""; + // Which language's template to seed from. `CustomRAGGenerationPrompt` takes + // only a template and never reads a language, so storing one in its params + // would be a control that silently does nothing. + const [seedLanguage, setSeedLanguage] = useState( + promptModel?.params?.language ?? "en", + ); + + useEffect(() => { + let cancelled = false; + getDefaultPrompts() + .then((options) => { + if (cancelled) return; + // CustomRAGGenerationPrompt is the base class a hand-written template + // is stored under, not a template to start from. + setSeeds(options.filter((o) => o.name !== CUSTOM_PROMPT_COMPONENT)); + const base = options.find((o) => o.name === CUSTOM_PROMPT_COMPONENT); + if (base?.metadata) { + setPlaceholderSpec({ + required: base.metadata.required_placeholders ?? FALLBACK_REQUIRED, + descriptions: base.metadata.placeholder_descriptions ?? {}, + }); + } + }) + .catch((error) => { + console.error("Failed to load prompt templates:", error); + if (!cancelled) setSeeds([]); + }); + return () => { + cancelled = true; + }; + }, []); + + const missing = useMemo( + () => placeholderSpec.required.filter((ph) => !template.includes(ph)), + [placeholderSpec, template], + ); + + useEffect(() => { + onValidityChange?.(missing.length === 0); + }, [missing, onValidityChange]); + + const update = useCallback( + (nextTemplate) => { + setPromptModel({ + component: CUSTOM_PROMPT_COMPONENT, + params: { template: nextTemplate }, + }); + }, + [setPromptModel], + ); + + /** + * Replaces the template with a registry template. + * + * Seeding is explicit rather than a side effect of picking a language: the + * language select used to silently overwrite whatever the user had written. + * + * @param {string} name - The seed component's name. + */ + const handleSeed = (name) => { + const seed = seeds.find((option) => option.name === name); + const seeded = seed?.metadata?.templates?.[seedLanguage]; + if (seeded === undefined) return; + update(seeded); + }; + + const insertPlaceholder = useCallback( + (placeholder) => { + const textarea = expanded + ? dialogTextareaRef.current + : textareaRef.current; + if (!textarea) { + update(template + placeholder); + return; + } + const start = textarea.selectionStart; + const end = textarea.selectionEnd; + const next = + template.substring(0, start) + placeholder + template.substring(end); + update(next); + requestAnimationFrame(() => { + const position = start + placeholder.length; + textarea.selectionStart = position; + textarea.selectionEnd = position; + textarea.focus(); + }); + }, + [expanded, template, update], + ); + + const editor = (ref, minRows) => ( + update(event.target.value)} + minRows={minRows} + placeholder={t("generative:rag.prompt.editor.templatePlaceholder")} + /> + ); + + return ( + + + handleSeed(event.target.value)} + helperText={t("generative:rag.prompt.editor.startFromHelp")} + > + {seeds.map((seed) => ( + + {seed.name} + + ))} + + setSeedLanguage(event.target.value)} + sx={{ minWidth: 96 }} + > + {LANGUAGE_CODES.map((code) => ( + + {code} + + ))} + + + + + + + + + {t("generative:rag.prompt.editor.template")} + + + setExpanded(true)} + aria-label={t("generative:rag.prompt.editor.expand")} + > + + + + + {editor(textareaRef, 8)} + + + {missing.length > 0 && ( + + {t("generative:rag.prompt.editor.missingPlaceholder", { + placeholders: missing.join(", "), + })} + + )} + + {/* The same controlled state, with room to read: eight rows in a panel + this narrow is not enough for a real prompt. Dressed like the + module's other dialogs -- titled row with a close affordance, divided + body, actions along the bottom. */} + setExpanded(false)} + maxWidth="md" + fullWidth + PaperProps={{ sx: { minHeight: "500px" } }} + > + + {t("generative:rag.prompt.editor.template")} + setExpanded(false)} + size="small" + sx={{ color: "text.secondary" }} + aria-label={t("common:close")} + > + + + + + + + + {editor(dialogTextareaRef, 18)} + {missing.length > 0 && ( + + {t("generative:rag.prompt.editor.missingPlaceholder", { + placeholders: missing.join(", "), + })} + + )} + + + + + + + + + + ); +} + +PromptEditor.propTypes = { + promptModel: PropTypes.shape({ + component: PropTypes.string, + params: PropTypes.object, + }), + setPromptModel: PropTypes.func.isRequired, + onValidityChange: PropTypes.func, +}; diff --git a/DashAI/front/src/components/generative/RAG/PromptEditor.test.jsx b/DashAI/front/src/components/generative/RAG/PromptEditor.test.jsx new file mode 100644 index 000000000..f4c04124a --- /dev/null +++ b/DashAI/front/src/components/generative/RAG/PromptEditor.test.jsx @@ -0,0 +1,107 @@ +import React from "react"; +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../test-utils/renderWithProviders"; +import PromptEditor from "./PromptEditor"; + +jest.mock("../../../api/rag", () => ({ + getDefaultPrompts: jest.fn(), +})); + +const { getDefaultPrompts } = require("../../../api/rag"); + +const BASE_PROMPT = { + name: "CustomRAGGenerationPrompt", + metadata: { + required_placeholders: ["{chunks}", "{input}"], + placeholder_descriptions: { + "{chunks}": "The retrieved passages", + "{input}": "The user's message", + }, + }, +}; + +const SEED_PROMPT = { + name: "DefaultRAGGenerationPrompt", + metadata: { templates: { en: "Context: {chunks}\nQuestion: {input}" } }, +}; + +/** + * Renders the editor with a controlled template, exposing what it wrote back. + * @param {string} template - The starting template. + * @returns {object} The mock setter and validity callback. + */ +function renderEditor(template) { + const setPromptModel = jest.fn(); + const onValidityChange = jest.fn(); + renderWithProviders( + , + ); + return { setPromptModel, onValidityChange }; +} + +beforeEach(() => { + jest.clearAllMocks(); + getDefaultPrompts.mockResolvedValue([BASE_PROMPT, SEED_PROMPT]); +}); + +test("reports a complete template as valid", async () => { + const { onValidityChange } = renderEditor("Use {chunks} to answer {input}"); + await waitFor(() => expect(getDefaultPrompts).toHaveBeenCalled()); + await waitFor(() => expect(onValidityChange).toHaveBeenLastCalledWith(true)); +}); + +test("reports a template missing a placeholder as invalid, and says which", async () => { + const { onValidityChange } = renderEditor("Answer {input}"); + await waitFor(() => expect(onValidityChange).toHaveBeenLastCalledWith(false)); + // Two alerts render the same message (panel and expanded editor share it), + // so assert that the user is told at all rather than matching one node. + expect((await screen.findAllByRole("alert")).length).toBeGreaterThan(0); +}); + +test("writes edits back as a self-contained component ref", async () => { + const { setPromptModel } = renderEditor("Use {chunks} for {input}"); + await waitFor(() => expect(getDefaultPrompts).toHaveBeenCalled()); + + const textarea = screen.getAllByRole("textbox")[0]; + await userEvent.type(textarea, "!"); + + expect(setPromptModel).toHaveBeenCalled(); + const written = setPromptModel.mock.calls.at(-1)[0]; + expect(written.component).toBe("CustomRAGGenerationPrompt"); + expect(written.params).toHaveProperty("template"); +}); + +test("changing the seed language does not touch the prompt", async () => { + const template = "Hand-written: {chunks} {input}"; + const { setPromptModel } = renderEditor(template); + await waitFor(() => expect(getDefaultPrompts).toHaveBeenCalled()); + + // The select only decides which template "Start from" would copy in. It used + // to reseed as a side effect, discarding whatever the user had written, and + // it used to store a `language` on a component that never reads one. + const selects = screen.getAllByRole("combobox"); + await userEvent.click(selects[selects.length - 1]); + const option = await screen.findByRole("option", { name: "es" }); + await userEvent.click(option); + + expect(setPromptModel).not.toHaveBeenCalled(); +}); + +test("writes only what the prompt component actually reads", async () => { + const { setPromptModel } = renderEditor("Use {chunks} for {input}"); + await waitFor(() => expect(getDefaultPrompts).toHaveBeenCalled()); + + const textarea = screen.getAllByRole("textbox")[0]; + await userEvent.type(textarea, "!"); + + const written = setPromptModel.mock.calls.at(-1)[0]; + expect(Object.keys(written.params)).toEqual(["template"]); +}); diff --git a/DashAI/front/src/components/generative/RAG/PromptParamsCard.jsx b/DashAI/front/src/components/generative/RAG/PromptParamsCard.jsx deleted file mode 100644 index 16ddd5dfa..000000000 --- a/DashAI/front/src/components/generative/RAG/PromptParamsCard.jsx +++ /dev/null @@ -1,525 +0,0 @@ -import { useState, useEffect, useCallback, useMemo, useRef } from "react"; -import { - Box, - Typography, - Card, - CardContent, - Autocomplete, - TextField, - Button, - MenuItem, - useTheme, -} from "@mui/material"; -import { useNavigate } from "react-router-dom"; -import { - ViewList as ViewListIcon, - Info as InfoIcon, - ExpandMore as ExpandMoreIcon, - AddCircleOutline as AddIcon, -} from "@mui/icons-material"; -import IconButton from "@mui/material/IconButton"; -import Tooltip from "@mui/material/Tooltip"; -import PropTypes from "prop-types"; -import { useTranslation } from "react-i18next"; -import { - getRAGPrompts, - getDefaultPrompts, - isGenerationPromptClass, -} from "../../../api/rag"; -import NewPromptModal from "../../../pages/generative/RAGSession/advanced/NewPromptModal"; -import RAGSectionColumn from "../../../pages/generative/RAGSession/components/RAGSectionColumn"; -import { - getDescription, - renderTemplateWithHighlights, -} from "../../../pages/generative/RAGSession/components/sectionUtils"; - -import { LANGUAGE_CODES } from "../../../constants/languages"; - -const CREATE_NEW_ID = "__create-new__"; -const DEFAULT_IDS = { - DefaultRAGGenerationPrompt: "default-generation", - DefaultQARAGGenerationPrompt: "default-QA", -}; - -/** - * Returns the translated display name for a default prompt option. - * @param {object} option - The prompt option object. - * @param {function} t - i18n translate function. - * @returns {string} - */ -function getDefaultDisplayName(option, t) { - // Use class_name (set explicitly by our code) rather than name - // (raw API field) to avoid potential encoding / serialization mismatches. - const cname = option.class_name || option.name || ""; - if (cname.includes("DefaultQARAGGenerationPrompt")) { - return t("generative:rag.prompt.defaultQAGenerationPrompt"); - } - if (cname.includes("DefaultRAGGenerationPrompt")) { - return t("generative:rag.prompt.defaultGenerationPrompt"); - } - return option.name || cname; -} - -/** - * Returns the display label for a given prompt option (default, custom, or "create new"). - * @param {object} option - The prompt option object. - * @param {function} t - i18n translate function. - * @returns {string} - */ -function getOptionLabel(option, t) { - if (option._isCreateNew) return option.name; - if (option._isDefault) return getDefaultDisplayName(option, t); - return option.name; -} - -/** - * Card for selecting and viewing a prompt template (default or custom), with - * language switcher, description toggle, and inline template preview with highlights. - * - * @param {object} props - * @param {object} props.promptModel - { component: string, params: { template, language, ... } } - * @param {function} props.setPromptModel - State setter for promptModel. - * @param {function} [props.onTokenCountChange] - Callback with estimated token count of the selected template. - * @returns {JSX.Element|null} - */ -export default function PromptParamsCard({ - promptModel, - setPromptModel, - onTokenCountChange, -}) { - const navigate = useNavigate(); - const goToPromptsDetail = () => navigate("/app/generative/rag/prompts"); - const { t, i18n } = useTranslation(["generative"]); - const theme = useTheme(); - const placeholderColors = useMemo( - () => ({ - bg: theme.palette.placeholder?.bg || theme.palette.warning.light, - text: theme.palette.placeholder?.text || theme.palette.warning.dark, - }), - [theme], - ); - const [showDescription, setShowDescription] = useState(false); - const [isExpanded, setIsExpanded] = useState(false); - - const platformLang = useMemo(() => { - const lang = (i18n.language || "en").split("-")[0]; - return ["en", "es", "pt"].includes(lang) ? lang : "en"; - }, [i18n.language]); - - const [customPrompts, setCustomPrompts] = useState([]); - const [defaultPrompts, setDefaultPrompts] = useState([]); - const [selectedPrompt, setSelectedPrompt] = useState(null); - const [newPromptModalOpen, setNewPromptModalOpen] = useState(false); - const [loading, setLoading] = useState(true); - const [selectedLanguage, setSelectedLanguage] = useState(platformLang); - const prevSelectedRef = useRef(null); - const isInitializedRef = useRef(false); - - /** - * Loads custom RAG prompts and default prompt templates from the API. - * Excludes system defaults from custom prompts and CustomRAGGenerationPrompt from defaults. - */ - const loadPrompts = useCallback(async () => { - try { - const dbPrompts = await getRAGPrompts(); - // Keep only user-created generation prompts: - // 1. Exclude system defaults (class_name starts with "Default") - // 2. Exclude augmentation prompts (wrong type for this selector) - setCustomPrompts( - (dbPrompts || []).filter( - (p) => - !p.class_name.startsWith("Default") && - isGenerationPromptClass(p.class_name), - ), - ); - } catch (error) { - console.error("Error loading custom RAG prompts:", error); - setCustomPrompts([]); - } - - try { - const defaultData = await getDefaultPrompts(); - // Filter out CustomRAGGenerationPrompt — it's a base class for - // user-created prompts, not a selectable template. It has no - // templates in its metadata and inherits a generic description - // ("Base class for RAG prompts.") from Prompt. - setDefaultPrompts( - (defaultData || []).filter( - (dp) => dp.name !== "CustomRAGGenerationPrompt", - ), - ); - } catch (error) { - console.error("Error loading default RAG prompts:", error); - setDefaultPrompts([]); - } - }, []); - - useEffect(() => { - const load = async () => { - await loadPrompts(); - setLoading(false); - }; - load(); - }, [loadPrompts]); - - const mergedOptions = useMemo(() => { - const defaults = defaultPrompts.map((dp) => ({ - ...dp, - id: DEFAULT_IDS[dp.name] || dp.name, - _isDefault: true, - class_name: dp.name, - })); - const customs = customPrompts.map((cp) => ({ - ...cp, - _isDefault: false, - })); - return [ - ...defaults, - ...customs, - { - id: CREATE_NEW_ID, - name: t("generative:rag.prompt.createNewPrompt"), - _isCreateNew: true, - _isDefault: false, - class_name: "", - }, - ]; - }, [defaultPrompts, customPrompts, t]); - - const currentTemplate = useMemo(() => { - if (!selectedPrompt) return ""; - if (selectedPrompt._isDefault) { - return selectedPrompt.metadata?.templates?.[selectedLanguage] || ""; - } - if (selectedPrompt.parameters?.templates) { - return selectedPrompt.parameters.templates[selectedLanguage] || ""; - } - return selectedPrompt.parameters?.template || ""; - }, [selectedPrompt, selectedLanguage]); - - const isDefault = selectedPrompt?._isDefault; - - useEffect(() => { - if (!selectedPrompt) { - if (onTokenCountChange) onTokenCountChange(0); - return; - } - - if ( - !isInitializedRef.current && - promptModel?.component === - (selectedPrompt.class_name || selectedPrompt.name) - ) { - isInitializedRef.current = true; - return; - } - isInitializedRef.current = true; - - setPromptModel({ - component: selectedPrompt.class_name || selectedPrompt.name, - params: { - template: currentTemplate, - language: selectedLanguage, - ...(selectedPrompt._isDefault || selectedPrompt.parameters?.templates - ? { templates: selectedPrompt.parameters?.templates } - : {}), - }, - }); - if (onTokenCountChange) { - const tokenCount = Math.ceil(currentTemplate.length / 4); - onTokenCountChange(tokenCount); - } - }, [selectedPrompt, selectedLanguage]); - - useEffect(() => { - const selectable = mergedOptions.filter((o) => !o._isCreateNew); - if (!selectable.length) return; - - if (promptModel?.component) { - const found = selectable.find((p) => { - if (p._isDefault) { - return p.class_name === promptModel.component; - } - return ( - p.class_name === promptModel.component && - p.parameters?.template === promptModel.params?.template - ); - }); - if (found?.id !== selectedPrompt?.id) { - setSelectedPrompt(found || null); - prevSelectedRef.current = found || null; - if (found?._isDefault) { - setSelectedLanguage(promptModel.params?.language || platformLang); - } - isInitializedRef.current = false; - } - return; - } - - if (!selectedPrompt) { - const firstDefault = - selectable.find((p) => p._isDefault) || selectable[0]; - if (firstDefault) { - setSelectedPrompt(firstDefault); - setSelectedLanguage(platformLang); - prevSelectedRef.current = firstDefault; - } - } - }, [mergedOptions, promptModel]); - - /** - * Handles selection change in the autocomplete. Intercepts the "Create new" option - * to open the creation modal instead of selecting it. - * @param {object} _event - The change event. - * @param {object|null} newValue - The newly selected option. - */ - const handlePromptChange = (_event, newValue) => { - if (newValue?._isCreateNew) { - setNewPromptModalOpen(true); - setSelectedPrompt(prevSelectedRef.current); - return; - } - prevSelectedRef.current = newValue; - setSelectedPrompt(newValue); - if (newValue?._isDefault) { - setSelectedLanguage(platformLang); - } - }; - - const handleLanguageChange = (event) => { - setSelectedLanguage(event.target.value); - }; - - /** - * Refetches prompts after creation, selects the new prompt, and syncs state. - * @param {number|string} newPromptId - ID of the newly created prompt. - */ - const handlePromptCreated = useCallback( - async (newPromptId) => { - const updatedPrompts = await getRAGPrompts(); - setCustomPrompts( - (updatedPrompts || []).filter( - (p) => - !p.class_name.startsWith("Default") && - isGenerationPromptClass(p.class_name), - ), - ); - - const newPrompt = (updatedPrompts || []).find( - (p) => p.id === newPromptId, - ); - if (newPrompt) { - const wrapped = { ...newPrompt, _isDefault: false }; - setSelectedPrompt(wrapped); - prevSelectedRef.current = wrapped; - // Sync language from the newly created prompt's saved language - // so local selectedLanguage stays aligned with the persisted value. - if (newPrompt.parameters?.language) { - setSelectedLanguage(newPrompt.parameters.language); - } - // Sync parent promptModel immediately to prevent infinite - // useEffect 1 <-> useEffect 2 ping-pong when the new prompt's - // class_name differs from the previously selected prompt. - setPromptModel({ - component: newPrompt.class_name, - params: { - template: newPrompt.parameters?.template || "", - language: newPrompt.parameters?.language || "", - }, - }); - const template = newPrompt.parameters?.template || ""; - const tokenCount = Math.ceil(template.length / 4); - onTokenCountChange?.(tokenCount); - } - - setNewPromptModalOpen(false); - }, - [onTokenCountChange, setPromptModel], - ); - - if (loading) { - return null; - } - - return ( - - - - - - {t("generative:rag.prompt.promptLabel")} - - - {isExpanded && ( - - setShowDescription((s) => !s)} - aria-label="prompt-info" - sx={{ color: "text.secondary" }} - > - - - - )} - - - - - - - setIsExpanded((s) => !s)} - aria-label="toggle-prompt-card" - > - - - - - - {isExpanded && showDescription && ( - - {t("generative:rag.prompt.description")} - - )} - - - - - getOptionLabel(option, t)} - isOptionEqualToValue={(option, value) => option.id === value.id} - renderOption={(props, option) => ( -
  • {getOptionLabel(option, t)}
    • - )} - renderInput={(params) => ( - - )} - sx={{}} - /> -     - - {isExpanded && - (isDefault || selectedPrompt?.parameters?.templates) && ( - - {LANGUAGE_CODES.map((code) => ( - - {t(`generative:rag.prompt.languages.${code}`)} - - ))} - - )} - - {isExpanded && selectedPrompt && ( - - - {t("generative:rag.prompt.selectedTemplate")} - - - - {renderTemplateWithHighlights( - currentTemplate, - placeholderColors, - theme.typography.code.fontFamily, - )} - - - {getDescription(selectedPrompt.description, i18n) && ( - - {getDescription(selectedPrompt.description, i18n)} - - )} - - )} - - {isExpanded && ( - - )} -     -     - - setNewPromptModalOpen(false)} - onPromptCreated={handlePromptCreated} - existingPrompts={customPrompts} - /> -     - ); -} - -PromptParamsCard.propTypes = { - promptModel: PropTypes.shape({ - component: PropTypes.string, - params: PropTypes.shape({ - template: PropTypes.string, - language: PropTypes.string, - }), - }), - setPromptModel: PropTypes.func.isRequired, - onTokenCountChange: PropTypes.func, -}; diff --git a/DashAI/front/src/components/generative/RAG/PromptSelectionTable.jsx b/DashAI/front/src/components/generative/RAG/PromptSelectionTable.jsx deleted file mode 100644 index b7ea0d6be..000000000 --- a/DashAI/front/src/components/generative/RAG/PromptSelectionTable.jsx +++ /dev/null @@ -1,291 +0,0 @@ -import React, { useState } from "react"; -import PropTypes from "prop-types"; -import { - Box, - Paper, - Tooltip, - IconButton, - Button, - Grid, - Typography, -} from "@mui/material"; -import { AddCircleOutline as AddIcon } from "@mui/icons-material"; -import VisibilityIcon from "@mui/icons-material/Visibility"; -import { DataGrid } from "@mui/x-data-grid"; -import { useTranslation } from "react-i18next"; -import { formatDate } from "../../../utils"; -import PromptViewModal from "./PromptViewModal"; -import NewPromptModal from "../../../pages/generative/RAGSession/advanced/NewPromptModal"; -import { getRAGPrompts } from "../../../api/rag"; - -/** - * Expand default prompts (with `templates` dict) into one row per language. - * Custom prompts (single `template`) remain as a single row. - * - * Each expanded row includes a `_parentPrompt` reference to the original - * prompt object, used by the view modal to show all language versions. - * - * @param {Array} prompts - Raw prompt objects from the API. - * @returns {Array} Flattened rows ready for the DataGrid. - */ -function expandPromptRows(prompts) { - const rows = []; - for (const prompt of prompts) { - const templates = prompt.parameters?.templates; - if (templates && Object.keys(templates).length > 0) { - for (const lang of Object.keys(templates)) { - rows.push({ - ...prompt, - id: `${prompt.id}-${lang}`, - _originalId: prompt.id, - language: lang, - _parentPrompt: prompt, - }); - } - } else { - rows.push({ - ...prompt, - language: prompt.parameters?.language || null, - _originalId: prompt.id, - _parentPrompt: prompt, - }); - } - } - return rows; -} - -/** - * DataGrid table of available prompts with expanded multi-language rows, - * view modal access, and "New Prompt" creation flow. - * - * @param {object} props - * @param {Array} [props.prompts=[]] - Initial prompt list. - * @param {boolean} [props.loading=false] - Whether the table data is loading. - * @param {Array} [props.rowSelectionModel=[]] - Currently selected row IDs. - * @param {function} [props.onRowSelectionModelChange] - Selection change callback. - * @param {boolean} [props.showTableTitle=false] - Whether to show the heading row. - * @param {function} [props.setSessionData] - State setter for session data (updates prompt_id). - * @returns {JSX.Element} - */ -export default function PromptSelectionTable({ - prompts = [], - loading = false, - rowSelectionModel = [], - onRowSelectionModelChange, - showTableTitle = false, - setSessionData, -}) { - const [modalOpen, setModalOpen] = useState(false); - const [selectedPrompt, setSelectedPrompt] = useState(null); - const [newPromptModalOpen, setNewPromptModalOpen] = useState(false); - const [promptRows, setPromptRows] = useState([]); - const [rawPrompts, setRawPrompts] = useState([]); - const { t } = useTranslation(["generative"]); - - React.useEffect(() => { - async function fetchPrompts() { - const initialPrompts = await getRAGPrompts(); - initialPrompts.sort((a, b) => new Date(b.created) - new Date(a.created)); - setRawPrompts(initialPrompts); - setPromptRows(expandPromptRows(initialPrompts)); - } - fetchPrompts(); - }, []); - - const handleViewPrompt = (row) => { - // Always open the full parent prompt so the modal shows all languages - setSelectedPrompt(row._parentPrompt || row); - setModalOpen(true); - }; - - const handleCloseModal = () => { - setModalOpen(false); - setSelectedPrompt(null); - }; - - const columns = React.useMemo( - () => [ - { - field: "id", - headerName: t("generative:rag.promptView.table.id"), - minWidth: 50, - flex: 0.3, - editable: false, - }, - { - field: "name", - headerName: t("generative:rag.promptView.table.name"), - minWidth: 140, - flex: 1, - editable: false, - }, - { - field: "class_name", - headerName: t("generative:rag.promptView.table.type"), - minWidth: 140, - flex: 1, - editable: false, - }, - { - field: "language", - headerName: t("generative:rag.promptView.table.language"), - minWidth: 100, - flex: 0.7, - editable: false, - valueGetter: (value, row) => { - if (row.language) { - return ( - t(`generative:rag.prompt.languages.${row.language}`) || - row.language - ); - } - return "-"; - }, - }, - { - field: "created", - headerName: t("generative:rag.promptView.table.created"), - minWidth: 140, - flex: 1, - editable: false, - valueGetter: (value) => formatDate(value), - }, - { - field: "last_modified", - headerName: t("generative:rag.promptView.table.edited"), - minWidth: 140, - flex: 1, - editable: false, - valueGetter: (value) => formatDate(value), - }, - { - field: "actions", - type: "actions", - headerName: t("generative:rag.promptView.table.actions"), - minWidth: 80, - flex: 0.4, - getActions: (params) => [ - - handleViewPrompt(params.row)} - > - - - , - ], - }, - ], - [t], - ); - - /** - * Refetches prompts after creation and selects the newly created one. - * @param {number|string} newPromptId - ID of the newly created prompt. - */ - const handlePromptCreated = async (newPromptId) => { - const updatedPrompts = await getRAGPrompts(); - updatedPrompts.sort((a, b) => new Date(b.created) - new Date(a.created)); - setRawPrompts(updatedPrompts); - setPromptRows(expandPromptRows(updatedPrompts)); - if (onRowSelectionModelChange && newPromptId) { - onRowSelectionModelChange([newPromptId]); - } - if (setSessionData && newPromptId) { - setSessionData((prev) => ({ - ...prev, - parameters: { - ...prev.parameters, - prompt_id: newPromptId, - }, - })); - } - setNewPromptModalOpen(false); - }; - - return ( - - {showTableTitle && ( - - - {t("generative:rag.promptView.table.currentPrompts")} - - - - )} - {!showTableTitle && ( - - - {t("generative:rag.promptView.table.choosePrompt")} - - - - )} - - - - setNewPromptModalOpen(false)} - onPromptCreated={handlePromptCreated} - existingPrompts={rawPrompts} - /> - - - ); -} - -PromptSelectionTable.propTypes = { - prompts: PropTypes.array, - loading: PropTypes.bool, - rowSelectionModel: PropTypes.array, - onRowSelectionModelChange: PropTypes.func, - showTableTitle: PropTypes.bool, - setSessionData: PropTypes.func, -}; diff --git a/DashAI/front/src/components/generative/RAG/PromptViewModal.jsx b/DashAI/front/src/components/generative/RAG/PromptViewModal.jsx deleted file mode 100644 index 42fc8e81a..000000000 --- a/DashAI/front/src/components/generative/RAG/PromptViewModal.jsx +++ /dev/null @@ -1,197 +0,0 @@ -import { useState, useMemo } from "react"; -import PropTypes from "prop-types"; -import { - Dialog, - DialogTitle, - DialogContent, - DialogActions, - Button, - TextField, - MenuItem, - Typography, - Box, - useTheme, -} from "@mui/material"; -import { useTranslation } from "react-i18next"; -import { renderTemplateWithHighlights } from "../../../pages/generative/RAGSession/components/sectionUtils"; -import { LANGUAGE_CODES } from "../../../constants/languages"; - -/** - * Dialog that displays prompt content with optional language selection. - * - * Supports two prompt shapes: - * - Single-template: content in `prompt.parameters.template` (string) - * - Multi-template: content in `prompt.parameters.templates` (dict of `{ [lang]: string }`) - * - * For multi-template prompts, a language selector is enabled so the user can - * switch between available language versions. For single-template prompts, - * the selector is disabled and shows either the stored language or - * "Language not available". - * - * The parent must pass a unique `key` prop (e.g. `prompt.id`) to ensure state - * resets when a different prompt is displayed (component is always mounted). - * - * @param {object} props - * @param {boolean} props.open - Whether the dialog is visible - * @param {function} props.handleClose - Callback when the dialog is closed - * @param {object} [props.prompt] - The prompt object to display - * @param {string} props.prompt.name - Display name (shown in title bar) - * @param {string} props.prompt.class_name - Component type (shown as "Type") - * @param {object} props.prompt.parameters - Parameters bag - * @param {string} [props.prompt.parameters.template] - Single template string - * @param {object} [props.prompt.parameters.templates] - Multi-language template dict - * @param {string} [props.prompt.parameters.language] - Default language code - */ -export default function PromptViewModal({ open, handleClose, prompt }) { - const { t } = useTranslation(["generative"]); - const theme = useTheme(); - - const placeholderColors = useMemo( - () => ({ - bg: theme.palette.placeholder?.bg || theme.palette.warning.light, - text: theme.palette.placeholder?.text || theme.palette.warning.dark, - }), - [theme], - ); - - const hasMultiTemplates = useMemo( - () => - !!prompt?.parameters?.templates && - Object.keys(prompt.parameters.templates).length > 0, - [prompt], - ); - - const [selectedLanguage, setSelectedLanguage] = useState(() => { - if (prompt?.parameters?.templates) { - return ( - prompt?.parameters?.language || - Object.keys(prompt.parameters.templates)[0] || - "" - ); - } - return ""; - }); - - /** Language options derived from available template keys for multi-template prompts. */ - const languageOptions = useMemo(() => { - if (!hasMultiTemplates || !prompt?.parameters?.templates) return []; - const codes = Object.keys(prompt.parameters.templates); - return codes.map((code) => ({ - code, - name: t(`generative:rag.prompt.languages.${code}`) || code, - })); - }, [hasMultiTemplates, prompt, t]); - - const currentTemplate = useMemo(() => { - if (hasMultiTemplates) { - return prompt?.parameters?.templates?.[selectedLanguage] || ""; - } - return prompt?.parameters?.template || ""; - }, [hasMultiTemplates, prompt, selectedLanguage]); - - return ( - - - {prompt?.name || t("generative:rag.promptView.untitledPrompt")} - - - - - {hasMultiTemplates ? ( - setSelectedLanguage(e.target.value)} - fullWidth - size="small" - sx={{ mb: 2 }} - > - {languageOptions.map((opt) => ( - - {opt.name} - - ))} - - ) : ( - - )} - - - {t("generative:rag.promptView.templateContent")} - - - {renderTemplateWithHighlights( - currentTemplate, - placeholderColors, - theme.typography.code.fontFamily, - ) ?? ( - - {hasMultiTemplates - ? t("generative:rag.promptView.languageNotAvailable") - : t("generative:rag.promptView.noContent")} - - )} - - - - - - - ); -} - -PromptViewModal.propTypes = { - open: PropTypes.bool.isRequired, - handleClose: PropTypes.func.isRequired, - prompt: PropTypes.shape({ - id: PropTypes.number, - name: PropTypes.string, - class_name: PropTypes.string, - parameters: PropTypes.shape({ - template: PropTypes.string, - templates: PropTypes.objectOf(PropTypes.string), - language: PropTypes.string, - }), - }), -}; diff --git a/DashAI/front/src/components/generative/RAG/RAGBreadcrumbs.jsx b/DashAI/front/src/components/generative/RAG/RAGBreadcrumbs.jsx index 809bcfbf1..d96a75627 100644 --- a/DashAI/front/src/components/generative/RAG/RAGBreadcrumbs.jsx +++ b/DashAI/front/src/components/generative/RAG/RAGBreadcrumbs.jsx @@ -61,40 +61,13 @@ function RAGBreadcrumbs({ sessionName }) { }, ]; - if (path === `${RAG_ROOT}/documents`) - return [ - ...base, - { - label: t("generative:rag.breadcrumbs.documents"), - path: null, - current: true, - }, - ]; - if (path === `${RAG_ROOT}/prompts`) - return [ - ...base, - { - label: t("generative:rag.breadcrumbs.prompts"), - path: null, - current: true, - }, - ]; - if (path === `${RAG_ROOT}/new`) - return [ - ...base, - { - label: t("generative:rag.create.title"), - path: null, - current: true, - }, - ]; - if (sessionName) return [ ...base, { label: sessionName, path: null, current: true, isSession: true }, ]; + // The RAG root is the creation form, so it is the end of the trail. base[1] = { ...base[1], path: null, current: true }; return base; }; diff --git a/DashAI/front/src/components/generative/RAG/RAGConfigPanel.jsx b/DashAI/front/src/components/generative/RAG/RAGConfigPanel.jsx index dff0fae7f..7cba389a6 100644 --- a/DashAI/front/src/components/generative/RAG/RAGConfigPanel.jsx +++ b/DashAI/front/src/components/generative/RAG/RAGConfigPanel.jsx @@ -6,22 +6,23 @@ import { Alert, Box, Button, - Chip, CircularProgress, - Collapse, Divider, IconButton, LinearProgress, Stack, + Tab, TextField, Tooltip, Typography, } from "@mui/material"; import EditIcon from "@mui/icons-material/Edit"; -import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; +import FiberManualRecordIcon from "@mui/icons-material/FiberManualRecord"; import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; import SideBar from "../../threeSectionLayout/panelContainers/SideBar"; -import PromptParamsCard from "./PromptParamsCard"; +import PillTabs from "../../shared/PillTabs"; +import PresetCardList from "./PresetCardList"; +import PromptEditor from "./PromptEditor"; import GeneratorPicker from "./GeneratorPicker"; import ChunkingAdvancedModal from "../../../pages/generative/RAGSession/advanced/ChunkingAdvancedModal"; import RetrieverAdvancedModal from "../../../pages/generative/RAGSession/advanced/RetrieverAdvancedModal"; @@ -37,83 +38,18 @@ import { import { updateGenerativeSession } from "../../../api/session"; import { getApiErrorMessage } from "../../../utils/apiError"; -/** Section keys, matching the RAG parameter keys the backend uses. */ +/** + * Section keys, matching the RAG parameter keys the backend uses, in the order + * the pipeline applies them: split the documents, retrieve from them, answer + * with a model, phrase it with a prompt. + */ const SECTIONS = [ "chunking_model", "retriever_model", - "prompt", "generation_model", + "prompt", ]; -/** - * A collapsible configuration section with a backend-supplied title. - * - * @param {object} props - * @param {string} props.title - Localized section name. - * @param {string} [props.summary] - One-line current value, shown when collapsed. - * @param {string} [props.info] - Contextual help, shown behind an info icon. - * @param {boolean} props.expanded - Whether the section is open. - * @param {Function} props.onToggle - Toggles the section. - * @param {JSX.Element} props.children - The section body. - * @returns {JSX.Element} The section. - */ -function ConfigSection({ title, summary, info, expanded, onToggle, children }) { - return ( - - - - - {title} - {info && ( - - - - )} - - {summary && ( - - {summary} - - )} - - - - - - - {children} - - - ); -} - -ConfigSection.propTypes = { - title: PropTypes.string.isRequired, - summary: PropTypes.string, - info: PropTypes.string, - expanded: PropTypes.bool.isRequired, - onToggle: PropTypes.func.isRequired, - children: PropTypes.node, -}; - /** * The single place a RAG session is configured. * @@ -125,8 +61,9 @@ ConfigSection.propTypes = { * @param {object} props * @param {number} props.sessionId - The RAG session being configured. * @param {object} [props.indexStatus] - Current indexing state, for the - * re-indexing warning. + * progress, re-indexing and failure notices. * @param {Function} [props.onSaved] - Called after parameters are persisted. + * @param {Function} [props.onRetryIndexing] - Called to restart a failed index. * @param {Function} [props.onSessionRenamed] - Called with the new name. * @returns {JSX.Element} The configuration panel. */ @@ -134,6 +71,7 @@ export default function RAGConfigPanel({ sessionId, indexStatus, onSaved, + onRetryIndexing, onSessionRenamed, }) { const { t } = useTranslation(["generative", "common"]); @@ -142,7 +80,8 @@ export default function RAGConfigPanel({ const [configuration, setConfiguration] = useState(null); const [loading, setLoading] = useState(true); const [saving, setSaving] = useState(false); - const [expanded, setExpanded] = useState({}); + const [activeSection, setActiveSection] = useState(SECTIONS[0]); + const [promptValid, setPromptValid] = useState(true); // Editable working copy of the parameters, seeded from the session. const [draft, setDraft] = useState(null); @@ -237,13 +176,20 @@ export default function RAGConfigPanel({ [draft, savedDraft], ); + // Which sections hold unsaved edits. One Save still sends the whole draft -- + // the endpoint replaces every parameter at once -- but with the sections + // behind tabs a pending change is otherwise invisible from another tab. + const dirtySections = useMemo(() => { + if (!draft || !savedDraft) return []; + return SECTIONS.filter( + (key) => JSON.stringify(draft[key]) !== JSON.stringify(savedDraft[key]), + ); + }, [draft, savedDraft]); + const updateSection = useCallback((key, value) => { setDraft((prev) => ({ ...prev, [key]: value })); }, []); - const toggleSection = (key) => - setExpanded((prev) => ({ ...prev, [key]: !prev[key] })); - const handleSave = async () => { if (!dirty || saving) return; setSaving(true); @@ -270,6 +216,8 @@ export default function RAGConfigPanel({ } }; + const handleDiscard = () => setDraft(savedDraft); + const handleSaveMetadata = async () => { const trimmed = name.trim(); if (!trimmed) { @@ -399,26 +347,18 @@ export default function RAGConfigPanel({ const sectionBody = (key) => { if (key === "chunking_model") { - const active = activePresetKey("chunking_model", chunkingPresets); return ( - - {chunkingPresets.map((preset) => ( - - updateSection("chunking_model", { - component: preset.component, - params: preset.params, - }) - } - /> - ))} - + + updateSection("chunking_model", { + component: preset.component, + params: preset.params, + }) + } + /> + ) + } + > + {indexStatus.job.error || t("generative:rag.index.indexFailed")} + + )} - + {/* Two by two rather than one scrolling row: the panel is 15-40% of + the viewport and the labels come from the backend, so four abreast + either wrap or hide half of themselves behind a scroll button. On + two rows all four sections are legible and reachable at once. + The indicator cannot follow a grid, so a selected tab is marked by + its own surface and underline instead. */} + setActiveSection(value)} + minHeight={36} + slotProps={{ + // The grid goes on the list slot itself rather than through a + // descendant selector, so it beats the row MUI lays out there. + list: { + sx: { + display: "grid", + gridTemplateColumns: "repeat(2, 1fr)", + gap: 0.5, + }, + }, + }} + sx={{ + p: 0.5, + "& .MuiTabs-indicator": { display: "none" }, + "& .MuiTab-root": { + minWidth: 0, + px: 1, + maxWidth: "none", + "&.Mui-selected": { + bgcolor: "background.paper", + fontWeight: 600, + borderBottom: 2, + borderColor: "primary.main", + }, + }, + }} + > {SECTIONS.map((key) => ( - toggleSection(key)} - > - {sectionBody(key)} - + value={key} + sx={ + key === "prompt" && !promptValid + ? { color: "error.main" } + : undefined + } + label={ + + {configuration[key].section_name} + {dirtySections.includes(key) && ( + + )} + + } + /> ))} + - {/* Context budget, computed by the backend from the live config. */} - - - {t("generative:rag.config.contextBudget")} - - - - {t("generative:validation.contextSpace", { - availableChars: budget.available.toLocaleString(), - })} - - {!budget.is_valid && ( - - {t("generative:validation.insufficientContextDescription")} - - )} - + + {/* Every section stays mounted, hidden rather than unrendered: the + generator reports whether its model is usable through a callback, + so a tab the user never opens would leave Save enabled for a + model that cannot run. */} + {SECTIONS.map((key) => ( + + ))} - + + {/* Context budget, computed by the backend from the live config. Every + section feeds into it, so it belongs beside Save rather than at the + end of one tab. */} + + + {t("generative:rag.config.contextBudget")} + + + + {t("generative:validation.contextSpace", { + availableChars: budget.available.toLocaleString(), + })} + + {!budget.is_valid && ( + + {t("generative:validation.insufficientContextDescription")} + + )} + + + {dirtySections.some((key) => key !== activeSection) && ( + + {t("generative:rag.config.unsavedIn", { + sections: dirtySections + .map((key) => configuration[key].section_name) + .join(", "), + })} + + )} + + + @@ -622,5 +689,6 @@ RAGConfigPanel.propTypes = { .isRequired, indexStatus: PropTypes.object, onSaved: PropTypes.func, + onRetryIndexing: PropTypes.func, onSessionRenamed: PropTypes.func, }; diff --git a/DashAI/front/src/components/generative/RAG/RAGConfigPanel.test.jsx b/DashAI/front/src/components/generative/RAG/RAGConfigPanel.test.jsx new file mode 100644 index 000000000..42bd74475 --- /dev/null +++ b/DashAI/front/src/components/generative/RAG/RAGConfigPanel.test.jsx @@ -0,0 +1,341 @@ +import React from "react"; +import { screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../test-utils/renderWithProviders"; +import RAGConfigPanel from "./RAGConfigPanel"; + +// One stable object: the panel's loader depends on `enqueueSnackbar`, so a +// fresh reference per render would restart the load on every render. +jest.mock("notistack", () => { + const snackbar = { enqueueSnackbar: jest.fn() }; + return { useSnackbar: () => snackbar }; +}); + +jest.mock("../../../api/rag", () => ({ + getChunkingPresets: jest.fn(), + getRAGSession: jest.fn(), + getRetrieverComponents: jest.fn(), + getRetrieverPresets: jest.fn(), + getSessionConfiguration: jest.fn(), + updateGenerativeSessionParams: jest.fn(), + getDefaultPrompts: jest.fn(), +})); +jest.mock("../../../api/session", () => ({ + updateGenerativeSession: jest.fn(), +})); +jest.mock("../../../pages/generative/RAGSession/retrieverKinds", () => ({ + loadRetrieverKinds: jest.fn(), +})); +// The generator picker fetches components and download state of its own; the +// panel only cares about the availability it reports back. +jest.mock("./GeneratorPicker", () => { + const MockGeneratorPicker = () =>
    ; + return MockGeneratorPicker; +}); +jest.mock( + "../../../pages/generative/RAGSession/advanced/ChunkingAdvancedModal", + () => { + const MockChunkingAdvancedModal = () => null; + return MockChunkingAdvancedModal; + }, +); +jest.mock( + "../../../pages/generative/RAGSession/advanced/RetrieverAdvancedModal", + () => { + const MockRetrieverAdvancedModal = () => null; + return MockRetrieverAdvancedModal; + }, +); + +const api = require("../../../api/rag"); +const { + loadRetrieverKinds, +} = require("../../../pages/generative/RAGSession/retrieverKinds"); + +const CHUNKING_PRESETS = [ + { + key: "balanced", + display_name: "Balanced", + description: "A middling chunk size", + component: "CharacterChunkModel", + params: { chunk_size: 400, chunk_overlap: 40 }, + }, + { + key: "fine", + display_name: "Fine", + description: "Smaller chunks", + component: "CharacterChunkModel", + params: { chunk_size: 200, chunk_overlap: 20 }, + }, +]; + +const RETRIEVER_PRESETS = [ + { + key: "keyword", + display_name: "Keyword", + description: "BM25", + component: "BM25Retriever", + params: { top_k: 5 }, + }, +]; + +const SESSION = { + id: 1, + name: "A session", + description: "", + parameters: { + documents: [], + chunking_model: CHUNKING_PRESETS[0], + retriever_model: RETRIEVER_PRESETS[0], + prompt: { + component: "CustomRAGGenerationPrompt", + params: { template: "Use {chunks} for {input}", language: "en" }, + }, + generation_model: { component: "StubLLM", params: {} }, + }, +}; + +/** The shape `getSessionConfiguration` returns, with backend-localized names. */ +const CONFIGURATION = { + chunking_model: { + section_name: "Chunking", + component: "CharacterChunkModel", + display_name: "Character", + description: "How documents are split", + registered: true, + params: [], + preset_key: "balanced", + preset_display_name: "Balanced", + summary: "400 characters", + }, + retriever_model: { + section_name: "Retrieval", + component: "BM25Retriever", + display_name: "BM25", + description: "How passages are found", + registered: true, + params: [], + preset_key: "keyword", + preset_display_name: "Keyword", + top_k: 5, + }, + generation_model: { + section_name: "Model", + component: "StubLLM", + display_name: "Stub", + description: "Which model answers", + registered: true, + params: [], + }, + prompt: { + section_name: "Prompt", + component: "CustomRAGGenerationPrompt", + display_name: "Custom", + description: "How the question is phrased", + registered: true, + params: [], + }, + context_budget: { + context_window: 4096, + available: 3000, + is_valid: true, + }, +}; + +beforeEach(() => { + jest.clearAllMocks(); + api.getRAGSession.mockResolvedValue(SESSION); + api.getSessionConfiguration.mockResolvedValue(CONFIGURATION); + api.getChunkingPresets.mockResolvedValue(CHUNKING_PRESETS); + api.getRetrieverPresets.mockResolvedValue(RETRIEVER_PRESETS); + api.getRetrieverComponents.mockResolvedValue([]); + api.updateGenerativeSessionParams.mockResolvedValue({}); + api.getDefaultPrompts.mockResolvedValue([]); + loadRetrieverKinds.mockResolvedValue({}); +}); + +/** Renders the panel and waits for its initial load. */ +async function renderPanel(props = {}) { + const onSaved = jest.fn(); + const onRetryIndexing = jest.fn(); + renderWithProviders( + , + ); + await screen.findByRole("tab", { name: /Chunking/ }); + return { onSaved, onRetryIndexing }; +} + +test("renders one tab per section, labelled by the backend", async () => { + await renderPanel(); + for (const label of ["Chunking", "Retrieval", "Model", "Prompt"]) { + expect( + screen.getByRole("tab", { name: new RegExp(label) }), + ).toBeInTheDocument(); + } +}); + +test("saving is offered only once something changed", async () => { + await renderPanel(); + const save = screen.getByRole("button", { name: /Save/i }); + expect(save).toBeDisabled(); + + await userEvent.click(screen.getByText("Fine")); + await waitFor(() => expect(save).toBeEnabled()); +}); + +test("an edit on a hidden tab is still reported and still saved", async () => { + await renderPanel(); + + // Change chunking, then move to a different tab: the change is now off + // screen, which is exactly when the panel has to keep saying it exists. + await userEvent.click(screen.getByText("Fine")); + await userEvent.click(screen.getByRole("tab", { name: /Model/ })); + + const chunkingTab = screen.getByRole("tab", { name: /Chunking/ }); + expect( + within(chunkingTab).getByTestId("FiberManualRecordIcon"), + ).toBeInTheDocument(); + + const save = screen.getByRole("button", { name: /Save/i }); + await waitFor(() => expect(save).toBeEnabled()); + await userEvent.click(save); + + // One request carries the whole draft: the endpoint replaces every + // parameter at once, so a per-tab save would be a lie. + await waitFor(() => + expect(api.updateGenerativeSessionParams).toHaveBeenCalledTimes(1), + ); + const [, sent] = api.updateGenerativeSessionParams.mock.calls[0]; + expect(sent.chunking_model.params.chunk_size).toBe(200); + expect(sent.retriever_model).toEqual(RETRIEVER_PRESETS[0]); + expect(sent.prompt.params.template).toBe("Use {chunks} for {input}"); +}); + +test("discarding puts every tab back to the saved configuration", async () => { + await renderPanel(); + + await userEvent.click(screen.getByText("Fine")); + const discard = screen.getByRole("button", { name: /Discard/i }); + await waitFor(() => expect(discard).toBeEnabled()); + + await userEvent.click(discard); + + await waitFor(() => + expect(screen.getByRole("button", { name: /Save/i })).toBeDisabled(), + ); + expect(api.updateGenerativeSessionParams).not.toHaveBeenCalled(); +}); + +test("the four sections sit two by two rather than in one scrolling row", async () => { + await renderPanel(); + + // The panel can be squeezed to 15% of the window and the labels come from the + // backend -- Spanish "Fragmentación"/"Recuperación" is the worst case at 37 + // characters across four tabs. Four abreast either wrap or hide half of + // themselves behind a scroll button; two rows of two keep all four legible. + const tablist = screen.getByRole("tablist"); + const style = getComputedStyle(tablist); + expect(style.display).toBe("grid"); + expect(style.gridTemplateColumns).toBe("repeat(2, 1fr)"); + + expect(screen.getAllByRole("tab")).toHaveLength(4); + // Not the scrollable variant: nothing is parked off screen. + expect(tablist.parentElement.className).not.toMatch(/scrollableX/); +}); + +test("every section is mounted, so a tab never opened still validates", async () => { + await renderPanel(); + // The generator reports whether its model can run through a callback. If it + // only mounted once its tab was opened, Save would stay enabled for a model + // that cannot answer. + expect(screen.getByTestId("generator-picker")).toBeInTheDocument(); +}); + +// ─── Indexing notices ────────────────────────────────────────────────── +// Indexing now runs up front rather than on the first message, so the panel +// is where its progress and its failures surface. + +test("a running index shows its progress, using the backend's wording", async () => { + await renderPanel({ + indexStatus: { + status: "indexing", + message: "Indexing the documents…", + job: { status: "started", progress: 40 }, + }, + }); + + // Scoped to the notice: the panel also draws a context-budget bar, and a + // bare progressbar query would not tell the two apart. + const notice = screen.getByRole("alert"); + expect( + within(notice).getByText("Indexing the documents…"), + ).toBeInTheDocument(); + expect(within(notice).getByRole("progressbar")).toHaveAttribute( + "aria-valuenow", + "40", + ); +}); + +test("an index with no progress yet shows an indeterminate bar", async () => { + // A determinate bar sitting at 0% reads as stalled rather than as starting. + await renderPanel({ + indexStatus: { + status: "indexing", + message: "Indexing the documents…", + job: { status: "not_started", progress: null }, + }, + }); + + const notice = screen.getByRole("alert"); + expect(within(notice).getByRole("progressbar")).not.toHaveAttribute( + "aria-valuenow", + ); +}); + +test("a stale index warns without claiming to be working", async () => { + await renderPanel({ + indexStatus: { + status: "stale", + message: "The configuration changed.", + job: null, + }, + }); + + const notice = screen.getByRole("alert"); + expect( + within(notice).getByText("The configuration changed."), + ).toBeInTheDocument(); + expect(within(notice).queryByRole("progressbar")).not.toBeInTheDocument(); +}); + +test("a failed index stays visible and offers a retry", async () => { + const { onRetryIndexing } = await renderPanel({ + indexStatus: { + status: "not_indexed", + message: "These documents are not indexed yet.", + job: { status: "error", error: "Out of memory" }, + }, + }); + + const notice = screen.getByRole("alert"); + expect(within(notice).getByText("Out of memory")).toBeInTheDocument(); + await userEvent.click(within(notice).getByRole("button", { name: /Retry/i })); + expect(onRetryIndexing).toHaveBeenCalledTimes(1); +}); + +test("a job that finished cleanly raises no alarm", async () => { + await renderPanel({ + indexStatus: { + status: "indexed", + message: "The documents are indexed.", + job: { status: "finished", progress: 100, error: null }, + }, + }); + + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); +}); diff --git a/DashAI/front/src/components/generative/RAG/RAGDocumentsPanel.jsx b/DashAI/front/src/components/generative/RAG/RAGDocumentsPanel.jsx deleted file mode 100644 index bcbeba19f..000000000 --- a/DashAI/front/src/components/generative/RAG/RAGDocumentsPanel.jsx +++ /dev/null @@ -1,39 +0,0 @@ -import PropTypes from "prop-types"; -import DocumentsBar from "./DocumentsBar"; -import { RAG_TASK_NAME } from "../../../api/rag"; - -/** - * Unified RAG Documents Panel wrapper. - * Handles consistent DocumentsBar rendering across all RAG contexts. - * - * @param {object} props - * @param {string} [props.selectedSessionId] - Session ID for session-specific documents. - * @param {object} [props.indexStatus] - Indexing state for the session, used - * to badge each document with its chunk count. - * @param {function} [props.onDocumentChange] - Optional callback for document changes. - * @param {boolean} [props.showSearch=false] - Whether to show the search bar. - * @returns {JSX.Element} The DocumentsBar component. - */ -export default function RAGDocumentsPanel({ - selectedSessionId, - indexStatus, - onDocumentChange, - showSearch = false, -}) { - return ( - - ); -} - -RAGDocumentsPanel.propTypes = { - selectedSessionId: PropTypes.oneOfType([PropTypes.string, PropTypes.number]), - indexStatus: PropTypes.object, - onDocumentChange: PropTypes.func, - showSearch: PropTypes.bool, -}; diff --git a/DashAI/front/src/components/generative/SessionBar.jsx b/DashAI/front/src/components/generative/SessionBar.jsx index 3aea32650..a697dba7b 100644 --- a/DashAI/front/src/components/generative/SessionBar.jsx +++ b/DashAI/front/src/components/generative/SessionBar.jsx @@ -1,15 +1,14 @@ -import { Box, Typography, Divider } from "@mui/material"; +import { Box, Divider } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import { useNavigate } from "react-router-dom"; import FolderIcon from "@mui/icons-material/Folder"; -import ViewModuleIcon from "@mui/icons-material/ViewModule"; import SearchBar from "../threeSectionLayout/SearchBar"; import { useEffect, useMemo, useState } from "react"; import InfoSessionModal from "./InfoSessionModal"; import GroupedCollapsibleList from "../threeSectionLayout/GroupedCollapsibleList"; import Footer from "../threeSectionLayout/Footer"; -import NewItemButton from "../threeSectionLayout/NewItemButton"; import SideBar from "../threeSectionLayout/panelContainers/SideBar"; +import GenerativeHubHeader from "./GenerativeHubHeader"; import { useTranslation } from "react-i18next"; import { useGenerative } from "./GenerativeContext"; import { standaloneRouteFor } from "./standaloneEntryPoints"; @@ -25,6 +24,9 @@ import { standaloneRouteFor } from "./standaloneEntryPoints"; * @param {Function} [props.handleNewSessionButton] - Overrides the new-session action. * @param {Function} [props.handleSessionDelete] - Overrides delete behaviour. * @param {boolean} [props.showSearch=true] - Whether to show the search field. + * @param {boolean} [props.showHeader=true] - Whether to render the hub header. + * A view that already puts the header above its own layout turns this off, so + * the row is not repeated part-way down the panel. * @param {string} [props.title] - Heading for the list. Defaults to the module * name; a view scoped to one task passes that task's display name. * @returns {JSX.Element} The session sidebar. @@ -37,6 +39,7 @@ export default function SessionBar({ handleNewSessionButton: handleNewSessionButtonProp, handleSessionDelete: handleSessionDeleteProp, showSearch = true, + showHeader = true, title, }) { const theme = useTheme(); @@ -89,10 +92,12 @@ export default function SessionBar({ const prevKeys = Object.keys(prev).sort().join(","); const newKeys = uniqueDisplayNames.slice().sort().join(","); if (prevKeys === newKeys) return prev; - // Preserve existing open/close state; initialize new keys as closed + // Preserve existing open/close state; initialize new keys as open, so + // every task's sessions -- the shared ones and a standalone task's, such + // as RAG -- are visible on arrival rather than behind a closed header. const merged = {}; uniqueDisplayNames.forEach((displayName) => { - merged[displayName] = displayName in prev ? prev[displayName] : false; + merged[displayName] = displayName in prev ? prev[displayName] : true; }); return merged; }); @@ -220,23 +225,12 @@ export default function SessionBar({ justifyContent={"flex-start"} minHeight={0} > - - {/* Create new session button */} - {selectedSessionId ? ( - - ) : ( - - {t("generative:label.generativeModule")} - - )} - + {showHeader && ( + + )} {/* Search Bar */} {showSearch && sessions.length > SEARCH_THRESHOLD && ( diff --git a/DashAI/front/src/components/generative/SessionBox.jsx b/DashAI/front/src/components/generative/SessionBox.jsx deleted file mode 100644 index f41caf844..000000000 --- a/DashAI/front/src/components/generative/SessionBox.jsx +++ /dev/null @@ -1,65 +0,0 @@ -import React from "react"; -import { Box, Typography } from "@mui/material"; -import SessionMenu from "./SessionMenu"; -const { useTranslation } = require("react-i18next"); - -export default function SessionBox({ - isSelected, - name, - modelName, - id, - onClick, - onDelete, - onInfo, -}) { - const { t } = useTranslation(["generative"]); - - return ( - - - - - {name ? name : t("generative:label.untitledSession")} - - - {modelName} - - - - - - ); -} diff --git a/DashAI/front/src/pages/generative/GenerativeContent.jsx b/DashAI/front/src/pages/generative/GenerativeContent.jsx index 7aee51179..3b919445f 100644 --- a/DashAI/front/src/pages/generative/GenerativeContent.jsx +++ b/DashAI/front/src/pages/generative/GenerativeContent.jsx @@ -30,6 +30,7 @@ export default function GenerativeContent() { setStepIndex, sessions, tasks, + fetchSessions, } = useGenerative(); const tourContext = useTourContext(); const { setDisabled } = tourContext ?? {}; @@ -39,6 +40,14 @@ export default function GenerativeContent() { "/app/generative/sessions/new", ); + // The module's list holds every task's sessions, but the app-level provider + // fetches it once at start-up. A task with its own entry point scopes a + // provider of its own, so a RAG session created (or deleted) there would be + // missing from this list until a reload — refresh on entering the module. + useEffect(() => { + fetchSessions?.(); + }, [fetchSessions]); + useEffect(() => { const path = location.pathname; diff --git a/DashAI/front/src/pages/generative/RAG/RAGCreatePage.jsx b/DashAI/front/src/pages/generative/RAG/RAGCreatePage.jsx index bbfdf0086..3287eb2ac 100644 --- a/DashAI/front/src/pages/generative/RAG/RAGCreatePage.jsx +++ b/DashAI/front/src/pages/generative/RAG/RAGCreatePage.jsx @@ -3,9 +3,7 @@ import { useNavigate } from "react-router-dom"; import { useTranslation } from "react-i18next"; import { useSnackbar } from "notistack"; import { - Alert, Box, - Chip, CircularProgress, Stack, TextField, @@ -17,7 +15,6 @@ import CenterPanel from "../../../components/threeSectionLayout/panels/CenterPan import SessionBar from "../../../components/generative/SessionBar"; import StepperNavigationFooter from "../../../components/shared/StepperNavigationFooter"; import ComponentSelector from "../../../components/custom/ComponentSelector"; -import DocumentSelector from "../../../components/generative/RAG/DocumentSelector"; import RAGBreadcrumbs from "../../../components/generative/RAG/RAGBreadcrumbs"; import { useCredentialStatuses, @@ -28,7 +25,6 @@ import { RAG_TASK_NAME, createRAGSession, getGeneratorComponents, - getSessionDefaults, } from "../../../api/rag"; import { useGenerative } from "../../../components/generative/GenerativeContext"; import { useTaskDisplayName } from "../../../hooks/generative/useTaskDisplayName"; @@ -39,7 +35,14 @@ import { generateSequentialName } from "../../../utils/nameGenerator"; import { getApiErrorMessage } from "../../../utils/apiError"; /** - * Minimal RAG session creation: a name, some documents, and a model. + * The RAG entry point: create a session from a name and a model. + * + * This is what `/app/generative/rag` renders, so arriving from the hub puts + * the cursor straight in the form. Existing sessions are listed on the left. + * + * Documents are uploaded into the session once it exists, so there is nothing + * to pick here; everything else the pipeline needs has a backend default the + * session view can change. * * Chunking, retrieval and the prompt template are filled in by the backend and * stay editable in the session view, so creating a session is three decisions @@ -57,11 +60,9 @@ export default function RAGCreatePage() { const [name, setName] = useState(""); const [isNameTouched, setIsNameTouched] = useState(false); - const [documentIds, setDocumentIds] = useState([]); const [models, setModels] = useState([]); const [loadingModels, setLoadingModels] = useState(true); const [selectedModel, setSelectedModel] = useState(null); - const [defaults, setDefaults] = useState(null); const [submitting, setSubmitting] = useState(false); const { statuses, loaded: credentialsLoaded } = useCredentialStatuses(); @@ -97,22 +98,6 @@ export default function RAGCreatePage() { }; }, [enqueueSnackbar, t]); - // The defaults preview comes from the same endpoint the backend applies on - // create, so what the user reads here is what the session will actually get. - useEffect(() => { - let cancelled = false; - getSessionDefaults() - .then((data) => { - if (!cancelled) setDefaults(data); - }) - .catch((error) => { - console.error("Failed to load session defaults:", error); - }); - return () => { - cancelled = true; - }; - }, []); - /** * Flip a model's downloaded flag in place after an inline download, so the * list updates without a refetch that would reset the scroll position. @@ -127,10 +112,6 @@ export default function RAGCreatePage() { ); }, []); - const handleDocumentSelectionChange = useCallback((selectedDocs) => { - setDocumentIds(selectedDocs.map((doc) => doc.id)); - }, []); - // Read from the live list so an inline download immediately ungates Create. const selectedModelState = useMemo( () => models.find((m) => m.name === selectedModel?.name) || selectedModel, @@ -152,7 +133,6 @@ export default function RAGCreatePage() { const canCreate = Boolean(name.trim()) && - documentIds.length > 0 && Boolean(selectedModel) && !modelUnavailable && !submitting; @@ -167,7 +147,6 @@ export default function RAGCreatePage() { task_name: RAG_TASK_NAME, model_name: RAG_MODEL_NAME, parameters: { - documents: documentIds, generation_model: { component: selectedModel.name, params: {} }, }, }); @@ -187,14 +166,6 @@ export default function RAGCreatePage() { } }; - const defaultsSummary = defaults - ? [ - defaults.chunking_model?.display_name, - defaults.retriever_model?.display_name, - defaults.prompt?.display_name, - ].filter(Boolean) - : []; - return ( @@ -205,7 +176,7 @@ export default function RAGCreatePage() { handleSessionClick={(sessionId) => navigate(`/app/generative/rag/sessions/${sessionId}`) } - handleNewSessionButton={() => navigate("/app/generative/rag")} + handleNewSessionButton={() => navigate("/app/generative")} handleSessionDelete={deleteSessionById} onToggle={threePanelLayout.handleToggleLeft} showSearch={false} @@ -233,7 +204,18 @@ export default function RAGCreatePage() { - + {/* pt leaves room for the name field's floating label: it sits + above the input's border, and a scroll container that starts + flush with it clips the label against the subtitle. */} + - - - {t("generative:rag.setup.selectDocuments")} - - - - {t("generative:rag.create.selectModel")} @@ -277,33 +249,16 @@ export default function RAGCreatePage() { onSelect={setSelectedModel} onDownloadChange={handleDownloadChange} flat + showFooter={false} searchPlaceholder={t("generative:label.searchModels")} /> )} - - {defaultsSummary.length > 0 && ( - - - {t("generative:rag.create.defaultsNotice")} - - - {defaultsSummary.map((label) => ( - - ))} - - - )} navigate("/app/generative/rag")} + onBack={() => navigate("/app/generative")} onNext={handleCreate} backDisabled={submitting} nextDisabled={!canCreate} diff --git a/DashAI/front/src/pages/generative/RAG/RAGCreatePage.test.jsx b/DashAI/front/src/pages/generative/RAG/RAGCreatePage.test.jsx new file mode 100644 index 000000000..13326ae24 --- /dev/null +++ b/DashAI/front/src/pages/generative/RAG/RAGCreatePage.test.jsx @@ -0,0 +1,132 @@ +import React from "react"; +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../test-utils/renderWithProviders"; +import RAGCreatePage from "./RAGCreatePage"; + +// One stable object: an effect depends on `enqueueSnackbar`, so a fresh +// reference per render would restart it on every render. +jest.mock("notistack", () => { + const snackbar = { enqueueSnackbar: jest.fn() }; + return { useSnackbar: () => snackbar }; +}); + +jest.mock("../../../api/rag", () => ({ + RAG_TASK_NAME: "RAGTask", + RAG_MODEL_NAME: "RAGPipeline", + createRAGSession: jest.fn(), + getGeneratorComponents: jest.fn(), +})); + +// Jest only lets a mock factory close over `mock`-prefixed names. +const mockGenerative = { + sessions: [], + setSessions: jest.fn(), + deleteSessionById: jest.fn(), + tasks: [], + taskDisplayNameMap: {}, +}; +jest.mock("../../../components/generative/GenerativeContext", () => ({ + useGenerative: () => mockGenerative, +})); + +jest.mock("../../../components/credentials/credentialStatus", () => ({ + useCredentialStatuses: () => ({ statuses: {}, loaded: true }), + // Same shape the real helper returns; the card reads the lists directly. + getComponentCredentialState: () => ({ + requiredCredentials: [], + optionalCredentials: [], + credentialsSatisfied: true, + locked: false, + requiredPlatforms: "", + optionalPlatforms: "", + }), +})); + +jest.mock("../../../hooks/generative/useTaskDisplayName", () => ({ + useTaskDisplayName: () => "RAG", +})); + +// The session list is not what this page is being tested for, and it pulls in +// the whole generative context to render. +jest.mock("../../../components/generative/SessionBar", () => { + const MockSessionBar = () =>
    ; + return MockSessionBar; +}); + +const api = require("../../../api/rag"); + +const MODELS = [ + { + name: "StubLLM", + display_name: "Stub LLM", + description: "A model that needs nothing", + metadata: { requires_download: false }, + downloaded: true, + }, + { + name: "OtherLLM", + display_name: "Other LLM", + description: "Another one", + metadata: { requires_download: false }, + downloaded: true, + }, +]; + +beforeEach(() => { + jest.clearAllMocks(); + api.getGeneratorComponents.mockResolvedValue(MODELS); + api.createRAGSession.mockResolvedValue({ id: 7 }); +}); + +test("loads the models instead of spinning forever", async () => { + renderWithProviders(); + + // A regression guard: the effect that fetches the models was once deleted by + // accident, leaving `loadingModels` true and the page on a permanent spinner. + await waitFor(() => expect(api.getGeneratorComponents).toHaveBeenCalled()); + expect(await screen.findByText("Stub LLM")).toBeInTheDocument(); + expect(screen.getByText("Other LLM")).toBeInTheDocument(); + expect(screen.queryByRole("progressbar")).not.toBeInTheDocument(); +}); + +test("does not tack a component count onto the form", async () => { + renderWithProviders(); + await screen.findByText("Stub LLM"); + + // The picker's footer counts the options it is showing, which reads as + // clutter on a two-field form. + expect(screen.queryByText(/components available/i)).not.toBeInTheDocument(); + expect(screen.queryByText(/componentsAvailable/)).not.toBeInTheDocument(); +}); + +test("suggests a session name so the form is ready to submit", async () => { + renderWithProviders(); + await screen.findByText("Stub LLM"); + + const named = screen + .getAllByRole("textbox") + .some((input) => /RAG_Session/.test(input.value)); + expect(named).toBe(true); +}); + +test("a name and a model are enough to create, and no documents are sent", async () => { + renderWithProviders(); + await screen.findByText("Stub LLM"); + + const create = screen.getByRole("button", { name: /create/i }); + expect(create).toBeDisabled(); + + await userEvent.click(screen.getByText("Stub LLM")); + await waitFor(() => expect(create).toBeEnabled()); + + await userEvent.click(create); + + await waitFor(() => expect(api.createRAGSession).toHaveBeenCalledTimes(1)); + const payload = api.createRAGSession.mock.calls[0][0]; + expect(payload.task_name).toBe("RAGTask"); + expect(payload.parameters.generation_model.component).toBe("StubLLM"); + // Documents are uploaded into the session afterwards; sending them here is + // rejected by the backend. + expect(payload.parameters).not.toHaveProperty("documents"); +}); diff --git a/DashAI/front/src/pages/generative/RAG/RAGDocumentsPage.jsx b/DashAI/front/src/pages/generative/RAG/RAGDocumentsPage.jsx deleted file mode 100644 index 1966c6740..000000000 --- a/DashAI/front/src/pages/generative/RAG/RAGDocumentsPage.jsx +++ /dev/null @@ -1,268 +0,0 @@ -import { useState, useEffect, useCallback } from "react"; -import { useNavigate } from "react-router-dom"; -import { useTranslation } from "react-i18next"; -import { Typography, Box, Paper, CircularProgress, Alert } from "@mui/material"; -import ModuleContainer from "../../../components/layout/ModuleContainer"; -import LeftPanel from "../../../components/threeSectionLayout/panels/LeftPanel"; -import CenterPanel from "../../../components/threeSectionLayout/panels/CenterPanel"; -import RightPanel from "../../../components/threeSectionLayout/panels/RightPanel"; -import SessionBar from "../../../components/generative/SessionBar"; -import RAGBreadcrumbs from "../../../components/generative/RAG/RAGBreadcrumbs"; -import DocumentTable from "../../../components/generative/RAG/DocumentTable"; -import { - loadDocuments, - deleteDocument, - extractDocumentText, -} from "../../../api/rag"; -import { getSessions, removeSession } from "../../../api/session"; -import { useThreePanelLayout } from "../../../hooks/useThreePanelsLayout"; -import { ThreePanelLayoutContext } from "../../../components/threeSectionLayout/panels/ThreePanelLayoutContext"; -import { FormSchemaProvider } from "../../../contexts/schema"; - -/** - * RAG documents management page. - * Shows a document table in the center, sessions in the left panel, - * and allows document upload / deletion. - * @returns {JSX.Element} Three-panel documents page. - */ -function RAGDocumentsPage() { - const navigate = useNavigate(); - const threePanelLayout = useThreePanelLayout(); - const { t } = useTranslation(["generative"]); - const [allDocuments, setAllDocuments] = useState([]); - const [documentsLoading, setDocumentsLoading] = useState(true); - const [sessions, setSessions] = useState([]); - const [selectedDocument, setSelectedDocument] = useState(null); - const [selectedContent, setSelectedContent] = useState(""); - const [contentLoading, setContentLoading] = useState(false); - const [contentError, setContentError] = useState(""); - - /** - * Fetch all sessions from the API. - */ - const loadSessions = useCallback(async () => { - try { - const allSessions = await getSessions(); - setSessions(allSessions); - } catch (error) { - console.error("RAGDocumentsPage: Error loading sessions:", error); - } - }, []); - - /** - * Load all RAG documents from the API. - */ - const fetchAllDocuments = useCallback(async () => { - setDocumentsLoading(true); - try { - const docs = await loadDocuments(); - setAllDocuments(docs); - } catch (error) { - console.error("RAGDocumentsPage: Error loading all documents:", error); - } finally { - setDocumentsLoading(false); - } - }, []); - - useEffect(() => { - loadSessions(); - fetchAllDocuments(); - }, [loadSessions, fetchAllDocuments]); - - /** - * Fetch extracted content when a document is selected. - * Uses the document's stored extractor configuration. - */ - useEffect(() => { - if (!selectedDocument) { - setSelectedContent(""); - setContentError(""); - return; - } - const extractorRef = selectedDocument.extractor?.component - ? { - component: selectedDocument.extractor.component, - params: selectedDocument.extractor.params || {}, - } - : null; - if (!extractorRef) { - setSelectedContent(""); - setContentError(""); - return; - } - let cancelled = false; - setContentLoading(true); - setContentError(""); - setSelectedContent(""); - extractDocumentText(Number(selectedDocument.id), extractorRef, false) // Preview mode - .then((result) => { - if (!cancelled) setSelectedContent(result.text); - }) - .catch((e) => { - if (!cancelled) setContentError(e.message || "Extraction failed"); - }) - .finally(() => { - if (!cancelled) setContentLoading(false); - }); - return () => { - cancelled = true; - }; - }, [selectedDocument]); - - /** - * Delete a document via the API and refresh the list. - * @param {number} id - Document id to delete. - */ - const handleRemoveDocumentFromTable = useCallback( - async (id) => { - try { - await deleteDocument(id); - await fetchAllDocuments(); - } catch (error) { - console.error("RAGDocumentsPage: Failed to delete document:", error); - } - }, - [fetchAllDocuments], - ); - - /** - * Prepend a newly uploaded document to the local list. - * @param {object} newDoc - The document returned by the upload API. - */ - const handleAddDocument = useCallback((newDoc) => { - setAllDocuments((prev) => [newDoc, ...prev]); - }, []); - - /** - * Navigate to the main generative page with the selected session pre-selected. - * @param {number} sessionId - * @param {string} taskName - * @param {string} taskDisplayName - */ - const handleSessionClick = (sessionId, taskName, taskDisplayName) => { - navigate("/app/generative", { - state: { - selectedSessionId: sessionId, - selectedTaskName: taskName, - selectedDisplayName: taskDisplayName, - }, - }); - }; - - const handleNewSessionButton = () => { - navigate("/app/generative"); - }; - - /** - * Remove a session optimistically and persist the deletion. - * @param {number} id - Session id to delete. - */ - const handleSessionDelete = async (id) => { - setSessions((prev) => prev.filter((s) => s.id !== id)); - await removeSession(id); - }; - - return ( - - - - - - - - - - - {t("generative:ragDocumentsPage.title")} - - - {t("generative:ragDocumentsPage.description")} - - - new Date(b.created || b.createdAt || 0) - - new Date(a.created || a.createdAt || 0), - ) - .map((doc) => ({ - ...doc, - id: String(doc.id), - name: doc.file_name, - createdAt: doc.created || doc.createdAt || "", - preview: doc.preview_url, - file_type: doc.file_name - ? doc.file_name.split(".").pop().toLowerCase() - : "", - }))} - onRemove={handleRemoveDocumentFromTable} - onAddDocument={handleAddDocument} - onSelectDocument={setSelectedDocument} - onExtractorChanged={fetchAllDocuments} - isLoading={documentsLoading} - showTableTitle={false} - /> - - - - - {!selectedDocument ? ( - - - {t( - "generative:ragDocumentsPage.contentPanel.noDocumentSelected", - )} - - - ) : contentLoading ? ( - - - - {t( - "generative:ragDocumentsPage.contentPanel.loadingContent", - )} - - - ) : contentError ? ( - {contentError} - ) : ( - - {selectedContent || - t("generative:ragDocumentsPage.contentPanel.noContent")} - - )} - - - - - - ); -} - -export default RAGDocumentsPage; diff --git a/DashAI/front/src/pages/generative/RAG/RAGHomePage.jsx b/DashAI/front/src/pages/generative/RAG/RAGHomePage.jsx deleted file mode 100644 index 51b840eef..000000000 --- a/DashAI/front/src/pages/generative/RAG/RAGHomePage.jsx +++ /dev/null @@ -1,104 +0,0 @@ -import { useCallback } from "react"; -import { useNavigate } from "react-router-dom"; -import { useTranslation } from "react-i18next"; -import ChatIcon from "@mui/icons-material/Chat"; -import DescriptionIcon from "@mui/icons-material/Description"; -import EditNoteIcon from "@mui/icons-material/EditNote"; -import ModuleContainer from "../../../components/layout/ModuleContainer"; -import LeftPanel from "../../../components/threeSectionLayout/panels/LeftPanel"; -import CenterPanel from "../../../components/threeSectionLayout/panels/CenterPanel"; -import SelectOptionMenu from "../../../components/threeSectionLayout/SelectOptionMenu"; -import SessionBar from "../../../components/generative/SessionBar"; -import RAGBreadcrumbs from "../../../components/generative/RAG/RAGBreadcrumbs"; -import { RAG_TASK_NAME } from "../../../api/rag"; -import { useGenerative } from "../../../components/generative/GenerativeContext"; -import { useTaskDisplayName } from "../../../hooks/generative/useTaskDisplayName"; -import { useThreePanelLayout } from "../../../hooks/useThreePanelsLayout"; -import { ThreePanelLayoutContext } from "../../../components/threeSectionLayout/panels/ThreePanelLayoutContext"; - -const NEW_SESSION = "new_session"; -const DOCUMENTS = "documents"; -const PROMPTS = "prompts"; - -const ROUTES = { - [NEW_SESSION]: "/app/generative/rag/new", - [DOCUMENTS]: "/app/generative/rag/documents", - [PROMPTS]: "/app/generative/rag/prompts", -}; - -/** - * Home of the RAG entry point. - * - * Lists the RAG sessions on the left and the three things you can do from here - * in the centre. Sessions are scoped to RAG by the provider that wraps this - * route, so the shared generative session list stays separate. - * - * @returns {JSX.Element} The RAG home page. - */ -export default function RAGHomePage() { - const navigate = useNavigate(); - const { t } = useTranslation(["generative"]); - const threePanelLayout = useThreePanelLayout({ storageKey: "rag" }); - const { sessions, deleteSessionById } = useGenerative(); - const ragTitle = useTaskDisplayName(RAG_TASK_NAME); - - const handleOption = useCallback( - (name) => navigate(ROUTES[name] ?? ROUTES[NEW_SESSION]), - [navigate], - ); - - const handleSessionClick = useCallback( - (sessionId) => navigate(`/app/generative/rag/sessions/${sessionId}`), - [navigate], - ); - - const options = [ - { - name: NEW_SESSION, - display_name: t("generative:rag.home.newSession"), - description: t("generative:rag.home.newSessionDescription"), - Icon: ChatIcon, - }, - { - name: DOCUMENTS, - display_name: t("generative:rag.home.documents"), - description: t("generative:rag.home.documentsDescription"), - Icon: DescriptionIcon, - }, - { - name: PROMPTS, - display_name: t("generative:rag.home.prompts"), - description: t("generative:rag.home.promptsDescription"), - Icon: EditNoteIcon, - }, - ]; - - return ( - - - - handleOption(NEW_SESSION)} - handleSessionDelete={deleteSessionById} - onToggle={threePanelLayout.handleToggleLeft} - showSearch={false} - title={ragTitle} - /> - - - - - - - - - ); -} diff --git a/DashAI/front/src/pages/generative/RAG/RAGPromptsPage.jsx b/DashAI/front/src/pages/generative/RAG/RAGPromptsPage.jsx deleted file mode 100644 index 2b161d582..000000000 --- a/DashAI/front/src/pages/generative/RAG/RAGPromptsPage.jsx +++ /dev/null @@ -1,143 +0,0 @@ -import { useState, useEffect, useCallback } from "react"; -import { useNavigate } from "react-router-dom"; -import { useTranslation } from "react-i18next"; -import { Box, Typography } from "@mui/material"; -import ModuleContainer from "../../../components/layout/ModuleContainer"; -import LeftPanel from "../../../components/threeSectionLayout/panels/LeftPanel"; -import CenterPanel from "../../../components/threeSectionLayout/panels/CenterPanel"; -import RightPanel from "../../../components/threeSectionLayout/panels/RightPanel"; -import SessionBar from "../../../components/generative/SessionBar"; -import RAGBreadcrumbs from "../../../components/generative/RAG/RAGBreadcrumbs"; -import RAGDocumentsPanel from "../../../components/generative/RAG/RAGDocumentsPanel"; -import PromptSelectionTable from "../../../components/generative/RAG/PromptSelectionTable"; -import { getSessions, removeSession } from "../../../api/session"; -import { useThreePanelLayout } from "../../../hooks/useThreePanelsLayout"; -import { ThreePanelLayoutContext } from "../../../components/threeSectionLayout/panels/ThreePanelLayoutContext"; -import { FormSchemaProvider } from "../../../contexts/schema"; - -/** - * RAG prompts management page. - * Displays a prompt selection table in the center, sessions in the left panel, - * and a document panel on the right. - * @returns {JSX.Element} Three-panel prompts page. - */ -function RAGPromptsPage() { - const navigate = useNavigate(); - const threePanelLayout = useThreePanelLayout(); - const { t } = useTranslation(["generative"]); - const [rowSelectionModel, setRowSelectionModel] = useState([]); - const [sessions, setSessions] = useState([]); - const [documentRefreshTrigger, setDocumentRefreshTrigger] = useState(0); - - /** - * Fetch all sessions from the API. - */ - const loadSessions = useCallback(async () => { - try { - const allSessions = await getSessions(); - setSessions(allSessions); - } catch (error) { - console.error("RAGPromptsPage: Error loading sessions:", error); - } - }, []); - - useEffect(() => { - loadSessions(); - }, [loadSessions]); - - /** - * Navigate to the main generative page with the selected session pre-selected. - * @param {number} sessionId - * @param {string} taskName - * @param {string} taskDisplayName - */ - const handleSessionClick = (sessionId, taskName, taskDisplayName) => { - navigate("/app/generative", { - state: { - selectedSessionId: sessionId, - selectedTaskName: taskName, - selectedDisplayName: taskDisplayName, - }, - }); - }; - - const handleNewSessionButton = () => { - navigate("/app/generative"); - }; - - /** - * Remove a session optimistically and persist the deletion. - * @param {number} id - Session id to delete. - */ - const handleSessionDelete = async (id) => { - setSessions((prev) => prev.filter((s) => s.id !== id)); - await removeSession(id); - }; - - const handleRowSelectionModelChange = (newSelection) => { - setRowSelectionModel(newSelection); - }; - - const handleDocumentChange = () => { - setDocumentRefreshTrigger((prev) => prev + 1); - }; - - return ( - - - - - - - - - - - {t("generative:ragPromptsPage.title")} - - - {t("generative:ragPromptsPage.description")} - - - - - - - - - - - - - ); -} - -export default RAGPromptsPage; diff --git a/DashAI/front/src/pages/generative/RAGSession/RAGSessionPage.jsx b/DashAI/front/src/pages/generative/RAGSession/RAGSessionPage.jsx index 777c3475f..416af5148 100644 --- a/DashAI/front/src/pages/generative/RAGSession/RAGSessionPage.jsx +++ b/DashAI/front/src/pages/generative/RAGSession/RAGSessionPage.jsx @@ -1,17 +1,19 @@ import { useCallback, useEffect, useState } from "react"; import { useNavigate, useParams } from "react-router-dom"; import { useTranslation } from "react-i18next"; -import { Box, CircularProgress, Typography } from "@mui/material"; +import { Box, CircularProgress, Divider, Typography } from "@mui/material"; import ModuleContainer from "../../../components/layout/ModuleContainer"; import LeftPanel from "../../../components/threeSectionLayout/panels/LeftPanel"; import CenterPanel from "../../../components/threeSectionLayout/panels/CenterPanel"; import RightPanel from "../../../components/threeSectionLayout/panels/RightPanel"; import SessionBar from "../../../components/generative/SessionBar"; +import GenerativeHubHeader from "../../../components/generative/GenerativeHubHeader"; import GenerativeChat from "../../../components/generative/GenerativeChat"; -import RAGDocumentsPanel from "../../../components/generative/RAG/RAGDocumentsPanel"; +import DocumentsBar from "../../../components/generative/RAG/DocumentsBar"; +import RAGBreadcrumbs from "../../../components/generative/RAG/RAGBreadcrumbs"; import RAGConfigPanel from "../../../components/generative/RAG/RAGConfigPanel"; import { getGenerativeSession } from "../../../api/generativeTask"; -import { getSessionIndexStatus } from "../../../api/rag"; +import { getSessionIndexStatus, startSessionIndexing } from "../../../api/rag"; import { RAG_TASK_NAME } from "../../../api/rag"; import { useGenerative } from "../../../components/generative/GenerativeContext"; import { useTaskDisplayName } from "../../../hooks/generative/useTaskDisplayName"; @@ -49,6 +51,7 @@ export default function RAGSessionPage() { const [notFound, setNotFound] = useState(false); const [indexStatus, setIndexStatus] = useState(null); + const [sessionName, setSessionName] = useState(null); const sessionId = Number(urlSessionId); const isValidId = Number.isFinite(sessionId) && sessionId > 0; @@ -65,6 +68,7 @@ export default function RAGSessionPage() { getGenerativeSession(sessionId) .then((session) => { if (cancelled || !session) return; + setSessionName(session.name ?? null); setSelectedSessionId?.(sessionId); setSelectedTaskName?.(session.task_name); setSelectedDisplayName?.(session.display_name ?? null); @@ -96,6 +100,30 @@ export default function RAGSessionPage() { refreshIndexStatus(); }, [refreshIndexStatus]); + // Indexing runs up front rather than on the first message, so every change + // that could invalidate it ends here. The backend decides whether there is + // actually anything to do, and answers with the resulting status. + const startIndexing = useCallback(() => { + if (!isValidId) return; + startSessionIndexing(sessionId) + .then(setIndexStatus) + .catch((error) => { + console.error("Failed to start RAG indexing:", error); + // Fall back to reading the status: the user still needs to see where + // the session stands, even if starting the run failed. + refreshIndexStatus(); + }); + }, [sessionId, isValidId, refreshIndexStatus]); + + // Poll only while a job is actually running, so this stops on its own. The + // status is recomputed against the queue on every call, which is what lets + // it recover from a job that was cancelled or killed. + useEffect(() => { + if (indexStatus?.status !== "indexing") return undefined; + const interval = setInterval(refreshIndexStatus, 1500); + return () => clearInterval(interval); + }, [indexStatus?.status, refreshIndexStatus]); + const handleSessionClick = useCallback( (clickedId) => navigate(`/app/generative/rag/sessions/${clickedId}`), [navigate], @@ -109,9 +137,14 @@ export default function RAGSessionPage() { [deleteSessionById, navigate, sessionId], ); - const handleSessionRenamed = useCallback(() => { - fetchSessions?.(); - }, [fetchSessions]); + const handleSessionRenamed = useCallback( + (newName) => { + // Keep the breadcrumb in step without waiting for a reload. + if (newName) setSessionName(newName); + fetchSessions?.(); + }, + [fetchSessions], + ); if (notFound) { return ( @@ -151,27 +184,38 @@ export default function RAGSessionPage() { display: "flex", flexDirection: "column", height: "100%", - gap: 1, + minHeight: 0, + bgcolor: "background.box", }} > - - navigate("/app/generative")} + /> + + {/* Both halves may shrink, and each scrolls its own content: + a fixed basis with an outer scroll pushed the header out of + view as the lists grew. */} + + - + + - navigate("/app/generative/rag/new") - } handleSessionDelete={handleSessionDelete} onToggle={threePanelLayout.handleToggleLeft} showSearch={false} + showHeader={false} title={ragTitle} /> @@ -179,14 +223,32 @@ export default function RAGSessionPage() { - + + {/* Page chrome, level with the other RAG tabs. It used to be + rendered by the chat, which sits lower and is shared with + every other generative task. */} + + + + + + + diff --git a/DashAI/front/src/pages/generative/RAGSession/RAGSessionPage.test.jsx b/DashAI/front/src/pages/generative/RAGSession/RAGSessionPage.test.jsx new file mode 100644 index 000000000..81bce54ab --- /dev/null +++ b/DashAI/front/src/pages/generative/RAGSession/RAGSessionPage.test.jsx @@ -0,0 +1,217 @@ +import React from "react"; +import { screen, waitFor, act } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../test-utils/renderWithProviders"; +import RAGSessionPage from "./RAGSessionPage"; + +// Indexing runs up front now, so this page owns when it starts and how long it +// keeps watching. Everything else on the page is mocked away: what is under +// test is that wiring, not the three-panel chrome. + +jest.mock("react-router-dom", () => ({ + ...jest.requireActual("react-router-dom"), + useParams: () => ({ id: "7" }), + useNavigate: () => jest.fn(), +})); + +jest.mock("../../../api/rag", () => ({ + RAG_TASK_NAME: "RAGTask", + getSessionIndexStatus: jest.fn(), + startSessionIndexing: jest.fn(), +})); + +jest.mock("../../../api/generativeTask", () => ({ + getGenerativeSession: jest.fn(), +})); + +jest.mock("../../../components/generative/GenerativeContext", () => ({ + useGenerative: () => ({ + sessions: [], + selectedSessionId: 7, + setSelectedSessionId: jest.fn(), + setSelectedTaskName: jest.fn(), + setSelectedDisplayName: jest.fn(), + deleteSessionById: jest.fn(), + fetchSessions: jest.fn(), + }), +})); + +jest.mock("../../../hooks/generative/useTaskDisplayName", () => ({ + useTaskDisplayName: () => "RAG", +})); + +// The documents bar is the trigger under test: it reports every change that +// could invalidate the index through one callback. +jest.mock("../../../components/generative/RAG/DocumentsBar", () => { + const MockDocumentsBar = ({ onDocumentChange, indexStatus }) => ( +
    + + {indexStatus?.status ?? "none"} +
    + ); + return MockDocumentsBar; +}); + +jest.mock("../../../components/generative/RAG/RAGConfigPanel", () => { + const MockConfigPanel = ({ onSaved }) => ( + + ); + return MockConfigPanel; +}); + +jest.mock("../../../components/generative/GenerativeChat", () => { + const MockChat = ({ indexStatus }) => ( +
    {indexStatus?.status ?? "none"}
    + ); + return MockChat; +}); + +jest.mock("../../../components/generative/SessionBar", () => { + const MockSessionBar = () => null; + return MockSessionBar; +}); + +jest.mock("../../../components/generative/RAG/RAGBreadcrumbs", () => { + const MockBreadcrumbs = () => null; + return MockBreadcrumbs; +}); + +const api = require("../../../api/rag"); +const { getGenerativeSession } = require("../../../api/generativeTask"); + +const status = (value) => ({ + status: value, + chunk_set_id: null, + total_chunks: 0, + retriever_ready: false, + documents: [], + message: `state: ${value}`, + job_id: null, + job: null, +}); + +beforeEach(() => { + jest.clearAllMocks(); + getGenerativeSession.mockResolvedValue({ id: 7, name: "A session" }); + api.getSessionIndexStatus.mockResolvedValue(status("not_indexed")); + api.startSessionIndexing.mockResolvedValue(status("indexing")); +}); + +afterEach(() => { + jest.useRealTimers(); +}); + +async function renderPage() { + renderWithProviders(); + await waitFor(() => expect(api.getSessionIndexStatus).toHaveBeenCalled()); +} + +test("reads the index status on open without starting a run", async () => { + await renderPage(); + // Opening a session must not enqueue work; the read is safe to repeat. + expect(api.startSessionIndexing).not.toHaveBeenCalled(); +}); + +test("a document change starts indexing exactly once", async () => { + await renderPage(); + + await userEvent.click(screen.getByText("documents-changed")); + + // The documents bar batches a whole upload behind one callback. Starting a + // run per file would rewrite the document set each time, and each rewrite + // re-chunks and re-embeds every document. + await waitFor(() => + expect(api.startSessionIndexing).toHaveBeenCalledTimes(1), + ); + expect(api.startSessionIndexing).toHaveBeenCalledWith(7); +}); + +test("saving the configuration starts indexing too", async () => { + await renderPage(); + + await userEvent.click(screen.getByText("config-saved")); + + // Called unconditionally: the backend owns the rule for which settings + // invalidate the index, and short-circuits when there is nothing to do. + await waitFor(() => + expect(api.startSessionIndexing).toHaveBeenCalledTimes(1), + ); +}); + +test("the started run's status is shown without a further fetch", async () => { + await renderPage(); + const readsBefore = api.getSessionIndexStatus.mock.calls.length; + + await userEvent.click(screen.getByText("documents-changed")); + + await waitFor(() => + expect(screen.getByTestId("bar-status")).toHaveTextContent("indexing"), + ); + // The POST answers with the resulting status, so no follow-up GET is needed + // to render it. + expect(api.getSessionIndexStatus.mock.calls.length).toBe(readsBefore); +}); + +test("a failure to start still leaves the user with a status", async () => { + api.startSessionIndexing.mockRejectedValue(new Error("boom")); + jest.spyOn(console, "error").mockImplementation(() => {}); + await renderPage(); + const readsBefore = api.getSessionIndexStatus.mock.calls.length; + + await userEvent.click(screen.getByText("documents-changed")); + + await waitFor(() => + expect(api.getSessionIndexStatus.mock.calls.length).toBeGreaterThan( + readsBefore, + ), + ); + console.error.mockRestore(); +}); + +// Fake timers have to be installed before the page mounts, or the interval is +// scheduled against the real clock and never advances. +async function renderPageWithFakeTimers() { + jest.useFakeTimers(); + renderWithProviders(); + // Flush the mount effects and their promises without a timer-based waitFor, + // which would deadlock against the fake clock. + await act(async () => {}); +} + +test("polls while indexing and stops as soon as it is done", async () => { + api.getSessionIndexStatus.mockResolvedValue(status("indexing")); + await renderPageWithFakeTimers(); + expect(screen.getByTestId("chat-status")).toHaveTextContent("indexing"); + + const during = api.getSessionIndexStatus.mock.calls.length; + await act(async () => { + jest.advanceTimersByTime(1600); + }); + expect(api.getSessionIndexStatus.mock.calls.length).toBeGreaterThan(during); + + // Once the status leaves "indexing" the interval must clear itself, rather + // than polling a finished job forever. + api.getSessionIndexStatus.mockResolvedValue(status("indexed")); + await act(async () => { + jest.advanceTimersByTime(1600); + }); + expect(screen.getByTestId("chat-status")).toHaveTextContent("indexed"); + + const settled = api.getSessionIndexStatus.mock.calls.length; + await act(async () => { + jest.advanceTimersByTime(10000); + }); + expect(api.getSessionIndexStatus.mock.calls.length).toBe(settled); +}); + +test("does not poll while the session is idle", async () => { + // Default mock is "not_indexed": nothing is running, so nothing is watched. + await renderPageWithFakeTimers(); + const before = api.getSessionIndexStatus.mock.calls.length; + + await act(async () => { + jest.advanceTimersByTime(10000); + }); + + expect(api.getSessionIndexStatus.mock.calls.length).toBe(before); +}); diff --git a/DashAI/front/src/pages/generative/RAGSession/advanced/NewPromptModal.jsx b/DashAI/front/src/pages/generative/RAGSession/advanced/NewPromptModal.jsx deleted file mode 100644 index 040d53422..000000000 --- a/DashAI/front/src/pages/generative/RAGSession/advanced/NewPromptModal.jsx +++ /dev/null @@ -1,285 +0,0 @@ -import React, { useState, useCallback, useEffect, useRef } from "react"; -import CloseIcon from "@mui/icons-material/Close"; -import { - Dialog, - DialogTitle, - DialogContent, - DialogActions, - Button, - Typography, - Grid, - IconButton, - TextField, - MenuItem, - Box, -} from "@mui/material"; -import { useSnackbar } from "notistack"; -import { useTranslation } from "react-i18next"; -import { generateSequentialName } from "../../../../utils/nameGenerator"; -import PlaceholdersList from "../../../../components/generative/RAG/PlaceholdersList"; -import HighlightedTextarea from "../../../../components/generative/RAG/HighlightedTextarea"; -import { getCustomPrompts, createRAGPrompt } from "../../../../api/rag"; -import { LANGUAGE_CODES } from "../../../../constants/languages"; - -/** - * Modal dialog for creating a new custom RAG generation prompt. - * Provides a form for the prompt name, language, template with placeholder insertion. - * - * @param {object} props - * @param {boolean} props.open - Whether the dialog is open. - * @param {function} props.handleClose - Callback to close the dialog. - * @param {function} props.onPromptCreated - Callback invoked with the new prompt ID on success. - * @param {Array} [props.existingPrompts=[]] - List of existing prompts for name generation. - * @returns {JSX.Element} The new prompt modal. - */ -export default function NewPromptModal({ - open, - handleClose, - onPromptCreated, - existingPrompts = [], -}) { - const { enqueueSnackbar } = useSnackbar(); - const { t } = useTranslation(["generative"]); - const [promptTypes, setPromptTypes] = useState([]); - const selectedPromptType = "CustomRAGGenerationPrompt"; - const [promptName, setPromptName] = useState(""); - const [promptTemplate, setPromptTemplate] = useState(""); - const [promptLanguage, setPromptLanguage] = useState("en"); - const [defaultPromptName, setDefaultPromptName] = useState(""); - const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false); - const textareaRef = useRef(null); - - /** - * Checks whether the prompt template contains all required placeholders. - * @returns {boolean} True if every required placeholder is present in the template. - */ - const allRequiredPresent = React.useMemo(() => { - if (!selectedPromptType) return false; - const selectedType = promptTypes.find( - (type) => type.name === selectedPromptType, - ); - if (!selectedType || !selectedType.metadata) return false; - const required = selectedType.metadata.required_placeholders || []; - return required.every((ph) => promptTemplate.includes(ph)); - }, [selectedPromptType, promptTypes, promptTemplate]); - - const canSave = Boolean( - selectedPromptType && promptName.trim() && allRequiredPresent, - ); - - useEffect(() => { - if (open) { - getCustomPrompts() - .then((data) => setPromptTypes(data)) - .catch(() => setPromptTypes([])); - - const generatedName = generateSequentialName({ - base: "Prompt", - items: existingPrompts, - }); - setDefaultPromptName(generatedName.defaultName); - setPromptName(generatedName.defaultName); - setPromptTemplate(""); - setPromptLanguage("en"); - setHasUnsavedChanges(false); - } - }, [open, existingPrompts]); - - /** - * Updates the prompt name on user input. - * @param {object} e - The input change event. - */ - const handlePromptNameChange = useCallback((e) => { - setPromptName(e.target.value); - setHasUnsavedChanges(true); - }, []); - - /** - * Updates the prompt template text on user input. - * @param {object} e - The textarea change event. - */ - const handlePromptTemplateChange = useCallback((e) => { - setPromptTemplate(e.target.value); - setHasUnsavedChanges(true); - }, []); - - /** - * Inserts a placeholder string at the current cursor position in the - * prompt template textarea. Replaces any selected text with the placeholder. - * @param {string} placeholder - The placeholder string to insert (e.g. "{chunks}"). - */ - const handleInsertPlaceholder = useCallback( - (placeholder) => { - const textarea = textareaRef.current; - if (!textarea) return; - - const start = textarea.selectionStart; - const end = textarea.selectionEnd; - const before = promptTemplate.substring(0, start); - const after = promptTemplate.substring(end); - - setPromptTemplate(before + placeholder + after); - setHasUnsavedChanges(true); - - // Restore cursor position right after the inserted placeholder - requestAnimationFrame(() => { - const pos = start + placeholder.length; - textarea.selectionStart = pos; - textarea.selectionEnd = pos; - textarea.focus(); - }); - }, - [promptTemplate], - ); - - /** - * Closes the modal after checking for unsaved changes; prompts for confirmation if needed. - */ - const handleConfirmClose = useCallback(() => { - if (hasUnsavedChanges) { - const confirmed = window.confirm( - t("generative:rag.newPrompt.unsavedChanges"), - ); - if (!confirmed) return; - } - setHasUnsavedChanges(false); - handleClose(); - }, [hasUnsavedChanges, handleClose, t]); - - /** - * Creates the prompt via the API and calls onPromptCreated with the new ID on success. - */ - const handleSave = useCallback(async () => { - try { - const result = await createRAGPrompt({ - class_name: selectedPromptType, - name: promptName, - parameters: { - template: promptTemplate, - ...(promptLanguage ? { language: promptLanguage } : {}), - }, - }); - if (result && result.id) { - enqueueSnackbar(t("generative:rag.newPrompt.success"), { - variant: "success", - }); - setHasUnsavedChanges(false); - await onPromptCreated(result.id); - } - } catch (error) { - console.error("Error creating prompt:", error); - enqueueSnackbar(t("generative:rag.newPrompt.error"), { - variant: "error", - }); - } - }, [ - promptName, - promptTemplate, - selectedPromptType, - onPromptCreated, - enqueueSnackbar, - t, - ]); - - const selectedType = promptTypes.find( - (type) => type.name === selectedPromptType, - ); - - return ( - - - - - - {t("generative:rag.newPrompt.title")} - - - - - - - - - - - - - {t("generative:rag.newPrompt.description")} - - - {t("generative:rag.newPrompt.placeholdersInfo")} - - - - - - { - setPromptLanguage(e.target.value); - setHasUnsavedChanges(true); - }} - sx={{ mb: 2 }} - > - - {t("generative:rag.promptView.languageNone")} - - {LANGUAGE_CODES.map((code) => ( - - {t(`generative:rag.prompt.languages.${code}`)} - - ))} - - - {selectedType && selectedType.metadata && ( - - )} - - - - - - - - - ); -} diff --git a/DashAI/front/src/pages/generative/RAGSession/components/RAGSectionColumn.jsx b/DashAI/front/src/pages/generative/RAGSession/components/RAGSectionColumn.jsx deleted file mode 100644 index 1bd79a36b..000000000 --- a/DashAI/front/src/pages/generative/RAGSession/components/RAGSectionColumn.jsx +++ /dev/null @@ -1,37 +0,0 @@ -import { Box } from "@mui/material"; -import PropTypes from "prop-types"; - -/** - * A flex-column layout wrapper used inside RAG sections for consistent spacing. - * - * @param {object} props - * @param {React.ReactNode} [props.children] - Content to render. - * @param {number|string} [props.gap] - Gap between children (default 4). - * @param {object|Array} [props.sx] - Additional MUI sx overrides. - * @param {object} [props] - Additional props spread to the Box. - * @returns {JSX.Element} The column wrapper. - */ -export default function RAGSectionColumn({ children, gap = 4, sx, ...props }) { - return ( - - {children} - - ); -} - -RAGSectionColumn.propTypes = { - children: PropTypes.node, - gap: PropTypes.oneOfType([PropTypes.number, PropTypes.string]), - sx: PropTypes.oneOfType([PropTypes.array, PropTypes.object, PropTypes.func]), -}; diff --git a/DashAI/front/src/types/ragConfiguration.ts b/DashAI/front/src/types/ragConfiguration.ts index 98e984700..979135f34 100644 --- a/DashAI/front/src/types/ragConfiguration.ts +++ b/DashAI/front/src/types/ragConfiguration.ts @@ -57,15 +57,29 @@ export interface IRAGDocumentIndexState { indexed: boolean; } +/** Queue state of the indexing job a session most recently started. */ +export interface IRAGIndexJobState { + status: string; + /** Percentage 0-100, or null while the total work is unknown. */ + progress: number | null; + progress_message: string | null; + error: string | null; +} + /** Whether a session's documents are indexed for its current configuration. */ export interface IRAGIndexStatus { - status: "not_indexed" | "stale" | "indexed"; + /** `no_documents` until the session has something to index. */ + status: "no_documents" | "not_indexed" | "stale" | "indexing" | "indexed"; chunk_set_id: number | null; total_chunks: number; retriever_ready: boolean; documents: IRAGDocumentIndexState[]; /** Localized, ready to render as-is. */ message: string; + /** Null once the job is dismissed from the queue, or if none ever ran. */ + job_id: string | null; + /** Kept after the job ends so a failure survives a page reload. */ + job: IRAGIndexJobState | null; } /** A ready-to-apply component configuration offered as a named preset. */ @@ -76,10 +90,3 @@ export interface IRAGPreset { component: string; params: Record; } - -/** The configuration a new session gets when the user picks nothing. */ -export interface IRAGSessionDefaults { - chunking_model: { component: string; display_name: string; params: object }; - retriever_model: { component: string; display_name: string; params: object }; - prompt: { component: string; display_name: string; params: object }; -} diff --git a/DashAI/front/src/types/ragPrompt.ts b/DashAI/front/src/types/ragPrompt.ts deleted file mode 100644 index 9bc493987..000000000 --- a/DashAI/front/src/types/ragPrompt.ts +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Represents a RAG prompt as returned by the /v1/prompt/ API. - * - * Supports both single-template (via `template`) and multi-template - * (via `templates` keyed by language code) prompt shapes. - */ -export interface IRAGPrompt { - /** Unique identifier for the prompt. */ - id: number; - /** Fully qualified component class name (e.g. "CustomRAGGenerationPrompt"). */ - class_name: string; - /** Human-readable display name. */ - name: string; - /** ISO timestamp of creation. */ - created?: string; - /** ISO timestamp of last modification. */ - last_modified?: string; - /** Parameter bag containing the template(s) and language. */ - parameters: { - /** Single template string (used for custom/simple prompts). */ - template?: string; - /** Multi-language template dictionary keyed by language code (used for default prompts). */ - templates?: Record; - /** The language code for the active template. */ - language?: string; - }; -} diff --git a/DashAI/front/src/utils/i18n/locales/de/common.json b/DashAI/front/src/utils/i18n/locales/de/common.json index b3ca550d1..ca3c3016b 100644 --- a/DashAI/front/src/utils/i18n/locales/de/common.json +++ b/DashAI/front/src/utils/i18n/locales/de/common.json @@ -1,5 +1,6 @@ { "actions": "Aktionen", + "copy": "Kopieren", "holdout": "Zurückgehalten", "hub": "Hub", "add": "Hinzufügen", diff --git a/DashAI/front/src/utils/i18n/locales/de/generative.json b/DashAI/front/src/utils/i18n/locales/de/generative.json index 5f405df0d..17afc93ed 100644 --- a/DashAI/front/src/utils/i18n/locales/de/generative.json +++ b/DashAI/front/src/utils/i18n/locales/de/generative.json @@ -13,14 +13,18 @@ }, "documentsBar": { "addDocuments": "Dokumente hinzufügen", - "available": " verfügbar", - "detailedView": "Detailansicht anzeigen", + "alreadyInSession": "„{{file}}“ ist bereits in dieser Sitzung", + "delete": "Aus der Sitzung entfernen", + "deleteWarning": "Der extrahierte Text und die Textabschnitte werden ebenfalls entfernt. Die Sitzung wird bei Ihrer nächsten Nachricht ohne das Dokument neu indexiert.", + "deleted": "„{{file}}“ entfernt", "documentCount_one": "{{count}} Dokument", "documentCount_other": "{{count}} Dokumente", + "failedDelete": "Das Dokument konnte nicht entfernt werden", "failedFetch": "Dokumente konnten nicht abgerufen werden", + "failedPreview": "Die Datei konnte nicht geladen werden", "failedUpload": "Dokument(e) konnten nicht hochgeladen werden", "inCurrentSession": " in aktueller Sitzung", - "noDocumentsAvailable": "Keine Dokumente verfügbar", + "inspect": "Textextraktion", "noDocumentsFound": "Keine Dokumente gefunden", "noDocumentsInSession": "Keine Dokumente in dieser Sitzung", "searchPlaceholder": "Dokumente suchen", @@ -39,6 +43,7 @@ "failedToFetchSessions": "Sitzungen konnten nicht abgerufen werden", "failedToFetchTasks": "Generative Aufgaben konnten nicht abgerufen werden", "failedToLoadModels": "Modelle konnten nicht geladen werden", + "failedToSendMessage": "Die Nachricht konnte nicht gesendet werden", "failedToUpdateSession": "Sitzung konnte nicht aktualisiert werden", "modelSwitchFailed": "Sitzungsmodell konnte nicht geändert werden", "nameRequired": "Name ist erforderlich", @@ -108,10 +113,8 @@ "retrieverTitle": "Erweiterte Retriever-Konfiguration" }, "breadcrumbs": { - "documents": "Dokumente", "generative": "Generativ", "goBack": "Zurück", - "prompts": "Prompts", "rag": "RAG", "sessionSuffix": "Sitzung" }, @@ -149,12 +152,14 @@ "advanced": "Erweiterte Konfiguration", "chunkCount_one": "{{count}} Chunk", "chunkCount_other": "{{count}} Chunks", - "contextBudget": "Kontextbudget" + "contextBudget": "Kontextbudget", + "discard": "Verwerfen", + "unsavedIn": "Nicht gespeicherte Änderungen in: {{sections}}" }, "create": { - "defaultsNotice": "Diese Einstellungen gelten standardmäßig und lassen sich in der Sitzung ändern:", + "newSession": "Neue RAG-Sitzung", "selectModel": "Sprachmodell", - "subtitle": "Ein Name, deine Dokumente und ein Modell. Alles andere hat einen sinnvollen Standardwert, den du später ändern kannst.", + "subtitle": "Ein Name und ein Modell. Fügen Sie Ihre Dokumente hinzu, sobald die Sitzung besteht; alles andere hat einen sinnvollen Standardwert, den Sie später ändern können.", "title": "Neue RAG-Sitzung" }, "documentPreview": { @@ -163,16 +168,6 @@ "title": "Dokumentvorschau" }, "documents": { - "duplicate": { - "affectedSessions": "Betroffene Sitzungen:", - "cancel": "Abbrechen", - "confirm": "Aktualisieren", - "message": "Diese Datei wurde bereits hochgeladen. Möchten Sie sie aktualisieren?", - "noAffectedSessions": "Derzeit verwendet keine Sitzung dieses Dokument.", - "title": "Datei existiert bereits", - "warning": "Dadurch werden die angepassten Modelle (Embeddings, Retriever) dieser Sitzungen gelöscht." - }, - "emptyUploadText": "Dokument(e) hochladen", "extractorModal": { "close": "Schließen", "explanation": "Es gibt verschiedene Möglichkeiten, Text aus einer PDF zu extrahieren. Jeder Extraktor kann je nach Dokumentstruktur unterschiedliche Ergebnisse liefern — probieren Sie verschiedene Optionen aus und vergleichen Sie die Ausgabe. Wichtig: RAG verwendet nur Text. Wenn Ihre PDF Bilder mit Text enthalten (gescannte Dokumente, Diagramme), müssen Sie einen OCR-fähigen Extraktor wie EasyOCR verwenden, um diese Bilder in Text umzuwandeln, den RAG nutzen kann. Ohne OCR werden Bilder ignoriert.", @@ -186,25 +181,9 @@ "upToDate": "Bereit" }, "table": { - "actions": "Aktionen", - "addDocument": "Dokument hinzufügen", "configureExtractor": "Extraktor konfigurieren", - "created": "Erstellt", - "currentDocuments": "Aktuelle Dokumente", - "delete": "Löschen", - "emptyUploadText": "Dokument(e) hochladen", - "extractor": "Extraktor", - "id": "ID", - "lastModified": "Zuletzt geändert", - "name": "Name", - "noDocumentsAvailable": "Keine Dokumente verfügbar.", - "preview": "Vorschau", - "type": "Typ", "unknownType": "Unbekannt" - }, - "uploadButton": "Dokumente hochladen", - "uploadFailed": "Das Dokument konnte nicht hochgeladen werden.", - "uploadFailedReason": "Konnte \"{{file}}\" nicht hochladen: {{reason}}" + } }, "generator": { "advancedButton": "Erweiterte Konfiguration öffnen", @@ -215,40 +194,18 @@ "configureTitle": "Generator-Modell konfigurieren (LLM)", "modelLabel": "Generator-Modell" }, - "home": { - "documents": "Dokumente", - "documentsDescription": "Dokumente hochladen und die Textextraktion wählen.", - "newSession": "Neue RAG-Sitzung", - "newSessionDescription": "Dokumente und ein Modell wählen und loslegen.", - "prompts": "Prompts", - "promptsDescription": "Prompt-Vorlagen für deine Sitzungen verwalten.", - "subtitle": "Chatte mit deinen eigenen Dokumenten.", - "title": "Retrieval-Augmented Generation" - }, "index": { "chunkCount_one": "{{count}} Chunk", "chunkCount_other": "{{count}} Chunks", + "indexFailed": "Indexierung fehlgeschlagen", + "indexing": "Wird indexiert…", "indexingInProgress": "Dokumente werden indexiert…", - "notIndexed": "Nicht indexiert" + "notIndexed": "Nicht indexiert", + "retryIndexing": "Wiederholen" }, "messages": { "success": "RAG-Sitzung erfolgreich erstellt" }, - "newPrompt": { - "cancel": "Abbrechen", - "description": "Die Prompt-Vorlage definiert, wie die Chunks (Dokumentteile) und Chat-Nachrichten integriert werden, um Antworten zu generieren. Passen Sie den Prompt an, um das Verhalten Ihrer RAG-Sitzungen anzupassen.", - "error": "Prompt konnte nicht erstellt werden", - "languageLabel": "Sprache (optional)", - "nameLabel": "Prompt-Name", - "nameRequired": "Prompt-Name ist erforderlich", - "placeholdersInfo": "Verwenden Sie {chunks} für die Stelle, an der die abgerufenen Dokument-Chunks eingefügt werden, und {input} für die Benutzernachricht.", - "promptLabel": "Prompt", - "promptPlaceholder": "Hier können Sie den Prompt ändern, zum Beispiel:\nJede Benutzernachricht wird als {input} hinzugefügt\nDie Quellen werden als {chunks} hinzugefügt", - "save": "Speichern", - "success": "Prompt erfolgreich erstellt!", - "title": "Neuen Prompt erstellen", - "unsavedChanges": "Sie haben nicht gespeicherte Änderungen. Möchten Sie wirklich abbrechen?" - }, "paramsPanel": { "failedToLoad": "RAG-Sitzung konnte nicht geladen werden", "failedToUpdate": "RAG-Parameter konnten nicht aktualisiert werden", @@ -260,48 +217,20 @@ "title": "Erforderliche Platzhalter" }, "prompt": { - "collapse": "Einklappen", - "createNewPrompt": "Neuen Prompt erstellen", - "defaultGenerationPrompt": "Standard-Generierungsprompt", - "defaultQAGenerationPrompt": "Standard-Q&A-Generierungsprompt", - "description": "Wählen Sie eine Prompt-Vorlage, die definiert, wie der abgerufene Kontext und die Chat-Nachrichten kombiniert werden, um Antworten zu generieren.", - "descriptionToggle": "Beschreibung", - "expand": "Ausklappen", - "language": "Sprache", + "editor": { + "expand": "In größerem Editor öffnen", + "missingPlaceholder": "In der Vorlage fehlt noch: {{placeholders}}", + "seedLanguage": "Vorlagensprache", + "startFrom": "Ausgehen von", + "startFromHelp": "Ersetzt die Vorlage unten. Ihre bleibt erhalten, bis Sie eine auswählen.", + "template": "Vorlage", + "templatePlaceholder": "Jede Nutzernachricht kommt als {input} an\nDie gefundenen Textabschnitte als {chunks}" + }, "languages": { "en": "English", "es": "Español", "pt": "Português" - }, - "newPromptButton": "Neuer Prompt", - "openPrompts": "Prompt-Bibliothek öffnen", - "promptLabel": "Prompt", - "selectTemplate": "Prompt-Vorlage auswählen", - "selectTemplatePlaceholder": "z. B. Standard-Prompt, Benutzerdefinierte Anweisung", - "selectedTemplate": "Ausgewählte Prompt-Vorlage:" - }, - "promptView": { - "close": "Schließen", - "language": "Sprache", - "languageNone": "Keine Sprache", - "languageNotAvailable": "Sprache nicht verfügbar", - "noContent": "Kein Vorlageninhalt", - "table": { - "actions": "Aktionen", - "choosePrompt": "Wählen oder passen Sie Ihren Prompt an, um das Modellverhalten zu definieren.", - "created": "Erstellt", - "currentPrompts": "Aktuelle Prompts", - "edited": "Bearbeitet", - "id": "ID", - "language": "Sprache", - "name": "Name", - "newPrompt": "Neuer Prompt", - "type": "Typ", - "viewPrompt": "Prompt anzeigen" - }, - "templateContent": "Vorlageninhalt", - "type": "Typ", - "untitledPrompt": "Prompt" + } }, "retrieverConfig": { "modelLabel": "Retriever-Modell" @@ -310,8 +239,6 @@ "notFound": "Sitzung nicht gefunden" }, "setup": { - "selectDocuments": "Dokumente auswählen", - "selectDocumentsDescription": "Laden Sie neue Dokumente hoch oder wählen Sie aus vorhandenen, um sie für RAG zu verwenden.", "sessionName": "Sitzungsname *" }, "summary": { @@ -324,39 +251,9 @@ "sessionUpdated": "Sitzung erfolgreich aktualisiert" }, "validation": { - "modelComponentMissing": "Eine Modellkomponente hat keinen ausgewählten Namen.", - "modelParamsIncomplete": "\"{{model}}\"-Parameter sind unvollständig. Bitte konfigurieren Sie sie erneut in den erweiterten Einstellungen.", "nameRequired": "Sitzungsname ist erforderlich" } }, - "ragDocumentsPage": { - "contentPanel": { - "loadingContent": "Inhalt wird geladen...", - "noContent": "Kein Inhalt extrahiert", - "noDocumentSelected": "Wählen Sie ein Dokument aus, um seinen verarbeiteten Inhalt anzuzeigen" - }, - "description": "Verwalten Sie Dokumente für Ihre RAG-Sitzungen. Denken Sie daran, dass RAG nur Text verwendet: Bewerten Sie die verfügbaren Extraktoren und wählen Sie den am besten geeigneten für jeden Dokumenttyp (z. B. OCR-fähige Extraktoren für gescannte Dokumente).", - "detailPanel": { - "changeExtractorConfirmBody_one": "Dieses Dokument hat bereits einen konfigurierten Extraktor. Möchten Sie ihn ändern?", - "changeExtractorConfirmBody_other": "Dieses Dokument hat bereits einen konfigurierten Extraktor. Möchten Sie ihn ändern?", - "changeExtractorConfirmTitle": "Extraktor ändern", - "contentPreview": "Inhaltsvorschau", - "documentInfo": "Dokumentinformationen", - "extracting": "Text wird extrahiert...", - "extractor": "Extraktor", - "name": "Name", - "noContent": "Noch kein Inhalt extrahiert. Klicken Sie auf \"Dokument verarbeiten\", um Text zu extrahieren.", - "noDocumentSelected": "Wählen Sie ein Dokument aus, um Details anzuzeigen", - "processAndShowContent": "Dokument verarbeiten und Inhalt anzeigen", - "saveExtractor": "Extraktor speichern", - "type": "Typ" - }, - "title": "RAG-Dokumente" - }, - "ragPromptsPage": { - "description": "Verwalten Sie Prompts für Ihre RAG-Sitzungen: Zeigen Sie alle verfügbaren Prompts an und erstellen Sie neue, um Ihre KI-Interaktionen zu verbessern.", - "title": "RAG-Prompts" - }, "sourcesDisplay": { "chunk_one": "Chunk", "chunk_other": "Chunks", diff --git a/DashAI/front/src/utils/i18n/locales/en/common.json b/DashAI/front/src/utils/i18n/locales/en/common.json index 3dcb81fcd..1d5689910 100644 --- a/DashAI/front/src/utils/i18n/locales/en/common.json +++ b/DashAI/front/src/utils/i18n/locales/en/common.json @@ -1,5 +1,6 @@ { "actions": "Actions", + "copy": "Copy", "holdout": "Holdout", "hub": "Hub", "add": "Add", diff --git a/DashAI/front/src/utils/i18n/locales/en/generative.json b/DashAI/front/src/utils/i18n/locales/en/generative.json index 95838b78f..25e76b0f6 100644 --- a/DashAI/front/src/utils/i18n/locales/en/generative.json +++ b/DashAI/front/src/utils/i18n/locales/en/generative.json @@ -13,14 +13,18 @@ }, "documentsBar": { "addDocuments": "Add documents", - "available": " available", - "detailedView": "See detailed view", + "alreadyInSession": "\"{{file}}\" is already in this session", + "delete": "Remove from session", + "deleteWarning": "Its extracted text and chunks are removed too. The session re-indexes without it on your next message.", + "deleted": "Removed \"{{file}}\"", "documentCount_one": "{{count}} document", "documentCount_other": "{{count}} documents", + "failedDelete": "Failed to remove the document", "failedFetch": "Failed to fetch documents", + "failedPreview": "Failed to load the file", "failedUpload": "Failed to upload document(s)", "inCurrentSession": " in current session", - "noDocumentsAvailable": "No documents available", + "inspect": "Text extraction", "noDocumentsFound": "No documents found", "noDocumentsInSession": "No documents in this session", "searchPlaceholder": "Search documents", @@ -39,6 +43,7 @@ "failedToFetchSessions": "Failed to fetch sessions", "failedToFetchTasks": "Failed to fetch generative tasks", "failedToLoadModels": "Failed to load models", + "failedToSendMessage": "Could not send the message", "failedToUpdateSession": "Failed to update session", "modelSwitchFailed": "Failed to change the session model", "nameRequired": "Name is required", @@ -108,10 +113,8 @@ "retrieverTitle": "Advanced retriever configuration" }, "breadcrumbs": { - "documents": "Documents", "generative": "Generative", "goBack": "Go back", - "prompts": "Prompts", "rag": "RAG", "sessionSuffix": "session" }, @@ -149,12 +152,14 @@ "advanced": "Advanced configuration", "chunkCount_one": "{{count}} chunk", "chunkCount_other": "{{count}} chunks", - "contextBudget": "Context budget" + "contextBudget": "Context budget", + "discard": "Discard", + "unsavedIn": "Unsaved changes in: {{sections}}" }, "create": { - "defaultsNotice": "These settings are applied by default and can be changed in the session:", + "newSession": "New RAG session", "selectModel": "Language model", - "subtitle": "A name, your documents, and a model. Everything else has a sensible default you can change later.", + "subtitle": "A name and a model. Add your documents once the session exists; everything else has a sensible default you can change later.", "title": "New RAG session" }, "documentPreview": { @@ -163,16 +168,6 @@ "title": "Preview Document" }, "documents": { - "duplicate": { - "affectedSessions": "Affected sessions:", - "cancel": "Cancel", - "confirm": "Update", - "message": "This file has already been uploaded. Do you want to update it?", - "noAffectedSessions": "No sessions are currently using this document.", - "title": "File already exists", - "warning": "This will delete the fitted models (embeddings, retrievers) of these sessions." - }, - "emptyUploadText": "Upload your document(s)", "extractorModal": { "close": "Close", "explanation": "There are different ways to process text from a PDF. Each extractor may produce different results depending on the document structure — try different options and compare the output. Important: RAG only uses text. If your PDF contains images with text (scanned documents, diagrams), you must use an OCR-capable extractor like EasyOCR to convert those images into text that RAG can use. Without OCR, images are ignored.", @@ -186,25 +181,9 @@ "upToDate": "Up to date" }, "table": { - "actions": "Actions", - "addDocument": "Add new document", "configureExtractor": "Configure extractor", - "created": "Created", - "currentDocuments": "Current documents", - "delete": "Delete", - "emptyUploadText": "Upload your document(s)", - "extractor": "Extractor", - "id": "ID", - "lastModified": "Last Modified", - "name": "Name", - "noDocumentsAvailable": "No documents available.", - "preview": "Preview", - "type": "Type", "unknownType": "Unknown" - }, - "uploadButton": "Upload documents", - "uploadFailed": "The document could not be uploaded.", - "uploadFailedReason": "Could not upload \"{{file}}\": {{reason}}" + } }, "generator": { "advancedButton": "Open advanced configuration", @@ -215,40 +194,18 @@ "configureTitle": "Configure Generator Model (LLM)", "modelLabel": "Generator Model" }, - "home": { - "documents": "Documents", - "documentsDescription": "Upload documents and choose how their text is extracted.", - "newSession": "New RAG session", - "newSessionDescription": "Pick documents and a model, then start chatting.", - "prompts": "Prompts", - "promptsDescription": "Manage the prompt templates your sessions can use.", - "subtitle": "Chat with your own documents.", - "title": "Retrieval-Augmented Generation" - }, "index": { "chunkCount_one": "{{count}} chunk", "chunkCount_other": "{{count}} chunks", + "indexFailed": "Indexing failed", + "indexing": "Indexing…", "indexingInProgress": "Indexing documents…", - "notIndexed": "Not indexed" + "notIndexed": "Not indexed", + "retryIndexing": "Retry" }, "messages": { "success": "RAG Session created successfully" }, - "newPrompt": { - "cancel": "Cancel", - "description": "Prompt template defines how the chunks (pieces of documents) and chat messages are integrated to generate responses. Customize the prompt to tailor the behavior of your RAG sessions.", - "error": "Failed to create prompt", - "languageLabel": "Language (optional)", - "nameLabel": "Prompt Name", - "nameRequired": "Prompt name is required", - "placeholdersInfo": "Use {chunks} to represent where the retrieved document chunks will be inserted, and {input} for the user message.", - "promptLabel": "Prompt", - "promptPlaceholder": "Here you can modify the prompt, for example:\nEach user message is added as {input}\nThe sources are added as {chunks}", - "save": "Save", - "success": "Prompt created successfully!", - "title": "Create a new prompt", - "unsavedChanges": "You have unsaved changes. Are you sure you want to cancel?" - }, "paramsPanel": { "failedToLoad": "Failed to load RAG session", "failedToUpdate": "Failed to update RAG parameters", @@ -260,48 +217,20 @@ "title": "Required Placeholders" }, "prompt": { - "collapse": "Collapse", - "createNewPrompt": "Create new prompt", - "defaultGenerationPrompt": "Default Generation Prompt", - "defaultQAGenerationPrompt": "Default Q&A Generation Prompt", - "description": "Select a prompt template that defines how the retrieved context and chat messages are combined to generate responses.", - "descriptionToggle": "Description", - "expand": "Expand", - "language": "Language", + "editor": { + "expand": "Open in a larger editor", + "missingPlaceholder": "The template still needs: {{placeholders}}", + "seedLanguage": "Seed language", + "startFrom": "Start from", + "startFromHelp": "Replaces the template below. Yours is kept until you pick one.", + "template": "Template", + "templatePlaceholder": "Each user message arrives as {input}\nThe retrieved passages arrive as {chunks}" + }, "languages": { "en": "English", "es": "Español", "pt": "Português" - }, - "newPromptButton": "New prompt", - "openPrompts": "Open prompt library", - "promptLabel": "Prompt", - "selectTemplate": "Select prompt template", - "selectTemplatePlaceholder": "e.g., Default Prompt, Custom Instruction", - "selectedTemplate": "Selected prompt template:" - }, - "promptView": { - "close": "Close", - "language": "Language", - "languageNone": "No language", - "languageNotAvailable": "Language not available", - "noContent": "No template content", - "table": { - "actions": "Actions", - "choosePrompt": "Choose or customize your prompt to define the model's behavior.", - "created": "Created", - "currentPrompts": "Current prompts", - "edited": "Edited", - "id": "ID", - "language": "Language", - "name": "Name", - "newPrompt": "New Prompt", - "type": "Type", - "viewPrompt": "View prompt" - }, - "templateContent": "Template content", - "type": "Type", - "untitledPrompt": "Prompt" + } }, "retrieverConfig": { "modelLabel": "Retriever model" @@ -310,8 +239,6 @@ "notFound": "Session not found" }, "setup": { - "selectDocuments": "Select documents", - "selectDocumentsDescription": "Upload new documents or select from existing ones to be used for RAG.", "sessionName": "Session Name *" }, "summary": { @@ -324,39 +251,9 @@ "sessionUpdated": "Session updated successfully" }, "validation": { - "modelComponentMissing": "A model component has no name selected.", - "modelParamsIncomplete": "\"{{model}}\" parameters are incomplete. Please reconfigure it in the advanced settings.", "nameRequired": "Session name is required" } }, - "ragDocumentsPage": { - "contentPanel": { - "loadingContent": "Loading content...", - "noContent": "No content extracted", - "noDocumentSelected": "Select a document to view its processed content" - }, - "description": "Manage documents for your RAG sessions. Remember that RAG only uses text: evaluate the available extractors and choose the most suitable one for each document type (e.g., use OCR-capable extractors for scanned documents).", - "detailPanel": { - "changeExtractorConfirmBody_one": "ragDocumentsPage.detailPanel.changeExtractorConfirmBody", - "changeExtractorConfirmBody_other": "ragDocumentsPage.detailPanel.changeExtractorConfirmBody", - "changeExtractorConfirmTitle": "Change extractor", - "contentPreview": "Content Preview", - "documentInfo": "Document Info", - "extracting": "Extracting text...", - "extractor": "Extractor", - "name": "Name", - "noContent": "No content extracted yet. Click \"Process document\" to extract text.", - "noDocumentSelected": "Select a document to view details", - "processAndShowContent": "Process document and show content", - "saveExtractor": "Save extractor", - "type": "Type" - }, - "title": "RAG documents" - }, - "ragPromptsPage": { - "description": "Manage prompts for your RAG sessions: view all available prompts and create new ones to improve your AI interactions.", - "title": "RAG prompts" - }, "sourcesDisplay": { "chunk_one": "chunk", "chunk_other": "chunks", diff --git a/DashAI/front/src/utils/i18n/locales/es/common.json b/DashAI/front/src/utils/i18n/locales/es/common.json index 4fef5381b..f327b1348 100644 --- a/DashAI/front/src/utils/i18n/locales/es/common.json +++ b/DashAI/front/src/utils/i18n/locales/es/common.json @@ -1,5 +1,6 @@ { "actions": "Acciones", + "copy": "Copiar", "holdout": "Reservado", "hub": "Hub", "add": "Agregar", diff --git a/DashAI/front/src/utils/i18n/locales/es/generative.json b/DashAI/front/src/utils/i18n/locales/es/generative.json index 5cc01898a..f2cefaf7b 100644 --- a/DashAI/front/src/utils/i18n/locales/es/generative.json +++ b/DashAI/front/src/utils/i18n/locales/es/generative.json @@ -14,15 +14,19 @@ }, "documentsBar": { "addDocuments": "Agregar documentos", - "available": " disponibles", - "detailedView": "Ver vista detallada", + "alreadyInSession": "\"{{file}}\" ya está en esta sesión", + "delete": "Quitar de la sesión", + "deleteWarning": "También se eliminan su texto extraído y sus fragmentos. La sesión se reindexará sin él en tu próximo mensaje.", + "deleted": "Se quitó \"{{file}}\"", "documentCount_many": "{{count}} documentos", "documentCount_one": "{{count}} documento", "documentCount_other": "{{count}} documentos", + "failedDelete": "No se pudo quitar el documento", "failedFetch": "Error al obtener documentos", + "failedPreview": "No se pudo cargar el archivo", "failedUpload": "Error al subir documento(s)", "inCurrentSession": " en la sesión actual", - "noDocumentsAvailable": "No hay documentos disponibles", + "inspect": "Extracción de texto", "noDocumentsFound": "No se encontraron documentos", "noDocumentsInSession": "No hay documentos en esta sesión", "searchPlaceholder": "Buscar documentos", @@ -42,6 +46,7 @@ "failedToFetchSessions": "Error al obtener sesiones", "failedToFetchTasks": "Error al obtener tareas generativas", "failedToLoadModels": "Error al cargar los modelos", + "failedToSendMessage": "No se pudo enviar el mensaje", "failedToUpdateSession": "Error al actualizar la sesión", "modelSwitchFailed": "No se pudo cambiar el modelo de la sesión", "nameRequired": "Se requiere un nombre", @@ -112,10 +117,8 @@ "retrieverTitle": "Configuración avanzada de recuperación" }, "breadcrumbs": { - "documents": "Documentos", "generative": "Generativo", "goBack": "Volver", - "prompts": "Prompts", "rag": "RAG", "sessionSuffix": "sesión" }, @@ -153,12 +156,14 @@ "advanced": "Configuración avanzada", "chunkCount_one": "{{count}} fragmento", "chunkCount_other": "{{count}} fragmentos", - "contextBudget": "Presupuesto de contexto" + "contextBudget": "Presupuesto de contexto", + "discard": "Descartar", + "unsavedIn": "Cambios sin guardar en: {{sections}}" }, "create": { - "defaultsNotice": "Esta configuración se aplica por defecto y puedes cambiarla en la sesión:", + "newSession": "Nueva sesión RAG", "selectModel": "Modelo de lenguaje", - "subtitle": "Un nombre, tus documentos y un modelo. Todo lo demás tiene un valor por defecto que puedes cambiar después.", + "subtitle": "Un nombre y un modelo. Agrega tus documentos una vez creada la sesión; todo lo demás tiene un valor por defecto que puedes cambiar después.", "title": "Nueva sesión RAG" }, "documentPreview": { @@ -167,16 +172,6 @@ "title": "Vista previa del documento" }, "documents": { - "duplicate": { - "affectedSessions": "Sesiones afectadas:", - "cancel": "Cancelar", - "confirm": "Actualizar", - "message": "Este archivo ya ha sido subido. ¿Desea actualizarlo?", - "noAffectedSessions": "Ninguna sesión está usando este documento actualmente.", - "title": "El archivo ya existe", - "warning": "Esto eliminará los modelos ajustados (embeddings, retrievers) de estas sesiones." - }, - "emptyUploadText": "Sube tu(s) documento(s)", "extractorModal": { "close": "Cerrar", "explanation": "Existen distintas formas de procesar el texto de un PDF. Cada extractor puede producir resultados diferentes según la estructura del documento — prueba distintas opciones y compara el resultado. Importante: RAG solo utiliza texto. Si tu PDF contiene imágenes con texto (documentos escaneados, diagramas), debes usar un extractor con capacidad OCR como EasyOCR para convertir esas imágenes en texto que RAG pueda utilizar. Sin OCR, las imágenes son ignoradas.", @@ -190,25 +185,9 @@ "upToDate": "Listo" }, "table": { - "actions": "Acciones", - "addDocument": "Añadir documento", "configureExtractor": "Configurar extractor", - "created": "Creado", - "currentDocuments": "Documentos actuales", - "delete": "Eliminar", - "emptyUploadText": "Sube tu(s) documento(s)", - "extractor": "Extractor", - "id": "ID", - "lastModified": "Última modificación", - "name": "Nombre", - "noDocumentsAvailable": "No hay documentos disponibles.", - "preview": "Vista previa", - "type": "Tipo", "unknownType": "Desconocido" - }, - "uploadButton": "Subir documentos", - "uploadFailed": "No se pudo subir el documento.", - "uploadFailedReason": "No se pudo subir \"{{file}}\": {{reason}}" + } }, "generator": { "advancedButton": "Abrir configuración avanzada", @@ -219,40 +198,18 @@ "configureTitle": "Configurar modelo generador (LLM)", "modelLabel": "Modelo generador" }, - "home": { - "documents": "Documentos", - "documentsDescription": "Sube documentos y elige cómo se extrae su texto.", - "newSession": "Nueva sesión RAG", - "newSessionDescription": "Elige documentos y un modelo, y empieza a conversar.", - "prompts": "Prompts", - "promptsDescription": "Administra las plantillas de prompt que pueden usar tus sesiones.", - "subtitle": "Conversa con tus propios documentos.", - "title": "Generación Aumentada por Recuperación" - }, "index": { "chunkCount_one": "{{count}} fragmento", "chunkCount_other": "{{count}} fragmentos", + "indexFailed": "Falló la indexación", + "indexing": "Indexando…", "indexingInProgress": "Indexando documentos…", - "notIndexed": "Sin indexar" + "notIndexed": "Sin indexar", + "retryIndexing": "Reintentar" }, "messages": { "success": "Sesión RAG creada exitosamente" }, - "newPrompt": { - "cancel": "Cancelar", - "description": "La plantilla de prompt define cómo se integran los fragmentos (piezas de documentos) y los mensajes del chat para generar respuestas. Personaliza el prompt para adaptar el comportamiento de tus sesiones RAG.", - "error": "Error al crear el prompt", - "languageLabel": "Idioma (opcional)", - "nameLabel": "Nombre del prompt", - "nameRequired": "El nombre del prompt es obligatorio", - "placeholdersInfo": "Usa {chunks} para representar dónde se insertarán los fragmentos del documento recuperado, e {input} para el mensaje del usuario.", - "promptLabel": "Prompt", - "promptPlaceholder": "Aquí puedes modificar el prompt, por ejemplo:\nCada mensaje del usuario se añade como {input}\nLas fuentes se añaden como {chunks}", - "save": "Guardar", - "success": "¡Prompt creado exitosamente!", - "title": "Crear una nueva plantilla", - "unsavedChanges": "Tienes cambios sin guardar. ¿Estás seguro de que quieres cancelar?" - }, "paramsPanel": { "failedToLoad": "Error al cargar la sesión RAG", "failedToUpdate": "Error al actualizar los parámetros RAG", @@ -264,48 +221,20 @@ "title": "Marcadores obligatorios" }, "prompt": { - "collapse": "Contraer", - "createNewPrompt": "Crear nuevo prompt", - "defaultGenerationPrompt": "Prompt de generación predeterminado", - "defaultQAGenerationPrompt": "Prompt predeterminado para preguntas y respuestas", - "description": "Selecciona una plantilla de prompt que define cómo se combinan el contexto recuperado y los mensajes del chat para generar respuestas.", - "descriptionToggle": "Descripción", - "expand": "Expandir", - "language": "Idioma", + "editor": { + "expand": "Abrir en un editor más grande", + "missingPlaceholder": "A la plantilla le falta: {{placeholders}}", + "seedLanguage": "Idioma de la plantilla", + "startFrom": "Partir de", + "startFromHelp": "Reemplaza la plantilla de abajo. La tuya se conserva hasta que elijas una.", + "template": "Plantilla", + "templatePlaceholder": "Cada mensaje del usuario llega como {input}\nLos fragmentos recuperados llegan como {chunks}" + }, "languages": { "en": "English", "es": "Español", "pt": "Português" - }, - "newPromptButton": "Nuevo prompt", - "openPrompts": "Abrir biblioteca de prompts", - "promptLabel": "Prompt", - "selectTemplate": "Seleccionar plantilla de prompt", - "selectTemplatePlaceholder": "p. ej., Prompt por defecto, Instrucción personalizada", - "selectedTemplate": "Plantilla de prompt seleccionada:" - }, - "promptView": { - "close": "Cerrar", - "language": "Idioma", - "languageNone": "Sin idioma", - "languageNotAvailable": "Idioma no disponible", - "noContent": "Sin contenido de plantilla", - "table": { - "actions": "Acciones", - "choosePrompt": "Elige o personaliza tu prompt para definir el comportamiento del modelo.", - "created": "Creado", - "currentPrompts": "Prompts actuales", - "edited": "Editado", - "id": "ID", - "language": "Idioma", - "name": "Nombre", - "newPrompt": "Nuevo prompt", - "type": "Tipo", - "viewPrompt": "Ver prompt" - }, - "templateContent": "Contenido de la plantilla", - "type": "Tipo", - "untitledPrompt": "Prompt" + } }, "retrieverConfig": { "modelLabel": "Modelo de recuperación" @@ -314,8 +243,6 @@ "notFound": "Sesión no encontrada" }, "setup": { - "selectDocuments": "Seleccionar documentos", - "selectDocumentsDescription": "Sube nuevos documentos o selecciona de los existentes para utilizarlos en RAG.", "sessionName": "Nombre de la sesión *" }, "summary": { @@ -328,40 +255,9 @@ "sessionUpdated": "Sesión actualizada correctamente" }, "validation": { - "modelComponentMissing": "Un componente del modelo no tiene nombre seleccionado.", - "modelParamsIncomplete": "Los parámetros de \"{{model}}\" están incompletos. Reconfigúralo en la configuración avanzada.", "nameRequired": "El nombre de la sesión es obligatorio" } }, - "ragDocumentsPage": { - "contentPanel": { - "loadingContent": "Cargando contenido...", - "noContent": "Sin contenido extraído", - "noDocumentSelected": "Selecciona un documento para ver su contenido procesado" - }, - "description": "Administra documentos para tus sesiones RAG. Recuerda que RAG solo utiliza texto: evalúa los extractores disponibles y elige el más apropiado según el tipo de documento (por ejemplo, usa extractores con OCR para documentos escaneados).", - "detailPanel": { - "changeExtractorConfirmBody_many": "Este documento ya tiene un extractor configurado. ¿Deseas cambiarlo?", - "changeExtractorConfirmBody_one": "Este documento ya tiene un extractor configurado. ¿Deseas cambiarlo?", - "changeExtractorConfirmBody_other": "Este documento ya tiene un extractor configurado. ¿Deseas cambiarlo?", - "changeExtractorConfirmTitle": "Cambiar extractor", - "contentPreview": "Vista previa del contenido", - "documentInfo": "Información del documento", - "extracting": "Extrayendo texto...", - "extractor": "Extractor", - "name": "Nombre", - "noContent": "Sin contenido extraído aún. Haz clic en \"Procesar documento\" para extraer texto.", - "noDocumentSelected": "Selecciona un documento para ver sus detalles", - "processAndShowContent": "Procesar documento y mostrar contenido", - "saveExtractor": "Guardar extractor", - "type": "Tipo" - }, - "title": "Documentos RAG" - }, - "ragPromptsPage": { - "description": "Gestiona los prompts para tus sesiones RAG: ve todos los prompts disponibles y crea nuevos para mejorar tus interacciones de IA.", - "title": "Prompts RAG" - }, "sourcesDisplay": { "chunk_many": "fragmentos", "chunk_one": "fragmento", diff --git a/DashAI/front/src/utils/i18n/locales/pt/common.json b/DashAI/front/src/utils/i18n/locales/pt/common.json index a6fa8c12d..a13e26160 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/common.json +++ b/DashAI/front/src/utils/i18n/locales/pt/common.json @@ -1,5 +1,6 @@ { "actions": "Ações", + "copy": "Copiar", "holdout": "Reservado", "hub": "Hub", "add": "Adicionar", diff --git a/DashAI/front/src/utils/i18n/locales/pt/generative.json b/DashAI/front/src/utils/i18n/locales/pt/generative.json index 27e9309b9..c1f3aaba8 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/generative.json +++ b/DashAI/front/src/utils/i18n/locales/pt/generative.json @@ -14,15 +14,19 @@ }, "documentsBar": { "addDocuments": "Adicionar documentos", - "available": " disponíveis", - "detailedView": "Ver visualização detalhada", + "alreadyInSession": "\"{{file}}\" já está nesta sessão", + "delete": "Remover da sessão", + "deleteWarning": "O texto extraído e os fragmentos também são removidos. A sessão será reindexada sem ele na sua próxima mensagem.", + "deleted": "\"{{file}}\" removido", "documentCount_many": "{{count}} documentos", "documentCount_one": "{{count}} documento", "documentCount_other": "{{count}} documentos", + "failedDelete": "Não foi possível remover o documento", "failedFetch": "Falha ao carregar documentos", + "failedPreview": "Não foi possível carregar o arquivo", "failedUpload": "Falha ao enviar documento(s)", "inCurrentSession": " na sessão atual", - "noDocumentsAvailable": "Nenhum documento disponível", + "inspect": "Extração de texto", "noDocumentsFound": "Nenhum documento encontrado", "noDocumentsInSession": "Nenhum documento nesta sessão", "searchPlaceholder": "Pesquisar documentos", @@ -42,6 +46,7 @@ "failedToFetchSessions": "Erro ao obter sessões", "failedToFetchTasks": "Erro ao obter tarefas generativas", "failedToLoadModels": "Erro ao carregar os modelos", + "failedToSendMessage": "Não foi possível enviar a mensagem", "failedToUpdateSession": "Erro ao atualizar a sessão", "modelSwitchFailed": "Falha ao alterar o modelo da sessão", "nameRequired": "É necessário um nome", @@ -112,10 +117,8 @@ "retrieverTitle": "Configuração avançada de recuperação" }, "breadcrumbs": { - "documents": "Documentos", "generative": "Generativo", "goBack": "Voltar", - "prompts": "Prompts", "rag": "RAG", "sessionSuffix": "sessão" }, @@ -153,12 +156,14 @@ "advanced": "Configuração avançada", "chunkCount_one": "{{count}} fragmento", "chunkCount_other": "{{count}} fragmentos", - "contextBudget": "Orçamento de contexto" + "contextBudget": "Orçamento de contexto", + "discard": "Descartar", + "unsavedIn": "Alterações não salvas em: {{sections}}" }, "create": { - "defaultsNotice": "Esta configuração é aplicada por padrão e pode ser alterada na sessão:", + "newSession": "Nova sessão RAG", "selectModel": "Modelo de linguagem", - "subtitle": "Um nome, seus documentos e um modelo. O resto tem um valor padrão que você pode mudar depois.", + "subtitle": "Um nome e um modelo. Adicione seus documentos depois de criar a sessão; todo o resto tem um padrão que você pode mudar depois.", "title": "Nova sessão RAG" }, "documentPreview": { @@ -167,16 +172,6 @@ "title": "Visualização do documento" }, "documents": { - "duplicate": { - "affectedSessions": "Sessões afetadas:", - "cancel": "Cancelar", - "confirm": "Atualizar", - "message": "Este arquivo já foi enviado. Deseja atualizá-lo?", - "noAffectedSessions": "Nenhuma sessão está usando este documento no momento.", - "title": "O arquivo já existe", - "warning": "Isso excluirá os modelos ajustados (embeddings, retrievers) dessas sessões." - }, - "emptyUploadText": "Faça upload do(s) seu(s) documento(s)", "extractorModal": { "close": "Fechar", "explanation": "Existem diferentes formas de processar texto de um PDF. Cada extrator pode produzir resultados diferentes dependendo da estrutura do documento — experimente diferentes opções e compare o resultado. Importante: RAG usa apenas texto. Se o seu PDF contém imagens com texto (documentos digitalizados, diagramas), você deve usar um extrator com capacidade OCR como EasyOCR para converter essas imagens em texto que o RAG possa usar. Sem OCR, as imagens são ignoradas.", @@ -190,25 +185,9 @@ "upToDate": "Pronto" }, "table": { - "actions": "Ações", - "addDocument": "Adicionar documento", "configureExtractor": "Configurar extrator", - "created": "Criado", - "currentDocuments": "Documentos atuais", - "delete": "Excluir", - "emptyUploadText": "Envie seu(s) documento(s)", - "extractor": "Extrator", - "id": "ID", - "lastModified": "Última modificação", - "name": "Nome", - "noDocumentsAvailable": "Nenhum documento disponível.", - "preview": "Visualizar", - "type": "Tipo", "unknownType": "Desconhecido" - }, - "uploadButton": "Upload de documentos", - "uploadFailed": "Não foi possível enviar o documento.", - "uploadFailedReason": "Não foi possível enviar \"{{file}}\": {{reason}}" + } }, "generator": { "advancedButton": "Abrir configuração avançada", @@ -219,40 +198,18 @@ "configureTitle": "Configurar modelo gerador (LLM)", "modelLabel": "Modelo gerador" }, - "home": { - "documents": "Documentos", - "documentsDescription": "Envie documentos e escolha como o texto é extraído.", - "newSession": "Nova sessão RAG", - "newSessionDescription": "Escolha documentos e um modelo, e comece a conversar.", - "prompts": "Prompts", - "promptsDescription": "Gerencie os modelos de prompt que suas sessões podem usar.", - "subtitle": "Converse com os seus próprios documentos.", - "title": "Geração Aumentada por Recuperação" - }, "index": { "chunkCount_one": "{{count}} fragmento", "chunkCount_other": "{{count}} fragmentos", + "indexFailed": "Falha na indexação", + "indexing": "Indexando…", "indexingInProgress": "Indexando documentos…", - "notIndexed": "Não indexado" + "notIndexed": "Não indexado", + "retryIndexing": "Tentar novamente" }, "messages": { "success": "Sessão RAG criada com sucesso" }, - "newPrompt": { - "cancel": "Cancelar", - "description": "O template do prompt define como os fragmentos (partes dos documentos) e as mensagens do chat são integrados para gerar respostas. Personalize o prompt para ajustar o comportamento das suas sessões RAG.", - "error": "Falha ao criar o prompt", - "languageLabel": "Idioma (opcional)", - "nameLabel": "Nome do Prompt", - "nameRequired": "O nome do prompt é obrigatório", - "placeholdersInfo": "Use {chunks} para representar onde os fragmentos dos documentos recuperados serão inseridos, e {input} para a mensagem do usuário.", - "promptLabel": "Prompt", - "promptPlaceholder": "Aqui você pode modificar o prompt, por exemplo:\nCada mensagem do usuário é adicionada como {input}\nAs fontes são adicionadas como {chunks}", - "save": "Salvar", - "success": "Prompt criado com sucesso!", - "title": "Criar um novo prompt", - "unsavedChanges": "Você tem alterações não salvas. Tem certeza de que deseja cancelar?" - }, "paramsPanel": { "failedToLoad": "Erro ao carregar a sessão RAG", "failedToUpdate": "Erro ao atualizar os parâmetros RAG", @@ -264,48 +221,20 @@ "title": "Marcadores obrigatórios" }, "prompt": { - "collapse": "Recolher", - "createNewPrompt": "Criar novo prompt", - "defaultGenerationPrompt": "Prompt de Geração Padrão", - "defaultQAGenerationPrompt": "Prompt de Geração Q&A Padrão", - "description": "Selecione um modelo de prompt que define como o contexto recuperado e as mensagens do chat são combinados para gerar respostas.", - "descriptionToggle": "Descrição", - "expand": "Expandir", - "language": "Idioma", + "editor": { + "expand": "Abrir em um editor maior", + "missingPlaceholder": "O modelo ainda precisa de: {{placeholders}}", + "seedLanguage": "Idioma do modelo", + "startFrom": "Partir de", + "startFromHelp": "Substitui o modelo abaixo. O seu é mantido até você escolher um.", + "template": "Modelo", + "templatePlaceholder": "Cada mensagem do usuário chega como {input}\nOs trechos recuperados chegam como {chunks}" + }, "languages": { "en": "English", "es": "Español", "pt": "Português" - }, - "newPromptButton": "Novo prompt", - "openPrompts": "Abrir biblioteca de prompts", - "promptLabel": "Prompt", - "selectTemplate": "Selecionar modelo de prompt", - "selectTemplatePlaceholder": "ex. Prompt Padrão, Instrução Personalizada", - "selectedTemplate": "Modelo de prompt selecionado:" - }, - "promptView": { - "close": "Fechar", - "language": "Idioma", - "languageNone": "Sem idioma", - "languageNotAvailable": "Idioma não disponível", - "noContent": "Sem conteúdo de template", - "table": { - "actions": "Ações", - "choosePrompt": "Escolha ou personalize seu prompt para definir o comportamento do modelo.", - "created": "Criado", - "currentPrompts": "Prompts atuais", - "edited": "Editado", - "id": "ID", - "language": "Idioma", - "name": "Nome", - "newPrompt": "Novo prompt", - "type": "Tipo", - "viewPrompt": "Ver prompt" - }, - "templateContent": "Conteúdo do template", - "type": "Tipo", - "untitledPrompt": "Prompt" + } }, "retrieverConfig": { "modelLabel": "Modelo de recuperação" @@ -314,8 +243,6 @@ "notFound": "Sessão não encontrada" }, "setup": { - "selectDocuments": "Selecionar documentos", - "selectDocumentsDescription": "Faça upload de novos documentos ou selecione entre os existentes para usar no RAG.", "sessionName": "Nome da sessão *" }, "summary": { @@ -328,40 +255,9 @@ "sessionUpdated": "Sessão atualizada com sucesso" }, "validation": { - "modelComponentMissing": "Um componente do modelo não tem nome selecionado.", - "modelParamsIncomplete": "Os parâmetros de \"{{model}}\" estão incompletos. Reconfigure-o nas configurações avançadas.", "nameRequired": "O nome da sessão é obrigatório" } }, - "ragDocumentsPage": { - "contentPanel": { - "loadingContent": "Carregando conteúdo...", - "noContent": "Nenhum conteúdo extraído", - "noDocumentSelected": "Selecione um documento para ver seu conteúdo processado" - }, - "description": "Gerencie documentos para suas sessões RAG. Lembre-se de que o RAG usa apenas texto: avalie os extratores disponíveis e escolha o mais adequado para cada tipo de documento (por exemplo, use extratores com OCR para documentos digitalizados).", - "detailPanel": { - "changeExtractorConfirmBody_many": "Este documento já tem um extrator configurado. Deseja alterá-lo?", - "changeExtractorConfirmBody_one": "Este documento já tem um extrator configurado. Deseja alterá-lo?", - "changeExtractorConfirmBody_other": "Este documento já tem um extrator configurado. Deseja alterá-lo?", - "changeExtractorConfirmTitle": "Alterar extrator", - "contentPreview": "Pré-visualização do conteúdo", - "documentInfo": "Informações do documento", - "extracting": "Extraindo texto...", - "extractor": "Extrator", - "name": "Nome", - "noContent": "Nenhum conteúdo extraído ainda. Clique em \"Processar documento\" para extrair texto.", - "noDocumentSelected": "Selecione um documento para ver detalhes", - "processAndShowContent": "Processar documento e mostrar conteúdo", - "saveExtractor": "Salvar extrator", - "type": "Tipo" - }, - "title": "Documentos RAG" - }, - "ragPromptsPage": { - "description": "Gerencie os prompts para suas sessões RAG: veja todos os prompts disponíveis e crie novos para melhorar suas interações de IA.", - "title": "Prompts RAG" - }, "sourcesDisplay": { "chunk_many": "fragmentos", "chunk_one": "fragmento", diff --git a/DashAI/front/src/utils/i18n/locales/zh/common.json b/DashAI/front/src/utils/i18n/locales/zh/common.json index df958e9f0..d66d17b9e 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/common.json +++ b/DashAI/front/src/utils/i18n/locales/zh/common.json @@ -1,5 +1,6 @@ { "actions": "操作", + "copy": "复制", "holdout": "保留数据", "hub": "数据中心", "add": "添加", diff --git a/DashAI/front/src/utils/i18n/locales/zh/generative.json b/DashAI/front/src/utils/i18n/locales/zh/generative.json index aff10ace0..937864c9a 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/generative.json +++ b/DashAI/front/src/utils/i18n/locales/zh/generative.json @@ -13,14 +13,18 @@ }, "documentsBar": { "addDocuments": "添加文档", - "available": " 可用", - "detailedView": "查看详细视图", + "alreadyInSession": "“{{file}}”已在此会话中", + "delete": "从会话中移除", + "deleteWarning": "其提取的文本和分块也会一并删除。会话将在你的下一条消息时重新建立索引。", + "deleted": "已移除“{{file}}”", "documentCount_one": "{{count}} 个文档", "documentCount_other": "{{count}} 个文档", + "failedDelete": "无法移除该文档", "failedFetch": "获取文档失败", + "failedPreview": "无法加载该文件", "failedUpload": "上传文档失败", "inCurrentSession": " 在当前会话中", - "noDocumentsAvailable": "没有可用的文档", + "inspect": "文本提取", "noDocumentsFound": "未找到文档", "noDocumentsInSession": "此会话中没有文档", "searchPlaceholder": "搜索文档", @@ -39,6 +43,7 @@ "failedToFetchSessions": "获取会话列表失败", "failedToFetchTasks": "获取生成式任务失败", "failedToLoadModels": "加载模型失败", + "failedToSendMessage": "无法发送消息", "failedToUpdateSession": "更新会话失败", "modelSwitchFailed": "更改会话模型失败", "nameRequired": "名称为必填项", @@ -108,10 +113,8 @@ "retrieverTitle": "高级检索配置" }, "breadcrumbs": { - "documents": "文档", "generative": "生成式", "goBack": "返回", - "prompts": "提示词", "rag": "RAG", "sessionSuffix": "会话" }, @@ -148,12 +151,14 @@ "config": { "advanced": "高级配置", "chunkCount_other": "{{count}} 个块", - "contextBudget": "上下文预算" + "contextBudget": "上下文预算", + "discard": "放弃更改", + "unsavedIn": "以下部分有未保存的更改：{{sections}}" }, "create": { - "defaultsNotice": "以下设置默认应用，可在会话中修改：", + "newSession": "新建 RAG 会话", "selectModel": "语言模型", - "subtitle": "一个名称、你的文档和一个模型。其余设置都有合理的默认值，之后可以修改。", + "subtitle": "一个名称和一个模型。会话创建后再添加文档；其余设置都有合理的默认值，之后可以更改。", "title": "新建 RAG 会话" }, "documentPreview": { @@ -162,16 +167,6 @@ "title": "文档预览" }, "documents": { - "duplicate": { - "affectedSessions": "受影响的会话：", - "cancel": "取消", - "confirm": "更新", - "message": "此文件已上传。是否要更新它？", - "noAffectedSessions": "当前没有会话使用此文档。", - "title": "文件已存在", - "warning": "这将删除这些会话的已拟合模型（嵌入、检索器）。" - }, - "emptyUploadText": "上传您的文档", "extractorModal": { "close": "关闭", "explanation": "处理PDF文本有多种方式。每个提取器可能根据文档结构产生不同的结果——尝试不同选项并比较输出结果。重要提示：RAG仅使用文本。如果您的PDF包含带有文字的图像（扫描文档、图表），您必须使用支持OCR的提取器（如EasyOCR）将这些图像转换为RAG可以使用的文本。没有OCR，图像将被忽略。", @@ -185,25 +180,9 @@ "upToDate": "已就绪" }, "table": { - "actions": "操作", - "addDocument": "添加文档", "configureExtractor": "配置提取器", - "created": "创建时间", - "currentDocuments": "当前文档", - "delete": "删除", - "emptyUploadText": "上传您的文档", - "extractor": "提取器", - "id": "ID", - "lastModified": "最后修改", - "name": "名称", - "noDocumentsAvailable": "没有可用的文档。", - "preview": "预览", - "type": "类型", "unknownType": "未知" - }, - "uploadButton": "上传文档", - "uploadFailed": "无法上传该文档。", - "uploadFailedReason": "无法上传“{{file}}”：{{reason}}" + } }, "generator": { "advancedButton": "打开高级配置", @@ -214,39 +193,17 @@ "configureTitle": "配置生成器模型 (LLM)", "modelLabel": "生成器模型" }, - "home": { - "documents": "文档", - "documentsDescription": "上传文档并选择文本提取方式。", - "newSession": "新建 RAG 会话", - "newSessionDescription": "选择文档和模型，即可开始对话。", - "prompts": "提示词", - "promptsDescription": "管理会话可用的提示词模板。", - "subtitle": "与你自己的文档对话。", - "title": "检索增强生成" - }, "index": { "chunkCount_other": "{{count}} 个块", + "indexFailed": "索引失败", + "indexing": "正在建立索引…", "indexingInProgress": "正在为文档建立索引…", - "notIndexed": "未建立索引" + "notIndexed": "未建立索引", + "retryIndexing": "重试" }, "messages": { "success": "RAG 会话创建成功" }, - "newPrompt": { - "cancel": "取消", - "description": "提示词模板定义了如何集成文档块和聊天消息以生成回复。自定义提示词以调整您的 RAG 会话行为。", - "error": "创建提示词失败", - "languageLabel": "语言（可选）", - "nameLabel": "提示词名称", - "nameRequired": "提示词名称为必填项", - "placeholdersInfo": "使用 {chunks} 表示检索到的文档块将被插入的位置，使用 {input} 表示用户消息。", - "promptLabel": "提示词", - "promptPlaceholder": "您可以在此修改提示词，例如：\n每条用户消息添加为 {input}\n来源添加为 {chunks}", - "save": "保存", - "success": "提示词创建成功！", - "title": "创建新提示词", - "unsavedChanges": "您有未保存的更改。确定要取消吗？" - }, "paramsPanel": { "failedToLoad": "加载 RAG 会话失败", "failedToUpdate": "更新 RAG 参数失败", @@ -258,48 +215,20 @@ "title": "必填占位符" }, "prompt": { - "collapse": "收起", - "createNewPrompt": "创建新提示词", - "defaultGenerationPrompt": "默认生成提示词", - "defaultQAGenerationPrompt": "默认问答生成提示词", - "description": "选择一种提示词模板，定义如何组合检索到的上下文和聊天消息以生成回复。", - "descriptionToggle": "描述", - "expand": "展开", - "language": "语言", + "editor": { + "expand": "在更大的编辑器中打开", + "missingPlaceholder": "模板还缺少：{{placeholders}}", + "seedLanguage": "模板语言", + "startFrom": "以此为起点", + "startFromHelp": "会替换下方的模板。在你选择之前，你的内容会保留。", + "template": "模板", + "templatePlaceholder": "每条用户消息以 {input} 传入\n检索到的片段以 {chunks} 传入" + }, "languages": { "en": "English", "es": "Español", "pt": "Português" - }, - "newPromptButton": "新建提示词", - "openPrompts": "打开提示词库", - "promptLabel": "提示词", - "selectTemplate": "选择提示词模板", - "selectTemplatePlaceholder": "例如：默认提示词、自定义指令", - "selectedTemplate": "已选择的提示词模板：" - }, - "promptView": { - "close": "关闭", - "language": "语言", - "languageNone": "无语言", - "languageNotAvailable": "语言不可用", - "noContent": "无模板内容", - "table": { - "actions": "操作", - "choosePrompt": "选择或自定义您的提示词以定义模型的行为。", - "created": "创建时间", - "currentPrompts": "当前提示词", - "edited": "编辑时间", - "id": "ID", - "language": "语言", - "name": "名称", - "newPrompt": "新建提示词", - "type": "类型", - "viewPrompt": "查看提示词" - }, - "templateContent": "模板内容", - "type": "类型", - "untitledPrompt": "提示词" + } }, "retrieverConfig": { "modelLabel": "检索模型" @@ -308,8 +237,6 @@ "notFound": "未找到会话" }, "setup": { - "selectDocuments": "选择文档", - "selectDocumentsDescription": "上传新文档或从现有文档中选择以用于 RAG。", "sessionName": "会话名称 *" }, "summary": { @@ -322,38 +249,9 @@ "sessionUpdated": "会话更新成功" }, "validation": { - "modelComponentMissing": "模型组件没有选择名称。", - "modelParamsIncomplete": "\"{{model}}\" 的参数不完整。请在高级设置中重新配置。", "nameRequired": "会话名称为必填项" } }, - "ragDocumentsPage": { - "contentPanel": { - "loadingContent": "正在加载内容...", - "noContent": "未提取到内容", - "noDocumentSelected": "选择一个文档以查看其处理后的内容" - }, - "description": "管理 RAG 会话的文档。请注意，RAG 仅使用文本：评估可用的提取器并为每种文档类型选择最合适的提取器（例如，对扫描文档使用支持 OCR 的提取器）。", - "detailPanel": { - "changeExtractorConfirmBody_other": "此文档已配置提取器。是否要更改它？", - "changeExtractorConfirmTitle": "更改提取器", - "contentPreview": "内容预览", - "documentInfo": "文档信息", - "extracting": "正在提取文本...", - "extractor": "提取器", - "name": "名称", - "noContent": "尚未提取内容。请点击\"处理文档\"以提取文本。", - "noDocumentSelected": "选择一个文档以查看详情", - "processAndShowContent": "处理文档并显示内容", - "saveExtractor": "保存提取器", - "type": "类型" - }, - "title": "RAG 文档" - }, - "ragPromptsPage": { - "description": "管理 RAG 会话的提示词：查看所有可用的提示词并创建新的，以提升您的 AI 交互体验。", - "title": "RAG 提示词" - }, "sourcesDisplay": { "chunk_one": "个块", "chunk_other": "个块", diff --git a/DashAI/front/src/utils/ragValidation.js b/DashAI/front/src/utils/ragValidation.js deleted file mode 100644 index 293f9ce5d..000000000 --- a/DashAI/front/src/utils/ragValidation.js +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Validates that a RAG model configuration has complete parameters. - * A model config is `{ component: string, params: object }`. - * - * Checks: - * - component is a non-empty string - * - params is a non-empty object (at least 1 key) - * - Recursively checks sub-components in: - * - Canonical format: `{ component: "X", params: {...} }` - * - Properties wrapper format: `{ properties: { component, params: { comp: { component, params } } } }` - * - Arrays of sub-configs (composite retriever `children`) - * - Guards against null params - * - * @param {object} model - The model config { component, params }. - * @param {Function} t - i18n translate function. - * @returns {{ valid: boolean, errors: string[] }} Validation result with translated error messages. - */ -export function validateModelConfig(model, t) { - const errors = []; - - if ( - !model || - !model.component || - typeof model.component !== "string" || - model.component.trim() === "" - ) { - errors.push(t("generative:rag.validation.modelComponentMissing")); - return { valid: false, errors }; - } - - const params = model.params; - if ( - !params || - typeof params !== "object" || - Array.isArray(params) || - Object.keys(params).length === 0 - ) { - errors.push( - t("generative:rag.validation.modelParamsIncomplete", { - model: model.component, - }), - ); - return { valid: false, errors }; - } - - _checkSubComponents(params, t, errors); - - return { valid: errors.length === 0, errors }; -} - -/** - * Recursively inspect sub-component values for empty params or - * empty children arrays in composite retrievers. - * - * @param {object} value - The params dict or sub-config value to inspect. - * @param {Function} t - i18n translate function. - * @param {string[]} errors - Accumulator for translated error messages. - */ -function _checkSubComponents(value, t, errors) { - if (!value || typeof value !== "object") return; - - for (const [key, v] of Object.entries(value)) { - // Skip scalar values and nulls - if (!v || typeof v !== "object") continue; - - // ── Arrays (e.g. composite retriever children) ── - if (Array.isArray(v)) { - for (let i = 0; i < v.length; i++) { - const item = v[i]; - if (item && typeof item === "object") { - if (item.component && _isEmptyParams(item.params)) { - errors.push( - t("generative:rag.validation.modelParamsIncomplete", { - model: item.component, - }), - ); - } else if (item.component) { - // Recurse into child's own params - if ( - item.params && - typeof item.params === "object" && - !Array.isArray(item.params) - ) { - _checkSubComponents(item.params, t, errors); - } - } - } - } - continue; - } - - // ── Properties wrapper format ── - if (v.properties) { - const comp = v.properties?.params?.comp; - if (comp && comp.component && comp.params !== undefined) { - if (_isEmptyParams(comp.params)) { - errors.push( - t("generative:rag.validation.modelParamsIncomplete", { - model: comp.component, - }), - ); - } else if ( - comp.params && - typeof comp.params === "object" && - !Array.isArray(comp.params) - ) { - _checkSubComponents(comp.params, t, errors); - } - } - continue; - } - - // ── Canonical format { component, params } ── - if (v.component) { - if (_isEmptyParams(v.params)) { - errors.push( - t("generative:rag.validation.modelParamsIncomplete", { - model: v.component, - }), - ); - } else if ( - v.params && - typeof v.params === "object" && - !Array.isArray(v.params) - ) { - _checkSubComponents(v.params, t, errors); - } - continue; - } - - // ── Nested plain object — recurse ── - if (!Array.isArray(v)) { - _checkSubComponents(v, t, errors); - } - } -} - -/** - * Returns true when `p` is missing, null, or an empty object. - * Arrays are considered populated (empty arrays are valid configs). - */ -function _isEmptyParams(p) { - if (p === null || p === undefined) return true; - if (typeof p !== "object") return true; - if (Array.isArray(p)) return false; - return Object.keys(p).length === 0; -} diff --git a/docs/RAG/01-overview.md b/docs/RAG/01-overview.md index a9bcff688..b064618a1 100644 --- a/docs/RAG/01-overview.md +++ b/docs/RAG/01-overview.md @@ -8,7 +8,7 @@ | Jobs | `DashAI/back/job/RAG_job.py` | | Task | `DashAI/back/tasks/RAG_task.py` | | Core | `DashAI/back/core/component_validation.py` | -| Frontend | `DashAI/front/src/pages/generative/RAGSession/` | +| Frontend pages | `DashAI/front/src/pages/generative/RAG/` (entry point, creation) and `pages/generative/RAGSession/` (the session view) | | Frontend components | `DashAI/front/src/components/generative/RAG/` | ## Quick Architecture diff --git a/docs/RAG/02-backend-architecture.md b/docs/RAG/02-backend-architecture.md index 0d4017793..f3016d73d 100644 --- a/docs/RAG/02-backend-architecture.md +++ b/docs/RAG/02-backend-architecture.md @@ -21,7 +21,9 @@ RAGSessionValidationService (services/RAG/) ▼ Services (services/RAG/) │ - ├── SetupService — pipeline assembly (build_pipeline) + ├── SetupService — indexing (build_index) + pipeline assembly + ├── IndexStatusService — read-only view of a session's index state + ├── IndexJobService — resolving / cancelling a live indexing job ├── DocumentService — document CRUD, file I/O, extractors, hydration ├── ChunkingService — chunk set identity (SHA-256), chunking lifecycle ├── PromptService — prompt CRUD, get_or_create via parameters_hash @@ -68,11 +70,16 @@ registered in the DI container — they are instantiated per request/job. Centralized validation for RAG session creation (POST) and parameter updates (PUT). Two public methods with shared private helpers: -- `prepare_RAG_params()` — POST validation. All model keys (`prompt`/`prompt_id`, - `chunking_model`, `retriever_model`, `generation_model`) and `documents` are - required. Resolves `prompt_id` → `prompt` **before** structural validation. +- `prepare_RAG_params()` — POST validation. Only `generation_model` is + required; `prompt`, `chunking_model` and `retriever_model` are filled from + backend defaults when omitted. Resolves `prompt_id` → `prompt` **before** + structural validation. `documents` always starts empty and is *rejected* if + sent: there is no session to attach documents to yet, and saying so beats + silently dropping the list. - `validate_update_payload()` — PUT validation. Only validates keys present in - the partial payload; documents are optional. + the partial payload, and rejects `documents` outright — the foreign key on + `document` is the authority, and the document endpoints keep the session's + list in step with it. Both methods use `_validate_component_params()` which **recursively** validates every `{component, params}` reference (including nested sub-components like @@ -83,16 +90,33 @@ every `{component, params}` reference (including nested sub-components like ### SetupService -(Formerly `RAGSetupService`.) Single responsibility: `build_pipeline()`. -Assembles the complete RAG pipeline returning a `RAGPipeline` instance. -Sequence: documents → chunk set → chunking → retriever → LLM → prompt. +(Formerly `RAGSetupService`.) Two entry points, one code path: + +- `build_index()` — documents → chunk set → chunking → retriever. Everything + that makes documents retrievable, returned as an `IndexResult`. Takes an + optional `progress(fraction, message)` callback so a job can report progress + without this service knowing the job system exists. +- `build_pipeline()` — calls `build_index()`, then adds LLM → prompt and + returns a `RAGPipeline` instance. + +The split exists because `LLMService.get_or_create` *instantiates* the +generation model: indexing that went through `build_pipeline` would load LLM +weights it never uses. `build_pipeline` delegates rather than repeating the +steps, so the two paths cannot drift. + No validation logic — that lives in `RAGSessionValidationService`. ### DocumentService CRUD for documents + file storage + document hydration (replaces `DocumentLoader`). -Key methods: `upload()`, `load()`, `validate_exist()`, `get_by_session()`. +Key methods: `upload()` (session-scoped), `load()`, `validate_exist()`, +`validate_belong_to_session()`, `get_by_session()`, `delete()`, +`delete_by_session()`, `extract_text()`, `update_extractor()`. + +Documents belong to exactly one session; see +[`06-document-processing.md`](./06-document-processing.md) for ownership, the +content-addressed file layout, and the extractor lifecycle. File type mapping uses `DocumentFileType` enum from `models/RAG/documents/file_type.py` for single-source-of-truth strings. @@ -139,6 +163,31 @@ recursive child setup. Cascade deletion of RAG resources when a session is deleted or parameters change. Retriever cleanup BEFORE chunking cleanup (critical ordering). +`invalidate_document_artifacts()` takes `commit=False` and `defer_paths` so a +caller can fold it into a larger transaction — `update_extractor()` needs the +extractor reassignment and the re-extraction to succeed or fail together, and +`rmtree` cannot be rolled back, so the paths are deleted only after the +caller's commit. + +`_other_sessions_with_same_config()` is gone, but the protection it was meant +to provide is not — it is just keyed on the right thing now. That function +compared `documents`, which per-session ownership makes unique, so it could +only ever return `False`; and even before that it was the wrong question, since +it also blocked cleanup for two sessions that merely shared a configuration. + +Which rows are actually shared decides the guard: + +- **Per chunk set** — retrievers, embedding matrices, chunks. A chunk set + belongs to one session now, so nothing else can be using them and they are + deleted outright. +- **Per configuration** — `rag_chunking_model` and `rag_embedding_model` are + matched by `(class_name, parameters)` alone, so every session that settled on + the same components shares one row. A new session takes the backend defaults, + which makes sharing the ordinary case rather than a corner one. Each is + deleted only once nothing references it: no other session's `rag_pipeline` + for the chunking model, and no dense retriever or embedding matrix for the + embedding model. + ## Pure Factories (no DB or FS) After the refactor, all four sub-factories are **pure** — they only construct @@ -206,7 +255,7 @@ Two functions that work for ANY parameter structure, not just RAG: Delegated to `RAGSessionValidationService.prepare_RAG_params()`: 1. Model and task are checked against the component registry. -2. Documents must be non-empty and all IDs must exist in the DB. +2. `documents` is forced to `[]`; sending a non-empty list is an error. 3. Parameters are normalized via `normalize_payload()`. 4. If `prompt_id` is present, resolved to a `prompt` component ref **before** structural validation. @@ -228,7 +277,7 @@ Delegated entirely to `RAGSessionValidationService.validate_update_payload()`: 3. Validate structure of each sent component ref (`component` + `params` keys). 4. **Recursive schema validation** of every present component ref. 5. `validate_component_refs()` — validate all components exist in registry. -6. Validate documents (if sent): must be non-empty + all IDs exist in DB. +6. Reject `documents` if sent (managed by the document endpoints). 7. Returns validated dict. Then the endpoint merges with old params and calls `CleanupService.cleanup_orphaned_resources()`. @@ -284,6 +333,39 @@ setup_service = SetupService(db, component_registry, config["RAG_PATH"]) model = setup_service.build_pipeline(pipeline_config) ``` +Since indexing is content-addressed and idempotent, this is a cache hit for a +session that was already indexed — and remains the fallback for one that was +not. + +## RAGIndexJob + +Indexing runs up front rather than as a side effect of the first message, so a +user who has just uploaded a document is not paying for it on their first +question. + +```python +POST /api/v1/rag/sessions/{id}/index → enqueues RAGIndexJob(session_id=...) +GET /api/v1/rag/sessions/{id}/index-status +``` + +The endpoint is idempotent and coalescing: no documents, already indexed, and +already indexing all return the current state without enqueueing. Callers +therefore fire it after *any* change rather than deciding for themselves which +settings invalidate the index — the chunk-set signature already owns that rule. + +`GenerativeSession.index_job_id` points at the run. It is only a pointer: the +queue's `task_copy` table stays authoritative for whether that job is alive, so +a stale id resolves to nothing and is overwritten by the next request. The +queue's watchdog flips a dead job to `killed` within ~10s, so nothing gets +stuck. + +Four write paths cancel a live run *before* mutating, because +`CleanupService` would otherwise delete the very rows the job is writing: +`PUT /generative-session/{id}/parameters`, `DELETE /document/{id}`, +`PUT /document/{id}/extractor`, and `DELETE /generative-session/{id}`. Upload +does not — it only appends, and the running job's output stays valid for the +signature it is working on. + ## RAGPipelineConfig (unchanged) Validated parameter dataclass. Used by `RAGSetupService` and `RAGJob`. @@ -407,9 +489,11 @@ a `_default_extract()` fallback. Endpoints: -- `POST /api/v1/document/{id}/extract` — on-demand extraction (does not persist) -- `PUT /api/v1/document/{id}/extractor` — commit extractor choice (with - force option to invalidate linked pipeline artifacts) +- `POST /api/v1/document/{id}/extract` — on-demand extraction (`persist=false` + for preview) +- `PUT /api/v1/document/{id}/extractor` — commit extractor choice. One + transaction, extraction first, artifacts invalidated unconditionally; `422` + when extraction fails, having changed nothing. ## Parameters Hash diff --git a/docs/RAG/03-frontend-architecture.md b/docs/RAG/03-frontend-architecture.md index ef0d7f861..f18105ada 100644 --- a/docs/RAG/03-frontend-architecture.md +++ b/docs/RAG/03-frontend-architecture.md @@ -2,121 +2,162 @@ ## Routes -| Path | Component | Purpose | -| ------------------------------ | ---------------- | ----------------------------- | -| `/app/generative` | `SessionRouter` | Routes to RAG or non-RAG view | -| `/app/generative/sessions/:id` | `RAGSessionPage` | RAG session detail + chat | - -## Main RAG Page Flow - -All files under `pages/generative/RAGSession/`: - -1. **`RAGSessionSetup.jsx`** — Session creation form with accordion sections: - - Document selection, chunking config, retriever config (3-preset card - system), prompt template selection, generator (LLM) config. - - Uses `RAGCard`, `SectionCard`, and `RAGSectionColumn` layout components. - -2. **`RAGSessionPage.jsx`** — 3-panel orchestrator (uses `ThreePanelLayout`): - - Left: session list + `RAGDocumentsPanel` (document manager) - - Center: `RAGSessionSetup` form / `RAGSessionSummary` view / `GenerativeChat` view - - Right: `RAGInfoBar` (educational) / `RAGParamsPanel` (parameter editing) - -3. **`GenerativeChat`** (`components/generative/GenerativeChat.jsx`) — Active chat - view shared with non-RAG sessions. - -### Per-Stage Config Sections - -Each pipeline stage has a section component in `sections/`: - -- `ChunkingSection.jsx` -- `RetrieverSection.jsx` -- `GeneratorSection.jsx` -- `PromptSection.jsx` - -Each section renders inside a `SectionCard` layout wrapper using `RAGSectionColumn`. - -### Page-Level Shared Components - -In `pages/generative/RAGSession/components/`: - -- `RAGCard.jsx` — Accordion-based card with expand/collapse and step indicators -- `SectionCard.jsx` — Flexbox layout wrapper for section content -- `RAGSectionColumn.jsx` — Vertical column layout for stacked sections -- `PresetCard.jsx` — Clickable preset selection card (Keyword/Semantic/Hybrid) -- `GeneratorBody.jsx` — Generator configuration content -- `AdvancedConfigCard.jsx` — Card with navigate-to-advanced-modal button -- `sectionUtils.jsx` — Utility functions (`getDescription`, `renderTemplateWithHighlights`) - -### Advanced Configuration Modals - -In `advanced/` (9 files): - -- `CompositeRetrieverBuilder.jsx` — Visual builder for composite retriever trees -- `RetrieverConfigurationStep.jsx` — Step within composite builder -- `RetrieverAdvancedModal.jsx` — Advanced retriever settings dialog -- `RetrieverNodeConfig.jsx` — Configuration panel for individual retriever nodes -- `ChunkingConfigurationStep.jsx` — Step-level chunking config -- `ChunkingAdvancedModal.jsx` — Advanced chunking settings dialog -- `GeneratorConfigurationStep.jsx` — Step-level generator config -- `GeneratorAdvancedModal.jsx` — Advanced generator settings dialog -- `NewPromptModal.jsx` — Custom prompt creation dialog - -### Supporting Components - -In `components/generative/RAG/`: - -- **Session & summary:** `RAGSessionSummary.jsx`, `RAGBreadcrumbs.jsx` -- **Info & params:** `RAGInfoBar.jsx`, `RAGParamsPanel.jsx` -- **Documents:** `DocumentSelector.jsx`, `DocumentList.jsx`, `DocumentListItem.jsx`, - `DocumentPreviewModal.jsx`, `DocumentsBar.jsx`, `DocumentTable.jsx`, - `RAGDocumentsPanel.jsx`, `DocumentDetailPanel.jsx` -- **Generator:** `GeneratorParamsCard.jsx` -- **Prompts:** `PromptParamsCard.jsx`, `PromptSelectionTable.jsx`, - `PromptViewModal.jsx`, `PlaceholdersList.jsx` -- **Utilities:** `HighlightedTextarea.jsx`, `ragValidation.js` - -A `setup/` directory exists with empty `sections/`, `components/`, and `advanced/` -subdirectories, reserved for a future setup-component refactor. - -## Key Features - -- **Retriever Presets** — 3-card system: Keyword (BM25), Semantic (Dense), - Hybrid (Sequential BM25 + Dense). -- **Retriever tree view** — `CompositeRetrieverBuilder` renders a tree with - vertical spine + horizontal connectors per child. Operation cards (reranking, - chunk fusion) appear as final clickable nodes with per-type summaries - (MMR: lambda + top_k, CrossEncoder: model_name, Parallel: merge strategy). - All nodes and operation cards are clickable to open `RetrieverNodeConfig`. -- **Document Selection UI** — Full document table with search, selection, - preview modal, multi-select, and collapsible `DocumentDetailPanel` with - extractor selector and schema-driven form. -- **Pre-save validation** — `RAGSessionSetup.validateConfiguration()` recursively - checks all `{component, params}` refs for completeness before saving the - session, showing snackbar warnings and blocking the save. -- **Error propagation** — `resolveDefaults` throwOnError option propagates API - failures instead of silently returning `{}`; `RetrieverSection` shows an - error state instead of building presets with incomplete configs. -- **Context Window Validation** — Validates that - `chunk_size * top_k + prompt_tokens <= context_window`. -- **Multi-Language Prompts** — Templates in en/es/pt/de/zh, selected via - dropdown. -- **Template Highlighting** — `renderTemplateWithHighlights()` renders - `{placeholders}` with colored backgrounds for visual clarity. -- **Translation Keys** — All RAG translations use the `generative:rag.*` - namespace. - -## API Layer - -All RAG API calls use standard DashAI endpoints: +RAG is a standalone entry point of the generative module, not a step inside +generic session creation. All its routes are declared in +`DashAI/front/src/App.jsx` and wrapped in a `RAGScope`, which provides a +`GenerativeProvider` filtered to `RAGTask` so the shared session list stays +separate. Route matching is case-insensitive, so older `/RAG/...` links keep +working. + +| Path | Component | Purpose | +| ---------------------------------- | ---------------- | ------------------------------- | +| `/app/generative/rag` | `RAGCreatePage` | Create a session (name + model) | +| `/app/generative/rag/sessions/:id` | `RAGSessionPage` | Documents, chat, configuration | +| `/app/generative/rag/new` | redirect | → `/app/generative/rag` | +| `/app/generative/rag/documents` | redirect | → `/app/generative/rag` | +| `/app/generative/rag/prompts` | redirect | → `/app/generative/rag` | + +**The entry point is the creation form.** Picking RAG in the hub used to land on +a menu whose only remaining card was "new session" — a leftover from when +documents and prompts sat beside it — so starting a session took two clicks. +Existing sessions are listed in the left panel of that same screen. + +`/app/generative/sessions/:id` is served by `SessionRouter`, which redirects a +`RAGTask` session to its own route. The map from a standalone task to its route +lives in `components/generative/standaloneEntryPoints.js`; the backend decides +*which* tasks are standalone, via each task's `metadata.entry_point`. + +The redirects exist because there is no catch-all route: without them a bookmark +of `/rag/new` or of the removed documents and prompts pages would render a blank +page. + +## The session view + +`pages/generative/RAGSession/RAGSessionPage.jsx` is a three-panel layout: + +``` +LeftPanel + GenerativeHubHeader ← 64px, above the split + DocumentsBar ← flex 1 1 55% + SessionBar showHeader={false} ← flex 1 1 45% +CenterPanel + RAGBreadcrumbs ← page chrome, px:4 pt:4 + GenerativeChat +RightPanel + RAGConfigPanel +``` + +Two things about this shape are deliberate: + +- **The header sits above the split.** It used to live inside `SessionBar`, + which RAG mounted in the lower 40% of the column inside an `overflow: auto` + box — so the way back to the hub rendered half-way down and scrolled out of + sight. `GenerativeHubHeader` is now its own component, `SessionBar` takes + `showHeader={false}` here, and both halves may shrink (`1 1 X%`, not `0 0 X%`) + rather than forcing an outer scroll. +- **The chat is the only centre content.** Opening a session lands straight in + the conversation, and adjusting retrieval or the model never takes it off + screen. Reading a document happens in a modal for the same reason. + +`RAGBreadcrumbs` is rendered by the page. It used to be rendered by +`GenerativeChat`, which is shared with every generative task and so carried a +`taskName === "RAGTask"` check — and drew the trail inside its own centred +column, lower than the same trail on every other RAG screen. + +## The configuration panel + +`components/generative/RAG/RAGConfigPanel.jsx`, fed by +`GET /v1/rag/sessions/{id}/configuration` (typed `IRAGConfiguration`). + +``` +Fixed header: session name (editable) · stale-index alert · PillTabs +Body: the active tab — summary line, info tooltip, content +Fixed footer: context budget · "unsaved changes in …" · Discard · Save +``` + +- The four sections are tabs, in pipeline order: chunking, retrieval, model, + prompt. Labels come from `configuration[key].section_name`, already localized + by the backend, so the tabs need no translation keys of their own. +- `PillTabs` is used `variant="scrollable"`, never `fullWidth`: the panel can be + 15% of the viewport and the labels are backend-supplied, so a fixed-width row + would wrap. +- **Every tab body stays mounted**, hidden rather than unrendered. + `GeneratorPicker` reports whether its model can actually run through a + callback, so a tab the user never opened would leave Save enabled for a model + that cannot answer. +- One Save sends the whole draft, because + `PUT /generative-session/{id}/parameters` replaces every parameter at once. + Since tabs hide pending edits, each edited tab gets a dot, a line above Save + names them, and there is a Discard button. + +`PresetCardList` renders the chunking and retrieval presets as cards in +`ComponentSelector`'s idiom — flat `Paper`, primary border when active, a tick. +It does not reuse that component: the search field, category chips, download +controls and viewport-breakpoint grid it also brings do not apply to a preset +recipe, and two columns are unreadable at this width. + +## The prompt + +`components/generative/RAG/PromptEditor.jsx` edits the session's own template. +There is no shared prompt library: `rag_prompt` rows are deduplicated by a hash +of their parameters, so two sessions that chose the same template shared one +row, and editing it rewrote the other session's prompt. + +The registry's built-in templates (`getDefaultPrompts`) remain, but only to +*seed* the template, and seeding is an explicit choice — the language select +used to overwrite whatever the user had written as a side effect. Nothing is +sent while typing; the panel's Save writes the whole draft. + +`HighlightedTextarea`, `PlaceholdersList` and `renderTemplateWithHighlights` are +reused unchanged. A template missing `{chunks}` or `{input}` marks its tab and +blocks Save. + +## Creating a session + +`pages/generative/RAG/RAGCreatePage.jsx` is what `/app/generative/rag` renders, +and it asks for a name and a model. Documents are uploaded into the session once +it exists, and the other three components come from backend defaults +(`GET /v1/rag/session-defaults` seeds them server-side) that the session view +can change. + +"Back" on this page leaves RAG for the generative hub, because this page is the +RAG root — there is no longer a menu above it to return to. + +## Advanced configuration + +In `pages/generative/RAGSession/advanced/`: + +- `ChunkingAdvancedModal` / `ChunkingConfigurationStep` +- `RetrieverAdvancedModal` / `RetrieverConfigurationStep` +- `GeneratorAdvancedModal` / `GeneratorConfigurationStep` +- `CompositeRetrieverBuilder` / `RetrieverNodeConfig` — the composite retriever + tree, with a vertical spine and clickable operation nodes. + +## API layer | Endpoint | Purpose | | ------------------------------------------------------- | --------------------------- | | `/api/v1/generative-session/` | Session CRUD | -| `/api/v1/generative-process/` | Process CRUD | +| `/api/v1/generative-session/{id}/parameters` | Configuration (whole-set) | +| `/api/v1/generative-process/` | A chat turn | | `/api/v1/job/` | Job dispatch | -| `/api/v1/document/` | Document management | +| `/api/v1/document/session/{id}` | A session's documents | | `/api/v1/document/{id}/view` | Document preview (inline) | | `/api/v1/document/{id}/extract` | On-demand extraction | -| `/api/v1/document/{id}/extractor` | Update extractor assignment | -| `/api/v1/prompt/` | Prompt management | +| `/api/v1/document/{id}/extractor` | Commit extractor choice | +| `/api/v1/rag/sessions/{id}/configuration` | Resolved configuration | +| `/api/v1/rag/sessions/{id}/index-status` | Whether documents are indexed | +| `/api/v1/rag/{chunking,retriever}-presets` | Preset recipes | | `/api/v1/component/{name}/children/?include_flags=true` | Child components with flags | + +## Tests + +`yarn test`, using `src/test-utils/renderWithProviders.jsx` — which supplies the +real theme, needed because `PillTabs` reads `theme.palette.ui.box` and a bare +`createTheme()` does not have it. + +- `RAGConfigPanel.test.jsx` — tabs, the dirty/Discard model, that one Save + carries every section, and that all sections stay mounted. +- `PromptEditor.test.jsx` — placeholder validation, that edits are written back + as a self-contained component ref, and that changing the language leaves the + template alone. diff --git a/docs/RAG/04-execution-flow.md b/docs/RAG/04-execution-flow.md index 41b17fb97..eaaad2d6b 100644 --- a/docs/RAG/04-execution-flow.md +++ b/docs/RAG/04-execution-flow.md @@ -2,25 +2,19 @@ ## Step-by-Step -### 1. Session Configuration +### 1. Session Creation -The user fills the `RAGSessionSetup` form: +The user gives the session a name and picks an LLM (`RAGCreatePage`). Nothing +else is asked: chunking, retrieval and the prompt come from backend defaults, +and documents are uploaded into the session once it exists. -- Selects documents from the repository. -- Configures the chunking model (type, chunk size, overlap). -- Chooses a retriever (preset or custom composite). -- Selects a prompt template (language, optional custom template). -- Picks an LLM (model name, parameters like temperature, context window). - -### 2. Session Creation - -Frontend calls `POST /api/v1/generative-session/` with the full parameter -payload. The endpoint delegates to `RAGSessionValidationService`: +Frontend calls `POST /api/v1/generative-session/`. The endpoint delegates to +`RAGSessionValidationService`: 1. **Model & task validation** — checks `model_name` and `task_name` exist in the component registry. -2. **Document validation** — ensures documents list is non-empty and all IDs - exist in the database. +2. **Documents** — forced to `[]`. A non-empty list is rejected: there is no + session id to attach documents to yet. 3. **Parameter normalization** — `normalize_payload()` transforms frontend-style property wrappers. 4. **Prompt resolution** — if `prompt_id` is provided, it is resolved to a @@ -40,9 +34,52 @@ On success, a `GenerativeSession` record is persisted with: - `task_name` — Set to `"RAGTask"` for RAG sessions. - `model_name` — Set to `"RAGPipeline"`. -- `parameters` — The validated configuration dict. +- `parameters` — The validated configuration dict, with `documents: []`. + +### 2. Adding Documents + +The user uploads documents into the session from its left panel, which calls +`POST /api/v1/document/session/{session_id}`. Each upload appends the new id to +the session's `parameters["documents"]` and historizes the change, so the +pipeline, the chunk-set signature and the index status all see it. + +The user may also adjust the configuration at any point, through +`PUT /api/v1/generative-session/{id}/parameters`. That replaces every +parameter at once, and `CleanupService.cleanup_orphaned_resources` drops +whatever the previous configuration had fitted. ### 3. Process Creation When the user sends a message, the frontend calls -`POST /api/v1/generative-process/` with the input text. This creates a +`POST /api/v1/generative-process/` with the input text. The endpoint refuses a +RAG session that still has no documents — otherwise the job would fail deep +inside the retriever, fitting an index over no text. Otherwise this creates a +`GenerativeProcess` row, and the frontend then dispatches a job through +`POST /api/v1/job/`. + +### 4. Job Execution + +Huey picks up the job. `GenerativeJob` sees the session uses `RAGTask` and +delegates to `RAGJob`, which: + +1. Builds a `RAGPipelineConfig` from `session.parameters` alone — the filter + `_RAG_PARAM_KEYS` decides what reaches the pipeline, which is why + `parameters["documents"]` has to be kept in step with the foreign key. +2. Calls `SetupService.build_pipeline()`: documents → chunk set → chunking → + retriever → LLM → prompt. Chunk sets and fitted retrievers are reused when + their signature already exists, so only a changed configuration pays to + re-index. +3. Runs `RAGPipeline.generate()` and hands the output to + `RAGTask.process_output()`, which serializes the answer and its chunk + references. + +### 5. Displaying the Answer + +The frontend polls `GET /api/v1/jobs/{job_id}`. When the job is delivered, the +chat renders the message plus its `referenceOutput`, and `SourcesDisplay` / +`DocumentReferencesModal` show which passages were retrieved. + +`GET /api/v1/rag/sessions/{id}/index-status` reports whether the documents are +indexed for the *current* configuration, distinguishing `no_documents`, +`not_indexed`, `stale` (indexed before, under a different configuration) and +`indexed`. The message it returns is already localized. diff --git a/docs/RAG/05-known-limitations.md b/docs/RAG/05-known-limitations.md index af19c4273..681a25f4e 100644 --- a/docs/RAG/05-known-limitations.md +++ b/docs/RAG/05-known-limitations.md @@ -6,6 +6,13 @@ documents but will not scale to millions. No out-of-core or approximate indexing. +- **Chunk sets are not shared between sessions.** Documents belong to exactly + one session, so `RAGChunkSet.signature` — which hashes the document ids — + always differs. Two sessions configured identically over the same file each + chunk it and fit their own retriever. This is the accepted cost of session + isolation; the bytes on disk are still shared, since files are stored + content-addressed. + - **No streaming.** The frontend waits for the full LLM response before displaying it. Streaming support is not implemented. @@ -26,6 +33,13 @@ lock or upsert. Safe for single-user usage but could create duplicate chunk sets under concurrent requests. +- **SQLite foreign keys are not enforced.** The engine is created without + `PRAGMA foreign_keys=ON` and nothing sets it per connection + (`dependencies/database/sqlite_database.py`), so every `ondelete` clause in + the models is documentation rather than behaviour. Cascading deletes have to + come from SQLAlchemy relationships — which is why, for example, + `GenerativeSession.documents` carries `cascade="all, delete-orphan"`. + - **Huey consumer runs in-process.** In dev mode it spawns as a subprocess; in PyInstaller bundles it runs as a daemon thread. Both models limit parallelism. @@ -86,15 +100,35 @@ ## Extractors - **PypdfExtractor** default `strict=True` rejects malformed PDFs (xref - errors). Use `strict=False` for broken PDFs. + errors). Use `strict=False` for broken PDFs. Committing it for a PDF it + cannot read now fails with `422` and changes nothing, rather than leaving the + document pointing at it. - **EasyOCRExtractor** requires the `easyocr` dependency (heavy, downloads models on first use). ## Validation -- **Strict validation** requires ALL sub-component params to be complete at - session creation. The frontend must send full configs; the backend never - fills defaults (except prompt templates for default prompts). - **Parameters hash** only covers `rag_prompt` and `rag_generation_model`. `rag_chunking_model`, `rag_embedding_model`, and retriever tables still compare JSON columns directly — fragile to key ordering in SQLite. +- **Orphaned `rag_prompt` rows are not cleaned up.** + `CleanupService.cleanup_orphaned_resources` handles retrievers and chunking + models only. Rows are deduplicated by `parameters_hash` and the only writer + is `get_or_create`, so orphans are bounded and harmless — but they do + accumulate. +- **`RAGPrompt.pipelines` no longer cascades**, because a hash-deduplicated row + is shared: deleting one prompt would have deleted other sessions' + `rag_pipeline` rows. There is no DELETE endpoint for prompts; do not add one + without re-auditing that. + +## Empty sessions + +A RAG session is created with no documents, so several things have to cope with +an empty `documents` list: + +- `IndexStatusService` reports a `no_documents` status, rather than promising an + indexing run that cannot happen. +- `POST /generative-process/` refuses a chat turn on an empty RAG session. That + guard is load-bearing: without it `SetupService.build_pipeline` would create a + chunk set over zero documents and fit a retriever on nothing, failing opaquely + inside the vectorizer. diff --git a/docs/RAG/06-document-processing.md b/docs/RAG/06-document-processing.md index 66bcd1687..90b9fab23 100644 --- a/docs/RAG/06-document-processing.md +++ b/docs/RAG/06-document-processing.md @@ -5,6 +5,34 @@ text in the RAG module — the _Document Loading_ stage of the pipeline. It cove supported file types, the extractor system, the storage model, extraction caching, invalidation, the REST API, and the frontend document manager. +## Ownership + +**A document belongs to exactly one RAG session.** `document.session_id` is a +NOT NULL foreign key to `generative_session`, and `UNIQUE(session_id, +file_hash)` replaces what used to be a global `UNIQUE(file_hash)`. Uploading the +same file into two sessions therefore creates two documents, each free to pick +its own extractor without disturbing the other. + +Documents used to be a global library, with membership expressed by the JSON +list `GenerativeSession.parameters["documents"]`. That list is still what the +pipeline, the chunk-set signature and the parameter history read, so it is kept +in step with the foreign key — but only by the document endpoints. Clients +cannot set it: session creation always starts empty, and both +`POST /generative-session/` and `PUT /generative-session/{id}/parameters` +reject the key rather than silently dropping it. + +Two consequences worth knowing: + +- **Chunk sets are no longer shared between sessions.** `RAGChunkSet.signature` + hashes the document ids, which now always differ, so two sessions with + identical configurations each chunk and fit their own retriever. That costs + CPU and disk in exchange for isolation. +- **Deleting a session deletes its documents**, their files and their fitted + artifacts. SQLite foreign keys are not enforced in this application (there is + no `PRAGMA foreign_keys=ON`), so this comes from the ORM cascade on + `GenerativeSession.documents` plus `DocumentService.delete_by_session`, not + from the `ondelete` clauses. + ## Supported File Types `DocumentFileType` (`models/RAG/documents/file_type.py`) is the single source of @@ -61,28 +89,49 @@ the default extractor is materialized as a `rag_extractor` row, so the ## Storage Model +- **`document.session_id`** — FK → `generative_session.id`, NOT NULL. The owning + session, unique together with `file_hash`. - **`rag_extractor`** — canonical extractor configuration: `id`, - `component_name` (NOT NULL), `params` (JSON, nullable). Multiple documents can - reference the same configuration via a foreign key. -- **`document.extractor_id`** — FK → `rag_extractor.id`, NOT NULL, with - `ondelete=RESTRICT`. Assigned at upload, never ambiguous. + `component_name` (NOT NULL), `params` (JSON). Rows are immutable value objects + and are reused: `_get_or_create_extractor_row` looks one up by + `(component_name, params)` before inserting, and an unreferenced row is + dropped once the last document stops pointing at it. +- **`document.extractor_id`** — FK → `rag_extractor.id`, NOT NULL. Assigned at + upload, never ambiguous. - **`processed_document_content`** — a 1:1 cache of extracted text (one row per document, enforced by a unique constraint on `document_id`): `content`, `signature`, `char_count`. +### Files on disk + +Files are stored **content-addressed** at +`/blobs/`. Naming them after `file_name` alone meant +two different files called `report.pdf` hashed differently — so both got a +document row — but resolved to the same path, and the second upload overwrote +the first one's bytes. Per-session copies would have made that collision +routine. + +One consequence: sessions holding identical files share one file on disk, so +deletion is reference-counted. `DocumentService._unlink_if_unreferenced` removes +the blob only when no other row points at it. Documents migrated from the old +global library may also share a path, which is the other reason the guard is +required. + ## Upload -`DocumentService.upload()`: - -1. Computes a SHA-256 content hash of the file bytes (`hash_function`). -2. Deduplicates by hash: - - Duplicate and `force=False` → returns the existing document plus its - related sessions (the endpoint surfaces this as `409 Conflict`). - - `force=True` → overwrites the file, invalidates RAG artifacts, and - re-extracts. -3. Writes the file to the configured `DOCUMENTS_PATH`. -4. Creates the default `rag_extractor` record for the file type and assigns it. -5. Commits, then pre-extracts (warms the cache) when a component registry is +`DocumentService.upload(..., session_id)`: + +1. Validates that the session exists and is a RAG session, before writing + anything. +2. Computes a SHA-256 content hash of the file bytes (`hash_function`). +3. Deduplicates **within the session**: the same bytes already present is + reported as a duplicate (the endpoint surfaces `409 Conflict`) and nothing is + modified. The same bytes in another session is a new document. +4. Writes the blob, if it is not already there. +5. Reuses or creates the default `rag_extractor` record for the file type. +6. Appends the new id to the session's `documents` list, with a parameter + history entry, in the same transaction as the insert. +7. Commits, then pre-extracts (warms the cache) when a component registry is available. Extraction failures are raised as `RAGDocumentExtractionError`. ## Text Extraction & Caching @@ -90,8 +139,10 @@ the default extractor is materialized as a `rag_extractor` row, so the `DocumentService.extract_text()` implements on-demand extraction with a 1:1 cache: -1. Resolves the extractor: explicit `{component, params}` ref → stored record → - file-type default. +1. Resolves the extractor **and how it is configured** in one place, + `_resolve_extractor_ref`: explicit `{component, params}` ref → stored record → + file-type default. Deriving the two separately is what made the signature + disagree with itself (see below). 2. Validates `SUPPORTED_FILE_TYPES` compatibility; incompatible extractors raise an error. 3. Builds a cache signature: @@ -99,65 +150,102 @@ cache: 4. `persist=False` (preview mode) → extracts without persisting or invalidating. 5. Cache hit (matching signature) → returns the stored text with `cached=True`. 6. Cache miss → extracts, then overwrites the single row in place (or creates - it). When the content changes because a different extractor/params produced a - new signature, RAG artifacts of the related sessions are invalidated. + it). When an existing row is replaced, the artifacts fitted over the old text + are invalidated. A *first* extraction invalidates nothing, because chunking + needs extracted text and so there is nothing to invalidate yet. + +> **Fixed:** the signature used to be built from empty params while the +> extractor was instantiated with the stored ones, so for any non-default +> configuration it never matched itself. Every empty-body `POST /extract` was a +> cache miss *and* destroyed the chunk set — a preview silently un-indexed the +> session. ## Changing the Extractor -`DocumentService.update_extractor()`: +`DocumentService.update_extractor()` is one transaction, and the extraction runs +before anything is mutated: 1. Validates the extractor exists in the registry and is compatible with the document's file type. -2. If the document is linked to RAG pipelines and `force=False`, refuses and - reports the affected sessions (the endpoint surfaces this as `409 Conflict`). -3. Creates a new `rag_extractor` record, reassigns `extractor_id`, and — with - `force=True` — invalidates artifacts. -4. Re-extracts with the new extractor to keep the 1:1 `processed_document_content` - invariant. +2. Returns early when nothing changed — the same component, the same params and + a matching cache signature. With invalidation now unconditional, saving an + unchanged choice would otherwise throw away a perfectly good index. +3. Extracts the text. This can fail, and nothing has been touched yet. +4. Reuses or creates the `rag_extractor` record, reassigns `extractor_id`, + invalidates artifacts, writes the new content, and drops the previous + extractor record if it is now unreferenced — then commits **once**. + +There is no `force` parameter. It existed to make the user confirm a destructive +re-index, but it asked `RAGDocumentPipelineSessionLink` which sessions were +affected — a table nothing ever wrote to — so the confirmation was unreachable +and the invalidation it guarded never ran. A document now belongs to one +session, so there is nobody else to warn. + +Committing the reassignment first was also how a failed extraction left a +document pointing at an extractor that had never produced its text, still +serving the previous extractor's chunks. A failure now returns `422` and changes +nothing. ## Invalidation -Changing an extractor (or force re-uploading a document) calls +Changing an extractor, or deleting a document or its session, calls `CleanupService.invalidate_document_artifacts(document_id)`, which deletes the document's chunks, retrievers, embedding matrices, and related disk artifacts. -Nothing is eagerly recomputed — the next pipeline run for an affected session -re-chunks and rebuilds retrieval automatically. +Nothing is eagerly recomputed — the next pipeline run re-chunks and rebuilds +retrieval automatically. + +The method takes two flags so a caller can make it part of a larger unit of +work: `commit=False` hands the transaction back, and `defer_paths` collects the +on-disk paths instead of removing them, since `rmtree` cannot be rolled back. +The caller deletes them after its commit succeeds. + +Note that it deletes the **whole chunk set**, including the chunks of sibling +documents in it. With documents owned by one session, a chunk set is too, so +re-chunking the set is exactly what has to happen. ## API All endpoints live in `DashAI/back/api/api_v1/endpoints/documents.py` under the `/api/v1/document` prefix: -| Method | Path | Purpose | -| ------ | ---------------------------------------- | ---------------------------------------------- | -| GET | `/api/v1/document/` | List all documents | -| POST | `/api/v1/document/` | Upload (multipart file + metadata JSON) | -| GET | `/api/v1/document/{id}` | Document metadata | -| GET | `/api/v1/document/{id}/download` | Download the file | -| GET | `/api/v1/document/{id}/view` | Inline preview | -| GET | `/api/v1/document/session/{session_id}` | Documents of a RAG session | -| GET | `/api/v1/document/related-sessions/{id}` | Session IDs linked to a document | -| DELETE | `/api/v1/document/{id}` | Delete document + file | -| PUT | `/api/v1/document/{id}` | Update metadata | -| POST | `/api/v1/document/{id}/extract` | On-demand extraction (`extractor`, `persist`) | -| PUT | `/api/v1/document/{id}/extractor` | Commit extractor choice (`extractor`, `force`) | +| Method | Path | Purpose | +| ------ | --------------------------------------------- | --------------------------------------------- | +| POST | `/api/v1/document/session/{session_id}` | Upload into a session (multipart + metadata) | +| GET | `/api/v1/document/session/{session_id}` | Documents of a RAG session | +| GET | `/api/v1/document/{id}` | Document metadata | +| GET | `/api/v1/document/{id}/download` | Download the file | +| GET | `/api/v1/document/{id}/view` | Inline preview | +| DELETE | `/api/v1/document/{id}` | Delete document, artifacts and file | +| PUT | `/api/v1/document/{id}` | Update metadata | +| POST | `/api/v1/document/{id}/extract` | On-demand extraction (`extractor`, `persist`) | +| PUT | `/api/v1/document/{id}/extractor` | Commit extractor choice (`extractor`) | + +`GET /api/v1/document/` and `GET /api/v1/document/related-sessions/{id}` are +gone: there is no global document list any more, and the latter read the dead +link table, so it always answered `[]`. ## Frontend -The document manager lives under `components/generative/RAG/`: - -- `RAGDocumentsPage` — document library page with `DocumentTable` and a detail - panel. -- `DocumentDetailPanel` — document info, an extractor selector filtered by - `supported_file_types`, and on-demand content display. -- `DocumentExtractorModal` — schema-driven extractor configuration. -- `DocumentPreviewModal` — inline preview. -- `DuplicateDocumentDialog` — confirmation when re-uploading an existing file. -- `DocumentSelector`, `DocumentList`, `DocumentListItem`, `DocumentsBar`, - `RAGDocumentsPanel` — selection and list UIs for session setup. - -The API client (`api/rag.ts`) exposes `getExtractorOptions()`, -`extractDocumentText()`, and `updateDocumentExtractor()`. +Documents are managed from inside the session, in its left panel. There is no +standalone documents page: `/app/generative/rag/documents` redirects to the RAG +home. + +Under `components/generative/RAG/`: + +- `DocumentsBar` — the session's document panel: the list, upload, per-row + inspect and delete. +- `DocumentList` / `DocumentListItem` — presentational list and row; the row + reveals its actions on hover. +- `DocumentInspectorModal` — the old extractor modal, now the document + inspector: the original file beside its extracted text, an extractor selector + filtered by `supported_file_types`, and a schema-driven params form. It + extracts on open. A modal rather than a panel because the left panel is too + narrow to read extracted text in, and the centre column stays with the chat. +- `DocumentPreviewModal` — the plain "show me the file" view, on row click. + +The API client (`api/rag.ts`) exposes `getSessionDocuments()`, `addDocument()` +(session-scoped), `deleteDocument()`, `getExtractorOptions()`, +`extractDocumentText()` and `updateDocumentExtractor()`. ## Related Docs diff --git a/tests/back/RAG/conftest.py b/tests/back/RAG/conftest.py index d3bd0f1d1..f1ede0f50 100644 --- a/tests/back/RAG/conftest.py +++ b/tests/back/RAG/conftest.py @@ -22,7 +22,12 @@ from fastapi.testclient import TestClient from DashAI.back.app import create_app -from DashAI.back.dependencies.database.models import Document, RAGExtractor +from DashAI.back.dependencies.database.models import ( + Document, + GenerativeSession, + GenerativeSessionParameterHistory, + RAGExtractor, +) from DashAI.back.dependencies.job_queues.huey_job_queue import HueyJobQueue # Shared constants for RAG E2E tests @@ -64,8 +69,15 @@ def write_test_doc_file(suffix: str, text: str) -> str: return path -def _create_test_document(client: TestClient, suffix: str = "") -> int: - """Create a minimal test document in the DB and return its ID. +def _add_document_to_session( + client: TestClient, session_id: int, suffix: str = "" +) -> int: + """Attach a minimal test document to a RAG session and return its ID. + + Documents belong to exactly one session, so a document cannot exist before + the session that owns it. The session's ``parameters["documents"]`` list is + updated in the same transaction, mirroring what ``DocumentService.upload`` + does, so the pipeline and the index status see the document too. Uses ``tempfile.gettempdir()`` for a cross-platform temporary path. """ @@ -75,6 +87,7 @@ def _create_test_document(client: TestClient, suffix: str = "") -> int: db.add(extractor) db.flush() doc = Document( + session_id=session_id, file_name=f"test_doc{suffix}.txt", file_type="txt", file_path=os.path.join(tempfile.gettempdir(), f"test_doc{suffix}.txt"), @@ -82,11 +95,69 @@ def _create_test_document(client: TestClient, suffix: str = "") -> int: extractor_id=extractor.id, ) db.add(doc) + db.flush() + + session = db.get(GenerativeSession, session_id) + parameters = dict(session.parameters or {}) + parameters["documents"] = [*(parameters.get("documents") or []), doc.id] + session.parameters = parameters + # DocumentService.upload also historizes the change, and the index + # status reads that history to tell "stale" from "never indexed". + db.add( + GenerativeSessionParameterHistory( + session_id=session_id, parameters=parameters + ) + ) + db.commit() db.refresh(doc) return doc.id +def _create_test_document(client: TestClient, suffix: str = "") -> int: + """Create a test document in a throwaway session and return its ID. + + A document cannot exist without an owning session, but service-level tests + only need a document that exists -- they never assert which session holds + it. This provisions a minimal session to hang it off. Use + ``_add_document_to_session`` when the owning session matters. + """ + session_factory = client.app.container["session_factory"] + with session_factory() as db: + holder = GenerativeSession( + name=f"doc_holder{suffix}", + task_name="RAGTask", + model_name="RAGPipeline", + parameters={"documents": []}, + ) + db.add(holder) + db.commit() + session_id = holder.id + return _add_document_to_session(client, session_id, suffix=suffix) + + +def _create_rag_session(client: TestClient, payload: dict) -> int: + """Create a RAG session and return its ID, failing loudly on a bad payload. + + ``documents`` is stripped from the payload: a session is always created + empty and gains its documents afterwards, so sending them is rejected. + """ + body = {**payload, "parameters": dict(payload.get("parameters") or {})} + body["parameters"].pop("documents", None) + response = client.post("/api/v1/generative-session/", json=body) + assert response.status_code == 201, response.text + return response.json()["id"] + + +def _create_session_with_document( + client: TestClient, payload: dict, suffix: str = "" +) -> "tuple[int, int]": + """Create a RAG session, attach one document, and return both IDs.""" + session_id = _create_rag_session(client, payload) + document_id = _add_document_to_session(client, session_id, suffix=suffix) + return session_id, document_id + + def _mark_download_required_components_present(app) -> None: """Create each download-required component's repo folder so the download gate (reconciled against the filesystem) treats it as available without diff --git a/tests/back/RAG/test_RAG_component_api_configs.py b/tests/back/RAG/test_RAG_component_api_configs.py index f0362595a..42089c258 100644 --- a/tests/back/RAG/test_RAG_component_api_configs.py +++ b/tests/back/RAG/test_RAG_component_api_configs.py @@ -14,18 +14,8 @@ pipeline runtime, not during session creation. """ -import pytest from fastapi.testclient import TestClient -from tests.back.RAG.conftest import _create_test_document - - -@pytest.fixture(scope="module") -def test_doc_id(client: TestClient) -> int: - """Module-scoped test document ID shared across all component config tests.""" - return _create_test_document(client, suffix="_component_configs") - - # --------------------------------------------------------------------------- # helpers # --------------------------------------------------------------------------- @@ -33,13 +23,12 @@ def test_doc_id(client: TestClient) -> int: ST_MINI_LM = "sentence-transformers/all-MiniLM-L6-v2" -def _base_params(test_doc_id: int) -> dict: +def _base_params() -> dict: """Return the minimal default RAG session payload.""" return { "model_name": "RAGPipeline", "task_name": "RAGTask", "parameters": { - "documents": [test_doc_id], "chunking_model": { "component": "CharacterChunkModel", "params": {"chunk_size": 400, "chunk_overlap": 40}, @@ -159,9 +148,9 @@ class TestEncodingModels: """6 configurations: sparse and dense encoding models with hyperparameter variations.""" - def test_encoding_bm25_default(self, client: TestClient, test_doc_id: int): + def test_encoding_bm25_default(self, client: TestClient): """BM25Retriever with all default hyperparams.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Enc BM25 Default" stored = _post_and_get(client, params) ret = stored["parameters"]["retriever_model"] @@ -172,12 +161,10 @@ def test_encoding_bm25_default(self, client: TestClient, test_doc_id: int): assert ret["params"]["similarity_function"] == "cosine" assert ret["params"]["top_k"] == 5 - def test_encoding_bm25_custom_hyperparams( - self, client: TestClient, test_doc_id: int - ): + def test_encoding_bm25_custom_hyperparams(self, client: TestClient): """BM25Retriever with custom hyperparams k1=2.0, b=0.5, delta=0.5, euclidean, top_k=7.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Enc BM25 Custom" params["parameters"]["retriever_model"] = { "component": "BM25Retriever", @@ -209,9 +196,9 @@ def test_encoding_bm25_custom_hyperparams( assert ret["params"]["similarity_function"] == "euclidean" assert ret["params"]["top_k"] == 7 - def test_encoding_tfidf_default(self, client: TestClient, test_doc_id: int): + def test_encoding_tfidf_default(self, client: TestClient): """TFIDFRetriever with default hyperparams.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Enc TFIDF Default" params["parameters"]["retriever_model"] = { "component": "TFIDFRetriever", @@ -243,9 +230,9 @@ def test_encoding_tfidf_default(self, client: TestClient, test_doc_id: int): assert ret["params"]["similarity_function"] == "cosine" assert ret["params"]["top_k"] == 5 - def test_encoding_tfidf_custom(self, client: TestClient, test_doc_id: int): + def test_encoding_tfidf_custom(self, client: TestClient): """TFIDFRetriever with similarity_function=manhattan, top_k=15.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Enc TFIDF Custom" params["parameters"]["retriever_model"] = { "component": "TFIDFRetriever", @@ -277,11 +264,9 @@ def test_encoding_tfidf_custom(self, client: TestClient, test_doc_id: int): assert ret["params"]["similarity_function"] == "manhattan" assert ret["params"]["top_k"] == 15 - def test_encoding_dense_sentence_transformer( - self, client: TestClient, test_doc_id: int - ): + def test_encoding_dense_sentence_transformer(self, client: TestClient): """DenseEmbeddingRetriever + SentenceTransformerEmbedding (all-MiniLM-L6-v2).""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Enc Dense ST" params["parameters"]["retriever_model"] = _dense_st_retriever(top_k=10) stored = _post_and_get(client, params) @@ -295,9 +280,9 @@ def test_encoding_dense_sentence_transformer( assert ret["params"]["similarity_metric"] == "cosine" assert ret["params"]["top_k"] == 10 - def test_encoding_dense_st_alt(self, client: TestClient, test_doc_id: int): + def test_encoding_dense_st_alt(self, client: TestClient): """DenseEmbeddingRetriever + SentenceTransformerEmbedding (all-MiniLM-L6-v2).""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Enc Dense ST Alt" params["parameters"]["retriever_model"] = _dense_st_alt_retriever(top_k=10) stored = _post_and_get(client, params) @@ -320,9 +305,9 @@ def test_encoding_dense_st_alt(self, client: TestClient, test_doc_id: int): class TestRankingFunctions: """4 ranking function configurations evaluated with different encoding models.""" - def test_ranking_dense_cosine(self, client: TestClient, test_doc_id: int): + def test_ranking_dense_cosine(self, client: TestClient): """DenseEmbeddingRetriever with cosine similarity.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Rank Cosine" params["parameters"]["retriever_model"] = _dense_st_retriever( similarity_metric="cosine", top_k=10 @@ -332,9 +317,9 @@ def test_ranking_dense_cosine(self, client: TestClient, test_doc_id: int): assert ret["params"]["similarity_metric"] == "cosine" assert ret["params"]["top_k"] == 10 - def test_ranking_dense_euclidean(self, client: TestClient, test_doc_id: int): + def test_ranking_dense_euclidean(self, client: TestClient): """DenseEmbeddingRetriever with euclidean similarity.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Rank Euclidean" params["parameters"]["retriever_model"] = _dense_st_retriever( similarity_metric="euclidean", top_k=10 @@ -344,9 +329,9 @@ def test_ranking_dense_euclidean(self, client: TestClient, test_doc_id: int): assert ret["params"]["similarity_metric"] == "euclidean" assert ret["params"]["top_k"] == 10 - def test_ranking_sparse_manhattan(self, client: TestClient, test_doc_id: int): + def test_ranking_sparse_manhattan(self, client: TestClient): """BM25Retriever with manhattan similarity.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Rank Manhattan" params["parameters"]["retriever_model"]["params"]["similarity_function"] = ( "manhattan" @@ -358,13 +343,13 @@ def test_ranking_sparse_manhattan(self, client: TestClient, test_doc_id: int): assert ret["params"]["similarity_function"] == "manhattan" assert ret["params"]["top_k"] == 10 - def test_ranking_mmr_reranker(self, client: TestClient, test_doc_id: int): + def test_ranking_mmr_reranker(self, client: TestClient): """MMRRerankerRetriever (lambda=0.7) wrapping DenseEmbeddingRetriever. The child's own ``top_k`` (40) defines the candidate set and the reranker selects ``top_k`` (10) of them. """ - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Rank MMR" params["parameters"]["retriever_model"] = { "component": "MMRRerankerRetriever", @@ -397,17 +382,17 @@ def test_ranking_mmr_reranker(self, client: TestClient, test_doc_id: int): class TestTopK: """6 top-K configurations using different encodings and ranking functions.""" - def test_topk_bm25_3(self, client: TestClient, test_doc_id: int): + def test_topk_bm25_3(self, client: TestClient): """BM25Retriever with top_k=3.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "TopK BM25 3" params["parameters"]["retriever_model"]["params"]["top_k"] = 3 stored = _post_and_get(client, params) assert stored["parameters"]["retriever_model"]["params"]["top_k"] == 3 - def test_topk_tfidf_5(self, client: TestClient, test_doc_id: int): + def test_topk_tfidf_5(self, client: TestClient): """TFIDFRetriever with top_k=5.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "TopK TFIDF 5" params["parameters"]["retriever_model"] = { "component": "TFIDFRetriever", @@ -436,34 +421,34 @@ def test_topk_tfidf_5(self, client: TestClient, test_doc_id: int): stored = _post_and_get(client, params) assert stored["parameters"]["retriever_model"]["params"]["top_k"] == 5 - def test_topk_dense_st_10(self, client: TestClient, test_doc_id: int): + def test_topk_dense_st_10(self, client: TestClient): """DenseEmbeddingRetriever + SentenceTransformer with top_k=10.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "TopK Dense ST 10" params["parameters"]["retriever_model"] = _dense_st_retriever(top_k=10) stored = _post_and_get(client, params) assert stored["parameters"]["retriever_model"]["params"]["top_k"] == 10 - def test_topk_dense_st_15(self, client: TestClient, test_doc_id: int): + def test_topk_dense_st_15(self, client: TestClient): """DenseEmbeddingRetriever + SentenceTransformerEmbedding with top_k=15.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "TopK Dense ST 15" params["parameters"]["retriever_model"] = _dense_st_alt_retriever(top_k=15) stored = _post_and_get(client, params) assert stored["parameters"]["retriever_model"]["params"]["top_k"] == 15 - def test_topk_bm25_20(self, client: TestClient, test_doc_id: int): + def test_topk_bm25_20(self, client: TestClient): """BM25Retriever with top_k=20.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "TopK BM25 20" params["parameters"]["retriever_model"]["params"]["top_k"] = 20 stored = _post_and_get(client, params) assert stored["parameters"]["retriever_model"]["params"]["top_k"] == 20 - def test_topk_mmr_12(self, client: TestClient, test_doc_id: int): + def test_topk_mmr_12(self, client: TestClient): """MMRRerankerRetriever with top_k=12; the child retrieves 36 via its own top_k and the reranker selects 12.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "TopK MMR 12" params["parameters"]["retriever_model"] = { "component": "MMRRerankerRetriever", @@ -488,9 +473,9 @@ class TestChunkingStrategies: """6 chunking strategy configurations with different algorithms and parameters (chunk_size, chunk_overlap).""" - def test_chunking_char_small(self, client: TestClient, test_doc_id: int): + def test_chunking_char_small(self, client: TestClient): """CharacterChunkModel with chunk_size=256, chunk_overlap=25.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Chunk Char 256/25" params["parameters"]["chunking_model"] = { "component": "CharacterChunkModel", @@ -502,9 +487,9 @@ def test_chunking_char_small(self, client: TestClient, test_doc_id: int): assert chunk["params"]["chunk_size"] == 256 assert chunk["params"]["chunk_overlap"] == 25 - def test_chunking_char_paragraph(self, client: TestClient, test_doc_id: int): + def test_chunking_char_paragraph(self, client: TestClient): """CharacterChunkModel with chunk_size=500, chunk_overlap=50.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Chunk Char 500/50" params["parameters"]["chunking_model"] = { "component": "CharacterChunkModel", @@ -516,9 +501,9 @@ def test_chunking_char_paragraph(self, client: TestClient, test_doc_id: int): assert chunk["params"]["chunk_size"] == 500 assert chunk["params"]["chunk_overlap"] == 50 - def test_chunking_char_page(self, client: TestClient, test_doc_id: int): + def test_chunking_char_page(self, client: TestClient): """CharacterChunkModel with chunk_size=2000, chunk_overlap=200.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Chunk Char 2000/200" params["parameters"]["chunking_model"] = { "component": "CharacterChunkModel", @@ -530,9 +515,9 @@ def test_chunking_char_page(self, client: TestClient, test_doc_id: int): assert chunk["params"]["chunk_size"] == 2000 assert chunk["params"]["chunk_overlap"] == 200 - def test_chunking_recursive_custom(self, client: TestClient, test_doc_id: int): + def test_chunking_recursive_custom(self, client: TestClient): """RecursiveCharacterChunkModel with custom separators.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Chunk Recursive Custom" params["parameters"]["chunking_model"] = { "component": "RecursiveCharacterChunkModel", @@ -549,9 +534,9 @@ def test_chunking_recursive_custom(self, client: TestClient, test_doc_id: int): assert chunk["params"]["chunk_overlap"] == 100 assert chunk["params"]["separators"] == ["\n\n", "\n", ".", " ", ""] - def test_chunking_token_e5_mistral(self, client: TestClient, test_doc_id: int): + def test_chunking_token_e5_mistral(self, client: TestClient): """TokenChunkModel with e5-mistral tokenizer, chunk_size=300, overlap=60.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Chunk Token E5" params["parameters"]["chunking_model"] = { "component": "TokenChunkModel", @@ -568,9 +553,9 @@ def test_chunking_token_e5_mistral(self, client: TestClient, test_doc_id: int): assert chunk["params"]["chunk_size"] == 300 assert chunk["params"]["chunk_overlap"] == 60 - def test_chunking_token_bert_spanish(self, client: TestClient, test_doc_id: int): + def test_chunking_token_bert_spanish(self, client: TestClient): """TokenChunkModel with BERT Spanish tokenizer, chunk_size=512, overlap=50.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Chunk Token BERT ES" params["parameters"]["chunking_model"] = { "component": "TokenChunkModel", @@ -599,18 +584,18 @@ def test_chunking_token_bert_spanish(self, client: TestClient, test_doc_id: int) class TestPrompts: """4 prompt configurations validating formatting and chat session consistency.""" - def test_prompt_default_rag_en(self, client: TestClient, test_doc_id: int): + def test_prompt_default_rag_en(self, client: TestClient): """DefaultRAGGenerationPrompt with language=en.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Prompt RAG EN" stored = _post_and_get(client, params) prompt = stored["parameters"]["prompt"] assert prompt["component"] == "DefaultRAGGenerationPrompt" assert prompt["params"]["language"] == "en" - def test_prompt_default_rag_es(self, client: TestClient, test_doc_id: int): + def test_prompt_default_rag_es(self, client: TestClient): """DefaultRAGGenerationPrompt with language=es.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Prompt RAG ES" params["parameters"]["prompt"] = { "component": "DefaultRAGGenerationPrompt", @@ -621,9 +606,9 @@ def test_prompt_default_rag_es(self, client: TestClient, test_doc_id: int): assert prompt["component"] == "DefaultRAGGenerationPrompt" assert prompt["params"]["language"] == "es" - def test_prompt_default_qna_en(self, client: TestClient, test_doc_id: int): + def test_prompt_default_qna_en(self, client: TestClient): """DefaultQARAGGenerationPrompt with language=en.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Prompt QnA EN" params["parameters"]["prompt"] = { "component": "DefaultQARAGGenerationPrompt", @@ -634,10 +619,10 @@ def test_prompt_default_qna_en(self, client: TestClient, test_doc_id: int): assert prompt["component"] == "DefaultQARAGGenerationPrompt" assert prompt["params"]["language"] == "en" - def test_prompt_custom_template(self, client: TestClient, test_doc_id: int): + def test_prompt_custom_template(self, client: TestClient): """CustomRAGGenerationPrompt with user-defined template.""" template_text = "Answer the question based on: {chunks}\n\nQuestion: {input}" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Prompt Custom" params["parameters"]["prompt"] = { "component": "CustomRAGGenerationPrompt", @@ -658,9 +643,9 @@ class TestGeneratorModels: """8 generator model configurations with different models and hyperparameters (≤8B).""" - def test_generator_llama_1b_default(self, client: TestClient, test_doc_id: int): + def test_generator_llama_1b_default(self, client: TestClient): """Llama 3.2-1B with default hyperparams.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Gen Llama 1B Default" stored = _post_and_get(client, params) gen = stored["parameters"]["generation_model"] @@ -671,9 +656,9 @@ def test_generator_llama_1b_default(self, client: TestClient, test_doc_id: int): assert gen["params"]["context_window"] == 512 assert gen["params"]["device"] == "CPU" - def test_generator_llama_3b_custom(self, client: TestClient, test_doc_id: int): + def test_generator_llama_3b_custom(self, client: TestClient): """Llama 3.2-3B with custom hyperparams.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Gen Llama 3B Custom" params["parameters"]["generation_model"] = { "component": "Llama32_3BInstruct", @@ -694,9 +679,9 @@ def test_generator_llama_3b_custom(self, client: TestClient, test_doc_id: int): assert gen["params"]["context_window"] == 2048 assert gen["params"]["device"] == "CPU" - def test_generator_mistral_7b_default(self, client: TestClient, test_doc_id: int): + def test_generator_mistral_7b_default(self, client: TestClient): """Mistral 7B v0.3 with default hyperparams.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Gen Mistral 7B Default" params["parameters"]["generation_model"] = { "component": "Mistral7BInstructV03", @@ -717,9 +702,9 @@ def test_generator_mistral_7b_default(self, client: TestClient, test_doc_id: int assert gen["params"]["context_window"] == 512 assert gen["params"]["device"] == "CPU" - def test_generator_qwen_0_5b_default(self, client: TestClient, test_doc_id: int): + def test_generator_qwen_0_5b_default(self, client: TestClient): """Qwen 2.5-0.5B with default hyperparams.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Gen Qwen 0.5B Default" params["parameters"]["generation_model"] = { "component": "Qwen25_05BInstruct", @@ -740,9 +725,9 @@ def test_generator_qwen_0_5b_default(self, client: TestClient, test_doc_id: int) assert gen["params"]["context_window"] == 512 assert gen["params"]["device"] == "CPU" - def test_generator_qwen_1_5b_custom(self, client: TestClient, test_doc_id: int): + def test_generator_qwen_1_5b_custom(self, client: TestClient): """Qwen 2.5-1.5B with custom hyperparams.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Gen Qwen 1.5B Custom" params["parameters"]["generation_model"] = { "component": "Qwen25_15BInstruct", @@ -763,9 +748,9 @@ def test_generator_qwen_1_5b_custom(self, client: TestClient, test_doc_id: int): assert gen["params"]["context_window"] == 1024 assert gen["params"]["device"] == "CPU" - def test_generator_smol_1_7b_default(self, client: TestClient, test_doc_id: int): + def test_generator_smol_1_7b_default(self, client: TestClient): """SmolLM2 1.7B with default hyperparams.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Gen SmolLM 1.7B Default" params["parameters"]["generation_model"] = { "component": "SmolLM2_17BInstruct", @@ -786,9 +771,9 @@ def test_generator_smol_1_7b_default(self, client: TestClient, test_doc_id: int) assert gen["params"]["context_window"] == 512 assert gen["params"]["device"] == "CPU" - def test_generator_phi4_mini_default(self, client: TestClient, test_doc_id: int): + def test_generator_phi4_mini_default(self, client: TestClient): """Phi4MiniInstructModel with default hyperparams.""" - params = _base_params(test_doc_id) + params = _base_params() params["name"] = "Gen Phi4 Mini Default" params["parameters"]["generation_model"] = { "component": "Phi4MiniInstructModel", diff --git a/tests/back/RAG/test_RAG_indexing.py b/tests/back/RAG/test_RAG_indexing.py new file mode 100644 index 000000000..178435b2a --- /dev/null +++ b/tests/back/RAG/test_RAG_indexing.py @@ -0,0 +1,478 @@ +"""Tests for eager RAG indexing: the job, the endpoint, and the status it reports. + +Indexing used to happen inside the chat job, so the first message paid for the +whole chunking and embedding run. It now runs up front, as ``RAGIndexJob``, +started through ``POST /rag/sessions/{id}/index``. What these tests pin is the +part that is easy to get wrong: the job must not touch the generation model, +the endpoint must not enqueue work twice, and the status must stay honest when +the job pointer goes stale. +""" + +import contextlib +import os +import sqlite3 +import tempfile +import uuid + +import pytest +from fastapi.testclient import TestClient + +from DashAI.back.core.schema_fields import BaseSchema +from DashAI.back.dependencies.database.models import GenerativeSession +from DashAI.back.job.base_job import JobError +from DashAI.back.job.RAG_index_job import RAGIndexJob +from DashAI.back.models.RAG.RAG_constants import RAG_PARAM_KEYS +from DashAI.back.models.RAG.RAG_models_factory import RAGModelsFactory +from DashAI.back.models.RAG.RAG_pipeline import RAGPipelineConfig +from DashAI.back.models.text_to_text_generation_model import ( + TextToTextGenerationTaskModel, +) +from DashAI.back.services.RAG.setup_service import SetupService +from tests.back.RAG.conftest import RAG_E2E_DOC_TEXT, _add_document_to_session + + +class StubIndexLLMSchema(BaseSchema): + """Empty schema — the stub model accepts any (empty) parameter set.""" + + +class StubIndexLLM(TextToTextGenerationTaskModel): + """Deterministic model, so indexing tests never run real inference.""" + + SCHEMA = StubIndexLLMSchema + + def __init__(self, **kwargs): + """Store parameters without initialising the base class.""" + self.parameters = {} + + def generate(self, prompt): + """Return a fixed stub answer.""" + return ["stub answer"] + + +@pytest.fixture(scope="module", autouse=True) +def _register_stub_llm(client: TestClient): + """Register the stub generation model for this module's sessions.""" + registry = client.app.container["component_registry"] + if "StubIndexLLM" not in registry: + registry.register_component(StubIndexLLM) + return + + +@pytest.fixture +def written_documents() -> list: + """Collect the document files written by a test, and clean them up.""" + paths: list = [] + yield paths + for path in paths: + with contextlib.suppress(OSError): + os.remove(path) + + +def _attach_indexable_document( + client: TestClient, session_id: int, written_documents: list +) -> int: + """Add a document to a session, with a real file the pipeline can chunk.""" + suffix = f"_index_{uuid.uuid4().hex[:8]}" + doc_id = _add_document_to_session(client, session_id, suffix=suffix) + path = os.path.join(tempfile.gettempdir(), f"test_doc{suffix}.txt") + with open(path, "w", encoding="utf-8") as handle: + handle.write(RAG_E2E_DOC_TEXT) + written_documents.append(path) + return doc_id + + +def _create_session(client: TestClient, name: str) -> int: + """Create an empty RAG session wired to the stub generation model.""" + response = client.post( + "/api/v1/generative-session/", + json={ + "model_name": "RAGPipeline", + "task_name": "RAGTask", + "name": f"{name}_{uuid.uuid4().hex[:8]}", + "parameters": { + "generation_model": {"component": "StubIndexLLM", "params": {}}, + }, + }, + ) + assert response.status_code == 201, response.text + return response.json()["id"] + + +def _create_session_with_document( + client: TestClient, name: str, written_documents: list +) -> "tuple[int, int]": + session_id = _create_session(client, name) + document_id = _attach_indexable_document(client, session_id, written_documents) + return session_id, document_id + + +@contextlib.contextmanager +def _setup_service(client: TestClient, session_id: int): + """Yield ``(service, make_config)`` for a session, sharing one DB session. + + ``RAGPipelineConfig`` holds the live SQLAlchemy session, so the config and + the service it is passed to have to be built inside the same ``with``. + """ + container = client.app.container + with container["session_factory"]() as db: + + def make_config() -> RAGPipelineConfig: + session = db.get(GenerativeSession, session_id) + clean = { + k: v + for k, v in dict(session.parameters or {}).items() + if k in RAG_PARAM_KEYS + } + return RAGPipelineConfig.from_kwargs( + db=db, + component_registry=container["component_registry"], + session_id=session_id, + env_RAG_path=container["config"]["RAG_PATH"], + **clean, + ) + + yield ( + SetupService( + db, + container["component_registry"], + container["config"]["RAG_PATH"], + ), + make_config, + ) + + +def _queue_rows(client: TestClient) -> list: + """Read the job queue's task_copy rows directly.""" + return client.app.container["job_queue"].to_list() + + +# =================================================================== +# build_index — the indexing half of the pipeline +# =================================================================== + + +def test_build_index_never_builds_the_generation_model( + client: TestClient, written_documents: list, monkeypatch: pytest.MonkeyPatch +): + """The whole point of splitting build_index out of build_pipeline. + + Indexing that instantiated the LLM would load model weights it never uses, + which is exactly the cost this change exists to avoid. + """ + session_id, _ = _create_session_with_document( + client, "index_no_llm", written_documents + ) + + def _explode(*args, **kwargs): + raise AssertionError("build_index must not instantiate the generation model") + + monkeypatch.setattr(RAGModelsFactory, "create_llm", _explode) + + with _setup_service(client, session_id) as (service, make_config): + result = service.build_index(make_config()) + + assert result.total_chunks > 0 + assert result.chunk_set_id + assert result.retriever.model is not None + + +def test_build_index_reports_progress_monotonically( + client: TestClient, written_documents: list +): + session_id, _ = _create_session_with_document( + client, "index_progress", written_documents + ) + seen: list = [] + + with _setup_service(client, session_id) as (service, make_config): + service.build_index(make_config(), progress=lambda f, m: seen.append((f, m))) + + fractions = [f for f, _ in seen] + assert fractions, "build_index reported no progress at all" + assert fractions == sorted(fractions) + assert fractions[-1] == 1.0 + assert all(message for _, message in seen) + + +def test_build_pipeline_reuses_what_build_index_created( + client: TestClient, written_documents: list +): + """The two paths must not drift: build_pipeline delegates, it does not redo.""" + session_id, _ = _create_session_with_document( + client, "index_shared_path", written_documents + ) + + with _setup_service(client, session_id) as (service, make_config): + indexed = service.build_index(make_config()) + pipeline = service.build_pipeline(make_config()) + + assert pipeline.pipeline_id == indexed.pipeline_id + assert pipeline.chunking_model_id == indexed.chunking_model_id + + data = client.get(f"/api/v1/rag/sessions/{session_id}/index-status").json() + assert data["status"] == "indexed", data + assert data["total_chunks"] == indexed.total_chunks + + +# =================================================================== +# RAGIndexJob +# =================================================================== + + +def test_index_job_makes_a_session_indexed(client: TestClient, written_documents: list): + session_id, _ = _create_session_with_document( + client, "index_job_run", written_documents + ) + + before = client.get(f"/api/v1/rag/sessions/{session_id}/index-status").json() + assert before["status"] == "not_indexed", before + + RAGIndexJob(session_id=session_id).run() + + after = client.get(f"/api/v1/rag/sessions/{session_id}/index-status").json() + assert after["status"] == "indexed", after + assert after["total_chunks"] > 0 + assert after["retriever_ready"] is True + + +def test_index_job_refuses_a_session_with_no_documents(client: TestClient): + session_id = _create_session(client, "index_job_empty") + + with pytest.raises(JobError, match="no documents"): + RAGIndexJob(session_id=session_id).run() + + +def test_index_job_refuses_an_unknown_session(client: TestClient): + with pytest.raises(JobError, match="not found"): + RAGIndexJob(session_id=999999).run() + + +def test_index_job_names_itself_after_the_session(client: TestClient): + session_id = _create_session(client, "index_job_naming") + name = RAGIndexJob(session_id=session_id).get_job_name() + assert name.startswith("Indexing: index_job_naming") + + +# =================================================================== +# The index endpoint +# =================================================================== + + +def test_index_endpoint_indexes_a_fresh_session( + client: TestClient, written_documents: list +): + session_id, _ = _create_session_with_document( + client, "index_endpoint", written_documents + ) + + response = client.post(f"/api/v1/rag/sessions/{session_id}/index") + assert response.status_code == 202, response.text + + # The queue runs jobs immediately in tests, so the work is already done. + data = client.get(f"/api/v1/rag/sessions/{session_id}/index-status").json() + assert data["status"] == "indexed", data + assert data["total_chunks"] > 0 + + +def test_index_endpoint_is_a_no_op_without_documents(client: TestClient): + session_id = _create_session(client, "index_endpoint_empty") + before = len(_queue_rows(client)) + + response = client.post(f"/api/v1/rag/sessions/{session_id}/index") + assert response.status_code == 202, response.text + assert response.json()["status"] == "no_documents" + assert response.json()["job_id"] is None + assert len(_queue_rows(client)) == before + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + assert db.get(GenerativeSession, session_id).index_job_id is None + + +def test_index_endpoint_short_circuits_when_already_indexed( + client: TestClient, written_documents: list +): + session_id, _ = _create_session_with_document( + client, "index_endpoint_twice", written_documents + ) + + assert client.post(f"/api/v1/rag/sessions/{session_id}/index").status_code == 202 + session_factory = client.app.container["session_factory"] + with session_factory() as db: + first_job = db.get(GenerativeSession, session_id).index_job_id + assert first_job + + # Nothing changed, so the second call must not queue a second run. + response = client.post(f"/api/v1/rag/sessions/{session_id}/index") + assert response.status_code == 202, response.text + assert response.json()["status"] == "indexed" + + with session_factory() as db: + assert db.get(GenerativeSession, session_id).index_job_id == first_job + + +def test_index_endpoint_404s_for_an_unknown_session(client: TestClient): + assert client.post("/api/v1/rag/sessions/999999/index").status_code == 404 + + +# =================================================================== +# Status reporting around a live / stale job pointer +# =================================================================== + + +def _write_queue_row(client: TestClient, job_id: str, status: str) -> None: + """Insert a task_copy row by hand. + + Far simpler than orchestrating a genuinely slow job, and it exercises the + exact thing the status service reads. + """ + queue = client.app.container["job_queue"] + with sqlite3.connect(queue.db_path) as conn: + conn.execute( + "INSERT OR REPLACE INTO task_copy " + "(id, task_type, job_name, enqueued_at, status, last_update, progress) " + "VALUES (?, ?, ?, STRFTIME('%Y-%m-%d %H:%M:%f','now'), ?, " + "STRFTIME('%Y-%m-%d %H:%M:%f','now'), ?)", + (job_id, "RAGIndexJob", "Indexing: test", status, 42.0), + ) + + +def _point_session_at_job(client: TestClient, session_id: int, job_id) -> None: + session_factory = client.app.container["session_factory"] + with session_factory() as db: + db.get(GenerativeSession, session_id).index_job_id = job_id + db.commit() + + +@pytest.mark.parametrize("queue_status", ["not_started", "started"]) +def test_status_reports_indexing_while_the_job_is_live( + client: TestClient, written_documents: list, queue_status: str +): + session_id, _ = _create_session_with_document( + client, f"index_live_{queue_status}", written_documents + ) + job_id = f"live-{queue_status}-{uuid.uuid4().hex[:8]}" + _write_queue_row(client, job_id, queue_status) + _point_session_at_job(client, session_id, job_id) + + data = client.get(f"/api/v1/rag/sessions/{session_id}/index-status").json() + assert data["status"] == "indexing", data + assert data["job_id"] == job_id + assert data["job"]["progress"] == 42.0 + + +def test_indexing_beats_indexed_so_a_reindex_is_not_reported_as_done( + client: TestClient, written_documents: list +): + """A re-index finds the old rows still in place; saying "ready" would lie.""" + session_id, _ = _create_session_with_document( + client, "index_precedence", written_documents + ) + assert client.post(f"/api/v1/rag/sessions/{session_id}/index").status_code == 202 + assert ( + client.get(f"/api/v1/rag/sessions/{session_id}/index-status").json()["status"] + == "indexed" + ) + + job_id = f"live-again-{uuid.uuid4().hex[:8]}" + _write_queue_row(client, job_id, "started") + _point_session_at_job(client, session_id, job_id) + + data = client.get(f"/api/v1/rag/sessions/{session_id}/index-status").json() + assert data["status"] == "indexing", data + + +def test_a_failed_job_stays_visible_without_blocking_the_status( + client: TestClient, written_documents: list +): + session_id, _ = _create_session_with_document( + client, "index_failed", written_documents + ) + job_id = f"failed-{uuid.uuid4().hex[:8]}" + _write_queue_row(client, job_id, "error") + _point_session_at_job(client, session_id, job_id) + + data = client.get(f"/api/v1/rag/sessions/{session_id}/index-status").json() + # The index genuinely is not there, so the status must say so... + assert data["status"] == "not_indexed", data + # ...but the failure has to remain visible after a reload. + assert data["job"]["status"] == "error" + + +def test_a_dangling_job_pointer_does_not_break_the_status( + client: TestClient, written_documents: list +): + """The column is a pointer, not the truth: a vanished job must not 500.""" + session_id, _ = _create_session_with_document( + client, "index_dangling", written_documents + ) + _point_session_at_job(client, session_id, "job-that-never-existed") + + response = client.get(f"/api/v1/rag/sessions/{session_id}/index-status") + assert response.status_code == 200, response.text + data = response.json() + assert data["status"] == "not_indexed", data + assert data["job"] is None + assert data["job_id"] is None + + +def test_a_dangling_pointer_does_not_block_a_new_index( + client: TestClient, written_documents: list +): + session_id, _ = _create_session_with_document( + client, "index_dangling_retry", written_documents + ) + _point_session_at_job(client, session_id, "job-that-never-existed") + + assert client.post(f"/api/v1/rag/sessions/{session_id}/index").status_code == 202 + data = client.get(f"/api/v1/rag/sessions/{session_id}/index-status").json() + assert data["status"] == "indexed", data + + +# =================================================================== +# Cancelling a live index before invalidating what it writes +# =================================================================== + + +def test_changing_parameters_cancels_a_live_index( + client: TestClient, written_documents: list +): + """The cleanup deletes the rows a running job is writing, so it must stop.""" + session_id, _ = _create_session_with_document( + client, "index_cancel_params", written_documents + ) + job_id = f"live-cancel-{uuid.uuid4().hex[:8]}" + _write_queue_row(client, job_id, "started") + _point_session_at_job(client, session_id, job_id) + + response = client.put( + f"/api/v1/generative-session/{session_id}/parameters", + json={ + "chunking_model": { + "component": "CharacterChunkModel", + "params": {"chunk_size": 250, "chunk_overlap": 25}, + } + }, + ) + assert response.status_code == 200, response.text + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + assert db.get(GenerativeSession, session_id).index_job_id is None + + +def test_deleting_a_document_cancels_a_live_index( + client: TestClient, written_documents: list +): + session_id, document_id = _create_session_with_document( + client, "index_cancel_delete", written_documents + ) + job_id = f"live-del-{uuid.uuid4().hex[:8]}" + _write_queue_row(client, job_id, "started") + _point_session_at_job(client, session_id, job_id) + + assert client.delete(f"/api/v1/document/{document_id}").status_code == 204 + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + assert db.get(GenerativeSession, session_id).index_job_id is None diff --git a/tests/back/RAG/test_RAG_pipeline_api_configs.py b/tests/back/RAG/test_RAG_pipeline_api_configs.py index 1e3a130c2..a6e3f092c 100644 --- a/tests/back/RAG/test_RAG_pipeline_api_configs.py +++ b/tests/back/RAG/test_RAG_pipeline_api_configs.py @@ -13,21 +13,12 @@ session creation. """ -import pytest from fastapi.testclient import TestClient -from tests.back.RAG.conftest import _create_test_document - ST_MINI_LM = "sentence-transformers/all-MiniLM-L6-v2" -@pytest.fixture(scope="module") -def test_doc_id(client: TestClient) -> int: - """Module-scoped test document ID shared across all pipeline config tests.""" - return _create_test_document(client, suffix="_pipeline_configs") - - -def test_publication_1_medical_fitness(client: TestClient, test_doc_id: int): +def test_publication_1_medical_fitness(client: TestClient): """Retrieval augmented generation for 10 large language models and its generalizability in assessing medical fitness. @@ -42,7 +33,6 @@ def test_publication_1_medical_fitness(client: TestClient, test_doc_id: int): "model_name": "RAGPipeline", "task_name": "RAGTask", "parameters": { - "documents": [test_doc_id], "chunking_model": { "component": "RecursiveCharacterChunkModel", "params": { @@ -110,7 +100,7 @@ def test_publication_1_medical_fitness(client: TestClient, test_doc_id: int): assert stored["parameters"]["prompt"]["component"] == "DefaultRAGGenerationPrompt" -def test_publication_2_ehr_summarization(client: TestClient, test_doc_id: int): +def test_publication_2_ehr_summarization(client: TestClient): """Applying generative AI with retrieval augmented generation to summarize and extract key clinical information from electronic health records. @@ -127,7 +117,6 @@ def test_publication_2_ehr_summarization(client: TestClient, test_doc_id: int): "model_name": "RAGPipeline", "task_name": "RAGTask", "parameters": { - "documents": [test_doc_id], "chunking_model": { "component": "CharacterChunkModel", "params": {"chunk_size": 600, "chunk_overlap": 40}, @@ -206,7 +195,7 @@ def test_publication_2_ehr_summarization(client: TestClient, test_doc_id: int): ) -def test_publication_3_case_study(client: TestClient, test_doc_id: int): +def test_publication_3_case_study(client: TestClient): """Development and Testing of Retrieval Augmented Generation in Large Language Models -- A Case Study Report. @@ -224,7 +213,6 @@ def test_publication_3_case_study(client: TestClient, test_doc_id: int): "model_name": "RAGPipeline", "task_name": "RAGTask", "parameters": { - "documents": [test_doc_id], "chunking_model": { "component": "RecursiveCharacterChunkModel", "params": { @@ -292,7 +280,7 @@ def test_publication_3_case_study(client: TestClient, test_doc_id: int): assert gen["component"] == "Llama32_1BInstruct" -def test_publication_4a_ragchecker_dense(client: TestClient, test_doc_id: int): +def test_publication_4a_ragchecker_dense(client: TestClient): """RAGChecker: A Fine-grained Framework for Diagnosing Retrieval-Augmented Generation (dense variant). @@ -307,7 +295,6 @@ def test_publication_4a_ragchecker_dense(client: TestClient, test_doc_id: int): "model_name": "RAGPipeline", "task_name": "RAGTask", "parameters": { - "documents": [test_doc_id], "chunking_model": { "component": "TokenChunkModel", "params": { @@ -373,7 +360,7 @@ def test_publication_4a_ragchecker_dense(client: TestClient, test_doc_id: int): assert stored["parameters"]["generation_model"]["component"] == "Llama32_3BInstruct" -def test_publication_4b_ragchecker_sparse(client: TestClient, test_doc_id: int): +def test_publication_4b_ragchecker_sparse(client: TestClient): """RAGChecker: A Fine-grained Framework for Diagnosing Retrieval-Augmented Generation (sparse variant). @@ -389,7 +376,6 @@ def test_publication_4b_ragchecker_sparse(client: TestClient, test_doc_id: int): "model_name": "RAGPipeline", "task_name": "RAGTask", "parameters": { - "documents": [test_doc_id], "chunking_model": { "component": "TokenChunkModel", "params": { diff --git a/tests/back/RAG/test_RAG_prompt_updates.py b/tests/back/RAG/test_RAG_prompt_updates.py index 8f7926316..9840e5805 100644 --- a/tests/back/RAG/test_RAG_prompt_updates.py +++ b/tests/back/RAG/test_RAG_prompt_updates.py @@ -1,4 +1,4 @@ -from DashAI.back.dependencies.database.models import GenerativeSession, RAGPrompt +from DashAI.back.dependencies.database.models import GenerativeSession def _get_prompt_list(client): @@ -9,7 +9,7 @@ def _get_prompt_list(client): def _create_rag_session(session_factory, prompt_id: int, name: str): params = { - "documents": [1], + "documents": [], "chunking_model": { "component": "CharacterChunkModel", "params": { @@ -69,74 +69,93 @@ def _create_rag_session(session_factory, prompt_id: int, name: str): return session.id -def test_update_prompt_in_place(client): +def test_session_prompt_is_edited_through_its_own_parameters(client): + """A session's prompt is part of its parameters, edited in place. + + There used to be a PATCH on the prompt itself, but ``rag_prompt`` rows are + deduplicated by a hash of their parameters, so one row is shared by every + session that landed on the same template: editing it rewrote the other + sessions' prompt too. + """ prompts = _get_prompt_list(client) - prompt = prompts[0] + session_factory = client.app.container["session_factory"] + session_id = _create_rag_session( + session_factory, prompts[0]["id"], "rag-prompt-edit" + ) - response = client.patch( - f"/api/v1/prompt/{prompt['id']}", - json={"name": f"{prompt['name']} (updated)"}, + response = client.put( + f"/api/v1/generative-session/{session_id}/parameters", + json={ + "prompt": { + "component": "CustomRAGGenerationPrompt", + "params": { + "template": "Answer using {chunks}. Question: {input}", + "language": "en", + }, + } + }, ) - assert response.status_code == 200 - data = response.json() - assert data["id"] == prompt["id"] - assert data["name"] == f"{prompt['name']} (updated)" + assert response.status_code == 200, response.text + prompt = response.json()["parameters"]["prompt"] + assert prompt["component"] == "CustomRAGGenerationPrompt" + assert prompt["params"]["template"].startswith("Answer using {chunks}") -def test_clone_prompt_for_session(client): +def test_editing_one_session_prompt_leaves_another_alone(client): + """Two sessions starting from the same template stay independent.""" prompts = _get_prompt_list(client) - prompt = prompts[1] session_factory = client.app.container["session_factory"] - session_id = _create_rag_session(session_factory, prompt["id"], "rag-session-clone") + first = _create_rag_session(session_factory, prompts[0]["id"], "rag-prompt-a") + second = _create_rag_session(session_factory, prompts[0]["id"], "rag-prompt-b") - response = client.post( - f"/api/v1/prompt/{prompt['id']}/sessions/{session_id}", json={} + shared = { + "component": "CustomRAGGenerationPrompt", + "params": {"template": "Shared: {chunks} {input}", "language": "en"}, + } + for session_id in (first, second): + response = client.put( + f"/api/v1/generative-session/{session_id}/parameters", + json={"prompt": shared}, + ) + assert response.status_code == 200, response.text + + edited = { + "component": "CustomRAGGenerationPrompt", + "params": {"template": "Only mine: {chunks} {input}", "language": "en"}, + } + response = client.put( + f"/api/v1/generative-session/{first}/parameters", json={"prompt": edited} ) + assert response.status_code == 200, response.text - assert response.status_code == 201 - data = response.json() - assert data["session_id"] == session_id - assert data["parameters"]["prompt_id"] == data["prompt"]["id"] - assert data["prompt"]["name"].endswith(f"session {session_id}") + untouched = client.get(f"/api/v1/generative-session/{second}").json() + assert untouched["parameters"]["prompt"]["params"]["template"] == ( + "Shared: {chunks} {input}" + ) -def test_session_parameter_prompt_reassignment(client): - """Verify that updating session parameters can switch the prompt. +def test_prompt_id_is_resolved_into_the_session(client): + """``prompt_id`` still works, and is copied rather than referenced. - The endpoint accepts a ``prompt_id`` in the payload and converts it to - a ``prompt`` dict with ``component`` and ``params`` keys. Prompt-level - cleanup is not implemented, so orphaned clones are left in the DB. + The id is a convenience for picking a registered template; what the + session stores is the resolved component and its params, so nothing later + depends on the shared row. """ prompts = _get_prompt_list(client) base_prompt = prompts[1] session_factory = client.app.container["session_factory"] session_id = _create_rag_session( - session_factory, base_prompt["id"], "rag-session-cleanup" + session_factory, base_prompt["id"], "rag-prompt-resolve" ) - clone_response = client.post( - f"/api/v1/prompt/{base_prompt['id']}/sessions/{session_id}", - json={}, - ) - assert clone_response.status_code == 201 - cloned_prompt_id = clone_response.json()["prompt"]["id"] - - update_response = client.put( + response = client.put( f"/api/v1/generative-session/{session_id}/parameters", json={"prompt_id": base_prompt["id"]}, ) - assert update_response.status_code == 200 - # The endpoint converts prompt_id → prompt dict with component + params - prompt_param = update_response.json()["parameters"]["prompt"] - assert prompt_param["component"] == base_prompt["class_name"] - assert "params" in prompt_param - - # The cloned prompt is NOT cleaned up by the API (orphan cleanup - # only handles retrievers and chunking models, not prompts). - with session_factory() as db: - orphan = db.get(RAGPrompt, cloned_prompt_id) - assert orphan is not None, ( - "Cloned prompt should still exist (no prompt cleanup)." - ) + assert response.status_code == 200, response.text + parameters = response.json()["parameters"] + # The id is resolved into a component ref the session owns outright. + assert parameters["prompt"]["component"] == base_prompt["class_name"] + assert "params" in parameters["prompt"] diff --git a/tests/back/RAG/test_RAG_prompts.py b/tests/back/RAG/test_RAG_prompts.py index 227ffe208..0b9b4bc46 100644 --- a/tests/back/RAG/test_RAG_prompts.py +++ b/tests/back/RAG/test_RAG_prompts.py @@ -7,10 +7,9 @@ - Prompt cloning to sessions """ -import pytest from fastapi.testclient import TestClient -from DashAI.back.dependencies.database.models import Document, RAGExtractor, RAGPrompt +from DashAI.back.dependencies.database.models import RAGPrompt from DashAI.back.services.RAG.prompt_service import PromptService # --------------------------------------------------------------------------- @@ -18,39 +17,12 @@ # --------------------------------------------------------------------------- -def _create_test_document(client: TestClient, suffix: str = "") -> int: - """Create a minimal test document in the DB and return its ID.""" - session_factory = client.app.container["session_factory"] - with session_factory() as db: - extractor = RAGExtractor(component_name="PlainTextExtractor", params={}) - db.add(extractor) - db.flush() - doc = Document( - file_name=f"test_doc{suffix}.txt", - file_type="txt", - file_path=f"/tmp/test_doc{suffix}.txt", - file_hash=f"test_hash_123_{suffix}" if suffix else "test_hash_123", - extractor_id=extractor.id, - ) - db.add(doc) - db.commit() - db.refresh(doc) - return doc.id - - -@pytest.fixture(scope="module") -def test_doc_id(client: TestClient) -> int: - """Module-scoped test document ID shared across all prompt tests.""" - return _create_test_document(client, suffix="_prompts") - - -def _base_session_params(test_doc_id: int) -> dict: +def _base_session_params() -> dict: """Return the minimal default RAG session payload.""" return { "model_name": "RAGPipeline", "task_name": "RAGTask", "parameters": { - "documents": [test_doc_id], "chunking_model": { "component": "CharacterChunkModel", "params": {"chunk_size": 400, "chunk_overlap": 40}, @@ -184,55 +156,6 @@ def test_create_duplicate_prompt_fails(self, client: TestClient): f"Duplicate prompt should be rejected: {resp2.text}" ) - def test_update_prompt_name(self, client: TestClient): - """PATCH /api/v1/prompt/{id} updates the prompt name.""" - # Create a prompt first - create_payload = { - "class_name": "CustomRAGGenerationPrompt", - "name": "Original Name", - "parameters": {"template": "A: {chunks} Q: {input}"}, - } - resp = client.post("/api/v1/prompt/", json=create_payload) - assert resp.status_code == 201, f"Creation failed: {resp.text}" - prompt_id = resp.json()["id"] - - # Update the name - patch_resp = client.patch( - f"/api/v1/prompt/{prompt_id}", json={"name": "Updated Name"} - ) - assert patch_resp.status_code == 200, f"PATCH failed: {patch_resp.text}" - updated = patch_resp.json() - assert updated["name"] == "Updated Name", "Name should be updated" - assert updated["id"] == prompt_id - - # Verify persistence - get_resp = client.get("/api/v1/prompt/") - prompts = get_resp.json() - match = [p for p in prompts if p["id"] == prompt_id] - assert len(match) == 1 - assert match[0]["name"] == "Updated Name" - - def test_update_prompt_parameters(self, client: TestClient): - """PATCH /api/v1/prompt/{id} updates the prompt template.""" - original_template = "Docs: {chunks}\nQuery: {input}" - create_payload = { - "class_name": "CustomRAGGenerationPrompt", - "name": "Params Test", - "parameters": {"template": original_template}, - } - resp = client.post("/api/v1/prompt/", json=create_payload) - assert resp.status_code == 201, f"Creation failed: {resp.text}" - prompt_id = resp.json()["id"] - - new_template = "Context: {chunks}\n\nUser: {input}\nAnswer:" - patch_resp = client.patch( - f"/api/v1/prompt/{prompt_id}", - json={"parameters": {"template": new_template}}, - ) - assert patch_resp.status_code == 200, f"PATCH failed: {patch_resp.text}" - updated = patch_resp.json() - assert updated["parameters"]["template"] == new_template - def test_create_prompt_missing_required_field(self, client: TestClient): """POST without class_name or without name returns 422.""" # Without class_name @@ -312,10 +235,10 @@ def test_service_get_or_create_reuses_existing(self, client: TestClient): class TestPromptSessionIntegration: """Prompt usage within generative RAG sessions.""" - def test_session_with_default_prompt_en(self, client: TestClient, test_doc_id: int): + def test_session_with_default_prompt_en(self, client: TestClient): """Session creation with DefaultRAGGenerationPrompt (language=en) stores prompt correctly.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "Session EN Prompt" stored = _post_and_get(client, params) prompt = stored["parameters"]["prompt"] @@ -324,10 +247,10 @@ def test_session_with_default_prompt_en(self, client: TestClient, test_doc_id: i ) assert prompt["params"]["language"] == "en", "Language should be en" - def test_session_with_default_prompt_es(self, client: TestClient, test_doc_id: int): + def test_session_with_default_prompt_es(self, client: TestClient): """Session creation with DefaultRAGGenerationPrompt (language=es) stores prompt correctly.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "Session ES Prompt" params["parameters"]["prompt"] = { "component": "DefaultRAGGenerationPrompt", @@ -338,10 +261,10 @@ def test_session_with_default_prompt_es(self, client: TestClient, test_doc_id: i assert prompt["component"] == "DefaultRAGGenerationPrompt" assert prompt["params"]["language"] == "es", "Language should be es" - def test_session_with_qna_prompt(self, client: TestClient, test_doc_id: int): + def test_session_with_qna_prompt(self, client: TestClient): """Session creation with DefaultQARAGGenerationPrompt (language=en) stores prompt correctly.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "Session QnA Prompt" params["parameters"]["prompt"] = { "component": "DefaultQARAGGenerationPrompt", @@ -354,13 +277,11 @@ def test_session_with_qna_prompt(self, client: TestClient, test_doc_id: int): ) assert prompt["params"]["language"] == "en" - def test_session_with_custom_prompt_template( - self, client: TestClient, test_doc_id: int - ): + def test_session_with_custom_prompt_template(self, client: TestClient): """Session creation with CustomRAGGenerationPrompt stores the custom template correctly.""" template_text = "Answer the question based on: {chunks}\n\nQuestion: {input}" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "Session Custom Prompt" params["parameters"]["prompt"] = { "component": "CustomRAGGenerationPrompt", @@ -373,53 +294,9 @@ def test_session_with_custom_prompt_template( "Custom template should be stored exactly as provided" ) - def test_clone_prompt_to_session(self, client: TestClient, test_doc_id: int): - """POST /api/v1/prompt/{id}/sessions/{session_id} - clones a prompt and attaches it to the session.""" - # Create a session first - params = _base_session_params(test_doc_id) - params["name"] = "Session For Clone" - session_data = _post_and_get(client, params) - session_id = session_data["id"] - - # Create a prompt via the prompt API - create_payload = { - "class_name": "CustomRAGGenerationPrompt", - "name": "Prompt To Clone", - "parameters": {"template": "Clone: {chunks}\nQ: {input}"}, - } - create_resp = client.post("/api/v1/prompt/", json=create_payload) - assert create_resp.status_code == 201, ( - f"Prompt creation failed: {create_resp.text}" - ) - prompt_id = create_resp.json()["id"] - - # Clone the prompt to the session - clone_resp = client.post( - f"/api/v1/prompt/{prompt_id}/sessions/{session_id}", - json={}, - ) - assert clone_resp.status_code == 201, ( - f"Clone failed: {clone_resp.status_code} {clone_resp.text}" - ) - clone_data = clone_resp.json() - assert "prompt" in clone_data, "Response should contain 'prompt'" - assert clone_data["session_id"] == session_id - new_prompt_id = clone_data["prompt"]["id"] - assert new_prompt_id is not None, "Cloned prompt should have an id" - assert new_prompt_id != prompt_id, "Cloned prompt should be a new record" - - # Verify session parameters now reference the cloned prompt - session_params = clone_data["parameters"] - assert session_params.get("prompt_id") == new_prompt_id, ( - "Session parameters should reference the cloned prompt ID" - ) - - def test_session_rejects_invalid_prompt_class( - self, client: TestClient, test_doc_id: int - ): + def test_session_rejects_invalid_prompt_class(self, client: TestClient): """Session creation rejects unknown prompt component names with 400.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "Session Invalid Prompt" params["parameters"]["prompt"] = { "component": "NonExistentPrompt", diff --git a/tests/back/RAG/test_RAG_session_configuration_api.py b/tests/back/RAG/test_RAG_session_configuration_api.py index e6d263654..632dbc637 100644 --- a/tests/back/RAG/test_RAG_session_configuration_api.py +++ b/tests/back/RAG/test_RAG_session_configuration_api.py @@ -23,7 +23,7 @@ from DashAI.back.models.text_to_text_generation_model import ( TextToTextGenerationTaskModel, ) -from tests.back.RAG.conftest import RAG_E2E_DOC_TEXT, _create_test_document +from tests.back.RAG.conftest import RAG_E2E_DOC_TEXT, _add_document_to_session STUB_ANSWER = "stub answer" @@ -52,17 +52,27 @@ def register_stub_llm(client: TestClient) -> None: client.app.container["component_registry"].register_component(StubConfigLLM) -@pytest.fixture(scope="module") -def indexed_document(client: TestClient) -> int: - """A document whose file actually exists, so the pipeline can chunk it.""" +@pytest.fixture +def written_documents() -> list: + """Collect the document files written by a test, and clean them up.""" + paths: list = [] + yield paths + for path in paths: + with contextlib.suppress(OSError): + os.remove(path) + + +def _attach_indexable_document( + client: TestClient, session_id: int, written_documents: list +) -> int: + """Add a document to a session, with a real file the pipeline can chunk.""" suffix = f"_config_{uuid.uuid4().hex[:8]}" - doc_id = _create_test_document(client, suffix=suffix) + doc_id = _add_document_to_session(client, session_id, suffix=suffix) path = os.path.join(tempfile.gettempdir(), f"test_doc{suffix}.txt") with open(path, "w", encoding="utf-8") as handle: handle.write(RAG_E2E_DOC_TEXT) - yield doc_id - with contextlib.suppress(OSError): - os.remove(path) + written_documents.append(path) + return doc_id def _base_generation_model() -> dict: @@ -70,8 +80,14 @@ def _base_generation_model() -> dict: return {"component": "StubConfigLLM", "params": {}} -def _create_minimal_session(client: TestClient, doc_id: int, name: str) -> int: - """Create a RAG session from the minimum the API accepts.""" +def _create_minimal_session( + client: TestClient, name: str, written_documents: list +) -> "tuple[int, int]": + """Create a RAG session from the minimum the API accepts, plus a document. + + A session is created empty and gains its documents afterwards, so the two + steps are bundled here and both ids returned. + """ response = client.post( "/api/v1/generative-session/", json={ @@ -79,13 +95,14 @@ def _create_minimal_session(client: TestClient, doc_id: int, name: str) -> int: "task_name": "RAGTask", "name": name, "parameters": { - "documents": [doc_id], "generation_model": {"component": "StubConfigLLM", "params": {}}, }, }, ) assert response.status_code == 201, response.text - return response.json()["id"] + session_id = response.json()["id"] + document_id = _attach_indexable_document(client, session_id, written_documents) + return session_id, document_id def _run_one_chat_turn(client: TestClient, session_id: int) -> None: @@ -115,10 +132,10 @@ def _run_one_chat_turn(client: TestClient, session_id: int) -> None: def test_configuration_never_exposes_class_names( - client: TestClient, indexed_document: int + client: TestClient, written_documents: list ): - session_id = _create_minimal_session( - client, indexed_document, "config_no_class_names" + session_id, _ = _create_minimal_session( + client, "config_no_class_names", written_documents ) response = client.get(f"/api/v1/rag/sessions/{session_id}/configuration") @@ -135,9 +152,9 @@ def test_configuration_never_exposes_class_names( def test_configuration_labels_every_parameter( - client: TestClient, indexed_document: int + client: TestClient, written_documents: list ): - session_id = _create_minimal_session(client, indexed_document, "config_labels") + session_id, _ = _create_minimal_session(client, "config_labels", written_documents) data = client.get(f"/api/v1/rag/sessions/{session_id}/configuration").json() chunking_params = {p["name"]: p for p in data["chunking_model"]["params"]} @@ -147,9 +164,9 @@ def test_configuration_labels_every_parameter( def test_configuration_names_the_matching_presets( - client: TestClient, indexed_document: int + client: TestClient, written_documents: list ): - session_id = _create_minimal_session(client, indexed_document, "config_presets") + session_id, _ = _create_minimal_session(client, "config_presets", written_documents) data = client.get(f"/api/v1/rag/sessions/{session_id}/configuration").json() assert data["chunking_model"]["preset_key"] == "paragraph" @@ -158,8 +175,10 @@ def test_configuration_names_the_matching_presets( assert data["retriever_model"]["preset_display_name"] == "Keyword" -def test_configuration_is_localized(client: TestClient, indexed_document: int): - session_id = _create_minimal_session(client, indexed_document, "config_localized") +def test_configuration_is_localized(client: TestClient, written_documents: list): + session_id, _ = _create_minimal_session( + client, "config_localized", written_documents + ) data = client.get( f"/api/v1/rag/sessions/{session_id}/configuration", headers={"Accept-Language": "es"}, @@ -172,9 +191,9 @@ def test_configuration_is_localized(client: TestClient, indexed_document: int): def test_configuration_reports_the_context_budget( - client: TestClient, indexed_document: int + client: TestClient, written_documents: list ): - session_id = _create_minimal_session(client, indexed_document, "config_budget") + session_id, _ = _create_minimal_session(client, "config_budget", written_documents) budget = client.get(f"/api/v1/rag/sessions/{session_id}/configuration").json()[ "context_budget" ] @@ -193,7 +212,7 @@ def test_configuration_reports_the_context_budget( def test_default_session_context_budget_is_usable( - client: TestClient, indexed_document: int + client: TestClient, written_documents: list ): """A session created from the defaults must actually fit in its context. @@ -202,8 +221,8 @@ def test_default_session_context_budget_is_usable( sessions; if that stops happening, a brand-new session opens with a red "insufficient context" warning. """ - session_id = _create_minimal_session( - client, indexed_document, "config_default_budget" + session_id, _ = _create_minimal_session( + client, "config_default_budget", written_documents ) budget = client.get(f"/api/v1/rag/sessions/{session_id}/configuration").json()[ "context_budget" @@ -216,7 +235,7 @@ def test_default_session_context_budget_is_usable( def test_explicit_context_window_survives_the_override( - client: TestClient, indexed_document: int + client: TestClient, written_documents: list ): """A context window the caller sets is never overridden.""" base = _base_generation_model() @@ -228,7 +247,6 @@ def test_explicit_context_window_survives_the_override( "task_name": "RAGTask", "name": "config_explicit_window", "parameters": { - "documents": [indexed_document], "generation_model": base, }, }, @@ -239,10 +257,10 @@ def test_explicit_context_window_survives_the_override( def test_configuration_survives_an_unregistered_component( - client: TestClient, indexed_document: int + client: TestClient, written_documents: list ): """An uninstalled plugin must degrade to a raw name, not break the page.""" - session_id = _create_minimal_session(client, indexed_document, "config_unknown") + session_id, _ = _create_minimal_session(client, "config_unknown", written_documents) response = client.put( f"/api/v1/generative-session/{session_id}/parameters", json={ @@ -278,8 +296,10 @@ def test_configuration_404s_for_an_unknown_session(client: TestClient): # =================================================================== -def test_index_status_starts_not_indexed(client: TestClient, indexed_document: int): - session_id = _create_minimal_session(client, indexed_document, "index_fresh") +def test_index_status_starts_not_indexed(client: TestClient, written_documents: list): + session_id, indexed_document = _create_minimal_session( + client, "index_fresh", written_documents + ) data = client.get(f"/api/v1/rag/sessions/{session_id}/index-status").json() assert data["status"] == "not_indexed" @@ -293,9 +313,11 @@ def test_index_status_starts_not_indexed(client: TestClient, indexed_document: i def test_index_status_becomes_indexed_then_stale( - client: TestClient, indexed_document: int + client: TestClient, written_documents: list ): - session_id = _create_minimal_session(client, indexed_document, "index_lifecycle") + session_id, _ = _create_minimal_session( + client, "index_lifecycle", written_documents + ) _run_one_chat_turn(client, session_id) diff --git a/tests/back/RAG/test_RAG_session_flow.py b/tests/back/RAG/test_RAG_session_flow.py index b04608472..45923c91d 100644 --- a/tests/back/RAG/test_RAG_session_flow.py +++ b/tests/back/RAG/test_RAG_session_flow.py @@ -20,29 +20,21 @@ - Parameter change history tracking """ -import pytest from fastapi.testclient import TestClient -from tests.back.RAG.conftest import _create_test_document +from tests.back.RAG.conftest import _add_document_to_session # --------------------------------------------------------------------------- # helpers # --------------------------------------------------------------------------- -@pytest.fixture(scope="module") -def test_doc_id(client: TestClient) -> int: - """Module-scoped test document ID shared across all flow tests.""" - return _create_test_document(client, suffix="_rag_session_flow") - - -def _base_session_params(test_doc_id: int) -> dict: +def _base_session_params() -> dict: """Return the minimal valid RAG session payload.""" return { "model_name": "RAGPipeline", "task_name": "RAGTask", "parameters": { - "documents": [test_doc_id], "chunking_model": { "component": "CharacterChunkModel", "params": {"chunk_size": 400, "chunk_overlap": 40}, @@ -88,13 +80,22 @@ def _base_session_params(test_doc_id: int) -> dict: } -def _create_session(client: TestClient, test_doc_id: int, name: str) -> dict: - """Create a minimal valid RAG session and return its JSON response.""" - params = _base_session_params(test_doc_id) +def _create_session(client: TestClient, name: str) -> dict: + """Create a minimal valid RAG session holding one document. + + A session is always created empty, so the document is attached afterwards + and the session re-read, giving callers a payload whose ``documents`` list + is already populated. + """ + params = _base_session_params() params["name"] = name resp = client.post("/api/v1/generative-session/", json=params) assert resp.status_code == 201, f"Session prereq failed: {resp.text}" - return resp.json() + session_id = resp.json()["id"] + _add_document_to_session(client, session_id, suffix=f"_flow_{session_id}") + refreshed = client.get(f"/api/v1/generative-session/{session_id}") + assert refreshed.status_code == 200, refreshed.text + return refreshed.json() def _create_prompt(client: TestClient, template: str, name: str) -> int: @@ -121,11 +122,10 @@ class TestParameterStateTransitions: def test_update_preserves_unmentioned_params( self, client: TestClient, - test_doc_id: int, ): """PUT with only ``generation_model`` → all other parameters preserved; only ``generation_model`` reflects the new values.""" - session = _create_session(client, test_doc_id, "flow_preserve_unmentioned") + session = _create_session(client, "flow_preserve_unmentioned") session_id = session["id"] new_gen = { @@ -154,7 +154,7 @@ def test_update_preserves_unmentioned_params( assert params["chunking_model"]["component"] == "CharacterChunkModel" assert params["chunking_model"]["params"]["chunk_size"] == 400 assert params["retriever_model"]["component"] == "BM25Retriever" - assert params["documents"] == [test_doc_id] + assert params["documents"] == session["parameters"]["documents"] # Verify persistence via GET (the response from PUT is the same shape) get_resp = client.get(f"/api/v1/generative-session/{session_id}") @@ -166,11 +166,10 @@ def test_update_preserves_unmentioned_params( def test_update_clears_old_retriever_when_changed( self, client: TestClient, - test_doc_id: int, ): """PUT to replace the retriever from BM25 → TFIDF results in the observable state showing the new retriever.""" - session = _create_session(client, test_doc_id, "flow_retriever_change") + session = _create_session(client, "flow_retriever_change") session_id = session["id"] new_retriever = { @@ -209,7 +208,6 @@ def test_update_clears_old_retriever_when_changed( def test_update_rollback_on_error( self, client: TestClient, - test_doc_id: int, ): """PUT with a structurally invalid component dict returns 400 and the session's stored parameters remain unchanged (no partial update). @@ -221,7 +219,7 @@ def test_update_rollback_on_error( field values (e.g. ``temperature`` type) are validated against their own schema during PUT as well. """ - session = _create_session(client, test_doc_id, "flow_rollback") + session = _create_session(client, "flow_rollback") session_id = session["id"] original_gen = dict(session["parameters"]["generation_model"]) @@ -251,11 +249,10 @@ def test_update_rollback_on_error( def test_update_invalid_then_valid( self, client: TestClient, - test_doc_id: int, ): """A failed PUT (400) does not corrupt the session — a subsequent valid PUT succeeds with the correct final state.""" - session = _create_session(client, test_doc_id, "flow_invalid_then_valid") + session = _create_session(client, "flow_invalid_then_valid") session_id = session["id"] # ---- invalid PUT: malformed component structure ---- @@ -318,7 +315,6 @@ class TestPromptIDLifecycle: def test_prompt_id_replaced_by_prompt_in_params( self, client: TestClient, - test_doc_id: int, ): """When both ``prompt`` and ``prompt_id`` are stored in session parameters (e.g. after prompt cloning), a PUT with an empty body @@ -329,7 +325,7 @@ def test_prompt_id_replaced_by_prompt_in_params( prompt_id = _create_prompt(client, template, "flow_resolve_initial") # ---- create a session that includes BOTH prompt and prompt_id ---- - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_prompt_id_resolve" params["parameters"]["prompt_id"] = prompt_id # prompt key is already present from _base_session_params @@ -369,7 +365,6 @@ def test_prompt_id_replaced_by_prompt_in_params( def test_update_prompt_id_resolves( self, client: TestClient, - test_doc_id: int, ): """PUT with a new ``prompt_id`` switches the session to a different prompt. The stored ``prompt`` config reflects the new prompt's @@ -382,7 +377,7 @@ def test_update_prompt_id_resolves( prompt_b_id = _create_prompt(client, template_b, "flow_prompt_switch_b") # ---- session with both default prompt AND prompt_a_id ---- - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_prompt_id_switch" params["parameters"]["prompt_id"] = prompt_a_id @@ -420,11 +415,10 @@ class TestSessionLifecycle: def test_create_then_delete_rag_session( self, client: TestClient, - test_doc_id: int, ): """A valid RAG session can be created (201), then deleted (204). Subsequent GET returns 404.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_create_delete" resp = client.post("/api/v1/generative-session/", json=params) assert resp.status_code == 201, resp.text @@ -456,11 +450,10 @@ def test_delete_nonexistent_session_bug( def test_multiple_updates_accumulate( self, client: TestClient, - test_doc_id: int, ): """Sequential PUTs for different parameters (A → B → C) all accumulate in the final session state.""" - session = _create_session(client, test_doc_id, "flow_multiple_updates") + session = _create_session(client, "flow_multiple_updates") session_id = session["id"] # ---- PUT A: change generation_model ---- @@ -520,7 +513,7 @@ def test_multiple_updates_accumulate( # Unchanged params still present assert p["retriever_model"]["component"] == "BM25Retriever" - assert p["documents"] == [test_doc_id] + assert p["documents"] == session["parameters"]["documents"] # =================================================================== @@ -548,11 +541,10 @@ class TestCrossComponentValidation: def test_component_missing_params_key( self, client: TestClient, - test_doc_id: int, ): """Component dict without ``params`` key fails structure validation → 400.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_missing_params" params["parameters"]["generation_model"] = { "component": "Llama32_1BInstruct", @@ -567,11 +559,10 @@ def test_component_missing_params_key( def test_component_missing_component_key( self, client: TestClient, - test_doc_id: int, ): """Component dict without ``component`` key fails structure validation → 400.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_missing_component" params["parameters"]["prompt"] = { "params": {"language": "en"}, @@ -586,11 +577,10 @@ def test_component_missing_component_key( def test_component_wrong_type( self, client: TestClient, - test_doc_id: int, ): """Component value that is not a dict fails structure validation → 400.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_wrong_component_type" params["parameters"]["chunking_model"] = "not_a_dict" resp = client.post("/api/v1/generative-session/", json=params) @@ -605,14 +595,13 @@ def test_component_wrong_type( def test_subcomponent_temperature_string_rejected( self, client: TestClient, - test_doc_id: int, ): """``temperature: "not-a-number"`` is REJECTED (400) at session creation — sub-component field types are validated recursively against ``LlamaSchema.temperature`` (``float_field(ge=0.0, le=1.0)``) at create/update time, not deferred to pipeline runtime. """ - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_temp_string_rejected" params["parameters"]["generation_model"]["params"]["temperature"] = ( "not-a-number" @@ -627,13 +616,12 @@ def test_subcomponent_temperature_string_rejected( def test_subcomponent_negative_chunk_size_rejected( self, client: TestClient, - test_doc_id: int, ): """``chunk_size: -1`` is REJECTED (400) at session creation — ``CharacterChunkModelSchema.chunk_size`` uses ``int_field(gt=1)`` and sub-component validation now propagates into nested schemas. """ - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_neg_chunk_rejected" params["parameters"]["chunking_model"]["params"]["chunk_size"] = -1 @@ -646,13 +634,12 @@ def test_subcomponent_negative_chunk_size_rejected( def test_subcomponent_overlap_equals_size_rejected( self, client: TestClient, - test_doc_id: int, ): """``chunk_overlap == chunk_size`` is REJECTED (400) at session creation — the cross-field validator in ``CharacterChunkModelSchema`` now runs at create/update time. """ - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_overlap_eq_rejected" params["parameters"]["chunking_model"]["params"]["chunk_size"] = 100 params["parameters"]["chunking_model"]["params"]["chunk_overlap"] = 100 @@ -666,13 +653,12 @@ def test_subcomponent_overlap_equals_size_rejected( def test_subcomponent_temperature_out_of_range_rejected( self, client: TestClient, - test_doc_id: int, ): """``temperature=2.5`` is REJECTED (400) at session creation — ``LlamaSchema.temperature`` has ``float_field(ge=0.0, le=1.0)`` and this constraint is now enforced at create/update time. """ - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_temp_range_rejected" params["parameters"]["generation_model"]["params"]["temperature"] = 2.5 @@ -695,11 +681,10 @@ class TestHistoryTracking: def test_parameter_update_logs_history( self, client: TestClient, - test_doc_id: int, ): """A PUT that changes parameters creates a history entry retrievable via the parameters-history endpoint.""" - session = _create_session(client, test_doc_id, "flow_history_basic") + session = _create_session(client, "flow_history_basic") session_id = session["id"] # PUT a change @@ -739,11 +724,10 @@ def test_parameter_update_logs_history( def test_history_contains_initial_state( self, client: TestClient, - test_doc_id: int, ): """Session creation also logs an initial history entry with the original parameters.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "flow_history_initial" resp = client.post("/api/v1/generative-session/", json=params) assert resp.status_code == 201, resp.text @@ -765,11 +749,10 @@ def test_history_contains_initial_state( def test_multiple_updates_create_multiple_history_entries( self, client: TestClient, - test_doc_id: int, ): """Three sequential PUTs produce distinct history entries, each capturing the parameter state at that point in time.""" - session = _create_session(client, test_doc_id, "flow_history_multiple") + session = _create_session(client, "flow_history_multiple") session_id = session["id"] # Three updates with different temperatures diff --git a/tests/back/RAG/test_RAG_session_validation.py b/tests/back/RAG/test_RAG_session_validation.py index 98ae0fd0a..1f0f8e744 100644 --- a/tests/back/RAG/test_RAG_session_validation.py +++ b/tests/back/RAG/test_RAG_session_validation.py @@ -29,17 +29,16 @@ @pytest.fixture(scope="module") def test_doc_id(client: TestClient) -> int: - """Module-scoped test document shared across all tests in this file.""" + """A document in its own session, for tests that need one to exist.""" return _create_test_document(client, suffix="_session_validation") -def _base_session_params(test_doc_id: int) -> dict: +def _base_session_params() -> dict: """Return the minimal valid RAG session payload (BM25 + Llama + DefaultPrompt).""" return { "model_name": "RAGPipeline", "task_name": "RAGTask", "parameters": { - "documents": [test_doc_id], "chunking_model": { "component": "CharacterChunkModel", "params": {"chunk_size": 400, "chunk_overlap": 40}, @@ -97,10 +96,10 @@ class TestCreateRAGSession: # Valid creation # ------------------------------------------------------------------ - def test_create_valid_rag_session(self, client: TestClient, test_doc_id: int): + def test_create_valid_rag_session(self, client: TestClient): """Creates a session with ALL valid RAG parameters, asserts 201 and the full response shape.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_create_valid_rag_session" response = client.post("/api/v1/generative-session/", json=params) @@ -124,17 +123,15 @@ def test_create_valid_rag_session(self, client: TestClient, test_doc_id: int): # --- parameters --- params_data = data["parameters"] - assert params_data["documents"] == [test_doc_id] + assert params_data["documents"] == [] # a session starts empty assert params_data["prompt"]["component"] == "DefaultRAGGenerationPrompt" assert params_data["chunking_model"]["component"] == "CharacterChunkModel" assert params_data["retriever_model"]["component"] == "BM25Retriever" assert params_data["generation_model"]["component"] == "Llama32_1BInstruct" - def test_create_rag_session_with_custom_description( - self, client: TestClient, test_doc_id: int - ): + def test_create_rag_session_with_custom_description(self, client: TestClient): """Session creation with a non-None description is stored correctly.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_create_with_description" params["description"] = "A test session for RAG validation" @@ -144,10 +141,10 @@ def test_create_rag_session_with_custom_description( @pytest.mark.parametrize("missing_key", ["model_name", "task_name", "name"]) def test_create_rag_session_missing_top_level_field( - self, client: TestClient, test_doc_id: int, missing_key: str + self, client: TestClient, missing_key: str ): """Omitting a top-level required field returns 422 Unprocessable Entity.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = f"test_missing_{missing_key}" del params[missing_key] @@ -163,13 +160,13 @@ def test_create_rag_session_missing_top_level_field( @pytest.mark.parametrize( "missing_param_key", - ["generation_model", "documents"], + ["generation_model"], ) def test_create_rag_session_missing_required_parameter( - self, client: TestClient, test_doc_id: int, missing_param_key: str + self, client: TestClient, missing_param_key: str ): """Omitting a key with no sensible default returns 400.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = f"test_missing_param_{missing_param_key}" del params["parameters"][missing_param_key] @@ -186,10 +183,10 @@ def test_create_rag_session_missing_required_parameter( ["prompt", "chunking_model", "retriever_model"], ) def test_create_rag_session_defaults_missing_parameter( - self, client: TestClient, test_doc_id: int, missing_param_key: str + self, client: TestClient, missing_param_key: str ): """Omitting a defaulted key succeeds and stores the backend default.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = f"test_defaulted_param_{missing_param_key}" del params["parameters"][missing_param_key] @@ -202,17 +199,14 @@ def test_create_rag_session_defaults_missing_parameter( assert stored["component"], f"{missing_param_key} was not resolved" assert isinstance(stored["params"], dict) - def test_create_rag_session_with_only_documents_and_model( - self, client: TestClient, test_doc_id: int - ): - """Name, documents and a generation model are enough to create a session.""" - base = _base_session_params(test_doc_id) + def test_create_rag_session_with_only_a_model(self, client: TestClient): + """A name and a generation model are enough to create a session.""" + base = _base_session_params() params = { "model_name": base["model_name"], "task_name": base["task_name"], "name": "test_minimal_creation", "parameters": { - "documents": base["parameters"]["documents"], "generation_model": base["parameters"]["generation_model"], }, } @@ -242,10 +236,10 @@ def test_create_rag_session_with_only_documents_and_model( ], ) def test_create_rag_session_bad_component_structure( - self, client: TestClient, test_doc_id: int, component_key: str, bad_value + self, client: TestClient, component_key: str, bad_value ): """Malformed component dict (missing keys / wrong type) returns 400.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = f"test_bad_struct_{component_key}" params["parameters"][component_key] = bad_value @@ -271,12 +265,11 @@ def test_create_rag_session_bad_component_structure( def test_create_rag_session_invalid_component_name( self, client: TestClient, - test_doc_id: int, component_key: str, invalid_name: str, ): """Non-existent component names now return 400 (validated against registry).""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = f"test_bad_comp_{component_key}" # Replace the target component with an invalid one @@ -297,11 +290,9 @@ def test_create_rag_session_invalid_component_name( # Invalid model / task name (caught by registry lookup → 400) # ------------------------------------------------------------------ - def test_create_rag_session_invalid_model_name( - self, client: TestClient, test_doc_id: int - ): + def test_create_rag_session_invalid_model_name(self, client: TestClient): """A non-registered model_name returns 400.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_invalid_model_name" params["model_name"] = "TotallyNotRealModel" @@ -311,11 +302,9 @@ def test_create_rag_session_invalid_model_name( f" {response.text}" ) - def test_create_rag_session_invalid_task_name( - self, client: TestClient, test_doc_id: int - ): + def test_create_rag_session_invalid_task_name(self, client: TestClient): """A non-registered task_name returns 400.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_invalid_task_name" params["task_name"] = "NonExistentTask" @@ -325,14 +314,12 @@ def test_create_rag_session_invalid_task_name( f" {response.text}" ) - def test_create_rag_session_model_not_generative( - self, client: TestClient, test_doc_id: int - ): + def test_create_rag_session_model_not_generative(self, client: TestClient): """A model that is not a subclass of BaseGenerativeModel returns 400. ``DummyClassifier`` is registered but is NOT a generative model. """ - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_model_not_generative" params["model_name"] = "DummyClassifier" # exists but is not generative @@ -346,62 +333,47 @@ def test_create_rag_session_model_not_generative( # Document validation # ------------------------------------------------------------------ - def test_create_rag_session_invalid_documents( - self, client: TestClient, test_doc_id: int - ): - """Non-existent document IDs return 400 (caught by endpoint).""" - params = _base_session_params(test_doc_id) - params["name"] = "test_invalid_docs" - params["parameters"]["documents"] = [99999] # does not exist + def test_create_rag_session_rejects_documents(self, client: TestClient): + """Sending a document list at creation is refused, not silently dropped. + + There is no session to attach documents to until it exists, so they are + uploaded into the session afterwards. + """ + params = _base_session_params() + params["name"] = "test_create_rejects_documents" + params["parameters"]["documents"] = [1] resp = client.post("/api/v1/generative-session/", json=params) - assert resp.status_code == 400 + assert resp.status_code == 400, resp.text + assert "documents" in resp.text.lower() - def test_create_rag_session_empty_documents( - self, client: TestClient, test_doc_id: int - ): - """Empty documents list is rejected with 400.""" - params = _base_session_params(test_doc_id) - params["name"] = "test_empty_docs" + def test_create_rag_session_accepts_empty_documents(self, client: TestClient): + """An explicit empty list is the state every session starts in.""" + params = _base_session_params() + params["name"] = "test_create_empty_documents" params["parameters"]["documents"] = [] - response = client.post("/api/v1/generative-session/", json=params) - assert response.status_code == 400, ( - f"Empty documents list should be rejected, " - f"got {response.status_code}: {response.text}" - ) - - def test_create_rag_session_document_zero( - self, client: TestClient, test_doc_id: int - ): - """Document ID = 0 returns 400 (caught by endpoint).""" - params = _base_session_params(test_doc_id) - params["name"] = "test_doc_id_zero" - params["parameters"]["documents"] = [0] - resp = client.post("/api/v1/generative-session/", json=params) - assert resp.status_code == 400 + assert resp.status_code == 201, resp.text + assert resp.json()["parameters"]["documents"] == [] - def test_create_rag_session_document_negative( - self, client: TestClient, test_doc_id: int - ): - """Negative document ID returns 400 (caught by endpoint).""" - params = _base_session_params(test_doc_id) - params["name"] = "test_doc_id_negative" - params["parameters"]["documents"] = [-5] + def test_create_rag_session_omitting_documents(self, client: TestClient): + """Omitting the key stores an empty list rather than failing.""" + params = _base_session_params() + params["name"] = "test_create_omits_documents" + params["parameters"].pop("documents", None) resp = client.post("/api/v1/generative-session/", json=params) - assert resp.status_code == 400 + assert resp.status_code == 201, resp.text + assert resp.json()["parameters"]["documents"] == [] # ------------------------------------------------------------------ # Miscellaneous edge-cases # ------------------------------------------------------------------ - def test_create_rag_session_duplicate_name( - self, client: TestClient, test_doc_id: int - ): + def test_create_rag_session_duplicate_name(self, client: TestClient): """Second creation with the same name returns 409 Conflict.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_duplicate_name" resp1 = client.post("/api/v1/generative-session/", json=params) @@ -412,10 +384,10 @@ def test_create_rag_session_duplicate_name( f"Expected 409 for duplicate name, got {resp2.status_code}: {resp2.text}" ) - def test_create_rag_session_empty_name(self, client: TestClient, test_doc_id: int): + def test_create_rag_session_empty_name(self, client: TestClient): """An empty or whitespace-only name may be treated differently by the schema — at minimum it should not crash.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "" response = client.post("/api/v1/generative-session/", json=params) @@ -426,16 +398,14 @@ def test_create_rag_session_empty_name(self, client: TestClient, test_doc_id: in f"Unexpected status for empty name: {response.status_code}: {response.text}" ) - def test_create_rag_session_unknown_parameter_key( - self, client: TestClient, test_doc_id: int - ): + def test_create_rag_session_unknown_parameter_key(self, client: TestClient): """Extra unknown keys inside ``parameters``. Pydantic v2 BaseModel with default config ignores extra fields during ``model_validate``, so the unknown key is silently accepted (201). The key is also stored in the session parameters. """ - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_unknown_param_key" params["parameters"]["unexpected_extra_key"] = "should_be_ignored" @@ -461,9 +431,9 @@ class TestUpdateRAGSessionParams: # ------------------------------------------------------------------ @staticmethod - def _create_session(client: TestClient, test_doc_id: int, name: str) -> dict: + def _create_session(client: TestClient, name: str) -> dict: """Create a minimal valid session and return its JSON response.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = name resp = client.post("/api/v1/generative-session/", json=params) assert resp.status_code == 201, f"Session prereq failed: {resp.text}" @@ -473,12 +443,12 @@ def _create_session(client: TestClient, test_doc_id: int, name: str) -> dict: # Successful updates # ------------------------------------------------------------------ - def test_update_rag_session_partial(self, client: TestClient, test_doc_id: int): + def test_update_rag_session_partial(self, client: TestClient): """Sends only ``generation_model`` change → 200 with merged params. Unchanged keys (prompt, chunking_model, …) must be preserved. """ - session = self._create_session(client, test_doc_id, "test_partial_update") + session = self._create_session(client, "test_partial_update") session_id = session["id"] new_gen = { @@ -512,15 +482,12 @@ def test_update_rag_session_partial(self, client: TestClient, test_doc_id: int): data["parameters"]["chunking_model"]["component"] == "CharacterChunkModel" ) - def test_update_rag_session_full_replacement( - self, client: TestClient, test_doc_id: int - ): + def test_update_rag_session_full_replacement(self, client: TestClient): """Replaces every RAG parameter at once → 200 with all new values.""" - session = self._create_session(client, test_doc_id, "test_full_replacement") + session = self._create_session(client, "test_full_replacement") session_id = session["id"] replacement = { - "documents": [test_doc_id], "chunking_model": { "component": "RecursiveCharacterChunkModel", "params": { @@ -583,9 +550,9 @@ def test_update_rag_session_full_replacement( assert p["retriever_model"]["params"]["top_k"] == 10 assert p["generation_model"]["component"] == "Llama32_1BInstruct" - def test_update_rag_session_prompt_only(self, client: TestClient, test_doc_id: int): + def test_update_rag_session_prompt_only(self, client: TestClient): """Updates only the prompt component → 200, prompt changed, others preserved.""" - session = self._create_session(client, test_doc_id, "test_update_prompt_only") + session = self._create_session(client, "test_update_prompt_only") session_id = session["id"] new_prompt = { @@ -606,9 +573,9 @@ def test_update_rag_session_prompt_only(self, client: TestClient, test_doc_id: i # Empty / no-op updates # ------------------------------------------------------------------ - def test_update_rag_session_empty_body(self, client: TestClient, test_doc_id: int): + def test_update_rag_session_empty_body(self, client: TestClient): """Empty dict ``{}`` → 200 no-op (merged params are identical to old).""" - session = self._create_session(client, test_doc_id, "test_update_empty_body") + session = self._create_session(client, "test_update_empty_body") session_id = session["id"] resp = client.put( @@ -639,15 +606,12 @@ def test_update_rag_session_empty_body(self, client: TestClient, test_doc_id: in def test_update_rag_session_invalid_component( self, client: TestClient, - test_doc_id: int, component_key: str, invalid_name: str, ): """Invalid component name in PUT now returns 400 (validated against registry).""" - session = self._create_session( - client, test_doc_id, f"test_update_invalid_{component_key}" - ) + session = self._create_session(client, f"test_update_invalid_{component_key}") session_id = session["id"] resp = client.put( @@ -690,14 +654,13 @@ def test_update_rag_session_invalid_component( def test_update_rag_session_bad_component_structure( self, client: TestClient, - test_doc_id: int, component_key: str, bad_value, idx: int, ): """Malformed component values → 400.""" session = self._create_session( - client, test_doc_id, f"test_update_bad_struct_{component_key}_{idx}" + client, f"test_update_bad_struct_{component_key}_{idx}" ) session_id = session["id"] @@ -714,46 +677,34 @@ def test_update_rag_session_bad_component_structure( # Document-related edge-cases # ------------------------------------------------------------------ - def test_update_rag_session_invalid_documents( + def test_update_rag_session_rejects_documents( self, client: TestClient, test_doc_id: int ): - """Updating ``documents`` to non-existent IDs now returns 400.""" - session = self._create_session(client, test_doc_id, "test_update_invalid_docs") - session_id = session["id"] - - resp = client.put( - f"/api/v1/generative-session/{session_id}/parameters", - json={"documents": [99999]}, - ) - assert resp.status_code == 400, ( - f"Non-existent doc ID should return 400, " - f"got {resp.status_code}: {resp.text}" - ) + """``documents`` cannot be set through session parameters. - def test_update_rag_session_documents_empty( - self, client: TestClient, test_doc_id: int - ): - """Updating documents to an empty list → 400 (empty not allowed).""" - session = self._create_session(client, test_doc_id, "test_update_docs_empty") + The foreign key on ``document`` decides which documents a session owns; + accepting the list here would let the two disagree. + """ + session = self._create_session(client, "test_update_rejects_documents") session_id = session["id"] - resp = client.put( - f"/api/v1/generative-session/{session_id}/parameters", - json={"documents": []}, - ) - assert resp.status_code == 400, ( - f"Empty documents should return 400, got {resp.status_code}: {resp.text}" - ) + for payload in ({"documents": [test_doc_id]}, {"documents": []}): + resp = client.put( + f"/api/v1/generative-session/{session_id}/parameters", + json=payload, + ) + assert resp.status_code == 400, ( + f"{payload} should be refused, got {resp.status_code}: {resp.text}" + ) + assert "documents" in resp.text.lower() # ------------------------------------------------------------------ # prompt_id edge cases # ------------------------------------------------------------------ - def test_update_rag_session_invalid_prompt_id( - self, client: TestClient, test_doc_id: int - ): + def test_update_rag_session_invalid_prompt_id(self, client: TestClient): """``prompt_id: 999`` (non-existent) now returns 400.""" - session = self._create_session(client, test_doc_id, "test_update_bad_prompt_id") + session = self._create_session(client, "test_update_bad_prompt_id") session_id = session["id"] resp = client.put( @@ -764,9 +715,7 @@ def test_update_rag_session_invalid_prompt_id( f"Invalid prompt_id should return 400, got {resp.status_code}: {resp.text}" ) - def test_update_rag_session_valid_prompt_id( - self, client: TestClient, test_doc_id: int - ): + def test_update_rag_session_valid_prompt_id(self, client: TestClient): """``prompt_id`` pointing to an existing prompt → prompt config resolved. The ``PromptService.resolve_prompt_id_to_component`` replaces the @@ -784,9 +733,7 @@ def test_update_rag_session_valid_prompt_id( ) prompt_id = prompt_resp.json()["id"] - session = self._create_session( - client, test_doc_id, "test_update_valid_prompt_id" - ) + session = self._create_session(client, "test_update_valid_prompt_id") session_id = session["id"] resp = client.put( @@ -804,13 +751,13 @@ def test_update_rag_session_valid_prompt_id( # Unknown keys # ------------------------------------------------------------------ - def test_update_rag_session_unknown_key(self, client: TestClient, test_doc_id: int): + def test_update_rag_session_unknown_key(self, client: TestClient): """Unknown key in PUT body is merged into parameters. Pydantic ``model_validate`` with default ``extra='ignore'`` tolerates extra keys, so the unknown key is persisted in the session. """ - session = self._create_session(client, test_doc_id, "test_update_unknown_key") + session = self._create_session(client, "test_update_unknown_key") session_id = session["id"] resp = client.put( @@ -846,12 +793,10 @@ def test_update_rag_session_nonexistent_session(self, client: TestClient): # Missing required keys after merge # ------------------------------------------------------------------ - def test_update_rag_session_remove_required_key( - self, client: TestClient, test_doc_id: int - ): + def test_update_rag_session_remove_required_key(self, client: TestClient): """Overwriting a required key with something that fails structure validation → 400.""" - session = self._create_session(client, test_doc_id, "test_remove_required_key") + session = self._create_session(client, "test_remove_required_key") session_id = session["id"] resp = client.put( @@ -1030,12 +975,12 @@ class TestRetrieverConfigRegression: # ------------------------------------------------------------------ def test_dense_embedding_retriever_preserves_component_name( - self, client: TestClient, test_doc_id: int + self, client: TestClient ): """Regression: creating a session with DenseEmbeddingRetriever must store ``component: "DenseEmbeddingRetriever"`` — NOT the embedding model name.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_dense_component_preserved" # Replace retriever with DenseEmbeddingRetriever + SentenceTransformer @@ -1077,11 +1022,11 @@ def test_dense_embedding_retriever_preserves_component_name( # ------------------------------------------------------------------ def test_composite_retriever_rejects_empty_child_component( - self, client: TestClient, test_doc_id: int + self, client: TestClient ): """Regression: a composite retriever with an empty-component child must be rejected with a clear validation error.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_composite_empty_child" # Build a ParallelRetriever with one valid child (BM25) and one @@ -1136,7 +1081,7 @@ def test_composite_retriever_rejects_empty_child_component( # ------------------------------------------------------------------ def test_bare_embedding_as_child_accepts_but_fails_at_runtime( - self, client: TestClient, test_doc_id: int + self, client: TestClient ): """Regression: a SentenceTransformerEmbedding used directly as a child of a composite retriever IS accepted during session creation @@ -1146,7 +1091,7 @@ def test_bare_embedding_as_child_accepts_but_fails_at_runtime( The frontend fix prevents this scenario by filtering embedding models out of the composite child selector in RetrieverNodeConfig. """ - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_bare_embedding_as_child" params["parameters"]["retriever_model"] = { @@ -1208,13 +1153,11 @@ def test_bare_embedding_as_child_accepts_but_fails_at_runtime( # Auto-save partial-data regression # ------------------------------------------------------------------ - def test_auto_save_partial_data_rejected( - self, client: TestClient, test_doc_id: int - ): + def test_auto_save_partial_data_rejected(self, client: TestClient): """When auto-save fires with only ``embedding_model`` (simulating the frontend bug where store formValues is empty), the missing ``similarity_metric``/``top_k`` fields cause validation to fail.""" - params = _base_session_params(test_doc_id) + params = _base_session_params() params["name"] = "test_autosave_partial" params["parameters"]["retriever_model"] = { "component": "DenseEmbeddingRetriever", diff --git a/tests/back/RAG/test_RAG_strict_validation.py b/tests/back/RAG/test_RAG_strict_validation.py index 98114af5b..3aa39e397 100644 --- a/tests/back/RAG/test_RAG_strict_validation.py +++ b/tests/back/RAG/test_RAG_strict_validation.py @@ -23,11 +23,8 @@ placeholders) are rejected with HTTP 400. """ -import pytest from fastapi.testclient import TestClient -from tests.back.RAG.conftest import _create_test_document - COMPLETE_BM25_VECTORIZER_PARAMS = { "strip_accents": None, "lowercase": True, @@ -43,7 +40,7 @@ # --------------------------------------------------------------------------- -def _complete_params(doc_id, name="strict_valid"): +def _complete_params(name="strict_valid"): """Return a fully-valid RAG session payload for the given document.""" return { "model_name": "RAGPipeline", @@ -51,7 +48,6 @@ def _complete_params(doc_id, name="strict_valid"): "name": name, "description": None, "parameters": { - "documents": [doc_id], "chunking_model": { "component": "CharacterChunkModel", "params": {"chunk_size": 200, "chunk_overlap": 20}, @@ -106,22 +102,16 @@ def _bm25_retriever_ref(vectorizer_params: dict) -> dict: } -@pytest.fixture(scope="module") -def test_doc_id(client: TestClient) -> int: - """Module-scoped test document shared across all tests in this file.""" - return _create_test_document(client, suffix="_strict_validation") - - # =================================================================== # POST /api/v1/generative-session/ # =================================================================== def test_create_session_with_empty_vectorizer_params_rejected( - client: TestClient, test_doc_id: int + client: TestClient, ) -> None: """Empty vectorizer ``params`` are rejected — backend must not fill gaps.""" - params = _complete_params(test_doc_id, name="strict_incomplete_vectorizer") + params = _complete_params(name="strict_incomplete_vectorizer") params["parameters"]["retriever_model"]["params"]["BM25Vectorizer"]["params"] = {} response = client.post("/api/v1/generative-session/", json=params) @@ -132,14 +122,14 @@ def test_create_session_with_empty_vectorizer_params_rejected( def test_create_session_with_empty_generation_model_params_filled( - client: TestClient, test_doc_id: int + client: TestClient, ) -> None: """Empty generation-model params are filled from the model's own schema. The generation model is picked by name — the creation flow asks *which* model, not how to tune it — so its parameters are resolved by the backend. """ - params = _complete_params(test_doc_id, name="strict_incomplete_llama") + params = _complete_params(name="strict_incomplete_llama") params["parameters"]["generation_model"]["params"] = {} response = client.post("/api/v1/generative-session/", json=params) @@ -153,10 +143,10 @@ def test_create_session_with_empty_generation_model_params_filled( def test_create_session_with_partial_generation_model_params_kept( - client: TestClient, test_doc_id: int + client: TestClient, ) -> None: """Explicit generation-model values survive the default filling.""" - params = _complete_params(test_doc_id, name="strict_partial_llama") + params = _complete_params(name="strict_partial_llama") params["parameters"]["generation_model"]["params"] = {"max_tokens": 77} response = client.post("/api/v1/generative-session/", json=params) @@ -167,11 +157,11 @@ def test_create_session_with_partial_generation_model_params_kept( def test_create_session_with_default_prompt_accepts_language_only( - client: TestClient, test_doc_id: int + client: TestClient, ) -> None: """A default prompt with only ``language`` is accepted and the injected template is persisted.""" - params = _complete_params(test_doc_id, name="strict_default_prompt_language_only") + params = _complete_params(name="strict_default_prompt_language_only") params["parameters"]["prompt"] = { "component": "DefaultRAGGenerationPrompt", "params": {"language": "en"}, @@ -188,11 +178,11 @@ def test_create_session_with_default_prompt_accepts_language_only( def test_create_session_with_custom_prompt_no_template_rejected( - client: TestClient, test_doc_id: int + client: TestClient, ) -> None: """A custom prompt without an explicit template is rejected — backend must not fill gaps.""" - params = _complete_params(test_doc_id, name="strict_custom_prompt_no_template") + params = _complete_params(name="strict_custom_prompt_no_template") params["parameters"]["prompt"] = { "component": "CustomRAGGenerationPrompt", "params": {}, @@ -205,11 +195,9 @@ def test_create_session_with_custom_prompt_no_template_rejected( ) -def test_create_session_with_invalid_subfield_rejected( - client: TestClient, test_doc_id: int -) -> None: +def test_create_session_with_invalid_subfield_rejected(client: TestClient) -> None: """A non-numeric ``temperature`` fails the recursive schema check → 400.""" - params = _complete_params(test_doc_id, name="strict_invalid_temperature") + params = _complete_params(name="strict_invalid_temperature") params["parameters"]["generation_model"]["params"]["temperature"] = "not-a-number" response = client.post("/api/v1/generative-session/", json=params) @@ -220,10 +208,10 @@ def test_create_session_with_invalid_subfield_rejected( def test_create_session_with_empty_default_prompt_template_normalized( - client: TestClient, test_doc_id: int + client: TestClient, ) -> None: """An empty default prompt template is replaced with the language template.""" - params = _complete_params(test_doc_id, name="strict_empty_default_template") + params = _complete_params(name="strict_empty_default_template") params["parameters"]["prompt"] = { "component": "DefaultRAGGenerationPrompt", "params": {"language": "en", "template": ""}, @@ -241,10 +229,10 @@ def test_create_session_with_empty_default_prompt_template_normalized( def test_create_session_with_whitespace_default_prompt_template_normalized( - client: TestClient, test_doc_id: int + client: TestClient, ) -> None: """A whitespace-only default prompt template is replaced.""" - params = _complete_params(test_doc_id, name="strict_whitespace_default_template") + params = _complete_params(name="strict_whitespace_default_template") params["parameters"]["prompt"] = { "component": "DefaultRAGGenerationPrompt", "params": {"language": "en", "template": " "}, @@ -262,10 +250,10 @@ def test_create_session_with_whitespace_default_prompt_template_normalized( def test_create_session_with_null_default_prompt_template_normalized( - client: TestClient, test_doc_id: int + client: TestClient, ) -> None: """A ``None`` default prompt template is replaced.""" - params = _complete_params(test_doc_id, name="strict_null_default_template") + params = _complete_params(name="strict_null_default_template") params["parameters"]["prompt"] = { "component": "DefaultRAGGenerationPrompt", "params": {"language": "en", "template": None}, @@ -283,10 +271,10 @@ def test_create_session_with_null_default_prompt_template_normalized( def test_create_session_with_custom_prompt_missing_placeholders_rejected( - client: TestClient, test_doc_id: int + client: TestClient, ) -> None: """A custom prompt template lacking the required placeholders → 400.""" - params = _complete_params(test_doc_id, name="strict_custom_prompt_no_placeholders") + params = _complete_params(name="strict_custom_prompt_no_placeholders") params["parameters"]["prompt"] = { "component": "CustomRAGGenerationPrompt", "params": {"template": "hello"}, @@ -300,11 +288,11 @@ def test_create_session_with_custom_prompt_missing_placeholders_rejected( def test_create_session_with_default_prompt_missing_language_rejected( - client: TestClient, test_doc_id: int + client: TestClient, ) -> None: """A default prompt without ``language`` is rejected — backend needs language to inject template.""" - params = _complete_params(test_doc_id, name="strict_default_prompt_no_language") + params = _complete_params(name="strict_default_prompt_no_language") params["parameters"]["prompt"] = { "component": "DefaultRAGGenerationPrompt", "params": {}, @@ -326,9 +314,9 @@ class TestUpdateStrictValidation: """Strict validation on parameter updates via PUT.""" @staticmethod - def _create_session(client: TestClient, test_doc_id: int, name: str) -> dict: + def _create_session(client: TestClient, name: str) -> dict: """Create a fully-valid session and return its JSON response.""" - params = _complete_params(test_doc_id, name=name) + params = _complete_params(name=name) response = client.post("/api/v1/generative-session/", json=params) assert response.status_code == 201, ( f"Session prereq failed: {response.status_code}: {response.text}" @@ -336,13 +324,11 @@ def _create_session(client: TestClient, test_doc_id: int, name: str) -> dict: return response.json() def test_update_parameters_rejects_incomplete_vectorizer( - self, client: TestClient, test_doc_id: int + self, client: TestClient ) -> None: """PUT with an empty vectorizer params dict is rejected — backend must not fill gaps.""" - session = self._create_session( - client, test_doc_id, "strict_update_incomplete_vectorizer" - ) + session = self._create_session(client, "strict_update_incomplete_vectorizer") response = client.put( f"/api/v1/generative-session/{session['id']}/parameters", @@ -354,12 +340,10 @@ def test_update_parameters_rejects_incomplete_vectorizer( ) def test_update_parameters_accepts_complete_vectorizer( - self, client: TestClient, test_doc_id: int + self, client: TestClient ) -> None: """PUT with a complete vectorizer is accepted and persisted as-is.""" - session = self._create_session( - client, test_doc_id, "strict_update_complete_vectorizer" - ) + session = self._create_session(client, "strict_update_complete_vectorizer") response = client.put( f"/api/v1/generative-session/{session['id']}/parameters", @@ -379,11 +363,11 @@ def test_update_parameters_accepts_complete_vectorizer( assert saved == COMPLETE_BM25_VECTORIZER_PARAMS def test_update_parameters_normalizes_nested_vectorizer_types( - self, client: TestClient, test_doc_id: int + self, client: TestClient ) -> None: """PUT with integer ``max_df``/``min_df`` persists them as floats.""" session = self._create_session( - client, test_doc_id, "strict_update_vectorizer_type_normalization" + client, "strict_update_vectorizer_type_normalization" ) vectorizer_params = { "strip_accents": None, @@ -411,11 +395,11 @@ def test_update_parameters_normalizes_nested_vectorizer_types( assert saved["min_df"] == 0.0 def test_update_parameters_with_default_prompt_accepts_language_only( - self, client: TestClient, test_doc_id: int + self, client: TestClient ) -> None: """PUT with a default prompt language-only body injects the template.""" session = self._create_session( - client, test_doc_id, "strict_update_default_prompt_language_only" + client, "strict_update_default_prompt_language_only" ) response = client.put( @@ -436,12 +420,10 @@ def test_update_parameters_with_default_prompt_accepts_language_only( assert "{chunks}" in template def test_update_parameters_normalizes_empty_default_prompt_template( - self, client: TestClient, test_doc_id: int + self, client: TestClient ) -> None: """PUT with an empty default prompt template replaces it.""" - session = self._create_session( - client, test_doc_id, "strict_update_empty_default_template" - ) + session = self._create_session(client, "strict_update_empty_default_template") response = client.put( f"/api/v1/generative-session/{session['id']}/parameters", @@ -462,7 +444,7 @@ def test_update_parameters_normalizes_empty_default_prompt_template( assert "{chunks}" in template def test_update_parameters_rejects_prompt_id_of_schema_invalid_prompt( - self, client: TestClient, test_doc_id: int + self, client: TestClient ) -> None: """A ``prompt_id`` whose resolved prompt is schema-invalid → 400. @@ -486,9 +468,7 @@ def test_update_parameters_rejects_prompt_id_of_schema_invalid_prompt( ) prompt_id = prompt_resp.json()["id"] - session = self._create_session( - client, test_doc_id, "strict_update_schema_invalid_prompt_id" - ) + session = self._create_session(client, "strict_update_schema_invalid_prompt_id") response = client.put( f"/api/v1/generative-session/{session['id']}/parameters", diff --git a/tests/back/RAG/test_cross_encoder_retriever.py b/tests/back/RAG/test_cross_encoder_retriever.py index 6a0562da6..428d1aa7e 100644 --- a/tests/back/RAG/test_cross_encoder_retriever.py +++ b/tests/back/RAG/test_cross_encoder_retriever.py @@ -8,10 +8,7 @@ import numpy as np import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from DashAI.back.dependencies.database.models import Base, GenerativeSession from DashAI.back.models.RAG.documents import Chunk from DashAI.back.models.RAG.exceptions import ( RAGRetrieverEmptyChildrenError, @@ -27,7 +24,6 @@ CrossEncoderRetriever, ) from DashAI.back.models.RAG.retrievers.retriever_model import RetrieverModel -from DashAI.back.services.RAG.cleanup_service import CleanupService def _make_chunk(chunk_id: int, doc_id: int = 1) -> Chunk: @@ -425,97 +421,3 @@ def test_mmr_falls_back_when_child_vectors_unavailable(): result = mmr.retrieve("query") assert [c.id for c in result] == [0, 1, 2] - - -def test_other_sessions_with_same_config_ignores_list_order(): - """Config matching treats document lists as equal regardless of order.""" - engine = create_engine("sqlite://") - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - db = Session() - try: - parameters = { - "documents": [1, 2], - "chunking_model": { - "component": "CharacterChunkModel", - "params": {"chunk_size": 400}, - }, - } - db.add( - GenerativeSession( - id=1, - task_name="RAGTask", - model_name="RAGPipeline", - parameters=parameters, - name="session-one", - ) - ) - db.add( - GenerativeSession( - id=2, - task_name="RAGTask", - model_name="RAGPipeline", - parameters={"documents": [2, 1]}, - name="session-two", - ) - ) - db.commit() - - service = CleanupService(db) - assert ( - service._other_sessions_with_same_config(1, parameters, keys=("documents",)) - is True - ) - finally: - db.close() - engine.dispose() - - -def test_other_sessions_with_same_config_ignores_current_session(): - """Cleanup config matching excludes the current session and finds others.""" - engine = create_engine("sqlite://") - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - db = Session() - try: - parameters = { - "documents": [1], - "chunking_model": { - "component": "CharacterChunkModel", - "params": {"chunk_size": 400}, - }, - } - db.add( - GenerativeSession( - id=1, - task_name="RAGTask", - model_name="RAGPipeline", - parameters=parameters, - name="session-one", - ) - ) - db.commit() - - service = CleanupService(db) - assert ( - service._other_sessions_with_same_config(1, parameters, keys=("documents",)) - is False - ) - - db.add( - GenerativeSession( - id=2, - task_name="RAGTask", - model_name="RAGPipeline", - parameters=parameters, - name="session-two", - ) - ) - db.commit() - assert ( - service._other_sessions_with_same_config(1, parameters, keys=("documents",)) - is True - ) - finally: - db.close() - engine.dispose() diff --git a/tests/back/RAG/test_cross_session_safety.py b/tests/back/RAG/test_cross_session_safety.py new file mode 100644 index 000000000..4da4259e2 --- /dev/null +++ b/tests/back/RAG/test_cross_session_safety.py @@ -0,0 +1,244 @@ +"""What one session's changes must not do to another. + +Documents belong to one session, but two of the rows the cleanup path deletes +are keyed by configuration alone -- `rag_chunking_model` and +`rag_embedding_model` -- so every session that settled on the same components +shares them. Since a new session takes the backend defaults, that is the +ordinary case rather than a corner one. +""" + +import json +import uuid + +import pytest +from fastapi.testclient import TestClient + +from DashAI.back.dependencies.database.models import ( + Document, + GenerativeSession, + RAGChunkingModel, + RAGPipeline, +) + + +def _new_session(client: TestClient, name: str) -> int: + """Create an empty RAG session and return its id.""" + session_factory = client.app.container["session_factory"] + with session_factory() as db: + session = GenerativeSession( + task_name="RAGTask", + model_name="RAGPipeline", + parameters={"documents": []}, + name=name, + ) + db.add(session) + db.commit() + return session.id + + +def _upload(client: TestClient, session_id: int, content: bytes, name: str): + """Upload one document into a session.""" + metadata = json.dumps({"file_name": name, "optional_metadata": {}}) + return client.post( + f"/api/v1/document/session/{session_id}", + files={"file": (name, content, "text/plain")}, + data={"metadata": metadata}, + ) + + +class TestSharedConfigRows: + """Cleanup must not delete rows another session's pipeline still uses.""" + + def test_chunking_model_survives_another_sessions_change( + self, client: TestClient + ) -> None: + """Two sessions on the same chunking share one row; one may re-configure. + + The cross-session guard this replaces compared document lists, which can + never match now, so it protected nothing. + """ + tag = uuid.uuid4().hex[:8] + keeper = _new_session(client, f"keeper_{tag}") + changer = _new_session(client, f"changer_{tag}") + + chunking = { + "component": "CharacterChunkModel", + "params": {"chunk_size": 400, "chunk_overlap": 40}, + } + session_factory = client.app.container["session_factory"] + with session_factory() as db: + shared = RAGChunkingModel( + class_name=chunking["component"], parameters=chunking["params"] + ) + db.add(shared) + db.flush() + # Both sessions' pipelines point at the one row, the way + # SetupService's lookup-or-create leaves them. + for session_id in (keeper, changer): + db.add( + RAGPipeline( + session_id=session_id, + name=f"pipeline_{session_id}", + chunking_model_id=shared.id, + ) + ) + db.get(GenerativeSession, session_id).parameters = { + "documents": [], + "chunking_model": chunking, + } + db.commit() + shared_id = shared.id + + response = client.put( + f"/api/v1/generative-session/{changer}/parameters", + json={ + "chunking_model": { + "component": "CharacterChunkModel", + "params": {"chunk_size": 200, "chunk_overlap": 20}, + } + }, + ) + assert response.status_code == 200, response.text + + with session_factory() as db: + assert db.get(RAGChunkingModel, shared_id) is not None, ( + "the other session's pipeline still points at this row" + ) + + def test_chunking_model_goes_once_nothing_uses_it(self, client: TestClient) -> None: + """The guard must not turn into a leak: a truly orphaned row is dropped.""" + tag = uuid.uuid4().hex[:8] + lonely = _new_session(client, f"lonely_{tag}") + + chunking = { + "component": "CharacterChunkModel", + "params": {"chunk_size": 411, "chunk_overlap": 41}, + } + session_factory = client.app.container["session_factory"] + with session_factory() as db: + row = RAGChunkingModel( + class_name=chunking["component"], parameters=chunking["params"] + ) + db.add(row) + db.flush() + db.add( + RAGPipeline( + session_id=lonely, + name=f"pipeline_{lonely}", + chunking_model_id=row.id, + ) + ) + db.get(GenerativeSession, lonely).parameters = { + "documents": [], + "chunking_model": chunking, + } + db.commit() + row_id = row.id + + response = client.put( + f"/api/v1/generative-session/{lonely}/parameters", + json={ + "chunking_model": { + "component": "CharacterChunkModel", + "params": {"chunk_size": 222, "chunk_overlap": 22}, + } + }, + ) + assert response.status_code == 200, response.text + + with session_factory() as db: + pipeline = db.query(RAGPipeline).filter_by(session_id=lonely).one_or_none() + # The pipeline is repointed or cleared; either way nothing references + # the old row, so it should not linger. + still_referenced = pipeline is not None and ( + pipeline.chunking_model_id == row_id + ) + if not still_referenced: + assert db.get(RAGChunkingModel, row_id) is None + + +class TestUploadRollback: + """A failed upload must not leave the session holding a text-less document.""" + + def test_failed_extraction_leaves_no_document_behind( + self, client: TestClient + ) -> None: + """Undecodable bytes fail extraction; the session must stay as it was.""" + tag = uuid.uuid4().hex[:8] + session_id = _new_session(client, f"rollback_{tag}") + + # Invalid UTF-8, which PlainTextExtractor cannot decode. + response = _upload( + client, session_id, b"caf\xe9 broken bytes", f"broken_{tag}.txt" + ) + assert response.status_code == 500, response.text + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + assert db.query(Document).filter_by(session_id=session_id).count() == 0, ( + "a document with no extractable text was left in the session" + ) + session = db.get(GenerativeSession, session_id) + assert session.parameters.get("documents") == [] + + listed = client.get(f"/api/v1/document/session/{session_id}") + assert listed.status_code == 200 + assert listed.json() == [] + + def test_a_good_upload_still_lands(self, client: TestClient) -> None: + """The rollback must not swallow successful uploads.""" + tag = uuid.uuid4().hex[:8] + session_id = _new_session(client, f"rollback_ok_{tag}") + + response = _upload(client, session_id, b"readable text", f"ok_{tag}.txt") + assert response.status_code == 201, response.text + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + assert db.query(Document).filter_by(session_id=session_id).count() == 1 + session = db.get(GenerativeSession, session_id) + assert session.parameters["documents"] == [response.json()["id"]] + + +class TestBulkDeleteAtomicity: + """Deleting several sessions is one transaction, files included.""" + + @pytest.mark.parametrize("count", [2]) + def test_bulk_delete_removes_every_session_and_its_files( + self, client: TestClient, count: int + ) -> None: + """The happy path still works with deletion deferred past the commit.""" + import os + + tag = uuid.uuid4().hex[:8] + session_ids = [] + paths = [] + for index in range(count): + session_id = _new_session(client, f"bulk_{tag}_{index}") + response = _upload( + client, + session_id, + f"bulk body {index}".encode(), + f"b_{tag}_{index}.txt", + ) + assert response.status_code == 201, response.text + session_ids.append(session_id) + session_factory = client.app.container["session_factory"] + with session_factory() as db: + paths.append(db.get(Document, response.json()["id"]).file_path) + + for path in paths: + assert os.path.exists(path) + + response = client.request( + "DELETE", "/api/v1/generative-session/", json={"ids": session_ids} + ) + assert response.status_code in (200, 204), response.text + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + for session_id in session_ids: + assert db.get(GenerativeSession, session_id) is None + assert db.query(Document).filter_by(session_id=session_id).count() == 0 + for path in paths: + assert not os.path.exists(path), "the blob outlived its only document" diff --git a/tests/back/RAG/test_deferred_fs.py b/tests/back/RAG/test_deferred_fs.py new file mode 100644 index 000000000..5cd1aa5f0 --- /dev/null +++ b/tests/back/RAG/test_deferred_fs.py @@ -0,0 +1,211 @@ +"""Filesystem removals must follow the transaction, not the call order. + +Deleting a file cannot be rolled back, so a removal issued while a transaction +is open leaves surviving rows pointing at nothing when that transaction fails. +These tests pin the guarantee down at the primitive, so no call site has to be +audited for it one at a time. +""" + +import os + +import pytest +from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy.orm import declarative_base, sessionmaker + +from DashAI.back.services.RAG.deferred_fs import remove_after_commit, remove_now + +Base = declarative_base() + + +class Row(Base): + """A stand-in for whatever row a file belongs to.""" + + __tablename__ = "deferred_fs_row" + id = Column(Integer, primary_key=True) + name = Column(String, nullable=False) + + +@pytest.fixture +def db(): + """An isolated in-memory session.""" + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine)() + yield session + session.close() + engine.dispose() + + +@pytest.fixture +def a_file(tmp_path): + """A file on disk, and its path.""" + path = tmp_path / "artifact.bin" + path.write_bytes(b"contents") + return str(path) + + +@pytest.fixture +def a_directory(tmp_path): + """A populated directory, and its path.""" + folder = tmp_path / "matrices" + folder.mkdir() + (folder / "matrix.npy").write_bytes(b"0123") + return str(folder) + + +def test_a_queued_path_survives_until_the_commit(db, a_file): + """Nothing is removed while the transaction is still open.""" + db.add(Row(id=1, name="keep")) + remove_after_commit(db, a_file) + + assert os.path.exists(a_file), "removed before the transaction committed" + + db.commit() + assert not os.path.exists(a_file) + + +def test_a_rollback_keeps_the_file(db, a_file): + """The rows that justified the removal survive, so the file must too.""" + db.add(Row(id=1, name="doomed")) + remove_after_commit(db, a_file) + + db.rollback() + + assert os.path.exists(a_file), ( + "the transaction failed, so the row still points at this file" + ) + + # And the queue is not carried into the next transaction. + db.add(Row(id=2, name="unrelated")) + db.commit() + assert os.path.exists(a_file) + + +def test_successive_commits_each_take_only_their_own(db, tmp_path): + """One session that commits repeatedly -- a job doing several operations. + + The listener is registered once, on the Session class, so it never + unregisters; what keeps successive commits honest is that the queue lives in + ``Session.info`` and is drained on each one. + """ + paths = [] + for index in range(3): + path = tmp_path / f"step{index}.bin" + path.write_bytes(b"x") + paths.append(str(path)) + + for index, path in enumerate(paths): + db.add(Row(id=index + 1, name=f"step{index}")) + remove_after_commit(db, path) + db.commit() + + assert not os.path.exists(path), "this commit's own path was not removed" + for later in paths[index + 1 :]: + assert os.path.exists(later), "a later commit's path was removed early" + + +def test_a_commit_with_an_empty_queue_is_a_no_op(db, a_file): + """A second commit must not re-run the removals the first one did.""" + db.add(Row(id=1, name="first")) + remove_after_commit(db, a_file) + db.commit() + assert not os.path.exists(a_file) + + # Re-create it: if the queue were not drained, this commit would remove it. + with open(a_file, "wb") as handle: + handle.write(b"recreated") + db.add(Row(id=2, name="second")) + db.commit() + assert os.path.exists(a_file) + + +def test_a_savepoint_rollback_keeps_the_outer_queue(db, a_file): + """Rolling back a savepoint must not cancel the whole transaction's queue. + + Both ``after_rollback`` and ``after_soft_rollback`` fire for a savepoint, so + treating either as "the transaction failed" drops paths the enclosing + transaction is still going to commit. + """ + db.add(Row(id=1, name="outer")) + remove_after_commit(db, a_file) + + savepoint = db.begin_nested() + db.add(Row(id=2, name="inner")) + savepoint.rollback() + + assert os.path.exists(a_file), "removed while the outer transaction lived" + + db.commit() + assert not os.path.exists(a_file), ( + "the savepoint rollback swallowed the outer transaction's queue" + ) + + +def test_a_committed_savepoint_still_defers_to_the_outer_commit(db, a_file): + """A path queued inside a savepoint waits for the real commit.""" + savepoint = db.begin_nested() + db.add(Row(id=1, name="inner")) + remove_after_commit(db, a_file) + savepoint.commit() + + assert os.path.exists(a_file), "a savepoint commit is not the transaction" + + db.commit() + assert not os.path.exists(a_file) + + +def test_directories_and_files_are_both_handled(db, a_file, a_directory): + """Artifact folders and document blobs travel the same queue.""" + remove_after_commit(db, a_directory) + remove_after_commit(db, a_file) + db.commit() + + assert not os.path.exists(a_directory) + assert not os.path.exists(a_file) + + +def test_queueing_nothing_is_harmless(db): + """A nullable column can be queued without the caller guarding it.""" + remove_after_commit(db, None) + remove_after_commit(db, "") + db.commit() # must not raise + + +def test_a_missing_path_does_not_break_the_commit(db, tmp_path): + """A path already gone is not an error: cleanup runs more than once.""" + remove_after_commit(db, str(tmp_path / "never-existed")) + db.add(Row(id=1, name="fine")) + db.commit() + + assert db.query(Row).count() == 1 + + +def test_each_session_carries_its_own_queue(a_file, tmp_path): + """One session committing must not remove another's pending paths.""" + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine) + first, second = factory(), factory() + + other = tmp_path / "second.bin" + other.write_bytes(b"x") + + remove_after_commit(first, a_file) + remove_after_commit(second, str(other)) + + first.commit() + assert not os.path.exists(a_file) + assert os.path.exists(other), "the other session's queue was flushed too" + + second.commit() + assert not os.path.exists(other) + + first.close() + second.close() + engine.dispose() + + +def test_remove_now_reports_rather_than_raises(tmp_path): + """The immediate helper is best-effort, so cleanup never fails a request.""" + remove_now(str(tmp_path / "absent")) + remove_now(None) diff --git a/tests/back/RAG/test_document_extractor_api.py b/tests/back/RAG/test_document_extractor_api.py index fd8e4f5e9..6a1ae3ea0 100644 --- a/tests/back/RAG/test_document_extractor_api.py +++ b/tests/back/RAG/test_document_extractor_api.py @@ -3,22 +3,42 @@ import json import os import tempfile +import uuid -from DashAI.back.dependencies.database.models import Document, RAGExtractor +from DashAI.back.dependencies.database.models import ( + Document, + GenerativeSession, + RAGExtractor, +) from DashAI.back.models.RAG.documents import DocumentFileType +def _create_session(db, name: str) -> int: + """Create an empty RAG session to own documents.""" + session = GenerativeSession( + task_name="RAGTask", + model_name="RAGPipeline", + parameters={"documents": []}, + name=name, + ) + db.add(session) + db.commit() + return session.id + + def _create_document( db, file_name: str, file_hash: str, content: str = "content" ) -> int: - """Create a txt test document (with extractor) in the DB and return its ID.""" + """Create a txt test document in its own session and return its ID.""" tmp_path = os.path.join(tempfile.gettempdir(), file_name) with open(tmp_path, "w", encoding="utf-8") as f: f.write(content) + session_id = _create_session(db, f"owner_of_{file_hash}") extractor = RAGExtractor(component_name="PlainTextExtractor", params={}) db.add(extractor) db.flush() doc = Document( + session_id=session_id, file_name=file_name, file_type="txt", file_path=tmp_path, @@ -26,6 +46,9 @@ def _create_document( extractor_id=extractor.id, ) db.add(doc) + db.flush() + session = db.get(GenerativeSession, session_id) + session.parameters = {"documents": [doc.id]} db.commit() db.refresh(doc) return doc.id @@ -97,8 +120,19 @@ def test_extract_document_not_found(self, client): class TestUploadWarmsExtractionCache: """Uploading a document warms the extraction cache (best-effort).""" - def _upload(self, client, file_name: str, content: bytes): - """POST a document via the upload endpoint.""" + def _session(self, client, name: str) -> int: + """Create an empty RAG session to upload into.""" + session_factory = client.app.container["session_factory"] + with session_factory() as db: + return _create_session(db, name) + + def _upload(self, client, file_name: str, content: bytes, session_id=None): + """POST a document into a session via the upload endpoint.""" + if session_id is None: + # Session names are unique, so keep them distinct per upload. + session_id = self._session( + client, f"upload_{file_name}_{uuid.uuid4().hex[:8]}" + ) metadata = json.dumps( { "file_name": file_name, @@ -106,7 +140,7 @@ def _upload(self, client, file_name: str, content: bytes): } ) return client.post( - "/api/v1/document/", + f"/api/v1/document/session/{session_id}", files={"file": (file_name, content, "text/plain")}, data={"metadata": metadata}, ) @@ -125,39 +159,29 @@ def test_upload_txt_warms_cache(self, client): assert data["text"] == "Cache warming text." assert data["extractor"]["component"] == "PlainTextExtractor" - def test_upload_dedup_returns_409_without_force(self, client): - """Re-uploading the same file (hash dedup) returns 409 + existing doc.""" - resp1 = self._upload(client, "dup_file.txt", b"Duplicate file content.") + def test_upload_dedup_within_a_session_returns_409(self, client): + """The same bytes twice in one session returns 409 + the existing doc.""" + session_id = self._session(client, "dedup_session") + resp1 = self._upload( + client, "dup_file.txt", b"Duplicate file content.", session_id + ) assert resp1.status_code == 201 - resp2 = self._upload(client, "dup_file_renamed.txt", b"Duplicate file content.") + resp2 = self._upload( + client, "dup_file_renamed.txt", b"Duplicate file content.", session_id + ) assert resp2.status_code == 409 detail = resp2.json()["detail"] - assert detail["detail"] == "Document already exists" assert detail["existing_document"]["file_hash"] == resp1.json()["file_hash"] - def test_upload_dedup_with_force_updates(self, client): - """Re-uploading the same file with force=true overwrites the existing doc.""" - resp1 = self._upload(client, "force_file.txt", b"Force overwrite content.") - assert resp1.status_code == 201 - resp2 = client.post( - "/api/v1/document/", - files={ - "file": ( - "force_file_renamed.txt", - b"Force overwrite content.", - "text/plain", - ) - }, # noqa: E501 - data={ - "metadata": json.dumps( - {"file_name": "force_file_renamed.txt", "optional_metadata": {}} - ) - }, - params={"force": "true"}, - ) - assert resp2.status_code == 200 - assert resp2.json()["id"] == resp1.json()["id"] - assert resp2.json()["file_name"] == "force_file_renamed.txt" + def test_upload_same_bytes_in_another_session_is_allowed(self, client): + """Dedup is per session: another session gets its own document.""" + content = b"Duplicate across sessions." + first = self._upload(client, "across.txt", content) + second = self._upload(client, "across.txt", content) + assert first.status_code == 201, first.text + assert second.status_code == 201, second.text + assert first.json()["id"] != second.json()["id"] + assert first.json()["session_id"] != second.json()["session_id"] def test_upload_unsupported_type_rejected(self, client): """Uploading an unsupported extension returns 400 before extraction.""" @@ -191,22 +215,22 @@ def test_upload_without_extension_rejected(self, client): class TestUpdateExtractorEndpoint: """Tests for PUT /api/v1/document/{id}/extractor.""" - def test_update_without_pipelines(self, client): - """Update extractor for a document not linked to any pipeline.""" + def test_update_extractor_needs_no_confirmation(self, client): + """Committing an extractor choice just works -- there is no force flag. + + A document belongs to exactly one session, so changing its extractor + cannot destroy anybody else's index. + """ session_factory = client.app.container["session_factory"] with session_factory() as db: doc_id = _create_document(db, "test_update_ext.txt", "update_ext_hash_010") resp = client.put( f"/api/v1/document/{doc_id}/extractor", - json={ - "extractor": {"component": "PlainTextExtractor", "params": {}}, - "force": False, - }, + json={"extractor": {"component": "PlainTextExtractor", "params": {}}}, ) - assert resp.status_code == 200 - data = resp.json() - assert data["extractor"]["component"] == "PlainTextExtractor" + assert resp.status_code == 200, resp.text + assert resp.json()["extractor"]["component"] == "PlainTextExtractor" def test_update_invalid_component(self, client): """Using a non-existent extractor name returns 400.""" diff --git a/tests/back/RAG/test_document_extractor_invariants.py b/tests/back/RAG/test_document_extractor_invariants.py new file mode 100644 index 000000000..7285c064f --- /dev/null +++ b/tests/back/RAG/test_document_extractor_invariants.py @@ -0,0 +1,389 @@ +"""Invariants of changing a document's extractor. + +Committing an extractor choice re-extracts the text and throws away everything +fitted over the previous extraction. These tests pin down the three properties +that used to be missing: it happens unconditionally, it is atomic, and doing it +twice with the same configuration is a no-op. +""" + +import json +import os +import uuid +from typing import ClassVar, Final, List + +import pytest +from fastapi.testclient import TestClient + +from DashAI.back.core.schema_fields import BaseSchema +from DashAI.back.dependencies.database.models import ( + Chunk, + GenerativeSession, + ProcessedDocumentContent, + RAGChunkSet, + RAGChunkSetDocument, + RAGExtractor, +) +from DashAI.back.models.RAG.extractors.base_extractor import BaseExtractor + + +class ExplodingExtractorSchema(BaseSchema): + """Empty schema — the stub takes no parameters.""" + + +class ExplodingExtractor(BaseExtractor): + """An extractor that always fails, standing in for a malformed file.""" + + TYPE: Final[str] = "Extractor" + SCHEMA: ClassVar[BaseSchema] = ExplodingExtractorSchema + SUPPORTED_FILE_TYPES: List[str] = ["txt"] + + def __init__(self, **kwargs): + """Accept and ignore any parameters.""" + + def extract(self, file_path: str) -> str: + """Fail the way a broken PDF would.""" + raise OSError("cannot read this file") + + +@pytest.fixture(scope="module", autouse=True) +def register_exploding_extractor(client: TestClient) -> None: + """Make the failing extractor selectable through the API.""" + client.app.container["component_registry"].register_component(ExplodingExtractor) + + +def _upload(client: TestClient, content: bytes) -> dict: + """Create a session and upload one txt document into it.""" + tag = uuid.uuid4().hex[:8] + session_factory = client.app.container["session_factory"] + with session_factory() as db: + session = GenerativeSession( + task_name="RAGTask", + model_name="RAGPipeline", + parameters={"documents": []}, + name=f"extractor_invariants_{tag}", + ) + db.add(session) + db.commit() + session_id = session.id + + metadata = json.dumps({"file_name": f"doc_{tag}.txt", "optional_metadata": {}}) + response = client.post( + f"/api/v1/document/session/{session_id}", + files={"file": (f"doc_{tag}.txt", content, "text/plain")}, + data={"metadata": metadata}, + ) + assert response.status_code == 201, response.text + return response.json() + + +def _give_it_a_chunk_set(client: TestClient, document_id: int) -> int: + """Attach a chunk set with one chunk, standing in for a built index.""" + session_factory = client.app.container["session_factory"] + with session_factory() as db: + chunk_set = RAGChunkSet(signature=f"cs_{uuid.uuid4().hex[:12]}", parameters={}) + db.add(chunk_set) + db.flush() + db.add(RAGChunkSetDocument(chunk_set_id=chunk_set.id, document_id=document_id)) + db.add( + Chunk( + chunk_set_id=chunk_set.id, + document_id=document_id, + chunk_index=0, + text="a chunk of the old extraction", + ) + ) + db.commit() + return chunk_set.id + + +def _put_extractor(client: TestClient, document_id: int, component: str, **params): + """Commit an extractor choice for a document.""" + return client.put( + f"/api/v1/document/{document_id}/extractor", + json={"extractor": {"component": component, "params": params}}, + ) + + +def test_changing_the_extractor_invalidates_the_index(client: TestClient): + """Chunks fitted over the old extraction go, with no confirmation step. + + The old ``force`` flag guarded this, but it asked a table nothing ever + wrote to which sessions were affected, so it never triggered. + """ + document = _upload(client, b"invalidation subject") + chunk_set_id = _give_it_a_chunk_set(client, document["id"]) + + response = _put_extractor( + client, document["id"], "PlainTextExtractor", encoding="latin-1" + ) + assert response.status_code == 200, response.text + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + assert db.get(RAGChunkSet, chunk_set_id) is None + assert db.query(Chunk).filter_by(chunk_set_id=chunk_set_id).count() == 0 + + +def test_a_failing_extraction_changes_nothing(client: TestClient): + """The document keeps its extractor, its text and its index. + + The extractor id used to be committed before re-extracting, so a failure + left the document pointing at an extractor that had never produced its + text, still serving the previous extractor's chunks. + """ + document = _upload(client, b"atomicity subject") + chunk_set_id = _give_it_a_chunk_set(client, document["id"]) + before = document["extractor"] + + response = _put_extractor(client, document["id"], "ExplodingExtractor") + assert response.status_code == 422, response.text + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + cached = ( + db.query(ProcessedDocumentContent) + .filter_by(document_id=document["id"]) + .one() + ) + assert cached.content == "atomicity subject" + assert db.get(RAGChunkSet, chunk_set_id) is not None + + current = client.get(f"/api/v1/document/{document['id']}").json() + assert current["extractor"] == before + + +def test_recommitting_the_same_extractor_is_a_no_op(client: TestClient): + """Saving an unchanged choice must not throw away a good index. + + With invalidation now unconditional, pressing Save without changing + anything would otherwise re-index the whole session for nothing. + """ + document = _upload(client, b"idempotence subject") + stored = document["extractor"] + chunk_set_id = _give_it_a_chunk_set(client, document["id"]) + + response = _put_extractor( + client, document["id"], stored["component"], **stored["params"] + ) + assert response.status_code == 200, response.text + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + assert db.get(RAGChunkSet, chunk_set_id) is not None + + +def test_extractor_records_are_reused(client: TestClient): + """One configuration means one ``rag_extractor`` row, not one per save. + + Every commit used to insert a new row, and none of them could be removed + while any document still referenced one. + """ + first = _upload(client, b"dedup subject one") + second = _upload(client, b"dedup subject two") + + def rows() -> int: + session_factory = client.app.container["session_factory"] + with session_factory() as db: + return ( + db.query(RAGExtractor) + .filter_by(component_name="PlainTextExtractor") + .filter(RAGExtractor.params == {"encoding": "cp1252"}) + .count() + ) + + for document in (first, second): + assert ( + _put_extractor( + client, document["id"], "PlainTextExtractor", encoding="cp1252" + ).status_code + == 200 + ) + + assert rows() == 1 + + +def test_previewing_twice_hits_the_cache(client: TestClient): + """A second preview is cached, and does not destroy the index. + + ``extract_text`` used to build its cache signature from empty params while + instantiating the extractor with the stored ones, so the signature never + matched itself: every preview missed the cache and re-invalidated the + index. + """ + document = _upload(client, b"signature subject") + assert ( + _put_extractor( + client, document["id"], "PlainTextExtractor", encoding="latin-1" + ).status_code + == 200 + ) + chunk_set_id = _give_it_a_chunk_set(client, document["id"]) + + first = client.post(f"/api/v1/document/{document['id']}/extract", json={}) + assert first.status_code == 200, first.text + second = client.post(f"/api/v1/document/{document['id']}/extract", json={}) + assert second.status_code == 200, second.text + assert second.json()["cached"] is True, second.json() + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + assert db.get(RAGChunkSet, chunk_set_id) is not None + + +def test_deleting_a_document_takes_its_artifacts(client: TestClient): + """Deleting a document removes its chunks, its blob and its extractor row.""" + document = _upload(client, b"deletion subject") + chunk_set_id = _give_it_a_chunk_set(client, document["id"]) + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + from DashAI.back.dependencies.database.models import Document + + file_path = db.get(Document, document["id"]).file_path + assert os.path.exists(file_path) + + response = client.delete(f"/api/v1/document/{document['id']}") + assert response.status_code == 204, response.text + + with session_factory() as db: + assert db.get(RAGChunkSet, chunk_set_id) is None + assert ( + db.query(ProcessedDocumentContent) + .filter_by(document_id=document["id"]) + .count() + == 0 + ) + assert ( + db.query(RAGChunkSetDocument).filter_by(document_id=document["id"]).count() + == 0 + ) + assert not os.path.exists(file_path) + + +def test_a_shared_blob_survives_deleting_one_copy(client: TestClient): + """Two sessions holding the same file share one blob, deleted once. + + The bytes are stored content-addressed, so removing one session's copy + must not pull the file out from under the other. + """ + content = b"shared blob subject" + first = _upload(client, content) + second = _upload(client, content) + assert first["id"] != second["id"] + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + from DashAI.back.dependencies.database.models import Document + + path_one = db.get(Document, first["id"]).file_path + path_two = db.get(Document, second["id"]).file_path + assert path_one == path_two, "identical content should resolve to one blob" + + assert client.delete(f"/api/v1/document/{first['id']}").status_code == 204 + assert os.path.exists(path_two), "the surviving document lost its file" + + still_readable = client.get(f"/api/v1/document/{second['id']}/view") + assert still_readable.status_code == 200 + assert still_readable.content == content + + assert client.delete(f"/api/v1/document/{second['id']}").status_code == 204 + assert not os.path.exists(path_two) + + +def test_documents_go_when_their_session_does(client: TestClient): + """Deleting a session removes its documents and their files.""" + document = _upload(client, b"session deletion subject") + session_id = document["session_id"] + + session_factory = client.app.container["session_factory"] + with session_factory() as db: + from DashAI.back.dependencies.database.models import Document + + file_path = db.get(Document, document["id"]).file_path + + response = client.delete(f"/api/v1/generative-session/{session_id}") + assert response.status_code in (200, 204), response.text + + with session_factory() as db: + from DashAI.back.dependencies.database.models import Document + + assert db.get(Document, document["id"]) is None + assert not os.path.exists(file_path) + + +def test_chatting_without_documents_is_refused(client: TestClient): + """An empty session says so, instead of failing inside the retriever.""" + session_factory = client.app.container["session_factory"] + with session_factory() as db: + session = GenerativeSession( + task_name="RAGTask", + model_name="RAGPipeline", + parameters={"documents": []}, + name=f"empty_session_{uuid.uuid4().hex[:8]}", + ) + db.add(session) + db.commit() + session_id = session.id + + response = client.post( + "/api/v1/generative-process/", + data={"session_id": str(session_id), "input_0": "what does it say?"}, + ) + assert response.status_code == 400, response.text + assert "document" in response.json()["detail"].lower() + + +def test_index_status_reports_an_empty_session(client: TestClient): + """A session with no documents is not merely "not indexed" yet.""" + session_factory = client.app.container["session_factory"] + with session_factory() as db: + session = GenerativeSession( + task_name="RAGTask", + model_name="RAGPipeline", + parameters={"documents": []}, + name=f"status_session_{uuid.uuid4().hex[:8]}", + ) + db.add(session) + db.commit() + session_id = session.id + + response = client.get(f"/api/v1/rag/sessions/{session_id}/index-status") + assert response.status_code == 200, response.text + body = response.json() + assert body["status"] == "no_documents", body + assert body["message"] + + +def test_uploading_into_a_non_rag_session_is_refused(client: TestClient): + """Only RAG sessions hold documents.""" + session_factory = client.app.container["session_factory"] + with session_factory() as db: + session = GenerativeSession( + task_name="TextToTextGenerationTask", + model_name="SomeModel", + parameters={}, + name=f"not_rag_{uuid.uuid4().hex[:8]}", + ) + db.add(session) + db.commit() + session_id = session.id + + metadata = json.dumps({"file_name": "nope.txt", "optional_metadata": {}}) + response = client.post( + f"/api/v1/document/session/{session_id}", + files={"file": ("nope.txt", b"not for you", "text/plain")}, + data={"metadata": metadata}, + ) + assert response.status_code == 400, response.text + + +def test_upload_writes_a_content_addressed_blob(client: TestClient): + """Files are named by content hash, so equal names cannot collide.""" + document = _upload(client, b"blob naming subject") + session_factory = client.app.container["session_factory"] + with session_factory() as db: + from DashAI.back.dependencies.database.models import Document + + stored = db.get(Document, document["id"]) + assert os.path.basename(stored.file_path) == stored.file_hash + assert os.path.basename(os.path.dirname(stored.file_path)) == "blobs" diff --git a/tests/back/RAG/test_extraction_cache_flow.py b/tests/back/RAG/test_extraction_cache_flow.py index 09b1dd78b..c2980fec5 100644 --- a/tests/back/RAG/test_extraction_cache_flow.py +++ b/tests/back/RAG/test_extraction_cache_flow.py @@ -10,7 +10,11 @@ import pytest -from DashAI.back.dependencies.database.models import Document, RAGExtractor +from DashAI.back.dependencies.database.models import ( + Document, + GenerativeSession, + RAGExtractor, +) _EXTRACTOR_BY_FILE_TYPE = { "pdf": "PyMuPDFExtractor", @@ -52,12 +56,22 @@ def _create_document(client, file_type: str, content: bytes | str) -> int: f.write(content) with session_factory() as db: + # A document cannot exist without an owning session. + session = GenerativeSession( + task_name="RAGTask", + model_name="RAGPipeline", + parameters={"documents": []}, + name=f"cache_flow_{unique_hash}", + ) + db.add(session) + db.flush() extractor = RAGExtractor( component_name=_EXTRACTOR_BY_FILE_TYPE[file_type], params={} ) db.add(extractor) db.flush() doc = Document( + session_id=session.id, file_name=f"test_cache.{ext}", file_type=file_type, file_path=tmp_path, @@ -65,6 +79,8 @@ def _create_document(client, file_type: str, content: bytes | str) -> int: extractor_id=extractor.id, ) db.add(doc) + db.flush() + session.parameters = {"documents": [doc.id]} db.commit() db.refresh(doc) return doc.id @@ -337,8 +353,23 @@ def test_pdf_extractor_rejected_for_txt(self, client): assert "does not support" in resp.json()["detail"] -def _upload_document(client, file_name: str, content: bytes, force: bool = False): - """POST a document via the upload endpoint.""" +def _create_rag_session(client) -> int: + """Create an empty RAG session to upload documents into.""" + session_factory = client.app.container["session_factory"] + with session_factory() as db: + session = GenerativeSession( + task_name="RAGTask", + model_name="RAGPipeline", + parameters={"documents": []}, + name=f"Test Session {uuid.uuid4().hex[:8]}", + ) + db.add(session) + db.commit() + return session.id + + +def _upload_document(client, file_name: str, content: bytes, session_id: int): + """POST a document into a session via the upload endpoint.""" import json metadata = json.dumps( @@ -348,103 +379,89 @@ def _upload_document(client, file_name: str, content: bytes, force: bool = False } ) return client.post( - "/api/v1/document/", + f"/api/v1/document/session/{session_id}", files={"file": (file_name, content, "application/octet-stream")}, data={"metadata": metadata}, - params={"force": "true"} if force else {}, ) -def _link_document_to_session(client, doc_id: int) -> int: - """Link a document to a RAG session+pipeline and return the session id.""" - from DashAI.back.dependencies.database.models import ( - GenerativeSession, - RAGDocumentPipelineSessionLink, - RAGPipeline, - ) - - session_factory = client.app.container["session_factory"] - with session_factory() as db: - session = GenerativeSession( - task_name="RAGTask", - model_name="RAGPipeline", - parameters={"documents": [doc_id]}, - name=f"Test Session {uuid.uuid4().hex[:8]}", - ) - db.add(session) - db.flush() - pipeline = RAGPipeline(session_id=session.id, name="Test Pipeline") - db.add(pipeline) - db.flush() - db.add( - RAGDocumentPipelineSessionLink( - document_id=doc_id, - session_id=session.id, - pipeline_id=pipeline.id, - ) - ) - db.commit() - return session.id - - class TestDocumentUploadFlow: """Upload flow: duplicate detection, force overwrite, extraction errors.""" - def test_duplicate_upload_returns_409(self, client): - """Re-uploading the same file returns 409 with existing doc + sessions.""" - resp1 = _upload_document(client, "dup.txt", b"unique content abc") + def test_duplicate_upload_in_same_session_returns_409(self, client): + """Re-uploading the same bytes into one session returns 409.""" + session_id = _create_rag_session(client) + resp1 = _upload_document(client, "dup.txt", b"unique content abc", session_id) assert resp1.status_code == 201 doc_id = resp1.json()["id"] - session_id = _link_document_to_session(client, doc_id) - resp2 = _upload_document(client, "dup_renamed.txt", b"unique content abc") + resp2 = _upload_document( + client, "dup_renamed.txt", b"unique content abc", session_id + ) assert resp2.status_code == 409 - body = resp2.json() - detail = body["detail"] - assert detail["detail"] == "Document already exists" + detail = resp2.json()["detail"] assert detail["existing_document"]["id"] == doc_id - assert session_id in {s["id"] for s in detail["affected_sessions"]} - def test_duplicate_upload_without_force_does_not_modify(self, client): + def test_same_file_in_two_sessions_creates_two_documents(self, client): + """The same bytes in a different session is a separate document. + + Each copy owns its own extractor choice, so changing one must not + disturb the other. The bytes themselves are stored once, so both rows + point at the same blob. + """ + content = b"shared across sessions" + first = _create_rag_session(client) + second = _create_rag_session(client) + + resp1 = _upload_document(client, "shared.txt", content, first) + resp2 = _upload_document(client, "shared.txt", content, second) + assert resp1.status_code == 201, resp1.text + assert resp2.status_code == 201, resp2.text + + doc1, doc2 = resp1.json(), resp2.json() + assert doc1["id"] != doc2["id"] + assert doc1["session_id"] == first + assert doc2["session_id"] == second + assert doc1["file_hash"] == doc2["file_hash"] + + def test_documents_with_the_same_name_do_not_collide(self, client): + """Two different files sharing a name must both stay readable. + + Files used to be stored under their original name, so the second + upload silently overwrote the first one's bytes. + """ + first = _create_rag_session(client) + second = _create_rag_session(client) + + resp1 = _upload_document(client, "report.txt", b"first report body", first) + resp2 = _upload_document(client, "report.txt", b"second report body", second) + assert resp1.status_code == 201, resp1.text + assert resp2.status_code == 201, resp2.text + + body1 = client.get(f"/api/v1/document/{resp1.json()['id']}/view") + body2 = client.get(f"/api/v1/document/{resp2.json()['id']}/view") + assert body1.content == b"first report body" + assert body2.content == b"second report body" + + def test_duplicate_upload_does_not_modify(self, client): """A 409 response must not change the existing document metadata.""" - resp1 = _upload_document(client, "keep.txt", b"keep this content") + session_id = _create_rag_session(client) + resp1 = _upload_document(client, "keep.txt", b"keep this content", session_id) assert resp1.status_code == 201 doc_id = resp1.json()["id"] - _upload_document(client, "keep_renamed.txt", b"keep this content") + _upload_document(client, "keep_renamed.txt", b"keep this content", session_id) resp = client.get(f"/api/v1/document/{doc_id}") assert resp.status_code == 200 assert resp.json()["file_name"] == "keep.txt" - def test_duplicate_upload_force_updates_content(self, client): - """Uploading the same file with force=true overwrites and re-extracts.""" - resp1 = _upload_document(client, "force.txt", b"force overwrite me") - assert resp1.status_code == 201 - doc_id = resp1.json()["id"] - - resp2 = _upload_document( - client, "force_renamed.txt", b"force overwrite me", force=True - ) - assert resp2.status_code == 200 - data = resp2.json() - assert data["id"] == doc_id - assert data["file_name"] == "force_renamed.txt" - - session_factory = client.app.container["session_factory"] - with session_factory() as db: - from DashAI.back.dependencies.database.models import ( - ProcessedDocumentContent, - ) - - entries = ( - db.query(ProcessedDocumentContent).filter_by(document_id=doc_id).all() - ) - assert len(entries) == 1 - def test_exactly_one_row_per_document_after_many_extractions(self, client): """Multiple extractions always leave exactly one content row.""" - resp = _upload_document(client, "one_row.txt", b"row invariant content") + session_id = _create_rag_session(client) + resp = _upload_document( + client, "one_row.txt", b"row invariant content", session_id + ) assert resp.status_code == 201 doc_id = resp.json()["id"] @@ -486,10 +503,12 @@ def test_change_extractor_updates_content_and_invalidates_models(self, client): RAGChunkSetDocument, ) - resp = _upload_document(client, "ext_switch.txt", b"extractor switch text") + session_id = _create_rag_session(client) + resp = _upload_document( + client, "ext_switch.txt", b"extractor switch text", session_id + ) assert resp.status_code == 201 doc_id = resp.json()["id"] - _link_document_to_session(client, doc_id) session_factory = client.app.container["session_factory"] with session_factory() as db: @@ -502,7 +521,9 @@ def test_change_extractor_updates_content_and_invalidates_models(self, client): db.commit() chunk_set_id = chunk_set.id - # Change extractor with force → content re-extracted, chunk set wiped. + # Changing the extractor re-extracts and wipes the chunk set. There is + # no force flag: a document belongs to one session, so there is nobody + # else whose index could be destroyed. resp = client.put( f"/api/v1/document/{doc_id}/extractor", json={ @@ -510,7 +531,6 @@ def test_change_extractor_updates_content_and_invalidates_models(self, client): "component": "PlainTextExtractor", "params": {"encoding": "utf-8"}, }, - "force": True, }, ) assert resp.status_code == 200 @@ -537,6 +557,9 @@ def test_change_extractor_updates_content_and_invalidates_models(self, client): def test_extraction_failure_during_upload_returns_error(self, client): """A file that fails pre-extraction makes the upload return 500.""" # Invalid UTF-8 bytes cannot be decoded by PlainTextExtractor (utf-8). - resp = _upload_document(client, "broken.txt", b"caf\xe9 broken bytes") + session_id = _create_rag_session(client) + resp = _upload_document( + client, "broken.txt", b"caf\xe9 broken bytes", session_id + ) assert resp.status_code == 500 assert "Failed to extract text" in resp.json()["detail"] diff --git a/tests/back/api/test_session_api.py b/tests/back/api/test_session_api.py index 50d08f243..64a06718b 100644 --- a/tests/back/api/test_session_api.py +++ b/tests/back/api/test_session_api.py @@ -211,29 +211,6 @@ def test_get_all_sessions( def test_update_generative_session_params_merges_and_logs_history(client: TestClient): """Test updating RAG parameters through the dedicated endpoint.""" - from DashAI.back.dependencies.database.models import Document, RAGExtractor - - session_factory = client.app.container["session_factory"] - - # Create documents in DB - with session_factory() as db: - doc_ids = [] - for i in range(2): - extractor = RAGExtractor(component_name="PlainTextExtractor", params={}) - db.add(extractor) - db.flush() - d = Document( - file_name=f"test_doc_{i}.txt", - file_type="txt", - file_path=f"/tmp/test_doc_{i}.txt", - file_hash=f"hash_doc_{i}_update", - extractor_id=extractor.id, - ) - db.add(d) - db.commit() - db.refresh(d) - doc_ids.append(d.id) - # Create a prompt first so prompt_id resolution works. prompt_payload = { "class_name": "DefaultRAGGenerationPrompt", @@ -253,7 +230,6 @@ def test_update_generative_session_params_merges_and_logs_history(client: TestCl "task_name": "RAGTask", "name": "rag-session-update-test", "parameters": { - "documents": doc_ids, "chunking_model": { "component": "CharacterChunkModel", "params": {"chunk_size": 256, "chunk_overlap": 40}, @@ -319,7 +295,8 @@ def test_update_generative_session_params_merges_and_logs_history(client: TestCl assert response.status_code == 200, f"Failed to update: {response.text}" data = response.json() assert data["id"] == session_id - assert data["parameters"]["documents"] == [1, 2] + # A session is created empty; documents are uploaded into it afterwards. + assert data["parameters"]["documents"] == [] assert data["parameters"]["chunking_model"] == { "component": "CharacterChunkModel", "params": {"chunk_size": 256, "chunk_overlap": 40}, @@ -333,7 +310,7 @@ def test_update_generative_session_params_merges_and_logs_history(client: TestCl with session_factory() as db: updated_session = db.get(GenerativeSession, session_id) assert updated_session is not None - assert updated_session.parameters["documents"] == [1, 2] + assert updated_session.parameters["documents"] == [] assert updated_session.parameters["generation_model"] == generation_update assert "prompt_id" not in updated_session.parameters assert "prompt" in updated_session.parameters