|
| 1 | +"""scope RAG documents to a single session |
| 2 | +
|
| 3 | +Gives ``document`` a ``session_id`` foreign key, so a document belongs to |
| 4 | +exactly one RAG session instead of being a globally deduplicated library |
| 5 | +entry. ``UNIQUE(file_hash)`` becomes ``UNIQUE(session_id, file_hash)``: the |
| 6 | +same file uploaded into two sessions is now two rows, each free to pick its |
| 7 | +own extractor without disturbing the other. |
| 8 | +
|
| 9 | +Also drops ``rag_document_pipeline_session_link``, which nothing in |
| 10 | +production ever wrote to. |
| 11 | +
|
| 12 | +This migration performs **no filesystem I/O**. Existing rows keep their |
| 13 | +current ``file_path``, and cloned rows deliberately share the path of the |
| 14 | +original; only uploads made after this migration use the content-addressed |
| 15 | +``blobs/`` layout. Deletion is reference-counted by ``file_path``, so a shared |
| 16 | +path is only unlinked once the last row pointing at it is gone. |
| 17 | +
|
| 18 | +Revision ID: k4l5m6n7o8p9 |
| 19 | +Revises: b7c1d4e9f206 |
| 20 | +Create Date: 2026-09-03 |
| 21 | +""" |
| 22 | + |
| 23 | +import json |
| 24 | +import logging |
| 25 | +from typing import Sequence, Union |
| 26 | + |
| 27 | +import sqlalchemy as sa |
| 28 | +from alembic import op |
| 29 | + |
| 30 | +revision: str = "k4l5m6n7o8p9" |
| 31 | +down_revision: Union[str, None] = "b7c1d4e9f206" |
| 32 | +branch_labels: Union[str, Sequence[str], None] = None |
| 33 | +depends_on: Union[str, Sequence[str], None] = None |
| 34 | + |
| 35 | +log = logging.getLogger("alembic.runtime.migration") |
| 36 | + |
| 37 | +#: Tables holding rows that reference a document, in deletion order. |
| 38 | +_DEPENDENT_TABLES = ( |
| 39 | + "chunk", |
| 40 | + "rag_embedding_matrix", |
| 41 | + "rag_chunk_set_document", |
| 42 | + "processed_document_content", |
| 43 | + "rag_document_pipeline_session_link", |
| 44 | +) |
| 45 | + |
| 46 | + |
| 47 | +def _load_parameters(raw) -> dict: |
| 48 | + """Return a session's ``parameters`` as a dict, whatever the driver gave us.""" |
| 49 | + if isinstance(raw, dict): |
| 50 | + return raw |
| 51 | + if not raw: |
| 52 | + return {} |
| 53 | + try: |
| 54 | + parsed = json.loads(raw) |
| 55 | + except (TypeError, ValueError): |
| 56 | + return {} |
| 57 | + return parsed if isinstance(parsed, dict) else {} |
| 58 | + |
| 59 | + |
| 60 | +def _document_owners(conn) -> dict: |
| 61 | + """Map each document id to the RAG sessions claiming it, lowest id first. |
| 62 | +
|
| 63 | + The pre-migration link between a document and a session is the JSON list |
| 64 | + ``generative_session.parameters["documents"]``. |
| 65 | + """ |
| 66 | + owners: dict = {} |
| 67 | + rows = conn.execute( |
| 68 | + sa.text( |
| 69 | + "SELECT id, parameters FROM generative_session " |
| 70 | + "WHERE task_name = 'RAGTask' ORDER BY id" |
| 71 | + ) |
| 72 | + ).fetchall() |
| 73 | + for session_id, raw in rows: |
| 74 | + for doc_id in _load_parameters(raw).get("documents") or []: |
| 75 | + if isinstance(doc_id, bool) or not isinstance(doc_id, int): |
| 76 | + continue |
| 77 | + owners.setdefault(doc_id, []).append(session_id) |
| 78 | + return owners |
| 79 | + |
| 80 | + |
| 81 | +def _delete_documents(conn, doc_ids) -> None: |
| 82 | + """Delete documents and every row referencing them.""" |
| 83 | + if not doc_ids: |
| 84 | + return |
| 85 | + present = set(sa.inspect(conn).get_table_names()) |
| 86 | + for doc_id in doc_ids: |
| 87 | + for table in _DEPENDENT_TABLES: |
| 88 | + if table in present: |
| 89 | + conn.execute( |
| 90 | + sa.text("DELETE FROM " + table + " WHERE document_id = :did"), |
| 91 | + {"did": doc_id}, |
| 92 | + ) |
| 93 | + conn.execute(sa.text("DELETE FROM document WHERE id = :did"), {"did": doc_id}) |
| 94 | + |
| 95 | + |
| 96 | +def _clone_document(conn, doc_id: int, session_id: int) -> int: |
| 97 | + """Copy a document row (and its cached text) over to another session. |
| 98 | +
|
| 99 | + The clone reuses the original's ``extractor_id`` -- ``rag_extractor`` rows |
| 100 | + are immutable value objects -- and its ``file_path``, since the bytes on |
| 101 | + disk are identical. Chunks, embeddings and chunk-set membership are |
| 102 | + deliberately *not* copied: changing a session's document list changes its |
| 103 | + chunk-set signature, so the session re-indexes by itself on its next |
| 104 | + message. |
| 105 | + """ |
| 106 | + conn.execute( |
| 107 | + sa.text( |
| 108 | + "INSERT INTO document (session_id, file_name, file_type, file_path, " |
| 109 | + "file_hash, optional_metadata, extractor_id, created, last_modified) " |
| 110 | + "SELECT :sid, file_name, file_type, file_path, file_hash, " |
| 111 | + "optional_metadata, extractor_id, created, last_modified " |
| 112 | + "FROM document WHERE id = :did" |
| 113 | + ), |
| 114 | + {"sid": session_id, "did": doc_id}, |
| 115 | + ) |
| 116 | + new_id = conn.execute(sa.text("SELECT last_insert_rowid()")).scalar() |
| 117 | + conn.execute( |
| 118 | + sa.text( |
| 119 | + "INSERT INTO processed_document_content " |
| 120 | + "(document_id, content, signature, char_count) " |
| 121 | + "SELECT :new_id, content, signature, char_count " |
| 122 | + "FROM processed_document_content WHERE document_id = :did" |
| 123 | + ), |
| 124 | + {"new_id": new_id, "did": doc_id}, |
| 125 | + ) |
| 126 | + return new_id |
| 127 | + |
| 128 | + |
| 129 | +def _replace_in_session_documents( |
| 130 | + conn, session_id: int, old_id: int, new_id: int |
| 131 | +) -> None: |
| 132 | + """Point a session's ``documents`` list at its own clone.""" |
| 133 | + raw = conn.execute( |
| 134 | + sa.text("SELECT parameters FROM generative_session WHERE id = :sid"), |
| 135 | + {"sid": session_id}, |
| 136 | + ).scalar() |
| 137 | + parameters = _load_parameters(raw) |
| 138 | + parameters["documents"] = [ |
| 139 | + new_id if doc_id == old_id else doc_id |
| 140 | + for doc_id in parameters.get("documents") or [] |
| 141 | + ] |
| 142 | + conn.execute( |
| 143 | + sa.text("UPDATE generative_session SET parameters = :params WHERE id = :sid"), |
| 144 | + {"params": json.dumps(parameters), "sid": session_id}, |
| 145 | + ) |
| 146 | + |
| 147 | + |
| 148 | +def upgrade() -> None: |
| 149 | + conn = op.get_bind() |
| 150 | + |
| 151 | + # The global UNIQUE(file_hash) has to go before the backfill, not after: |
| 152 | + # splitting a shared document into one row per session inserts rows that |
| 153 | + # deliberately repeat a hash. |
| 154 | + with op.batch_alter_table("document", schema=None) as batch_op: |
| 155 | + batch_op.add_column(sa.Column("session_id", sa.Integer(), nullable=True)) |
| 156 | + batch_op.drop_constraint("uq_document_file_hash", type_="unique") |
| 157 | + |
| 158 | + owners = _document_owners(conn) |
| 159 | + all_doc_ids = { |
| 160 | + row[0] for row in conn.execute(sa.text("SELECT id FROM document")).fetchall() |
| 161 | + } |
| 162 | + |
| 163 | + # Orphans: with the global documents page gone these are unreachable |
| 164 | + # forever, and a nullable session_id would defeat the whole invariant. |
| 165 | + orphans = sorted(doc_id for doc_id in all_doc_ids if doc_id not in owners) |
| 166 | + if orphans: |
| 167 | + abandoned = conn.execute( |
| 168 | + sa.text("SELECT id, file_path FROM document WHERE session_id IS NULL") |
| 169 | + ).fetchall() |
| 170 | + log.info( |
| 171 | + "Deleting %d RAG document(s) that no session references. Their files " |
| 172 | + "are left on disk for manual cleanup: %s", |
| 173 | + len(orphans), |
| 174 | + ", ".join( |
| 175 | + "#%s %s" % (doc_id, path) |
| 176 | + for doc_id, path in abandoned |
| 177 | + if doc_id in set(orphans) |
| 178 | + ), |
| 179 | + ) |
| 180 | + _delete_documents(conn, orphans) |
| 181 | + |
| 182 | + for doc_id, session_ids in sorted(owners.items()): |
| 183 | + if doc_id not in all_doc_ids: |
| 184 | + continue # stale id left behind in a session's parameters |
| 185 | + conn.execute( |
| 186 | + sa.text("UPDATE document SET session_id = :sid WHERE id = :did"), |
| 187 | + {"sid": session_ids[0], "did": doc_id}, |
| 188 | + ) |
| 189 | + for extra_session_id in session_ids[1:]: |
| 190 | + new_id = _clone_document(conn, doc_id, extra_session_id) |
| 191 | + _replace_in_session_documents(conn, extra_session_id, doc_id, new_id) |
| 192 | + |
| 193 | + # Anything still unclaimed (e.g. a document whose session was deleted |
| 194 | + # without its parameters being cleaned up) has nothing left to belong to. |
| 195 | + _delete_documents( |
| 196 | + conn, |
| 197 | + [ |
| 198 | + row[0] |
| 199 | + for row in conn.execute( |
| 200 | + sa.text("SELECT id FROM document WHERE session_id IS NULL") |
| 201 | + ).fetchall() |
| 202 | + ], |
| 203 | + ) |
| 204 | + |
| 205 | + # upload() wrote params={} while update_extractor() wrote NULL, which is one |
| 206 | + # reason extractor rows could never be deduplicated. Settle on {}. |
| 207 | + conn.execute(sa.text("UPDATE rag_extractor SET params = '{}' WHERE params IS NULL")) |
| 208 | + |
| 209 | + with op.batch_alter_table("document", schema=None) as batch_op: |
| 210 | + batch_op.alter_column("session_id", existing_type=sa.Integer(), nullable=False) |
| 211 | + batch_op.create_foreign_key( |
| 212 | + "fk_document_session_id_generative_session", |
| 213 | + "generative_session", |
| 214 | + ["session_id"], |
| 215 | + ["id"], |
| 216 | + ondelete="CASCADE", |
| 217 | + ) |
| 218 | + batch_op.create_unique_constraint( |
| 219 | + "uq_document_session_file_hash", ["session_id", "file_hash"] |
| 220 | + ) |
| 221 | + |
| 222 | + if "rag_document_pipeline_session_link" in sa.inspect(conn).get_table_names(): |
| 223 | + op.drop_table("rag_document_pipeline_session_link") |
| 224 | + |
| 225 | + |
| 226 | +def downgrade() -> None: |
| 227 | + """Restore the global document library. |
| 228 | +
|
| 229 | + Lossy: ``UNIQUE(file_hash)`` cannot be restored while per-session copies of |
| 230 | + the same file exist, so every copy but the lowest-id one is deleted. |
| 231 | + ``rag_document_pipeline_session_link`` comes back empty, which is the only |
| 232 | + state it was ever in. |
| 233 | + """ |
| 234 | + conn = op.get_bind() |
| 235 | + |
| 236 | + duplicates = [ |
| 237 | + row[0] |
| 238 | + for row in conn.execute( |
| 239 | + sa.text( |
| 240 | + "SELECT id FROM document WHERE id NOT IN " |
| 241 | + "(SELECT MIN(id) FROM document GROUP BY file_hash)" |
| 242 | + ) |
| 243 | + ).fetchall() |
| 244 | + ] |
| 245 | + _delete_documents(conn, duplicates) |
| 246 | + |
| 247 | + with op.batch_alter_table("document", schema=None) as batch_op: |
| 248 | + batch_op.drop_constraint("uq_document_session_file_hash", type_="unique") |
| 249 | + batch_op.drop_constraint( |
| 250 | + "fk_document_session_id_generative_session", type_="foreignkey" |
| 251 | + ) |
| 252 | + batch_op.drop_column("session_id") |
| 253 | + batch_op.create_unique_constraint("uq_document_file_hash", ["file_hash"]) |
| 254 | + |
| 255 | + op.create_table( |
| 256 | + "rag_document_pipeline_session_link", |
| 257 | + sa.Column("id", sa.Integer(), nullable=False), |
| 258 | + sa.Column("document_id", sa.Integer(), nullable=False), |
| 259 | + sa.Column("session_id", sa.Integer(), nullable=False), |
| 260 | + sa.Column("pipeline_id", sa.Integer(), nullable=False), |
| 261 | + sa.ForeignKeyConstraint(["document_id"], ["document.id"], ondelete="CASCADE"), |
| 262 | + sa.ForeignKeyConstraint( |
| 263 | + ["session_id"], ["generative_session.id"], ondelete="CASCADE" |
| 264 | + ), |
| 265 | + sa.ForeignKeyConstraint( |
| 266 | + ["pipeline_id"], ["rag_pipeline.id"], ondelete="CASCADE" |
| 267 | + ), |
| 268 | + sa.PrimaryKeyConstraint("id"), |
| 269 | + sa.UniqueConstraint("document_id", "session_id", name="uix_document_session"), |
| 270 | + sa.UniqueConstraint("session_id", "pipeline_id", name="uix_session_pipeline"), |
| 271 | + sa.UniqueConstraint("document_id", "pipeline_id", name="uix_document_pipeline"), |
| 272 | + ) |
0 commit comments