ETL Postrelease Improvements (#370)

* fix default etl config id

* chore: update submodules

* perf(alembic): stale etl

* perf(alembic): add new monitoring attrs

* chore: update submodules

* perf(alembic): new etl attrs

* chore: update submodules

* fix: default etl config id

* chore: update submodules

* perf(alembic): set default on etl deletion

* chore: update submodules

* perf: add file_reference_id to markdown_file.meta_data

* chore: update submodules

---------

Co-authored-by: andhreljaKern <andrea.hrelja@kern.ai>

Author: LennartSchmidtKern

alembic/versions/15f133dd208b_adds_etl_new_attributes.py

@@ -0,0 +1,59 @@
+"""adds etl new attributes
+
+Revision ID: 15f133dd208b
+Revises: 04cd434ed6eb
+Create Date: 2025-12-11 22:34:07.966633
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "15f133dd208b"
+down_revision = "04cd434ed6eb"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "etl_task",
+        sa.Column("full_config_hash", sa.String(), nullable=True),
+        schema="global",
+    )
+    op.add_column(
+        "etl_task", sa.Column("is_stale", sa.Boolean(), nullable=True), schema="global"
+    )
+    op.add_column(
+        "etl_task", sa.Column("llm_ops", sa.JSON(), nullable=True), schema="global"
+    )
+    op.add_column(
+        "etl_task",
+        sa.Column("updated_at", sa.DateTime(), nullable=True),
+        schema="global",
+    )
+    op.create_index(
+        op.f("ix_global_etl_task_full_config_hash"),
+        "etl_task",
+        ["full_config_hash"],
+        unique=False,
+        schema="global",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_global_etl_task_full_config_hash"),
+        table_name="etl_task",
+        schema="global",
+    )
+    op.drop_column("etl_task", "updated_at", schema="global")
+    op.drop_column("etl_task", "llm_ops", schema="global")
+    op.drop_column("etl_task", "is_stale", schema="global")
+    op.drop_column("etl_task", "full_config_hash", schema="global")
+    # ### end Alembic commands ###

alembic/versions/5186966ea4db_sets_md_file_etl_task_id_null_on_.py

@@ -0,0 +1,54 @@
+"""sets md file etl task id null on deletion
+
+Revision ID: 5186966ea4db
+Revises: 15f133dd208b
+Create Date: 2025-12-15 07:59:30.702200
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "5186966ea4db"
+down_revision = "15f133dd208b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "markdown_file_etl_task_id_fkey",
+        "markdown_file",
+        schema="cognition",
+        type_="foreignkey",
+    )
+    op.create_foreign_key(
+        None,
+        "markdown_file",
+        "etl_task",
+        ["etl_task_id"],
+        ["id"],
+        source_schema="cognition",
+        referent_schema="global",
+        ondelete="SET NULL",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, "markdown_file", schema="cognition", type_="foreignkey")
+    op.create_foreign_key(
+        "markdown_file_etl_task_id_fkey",
+        "markdown_file",
+        "etl_task",
+        ["etl_task_id"],
+        ["id"],
+        source_schema="cognition",
+        referent_schema="global",
+        ondelete="CASCADE",
+    )
+    # ### end Alembic commands ###

controller/transfer/cognition/minio_upload.py

@@ -1,5 +1,7 @@
 from typing import List
 
+import datetime
+
 from controller.task_master import manager as task_master_manager
 from submodules.model import enums, etl_utils
 from submodules.model.business_objects import general
@@ -120,6 +122,9 @@ def handle_cognition_file_upload(path_parts: List[str]):
             org_id=org_id,
             markdown_file_id=markdown_file.id,
             etl_task_id=etl_task.id,
+            meta_data={"file_reference_id": str(file_reference.id)},
+            started_at=datetime.datetime.now(datetime.UTC),
+            overwrite_meta_data=False,
         )
 
         task_master_manager.queue_task(