Knowledge graphs (#209)

* fix: show created by

* perf: knowledge graphs

* perf: knowledge graphs bo

* perf: add knowledge graph bo

* fix: is_integration_project

* perf: update knowledge graphs

* perf: knowledge graphs

* perf: fetching supported file extensions

* perf: add schema resolution for integrations

* perf: add knowledge graphs groupings

* fix: _asdict for model instances

* perf: get kg by id

* perf: rename knowledge graph to data block

* perf: update data blocks managers

* fix: data block

* perf: data blocks

* perf: enhance sanitized records getter

* perf: data blocks updates

* perf: add sql_schema to datablock

* fix: data block result retreival

* perf: always overwrite data block schema

* perf: data block attributes

* fix: auto created data block attrs

* Id and usercreated as part of the request

* Addion of sql_validator

* fix: data block schema state

* Adds sub select query validation

* Enhanced test cases and validator restructure

* Always true validation update

* perf: disallowed identifiers

* perf(ac-exec-env): data blocks run on 10 support

* more check and validation

* Default additionalConfig for LLM Response

* perf: data block updates

* perf: add limit support to validate_sql_clause

* fix: data block sql_data default

* perf: related attrs

* fix: pr comments

* Added project_id checks to the data blocks

* perf(data-block): get_all endpoint

* perf: dont convert keys in sqlalchemytodict

* perf: pr comments

* perf: save sql query to data block

* perf: pr comments

---------

Co-authored-by: andhreljaKern <[email protected]>
Co-authored-by: JWittmeyer <[email protected]>

Author: 3 people

business_objects/data_block.py

@@ -0,0 +1,145 @@
+from typing import Dict, List, Optional, Any
+
+from sqlalchemy.types import Text
+from sqlalchemy.orm.attributes import flag_modified
+
+from submodules.model.business_objects import general
+from submodules.model.session import session
+from submodules.model import DataBlock
+from submodules.model.enums import DataBlockType
+
+
+def get(org_id: str, project_id: str, id: str) -> DataBlock:
+    return (
+        session.query(DataBlock)
+        .filter(
+            DataBlock.organization_id == org_id,
+            DataBlock.project_id == project_id,
+            DataBlock.id == id,
+        )
+        .first()
+    )
+
+
+def get_by_id(data_block_id: str) -> DataBlock:
+    """Get a data block by ID without requiring org_id."""
+    return session.query(DataBlock).filter(DataBlock.id == data_block_id).first()
+
+
+def get_all_by_project_id(org_id: str, project_id: str) -> List[DataBlock]:
+    return (
+        session.query(DataBlock)
+        .filter(
+            DataBlock.organization_id == org_id,
+            DataBlock.project_id == project_id,
+        )
+        .all()
+    )
+
+
+def get_by_project_id_and_type(
+    org_id: str, project_id: str, type: DataBlockType
+) -> DataBlock:
+    return (
+        session.query(DataBlock)
+        .filter(
+            DataBlock.organization_id == org_id,
+            DataBlock.project_id == project_id,
+            DataBlock.type == type.value,
+        )
+        .first()
+    )
+
+
+def get_refinery_attribute_dependants(
+    project_id: str, refinery_attribute_name: str
+) -> List[DataBlock]:
+    return (
+        session.query(DataBlock)
+        .filter(
+            DataBlock.project_id == project_id,
+            DataBlock.type == DataBlockType.STABLE.value,
+            DataBlock.sql_config.cast(Text).like(
+                f"%data->>'{refinery_attribute_name}'%"
+            ),
+        )
+        .all()
+    )
+
+
+def create(
+    org_id: str,
+    user_id: str,
+    project_id: str,
+    name: str,
+    description: str,
+    type: DataBlockType,
+    with_commit: bool = True,
+) -> DataBlock:
+    data_block = DataBlock(
+        organization_id=org_id,
+        created_by=user_id,
+        project_id=project_id,
+        name=name,
+        description=description,
+        type=type.value,
+    )
+    general.add(data_block, with_commit)
+    return data_block
+
+
+def update(
+    org_id: str,
+    project_id: str,
+    data_block_id: str,
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    sql_config: Optional[Dict[str, Dict[str, Any]]] = None,
+    sql_data: Optional[List[Dict[str, str]]] = None,
+    overwrite_sql: bool = False,
+    with_commit: bool = True,
+) -> DataBlock:
+    data_block = get(org_id, project_id, data_block_id)
+
+    if name:
+        data_block.name = name
+    if description:
+        data_block.description = description
+
+    if overwrite_sql:
+        if sql_data is not None:
+            data_block.sql_data = sql_data
+        if sql_config is not None:
+            data_block.sql_config = sql_config
+    else:
+        if sql_data is not None:
+            if not data_block.sql_data:
+                data_block.sql_data = []
+            data_block.sql_data.extend(sql_data)
+            flag_modified(data_block, "sql_data")
+        if sql_config is not None:
+            if not data_block.sql_config:
+                data_block.sql_config = {}
+            data_block.sql_config.update(sql_config)
+            flag_modified(data_block, "sql_config")
+
+    general.add(data_block, with_commit)
+    return data_block
+
+
+def delete_many(
+    org_id: str, project_id: str, ids: List[str], with_commit: bool = False
+) -> None:
+    session.query(DataBlock).filter(
+        DataBlock.organization_id == org_id,
+        DataBlock.project_id == project_id,
+        DataBlock.id.in_(ids),
+    ).delete()
+    general.flush_or_commit(with_commit)
+
+
+def execute_query(data_block: DataBlock) -> List[Dict[str, Any]]:
+    assert (
+        "query" in data_block.sql_config
+    ), "Data block SQL config must contain a 'query' key"
+    return general.execute_all(data_block.sql_config["query"])