Fix #23853: AI Governance and Compliance Framework for AI Applications

bootstrap/sql/migrations/native/1.11.0/mysql/schemaChanges.sql

@@ -0,0 +1,90 @@
+-- AI Application and LLM Model entities for AI/LLM governance
+-- Version 1.11.0
+
+-- Create ai_application_entity table
+CREATE TABLE IF NOT EXISTS ai_application_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    fqnHash VARCHAR(768) NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
+    PRIMARY KEY (id),
+    UNIQUE KEY unique_name (fqnHash),
+    INDEX name_index (name),
+    INDEX deleted_index (deleted)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='AI Application entities';
+
+-- Create llm_model_entity table
+CREATE TABLE IF NOT EXISTS llm_model_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    fqnHash VARCHAR(768) NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
+    PRIMARY KEY (id),
+    UNIQUE KEY unique_name (fqnHash),
+    INDEX name_index (name),
+    INDEX deleted_index (deleted)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='LLM Model entities';
+
+-- Create prompt_template_entity table
+CREATE TABLE IF NOT EXISTS prompt_template_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    fqnHash VARCHAR(768) NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
+    PRIMARY KEY (id),
+    UNIQUE KEY unique_name (fqnHash),
+    INDEX name_index (name),
+    INDEX deleted_index (deleted)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Prompt Template entities';
+
+-- Create application_execution_entity table
+CREATE TABLE IF NOT EXISTS application_execution_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    applicationId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.applicationId') STORED NOT NULL,
+    json JSON NOT NULL,
+    timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
+    PRIMARY KEY (id),
+    INDEX application_index (applicationId),
+    INDEX timestamp_index (timestamp)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='AI Application Execution logs';
+
+-- Create ai_governance_policy_entity table
+CREATE TABLE IF NOT EXISTS ai_governance_policy_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    fqnHash VARCHAR(768) NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
+    PRIMARY KEY (id),
+    UNIQUE KEY unique_name (fqnHash),
+    INDEX name_index (name),
+    INDEX deleted_index (deleted)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='AI Governance Policy entities';
+
+-- Create llm_service_entity table
+CREATE TABLE IF NOT EXISTS llm_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.id'))) STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.name'))) VIRTUAL NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.serviceType'))) VIRTUAL NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.updatedAt'))) VIRTUAL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.updatedBy'))) VIRTUAL NOT NULL,
+    deleted TINYINT(1) GENERATED ALWAYS AS (json_extract(`json`, '$.deleted')) VIRTUAL,
+    nameHash VARCHAR(256) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
+    PRIMARY KEY (id),
+    UNIQUE KEY nameHash (nameHash),
+    INDEX name_index (name),
+    INDEX service_type_index (serviceType),
+    INDEX deleted_index (deleted)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='LLM Service entities';

bootstrap/sql/migrations/native/1.11.0/postgres/schemaChanges.sql

@@ -0,0 +1,108 @@
+-- AI Application and LLM Model entities for AI/LLM governance
+-- Version 1.11.0
+
+-- Create ai_application_entity table
+CREATE TABLE IF NOT EXISTS ai_application_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
+    fqnHash VARCHAR(768) NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fqnHash)
+);
+
+CREATE INDEX IF NOT EXISTS ai_application_name_index ON ai_application_entity(name);
+CREATE INDEX IF NOT EXISTS ai_application_deleted_index ON ai_application_entity(deleted);
+
+COMMENT ON TABLE ai_application_entity IS 'AI Application entities';
+
+-- Create llm_model_entity table
+CREATE TABLE IF NOT EXISTS llm_model_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
+    fqnHash VARCHAR(768) NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fqnHash)
+);
+
+CREATE INDEX IF NOT EXISTS llm_model_name_index ON llm_model_entity(name);
+CREATE INDEX IF NOT EXISTS llm_model_deleted_index ON llm_model_entity(deleted);
+
+COMMENT ON TABLE llm_model_entity IS 'LLM Model entities';
+
+-- Create prompt_template_entity table
+CREATE TABLE IF NOT EXISTS prompt_template_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
+    fqnHash VARCHAR(768) NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fqnHash)
+);
+
+CREATE INDEX IF NOT EXISTS prompt_template_name_index ON prompt_template_entity(name);
+CREATE INDEX IF NOT EXISTS prompt_template_deleted_index ON prompt_template_entity(deleted);
+
+COMMENT ON TABLE prompt_template_entity IS 'Prompt Template entities';
+
+-- Create application_execution_entity table
+CREATE TABLE IF NOT EXISTS application_execution_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
+    applicationId VARCHAR(36) GENERATED ALWAYS AS (json->>'applicationId') STORED NOT NULL,
+    json JSONB NOT NULL,
+    timestamp BIGINT GENERATED ALWAYS AS ((json->>'timestamp')::bigint) STORED NOT NULL,
+    PRIMARY KEY (id)
+);
+
+CREATE INDEX IF NOT EXISTS application_execution_application_index ON application_execution_entity(applicationId);
+CREATE INDEX IF NOT EXISTS application_execution_timestamp_index ON application_execution_entity(timestamp);
+
+COMMENT ON TABLE application_execution_entity IS 'AI Application Execution logs';
+
+-- Create ai_governance_policy_entity table
+CREATE TABLE IF NOT EXISTS ai_governance_policy_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
+    fqnHash VARCHAR(768) NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fqnHash)
+);
+
+CREATE INDEX IF NOT EXISTS ai_governance_policy_name_index ON ai_governance_policy_entity(name);
+CREATE INDEX IF NOT EXISTS ai_governance_policy_deleted_index ON ai_governance_policy_entity(deleted);
+
+COMMENT ON TABLE ai_governance_policy_entity IS 'AI Governance Policy entities';
+
+-- Create llm_service_entity table
+CREATE TABLE IF NOT EXISTS llm_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS ((json ->> 'id'::text)) STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'name'::text)) STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'serviceType'::text)) STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS (((json ->> 'updatedAt'::text))::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'updatedBy'::text)) STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (((json ->> 'deleted'::text))::boolean) STORED,
+    nameHash VARCHAR(256) NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (nameHash)
+);
+
+CREATE INDEX IF NOT EXISTS llm_service_name_index ON llm_service_entity(name);
+CREATE INDEX IF NOT EXISTS llm_service_type_index ON llm_service_entity(serviceType);
+CREATE INDEX IF NOT EXISTS llm_service_deleted_index ON llm_service_entity(deleted);
+
+COMMENT ON TABLE llm_service_entity IS 'LLM Service entities';

openmetadata-service/src/main/java/org/openmetadata/service/Entity.java

@@ -158,6 +158,7 @@ public final class Entity {
   public static final String SECURITY_SERVICE = "securityService";
   public static final String API_SERVICE = "apiService";
   public static final String DRIVE_SERVICE = "driveService";
+  public static final String LLM_SERVICE = "llmService";
   //
   // Data asset entities
   //
@@ -195,6 +196,15 @@ public final class Entity {
   public static final String TAG = "tag";
   public static final String CLASSIFICATION = "classification";
   public static final String TYPE = "type";
+
+  //
+  // AI entities
+  //
+  public static final String AI_APPLICATION = "aiApplication";
+  public static final String LLM_MODEL = "llmModel";
+  public static final String PROMPT_TEMPLATE = "promptTemplate";
+  public static final String AGENT_EXECUTION = "agentExecution";
+  public static final String AI_GOVERNANCE_POLICY = "aiGovernancePolicy";
   public static final String TEST_DEFINITION = "testDefinition";
   public static final String TEST_CONNECTION_DEFINITION = "testConnectionDefinition";
   public static final String TEST_SUITE = "testSuite";
@@ -287,6 +297,7 @@ public final class Entity {
     SERVICE_TYPE_ENTITY_MAP.put(ServiceType.SECURITY, SECURITY_SERVICE);
     SERVICE_TYPE_ENTITY_MAP.put(ServiceType.API, API_SERVICE);
     SERVICE_TYPE_ENTITY_MAP.put(ServiceType.DRIVE, DRIVE_SERVICE);
+    SERVICE_TYPE_ENTITY_MAP.put(ServiceType.LLM, LLM_SERVICE);
 
     ENTITY_SERVICE_TYPE_MAP.put(DATABASE, DATABASE_SERVICE);
     ENTITY_SERVICE_TYPE_MAP.put(DATABASE_SCHEMA, DATABASE_SERVICE);
@@ -308,6 +319,7 @@ public final class Entity {
     ENTITY_SERVICE_TYPE_MAP.put(FILE, DRIVE_SERVICE);
     ENTITY_SERVICE_TYPE_MAP.put(SPREADSHEET, DRIVE_SERVICE);
     ENTITY_SERVICE_TYPE_MAP.put(WORKSHEET, DRIVE_SERVICE);
+    ENTITY_SERVICE_TYPE_MAP.put(LLM_MODEL, LLM_SERVICE);
 
     PARENT_ENTITY_TYPES.addAll(
         listOf(
@@ -323,6 +335,7 @@ public final class Entity {
             SEARCH_SERVICE,
             SECURITY_SERVICE,
             DRIVE_SERVICE,
+            LLM_SERVICE,
             DATABASE,
             DATABASE_SCHEMA,
             CLASSIFICATION,
@@ -679,7 +692,9 @@ public static boolean supportsFeed(String entityType) {
             GLOSSARY,
             GLOSSARY_TERM,
             TAG,
-            CLASSIFICATION)
+            CLASSIFICATION,
+            AI_APPLICATION,
+            LLM_MODEL)
         .contains(entityType);
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/cache/CachedCollectionDAO.java

@@ -314,6 +314,36 @@ public MlModelServiceDAO mlModelServiceDAO() {
     return delegate.mlModelServiceDAO();
   }
 
+  @Override
+  public LLMServiceDAO llmServiceDAO() {
+    return delegate.llmServiceDAO();
+  }
+
+  @Override
+  public LLMModelDAO llmModelDAO() {
+    return delegate.llmModelDAO();
+  }
+
+  @Override
+  public AIApplicationDAO aiApplicationDAO() {
+    return delegate.aiApplicationDAO();
+  }
+
+  @Override
+  public PromptTemplateDAO promptTemplateDAO() {
+    return delegate.promptTemplateDAO();
+  }
+
+  @Override
+  public AIGovernancePolicyDAO aiGovernancePolicyDAO() {
+    return delegate.aiGovernancePolicyDAO();
+  }
+
+  @Override
+  public AgentExecutionDAO agentExecutionDAO() {
+    return delegate.agentExecutionDAO();
+  }
+
   @Override
   public DashboardServiceDAO dashboardServiceDAO() {
     return delegate.dashboardServiceDAO();