[sigevents] Nightshift northstar — memory workflows, onboarding, and gap detection

src/platform/packages/shared/kbn-workflows/managed/definitions/index.ts

@@ -12,9 +12,19 @@ import {
   STREAMS_KI_ONBOARDING_WORKFLOW,
   STREAMS_KI_QUERIES_GENERATION_WORKFLOW,
 } from './streams_ki';
+import {
+  STREAMS_MEMORY_CONSOLIDATION_WORKFLOW,
+  STREAMS_MEMORY_CONVERSATION_SCRAPER_WORKFLOW,
+  STREAMS_MEMORY_SYNTHESIS_WORKFLOW,
+} from './streams_memory';
 import { EXAMPLE_MANAGED_WORKFLOW } from './workflows_extensions_example';
 
 export { EXAMPLE_MANAGED_WORKFLOW_ID } from './workflows_extensions_example';
+export {
+  STREAMS_MEMORY_SYNTHESIS_WORKFLOW_ID,
+  STREAMS_MEMORY_CONSOLIDATION_WORKFLOW_ID,
+  STREAMS_MEMORY_CONVERSATION_SCRAPER_WORKFLOW_ID,
+} from './streams_memory';
 export {
   STREAMS_KI_FEATURES_IDENTIFICATION_WORKFLOW_ID,
   STREAMS_KI_QUERIES_GENERATION_WORKFLOW_ID,
@@ -23,6 +33,9 @@ export {
 
 export const managedWorkflowDefinitions = [
   EXAMPLE_MANAGED_WORKFLOW,
+  STREAMS_MEMORY_SYNTHESIS_WORKFLOW,
+  STREAMS_MEMORY_CONSOLIDATION_WORKFLOW,
+  STREAMS_MEMORY_CONVERSATION_SCRAPER_WORKFLOW,
   STREAMS_KI_FEATURES_IDENTIFICATION_WORKFLOW,
   STREAMS_KI_QUERIES_GENERATION_WORKFLOW,
   STREAMS_KI_ONBOARDING_WORKFLOW,

src/platform/packages/shared/kbn-workflows/managed/definitions/streams_memory/conversation_scraper.ts

@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { ManagedWorkflowDefinition } from '../../types';
+
+export const STREAMS_MEMORY_CONVERSATION_SCRAPER_WORKFLOW_ID =
+  'system-streams-memory-conversation-scraper';
+
+export const STREAMS_MEMORY_CONVERSATION_SCRAPER_WORKFLOW = {
+  id: STREAMS_MEMORY_CONVERSATION_SCRAPER_WORKFLOW_ID,
+  pluginId: 'streams',
+  version: 3,
+  yaml: `version: "1"
+
+name: Conversation Scraper
+description: >
+  Extracts durable knowledge from recent AI chat conversations and persists it as
+  memory wiki pages in .significant_events-memories. Runs on a 4h schedule.
+enabled: true
+settings:
+  timeout: "30m"
+  concurrency:
+    key: "streams-sigevents-memory-scraper"
+    strategy: drop
+    max: 1
+tags:
+  - observability
+  - streams
+  - sigevents
+  - memory
+  - conversation-scraper
+
+triggers:
+  - type: scheduled
+    with:
+      every: "4h"
+
+consts:
+  CONVERSATIONS_LOOKBACK: "now-8h"
+
+steps:
+  - name: get_recent_conversations
+    type: elasticsearch.request
+    with:
+      method: POST
+      path: '/.chat-conversations/_search'
+      body:
+        size: 50
+        sort:
+          - updated_at:
+              order: desc
+        _source:
+          - id
+          - agent_id
+          - title
+          - created_at
+          - updated_at
+          - rounds
+        query:
+          bool:
+            must:
+              - range:
+                  updated_at:
+                    gte: '{{ consts.CONVERSATIONS_LOOKBACK }}'
+    on-failure:
+      continue: true
+
+  - name: check_has_conversations
+    type: data.set
+    with:
+      has_conversations: "\${{ steps.get_recent_conversations.error == null and steps.get_recent_conversations.output.hits.total.value > 0 }}"
+      conversation_count: "\${{ steps.get_recent_conversations.output.hits.total.value | default: 0 }}"
+
+  - name: scrape_conversations
+    type: if
+    condition: "\${{ steps.check_has_conversations.output.has_conversations }}"
+    steps:
+      - name: extract_knowledge
+        type: ai.agent
+        connector-id: ".anthropic-claude-4.6-sonnet-chat_completion"
+        create-conversation: true
+        with:
+          timeout: 1800s
+          message: |
+            First, call the load_skill tool with skill path skills/platform/streams/streams-conversation-scraper.
+            Wait for the skill to load before calling any other tools.
+
+            Review the following {{ steps.check_has_conversations.output.conversation_count }} recent conversations and extract any durable knowledge into wiki pages.
+            Skip conversations that look like automated memory workflow runs (for example, messages that only load streams-memory-* skills).
+
+            {{ steps.get_recent_conversations.output.hits.hits | json }}
+          schema:
+            type: object
+            properties:
+              pages_written:
+                type: array
+                description: page_name values of pages written or updated.
+                items:
+                  type: string
+              pages_deleted:
+                type: array
+                description: page_name values of pages soft-deleted.
+                items:
+                  type: string
+              conversations_processed:
+                type: integer
+                description: Number of conversations that contained extractable knowledge.
+              summary:
+                type: string
+                description: Brief summary of what knowledge was extracted.
+            required:
+              - pages_written
+              - pages_deleted
+              - conversations_processed
+              - summary
+        on-failure:
+          continue: false
+`,
+  management: {
+    lifecycle: 'static',
+    versionStrategy: 'auto',
+    enablement: 'restorable',
+  },
+} as const satisfies ManagedWorkflowDefinition;

src/platform/packages/shared/kbn-workflows/managed/definitions/streams_memory/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export {
+  STREAMS_MEMORY_SYNTHESIS_WORKFLOW,
+  STREAMS_MEMORY_SYNTHESIS_WORKFLOW_ID,
+} from './memory_synthesis';
+export {
+  STREAMS_MEMORY_CONSOLIDATION_WORKFLOW,
+  STREAMS_MEMORY_CONSOLIDATION_WORKFLOW_ID,
+} from './memory_consolidation';
+export {
+  STREAMS_MEMORY_CONVERSATION_SCRAPER_WORKFLOW,
+  STREAMS_MEMORY_CONVERSATION_SCRAPER_WORKFLOW_ID,
+} from './conversation_scraper';

src/platform/packages/shared/kbn-workflows/managed/definitions/streams_memory/memory_consolidation.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { ManagedWorkflowDefinition } from '../../types';
+
+export const STREAMS_MEMORY_CONSOLIDATION_WORKFLOW_ID = 'system-streams-memory-consolidation';
+
+export const STREAMS_MEMORY_CONSOLIDATION_WORKFLOW = {
+  id: STREAMS_MEMORY_CONSOLIDATION_WORKFLOW_ID,
+  pluginId: 'streams',
+  version: 2,
+  yaml: `version: "1"
+
+name: Memory Consolidation
+description: >
+  Curates the memory wiki by merging duplicates, removing stale entries,
+  improving categorization, and adding cross-references via the streams-memory-consolidation
+  skill. Runs on a 24h schedule.
+enabled: true
+settings:
+  timeout: "30m"
+  concurrency:
+    key: "streams-sigevents-memory-consolidation"
+    strategy: drop
+    max: 1
+tags:
+  - observability
+  - streams
+  - sigevents
+  - memory
+  - consolidation
+
+triggers:
+  - type: scheduled
+    with:
+      every: "24h"
+
+steps:
+  - name: consolidate_memory
+    type: ai.agent
+    connector-id: ".anthropic-claude-4.6-sonnet-chat_completion"
+    create-conversation: true
+    with:
+      timeout: 1800s
+      message: |
+        First, call the load_skill tool with skill path skills/platform/streams/streams-memory-consolidation.
+        Wait for the skill to load before calling any other tools.
+
+        Then review the current state of the memory wiki. Start by listing all pages, then work through them methodically. Not every page needs changes — focus on the highest-impact improvements.
+      schema:
+        type: object
+        properties:
+          pages_written:
+            type: array
+            description: page_name values of pages written or updated.
+            items:
+              type: string
+          pages_deleted:
+            type: array
+            description: page_name values of pages soft-deleted.
+            items:
+              type: string
+          summary:
+            type: string
+            description: Brief summary of what was changed and why.
+        required:
+          - pages_written
+          - pages_deleted
+          - summary
+    on-failure:
+      continue: false
+`,
+  management: {
+    lifecycle: 'static',
+    versionStrategy: 'auto',
+    enablement: 'restorable',
+  },
+} as const satisfies ManagedWorkflowDefinition;

src/platform/packages/shared/kbn-workflows/managed/definitions/streams_memory/memory_synthesis.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { ManagedWorkflowDefinition } from '../../types';
+
+export const STREAMS_MEMORY_SYNTHESIS_WORKFLOW_ID = 'system-streams-memory-synthesis';
+
+export const STREAMS_MEMORY_SYNTHESIS_WORKFLOW = {
+  id: STREAMS_MEMORY_SYNTHESIS_WORKFLOW_ID,
+  pluginId: 'streams',
+  version: 2,
+  yaml: `version: "1"
+
+name: Memory Synthesis
+description: >
+  Synthesizes significant events knowledge indicators into wiki pages using the
+  streams-memory-synthesis skill. The default Agent Builder agent loads the skill,
+  reads existing pages and insights via skill tools, and writes updates via memory_write.
+enabled: true
+settings:
+  timeout: "30m"
+  concurrency:
+    key: "streams-sigevents-memory"
+    strategy: drop
+    max: 1
+tags:
+  - observability
+  - streams
+  - sigevents
+  - memory
+
+triggers:
+  - type: manual
+
+steps:
+  - name: synthesize_memory
+    type: ai.agent
+    connector-id: ".anthropic-claude-4.6-sonnet-chat_completion"
+    create-conversation: true
+    with:
+      timeout: 1800s
+      message: |
+        First, call the load_skill tool with skill path skills/platform/streams/streams-memory-synthesis.
+        Wait for the skill to load before calling any other tools.
+
+        Then review the current state of the memory wiki and the latest significant events insights, and synthesize new or updated wiki pages.
+      schema:
+        type: object
+        properties:
+          pages_written:
+            type: array
+            description: page_name values of pages written or updated in this run.
+            items:
+              type: string
+          pages_deleted:
+            type: array
+            description: page_name values of pages soft-deleted in this run.
+            items:
+              type: string
+          summary:
+            type: string
+            description: Brief summary of what was synthesized and why.
+        required:
+          - pages_written
+          - pages_deleted
+          - summary
+    on-failure:
+      continue: false
+`,
+  management: {
+    lifecycle: 'static',
+    versionStrategy: 'auto',
+    enablement: 'restorable',
+  },
+} as const satisfies ManagedWorkflowDefinition;

src/platform/plugins/shared/workflows_extensions/server/steps/ai/ai_feature_ids.ts

@@ -18,6 +18,5 @@ export const AI_CLASSIFY_FEATURE_ID = 'ai_classify';
  */
 export const WORKFLOWS_AI_RECOMMENDED_ENDPOINTS = [
   '.anthropic-claude-4.6-sonnet-chat_completion',
-  '.anthropic-claude-4.6-opus-chat_completion',
   '.openai-gpt-5.2-chat_completion',
 ];

x-pack/platform/packages/shared/agent-builder/agent-builder-common/constants.ts

@@ -11,7 +11,6 @@ export const AGENT_BUILDER_FAST_INFERENCE_FEATURE_ID = 'agent_builder_fast';
 
 export const AGENT_BUILDER_RECOMMENDED_ENDPOINTS = [
   '.anthropic-claude-4.6-sonnet-chat_completion',
-  '.anthropic-claude-4.6-opus-chat_completion',
   '.openai-gpt-5.2-chat_completion',
 ];
 

x-pack/platform/packages/shared/agent-builder/agent-builder-server/allow_lists.ts

@@ -64,6 +64,7 @@ export const AGENT_BUILDER_BUILTIN_TOOLS = [
   `${internalNamespaces.streams}.update_stream`,
   `${internalNamespaces.streams}.create_partition`,
   `${internalNamespaces.streams}.delete_stream`,
+  `${internalNamespaces.streams}.memory.write_page`,
 
   // Workflows
   `${internalNamespaces.workflows}.validate_workflow`,
@@ -91,6 +92,8 @@ export type AgentBuilderBuiltinTool = (typeof AGENT_BUILDER_BUILTIN_TOOLS)[numbe
 export const AGENT_BUILDER_BUILTIN_AGENTS = [
   `${internalNamespaces.search}.agent`,
   `${internalNamespaces.security}.agent`,
+  'sigevents.memory.system-onboarding',
+  'sigevents.memory.gap-detector',
 ] as const;
 
 export type AgentBuilderBuiltinAgent = (typeof AGENT_BUILDER_BUILTIN_AGENTS)[number];
@@ -124,9 +127,13 @@ export const AGENT_BUILDER_BUILTIN_SKILLS = [
 
   // Platform – Streams
   'streams-management',
-  'significant-events-memory',
   'knowledge-indicators-management',
   'ki-identification-management',
+  'streams-memory-synthesis',
+  'streams-memory-consolidation',
+  'streams-conversation-scraper',
+  'significant-events-onboarding',
+  'streams-gap-detection',
 
   // Platform – Workflows
   'workflow-authoring',