kafSIEM Ontology / Unmanned Systems (Drones) pack / Critical Infrastructure (SCADA) pack

This branch completes the ops-ontology / AgentOps foundation work for kafSIEM.

It moves AgentOps off the old JSON snapshot path onto a SQLite-backed runtime and typed analyst API, adds graph and pack primitives for operations-focused ontology workflows, replaces the legacy desk with the new runtime investigation UI, and documents the new Operations / Fusion product surface with demo support and screenshots.

Author: novatechflow

.env.example

@@ -61,7 +61,7 @@ AGENTOPS_TLS_INSECURE_SKIP_VERIFY=false
 AGENTOPS_POLICY_PATH=/config/agentops_policy.yaml
 AGENTOPS_REPLAY_ENABLED=true
 AGENTOPS_REPLAY_PREFIX=kafsiem-agentops-replay
-AGENTOPS_OUTPUT_PATH=public/agentops-state.json
+AGENTOPS_OUTPUT_PATH=public/agentops.db
 
 # UI operating mode.
 UI_MODE=OSINT

.gitignore

@@ -5,6 +5,7 @@ node_modules/
 dist/
 dist-ssr/
 coverage/
+*.lcov
 .tmp/
 /kafsiem-collector
 cmd/test-*/
@@ -58,6 +59,18 @@ internal/collector/run/public/
 
 # Tool caches
 .eslintcache
+.cache/
+.vite/
+.vite-inspect/
+.turbo/
+.parcel-cache/
+.next/
+.nuxt/
+.svelte-kit/
+storybook-static/
+playwright-report/
+test-results/
+junit.xml
 .npm/
 .pnpm-store/
 .yarn/

.spectral.yaml

@@ -0,0 +1,2 @@
+extends:
+  - spectral:oas

Dockerfile.collector

@@ -9,12 +9,15 @@ ARG TARGETARCH
 
 COPY go.mod go.sum ./
 RUN go mod download
+COPY api ./api
 COPY cmd ./cmd
 COPY internal ./internal
+COPY packs ./packs
 COPY registry ./registry
 COPY public ./public
 
 RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -o /out/kafsiem-collector ./cmd/kafsiem-collector
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -o /out/kafsiem-api ./cmd/kafsiem-api
 
 FROM alpine:3.20 AS geonames
 RUN apk add --no-cache curl unzip
@@ -31,8 +34,20 @@ WORKDIR /app
 COPY --from=build /out/kafsiem-collector /usr/local/bin/kafsiem-collector
 COPY --from=geonames /tmp/cities500.txt ./registry/cities500.txt
 COPY registry ./registry
+COPY packs /packs
 COPY docker/collector-entrypoint.sh /usr/local/bin/collector-entrypoint.sh
 
 RUN chmod +x /usr/local/bin/collector-entrypoint.sh
 
 ENTRYPOINT ["/usr/local/bin/collector-entrypoint.sh"]
+
+FROM alpine:3.20 AS api-runtime
+
+RUN apk add --no-cache ca-certificates
+
+WORKDIR /app
+
+COPY --from=build /out/kafsiem-api /usr/local/bin/kafsiem-api
+COPY packs /packs
+
+ENTRYPOINT ["/usr/local/bin/kafsiem-api"]

Makefile

@@ -22,11 +22,11 @@ CODEQL_GO_OUT ?= $(CODEQL_DIR)/go.sarif
 BRANCH ?= main
 RELEASE_LEVEL ?= patch
 
-.PHONY: help check check-commit install clean lint typecheck test build ci \
+.PHONY: help check check-commit install clean lint typecheck test build ci api-lint pack-lint pack-docs \
 	npm-install-if-needed \
 	go-fmt go-fmt-check go-test go-race go-cover go-vet go-codeql commit-check \
 	docker-build docker-up docker-down docker-logs docker-shell \
-	dev-start dev-stop dev-stop-clean dev-restart dev-restart-clean dev-logs dev-ensure-env \
+	dev-start dev-start-collector dev-start-api dev-stop dev-stop-clean dev-restart dev-restart-clean dev-logs dev-ensure-env \
 	code-ql code-ql-summary \
 	release-patch release-minor release-major \
 	branch-protection
@@ -84,7 +84,18 @@ test: ## Run repository test suite
 build: ## Build the production bundle
 	npm run build
 
-ci: check lint test build ## Run the full local CI suite
+api-lint: ## Lint OpenAPI with Spectral
+	npx --yes @stoplight/spectral-cli lint --fail-severity=error api/openapi.yaml
+
+pack-lint: ## Validate bundled packs
+	@mkdir -p $(GOCACHE_DIR) $(GOMODCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOMODCACHE=$(GOMODCACHE_DIR) go run ./cmd/kafsiem-api --validate-packs --packs ./packs
+
+pack-docs: ## Generate bundled pack reference docs
+	@mkdir -p $(GOCACHE_DIR) $(GOMODCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOMODCACHE=$(GOMODCACHE_DIR) go run ./cmd/pack-docs --packs ./packs --out docs/packs
+
+ci: check lint typecheck test build go-test api-lint pack-lint ## Run the full local CI suite
 
 go-fmt: ## Auto-format Go code
 	@mkdir -p $(GOCACHE_DIR) $(GOMODCACHE_DIR)
@@ -146,10 +157,17 @@ dev-ensure-env: ## Ensure local .env contains API_BEARER_TOKEN for local API pro
 
 dev-start: registry/cities500.txt dev-collector-bin dev-ensure-env ## Start the local HTTP dev stack on localhost
 	docker build -f Dockerfile.collector.dev -t kafsiem-collector:dev .
-	$(DOCKER_COMPOSE) build kafsiem
+	$(DOCKER_COMPOSE) build kafsiem kafsiem-api
 	KAFSIEM_COLLECTOR_IMAGE=kafsiem-collector:dev $(DOCKER_COMPOSE) up -d --no-build
 	@echo "kafSIEM available at http://localhost:$${KAFSIEM_HTTP_PORT:-8080}"
-	@open "http://localhost:$${KAFSIEM_HTTP_PORT:-8080}"
+
+dev-start-collector: registry/cities500.txt dev-collector-bin dev-ensure-env ## Start only the collector-side services
+	docker build -f Dockerfile.collector.dev -t kafsiem-collector:dev .
+	KAFSIEM_COLLECTOR_IMAGE=kafsiem-collector:dev $(DOCKER_COMPOSE) up -d --no-build browser collector
+
+dev-start-api: dev-ensure-env ## Start the API and web surfaces for debugging
+	$(DOCKER_COMPOSE) build kafsiem-api kafsiem
+	$(DOCKER_COMPOSE) up -d --no-build kafsiem-api kafsiem
 
 dev-stop: ## Stop the local dev stack, remove volumes, prune dangling images
 	$(DOCKER_COMPOSE) down --remove-orphans -v
@@ -163,20 +181,18 @@ dev-stop-clean: ## Stop stack, remove feed-data volume, and aggressively prune D
 dev-restart: registry/cities500.txt dev-collector-bin dev-ensure-env ## Restart the local dev stack (removes volumes, rebuilds)
 	$(DOCKER_COMPOSE) down --remove-orphans -v
 	docker build -f Dockerfile.collector.dev -t kafsiem-collector:dev .
-	$(DOCKER_COMPOSE) build kafsiem
+	$(DOCKER_COMPOSE) build kafsiem kafsiem-api
 	KAFSIEM_COLLECTOR_IMAGE=kafsiem-collector:dev $(DOCKER_COMPOSE) up -d --no-build
 	@echo "kafSIEM available at http://localhost:$${KAFSIEM_HTTP_PORT:-8080}"
-	@open "http://localhost:$${KAFSIEM_HTTP_PORT:-8080}"
 
 dev-restart-clean: registry/cities500.txt dev-collector-bin dev-ensure-env ## Restart from scratch (removes volumes, prunes caches)
 	$(DOCKER_COMPOSE) down --remove-orphans -v
 	@docker image prune -af >/dev/null 2>&1 || true
 	@docker builder prune -af >/dev/null 2>&1 || true
 	docker build --no-cache -f Dockerfile.collector.dev -t kafsiem-collector:dev .
-	$(DOCKER_COMPOSE) build --no-cache kafsiem
+	$(DOCKER_COMPOSE) build --no-cache kafsiem kafsiem-api
 	KAFSIEM_COLLECTOR_IMAGE=kafsiem-collector:dev $(DOCKER_COMPOSE) up -d --no-build
 	@echo "kafSIEM available at http://localhost:$${KAFSIEM_HTTP_PORT:-8080}"
-	@open "http://localhost:$${KAFSIEM_HTTP_PORT:-8080}"
 
 dev-sync-registry: ## Merge source_registry.json into the running DB (adds new feeds)
 	$(DOCKER_COMPOSE) exec collector kafsiem-collector --source-db /data/sources.db --curated-seed /app/registry/source_registry.json --source-db-merge-registry

README.md

@@ -1,30 +1,74 @@
 # kafSIEM
 
-kafSIEM is the open-source edition of our OSINT pipeline, used across multiple installations and packaged for local and server deployment.
+kafSIEM is an open-source operations and fusion analysis surface. It turns
+Kafka-observed operational traffic and selected OSINT context into an
+auditable entity graph for analyst workflows.
 
-It now ships with two distinct operating surfaces:
+The product can run standalone for teams that need a local, Docker-first
+analysis stack. It can also complement existing enterprise intelligence
+platforms through typed APIs, pack-defined ontology, provenance-preserving
+records, and exportable graph context.
+
+It now ships with three operating modes:
 
 - `OSINT`: the existing globe-first external intelligence workflow
-- `AgentOps`: Kafka-backed flow tracking for KafClaw agent communication
-- `HYBRID`: AgentOps plus selective external OSINT context
+- `Operations`: Kafka-backed flow tracking and internal operational workflows
+- `Fusion`: operations plus selective external OSINT context
 
 This repository has been prepared for public use by removing non-public, internal, and protected source integrations while keeping the operational pipeline structure intact.
 
+## Screenshots
+
+<p align="center">
+  <a href="public/images/Scalytics-OSINT-Prediction.png">
+    <img src="public/images/Scalytics-OSINT-Prediction.png" alt="kafSIEM OSINT mode screenshot" width="32%" />
+  </a>
+  <a href="public/images/Scalytics-Ontology-Drones.png">
+    <img src="public/images/Scalytics-Ontology-Drones.png" alt="kafSIEM Operations ontology drones screenshot" width="32%" />
+  </a>
+  <a href="public/images/Scalytics-Ontology-SCADA.png">
+    <img src="public/images/Scalytics-Ontology-SCADA.png" alt="kafSIEM Operations ontology SCADA screenshot" width="32%" />
+  </a>
+</p>
+<p align="center">
+  <strong>OSINT</strong> | <strong>Operations / Drones</strong> | <strong>Operations / SCADA</strong>
+</p>
+
 ## Open-Source Scope
 
 - Public-ready OSINT pipeline architecture
-- AgentOps flow tracking over Kafka for KafClaw-style agent traffic
+- Operations flow tracking over Kafka for KafClaw-style agent traffic
+- Entity, edge, provenance, map, and timeline APIs backed by SQLite
+- Pack-defined ontology for unmanned systems and SCADA / critical infrastructure workflows
 - Docker-first deployment for reproducible installs
-- Web dashboard + Go collector runtime
+- Web dashboard, Go collector runtime, and standalone analyst API service
 - Configurable ingestion and refresh cadence
 
+## Target Deployments
+
+kafSIEM is designed for two initial operations profiles:
+
+- unmanned systems teams that need readiness, sortie, EW, software, and
+  signoff evidence connected across fleet activity
+- SCADA and critical infrastructure teams that need plant, device, change,
+  alarm, firmware, vulnerability, and session evidence connected across
+  operational telemetry
+
+These profiles are shipped as data packs under `packs/`. The active pack
+contract is documented in [docs/packs/drones.md](https://github.com/scalytics/kafSIEM/blob/main/docs/packs/drones.md)
+and [docs/packs/scada.md](https://github.com/scalytics/kafSIEM/blob/main/docs/packs/scada.md).
+
 ## Operating Modes
 
 The runtime mode is driven by environment and mounted policy files.
 
 - `UI_MODE=OSINT` keeps the existing OSINT product behavior.
-- `UI_MODE=AGENTOPS` switches the desktop UI to the AgentOps flow desk.
-- `UI_MODE=HYBRID` keeps AgentOps primary and adds selective external-intel context.
+- `UI_MODE=AGENTOPS` switches the desktop UI to the Operations desk.
+- `UI_MODE=HYBRID` switches the desktop UI to Fusion mode with selective external-intel context.
+
+Current runtime values remain `OSINT`, `AGENTOPS`, and `HYBRID` for
+compatibility. User-facing product naming is `OSINT`, `Operations`, and
+`Fusion`.
 
 AgentOps is a separate bounded domain in the codebase:
 
@@ -33,6 +77,8 @@ AgentOps is a separate bounded domain in the codebase:
 
 It is not implemented as a generic plugin tree.
 
+Architecture details live in [docs/architecture.md](https://github.com/scalytics/kafSIEM/blob/main/docs/architecture.md).
+
 ## Run With Docker
 
 ```bash
@@ -54,13 +100,19 @@ make dev-restart
 make dev-logs
 ```
 
-For a local AgentOps demo with mocked Kafka-derived traffic and the real dashboard:
+The old JSON-backed AgentOps demo UI has been removed. Operations and Fusion
+development uses the live runtime desk against `kafsiem-api` and the
+collector-written SQLite store. For UI-only demos without Kafka or SQLite,
+the same runtime desk can run against typed mock streams:
 
 ```bash
-npm run demo:agentops
+npm run demo:ontology
+npm run demo:fusion
 ```
 
-This opens the desktop UI directly in `AgentOps` mode via `/?demo=agentops`, serves demo state from `public/demo/*.json`, and mocks the replay endpoint locally.
+These open `/?demo=ontology` and `/?demo=fusion`, which keep OSINT untouched
+and feed the Operations/Fusion ontology dashboard through mocked typed API
+resources.
 
 ## Remote Install (wget bootstrap)
 
@@ -71,17 +123,17 @@ wget -qO- https://raw.githubusercontent.com/scalytics/kafSIEM/main/deploy/instal
 The installer will:
 - verify Docker + Compose availability
 - clone or update the repo on the host
-- ask for the operating profile (`OSINT`, `AGENTOPS`, or `HYBRID`)
+- ask for the operating profile (`OSINT`, `Operations`, or `Fusion`)
 - set GHCR runtime images (`ghcr.io/scalytics/kafsiem-web` + `ghcr.io/scalytics/kafsiem-collector`)
 - prompt for install mode (`preserve` or `fresh` volume reset)
 - prompt for the common site setting (`KAFSIEM_SITE_ADDRESS`)
 - when domain mode is enabled, optionally check `ufw`/`firewalld` and validate local 80/443 availability
 - prompt only for the profile-relevant runtime keys
 - optionally run `docker compose pull` and start with `--no-build`
 
-- The release pipeline builds two images: a web image and a Go collector image.
+- The release pipeline builds three images: a web image, a Go collector image, and a Go analyst API image.
 - The scheduled feed refresh workflow runs the Go collector.
-- The web image uses Caddy, with collector output mounted into the web container at runtime.
+- The web image uses Caddy, with collector output mounted into the web container at runtime and `/api/*` reverse-proxied to the standalone analyst API service.
 - In Docker dev mode, the collector initializes empty JSON outputs on a fresh volume and writes live output on the first successful run.
 
 ## Run Locally Without Docker
@@ -117,11 +169,11 @@ The installer is profile-driven and only asks for the settings that matter for t
   - prompts for `KAFSIEM_SITE_ADDRESS`
   - prompts for OSINT credentials and optional LLM toggles
   - writes `UI_MODE=OSINT` and `PROFILE=osint-default`
-- `AGENTOPS`
+- `Operations`
   - prompts for `KAFSIEM_SITE_ADDRESS`
   - prompts for AgentOps Kafka brokers, auth mode, group identifiers, topic mode, replay, and optional reject mirroring
   - writes `UI_MODE=AGENTOPS` and `PROFILE=agentops-default`
-- `HYBRID`
+- `Fusion`
   - prompts for both the OSINT and AgentOps settings above
   - writes `UI_MODE=HYBRID` and `PROFILE=hybrid-ops`
 
@@ -137,16 +189,18 @@ AgentOps-specific runtime knobs include:
 - `AGENTOPS_REPLAY_ENABLED`
 - `AGENTOPS_REJECT_TOPIC`
 - `AGENTOPS_OUTPUT_PATH`
+- `KAFSIEM_PACKS_DIR`
 - `UI_MODE`
 - `PROFILE`
 - `UI_POLICY_PATH`
 
-When AgentOps is enabled, the collector writes `agentops-state.json` into the runtime data volume and the web UI reads that state directly.
+When AgentOps is enabled, the collector writes `agentops.db` into the runtime data volume and the analyst API serves typed `/api/v1/...` resources from that SQLite store.
 
 Mount contract:
 
 - `/config`: policy and UI steering files
-- `/data`: generated AgentOps state and replay metadata
+- `/data`: generated AgentOps SQLite state (`agentops.db` plus WAL/SHM sidecars), alerts JSON, and replay metadata
+- `/packs`: active bundled or mounted pack directories
 
 Content behavior is explicit:
 
@@ -157,6 +211,9 @@ Content behavior is explicit:
 - replay always uses a dedicated consumer group and never mutates the live tracking group
 
 Operator reference and examples live in [docs/agentops-operator-guide.md](https://github.com/scalytics/kafSIEM/blob/main/docs/agentops-operator-guide.md).
+The analyst API contract lives in [api/openapi.yaml](https://github.com/scalytics/kafSIEM/blob/main/api/openapi.yaml);
+client guidance is in [docs/api-clients.md](https://github.com/scalytics/kafSIEM/blob/main/docs/api-clients.md)
+and problem details are registered in [docs/agentops-api-errors.md](https://github.com/scalytics/kafSIEM/blob/main/docs/agentops-api-errors.md).
 
 ## Operations
 

api/cmd/specgen/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/scalytics/kafSIEM/api/specgen"
+)
+
+func main() {
+	root, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	root = filepath.Clean(filepath.Join(root, ".."))
+	for _, output := range specgen.Outputs() {
+		target := filepath.Join(root, output.Path)
+		if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+			log.Fatal(err)
+		}
+		if err := os.WriteFile(target, []byte(output.Content), 0o644); err != nil {
+			log.Fatal(err)
+		}
+	}
+}

api/doc.go

@@ -0,0 +1,3 @@
+// Package api holds the generated OpenAPI contract for the standalone
+// kafSIEM analyst API.
+package api

api/generate.go

@@ -0,0 +1,3 @@
+package api
+
+//go:generate go run ./cmd/specgen

api/generate_test.go

@@ -0,0 +1,23 @@
+package api
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/scalytics/kafSIEM/api/specgen"
+)
+
+func TestGeneratedArtifactsAreUpToDate(t *testing.T) {
+	root := filepath.Clean(filepath.Join(".."))
+	for _, output := range specgen.Outputs() {
+		path := filepath.Join(root, output.Path)
+		body, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatalf("read %s: %v", output.Path, err)
+		}
+		if string(body) != output.Content {
+			t.Fatalf("%s is stale; run `go generate ./api/...`", output.Path)
+		}
+	}
+}