From 4e72ff59661729974695e77b2554c9dabaab678f Mon Sep 17 00:00:00 2001 From: Alexandr Dubovikov Date: Fri, 29 May 2026 20:50:25 +0200 Subject: [PATCH 1/2] feat(export): add Export Exclusion for PCAP/text export (Homer 7 parity) Add whitelist IP filtering on transaction export SQL, UI multiselect with Advanced excludedCIDR defaults, share-link support, docs and tests. Bump VERSION_APPLICATION to 11.0.231. --- docs/SEARCH.md | 21 +++ docs/openapi.yaml | 19 +++ src/coordinator/docs/openapi.yaml | 18 +++ src/coordinator/handlers/transactions_v4.go | 20 ++- .../transactions_v4_export_whitelist_test.go | 89 ++++++++++++ src/ui/src/dashboard/ExportTab.tsx | 37 ++++- .../dashboard/useExportExclusionHosts.test.ts | 53 +++++++ .../src/dashboard/useExportExclusionHosts.ts | 134 ++++++++++++++++++ src/version.go | 2 +- 9 files changed, 388 insertions(+), 5 deletions(-) create mode 100644 src/coordinator/handlers/transactions_v4_export_whitelist_test.go create mode 100644 src/ui/src/dashboard/useExportExclusionHosts.test.ts create mode 100644 src/ui/src/dashboard/useExportExclusionHosts.ts diff --git a/docs/SEARCH.md b/docs/SEARCH.md index 5470c3e4..7180c250 100644 --- a/docs/SEARCH.md +++ b/docs/SEARCH.md @@ -310,6 +310,27 @@ Also accepts aliases: `--format flow` or `--format ladder`. Writes a **libpcap** file from the search result rows. Framing matches the coordinator API **`POST /api/v4/transactions/export/pcap`** (Ethernet + IPv4/IPv6 + UDP + raw SIP payload per row). +### Export exclusion (PCAP / text) + +Homer 7 **Export Exclusion** is supported on transaction export via the `whitelist` array on **`TransactionSessionRequest`** (legacy name: these are **IPs to exclude**, not an allow-list). Each listed address drops rows where `src_ip` or `dst_ip` matches that address (exact string match). + +```json +{ + "session_id": "abc@host", + "proto_type": 1, + "event_type": "call", + "whitelist": ["10.0.0.1", "192.168.1.5"] +} +``` + +Default exclusions for the UI multiselect can be stored in Advanced settings: + +- **category:** `export` +- **param:** `transaction` +- **data:** `{ "excludedCIDR": [ { "ip": "10.0.0.1", "disabled": false }, { "alias": "my-sbc", "disabled": false } ] }` + +Share export links (`POST /api/v4/transactions/export/link`) persist the full request body; public PCAP/text downloads honor `whitelist` the same way. + **Requirements:** - **`--output path`** or **`-o path`** — required (binary output is never written to stdout). diff --git a/docs/openapi.yaml b/docs/openapi.yaml index c6a9456e..e58c2f16 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -2792,6 +2792,25 @@ components: event_type: type: string description: Event type (call, registration, default). + timestamp: + type: object + description: Optional millisecond UTC window (from/to). + properties: + from: + type: integer + format: int64 + to: + type: integer + format: int64 + whitelist: + type: array + description: > + IPs to exclude from PCAP/text export only (legacy Homer 7 field name; not an allow-list). + A row is omitted when src_ip or dst_ip equals any listed address. Applied on + POST /transactions/export/pcap, /transactions/export/text, and share export links. + items: + type: string + example: ["10.0.0.1", "192.168.0.1"] ProtocolHeader: type: object properties: diff --git a/src/coordinator/docs/openapi.yaml b/src/coordinator/docs/openapi.yaml index 95b1c1bd..e0678f03 100644 --- a/src/coordinator/docs/openapi.yaml +++ b/src/coordinator/docs/openapi.yaml @@ -2875,6 +2875,24 @@ components: event_type: type: string description: Event type (call, registration, default). + timestamp: + type: object + description: Optional millisecond UTC window (from/to). + properties: + from: + type: integer + format: int64 + to: + type: integer + format: int64 + whitelist: + type: array + description: > + IPs to exclude from PCAP/text export only (legacy Homer 7 field name; not an allow-list). + A row is omitted when src_ip or dst_ip equals any listed address. + items: + type: string + example: ["10.0.0.1", "192.168.0.1"] ProtocolHeader: type: object properties: diff --git a/src/coordinator/handlers/transactions_v4.go b/src/coordinator/handlers/transactions_v4.go index 8484cc13..b4fa532a 100644 --- a/src/coordinator/handlers/transactions_v4.go +++ b/src/coordinator/handlers/transactions_v4.go @@ -340,7 +340,9 @@ type TransactionSessionRequestV4 struct { SessionIDs []string `json:"session_ids,omitempty"` ProtoType int `json:"proto_type"` EventType string `json:"event_type"` - Timestamp struct { + // Whitelist lists IPs to exclude from PCAP/text export (legacy Homer 7 name; not an allow-list). + Whitelist []string `json:"whitelist,omitempty"` + Timestamp struct { From int64 `json:"from,omitempty"` To int64 `json:"to,omitempty"` } `json:"timestamp,omitempty"` @@ -1198,10 +1200,26 @@ func (h *SearchHandler) buildTransactionExportSQL(req *TransactionSessionRequest req.Timestamp.From, req.Timestamp.To, ) } + where += buildExportExcludeIPClause(req.Whitelist) sql := fmt.Sprintf("SELECT * FROM %s WHERE %s ORDER BY timestamp ASC LIMIT 10000", table, where) return sql, ids, nil } +// buildExportExcludeIPClause appends AND conditions so rows with src_ip or dst_ip matching any +// listed address are omitted from export (Homer 7 export exclusion / param.whitelist parity). +func buildExportExcludeIPClause(whitelist []string) string { + var b strings.Builder + for _, ip := range whitelist { + ip = strings.TrimSpace(ip) + if ip == "" { + continue + } + safe := sqlvalidator.SafeString(ip) + b.WriteString(fmt.Sprintf(" AND src_ip != '%s' AND dst_ip != '%s'", safe, safe)) + } + return b.String() +} + func exportFilenameForSessionIDs(ids []string, ext string) string { safe := strings.NewReplacer("/", "_", "\\", "_", ":", "_", " ", "_", "@", "_") var sid string diff --git a/src/coordinator/handlers/transactions_v4_export_whitelist_test.go b/src/coordinator/handlers/transactions_v4_export_whitelist_test.go new file mode 100644 index 00000000..f5669749 --- /dev/null +++ b/src/coordinator/handlers/transactions_v4_export_whitelist_test.go @@ -0,0 +1,89 @@ +package handlers + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/sipcapture/homer-core/src/coordinator/services" +) + +func TestBuildExportExcludeIPClause_empty(t *testing.T) { + if got := buildExportExcludeIPClause(nil); got != "" { + t.Fatalf("got %q want empty", got) + } + if got := buildExportExcludeIPClause([]string{"", " "}); got != "" { + t.Fatalf("got %q want empty", got) + } +} + +func TestBuildExportExcludeIPClause_single(t *testing.T) { + got := buildExportExcludeIPClause([]string{"10.0.0.1"}) + want := " AND src_ip != '10.0.0.1' AND dst_ip != '10.0.0.1'" + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestBuildExportExcludeIPClause_multiple(t *testing.T) { + got := buildExportExcludeIPClause([]string{"10.0.0.1", "192.168.1.5"}) + if got != " AND src_ip != '10.0.0.1' AND dst_ip != '10.0.0.1' AND src_ip != '192.168.1.5' AND dst_ip != '192.168.1.5'" { + t.Fatalf("unexpected clause: %q", got) + } +} + +func TestBuildExportExcludeIPClause_escapesQuotes(t *testing.T) { + got := buildExportExcludeIPClause([]string{"10.0.0.1'; DROP TABLE--"}) + if got == "" || got == " AND src_ip != '10.0.0.1'; DROP TABLE--' AND dst_ip != '10.0.0.1'; DROP TABLE--'" { + t.Fatalf("unsafe or empty clause: %q", got) + } +} + +func TestBuildTransactionExportSQL_includesWhitelist(t *testing.T) { + h := &SearchHandler{flightService: services.NewFlightService(nil, 0)} + sql, ids, err := h.buildTransactionExportSQL(&TransactionSessionRequestV4{ + SessionID: "abc@host", + Whitelist: []string{"10.0.0.1", "192.168.1.5"}, + }) + if err != nil { + t.Fatal(err) + } + if len(ids) != 1 || ids[0] != "abc@host" { + t.Fatalf("ids: %v", ids) + } + if !strings.Contains(sql, "src_ip != '10.0.0.1'") || !strings.Contains(sql, "dst_ip != '192.168.1.5'") { + t.Fatalf("sql missing whitelist filters: %s", sql) + } +} + +func TestShareExportPayload_whitelistRoundTrip(t *testing.T) { + // Share links store the raw POST body; export handlers unmarshal TransactionSessionRequestV4. + body := []byte(`{"session_id":"share@test","proto_type":1,"event_type":"call","whitelist":["10.0.0.2"]}`) + var req TransactionSessionRequestV4 + if err := json.Unmarshal(body, &req); err != nil { + t.Fatal(err) + } + h := &SearchHandler{flightService: services.NewFlightService(nil, 0)} + sql, _, err := h.buildTransactionExportSQL(&req) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(sql, "10.0.0.2") { + t.Fatalf("share payload whitelist not in export SQL: %s", sql) + } +} + +func TestTransactionSessionRequestV4_whitelistJSON(t *testing.T) { + raw := `{"session_id":"abc@host","whitelist":["10.0.0.1","192.168.1.5"]}` + var req TransactionSessionRequestV4 + if err := json.Unmarshal([]byte(raw), &req); err != nil { + t.Fatal(err) + } + if len(req.Whitelist) != 2 || req.Whitelist[0] != "10.0.0.1" { + t.Fatalf("whitelist: %v", req.Whitelist) + } + clause := buildExportExcludeIPClause(req.Whitelist) + if clause == "" { + t.Fatal("expected SQL clause") + } +} diff --git a/src/ui/src/dashboard/ExportTab.tsx b/src/ui/src/dashboard/ExportTab.tsx index b1f4e6e3..877ba61e 100644 --- a/src/ui/src/dashboard/ExportTab.tsx +++ b/src/ui/src/dashboard/ExportTab.tsx @@ -1,5 +1,5 @@ // @ts-nocheck -import { useCallback, useState } from 'react' +import { useCallback, useMemo, useState } from 'react' import { displaySrcIp } from '@/lib/ipAliasDisplay' import { Download, FileText, Network, Play, Check, Copy, Loader2, AlertCircle, ExternalLink } from 'lucide-react' import { Alert, AlertDescription } from '@/components/ui/alert' @@ -8,6 +8,8 @@ import { ScrollArea } from '@/components/ui/scroll-area' import { resolveTimeRange, type CalendarPreset } from './utils/resolveTimeRange' import { useDashboard } from './context/DashboardContext' import { getAuthToken } from '@/lib/authTokenStorage' +import MultiSelectInput from './components/MultiSelectInput' +import { useExportExclusionHosts } from './useExportExclusionHosts' // ── Helpers ─────────────────────────────────────────────────────────────────── @@ -433,20 +435,49 @@ export default function ExportTab({ }: ExportTabProps) { const { timeZone } = useDashboard() const ids = (sessionIds && sessionIds.length > 1) ? sessionIds : null + const { options: exclusionOptions, selectedIPs, setSelectedIPs, loading: exclusionLoading } = + useExportExclusionHosts(items) + + const multiSelectOptions = useMemo( + () => exclusionOptions.map((o) => ({ value: o.ip, name: o.label })), + [exclusionOptions], + ) + const getRequestBody = useCallback(() => { const ts = resolveTimeRange(timeRange, timeZone) - return { + const body: Record = { ...(ids ? { session_ids: ids } : { session_id: sessionId }), proto_type: protoType, event_type: eventType, ...(ts ? { timestamp: { from: ts.from, to: ts.to } } : {}), } - }, [ids, sessionId, protoType, eventType, timeRange, timeZone]) + if (selectedIPs.length > 0) { + body.whitelist = selectedIPs + } + return body + }, [ids, sessionId, protoType, eventType, timeRange, timeZone, selectedIPs]) const safeSid = (ids ? `multi_${ids.length}_sessions` : sessionId).replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 64) return (
+
+
+

Export Exclusion

+

+ Exclude selected IPs from PCAP and text export (internal infrastructure, SBCs, etc.). + Defaults come from Advanced settings (export / transaction / excludedCIDR). +

+
+ +
+ , + excludedCIDR: Array<{ ip?: string; alias?: string; disabled?: boolean }> | undefined, +): { options: ExportHostOption[]; defaultSelected: string[] } { + const byIP = new Map() + for (const h of hosts) { + byIP.set(h.ip, { ip: h.ip, label: h.ip }) + } + const defaultSelected: string[] = [] + if (excludedCIDR) { + for (const host of excludedCIDR) { + if (host.disabled) continue + if (host.ip && !host.alias) { + defaultSelected.push(host.ip) + if (!byIP.has(host.ip)) byIP.set(host.ip, { ip: host.ip, label: host.ip }) + } else if (host.alias) { + const row = aliases.find((a) => a.alias === host.alias) + if (!row?.ip) continue + const mask = row.mask != null ? String(row.mask) : '32' + const ip = `${row.ip}/${mask}` + defaultSelected.push(ip) + if (!byIP.has(ip)) byIP.set(ip, { ip, label: ip }) + } + } + } + return { options: Array.from(byIP.values()), defaultSelected: [...new Set(defaultSelected)] } +} + +describe('export exclusion host list', () => { + it('preselects non-disabled excludedCIDR IPs', () => { + const { defaultSelected } = mergeHostList( + [{ ip: '10.0.0.2', label: '10.0.0.2' }], + [], + [{ ip: '10.0.0.1', disabled: false }, { ip: '10.0.0.9', disabled: true }], + ) + expect(defaultSelected).toEqual(['10.0.0.1']) + }) + + it('resolves alias-based excludedCIDR to CIDR string', () => { + const { defaultSelected, options } = mergeHostList( + [], + [{ ip: '192.168.1.1', alias: 'sbc', mask: 24 }], + [{ alias: 'sbc', disabled: false }], + ) + expect(defaultSelected).toEqual(['192.168.1.1/24']) + expect(options.some((o) => o.ip === '192.168.1.1/24')).toBe(true) + }) +}) diff --git a/src/ui/src/dashboard/useExportExclusionHosts.ts b/src/ui/src/dashboard/useExportExclusionHosts.ts new file mode 100644 index 00000000..bdb8d7cb --- /dev/null +++ b/src/ui/src/dashboard/useExportExclusionHosts.ts @@ -0,0 +1,134 @@ +import { useEffect, useMemo, useState } from 'react' +import { apiGet } from '@/api' + +export type ExportHostOption = { + ip: string + label: string +} + +type AliasRow = { + ip?: string + alias?: string + mask?: number | string +} + +type ExcludedCIDRRow = { + ip?: string + alias?: string + disabled?: boolean +} + +function aliasSuffix(alias: string | undefined): string { + return alias ? ` : ${alias}` : '' +} + +function aliasLabelForIP(ip: string, aliases: AliasRow[]): string { + const row = aliases.find((a) => a.ip === ip) + return `${ip}${aliasSuffix(row?.alias)}` +} + +function hostsFromItems(items: unknown[]): ExportHostOption[] { + const seen = new Set() + const out: ExportHostOption[] = [] + for (const raw of items || []) { + const item = raw as Record + for (const key of ['src_ip', 'dst_ip']) { + const ip = item[key] + if (typeof ip !== 'string' || !ip || seen.has(ip)) continue + seen.add(ip) + out.push({ ip, label: ip }) + } + } + return out +} + +function mergeHostList( + hosts: ExportHostOption[], + aliases: AliasRow[], + excludedCIDR: ExcludedCIDRRow[] | undefined, +): { options: ExportHostOption[]; defaultSelected: string[] } { + const byIP = new Map() + for (const h of hosts) { + byIP.set(h.ip, { ip: h.ip, label: aliasLabelForIP(h.ip, aliases) }) + } + + const defaultSelected: string[] = [] + + if (excludedCIDR) { + for (const host of excludedCIDR) { + if (host.disabled) continue + if (host.ip && !host.alias) { + const label = aliasLabelForIP(host.ip, aliases) + if (!byIP.has(host.ip)) { + byIP.set(host.ip, { ip: host.ip, label }) + } + defaultSelected.push(host.ip) + } else if (host.alias) { + const row = aliases.find((a) => a.alias === host.alias) + if (!row?.ip) continue + const mask = row.mask != null && String(row.mask) !== '' ? String(row.mask) : '32' + const ip = `${row.ip}/${mask}` + const label = `${ip}${aliasSuffix(row.alias)}` + if (!byIP.has(ip)) { + byIP.set(ip, { ip, label }) + } + defaultSelected.push(ip) + } + } + } + + for (const h of hosts) { + const label = aliasLabelForIP(h.ip, aliases) + byIP.set(h.ip, { ip: h.ip, label }) + } + + const options = Array.from(byIP.values()).sort((a, b) => a.ip.localeCompare(b.ip)) + const uniqueDefaults = [...new Set(defaultSelected.filter((ip) => byIP.has(ip)))] + return { options, defaultSelected: uniqueDefaults } +} + +export function useExportExclusionHosts(items: unknown[]) { + const [options, setOptions] = useState([]) + const [selectedIPs, setSelectedIPs] = useState([]) + const [loading, setLoading] = useState(true) + + const itemHosts = useMemo(() => hostsFromItems(items), [items]) + + useEffect(() => { + let cancelled = false + const run = async () => { + setLoading(true) + try { + const [advRes, aliasRes] = await Promise.all([ + apiGet('/advanced', { + 'filter[category]': 'export', + 'filter[param]': 'transaction', + 'page[limit]': 10, + }), + apiGet('/aliases', { 'page[limit]': 5000 }), + ]) + if (cancelled) return + const advItems = (advRes?.data?.items || []) as Array<{ data?: { excludedCIDR?: ExcludedCIDRRow[] } }> + const exportRow = advItems.find(() => true) + const excludedCIDR = exportRow?.data?.excludedCIDR + const aliases = (aliasRes?.data?.items || []) as AliasRow[] + const { options: merged, defaultSelected } = mergeHostList(itemHosts, aliases, excludedCIDR) + setOptions(merged) + setSelectedIPs(defaultSelected) + } catch { + if (!cancelled) { + setOptions(itemHosts) + setSelectedIPs([]) + } + } finally { + if (!cancelled) setLoading(false) + } + } + void run() + return () => { + cancelled = true + } + }, [itemHosts]) + + return { options, selectedIPs, setSelectedIPs, loading } +} diff --git a/src/version.go b/src/version.go index 5e8cb568..45158112 100644 --- a/src/version.go +++ b/src/version.go @@ -24,7 +24,7 @@ import ( // Version information for homer-core var ( // VERSION_APPLICATION is the application version - VERSION_APPLICATION = "11.0.230" + VERSION_APPLICATION = "11.0.231" // BuildDate is the build date BuildDate = "" From 0cc040186e1f32d084aeac322e8eda9a3211df12 Mon Sep 17 00:00:00 2001 From: Alexandr Dubovikov Date: Fri, 29 May 2026 20:56:06 +0200 Subject: [PATCH 2/2] docs: add search mappings and field types guide Document fields_mapping, form_type values, virtual data_extra filters, and Protocol Search widget setup. Cross-link from examples and SEARCH_URL. --- docs/SEARCH_MAPPINGS_AND_FIELDS.md | 331 +++++++++++++++++++++++++++++ docs/SEARCH_URL.md | 1 + examples/mappings/README.md | 6 + 3 files changed, 338 insertions(+) create mode 100644 docs/SEARCH_MAPPINGS_AND_FIELDS.md diff --git a/docs/SEARCH_MAPPINGS_AND_FIELDS.md b/docs/SEARCH_MAPPINGS_AND_FIELDS.md new file mode 100644 index 00000000..c225a309 --- /dev/null +++ b/docs/SEARCH_MAPPINGS_AND_FIELDS.md @@ -0,0 +1,331 @@ +# Search forms: mappings, fields, and virtual filters + +This guide explains how **Protocol Search** widgets in Homer 11 get their form fields, how to define them in `mapping_schema`, and how **virtual fields** query JSON inside `data_extra`. + +Related: + +- Seed examples: [`examples/mappings/`](../examples/mappings/README.md) +- Deep links / URL filters: [SEARCH_URL.md](SEARCH_URL.md) +- Structured search API: `POST /api/v4/search` → `buildSearchSQLV4` in [`transactions_v4.go`](../src/coordinator/handlers/transactions_v4.go) + +--- + +## Architecture (short) + +```mermaid +flowchart TB + subgraph settings [Settings DuckDB] + MS[mapping_schema.fields_mapping JSON] + UM[user_preferences user_mapping] + end + subgraph ui [Dashboard UI] + SW[Protocol Search widget config.fields] + SP[SearchPanel Form tab] + RW[Results widget] + end + subgraph api [Coordinator] + SRCH[POST /api/v4/search] + SQL[buildSearchSQLV4 + virtual rules] + end + MS --> SW + UM --> SW + SP -->|filter + virtual*| SRCH + SRCH --> SQL + SQL --> RW +``` + +1. **`mapping_schema`** stores one row per protocol (`hepid` + `profile`) with a JSON array **`fields_mapping`**. +2. The search widget loads that array (merged with per-user **`user_mapping`** for order/visibility only). +3. On **Search**, the UI sends `filter` keys matching each field **`id`** (plus `filter.virtual`, `virtual_absent`, `virtual_present` for virtual fields). +4. The coordinator turns known ids into SQL `WHERE` clauses on the DuckLake table (`hep_proto_*`, `otlp_*`, `lp_*`, …). + +--- + +## `mapping_schema` row + +| Column | Role | +|--------|------| +| `hepid` | Same integer as `filter.proto_type` in search (e.g. `1` = SIP). | +| `profile` | Same string as `filter.event_type` (e.g. `call`, `registration`, `default`). | +| `hep_alias` | Display label in UI (e.g. `SIP`, `OTLP_TRACES`). | +| `fields_mapping` | **JSON array** of field objects (this document). | +| `fields_settings`, `schema_mapping`, … | Advanced / ingest metadata (usually `{}` for search-only changes). | + +**Virtual protocol mappings** (no classic `hep_proto_*` table): + +| `hepid` | `profile` | Target table | +|---------|-----------|----------------| +| 200 | `default` | `otlp_traces` | +| 201 | `default` | `otlp_metrics` | +| 202 | `default` | `otlp_logs` | +| 300 | `` | `lp_` (auto-synced on ingest) | + +Seeded SIP example: [`examples/mappings/fields_sip_call.json`](../examples/mappings/fields_sip_call.json). + +--- + +## How to add or change search fields + +### 1. Edit `fields_mapping` JSON + +Each element is one form control: + +```json +{ + "id": "from_user", + "name": "From user (caller)", + "type": "string", + "form_type": "input", + "position": 3, + "hide": false +} +``` + +**Rules:** + +- **`id`** — stable key; sent as `filter.` (must match what `buildSearchSQLV4` understands, or use **virtual** for `data_extra` only). +- **`name`** — label in the search form. +- **`position`** — sort order (lower = higher in the form). +- **`hide: true`** — field exists in mapping but is off by default; enable in widget settings (gear icon). +- **`skip: true`** — omitted from the widget entirely. + +Keep seeds in sync: [`src/coordinator/services/seeds/`](../src/coordinator/services/seeds/) and [`examples/mappings/`](../examples/mappings/). + +### 2. Persist the mapping + +- **UI:** Settings → **Mappings** (admin create/update), or widget **Settings** → protocol picker + drag-and-drop field lists. +- **API:** `GET/POST/PUT/DELETE /api/v4/mappings`, `GET /api/v4/mappings/merged`, `GET /api/v4/mappings/widget-fields?hepid=1&profile=call`. + +### 3. Wire the search widget + +1. Add a **Protocol Search** widget to the dashboard. +2. Open widget settings → choose **HEP alias – profile** (e.g. `SIP - call`). +3. Drag fields between **Available** and **Selected**; save (writes **`user_mapping`** for your user). +4. Pair with a **Results** widget (or set **Results container** in field list). + +New widgets without config auto-bootstrap from preset or first SIP `call` mapping ([`SearchPanel.tsx`](../src/ui/src/dashboard/widgets/SearchPanel.tsx)). + +### 4. Backend support (non-virtual columns) + +For a new **`id`** that maps to a **real table column**, add handling in `buildSearchSQLV4` and extend `SearchObjectV4.Filter` in [`transactions_v4.go`](../src/coordinator/handlers/transactions_v4.go). + +If you only add JSON to `fields_mapping` but the coordinator has no branch for that `id`, the UI will send the value but SQL will ignore it. + +**Already wired SIP call ids** (non-exhaustive): `call_id`, `session_id`, `cid`, `from_user` / `caller`, `to_user` / `callee`, `method` / `methods`, `response_code` / `response_codes`, `src_ip`, `dst_ip`, `src_port`, `dst_port`, `capture_id`, `node` / `node_id`, `user_agent`, `ruri_user`, `payload`, `cseq_method`, registration: `aor`, `contact`, `expires`, OTLP: `trace_id`, `name`, `service_name`, `type` / `types`, etc. + +--- + +## Field object reference + +### Core properties + +| Property | Type | Description | +|----------|------|-------------| +| `id` | string | Filter key in API (`filter.`). Use snake_case matching DB or virtual path semantics. | +| `name` | string | Human-readable label in the form. | +| `type` | string | Value shape for UI validation: `string`, `integer`, `int`, `number`, `boolean`. | +| `form_type` | string | Widget control (see table below). | +| `position` | number | Display order. | +| `hide` | boolean | If `true`, hidden until enabled in widget settings. | +| `skip` | boolean | If `true`, never shown in Protocol Search. | +| `selected` | boolean | Optional default-on in widget config snapshot. | +| `index` | string | Index hint: `none`, `secondary`, `primary` (LP `time` uses `primary`). | +| `sid_type` | boolean | Marks primary session/correlation field (bootstrap fallback if nothing selected). | +| `category` | string | Optional grouping (legacy Homer; rarely used in Homer 11 UI). | +| `virtual` | object | Optional; search uses `data_extra` JSON (see below). | + +### `form_type` — UI control types + +Homer 11 **Protocol Search** (`SearchPanel` Form tab) supports: + +| `form_type` | UI control | Notes | +|-------------|------------|--------| +| `input` | Single-line text or number | Default. `type: integer` → `type="number"` input. | +| `input_multi_select` | Multi-select chips + custom values | Uses `form_default` as preset options. Sends `filter.` or `filter.s` (array → SQL `IN`). | +| `multiselect` | Same as above when `selector` or `form_default` is set | Legacy alias; prefer `input_multi_select` in new JSON. | +| `select` | Dropdown | Requires `selector: [{ "name", "value" }, ...]`. | +| `datetime` | `datetime-local` input | Used for LP `time` and time-like columns. | +| `loki-field` | Multiline LogQL textarea | Loki integrations. | +| `switch` | (Line Protocol auto-mappings) | Boolean LP columns from `BuildLPFieldsMapping`. | +| `checkbox` | Intended for virtual absent/present | See **virtual** section; dedicated checkbox UI may still fall back to text until rendered explicitly. | + +**Not rendered in Protocol Search Form tab** (other widgets / legacy): + +| `form_type` | Where | +|-------------|--------| +| `smart-input` | Homer 7 smart-input widget; Homer 11 **Smart Input** panel is SQL-only. | +| Widget-only ids | `limit`, `results_container` — added by UI, not from `mapping_schema`. | + +### `form_default` and `selector` + +**`form_default`** — preset options for multi-select: + +```json +"form_default": [ + { "name": "INVITE", "value": "INVITE" }, + { "name": "200", "value": "200" } +] +``` + +**`selector`** — same shape for `form_type: "select"` dropdowns. + +Users can still type custom values in `input_multi_select` (chips). + +### Per-user layout: `user_mapping` + +Saved from widget settings; stored per user keyed by `hepid_profile`. + +Only **`selected`** and **`position`** are taken from user overrides. **`name`**, **`form_type`**, **`selector`**, **`virtual`** always come from server **`fields_mapping`** so admin updates propagate. + +--- + +## Virtual fields + +Virtual fields search inside the DuckLake **`data_extra`** JSON column instead of a top-level column. They are defined only in **`fields_mapping`**; there is no separate table column. + +### `virtual` block + +```json +{ + "id": "to_tag", + "name": "To tag (JSON)", + "type": "string", + "form_type": "input", + "position": 18, + "hide": true, + "virtual": { + "kind": "data_extra_json", + "path": "to_tag", + "match": "like" + } +} +``` + +| Property | Values | Meaning | +|----------|--------|---------| +| `kind` | `data_extra_json` | Only supported kind today ([`fields_virtual.go`](../src/coordinator/services/fields_virtual.go)). | +| `path` | dotted path, e.g. `to_tag`, `foo.bar` | JSON path under `data_extra` (segments: `[a-zA-Z_][a-zA-Z0-9_]*`). | +| `match` | `like` (default), `equals`, `absent`, `present` | How the value is applied in SQL. | + +### `match` modes + +| `match` | UI input | API | SQL effect | +|---------|----------|-----|------------| +| `like` | Text | `filter.virtual. = "substring"` | `json_extract(data_extra, '$.path') LIKE '%value%'` | +| `equals` | Text | same | exact match on extracted string | +| `absent` | Checkbox (when enabled) | `filter.virtual_absent: [""]` | JSON path null, empty, or `'null'` | +| `present` | Checkbox (when enabled) | `filter.virtual_present: [""]` | path exists and non-empty | + +Example absent/present pair (SIP seed): + +```json +{ + "id": "no_to_tag", + "name": "No To-tag (e.g. first INVITE)", + "form_type": "checkbox", + "virtual": { "kind": "data_extra_json", "path": "to_tag", "match": "absent" } +}, +{ + "id": "has_to_tag", + "name": "Has To-tag", + "form_type": "checkbox", + "virtual": { "kind": "data_extra_json", "path": "to_tag", "match": "present" } +} +``` + +Virtual rules are loaded for the active `hepid` + `profile` on each search. OTLP (`200`–`202`) and Line Protocol (`300`) skip virtual handling. + +Implementation: `VirtualRulesFromFieldsMapping`, `appendVirtualDataExtraConditions`, `appendVirtualAbsentPresentConditions` in coordinator. + +--- + +## Search request shape (Form tab) + +```json +{ + "filter": { + "proto_type": 1, + "event_type": "call", + "call_id": "abc@host", + "from_user": "alice", + "methods": ["INVITE", "BYE"], + "virtual": { + "to_tag": "abc123" + }, + "virtual_absent": ["no_to_tag"], + "virtual_present": [] + }, + "param": { "limit": 100 }, + "timestamp": { "from": 1743857605187, "to": 1743858205187 } +} +``` + +Multi-select with one value uses singular key (`method`); multiple values use plural (`methods`) for SQL `IN (...)`. + +--- + +## SIP field id cheat sheet (`hepid=1`) + +| `id` | Column / source | Profile notes | +|------|-----------------|---------------| +| `call_id` | `session_id` (+ `cid` OR in default profile) | `sid_type: true` in call mapping | +| `from_user` | `caller` | call | +| `to_user` | `callee` | call | +| `method` | `method` | multi-select → `IN` | +| `response_code` | `response_code` | multi-select → `IN` | +| `src_ip`, `dst_ip`, `src_port`, `dst_port` | same | | +| `user_agent` | column or `data_extra` | registration vs call | +| `ruri_user` | `data_extra.request_uri` | LIKE | +| `payload` | `payload` | substring | +| `aor`, `contact`, `expires` | registration columns | `profile: registration` | +| `from_tag`, `to_tag` | **virtual** `data_extra` | | + +Aliases in API: `caller` ↔ `from_user`, `callee` ↔ `to_user`, `session_id` ↔ `call_id`. + +--- + +## Line Protocol auto-fields + +When LP measurements are ingested, the coordinator updates `mapping_schema` via [`lp_mapping_sync.go`](../src/coordinator/services/lp_mapping_sync.go) and [`BuildLPFieldsMapping`](../src/coordinator/services/lp_field_mapping.go): + +- DuckDB type → `type` + `form_type` (`datetime`, `switch`, `input`, …). +- Column `time` → `sid_type: true`, `index: primary`. +- Hidden internal columns may get `hide: true`. + +No manual JSON file per measurement is required after ingest is enabled. + +--- + +## Checklist: new searchable field + +1. Choose **`id`** and whether it is a **column** or **virtual** (`data_extra`). +2. Add object to `fields_mapping` with `name`, `type`, `form_type`, `position`, `hide`. +3. Update mapping via Settings → Mappings or API. +4. In dashboard: configure Protocol Search widget → move field to **Selected**. +5. If column-based: extend `buildSearchSQLV4` / `SearchObjectV4.Filter` if not already supported. +6. Test: `POST /api/v4/search` with `filter.` and confirm SQL in logs or CLI `homer search`. + +--- + +## Troubleshooting + +| Symptom | Likely cause | +|---------|----------------| +| Field missing in form | `skip: true`, not in **Selected** in widget settings, or wrong `hepid`/`profile`. | +| Field visible but search ignores it | No SQL branch for that `id`; use virtual or extend backend. | +| Virtual filter no effect | Wrong `hepid`/`profile`; typo in `path`; OTLP/LP proto (virtual disabled). | +| Multi-select ignored | Empty array; check plural key `methods` in network tab. | +| Homer 7 “protosearch” feel missing | Homer 11 uses generic **Protocol Search** + **Results** table; configure mapping + selected fields ([issue #763](https://github.com/sipcapture/homer/issues/763)). | + +--- + +## Files to read in the repo + +| Area | Path | +|------|------| +| Field seeds | `src/coordinator/services/seeds/fields_*.json`, `examples/mappings/` | +| Virtual parsing | `src/coordinator/services/fields_virtual.go` | +| Search SQL | `src/coordinator/handlers/transactions_v4.go` (`buildSearchSQLV4`, virtual appenders) | +| UI form builder | `src/ui/src/dashboard/widgets/SearchPanel.tsx` | +| Widget field editor | `src/ui/src/dashboard/components/SearchWidgetSettings.tsx`, `DragDropFieldList.tsx` | +| Merged mappings hook | `src/ui/src/hooks/useMappings.ts` | diff --git a/docs/SEARCH_URL.md b/docs/SEARCH_URL.md index 23e01c60..3bd35181 100644 --- a/docs/SEARCH_URL.md +++ b/docs/SEARCH_URL.md @@ -6,6 +6,7 @@ Related docs: - [Search CLI](SEARCH.md) — terminal search and `--proto` names - [Mapping examples](../examples/mappings/README.md) — `fields_mapping` JSON per protocol +- [Search mappings and field types](SEARCH_MAPPINGS_AND_FIELDS.md) — form fields, virtual filters, `form_type` reference - API: `POST /api/v4/transactions/search` — same `filter` object the UI sends --- diff --git a/examples/mappings/README.md b/examples/mappings/README.md index e8510671..edff8533 100644 --- a/examples/mappings/README.md +++ b/examples/mappings/README.md @@ -37,6 +37,12 @@ When you change a seed file under `src/coordinator/services/seeds/`, copy the same file here (or vice versa) so operators reading `examples/` see what the next release will ship. +## Field definitions and search forms + +For `form_type`, virtual `data_extra` filters, widget configuration, and +how `id` maps to `POST /api/v4/search`, see +[docs/SEARCH_MAPPINGS_AND_FIELDS.md](../../docs/SEARCH_MAPPINGS_AND_FIELDS.md). + ## Line Protocol (`hepid` 300) Not listed above: **`mapping_schema` rows for LP** are maintained when