Skip to content

Commit dd9e54d

Browse files
bbednarski9grahamking
bbednarski9
authored and
grahamking
committed
refactor(relay): use shared configured runner
Signed-off-by: Bryan Bednarski <bbednarski@nvidia.com>
1 parent 6cf9949 commit dd9e54d

10 files changed

Lines changed: 276 additions & 3452 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,15 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
88

99
### Added
1010

11-
- **NeMo Relay native plugin** — a dynamically loaded integration that runs
12-
libsy's weighted-random, LLM-classifier, escalation, and stage-router
13-
algorithms in process while Switchyard owns provider HTTP dispatch,
14-
credentials, translation, retries, and fallback. Managed calls require NeMo
15-
Relay 0.7 or newer and do not depend on `switchyard-server`. Target bindings
16-
accept non-secret `extra_body` provider defaults, preserved requests are
17-
re-encoded after routing mutations, and synthetic Relay gateway identities do
18-
not become shared router session state.
19-
20-
- **NeMo Relay routing-model usage marks** — classifier judges, escalation
21-
judges and discarded weak candidates, and failed routing candidates now emit
22-
`switchyard.routing.llm_call` ATOF marks with normalized token usage and
23-
latency. The final serving call remains represented only by Relay's outer LLM
24-
lifecycle event to prevent double-counting.
11+
- **NeMo Relay native plugin** — a dynamically loaded integration that loads
12+
Switchyard's standard TOML deployment and executes its named routes in
13+
process. Managed calls require NeMo Relay 0.8.0 or newer; unknown models use
14+
Relay's continuation unchanged.
15+
16+
- **NeMo Relay routing marks** — routing-model usage, measured routing
17+
overhead, and selected-model decisions are emitted as ATOF marks. The final
18+
serving call remains represented only by Relay's outer LLM lifecycle event to
19+
prevent double-counting.
2520

2621
- **Advisor-gate routing** — new `advisor` route type pairing the serving
2722
executor with a stronger judge-only advisor that reviews terminal turns:

‎crates/switchyard-nemo-relay-plugin/Cargo.toml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,13 @@ publish = false
1616
crate-type = ["cdylib"]
1717

1818
[dependencies]
19-
async-channel.workspace = true
20-
async-trait.workspace = true
2119
futures-util.workspace = true
2220
http.workspace = true
2321
nemo-relay-plugin.workspace = true
2422
serde.workspace = true
2523
serde_json.workspace = true
26-
switchyard-libsy.workspace = true
2724
switchyard-llm-client.workspace = true
2825
switchyard-protocol.workspace = true
26+
switchyard-runner.workspace = true
2927
switchyard-translation.workspace = true
3028
tokio.workspace = true

‎crates/switchyard-nemo-relay-plugin/README.md‎

Lines changed: 36 additions & 362 deletions
Large diffs are not rendered by default.

‎crates/switchyard-nemo-relay-plugin/config.schema.json‎

Lines changed: 8 additions & 198 deletions
Original file line numberDiff line numberDiff line change
@@ -1,210 +1,20 @@
11
{
22
"$schema": "https://json-schema.org/draft/2020-12/schema",
33
"title": "Switchyard NeMo Relay Plugin",
4-
"description": "In-process Switchyard routing with Switchyard-owned provider HTTP dispatch.",
4+
"description": "Runs a Switchyard deployment through its shared in-process runner.",
55
"type": "object",
66
"additionalProperties": false,
7-
"required": ["version", "algorithm", "targets", "default_targets"],
7+
"required": ["deployment_path"],
88
"properties": {
9-
"version": {
10-
"const": 2,
11-
"description": "Library-only Switchyard configuration version."
12-
},
139
"priority": {
1410
"type": "integer",
15-
"default": 0
16-
},
17-
"algorithm": {
18-
"description": "In-process random, capability, escalation, or stage-router configuration.",
19-
"oneOf": [
20-
{
21-
"type": "object",
22-
"additionalProperties": false,
23-
"required": ["kind"],
24-
"properties": {
25-
"kind": { "const": "random" },
26-
"seed": { "type": ["integer", "null"], "minimum": 0 }
27-
}
28-
},
29-
{
30-
"type": "object",
31-
"additionalProperties": false,
32-
"required": [
33-
"kind",
34-
"classifier_target",
35-
"weak_target",
36-
"strong_target",
37-
"base_threshold"
38-
],
39-
"properties": {
40-
"kind": { "const": "llm_classifier" },
41-
"mode": { "const": "capability", "default": "capability" },
42-
"classifier_target": {
43-
"type": "string",
44-
"minLength": 1,
45-
"description": "Semantic target name for the judge. The target must use openai_chat or openai_responses because the judge requires a JSON-schema response format."
46-
},
47-
"weak_target": { "type": "string", "minLength": 1 },
48-
"strong_target": { "type": "string", "minLength": 1 },
49-
"base_threshold": { "type": "number", "minimum": 0, "maximum": 1 },
50-
"threshold_step": { "type": "number", "minimum": 0, "default": 0 },
51-
"recent_turn_window": {
52-
"type": ["integer", "null"],
53-
"minimum": 0
54-
},
55-
"max_output_tokens": {
56-
"type": "integer",
57-
"minimum": 1,
58-
"default": 4096
59-
},
60-
"prompt": { "type": "string" },
61-
"session_affinity": { "type": "boolean", "default": false },
62-
"message_hash_fallback": { "type": "boolean", "default": false }
63-
}
64-
},
65-
{
66-
"type": "object",
67-
"additionalProperties": false,
68-
"required": [
69-
"kind",
70-
"mode",
71-
"classifier_target",
72-
"weak_target",
73-
"strong_target",
74-
"escalation"
75-
],
76-
"properties": {
77-
"kind": { "const": "llm_classifier" },
78-
"mode": { "const": "escalation" },
79-
"classifier_target": {
80-
"type": "string",
81-
"minLength": 1,
82-
"description": "Semantic target name for the trajectory judge. The target must use openai_chat or openai_responses."
83-
},
84-
"weak_target": { "type": "string", "minLength": 1 },
85-
"strong_target": { "type": "string", "minLength": 1 },
86-
"prompt": { "type": "string" },
87-
"max_output_tokens": {
88-
"type": "integer",
89-
"minimum": 1,
90-
"default": 4096
91-
},
92-
"escalation": {
93-
"type": "object",
94-
"additionalProperties": false,
95-
"properties": {
96-
"confirmations": { "type": "integer", "minimum": 1, "default": 2 },
97-
"recent_turn_window": { "type": "integer", "minimum": 1, "default": 28 },
98-
"window_message_chars": { "type": "integer", "minimum": 50, "default": 500 }
99-
}
100-
}
101-
}
102-
},
103-
{
104-
"type": "object",
105-
"additionalProperties": false,
106-
"required": [
107-
"kind",
108-
"capable_target",
109-
"efficient_target",
110-
"picker",
111-
"confidence_threshold"
112-
],
113-
"properties": {
114-
"kind": { "const": "stage_router" },
115-
"capable_target": { "type": "string", "minLength": 1 },
116-
"efficient_target": { "type": "string", "minLength": 1 },
117-
"picker": { "enum": ["capable_first", "efficient_first"] },
118-
"confidence_threshold": { "type": "number", "minimum": 0, "maximum": 1 },
119-
"recent_turn_window": {
120-
"type": ["integer", "null"],
121-
"minimum": 0,
122-
"description": "Trailing tool results scored for the current turn. Null uses libsy's default window."
123-
},
124-
"capable_system_prompt": { "type": "string" },
125-
"efficient_system_prompt": { "type": "string" },
126-
"handoff_notes": {
127-
"type": "object",
128-
"additionalProperties": false,
129-
"required": ["escalation_note"],
130-
"properties": {
131-
"escalation_note": { "type": "string", "minLength": 1 },
132-
"deescalation_note": { "type": ["string", "null"], "minLength": 1 },
133-
"only_on_wrong_signal_escalation": { "type": "boolean", "default": true }
134-
}
135-
},
136-
"classifier": {
137-
"type": "object",
138-
"additionalProperties": false,
139-
"required": ["target", "base_threshold"],
140-
"properties": {
141-
"target": {
142-
"type": "string",
143-
"minLength": 1,
144-
"description": "Judge target used only when stage signals are ambiguous. It must use openai_chat or openai_responses."
145-
},
146-
"base_threshold": { "type": "number", "minimum": 0, "maximum": 1 },
147-
"threshold_step": { "type": "number", "minimum": 0, "default": 0 },
148-
"recent_turn_window": { "type": ["integer", "null"], "minimum": 0 },
149-
"prompt": { "type": "string" },
150-
"max_output_tokens": { "type": "integer", "minimum": 1, "default": 4096 }
151-
}
152-
}
153-
}
154-
}
155-
]
156-
},
157-
"targets": {
158-
"type": "object",
159-
"minProperties": 1,
160-
"additionalProperties": {
161-
"type": "object",
162-
"additionalProperties": false,
163-
"required": ["model", "protocol", "base_url"],
164-
"properties": {
165-
"model": { "type": "string", "minLength": 1 },
166-
"protocol": {
167-
"enum": ["openai_chat", "openai_responses", "anthropic_messages"]
168-
},
169-
"endpoint": {
170-
"type": "string",
171-
"pattern": "^$|^/",
172-
"description": "Optional provider endpoint override. The resolved URL must end in the canonical route for the selected protocol."
173-
},
174-
"base_url": {
175-
"type": "string",
176-
"pattern": "^https?://"
177-
},
178-
"weight": { "type": "number", "minimum": 0, "default": 1 },
179-
"drop_caller_extra_body": {
180-
"type": "boolean",
181-
"default": false,
182-
"description": "Drop an intercepted OpenAI SDK extra_body wrapper instead of forwarding it to targets that reject caller-specific extensions."
183-
},
184-
"extra_body": {
185-
"type": "object",
186-
"default": {},
187-
"description": "Non-secret provider request defaults, such as judge reasoning controls. Caller-provided fields take precedence.",
188-
"additionalProperties": true
189-
},
190-
"header_env": {
191-
"type": "object",
192-
"description": "Sole custom provider-header source. Maps header names to environment-variable names resolved by the plugin process so literal values are never stored in configuration.",
193-
"additionalProperties": { "type": "string", "minLength": 1 }
194-
}
195-
}
196-
}
11+
"default": 0,
12+
"description": "NeMo Relay execution-intercept priority."
19713
},
198-
"default_targets": {
199-
"type": "object",
200-
"description": "Maps each managed inbound protocol to its trusted fallback target.",
201-
"minProperties": 1,
202-
"additionalProperties": false,
203-
"properties": {
204-
"openai_chat": { "type": "string", "minLength": 1 },
205-
"openai_responses": { "type": "string", "minLength": 1 },
206-
"anthropic_messages": { "type": "string", "minLength": 1 }
207-
}
14+
"deployment_path": {
15+
"type": "string",
16+
"minLength": 1,
17+
"description": "Path to a Switchyard version-1 TOML deployment shared with switchyard-server."
20818
}
20919
}
21020
}

0 commit comments

Comments
 (0)