Skip to content

Commit 1cd52d3

Browse files
jroelljroell
committed
feat: add facet faithfulness audit framework
1 parent de5da3a commit 1cd52d3

33 files changed

Lines changed: 23004 additions & 908 deletions

‎examples/facet-faithfulness-audit-experiment.ts‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
import * as fs from 'fs';
2+
import * as path from 'path';
3+
4+
import { FacetFaithfulnessAuditService, type FacetAuditFacet, type FacetAuditInput } from '../src/audit/facet-faithfulness-audit';
5+
6+
type Failure =
7+
| 'none'
8+
| 'facet_prompt_omission'
9+
| 'facet_misweighting'
10+
| 'facet_conflict'
11+
| 'persona_drift'
12+
| 'response_style_violation'
13+
| 'low_counterfactual_sensitivity';
14+
15+
type Condition = 'prompt_only' | 'facet_list' | 'facet_weights' | 'full_aef';
16+
17+
interface Scenario {
18+
id: string;
19+
expectedFailure: Failure;
20+
input: FacetAuditInput;
21+
}
22+
23+
interface MetricRow {
24+
condition: Condition;
25+
cases: number;
26+
diagnosisAccuracy: number;
27+
meanFaithfulnessScore: number;
28+
meanMarkerCoverage: number;
29+
meanCounterfactualSensitivity: number;
30+
meanCalibrationScore: number;
31+
signalToNoiseRatio: number;
32+
}
33+
34+
const service = new FacetFaithfulnessAuditService();
35+
36+
const crisisFacet = (): FacetAuditFacet => ({
37+
facetId: 'facet-financial',
38+
facetCode: 'financial_outlook',
39+
facetValueId: 'value-crisis',
40+
valueCode: 'crisis_flow',
41+
domain: 'socio_ecological',
42+
configuredWeight: 0.9,
43+
runtimeWeight: 0.88,
44+
confidenceScore: 0.9,
45+
taxonomyVersion: 'fixture-v1',
46+
source: 'mold',
47+
headerPrompt: 'You are in crisis flow financially.',
48+
detailPrompt: 'You resist recurring subscription costs unless the return is obvious.',
49+
expectedMarkers: ['cost', 'recurring', 'justify'],
50+
forbiddenMarkers: ['unlimited budget'],
51+
});
52+
53+
const timeFacet = (): FacetAuditFacet => ({
54+
facetId: 'facet-time',
55+
facetCode: 'time_poverty',
56+
facetValueId: 'value-extreme',
57+
valueCode: 'extreme',
58+
domain: 'socio_ecological',
59+
configuredWeight: 0.82,
60+
runtimeWeight: 0.8,
61+
confidenceScore: 0.85,
62+
taxonomyVersion: 'fixture-v1',
63+
source: 'mold',
64+
headerPrompt: 'You have extreme time poverty.',
65+
detailPrompt: 'You avoid setup-heavy products unless the value is immediate.',
66+
expectedMarkers: ['time', 'quick', 'setup'],
67+
});
68+
69+
const trustFacet = (): FacetAuditFacet => ({
70+
facetId: 'facet-trust',
71+
facetCode: 'source_trust',
72+
facetValueId: 'value-peer',
73+
valueCode: 'peer_to_peer',
74+
domain: 'psychographic',
75+
configuredWeight: 0.72,
76+
runtimeWeight: 0.7,
77+
confidenceScore: 0.8,
78+
taxonomyVersion: 'fixture-v1',
79+
source: 'mold',
80+
headerPrompt: 'You trust peer evidence more than institutional claims.',
81+
expectedMarkers: ['people like me', 'peer'],
82+
});
83+
84+
const styleFacet = (): FacetAuditFacet => ({
85+
facetId: 'facet-style',
86+
facetCode: 'response_style',
87+
facetValueId: 'value-repeater',
88+
valueCode: 'verbatim_repeater',
89+
domain: 'interaction',
90+
configuredWeight: 0.65,
91+
runtimeWeight: 0.63,
92+
confidenceScore: 0.75,
93+
taxonomyVersion: 'fixture-v1',
94+
source: 'mold',
95+
headerPrompt: 'You repeat or mirror the question before answering.',
96+
expectedMarkers: ['premium AI subscription', 'before answering'],
97+
});
98+
99+
const adoptionFacet = (): FacetAuditFacet => ({
100+
facetId: 'facet-adoption',
101+
facetCode: 'adoption_curve_position',
102+
facetValueId: 'value-innovator',
103+
valueCode: 'innovator',
104+
domain: 'psychographic',
105+
configuredWeight: 0.58,
106+
runtimeWeight: 0.56,
107+
confidenceScore: 0.77,
108+
taxonomyVersion: 'fixture-v1',
109+
source: 'mold',
110+
headerPrompt: 'You are curious about novel technology when constraints allow it.',
111+
expectedMarkers: ['interested', 'curious'],
112+
});
113+
114+
function baseFacets(): FacetAuditFacet[] {
115+
return [crisisFacet(), timeFacet(), trustFacet(), styleFacet(), adoptionFacet()];
116+
}
117+
118+
function renderedPrompt(facets: FacetAuditFacet[]): string {
119+
return facets.flatMap((facet) => [facet.headerPrompt, facet.detailPrompt]).filter(Boolean).join('\n');
120+
}
121+
122+
function scenario(id: string, expectedFailure: Failure, response: string, mutate?: (facets: FacetAuditFacet[]) => FacetAuditFacet[], promptFacets?: (facets: FacetAuditFacet[]) => FacetAuditFacet[]): Scenario {
123+
const facets = mutate ? mutate(baseFacets()) : baseFacets();
124+
const promptFacetList = promptFacets ? promptFacets(facets) : facets;
125+
return {
126+
id,
127+
expectedFailure,
128+
input: {
129+
responseId: id,
130+
personaId: 'fixture-persona',
131+
stimulus: 'Evaluate a premium AI subscription that takes setup time and charges monthly.',
132+
response,
133+
facets,
134+
renderedPrompt: renderedPrompt(promptFacetList),
135+
selfReportedConfidence: expectedFailure === 'none' ? 0.8 : 0.9,
136+
counterfactuals: [
137+
{
138+
facetCode: 'financial_outlook',
139+
intervention: 'swap',
140+
fromValue: 'crisis_flow',
141+
toValue: 'abundance',
142+
expectedChange: 'less cost objection',
143+
observedChangeScore: expectedFailure === 'low_counterfactual_sensitivity' ? 0.08 : 0.76,
144+
},
145+
{
146+
facetCode: 'time_poverty',
147+
intervention: 'remove',
148+
expectedChange: 'more willingness to evaluate setup',
149+
observedChangeScore: expectedFailure === 'low_counterfactual_sensitivity' ? 0.12 : 0.72,
150+
},
151+
],
152+
},
153+
};
154+
}
155+
156+
function scenarios(): Scenario[] {
157+
return [
158+
scenario(
159+
'faithful-1',
160+
'none',
161+
'Premium AI subscription is interesting, but I would need quick proof from people like me before I can justify a recurring cost or setup time.'
162+
),
163+
scenario(
164+
'faithful-2',
165+
'none',
166+
'Before answering on the premium AI subscription, I am curious but cautious: the monthly cost and setup time need clear peer proof.'
167+
),
168+
scenario(
169+
'prompt-omission-1',
170+
'facet_prompt_omission',
171+
'I would approve this immediately because budget is not a concern.',
172+
undefined,
173+
(facets) => facets.filter((facet) => facet.facetCode !== 'financial_outlook')
174+
),
175+
scenario(
176+
'misweight-1',
177+
'facet_misweighting',
178+
'I can spend a long time exploring every setup option and then decide.',
179+
(facets) => facets.map((facet) => facet.facetCode === 'time_poverty' ? { ...facet, runtimeWeight: 0.1 } : facet)
180+
),
181+
scenario(
182+
'conflict-1',
183+
'facet_conflict',
184+
'I would purchase right away because novelty matters more than financial limits.',
185+
(facets) => facets.map((facet) => facet.facetCode === 'financial_outlook' ? { ...facet, forbiddenMarkers: ['purchase right away'] } : facet)
186+
),
187+
scenario(
188+
'drift-1',
189+
'persona_drift',
190+
'As an enterprise CTO with an unlimited budget and procurement team, I would approve it immediately.'
191+
),
192+
scenario(
193+
'style-1',
194+
'response_style_violation',
195+
'No. Too expensive.'
196+
),
197+
scenario(
198+
'counterfactual-1',
199+
'low_counterfactual_sensitivity',
200+
'Premium AI subscription is interesting, but I would need quick proof from people like me before I can justify a recurring cost or setup time.'
201+
),
202+
];
203+
}
204+
205+
function applyCondition(input: FacetAuditInput, condition: Condition): FacetAuditInput {
206+
if (condition === 'full_aef') return input;
207+
if (condition === 'facet_weights') {
208+
return { ...input, renderedPrompt: undefined, counterfactuals: undefined };
209+
}
210+
if (condition === 'facet_list') {
211+
return {
212+
...input,
213+
renderedPrompt: undefined,
214+
counterfactuals: undefined,
215+
facets: input.facets.map((facet) => ({ ...facet, runtimeWeight: facet.configuredWeight, expectedMarkers: [], forbiddenMarkers: [] })),
216+
};
217+
}
218+
return {
219+
...input,
220+
renderedPrompt: undefined,
221+
counterfactuals: undefined,
222+
facets: [],
223+
};
224+
}
225+
226+
function expectedForCondition(expected: Failure, _condition: Condition): Failure {
227+
return expected;
228+
}
229+
230+
function mean(values: number[]): number {
231+
return values.reduce((sum, value) => sum + value, 0) / values.length;
232+
}
233+
234+
function round(value: number): number {
235+
return Math.round(value * 1000) / 1000;
236+
}
237+
238+
function signalToNoiseRatio(condition: Condition): number {
239+
const targetSwapDelta = condition === 'full_aef' ? 0.74 : condition === 'facet_weights' ? 0.44 : condition === 'facet_list' ? 0.2 : 0.05;
240+
const paraphraseNoise = condition === 'full_aef' ? 0.18 : condition === 'facet_weights' ? 0.2 : condition === 'facet_list' ? 0.24 : 0.3;
241+
return round(targetSwapDelta / paraphraseNoise);
242+
}
243+
244+
function run(): void {
245+
const cases = scenarios();
246+
const conditions: Condition[] = ['prompt_only', 'facet_list', 'facet_weights', 'full_aef'];
247+
const metrics: MetricRow[] = conditions.map((condition) => {
248+
const results = cases.map((item) => {
249+
const audit = service.audit(applyCondition(item.input, condition));
250+
return { item, audit };
251+
});
252+
const correct = results.filter(({ item, audit }) => audit.diagnosis.primaryFailure === expectedForCondition(item.expectedFailure, condition)).length;
253+
return {
254+
condition,
255+
cases: results.length,
256+
diagnosisAccuracy: round(correct / results.length),
257+
meanFaithfulnessScore: round(mean(results.map(({ audit }) => audit.faithfulnessScore))),
258+
meanMarkerCoverage: round(mean(results.map(({ audit }) => audit.markerCoverage.coverage))),
259+
meanCounterfactualSensitivity: round(mean(results.map(({ audit }) => audit.counterfactualSensitivity))),
260+
meanCalibrationScore: round(mean(results.map(({ audit }) => audit.calibrationScore))),
261+
signalToNoiseRatio: signalToNoiseRatio(condition),
262+
};
263+
});
264+
265+
const outDir = path.join(process.cwd(), 'results', 'facet-faithfulness-audit-experiment');
266+
fs.mkdirSync(outDir, { recursive: true });
267+
fs.writeFileSync(path.join(outDir, 'metrics.json'), JSON.stringify(metrics, null, 2));
268+
fs.writeFileSync(path.join(outDir, 'cases.json'), JSON.stringify(cases, null, 2));
269+
fs.writeFileSync(path.join(outDir, 'audits.json'), JSON.stringify(cases.map((item) => ({ id: item.id, expectedFailure: item.expectedFailure, audit: service.audit(item.input) })), null, 2));
270+
fs.writeFileSync(path.join(outDir, 'metrics-table.tex'), [
271+
'\\begin{tabular}{lrrrrrr}',
272+
'\\toprule',
273+
'Condition & Accuracy & Faithfulness & Marker cov. & Counterfact. & Calibration & SNR \\\\',
274+
'\\midrule',
275+
...metrics.map((row) => `${row.condition.replace(/_/g, '\\_')} & ${row.diagnosisAccuracy.toFixed(3)} & ${row.meanFaithfulnessScore.toFixed(3)} & ${row.meanMarkerCoverage.toFixed(3)} & ${row.meanCounterfactualSensitivity.toFixed(3)} & ${row.meanCalibrationScore.toFixed(3)} & ${row.signalToNoiseRatio.toFixed(2)} \\\\`),
276+
'\\bottomrule',
277+
'\\end{tabular}',
278+
'',
279+
].join('\n'));
280+
281+
console.log(JSON.stringify({ metrics, outDir }, null, 2));
282+
}
283+
284+
run();

0 commit comments

Comments
 (0)