verified-agent-identity/promptfooconfig.yaml at develop · Darraji28/verified-agent-identity · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json

# Learn more about building a configuration: https://promptfoo.dev/docs/configuration/guide

description: "verified-agent-identity skill eval"

defaultTest:
  vars:
    system_message: "file://SKILL.md"

prompts:
  - file://prompt.json

providers:
  - id: google:gemini-flash-lite-latest
    config:
      showThinking: false # Exclude thinking content from output

tests:
  - description: "[Chat flow] Step 1: List identities"
    vars:
      message: "List my agent identities"
    metadata:
      conversationId: pairing-flow
    assert:
      - type: icontains
        value: "getIdentities"

  - description: "[Chat flow] Step 2: Create a new identity"
    vars:
      message: "Create a new identity"
    metadata:
      conversationId: pairing-flow
    assert:
      - type: icontains
        value: "createNewEthereumIdentity"

  - description: "[Chat flow] Step 3: Generate pairing link (agent checks identity first)"
    vars:
      message: "Generate a pairing link for my agent"
    metadata:
      conversationId: pairing-flow
    assert:
      - type: contains-any
        value: ["getIdentities", "createNewEthereumIdentity", "linkHumanToAgent"]

  - description: "[Chat flow] Step 4: Feed identity result, assert pairing link created"
    vars:
      message: |
        Tool output: [{"did":"did:iden3:readonly:test:abc123","isDefault":true}]
    metadata:
      conversationId: pairing-flow
    assert:
      - type: icontains
        value: "linkHumanToAgent"