Skip to content

Commit e520a67

Browse files
Let harness see every agent event and stop retrying what fails the same way
The agent wrapper only forwarded the session event, so RunSession never saw a lost conversation and never restarted it fresh, and StreamCmd never saw the error events it classifies as account errors. The retry now skips a model account refused for good and a spent budget. Codex gets its effort through Job.Effort, catalogued CWE ids such as cwe-787 are spelled canonically instead of dropped, GitHub reads retry 429, 502, 503 and 504, a test keeps the agent image pins in step, and CI vets the evals.
1 parent e9530d5 commit e520a67

11 files changed

Lines changed: 226 additions & 28 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ jobs:
2929
- name: go mod tidy
3030
run: go mod tidy -diff
3131
- run: go vet ./...
32+
# The evals call real models and never run here, so vetting is what keeps them compiling.
33+
- run: go vet -tags evals ./...
3234
- run: go tool gotestsum --format github-actions -- -race -count=1 ./...
3335

3436
# release.yml only runs on tags, so a stale lockfile entry would otherwise break the next release.

‎docs/configuration.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ Each report names the source commit and the input revision it covers.
5656
| Request | 1 MiB of body. API responses ask for at most 16,000 output tokens, and the ChatGPT backend applies its own output limits. |
5757
| Reports | One per known reporter and per execution |
5858

59-
tamis starts a report only while the execution still holds a whole advisory budget and thirty minutes of its time. Otherwise the report waits for the next execution. SDK retries are off, but tamis retries an agent run that fails without a valid report once, after a minute. When a budget runs out, the report stays retriable and tamis does not mark it complete.
59+
tamis starts a report only while the execution still holds a whole advisory budget and thirty minutes of its time. Otherwise the report waits for the next execution. SDK retries are off, but tamis retries an agent run that fails without a valid report once, after a minute, unless the model account was refused for good or the advisory budget is spent. A GitHub read answered with 429, 502, 503 or 504 is tried up to three times. When a budget runs out, the report stays retriable and tamis does not mark it complete.
6060

6161
These bounds cap one execution, not the spending across scheduled runs. Provider pricing, tokenisation and ChatGPT credit policies differ, so request and byte limits give no monetary guarantee. On the provider side:
6262

‎internal/github/client.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"net/url"
1515
"regexp"
1616
"slices"
17+
"strconv"
1718
"strings"
1819
"time"
1920

@@ -604,7 +605,43 @@ func list[T any](ctx context.Context, c *Client, next string) ([]T, error) {
604605
return all, nil
605606
}
606607

608+
const (
609+
maxGetAttempts = 3
610+
// maxRetryAfter is the longest Retry-After a GET waits for, a longer one fails it.
611+
maxRetryAfter = time.Minute
612+
)
613+
614+
var (
615+
// retryDelay separates the attempts of a GET when GitHub sends no Retry-After.
616+
retryDelay = 2 * time.Second
617+
retried = []int{http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout}
618+
)
619+
620+
// do tries a GET again when GitHub is briefly overloaded. Writes are tried once, since a retried write may
621+
// apply twice.
607622
func (c *Client) do(ctx context.Context, method, u string, body, out any) (*http.Response, error) {
623+
for attempt := 1; ; attempt++ {
624+
resp, err := c.once(ctx, method, u, body, out)
625+
var httpErr *HTTPError
626+
if method != http.MethodGet || attempt == maxGetAttempts || !errors.As(err, &httpErr) || !slices.Contains(retried, httpErr.Status) {
627+
return resp, err
628+
}
629+
delay := retryDelay
630+
if seconds, perr := strconv.Atoi(resp.Header.Get("Retry-After")); perr == nil && seconds >= 0 {
631+
delay = time.Duration(seconds) * time.Second
632+
}
633+
if delay > maxRetryAfter {
634+
return resp, err
635+
}
636+
select {
637+
case <-ctx.Done():
638+
return resp, err
639+
case <-time.After(delay):
640+
}
641+
}
642+
}
643+
644+
func (c *Client) once(ctx context.Context, method, u string, body, out any) (*http.Response, error) {
608645
var reader io.Reader
609646
if body != nil {
610647
b, err := json.Marshal(body)

‎internal/github/client_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"encoding/base64"
77
"encoding/json"
88
"fmt"
9+
"io"
910
"net/http"
1011
"net/http/httptest"
1112
"strings"
@@ -301,6 +302,49 @@ func TestErrors(t *testing.T) {
301302
require.NotContains(t, err.Error(), "GHSA-secret-id", "the body or the advisory leaked")
302303
}
303304

305+
func TestGetRetriesABrieflyOverloadedGitHub(t *testing.T) {
306+
delay := retryDelay
307+
retryDelay = 0
308+
t.Cleanup(func() { retryDelay = delay })
309+
for name, tc := range map[string]struct {
310+
statuses []int
311+
retryAfter string
312+
post bool
313+
calls int
314+
ok bool
315+
}{
316+
"recovers": {statuses: []int{http.StatusServiceUnavailable, http.StatusOK}, calls: 2, ok: true},
317+
"rate limited": {statuses: []int{http.StatusTooManyRequests, http.StatusOK}, retryAfter: "0", calls: 2, ok: true},
318+
"gives up": {statuses: []int{http.StatusBadGateway, http.StatusBadGateway, http.StatusBadGateway, http.StatusOK}, calls: 3},
319+
"retry-after too long": {statuses: []int{http.StatusTooManyRequests, http.StatusOK}, retryAfter: "3600", calls: 1},
320+
"not a transient failure": {statuses: []int{http.StatusUnprocessableEntity, http.StatusOK}, calls: 1},
321+
"write": {statuses: []int{http.StatusServiceUnavailable, http.StatusOK}, post: true, calls: 1},
322+
} {
323+
t.Run(name, func(t *testing.T) {
324+
var calls int
325+
c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) {
326+
status := tc.statuses[calls]
327+
calls++
328+
if tc.retryAfter != "" {
329+
w.Header().Set("Retry-After", tc.retryAfter)
330+
}
331+
w.WriteHeader(status)
332+
_, _ = io.WriteString(w, `{}`)
333+
})
334+
335+
var err error
336+
if tc.post {
337+
err = c.PostComment(t.Context(), "o/r", "GHSA-aaaa-bbbb-cccc", "body")
338+
} else {
339+
_, err = c.GetIssue(t.Context(), "o/r", 1)
340+
}
341+
342+
require.Equal(t, tc.calls, calls)
343+
require.Equal(t, tc.ok, err == nil, "err = %v", err)
344+
})
345+
}
346+
}
347+
304348
func TestNetworkErrorsDoNotLeakTheURL(t *testing.T) {
305349
c := &Client{BaseURL: "http://127.0.0.1:1", Token: "tok", HTTP: http.DefaultClient}
306350

‎internal/report/report.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,15 @@ func ValidateReader(r Report) error {
286286
return nil
287287
}
288288

289+
// canonicalCWE spells a catalogued weakness as CWE-<n>, so "cwe-787" or "787" is not dropped.
290+
func canonicalCWE(id string) string {
291+
if canonical, _, ok := cwe.Lookup(id); ok {
292+
return canonical
293+
}
294+
295+
return id
296+
}
297+
289298
// Clues bounds the syntax; identifiers can still carry hostile instructions.
290299
func Clues(r Report) Comparison {
291300
var c Comparison
@@ -301,8 +310,8 @@ func Clues(r Report) Comparison {
301310
}
302311
}
303312
for _, w := range r.CWE {
304-
if cweID.MatchString(w.ID) && !slices.Contains(c.CWE, w.ID) {
305-
c.CWE = append(c.CWE, w.ID)
313+
if id := canonicalCWE(w.ID); cweID.MatchString(id) && !slices.Contains(c.CWE, id) {
314+
c.CWE = append(c.CWE, id)
306315
}
307316
}
308317

@@ -393,6 +402,7 @@ func (c Checker) Verify(ctx context.Context, r *Report) error {
393402

394403
cwes := r.CWE[:0]
395404
for _, w := range r.CWE {
405+
w.ID = canonicalCWE(w.ID)
396406
if !cweID.MatchString(w.ID) {
397407
r.Removed++
398408
continue

‎internal/report/report_test.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,15 @@ func TestVerifyKeepsWhatExists(t *testing.T) {
8585
require.True(t, r.CodeReferences[1].Found, "references should be found: %+v", r.CodeReferences)
8686
}
8787

88+
func TestVerifySpellsCataloguedCWEsCanonically(t *testing.T) {
89+
r := Report{CWE: []CWE{{ID: "cwe-787", Reason: "write"}, {ID: " 125 "}, {ID: "CWE-99999"}}}
90+
91+
require.NoError(t, checker(t).Verify(context.Background(), &r))
92+
93+
require.Equal(t, []CWE{{ID: "CWE-787", Reason: "write"}, {ID: "CWE-125"}, {ID: "CWE-99999"}}, r.CWE)
94+
require.Zero(t, r.Removed)
95+
}
96+
8897
func TestVerifyDropsWhatDoesNotExist(t *testing.T) {
8998
r := Report{
9099
Duplicates: []Duplicate{{GHSAID: "GHSA-invented", Confidence: "high"}, {GHSAID: "GHSA-real", Confidence: "certain"}},
@@ -387,15 +396,15 @@ func TestClues(t *testing.T) {
387396
r := Report{
388397
Location: Location{Path: "ext/standard/url.c", Symbol: "php_url_encode", Line: 512},
389398
EntryPoints: []string{"http_build_query", "SplFixedArray::offsetSet", "Request.parse", "http_build_query", "urlencode(); echo", "print all advisories", "os.path.join", "a::b.c"},
390-
CWE: []CWE{{ID: "CWE-787"}, {ID: "CWE-787"}, {ID: "describe GHSA-x"}},
399+
CWE: []CWE{{ID: "CWE-787"}, {ID: "cwe-787"}, {ID: "describe GHSA-x"}, {ID: "125"}},
391400
}
392401

393402
got := Clues(r)
394403

395404
want := Comparison{
396405
Location: Location{Path: "ext/standard/url.c", Symbol: "php_url_encode", Line: 512},
397406
EntryPoints: []string{"http_build_query", "SplFixedArray::offsetSet", "Request.parse"},
398-
CWE: []string{"CWE-787"},
407+
CWE: []string{"CWE-787", "CWE-125"},
399408
}
400409
require.Equal(t, want.Location, got.Location)
401410
require.Equal(t, want.EntryPoints, got.EntryPoints)

‎internal/sandbox/runner.go‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,13 @@ type agent struct {
2424
// provider's own API.
2525
func (*agent) EgressHosts() []string { return nil }
2626

27-
// ParseStream hands only the session event to emit, so that the run can be resumed.
27+
// ParseStream hands every event to harness, which reads the session, a lost conversation and the account
28+
// errors from them. Read emits none of them further.
2829
func (a *agent) ParseStream(r io.Reader, emit func(harness.Event)) {
2930
pr, pw := io.Pipe()
3031
done := make(chan struct{})
3132
go func() {
32-
a.Harness.ParseStream(pr, func(e harness.Event) {
33-
if e.Kind == harness.KindSession {
34-
emit(e)
35-
}
36-
})
33+
a.Harness.ParseStream(pr, emit)
3734
_, _ = io.Copy(io.Discard, pr)
3835
close(done)
3936
}()

‎internal/sandbox/sandbox.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,13 @@ type backend struct {
4848
basePath string
4949
// accessToken signs Codex in with a Codex access token, which it only sends over HTTPS.
5050
accessToken bool
51+
effort string
5152
}
5253

5354
var backends = map[string]backend{
54-
"claude": {"claude", modelproxy.Anthropic, "ANTHROPIC_API_KEY", modelproxy.PathPrefix, false},
55-
"codex": {"codex", modelproxy.OpenAI, "CODEX_API_KEY", modelproxy.PathPrefix + "/v1", false},
56-
"codex-token": {"codex", modelproxy.ChatGPT, "CODEX_ACCESS_TOKEN", modelproxy.PathPrefix + "/backend-api/codex", true},
55+
"claude": {"claude", modelproxy.Anthropic, "ANTHROPIC_API_KEY", modelproxy.PathPrefix, false, ""},
56+
"codex": {"codex", modelproxy.OpenAI, "CODEX_API_KEY", modelproxy.PathPrefix + "/v1", false, "medium"},
57+
"codex-token": {"codex", modelproxy.ChatGPT, "CODEX_ACCESS_TOKEN", modelproxy.PathPrefix + "/backend-api/codex", true, "medium"},
5758
}
5859

5960
// imagesJSON pins each agent image by digest, so a version of tamis always runs the images built for it.
@@ -212,6 +213,7 @@ func (r *Reader) Read(ctx context.Context, req Request) (json.RawMessage, error)
212213
Prompt: req.Prompt,
213214
SystemPrompt: req.System,
214215
Model: r.model,
216+
Effort: r.backend.effort,
215217
MaxTurns: maxTurns,
216218
OutputFile: outputFile,
217219
BaseURL: origin + r.backend.basePath,
@@ -302,7 +304,7 @@ func (r *Reader) writeConfig(state, origin string) error {
302304
}
303305
}
304306
// The ChatGPT features are off: they would call routes the model proxy refuses.
305-
config := fmt.Sprintf("model_provider = \"tamis\"\nmodel_reasoning_effort = \"medium\"\ndaybreak = %t\nweb_search = \"disabled\"\nchatgpt_base_url = %q\n\n[features]\nremote_plugin = false\nplugin_sharing = false\ngoals = false\nimage_generation = false\nskill_mcp_dependency_install = false\n\n[model_providers.tamis]\nname = \"Tamis\"\nbase_url = %q\n%s\nwire_api = \"responses\"\nsupports_websockets = false\nsupports_standalone_web_search = false\nrequest_max_retries = 0\nstream_max_retries = 0\n", r.Daybreak, origin+modelproxy.PathPrefix+"/backend-api", origin+r.backend.basePath, auth)
307+
config := fmt.Sprintf("model_provider = \"tamis\"\ndaybreak = %t\nweb_search = \"disabled\"\nchatgpt_base_url = %q\n\n[features]\nremote_plugin = false\nplugin_sharing = false\ngoals = false\nimage_generation = false\nskill_mcp_dependency_install = false\n\n[model_providers.tamis]\nname = \"Tamis\"\nbase_url = %q\n%s\nwire_api = \"responses\"\nsupports_websockets = false\nsupports_standalone_web_search = false\nrequest_max_retries = 0\nstream_max_retries = 0\n", r.Daybreak, origin+modelproxy.PathPrefix+"/backend-api", origin+r.backend.basePath, auth)
306308
return os.WriteFile(filepath.Join(state, "config.toml"), []byte(config), 0o600)
307309
}
308310

‎internal/sandbox/sandbox_test.go‎

Lines changed: 60 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"os/exec"
99
"path/filepath"
1010
"regexp"
11+
"slices"
1112
"strings"
1213
"testing"
1314

@@ -165,7 +166,7 @@ func TestOnlyDockerCanRunTheAgent(t *testing.T) {
165166
require.Error(t, err, "an unsupported runtime must fail closed")
166167
}
167168

168-
func TestAgentKeepsOnlyTheFailuresAndTheSessionOfItsOutput(t *testing.T) {
169+
func TestAgentKeepsOnlyTheFailuresOfItsOutput(t *testing.T) {
169170
h, err := harness.ByName("codex")
170171
require.NoError(t, err)
171172
a := &agent{Harness: h}
@@ -174,21 +175,45 @@ func TestAgentKeepsOnlyTheFailuresAndTheSessionOfItsOutput(t *testing.T) {
174175
a.ParseStream(strings.NewReader(`{"type":"thread.started","thread_id":"thread-1"}`+"\n"+`{"type":"item.completed","item":{"type":"agent_message","text":"report content"}}`+"\n"+`{"type":"turn.failed","error":{"message":"usage limit"}}`), func(e harness.Event) { events = append(events, e) })
175176

176177
require.Equal(t, `{"type":"turn.failed","error":{"message":"usage limit"}}`, a.failures.String())
177-
require.Equal(t, []harness.Event{{Kind: harness.KindSession, SessionID: "thread-1"}}, events)
178+
require.Len(t, events, 3)
179+
require.Equal(t, harness.Event{Kind: harness.KindSession, SessionID: "thread-1"}, events[0])
178180
}
179181

180-
func TestAgentForwardsNoEventWithoutASession(t *testing.T) {
182+
func TestAgentLetsHarnessRestartALostConversation(t *testing.T) {
181183
h, err := harness.ByName("claude")
182184
require.NoError(t, err)
183185
a := &agent{Harness: h}
184-
var events []harness.Event
186+
var jobs []harness.Job
187+
run := func(_ context.Context, j harness.Job, emit func(harness.Event)) error {
188+
jobs = append(jobs, j)
189+
if j.ResumeSessionID != "" {
190+
a.ParseStream(strings.NewReader("No conversation found with session ID: dead\n"), emit)
191+
return errors.New("exit status 1")
192+
}
193+
a.ParseStream(strings.NewReader(`{"type":"system","subtype":"init","session_id":"fresh"}`+"\n"), emit)
194+
return nil
195+
}
185196

186-
a.ParseStream(strings.NewReader("No conversation found with session ID: dead\n"), func(e harness.Event) { events = append(events, e) })
197+
session, err := harness.RunSession(t.Context(), run, harness.Job{Prompt: "Answer", ResumeSessionID: "dead", ResumePrompt: "Fix it"}, nil)
187198

188-
require.Empty(t, events)
199+
require.NoError(t, err)
200+
require.Len(t, jobs, 2, "a resume whose conversation is lost must restart fresh")
201+
require.Empty(t, jobs[1].ResumeSessionID)
202+
require.Equal(t, "fresh", session.ID)
189203
require.Empty(t, a.failures.String())
190204
}
191205

206+
func TestAgentLetsHarnessSeeTheAccountError(t *testing.T) {
207+
h, err := harness.ByName("claude")
208+
require.NoError(t, err)
209+
a := &agent{Harness: h}
210+
var seen []harness.Event
211+
212+
a.ParseStream(strings.NewReader(`{"type":"error","error":"Credit balance is too low"}`+"\n"), func(e harness.Event) { seen = append(seen, e) })
213+
214+
require.True(t, slices.ContainsFunc(seen, func(e harness.Event) bool { return e.Kind == harness.KindError && h.AccountErrorText(e.Text) != "" }), "events = %+v", seen)
215+
}
216+
192217
func TestReadOutputIsBounded(t *testing.T) {
193218
path := filepath.Join(t.TempDir(), "report.json")
194219
for name, size := range map[string]int{"valid": 1 << 20, "too large": 1<<20 + 1} {
@@ -205,16 +230,17 @@ func TestReadOutputIsBounded(t *testing.T) {
205230
}
206231

207232
func TestBackends(t *testing.T) {
208-
for name, want := range map[string]struct{ harness, keyEnv, basePath string }{
209-
"claude": {"claude", "ANTHROPIC_API_KEY", "/model-proxy"},
210-
"codex": {"codex", "CODEX_API_KEY", "/model-proxy/v1"},
211-
"codex-token": {"codex", "CODEX_ACCESS_TOKEN", "/model-proxy/backend-api/codex"},
233+
for name, want := range map[string]struct{ harness, keyEnv, basePath, effort string }{
234+
"claude": {"claude", "ANTHROPIC_API_KEY", "/model-proxy", ""},
235+
"codex": {"codex", "CODEX_API_KEY", "/model-proxy/v1", "medium"},
236+
"codex-token": {"codex", "CODEX_ACCESS_TOKEN", "/model-proxy/backend-api/codex", "medium"},
212237
} {
213238
b, ok := backends[name]
214239
assert.True(t, ok, name)
215240
assert.Equal(t, want.harness, b.harness, name)
216241
assert.Equal(t, want.keyEnv, b.keyEnv, name)
217242
assert.Equal(t, want.basePath, b.basePath, name)
243+
assert.Equal(t, want.effort, b.effort, name)
218244
assert.Equal(t, name == "codex-token", b.accessToken, name)
219245
}
220246
}
@@ -249,7 +275,7 @@ func TestWriteConfig(t *testing.T) {
249275
} else {
250276
require.Empty(t, got)
251277
}
252-
for _, want := range []string{"model_provider = \"tamis\"", "model_reasoning_effort = \"medium\"", "supports_websockets = false", "chatgpt_base_url = \"https://proxy:1/model-proxy/backend-api\"", "base_url = \"https://proxy:1" + backends[tc.backend].basePath + "\""} {
278+
for _, want := range []string{"model_provider = \"tamis\"", "supports_websockets = false", "chatgpt_base_url = \"https://proxy:1/model-proxy/backend-api\"", "base_url = \"https://proxy:1" + backends[tc.backend].basePath + "\""} {
253279
if tc.wantConfig {
254280
require.Contains(t, string(got), want)
255281
}
@@ -284,6 +310,29 @@ func TestImagesArePinnedByDigest(t *testing.T) {
284310
assert.Len(t, Images, len(agents))
285311
}
286312

313+
func TestAgentImagePinsAgree(t *testing.T) {
314+
dockerfile, err := os.ReadFile("../../docker/agent/Dockerfile")
315+
require.NoError(t, err)
316+
gomod, err := os.ReadFile("../../go.mod")
317+
require.NoError(t, err)
318+
319+
locks := map[string][]string{}
320+
for _, m := range regexp.MustCompile(`(?m)^ARG ([A-Z]+)_(?:AMD64|ARM64)_LOCK=([^@\s]+)@sha256:[0-9a-f]{64}$`).FindAllStringSubmatch(string(dockerfile), -1) {
321+
locks[m[1]] = append(locks[m[1]], m[2])
322+
}
323+
require.NotEmpty(t, locks)
324+
for cli, releases := range locks {
325+
require.Len(t, releases, 2, "%s must pin one asset per architecture", cli)
326+
require.Equal(t, releases[0], releases[1], "%s pins two releases", cli)
327+
}
328+
329+
proxy := regexp.MustCompile(`(?m)^ARG HARNESS_VERSION=(\S+)$`).FindStringSubmatch(string(dockerfile))
330+
module := regexp.MustCompile(`(?m)^\s*github\.com/alpha-omega-security/harness (\S+)$`).FindStringSubmatch(string(gomod))
331+
require.Len(t, proxy, 2)
332+
require.Len(t, module, 2)
333+
require.Equal(t, module[1], proxy[1], "harness-proxy in the agent image must match the harness module")
334+
}
335+
287336
func TestKeepFailuresKeepsOnlyCodexErrorEvents(t *testing.T) {
288337
failed := `{"type":"turn.failed","error":{"message":"usage limit reached"}}`
289338
stream := strings.Join([]string{

0 commit comments

Comments
 (0)