sdk-typescript/strandly/src/benchmark/configs.ts at cb20a77bcb3fa0e5427b1f481bb3fd5fef33fe44 · strands-agents/sdk-typescript · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
import { Agent } from '../../../strands-ts/src/agent/agent.js'
import { BedrockModel } from '../../../strands-ts/src/models/bedrock.js'
import { bash } from '../../../strands-ts/src/vended-tools/bash/bash.js'
import { SlidingWindowConversationManager } from '../../../strands-ts/src/conversation-manager/sliding-window-conversation-manager.js'
import { SummarizingConversationManager } from '../../../strands-ts/src/conversation-manager/summarizing-conversation-manager.js'
import { ContextOffloader } from '../../../strands-ts/src/vended-plugins/context-offloader/plugin.js'
import { InMemoryStorage } from '../../../strands-ts/src/vended-plugins/context-offloader/storage.js'
import type { BenchmarkConfig, ContextBenchTask } from './types.js'

const DEFAULT_MODEL = 'us.anthropic.claude-sonnet-4-20250514-v1:0'

// TODO: Update these configs once we have preset context management strategies
export function getConfigs(modelId?: string): BenchmarkConfig[] {
  const model = modelId ?? DEFAULT_MODEL

  return [
    {
      name: 'control',
      description: `SDK default (SlidingWindow ws=40, ${model})`,
      createAgent(task: ContextBenchTask): Agent {
        return new Agent({
          model: new BedrockModel({ modelId: model, stream: false }),
          tools: [bash],
          systemPrompt: task.prompt,
          printer: false,
        })
      },
    },
    {
      name: 'offloader',
      description: `Context offloading (maxResult=2500, preview=1000, ${model})`,
      createAgent(task: ContextBenchTask): Agent {
        return new Agent({
          model: new BedrockModel({ modelId: model, stream: false }),
          tools: [bash],
          plugins: [new ContextOffloader({ storage: new InMemoryStorage() })],
          systemPrompt: task.prompt,
          printer: false,
        })
      },
    },
    {
      name: 'offloader-aggressive',
      description: `Aggressive offloading (maxResult=500, preview=200, ${model})`,
      createAgent(task: ContextBenchTask): Agent {
        return new Agent({
          model: new BedrockModel({ modelId: model, stream: false }),
          tools: [bash],
          plugins: [new ContextOffloader({ storage: new InMemoryStorage(), maxResultTokens: 500, previewTokens: 200 })],
          systemPrompt: task.prompt,
          printer: false,
        })
      },
    },
    {
      name: 'summarizing',
      description: `Summarizing conversation manager (ratio=0.3, proactive, ${model})`,
      createAgent(task: ContextBenchTask): Agent {
        return new Agent({
          model: new BedrockModel({ modelId: model, stream: false }),
          tools: [bash],
          conversationManager: new SummarizingConversationManager({
            summaryRatio: 0.3,
            proactiveCompression: true,
          }),
          systemPrompt: task.prompt,
          printer: false,
        })
      },
    },
    {
      name: 'sliding-proactive',
      description: `Sliding window (ws=40) with proactive compression (${model})`,
      createAgent(task: ContextBenchTask): Agent {
        return new Agent({
          model: new BedrockModel({ modelId: model, stream: false }),
          tools: [bash],
          conversationManager: new SlidingWindowConversationManager({
            windowSize: 40,
            proactiveCompression: true,
          }),
          systemPrompt: task.prompt,
          printer: false,
        })
      },
    },
    {
      name: 'offloader-summarizing',
      description: `Offloading + summarizing combined (${model})`,
      createAgent(task: ContextBenchTask): Agent {
        return new Agent({
          model: new BedrockModel({ modelId: model, stream: false }),
          tools: [bash],
          plugins: [new ContextOffloader({ storage: new InMemoryStorage() })],
          conversationManager: new SummarizingConversationManager({
            summaryRatio: 0.3,
            proactiveCompression: true,
          }),
          systemPrompt: task.prompt,
          printer: false,
        })
      },
    },
  ]
}