Skip to content

feat(parser): migrate amp and zencoder providers #759

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mariusvniekerk wants to merge 1 commit into from
Closed

feat(parser): migrate amp and zencoder providers #759

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions internal/parser/amp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@ import (
"github.com/tidwall/gjson"
)

// ParseAmpSession parses an Amp thread JSON file.
// Each thread is a single JSON document at ~/.local/share/amp/threads/T-*.json.
func ParseAmpSession(
func parseAmpSession(
path, machine string,
) (*ParsedSession, []ParsedMessage, error) {
info, err := os.Stat(path)
Expand Down
63 changes: 63 additions & 0 deletions internal/parser/amp_provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package parser

import (
"context"
"path/filepath"
)

// Amp stores each thread as a single JSON file in a directory. It is a
// directory-of-files provider: discovery, watching, change classification,
// lookup, and fingerprinting come from JSONLSourceSet, and the ParseFile option
// makes that source set a full SourceSet so it rides the generic factory.
func newAmpProviderFactory(def AgentDef) ProviderFactory {
return newSourceSetFactory(
def,
ampProviderCapabilities(),
func(cfg ProviderConfig) SourceSet { return newAmpSourceSet(cfg.Roots) },
)
}

func newAmpSourceSet(roots []string) JSONLSourceSet {
return newJSONLSourceSet(AgentAmp, roots,
withExtensions(".json"),
withFollowSymlinkFiles(),
withContentHashing(),
withIncludePath(isAmpSourcePath),
withSessionIDFromPath(func(root, path string) string {
return ampThreadIDFromPath(path)
}),
withParseFile(ampParseFile),
)
}

func ampParseFile(
_ context.Context, path string, req ParseRequest,
) ([]ParseResult, []string, error) {
sess, msgs, err := parseAmpSession(path, req.Machine)
if err != nil {
return nil, nil, err
}
if sess == nil {
return nil, nil, nil
}
if req.Fingerprint.Hash != "" {
sess.File.Hash = req.Fingerprint.Hash
}
return []ParseResult{{Session: *sess, Messages: msgs}}, nil, nil
}

func isAmpSourcePath(root, path string) bool {
return IsAmpThreadFileName(filepath.Base(path))
}

func ampProviderCapabilities() Capabilities {
return Capabilities{
Source: jsonlFileProviderSourceCapabilities(),
Content: ContentCapabilities{
FirstMessage: CapabilitySupported,
Thinking: CapabilitySupported,
ToolCalls: CapabilitySupported,
ToolResults: CapabilitySupported,
},
}
}
75 changes: 55 additions & 20 deletions internal/parser/amp_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package parser

import (
"context"
"path/filepath"
"strings"
"testing"
"time"
Expand All @@ -15,10 +17,43 @@ func runAmpParserTest(
) (*ParsedSession, []ParsedMessage, error) {
t.Helper()
path := createTestFile(t, "T-test.json", content)
return ParseAmpSession(path, "local")
return parseAmpTestSession(t, path, "local")
}

func TestParseAmpSession_Basic(t *testing.T) {
func parseAmpTestSession(
t *testing.T,
path string,
machine string,
) (*ParsedSession, []ParsedMessage, error) {
t.Helper()

provider, ok := NewProvider(AgentAmp, ProviderConfig{
Roots: []string{filepath.Dir(path)},
Machine: machine,
})
require.True(t, ok)

outcome, err := provider.Parse(context.Background(), ParseRequest{
Source: SourceRef{
Provider: AgentAmp,
Key: path,
DisplayPath: path,
FingerprintKey: path,
Opaque: JSONLSource{
Root: filepath.Dir(path),
Path: path,
},
},
Machine: machine,
})
if err != nil || len(outcome.Results) == 0 {
return nil, nil, err
}
result := outcome.Results[0].Result
return &result.Session, result.Messages, nil
}

func TestAmpProviderParsesBasic(t *testing.T) {
threadID := "T-019ca26f-aaaa-bbbb-cccc-dddddddddddd"
content := `{
"v": 1,
Expand All @@ -43,7 +78,7 @@ func TestParseAmpSession_Basic(t *testing.T) {
}`

path := createTestFile(t, threadID+".json", content)
sess, msgs, err := ParseAmpSession(path, "local")
sess, msgs, err := parseAmpTestSession(t, path, "local")
require.NoError(t, err)
require.NotNil(t, sess)

Expand Down Expand Up @@ -72,7 +107,7 @@ func TestParseAmpSession_Basic(t *testing.T) {
assert.Equal(t, 1, msgs[1].Ordinal)
}

func TestParseAmpSession_ToolUseAndThinking(t *testing.T) {
func TestAmpProviderParsesToolUseAndThinking(t *testing.T) {
content := `{
"v": 1,
"id": "T-tooluse",
Expand Down Expand Up @@ -322,7 +357,7 @@ func TestExtractAmpToolResults(t *testing.T) {
}
}

func TestParseAmpSession_AmpToolResultSchema(t *testing.T) {
func TestAmpProviderParsesAmpToolResultSchema(t *testing.T) {
content := `{
"v": 1,
"id": "T-amp-tool-result-schema",
Expand All @@ -347,7 +382,7 @@ func TestParseAmpSession_AmpToolResultSchema(t *testing.T) {
assert.Equal(t, "Here is a complete breakdown", DecodeContent(msgs[1].ToolResults[0].ContentRaw))
}

func TestParseAmpSession_AmpToolResultDict(t *testing.T) {
func TestAmpProviderParsesAmpToolResultDict(t *testing.T) {
content := `{
"v": 1,
"id": "T-amp-tool-result-dict",
Expand All @@ -370,7 +405,7 @@ func TestParseAmpSession_AmpToolResultDict(t *testing.T) {
assert.Equal(t, "cmd output", DecodeContent(msgs[1].ToolResults[0].ContentRaw))
}

func TestParseAmpSession_NoEnv(t *testing.T) {
func TestAmpProviderParsesNoEnv(t *testing.T) {
content := `{
"v": 1,
"id": "T-noenv",
Expand All @@ -389,7 +424,7 @@ func TestParseAmpSession_NoEnv(t *testing.T) {
require.Equal(t, 1, len(msgs))
}

func TestParseAmpSession_NoTitle(t *testing.T) {
func TestAmpProviderParsesNoTitle(t *testing.T) {
content := `{
"v": 1,
"id": "T-notitle",
Expand All @@ -408,7 +443,7 @@ func TestParseAmpSession_NoTitle(t *testing.T) {
assert.Equal(t, "Fix the bug in main.go please.", sess.FirstMessage)
}

func TestParseAmpSession_NoMetaTraces(t *testing.T) {
func TestAmpProviderParsesNoMetaTraces(t *testing.T) {
content := `{
"v": 1,
"id": "T-notraces",
Expand All @@ -427,7 +462,7 @@ func TestParseAmpSession_NoMetaTraces(t *testing.T) {
assertZeroTimestamp(t, sess.EndedAt, "EndedAt")
}

func TestParseAmpSession_LastTraceWithoutEndTime(t *testing.T) {
func TestAmpProviderParsesLastTraceWithoutEndTime(t *testing.T) {
content := `{
"v": 1,
"id": "T-trace-end-missing",
Expand All @@ -451,7 +486,7 @@ func TestParseAmpSession_LastTraceWithoutEndTime(t *testing.T) {
assert.Equal(t, "2024-01-01T00:00:02Z", sess.EndedAt.UTC().Format(time.RFC3339))
}

func TestParseAmpSession_EmptyThread(t *testing.T) {
func TestAmpProviderParsesEmptyThread(t *testing.T) {
content := `{
"v": 1,
"id": "T-empty",
Expand All @@ -466,7 +501,7 @@ func TestParseAmpSession_EmptyThread(t *testing.T) {
assert.Nil(t, msgs)
}

func TestParseAmpSession_FirstMessageTruncation(t *testing.T) {
func TestAmpProviderParsesFirstMessageTruncation(t *testing.T) {
longText := strings.Repeat("a", 400)
content := `{"v":1,"id":"T-trunc","created":1704067200000,"messages":[` +
`{"role":"user","content":[{"type":"text","text":"` + longText + `"}]}]}`
Expand All @@ -478,7 +513,7 @@ func TestParseAmpSession_FirstMessageTruncation(t *testing.T) {
assert.Equal(t, 303, len(sess.FirstMessage))
}

func TestParseAmpSession_InvalidCreated(t *testing.T) {
func TestAmpProviderParsesInvalidCreated(t *testing.T) {
t.Run("missing created", func(t *testing.T) {
content := `{
"v": 1,
Expand Down Expand Up @@ -539,9 +574,9 @@ func TestParseAmpSession_InvalidCreated(t *testing.T) {
})
}

func TestParseAmpSession_Errors(t *testing.T) {
func TestAmpProviderParsesErrors(t *testing.T) {
t.Run("missing file", func(t *testing.T) {
_, _, err := ParseAmpSession("/nonexistent/T-xxx.json", "local")
_, _, err := parseAmpTestSession(t, "/nonexistent/T-xxx.json", "local")
assert.Error(t, err)
})

Expand All @@ -563,13 +598,13 @@ func TestParseAmpSession_Errors(t *testing.T) {
t.Run("missing id and invalid filename", func(t *testing.T) {
content := `{"v":1,"created":1704067200000,"messages":[]}`
path := createTestFile(t, "bad-name.json", content)
_, _, err := ParseAmpSession(path, "local")
_, _, err := parseAmpTestSession(t, path, "local")
assert.Error(t, err)
assert.Contains(t, err.Error(), "missing or invalid id")
})
}

func TestParseAmpSession_MismatchedID(t *testing.T) {
func TestAmpProviderParsesMismatchedID(t *testing.T) {
t.Run("invalid JSON id", func(t *testing.T) {
content := `{
"v": 1,
Expand All @@ -581,7 +616,7 @@ func TestParseAmpSession_MismatchedID(t *testing.T) {
}`

path := createTestFile(t, "T-fallback-uuid.json", content)
sess, _, err := ParseAmpSession(path, "local")
sess, _, err := parseAmpTestSession(t, path, "local")
require.NoError(t, err)
require.NotNil(t, sess)
assert.Equal(t, "amp:T-fallback-uuid", sess.ID)
Expand All @@ -600,7 +635,7 @@ func TestParseAmpSession_MismatchedID(t *testing.T) {
}`

path := createTestFile(t, "bad-name.json", content)
sess, _, err := ParseAmpSession(path, "local")
sess, _, err := parseAmpTestSession(t, path, "local")
require.NoError(t, err)
require.NotNil(t, sess)
assert.Equal(t, "amp:T-from-json", sess.ID)
Expand All @@ -620,7 +655,7 @@ func TestParseAmpSession_MismatchedID(t *testing.T) {
}`

path := createTestFile(t, "T-from-file.json", content)
sess, _, err := ParseAmpSession(path, "local")
sess, _, err := parseAmpTestSession(t, path, "local")
require.NoError(t, err)
require.NotNil(t, sess)
assert.Equal(t, "amp:T-from-file", sess.ID)
Expand Down
Loading