Skip to content

Commit 3e42efc

Browse files
codefromthecryptTyler-Hardin
authored andcommitted
feat: add image support and improve error resilience for Codex (aaif-goose#7033)
Signed-off-by: Adrian Cole <adrian@tetrate.io>
1 parent ae0caaf commit 3e42efc

File tree

2 files changed

+296
-118
lines changed

2 files changed

+296
-118
lines changed

crates/goose/src/providers/chatgpt_codex.rs

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,12 @@ fn build_input_items(messages: &[Message]) -> Result<Vec<Value>> {
111111
content_items.push(json!({ "type": content_type, "text": text.text }));
112112
}
113113
}
114+
MessageContent::Image(img) => {
115+
content_items.push(json!({
116+
"type": "input_image",
117+
"image_url": format!("data:{};base64,{}", img.mime_type, img.data),
118+
}));
119+
}
114120
MessageContent::ToolRequest(request) => {
115121
flush_text(&mut items, role, &mut content_items);
116122
if let Ok(tool_call) = &request.tool_call {
@@ -1000,6 +1006,9 @@ mod tests {
10001006
use wiremock::matchers::{body_string_contains, method, path};
10011007
use wiremock::{Mock, MockServer, ResponseTemplate};
10021008

1009+
/// 1x1 transparent PNG, base64-encoded.
1010+
const TINY_PNG_B64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=";
1011+
10031012
fn input_kinds(payload: &Value) -> Vec<String> {
10041013
payload["input"]
10051014
.as_array()
@@ -1095,13 +1104,40 @@ mod tests {
10951104
];
10961105
"includes tool error output"
10971106
)]
1107+
#[test_case(
1108+
vec![
1109+
Message::user()
1110+
.with_text("describe this")
1111+
.with_image(TINY_PNG_B64, "image/png"),
1112+
],
1113+
vec![
1114+
"message:user".to_string(),
1115+
];
1116+
"image content included in user message"
1117+
)]
10981118
fn test_codex_input_order(messages: Vec<Message>, expected: Vec<String>) {
10991119
let items = build_input_items(&messages).unwrap();
11001120
let payload = json!({ "input": items });
11011121
let kinds = input_kinds(&payload);
11021122
assert_eq!(kinds, expected);
11031123
}
11041124

1125+
#[test]
1126+
fn test_image_url_format() {
1127+
let messages = vec![Message::user().with_image(TINY_PNG_B64, "image/png")];
1128+
let items = build_input_items(&messages).unwrap();
1129+
// The image is inside the content array of the user message
1130+
let content = items[0]["content"].as_array().unwrap();
1131+
let image_item = &content[0];
1132+
assert_eq!(image_item["type"], "input_image");
1133+
let url = image_item["image_url"].as_str().unwrap();
1134+
assert!(
1135+
url.starts_with("data:image/png;base64,"),
1136+
"image_url should start with data:image/png;base64, but was: {}",
1137+
url
1138+
);
1139+
}
1140+
11051141
#[test_case(
11061142
JwtClaims {
11071143
chatgpt_account_id: Some("account-1".to_string()),

0 commit comments

Comments
 (0)