Merge pull request #103 from sloganking/codex/add-v2-media-playback-commands

Add media controls to assistant_v2

Author: sloganking

assistant_v2/Cargo.toml

@@ -23,3 +23,4 @@ clap = { version = "4.4.6", features = ["derive"] }
 colored = "2.0.4"
 clipboard = "0.5.0"
 open = "5.3.1"
+enigo = "0.1.3"

assistant_v2/FEATURE_PROGRESS.md

@@ -6,7 +6,7 @@ This document tracks which features from the original assistant have been implem
 | --- | --- |
 | Screen brightness control | Done |
 | System volume adjustment (Windows only) | Pending |
-| Media playback commands | Pending |
+| Media playback commands | Done |
 | Launch applications from voice | Pending |
 | Display log files | Pending |
 | Get system info | Pending |

assistant_v2/src/main.rs

@@ -13,6 +13,7 @@ use colored::Colorize;
 use dotenvy::dotenv;
 use futures::StreamExt;
 use open;
+use enigo::{Enigo, KeyboardControllable};
 use speakstream::ss::SpeakStream;
 use std::error::Error;
 use std::path::PathBuf;
@@ -145,6 +146,30 @@ async fn main() -> Result<(), Box<dyn Error>> {
                 strict: None,
             }
             .into(),
+            FunctionObject {
+                name: "media_controls".into(),
+                description: Some("Plays, pauses or seeks media.".into()),
+                parameters: Some(serde_json::json!({
+                    "type": "object",
+                    "properties": {
+                        "media_button": {
+                            "type": "string",
+                            "enum": [
+                                "MediaStop",
+                                "MediaNextTrack",
+                                "MediaPlayPause",
+                                "MediaPrevTrack",
+                                "VolumeUp",
+                                "VolumeDown",
+                                "VolumeMute"
+                            ]
+                        }
+                    },
+                    "required": ["media_button"]
+                })),
+                strict: None,
+            }
+            .into(),
             FunctionObject {
                 name: "set_clipboard".into(),
                 description: Some("Sets the clipboard to the given text.".into()),
@@ -484,6 +509,56 @@ async fn handle_requires_action(
                 });
             }
 
+            if tool.function.name == "media_controls" {
+                let button = match serde_json::from_str::<serde_json::Value>(&tool.function.arguments) {
+                    Ok(v) => v["media_button"].as_str().unwrap_or("").to_string(),
+                    Err(_) => String::new(),
+                };
+
+                let mut enigo = Enigo::new();
+                let msg = match button.as_str() {
+                    "MediaStop" => {
+                        enigo.key_click(enigo::Key::MediaStop);
+                        "MediaStop"
+                    }
+                    "MediaNextTrack" => {
+                        enigo.key_click(enigo::Key::MediaNextTrack);
+                        "MediaNextTrack"
+                    }
+                    "MediaPlayPause" => {
+                        enigo.key_click(enigo::Key::MediaPlayPause);
+                        "MediaPlayPause"
+                    }
+                    "MediaPrevTrack" => {
+                        enigo.key_click(enigo::Key::MediaPrevTrack);
+                        enigo.key_click(enigo::Key::MediaPrevTrack);
+                        "MediaPrevTrack"
+                    }
+                    "VolumeUp" => {
+                        for _ in 0..5 {
+                            enigo.key_click(enigo::Key::VolumeUp);
+                        }
+                        "VolumeUp"
+                    }
+                    "VolumeDown" => {
+                        for _ in 0..5 {
+                            enigo.key_click(enigo::Key::VolumeDown);
+                        }
+                        "VolumeDown"
+                    }
+                    "VolumeMute" => {
+                        enigo.key_click(enigo::Key::VolumeMute);
+                        "VolumeMute"
+                    }
+                    _ => "Unknown button",
+                };
+
+                tool_outputs.push(ToolsOutputs {
+                    tool_call_id: Some(tool.id.clone()),
+                    output: Some(msg.into()),
+                });
+            }
+
             if tool.function.name == "open_openai_billing" {
                 let result = open::that("https://platform.openai.com/usage");
                 let msg = match result {
@@ -711,6 +786,46 @@ mod tests {
         }));
     }
 
+    #[test]
+    fn includes_media_controls_function() {
+        let req = CreateAssistantRequestArgs::default()
+            .instructions("test")
+            .model("gpt-4o")
+            .tools(vec![FunctionObject {
+                name: "media_controls".into(),
+                description: Some("Plays, pauses or seeks media.".into()),
+                parameters: Some(serde_json::json!({
+                    "type": "object",
+                    "properties": {
+                        "media_button": {
+                            "type": "string",
+                            "enum": [
+                                "MediaStop",
+                                "MediaNextTrack",
+                                "MediaPlayPause",
+                                "MediaPrevTrack",
+                                "VolumeUp",
+                                "VolumeDown",
+                                "VolumeMute"
+                            ]
+                        }
+                    },
+                    "required": ["media_button"]
+                })),
+                strict: None,
+            }
+            .into()])
+            .build()
+            .unwrap();
+
+        let tools = req.tools.unwrap();
+        assert!(tools.iter().any(|t| match t {
+            async_openai::types::AssistantTools::Function(f) =>
+                f.function.name == "media_controls",
+            _ => false,
+        }));
+    }
+
     #[test]
     fn includes_mute_speech_function() {
         let req = CreateAssistantRequestArgs::default()