fix: Wire /api/vision endpoint to main branch

Root cause: Vision API code existed but was never merged from
feature/shimmy-vision-phase1 branch. This commit adds:

- /api/vision POST route in server.rs
- pub async fn vision() handler in api.rs
- vision + vision_license module exports in lib.rs and main.rs
- vision_license_manager field in AppState
- generate_vision() method on LoadedModel trait
- Remove shimmy-vision private crate dependency (use local code)

The endpoint was working when testing from the feature branch but
the main branch lacked the HTTP server wiring.

Author: Michael-A-Kuykendall

Cargo.toml

@@ -36,7 +36,7 @@ full = ["huggingface", "llama", "mlx"] # Full compilation - includes all backend
 gpu = ["huggingface", "llama-cuda", "llama-vulkan", "llama-opencl"] # GPU-optimized build
 apple = ["huggingface", "mlx"] # Apple Silicon optimized - MLX + HuggingFace
 coverage = ["huggingface"] # Coverage testing - minimal deps for faster builds
-vision = ["dep:shimmy-vision", "dep:image", "dep:base64", "dep:chromiumoxide", "dep:ed25519-dalek", "dep:hex", "dep:sha2"] # Optional vision feature for image/web analysis
+vision = ["dep:image", "dep:base64", "dep:chromiumoxide", "dep:ed25519-dalek", "dep:hex", "dep:sha2"] # Optional vision feature for image/web analysis
 
 [dependencies]
 anyhow = "1"
@@ -76,9 +76,6 @@ reqwest = { version = "0.11", features = ["json", "rustls-tls"], default-feature
 # llama.cpp bindings (optional) - published shimmy-llama-cpp-2 with MoE CPU offloading support
 shimmy-llama-cpp-2 = { version = "0.1.123", optional = true, default-features = false }
 
-# Private vision crate (optional) - licensed vision processing
-shimmy-vision = { git = "https://github.com/Michael-A-Kuykendall/shimmy-vision-private.git", optional = true }
-
 [dev-dependencies]
 tokio-tungstenite = "0.20"
 criterion = { version = "0.5", features = ["html_reports"] }

src/api.rs

@@ -1128,3 +1128,114 @@ mod tests {
         assert_eq!(request.messages.as_ref().unwrap().len(), 1);
     }
 }
+
+#[cfg(feature = "vision")]
+#[axum::debug_handler]
+pub async fn vision(
+    State(state): State<Arc<AppState>>,
+    Json(mut req): Json<crate::vision::VisionRequest>,
+) -> impl IntoResponse {
+    // Extract license from environment or request
+    if req.license.is_none() {
+        req.license = std::env::var("SHIMMY_LICENSE_KEY").ok();
+    }
+
+    // Use default vision model or specified one
+    let env_model = std::env::var("SHIMMY_VISION_MODEL").ok();
+    let model_name = req
+        .model
+        .as_deref()
+        .or(env_model.as_deref())
+        .unwrap_or("minicpm-v")
+        .to_string();
+
+    let Some(license_manager) = state.vision_license_manager.as_ref() else {
+        tracing::error!("Vision license manager not initialized");
+        return (
+            axum::http::StatusCode::INTERNAL_SERVER_ERROR,
+            Json(serde_json::json!({
+                "error": {
+                    "code": "VISION_LICENSE_MANAGER_MISSING",
+                    "message": "Vision subsystem not initialized",
+                }
+            })),
+        )
+            .into_response();
+    };
+
+    fn map_vision_error_status(message: &str) -> axum::http::StatusCode {
+        if message.contains("Either image_base64 or url must be provided") {
+            return axum::http::StatusCode::BAD_REQUEST;
+        }
+        if message.starts_with("Failed to decode base64 image") {
+            return axum::http::StatusCode::BAD_REQUEST;
+        }
+        if message.starts_with("Failed to preprocess image") {
+            return axum::http::StatusCode::UNPROCESSABLE_ENTITY;
+        }
+        if message.contains("Vision model '") && message.contains("not available in Ollama") {
+            return axum::http::StatusCode::UNPROCESSABLE_ENTITY;
+        }
+        if message.contains("Failed to fetch image from URL") {
+            if message.to_lowercase().contains("timed out") {
+                return axum::http::StatusCode::GATEWAY_TIMEOUT;
+            }
+            return axum::http::StatusCode::BAD_GATEWAY;
+        }
+        if message.contains("Vision inference timed out") {
+            return axum::http::StatusCode::GATEWAY_TIMEOUT;
+        }
+        if message.contains("Failed to load vision model")
+            || message.contains("Vision inference failed")
+        {
+            return axum::http::StatusCode::BAD_GATEWAY;
+        }
+
+        axum::http::StatusCode::INTERNAL_SERVER_ERROR
+    }
+
+    match crate::vision::process_vision_request(
+        req,
+        &model_name,
+        license_manager,
+        &state,
+    )
+    .await
+    {
+        Ok(response) => Json(response).into_response(),
+        Err(e) => {
+            // Check if it's a license error
+            if let Some(license_err) = e.downcast_ref::<crate::vision_license::VisionLicenseError>()
+            {
+                return (
+                    license_err.to_status_code(),
+                    Json(license_err.to_json_error()),
+                )
+                    .into_response();
+            }
+
+            let full_message = e.to_string();
+            let status = map_vision_error_status(&full_message);
+
+            tracing::error!(status = %status, "Vision processing error: {}", full_message);
+            // Expose client error messages (4xx) to help users fix their requests.
+            // Hide server error details (5xx) unless running in dev mode.
+            let message = if status.is_client_error() || std::env::var("SHIMMY_VISION_DEV_MODE").is_ok() {
+                full_message
+            } else {
+                "Vision processing error".to_string()
+            };
+
+            (
+                status,
+                Json(serde_json::json!({
+                    "error": {
+                        "code": "VISION_PROCESSING_ERROR",
+                        "message": message,
+                    }
+                })),
+            )
+                .into_response()
+        }
+    }
+}

src/engine/mod.rs

@@ -1,4 +1,4 @@
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
@@ -126,6 +126,17 @@ pub trait LoadedModel: Send + Sync {
         opts: GenOptions,
         on_token: Option<Box<dyn FnMut(String) + Send>>,
     ) -> Result<String>;
+
+    async fn generate_vision(
+        &self,
+        _image_data: &[u8],
+        _prompt: &str,
+        _opts: GenOptions,
+        _on_token: Option<Box<dyn FnMut(String) + Send>>,
+    ) -> Result<String> {
+        // Default implementation returns error - vision models should override
+        Err(anyhow!("Vision not supported by this model"))
+    }
 }
 
 pub mod llama;

src/lib.rs

@@ -22,6 +22,10 @@ pub mod safetensors_adapter;
 pub mod server;
 pub mod templates;
 pub mod tools;
+#[cfg(feature = "vision")]
+pub mod vision;
+#[cfg(feature = "vision")]
+pub mod vision_license;
 pub mod util {
     pub mod diag;
     pub mod memory;
@@ -42,6 +46,8 @@ pub struct AppState {
     pub registry: model_registry::Registry,
     pub observability: observability::ObservabilityManager,
     pub response_cache: cache::ResponseCache,
+    #[cfg(feature = "vision")]
+    pub vision_license_manager: Option<crate::vision_license::VisionLicenseManager>,
 }
 
 impl AppState {
@@ -54,6 +60,8 @@ impl AppState {
             registry,
             observability: observability::ObservabilityManager::new(),
             response_cache: cache::ResponseCache::new(),
+            #[cfg(feature = "vision")]
+            vision_license_manager: Some(crate::vision_license::VisionLicenseManager::new()),
         }
     }
 }

src/main.rs

@@ -16,6 +16,10 @@ mod openai_compat;
 mod port_manager;
 mod server;
 mod templates;
+#[cfg(feature = "vision")]
+mod vision;
+#[cfg(feature = "vision")]
+mod vision_license;
 mod util {
     pub mod diag;
     pub mod memory;
@@ -32,16 +36,28 @@ pub struct AppState {
     pub registry: Registry,
     pub observability: observability::ObservabilityManager,
     pub response_cache: cache::ResponseCache,
+    #[cfg(feature = "vision")]
+    pub vision_license_manager: Option<crate::vision_license::VisionLicenseManager>,
 }
 
 impl AppState {
     pub fn new(engine: Box<dyn engine::InferenceEngine>, registry: Registry) -> Self {
-        Self {
+        #[allow(unused_mut)]
+        let mut state = Self {
             engine,
             registry,
             observability: observability::ObservabilityManager::new(),
             response_cache: cache::ResponseCache::new(),
+            #[cfg(feature = "vision")]
+            vision_license_manager: None,
+        };
+
+        #[cfg(feature = "vision")]
+        {
+            state.vision_license_manager = Some(crate::vision_license::VisionLicenseManager::new());
         }
+
+        state
     }
 }
 

src/server.rs

@@ -130,7 +130,8 @@ async fn metrics_endpoint(State(state): State<Arc<AppState>>) -> Json<Value> {
 
 pub async fn run(addr: SocketAddr, state: Arc<AppState>) -> anyhow::Result<()> {
     let listener = tokio::net::TcpListener::bind(addr).await?;
-    let app = Router::new()
+    #[allow(unused_mut)]
+    let mut app = Router::new()
         .route("/health", get(health_check))
         .route("/metrics", get(metrics_endpoint))
         .route("/diag", get(diag_handler))
@@ -150,9 +151,14 @@ pub async fn run(addr: SocketAddr, state: Arc<AppState>) -> anyhow::Result<()> {
         )
         .route("/v1/models", get(openai_compat::models))
         // Anthropic Claude API compatibility
-        .route("/v1/messages", post(anthropic_compat::messages))
-        .layer(middleware::from_fn(cors_layer))
-        .with_state(state);
+        .route("/v1/messages", post(anthropic_compat::messages));
+
+    #[cfg(feature = "vision")]
+    {
+        app = app.route("/api/vision", post(api::vision));
+    }
+
+    let app = app.layer(middleware::from_fn(cors_layer)).with_state(state);
     axum::serve(listener, app).await?;
     Ok(())
 }