diff --git a/sdk/cs/src/Detail/CoreInterop.cs b/sdk/cs/src/Detail/CoreInterop.cs index a7a43447..b88f5597 100644 --- a/sdk/cs/src/Detail/CoreInterop.cs +++ b/sdk/cs/src/Detail/CoreInterop.cs @@ -203,7 +203,7 @@ public CallbackHelper(CallbackFn callback) } } - private static void HandleCallback(nint data, int length, nint callbackHelper) + private static int HandleCallback(nint data, int length, nint callbackHelper) { var callbackData = string.Empty; CallbackHelper? helper = null; @@ -221,14 +221,24 @@ private static void HandleCallback(nint data, int length, nint callbackHelper) helper = (CallbackHelper)GCHandle.FromIntPtr(callbackHelper).Target!; helper.Callback.Invoke(callbackData); + return 0; // continue } - catch (Exception ex) when (ex is not OperationCanceledException) + catch (OperationCanceledException ex) + { + if (helper != null && helper.Exception == null) + { + helper.Exception = ex; + } + return 1; // cancel + } + catch (Exception ex) { FoundryLocalManager.Instance.Logger.LogError(ex, $"Error in callback. Callback data: {callbackData}"); if (helper != null && helper.Exception == null) { helper.Exception = ex; } + return 1; // cancel on error } } diff --git a/sdk/cs/src/Detail/ICoreInterop.cs b/sdk/cs/src/Detail/ICoreInterop.cs index b493dfb7..74e2a8ad 100644 --- a/sdk/cs/src/Detail/ICoreInterop.cs +++ b/sdk/cs/src/Detail/ICoreInterop.cs @@ -40,8 +40,9 @@ protected unsafe struct ResponseBuffer } // native callback function signature + // Return: 0 = continue, 1 = cancel [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - protected unsafe delegate void NativeCallbackFn(nint data, int length, nint userData); + protected unsafe delegate int NativeCallbackFn(nint data, int length, nint userData); Response ExecuteCommand(string commandName, CoreInteropRequest? commandInput = null); Response ExecuteCommandWithCallback(string commandName, CoreInteropRequest? commandInput, CallbackFn callback); diff --git a/sdk/js/src/detail/coreInterop.ts b/sdk/js/src/detail/coreInterop.ts index 9b723e84..5af32421 100644 --- a/sdk/js/src/detail/coreInterop.ts +++ b/sdk/js/src/detail/coreInterop.ts @@ -29,7 +29,7 @@ koffi.struct('StreamingRequestBuffer', { BinaryDataLength: 'int32_t', }); -const CallbackType = koffi.proto('void CallbackType(void *data, int32_t length, void *userData)'); +const CallbackType = koffi.proto('int32_t CallbackType(void *data, int32_t length, void *userData)'); const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -198,8 +198,13 @@ export class CoreInterop { koffi.encode(dataBuf, 'char', dataStr, dataBytes.length + 1); const cb = koffi.register((data: any, length: number, userData: any) => { - const chunk = koffi.decode(data, 'char', length); - callback(chunk); + try { + const chunk = koffi.decode(data, 'char', length); + callback(chunk); + return 0; // continue + } catch { + return 1; // cancel on error + } }, koffi.pointer(CallbackType)); return new Promise((resolve, reject) => { diff --git a/sdk/python/requirements-winml.txt b/sdk/python/requirements-winml.txt index bcf02668..68b76b56 100644 --- a/sdk/python/requirements-winml.txt +++ b/sdk/python/requirements-winml.txt @@ -2,6 +2,6 @@ pydantic>=2.0.0 requests>=2.32.4 openai>=2.24.0 # WinML native binary packages from the ORT-Nightly PyPI feed. -foundry-local-core-winml==0.9.0.dev20260331004032 +foundry-local-core-winml==1.0.0rc1 onnxruntime-core==1.23.2.3 onnxruntime-genai-core==0.13.0 \ No newline at end of file diff --git a/sdk/python/requirements.txt b/sdk/python/requirements.txt index 26da243f..9295b832 100644 --- a/sdk/python/requirements.txt +++ b/sdk/python/requirements.txt @@ -2,6 +2,8 @@ pydantic>=2.0.0 requests>=2.32.4 openai>=2.24.0 # Standard native binary packages from the ORT-Nightly PyPI feed. -foundry-local-core==0.9.0.dev20260327060216 -onnxruntime-core==1.24.4 -onnxruntime-genai-core==0.13.0 \ No newline at end of file +foundry-local-core==1.0.0rc1 +onnxruntime-core==1.24.4; sys_platform != "linux" +onnxruntime-gpu==1.24.4; sys_platform == "linux" +onnxruntime-genai-core==0.13.0; sys_platform != "linux" +onnxruntime-genai-cuda==0.13.0; sys_platform == "linux" diff --git a/sdk/python/src/detail/core_interop.py b/sdk/python/src/detail/core_interop.py index 4f4ddb67..1cd53e33 100644 --- a/sdk/python/src/detail/core_interop.py +++ b/sdk/python/src/detail/core_interop.py @@ -79,9 +79,11 @@ def callback(data_ptr, length, self_ptr): data_bytes = ctypes.string_at(data_ptr, length) data_str = data_bytes.decode('utf-8') self._py_callback(data_str) + return 0 # continue except Exception as e: if self is not None and self.exception is None: self.exception = e # keep the first only as they are likely all the same + return 1 # cancel on error def __init__(self, py_callback: Callable[[str], None]): self._py_callback = py_callback @@ -103,8 +105,8 @@ class CoreInterop: instance = None # Callback function for native interop. - # This returns a string and its length, and an optional user provided object. - CALLBACK_TYPE = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p) + # Returns c_int: 0 = continue, 1 = cancel. + CALLBACK_TYPE = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p) @staticmethod def _initialize_native_libraries() -> 'NativeBinaryPaths': @@ -129,8 +131,9 @@ def _initialize_native_libraries() -> 'NativeBinaryPaths': logger.info("Native libraries found — Core: %s ORT: %s GenAI: %s", paths.core, paths.ort, paths.genai) - # Create the onnxruntime.dll symlink on Linux/macOS if needed. - # create_ort_symlinks(paths) + # Create compatibility symlinks on Linux/macOS so Core can resolve + # ORT/GenAI names regardless of package layout. + create_ort_symlinks(paths) os.environ["ORT_LIB_PATH"] = str(paths.ort) # For ORT-GENAI to find ORT dependency if sys.platform.startswith("win"): diff --git a/sdk/python/src/detail/model_data_types.py b/sdk/python/src/detail/model_data_types.py index 46525dc7..e000c9c8 100644 --- a/sdk/python/src/detail/model_data_types.py +++ b/sdk/python/src/detail/model_data_types.py @@ -4,7 +4,7 @@ # -------------------------------------------------------------------------- from typing import Optional, List -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from enum import StrEnum @@ -53,6 +53,8 @@ class ModelInfo(BaseModel): Fields are populated from the JSON response of the ``get_model_list`` command. """ + model_config = ConfigDict(protected_namespaces=()) + id: str = Field(alias="id", description="Unique identifier of the model. Generally :") name: str = Field(alias="name", description="Model variant name") version: int = Field(alias="version") diff --git a/sdk/python/src/detail/utils.py b/sdk/python/src/detail/utils.py index 5a054610..5780cfc9 100644 --- a/sdk/python/src/detail/utils.py +++ b/sdk/python/src/detail/utils.py @@ -12,7 +12,6 @@ import argparse import importlib.util -import json import logging import os import sys @@ -90,9 +89,9 @@ def _find_file_in_package(package_name: str, filename: str) -> Path | None: # Quick checks for well-known sub-directories first for candidate_dir in (pkg_root, pkg_root / "capi", pkg_root / "native", pkg_root / "lib", pkg_root / "bin"): - candidate = candidate_dir / filename - if candidate.exists(): - return candidate + candidates = list(candidate_dir.glob(f"*{filename}*")) + if candidates: + return candidates[0] # Recursive fallback for match in pkg_root.rglob(filename): @@ -144,8 +143,18 @@ def get_native_binary_paths() -> NativeBinaryPaths | None: # Probe WinML packages first; fall back to standard if not installed. core_path = _find_file_in_package("foundry-local-core-winml", core_name) or _find_file_in_package("foundry-local-core", core_name) - ort_path = _find_file_in_package("onnxruntime-core", ort_name) - genai_path = _find_file_in_package("onnxruntime-genai-core", genai_name) + + # On Linux, ORT is shipped by onnxruntime-gpu (libonnxruntime.so in capi/). + if sys.platform.startswith("linux"): + ort_path = _find_file_in_package("onnxruntime", ort_name) or _find_file_in_package("onnxruntime-core", ort_name) + else: + ort_path = _find_file_in_package("onnxruntime-core", ort_name) + + # On Linux, ORTGenAI is shipped by onnxruntime-genai-cuda (libonnxruntime-genai.so in the package root). + if sys.platform.startswith("linux"): + genai_path = _find_file_in_package("onnxruntime-genai", genai_name) or _find_file_in_package("onnxruntime-genai-core", genai_name) + else: + genai_path = _find_file_in_package("onnxruntime-genai-core", genai_name) if core_path and ort_path and genai_path: return NativeBinaryPaths(core=core_path, ort=ort_path, genai=genai_path) @@ -254,6 +263,9 @@ def foundry_local_install(args: list[str] | None = None) -> None: if parsed.winml: variant = "WinML" packages = ["foundry-local-core-winml", "onnxruntime-core", "onnxruntime-genai-core"] + elif sys.platform.startswith("linux"): + variant = "Linux (GPU)" + packages = ["foundry-local-core", "onnxruntime-gpu", "onnxruntime-genai-cuda"] else: variant = "standard" packages = ["foundry-local-core", "onnxruntime-core", "onnxruntime-genai-core"] @@ -271,10 +283,18 @@ def foundry_local_install(args: list[str] | None = None) -> None: else: if _find_file_in_package("foundry-local-core", core_name) is None: missing.append("foundry-local-core") - if _find_file_in_package("onnxruntime-core", ort_name) is None: + if sys.platform.startswith("linux"): + if _find_file_in_package("onnxruntime", ort_name) is None: + missing.append("onnxruntime-gpu") + else: + if _find_file_in_package("onnxruntime-core", ort_name) is None: missing.append("onnxruntime-core") - if _find_file_in_package("onnxruntime-genai-core", genai_name) is None: - missing.append("onnxruntime-genai-core") + if sys.platform.startswith("linux"): + if _find_file_in_package("onnxruntime-genai", genai_name) is None: + missing.append("onnxruntime-genai-cuda") + else: + if _find_file_in_package("onnxruntime-genai-core", genai_name) is None: + missing.append("onnxruntime-genai-core") print( "[foundry-local] ERROR: Could not locate native binaries after installation. " f"Missing: {', '.join(missing)}", @@ -289,6 +309,3 @@ def foundry_local_install(args: list[str] | None = None) -> None: print(f" Core : {paths.core}") print(f" ORT : {paths.ort}") print(f" GenAI : {paths.genai}") - - - diff --git a/sdk/rust/src/detail/core_interop.rs b/sdk/rust/src/detail/core_interop.rs index 75146164..43884d7f 100644 --- a/sdk/rust/src/detail/core_interop.rs +++ b/sdk/rust/src/detail/core_interop.rs @@ -52,7 +52,8 @@ impl ResponseBuffer { type ExecuteCommandFn = unsafe extern "C" fn(*const RequestBuffer, *mut ResponseBuffer); /// Signature for the streaming callback invoked by the native library. -type CallbackFn = unsafe extern "C" fn(*const u8, i32, *mut std::ffi::c_void); +/// Returns 0 to continue, 1 to cancel. +type CallbackFn = unsafe extern "C" fn(*const u8, i32, *mut std::ffi::c_void) -> i32; /// Signature for `execute_command_with_callback`. type ExecuteCommandWithCallbackFn = unsafe extern "C" fn( @@ -197,12 +198,12 @@ unsafe extern "C" fn streaming_trampoline( data: *const u8, length: i32, user_data: *mut std::ffi::c_void, -) { +) -> i32 { if data.is_null() || length <= 0 { - return; + return 0; } // catch_unwind prevents UB if the closure panics across the FFI boundary. - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { // SAFETY: `user_data` points to a `StreamingCallbackState` kept alive // by the caller of `execute_command_with_callback` for the duration of // the native call. @@ -212,6 +213,11 @@ unsafe extern "C" fn streaming_trampoline( let slice = std::slice::from_raw_parts(data, length as usize); state.push(slice); })); + if result.is_err() { + 1 + } else { + 0 + } } // ── CoreInterop ──────────────────────────────────────────────────────────────