Merge branch 'main' into feat/sdk-qwen35-gemma4-reasoning-budget

Author: gianni-cor

packages/sdk/client/api/index.ts

@@ -29,6 +29,7 @@ export { getLoadedModelInfo } from "./get-loaded-model-info";
 export { ocr } from "./ocr";
 export { invokePlugin, invokePluginStream } from "./invoke-plugin";
 export { diffusion, type DiffusionProgressTick } from "./diffusion";
+export { upscale } from "./upscale";
 export {
   modelRegistryList,
   modelRegistrySearch,

packages/sdk/client/api/upscale.ts

@@ -0,0 +1,122 @@
+import {
+  upscaleStreamResponseSchema,
+  type UpscaleClientParams,
+  type UpscaleStats,
+  type UpscaleStreamRequest,
+} from "@/schemas/sdcpp-config";
+import { stream as streamRpc } from "@/client/rpc/rpc-client";
+import { decodeBase64, encodeBase64 } from "@/utils/encoding";
+import { StreamEndedError } from "@/utils/errors-client";
+
+interface UpscaleResult {
+  outputs: Promise<Uint8Array[]>;
+  stats: Promise<UpscaleStats | undefined>;
+}
+
+/**
+ * Runs standalone ESRGAN upscaling on an arbitrary PNG/JPEG image.
+ *
+ * The model must have been loaded with `modelType: "diffusion"` and
+ * `modelConfig.mode: "upscale"` — calling `upscale()` against a model
+ * loaded in default (`mode: "diffusion"`) mode throws
+ * `ModelOperationNotSupportedError` upfront.
+ *
+ * `outputs` always resolves to length 1: `repeats` runs N passes
+ * internally and emits a single final image at `source * scale^repeats`
+ * dimensions. The `Uint8Array[]` shape reserves headroom for future
+ * multi-output variants.
+ *
+ * @param params - `{ modelId, image, repeats? }`. `image` is raw PNG/JPEG
+ *   bytes; the client base64-encodes them on the wire.
+ * @returns `{ outputs, stats }` — `outputs` resolves to a single-element
+ *   array containing the final upscaled PNG; `stats` resolves to
+ *   addon-side stats (load/upscale ms, final width/height, repeats
+ *   actually executed, etc.).
+ * @throws {ModelOperationNotSupportedError} If the model was not loaded
+ *   with `mode: "upscale"`.
+ * @throws {StreamEndedError} If the RPC stream closes without emitting a
+ *   terminal `done` chunk.
+ *
+ * @example
+ * ```ts
+ * const modelId = await loadModel(REALESRGAN_X4PLUS_ANIME_6B, {
+ *   modelType: "diffusion",
+ *   modelConfig: { mode: "upscale", upscaler: { tile_size: 128 } },
+ * });
+ * const pngBytes = fs.readFileSync("input.png");
+ * const { outputs, stats } = upscale({ modelId, image: pngBytes, repeats: 2 });
+ * const [upscaledPng] = await outputs;
+ * fs.writeFileSync("upscaled.png", upscaledPng);
+ * console.log(await stats);
+ * ```
+ */
+export function upscale(params: UpscaleClientParams): UpscaleResult {
+  const request: UpscaleStreamRequest = {
+    modelId: params.modelId,
+    image: encodeBase64(params.image),
+    ...(params.repeats !== undefined && { repeats: params.repeats }),
+    type: "upscaleStream",
+  };
+
+  let statsResolver: (value: UpscaleStats | undefined) => void = () => {};
+  let statsRejecter: (error: unknown) => void = () => {};
+  const statsPromise = new Promise<UpscaleStats | undefined>(
+    (resolve, reject) => {
+      statsResolver = resolve;
+      statsRejecter = reject;
+    },
+  );
+  statsPromise.catch(() => {});
+
+  let outputsResolver: (value: Uint8Array[]) => void = () => {};
+  let outputsRejecter: (error: unknown) => void = () => {};
+  const outputsPromise = new Promise<Uint8Array[]>((resolve, reject) => {
+    outputsResolver = resolve;
+    outputsRejecter = reject;
+  });
+  outputsPromise.catch(() => {});
+
+  const collectedBuffers: Uint8Array[] = [];
+
+  async function processResponses() {
+    let sawDone = false;
+    try {
+      for await (const response of streamRpc(request)) {
+        if (
+          response &&
+          typeof response === "object" &&
+          "type" in response &&
+          response.type === "upscaleStream"
+        ) {
+          const parsed = upscaleStreamResponseSchema.parse(response);
+
+          if (parsed.data) {
+            collectedBuffers.push(decodeBase64(parsed.data));
+          }
+
+          if (parsed.done) {
+            sawDone = true;
+            statsResolver(parsed.stats);
+            outputsResolver(collectedBuffers);
+          }
+        }
+      }
+
+      if (!sawDone) {
+        const error = new StreamEndedError();
+        statsRejecter(error);
+        outputsRejecter(error);
+      }
+    } catch (error) {
+      statsRejecter(error);
+      outputsRejecter(error);
+    }
+  }
+
+  void processResponses();
+
+  return {
+    outputs: outputsPromise,
+    stats: statsPromise,
+  };
+}

packages/sdk/index.ts

@@ -35,6 +35,7 @@ export {
   invokePluginStream,
   diffusion,
   type DiffusionProgressTick,
+  upscale,
   modelRegistryList,
   modelRegistrySearch,
   modelRegistryGetModel,
@@ -106,6 +107,9 @@ export {
   type DiffusionClientParams,
   type DiffusionStreamResponse,
   type DiffusionStats,
+  type UpscaleClientParams,
+  type UpscaleStreamResponse,
+  type UpscaleStats,
   definePlugin,
   defineHandler,
   defineDuplexHandler,

packages/sdk/package.json

@@ -173,7 +173,7 @@
   },
   "dependencies": {
     "@qvac/decoder-audio": "^0.3.7",
-    "@qvac/diffusion-cpp": "^0.6.0",
+    "@qvac/diffusion-cpp": "^0.7.0",
     "@qvac/embed-llamacpp": "^0.15.0",
     "@qvac/error": "^0.1.1",
     "@qvac/langdetect-text": "^0.1.2",

packages/sdk/schemas/common.ts

@@ -64,6 +64,8 @@ import { ocrStreamRequestSchema, ocrStreamResponseSchema } from "./ocr";
 import {
   diffusionStreamRequestSchema,
   diffusionStreamResponseSchema,
+  upscaleStreamRequestSchema,
+  upscaleStreamResponseSchema,
 } from "./sdcpp-config";
 import {
   finetuneRequestSchema,
@@ -110,6 +112,7 @@ export const requestSchema = z.union([
   getLoadedModelInfoRequestSchema,
   ocrStreamRequestSchema,
   diffusionStreamRequestSchema,
+  upscaleStreamRequestSchema,
   finetuneRequestSchema,
   pluginInvokeRequestSchema,
   pluginInvokeStreamRequestSchema,
@@ -146,6 +149,7 @@ export const responseSchema = z.discriminatedUnion("type", [
   getLoadedModelInfoResponseSchema,
   ocrStreamResponseSchema,
   diffusionStreamResponseSchema,
+  upscaleStreamResponseSchema,
   finetuneResponseSchema,
   finetuneProgressResponseSchema,
   pluginInvokeResponseSchema,

packages/sdk/schemas/plugin.ts

@@ -340,7 +340,7 @@ export const PLUGIN_OCR = "@qvac/sdk/onnx-ocr/plugin" as const;
 
 /**
  * Image generation plugin (stable-diffusion.cpp).
- * Provides: text-to-image generation.
+ * Provides: text-to-image generation and standalone ESRGAN image upscaling.
  */
 export const PLUGIN_DIFFUSION =
   "@qvac/sdk/sdcpp-generation/plugin" as const;

packages/sdk/schemas/sdcpp-config.ts

@@ -8,6 +8,15 @@ const ABSOLUTE_PATH_PATTERN = /^(\/|[A-Za-z]:[\\/]|\\\\)/;
 
 export const sdcppConfigSchema = z
   .object({
+    mode: z.enum(["diffusion", "upscale"]).default("diffusion")
+      .describe(
+        "Operation mode for the diffusion plugin. " +
+        "`'diffusion'` (default) builds a full SD / SDXL / SD3 / FLUX pipeline from " +
+        "the primary model plus optional auxiliary text encoders, VAE, and ESRGAN " +
+        "upscaler, and exposes diffusion({ ... }). " +
+        "`'upscale'` builds a standalone ESRGAN upscaler from the primary model " +
+        "file alone (auxiliary model sources are ignored) and exposes upscale({ ... }).",
+      ),
     threads: z.number().optional(),
     device: z.enum(["gpu", "cpu"]).optional(),
     prediction: z
@@ -52,17 +61,18 @@ export const sdcppConfigSchema = z
     upscaler: z.object({
       type: z.literal("esrgan").optional()
         .describe("Type of upscaler to use for post-generation upscaling when requested in diffusion({ upscale })."),
-      model_src: modelSrcInputSchema
+      model_src: modelSrcInputSchema.optional()
         .describe(
-          "ESRGAN upscaler model (e.g. RealESRGAN_x4plus_anime_6B.pth). When " +
-          "provided, generation requests can opt into post-generation upscale " +
-          "via diffusion({ upscale: true }) or diffusion({ upscale: { repeats } }).",
+          "ESRGAN upscaler model (e.g. RealESRGAN_x4plus_anime_6B.pth). " +
+          "Required in diffusion mode when this `upscaler` block is set — " +
+          "configures the post-generation upscaler invoked via diffusion({ upscale }). " +
+          "In `mode: 'upscale'` the primary modelSrc itself is the ESRGAN model, " +
+          "so this field is ignored.",
         ),
       tile_size: z.number().int().positive().optional()
         .describe(
           "ESRGAN upscaler tile size in pixels. Smaller tiles use less VRAM " +
-          "at the cost of more passes. Only used when upscaler.model_src is " +
-          "configured and diffusion({ upscale }) is requested.",
+          "at the cost of more passes.",
         ),
       direct: z.boolean().optional()
         .describe(
@@ -83,10 +93,19 @@ export const sdcppConfigSchema = z
           "Number of CPU threads dedicated to the ESRGAN upscaler. -1 = auto.",
         ),
     }).strict().optional()
-      .describe("Configuration for an optional upscaler that can be applied after diffusion generation when requested in diffusion({ upscale })."),
+      .describe(
+        "ESRGAN upscaler configuration. In diffusion mode this enables the " +
+        "post-generation upscale path invoked via diffusion({ upscale }) and " +
+        "requires `model_src`. In `mode: 'upscale'` only the tuning fields " +
+        "(tile_size, direct, offload_params_to_cpu, threads) are honored — " +
+        "the primary modelSrc IS the ESRGAN model in that mode and " +
+        "`model_src` here is ignored. Mode-dependent constraints (e.g. " +
+        "`model_src` required in diffusion mode) are enforced by the " +
+        "sdcpp-generation plugin at load time, not at the schema layer.",
+      ),
   });
 
-export type SdcppConfig = z.infer<typeof sdcppConfigSchema>;
+export type SdcppConfig = z.input<typeof sdcppConfigSchema>;
 
 export const diffusionStatsSchema = z.object({
   modelLoadMs: z
@@ -350,3 +369,94 @@ export type DiffusionClientParams = DiffusionClientParamsBase &
     | { init_image?: Uint8Array; init_images?: never }
     | { init_image?: never; init_images?: Uint8Array[] }
   );
+
+// ============================================
+// Standalone ESRGAN upscale (mode: "upscale")
+// ============================================
+
+export const upscaleStatsSchema = z.object({
+  modelLoadMs: z
+    .number()
+    .optional()
+    .describe("Wall-clock time in milliseconds spent loading the upscaler model."),
+  upscaleMs: z
+    .number()
+    .optional()
+    .describe("Wall-clock time in milliseconds for the most recent upscale job."),
+  totalUpscaleMs: z
+    .number()
+    .optional()
+    .describe("Cumulative upscale time in milliseconds across all jobs."),
+  totalWallMs: z
+    .number()
+    .optional()
+    .describe(
+      "Total wall-clock time in milliseconds including model load and upscaling.",
+    ),
+  totalUpscales: z
+    .number()
+    .optional()
+    .describe("Cumulative number of upscale calls."),
+  totalImages: z
+    .number()
+    .optional()
+    .describe("Cumulative number of images produced."),
+  totalPixels: z
+    .number()
+    .optional()
+    .describe("Cumulative number of pixels produced across all images."),
+  width: z.number().optional().describe("Width of the most recent emitted PNG."),
+  height: z.number().optional().describe("Height of the most recent emitted PNG."),
+  repeats: z
+    .number()
+    .optional()
+    .describe("Number of ESRGAN passes used by the most recent upscale job."),
+});
+
+export type UpscaleStats = z.infer<typeof upscaleStatsSchema>;
+
+export const upscaleRequestSchema = z.object({
+  modelId: z
+    .string()
+    .describe(
+      "Identifier of the loaded upscaler model. The model must have been loaded " +
+      "with `modelType: 'diffusion'` and `modelConfig.mode: 'upscale'`.",
+    ),
+  image: z
+    .string()
+    .min(1)
+    .regex(BASE64_PATTERN)
+    .describe("Base64-encoded PNG/JPEG bytes of the source image."),
+  repeats: z
+    .number()
+    .int()
+    .positive()
+    .optional()
+    .describe(
+      "Number of ESRGAN passes to run sequentially. Each pass multiplies " +
+      "dimensions by the model's native scale factor; only the final image " +
+      "is emitted (`outputs.length === 1`). Defaults to 1.",
+    ),
+});
+
+export type UpscaleRequest = z.input<typeof upscaleRequestSchema>;
+
+export const upscaleStreamRequestSchema = upscaleRequestSchema.extend({
+  type: z.literal("upscaleStream"),
+});
+
+export type UpscaleStreamRequest = z.input<typeof upscaleStreamRequestSchema>;
+
+export const upscaleStreamResponseSchema = z.object({
+  type: z.literal("upscaleStream"),
+  data: z.string().optional(),
+  outputIndex: z.number().optional(),
+  done: z.boolean().optional(),
+  stats: upscaleStatsSchema.optional(),
+});
+
+export type UpscaleStreamResponse = z.infer<typeof upscaleStreamResponseSchema>;
+
+export type UpscaleClientParams = Omit<UpscaleRequest, "image"> & {
+  image: Uint8Array;
+};