Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions server/convexProxy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ describe("Convex HTTP proxy", () => {
expect(metricCall?.[1]).toMatchObject({
method: "POST",
body: "signed-metric-capability",
signal: expect.any(AbortSignal),
});
const fetchedUrls = fetchMock.mock.calls.map(([input]) => input.toString());
expect(
Expand Down Expand Up @@ -644,4 +645,76 @@ describe("Convex HTTP proxy", () => {
expect(response.status).toBe(405);
expect(fetchMock).not.toHaveBeenCalled();
});

it("does not wait for download metrics before emitting zip bytes", async () => {
const storedBody = new TextEncoder().encode("# streamed skill\n");
const fetchMock = vi.fn(async (input: string | URL | Request) => {
const url = input.toString();
if (url.startsWith("https://preview-branch-123.convex.site/api/v1/download")) {
return Response.json(
{
schema: "clawhub.skill-archive-manifest.v1",
issuedAt: 1_000,
expiresAt: 31_000,
filename: "demo-1.0.0.zip",
meta: {
ownerId: "users:1",
slug: "demo",
version: "1.0.0",
publishedAt: 3,
},
entries: [
{
path: "SKILL.md",
url: "https://preview-branch-123.convex.cloud/api/storage/storage-1",
},
],
metricToken: "signed-metric-capability",
},
{ headers: { "content-type": ARCHIVE_MANIFEST_CONTENT_TYPE } },
);
}
if (url === "https://preview-branch-123.convex.cloud/api/storage/storage-1") {
return new Response(storedBody, { status: 200 });
}
if (url === "https://preview-branch-123.convex.site/api/internal/archive-download-metric") {
return new Promise<Response>(() => {});
}
throw new Error(`Unexpected fetch: ${url}`);
});
vi.stubGlobal("fetch", fetchMock);
vi.spyOn(Date, "now").mockReturnValue(2_000);

const response = await proxyConvexRequest(
mockEvent("https://preview.example/api/v1/download?slug=demo"),
{
VERCEL_ENV: "preview",
VITE_CONVEX_SITE_URL: "https://preview-branch-123.convex.site",
VITE_CONVEX_URL: "https://preview-branch-123.convex.cloud",
},
TEST_ARCHIVE_DEPENDENCIES,
);

expect(response.status).toBe(200);
expect(response.body).not.toBeNull();
const reader = response.body!.getReader();
const firstChunk = await Promise.race([
reader.read(),
new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error("zip first byte blocked by download metric")), 500);
}),
]);

expect(firstChunk.done).toBe(false);
expect(firstChunk.value?.byteLength).toBeGreaterThan(0);
expect(
fetchMock.mock.calls.some(
([input]) =>
input.toString() ===
"https://preview-branch-123.convex.site/api/internal/archive-download-metric",
),
).toBe(true);

await reader.cancel();
});
});
26 changes: 16 additions & 10 deletions server/convexProxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ const MAX_ARCHIVE_MANIFEST_ENTRIES = 8_192;
const MAX_ARCHIVE_ENTRY_URL_LENGTH = 4_096;
const MAX_ARCHIVE_MANIFEST_BYTES = 4 * 1024 * 1024;
const ARCHIVE_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._+-]{0,499}\.zip$/;
const ARCHIVE_METRIC_FETCH_TIMEOUT_MS = 1_500;
const ARCHIVE_REPRESENTATION_HEADERS = [
"accept-ranges",
"content-digest",
Expand Down Expand Up @@ -217,18 +218,23 @@ async function streamSkillArchive(
}

let metricRecorded = false;
const recordMetric = async () => {
const recordMetric = () => {
if (metricRecorded || !manifest.metricToken) return;
metricRecorded = true;
try {
await fetch(new URL("/api/internal/archive-download-metric", target), {
method: "POST",
headers: { "content-type": "application/jose" },
body: manifest.metricToken,
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), ARCHIVE_METRIC_FETCH_TIMEOUT_MS);
void fetch(new URL("/api/internal/archive-download-metric", target), {
method: "POST",
headers: { "content-type": "application/jose" },
body: manifest.metricToken,
signal: controller.signal,
})
.catch(() => {
// Download metrics remain best-effort and never interrupt archive bytes.
})
.finally(() => {
clearTimeout(timeout);
});
} catch {
// Download metrics remain best-effort and never interrupt archive bytes.
}
};

const stream = buildDeterministicZipStream(
Expand All @@ -240,7 +246,7 @@ async function streamSkillArchive(
if (!response.ok || !response.body) {
throw new Error(`Failed to fetch archive entry: ${response.status}`);
}
await recordMetric();
recordMetric();
return response.body;
},
})),
Expand Down
47 changes: 46 additions & 1 deletion server/og/fetchPluginOgMeta.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,29 @@

import { afterEach, describe, expect, it, vi } from "vitest";
import { fetchPluginOgMeta } from "./fetchPluginOgMeta";
import { OG_FETCH_TIMEOUT_MS } from "./ogFetchTimeout";

function hangUntilAborted(_input: unknown, init?: RequestInit) {
return new Promise<Response>((_resolve, reject) => {
const signal = init?.signal;
if (!signal) return;
signal.addEventListener(
"abort",
() => {
reject(
signal.reason instanceof Error
? signal.reason
: new DOMException("The operation was aborted.", "AbortError"),
);
},
{ once: true },
);
});
}

describe("fetchPluginOgMeta", () => {
afterEach(() => {
vi.useRealTimers();
vi.unstubAllGlobals();
});

Expand All @@ -29,8 +49,33 @@ describe("fetchPluginOgMeta", () => {

expect(fetchMock).toHaveBeenCalledWith(
"https://clawhub.ai/api/v1/packages/%40openclaw%2Fcodex",
{ headers: { Accept: "application/json" } },
{
headers: { Accept: "application/json" },
signal: expect.any(AbortSignal),
},
);
expect(meta?.stats.downloads).toBe(99);
});

it("aborts a hanging public package API fetch after the OG timeout", async () => {
vi.useFakeTimers();
let usedSignal: AbortSignal | undefined;
const fetchMock = vi.fn((input: unknown, init?: RequestInit) => {
usedSignal = init?.signal ?? undefined;
return hangUntilAborted(input, init);
});
vi.stubGlobal("fetch", fetchMock);

const pending = fetchPluginOgMeta("@openclaw/codex", "https://clawhub.ai");
await Promise.resolve();
expect(usedSignal).toBeInstanceOf(AbortSignal);
expect(usedSignal?.aborted).toBe(false);

await vi.advanceTimersByTimeAsync(OG_FETCH_TIMEOUT_MS - 1);
expect(usedSignal?.aborted).toBe(false);

await vi.advanceTimersByTimeAsync(1);
await expect(pending).resolves.toBeNull();
expect(usedSignal?.aborted).toBe(true);
});
});
6 changes: 5 additions & 1 deletion server/og/fetchPluginOgMeta.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { withOgFetchTimeout } from "./ogFetchTimeout";

export type PluginOgMeta = {
name: string | null;
displayName: string | null;
Expand All @@ -19,7 +21,9 @@ export async function fetchPluginOgMeta(
): Promise<PluginOgMeta | null> {
try {
const url = new URL(`/api/v1/packages/${encodeURIComponent(packageName)}`, apiBase);
const response = await fetch(url.toString(), { headers: { Accept: "application/json" } });
const response = await withOgFetchTimeout((signal) =>
fetch(url.toString(), { headers: { Accept: "application/json" }, signal }),
);
if (!response.ok) return null;
const payload = (await response.json()) as {
package?: {
Expand Down
21 changes: 21 additions & 0 deletions server/og/fetchPublisherOgMeta.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ describe("fetchPublisherOgMeta", () => {
});

afterEach(() => {
vi.useRealTimers();
vi.resetModules();
});

Expand Down Expand Up @@ -62,4 +63,24 @@ describe("fetchPublisherOgMeta", () => {
{ handle: "github", displayName: "GitHub", image: "https://example.com/github.png" },
]);
});

it("returns null when the Convex publisher query hangs past the OG timeout", async () => {
vi.useFakeTimers();
queryMock.mockImplementation(() => new Promise(() => {}));

const { fetchPublisherOgMeta } = await import("./fetchPublisherOgMeta");
const { OG_FETCH_TIMEOUT_MS } = await import("./ogFetchTimeout");
const pending = fetchPublisherOgMeta("openclaw", "https://example.convex.cloud");

await vi.advanceTimersByTimeAsync(OG_FETCH_TIMEOUT_MS - 1);
let settled = false;
void pending.then(() => {
settled = true;
});
await Promise.resolve();
expect(settled).toBe(false);

await vi.advanceTimersByTimeAsync(1);
await expect(pending).resolves.toBeNull();
});
});
11 changes: 8 additions & 3 deletions server/og/fetchPublisherOgMeta.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { ConvexHttpClient } from "convex/browser";
import type { FunctionReturnType } from "convex/server";
import { api } from "../../convex/_generated/api";
import { withOgFetchTimeout } from "./ogFetchTimeout";

export type PublisherOgMeta = {
handle: string | null;
Expand All @@ -27,9 +28,13 @@ export async function fetchPublisherOgMeta(
convexUrl: string,
): Promise<PublisherOgMeta | null> {
try {
const client = new ConvexHttpClient(convexUrl);
const profile = await client.query(api.publishers.getOgMetaByHandle, {
handle,
const profile = await withOgFetchTimeout((signal) => {
const client = new ConvexHttpClient(convexUrl, {
fetch: (input, init) => fetch(input, { ...init, signal }),
});
return client.query(api.publishers.getOgMetaByHandle, {
handle,
});
});
if (!profile) return null;
return {
Expand Down
43 changes: 43 additions & 0 deletions server/og/fetchSkillOgMeta.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,29 @@

import { afterEach, describe, expect, it, vi } from "vitest";
import { fetchSkillOgMeta } from "./fetchSkillOgMeta";
import { OG_FETCH_TIMEOUT_MS } from "./ogFetchTimeout";

function hangUntilAborted(_input: unknown, init?: RequestInit) {
return new Promise<Response>((_resolve, reject) => {
const signal = init?.signal;
if (!signal) return;
signal.addEventListener(
"abort",
() => {
reject(
signal.reason instanceof Error
? signal.reason
: new DOMException("The operation was aborted.", "AbortError"),
);
},
{ once: true },
);
});
}

describe("fetchSkillOgMeta", () => {
afterEach(() => {
vi.useRealTimers();
vi.unstubAllGlobals();
});

Expand Down Expand Up @@ -32,9 +52,32 @@ describe("fetchSkillOgMeta", () => {
"https://clawhub.ai/api/v1/skills/gifgrep?ownerHandle=steipete",
{
headers: { Accept: "application/json" },
signal: expect.any(AbortSignal),
},
);
expect(meta?.stats.downloads).toBe(1200);
expect(meta?.icon).toBe(`https://clawhub.ai/api/v1/skill-icons/${"a".repeat(64)}`);
});

it("aborts a hanging public skill API fetch after the OG timeout", async () => {
vi.useFakeTimers();
let usedSignal: AbortSignal | undefined;
const fetchMock = vi.fn((input: unknown, init?: RequestInit) => {
usedSignal = init?.signal ?? undefined;
return hangUntilAborted(input, init);
});
vi.stubGlobal("fetch", fetchMock);

const pending = fetchSkillOgMeta("gifgrep", "https://clawhub.ai");
await Promise.resolve();
expect(usedSignal).toBeInstanceOf(AbortSignal);
expect(usedSignal?.aborted).toBe(false);

await vi.advanceTimersByTimeAsync(OG_FETCH_TIMEOUT_MS - 1);
expect(usedSignal?.aborted).toBe(false);

await vi.advanceTimersByTimeAsync(1);
await expect(pending).resolves.toBeNull();
expect(usedSignal?.aborted).toBe(true);
});
});
5 changes: 4 additions & 1 deletion server/og/fetchSkillOgMeta.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { readCanonicalStat, type SkillStatReadable } from "../../convex/lib/skillStats";
import { withOgFetchTimeout } from "./ogFetchTimeout";

type SkillApiPayload = SkillStatReadable & {
displayName?: string;
Expand Down Expand Up @@ -43,7 +44,9 @@ export async function fetchSkillOgMeta(
const url = new URL(`/api/v1/skills/${encodeURIComponent(slug)}`, apiBase);
const owner = ownerHandle?.trim().replace(/^@+/, "");
if (owner) url.searchParams.set("ownerHandle", owner);
const response = await fetch(url.toString(), { headers: { Accept: "application/json" } });
const response = await withOgFetchTimeout((signal) =>
fetch(url.toString(), { headers: { Accept: "application/json" }, signal }),
);
if (!response.ok) return null;
const payload = (await response.json()) as {
skill?: SkillApiPayload | null;
Expand Down
31 changes: 31 additions & 0 deletions server/og/ogFetchTimeout.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Match fetchImageDataUrl: public OG cards must not wait forever on a hung origin.
export const OG_FETCH_TIMEOUT_MS = 1_500;

export async function withOgFetchTimeout<T>(
work: (signal: AbortSignal) => Promise<T>,
timeoutMs = OG_FETCH_TIMEOUT_MS,
): Promise<T> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
const aborted = new Promise<never>((_, reject) => {
const rejectOnAbort = () => {
reject(
controller.signal.reason instanceof Error
? controller.signal.reason
: new DOMException("The operation was aborted.", "AbortError"),
);
};
if (controller.signal.aborted) {
rejectOnAbort();
return;
}
controller.signal.addEventListener("abort", rejectOnAbort, { once: true });
});
// Mark handled so abort cannot become an unhandled rejection if work() ignores the signal.
void aborted.catch(() => undefined);
try {
return await Promise.race([work(controller.signal), aborted]);
} finally {
clearTimeout(timeout);
}
}
1 change: 1 addition & 0 deletions specs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ into `docs/` and leave only the design record here.
- `github-backed-skills.md`: source-backed GitHub skills catalog and install invariants.
- `diffing.md`: skill version diffing UI/API design.
- `slug-routing.md`: internal web route precedence and plugin alias contract.
- `og-routes.md`: public OG card metadata fetches and hang timeouts.
- `catalog-taxonomy.md`: stored category, author topic, browse ordering, and correction contract.
- `ci.md`: PR check and production deploy audit-tag policy.
- `manual-testing.md`: maintainer CLI smoke checklist.
Expand Down
Loading
Loading