Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 73 additions & 2 deletions packages/cli/src/lib/download.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { CliError } from "./cli-error";
import {
assertEmptyDirectory,
downloadSnapshots,
getOriginalFileUrl,
getSnapshotPath,
getSnapshotPaths,
} from "./download";
Expand Down Expand Up @@ -67,6 +68,32 @@ describe("getSnapshotPaths", () => {
});
});

describe("getOriginalFileUrl", () => {
it("asks the image CDN for the original file", () => {
expect(getOriginalFileUrl("https://files.argos-ci.com/abc")).toBe(
"https://files.argos-ci.com/abc?tr=orig-true",
);
});

it("keeps the rest of the query string", () => {
expect(getOriginalFileUrl("https://files.argos-ci.com/abc?v=2")).toBe(
"https://files.argos-ci.com/abc?v=2&tr=orig-true",
);
// A transformation would change the file too: the original replaces it.
expect(getOriginalFileUrl("https://files.argos-ci.com/abc?tr=w-100")).toBe(
"https://files.argos-ci.com/abc?tr=orig-true",
);
});

it("leaves other URLs alone", () => {
// An added parameter would break the signature of a presigned S3 URL.
const presigned =
"https://argos.s3.us-east-1.amazonaws.com/abc?X-Amz-Signature=def";
expect(getOriginalFileUrl(presigned)).toBe(presigned);
expect(getOriginalFileUrl("not a url")).toBe("not a url");
});
});

describe("assertEmptyDirectory", () => {
let root: string;

Expand Down Expand Up @@ -113,7 +140,9 @@ describe("downloadSnapshots", () => {
throw status;
}
return status === 200
? new Response(`bytes of ${String(input).split("/").pop()}`)
? new Response(
`bytes of ${new URL(String(input)).pathname.split("/").pop()}`,
)
: new Response(null, { status });
});
vi.stubGlobal("fetch", fetchMock);
Expand Down Expand Up @@ -155,6 +184,48 @@ describe("downloadSnapshots", () => {
expect(onProgress).toHaveBeenLastCalledWith(2, 2);
});

it("writes the original bytes of an image the CDN would convert", async () => {
// Like ImageKit: the format of the response is the CDN's pick, unless the
// original is asked for.
const fetchMock = vi.fn<typeof fetch>(async (input) =>
new URL(String(input)).searchParams.get("tr") === "orig-true"
? new Response("original PNG", {
headers: { "content-type": "image/png" },
})
: new Response("converted JPEG", {
headers: { "content-type": "image/jpeg" },
}),
);
vi.stubGlobal("fetch", fetchMock);
const [entry] = await downloadSnapshots(
[file("chromium/home.png")],
directory,
);
await expect(
readFile(join(directory, "chromium/home.png"), "utf8"),
).resolves.toBe("original PNG");
expect(fetchMock).toHaveBeenCalledWith(
"https://files.argos-ci.com/chromium-home.png?tr=orig-true",
expect.anything(),
);
// The result lists the snapshot's URL as Argos gave it.
expect(entry?.url).toBe("https://files.argos-ci.com/chromium-home.png");
});

it("fetches a presigned URL as it is", async () => {
const fetchMock = mockFetch();
const url =
"https://bucket.s3.us-east-1.amazonaws.com/tree?X-Amz-Signature=abc";
await downloadSnapshots(
[{ name: "tree.aria.yml", url, contentType: "text/yaml" }],
directory,
);
expect(fetchMock).toHaveBeenCalledWith(url, expect.anything());
await expect(
readFile(join(directory, "tree.aria.yml"), "utf8"),
).resolves.toBe("bytes of tree");
});

it("keeps the fields of each snapshot in the result", async () => {
mockFetch();
const [entry] = await downloadSnapshots(
Expand All @@ -178,7 +249,7 @@ describe("downloadSnapshots", () => {
await expect(
downloadSnapshots([file("home.png")], directory),
).rejects.toThrow(
'Could not download "home.png" (HTTP 500): https://files.argos-ci.com/home.png',
'Could not download "home.png" (HTTP 500): https://files.argos-ci.com/home.png?tr=orig-true',
);
expect(fetchMock).toHaveBeenCalledTimes(3);
});
Expand Down
30 changes: 28 additions & 2 deletions packages/cli/src/lib/download.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ const EXTENSION_BY_CONTENT_TYPE: Record<string, string> = {

const IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".webp"]);

/** Host of the image CDN (ImageKit) Argos serves most screenshots from. */
const IMAGE_CDN_HOSTNAME = "files.argos-ci.com";

// Characters Windows refuses in a file name.
// eslint-disable-next-line no-control-regex
const WINDOWS_RESERVED_CHARACTERS = /[<>:"|?*\u0000-\u001f]/g;
Expand Down Expand Up @@ -140,6 +143,28 @@ export async function assertEmptyDirectory(directory: string): Promise<void> {
}
}

/**
* Get the URL that serves a snapshot's file exactly as it was uploaded.
*
* Argos serves images through ImageKit, which picks the format of each
* response on the fly: a small PNG can come back as a JPEG, so the file
* written under `home.png` would hold lossy JPEG bytes. `tr=orig-true` asks it
* for the original instead. Other URLs (large images and text snapshots) are
* presigned S3 URLs: they already serve the original, and any added parameter
* would break their signature, so they are left alone.
*/
export function getOriginalFileUrl(url: string): string {
if (!URL.canParse(url)) {
return url;
}
const parsed = new URL(url);
if (parsed.hostname !== IMAGE_CDN_HOSTNAME) {
return url;
}
parsed.searchParams.set("tr", "orig-true");
return parsed.href;
}

/** A failed response, and whether trying again may succeed. */
class HTTPStatusError extends Error {
retryable: boolean;
Expand All @@ -155,10 +180,11 @@ async function downloadFile(
path: string,
signal: AbortSignal,
): Promise<void> {
const url = getOriginalFileUrl(file.url);
for (let attempt = 1; ; attempt++) {
signal.throwIfAborted();
try {
const response = await fetch(file.url, { signal });
const response = await fetch(url, { signal });
if (!response.ok || !response.body) {
await response.body?.cancel();
throw new HTTPStatusError(response.status);
Expand All @@ -177,7 +203,7 @@ async function downloadFile(
const retryable = !(error instanceof HTTPStatusError) || error.retryable;
if (!retryable || attempt >= MAX_ATTEMPTS) {
const reason = error instanceof Error ? error.message : String(error);
fail(`Could not download "${file.name}" (${reason}): ${file.url}`);
fail(`Could not download "${file.name}" (${reason}): ${url}`);
}
await sleep(RETRY_DELAY_MS * attempt, undefined, { signal });
}
Expand Down
Loading