Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/api-client/src/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1846,6 +1846,8 @@ export interface operations {
parallel?: boolean | null;
parallelTotal?: number | null;
parallelIndex?: number | null;
/** @description Only used for non-parallel builds. Indicates that this is the last request of the build, so Argos can finalize it. A build whose screenshots are too large to fit in a single request can split them across several sequential requests, leaving `final` falsy on every request but the last. Defaults to `true`. */
final?: boolean | null;
metadata?: components["schemas"]["BuildMetadata"];
};
};
Expand Down
45 changes: 45 additions & 0 deletions packages/core/src/upload.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,51 @@ describe("#upload", () => {
})();
});

it("marks the update build request as final and carries the metadata", () => {
return server.boundary(async () => {
const updateBuildBodies: {
final?: boolean | null;
metadata?: unknown;
screenshots: unknown[];
}[] = [];

server.use(
http.put(
"https://api.argos-ci.dev/builds/:buildId",
async ({ request }) => {
updateBuildBodies.push(
(await request.json()) as {
final?: boolean | null;
metadata?: unknown;
screenshots: unknown[];
},
);
return HttpResponse.json({
build: { id: "123", url: "https://app.argos-ci.dev/builds/123" },
});
},
),
);

await upload({
branch: "main",
apiBaseUrl: "https://api.argos-ci.dev",
root: join(__dirname, "../../../__fixtures__/screenshots"),
commit: "f16f980bd17cccfa93a1ae7766727e67950773d0",
token: "92d832e0d22ab113c8979d73a87a11130eaa24a9",
metadata: {
testReport: { status: "passed" },
},
});

// The fixtures are small, so everything fits in a single request.
expect(updateBuildBodies).toHaveLength(1);
const last = updateBuildBodies.at(-1);
expect(last?.final).toBe(true);
expect(last?.metadata).toEqual({ testReport: { status: "passed" } });
})();
});

it("retries", () => {
return server.boundary(async () => {
let reqCount = 0;
Expand Down
95 changes: 69 additions & 26 deletions packages/core/src/upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { hashFile } from "./hashing";
import { resolveArgosToken } from "./auth";
import { uploadFileWithPresignedPost } from "./s3";
import { debug, debugTime, debugTimeEnd } from "./debug";
import { chunk, mapInChunks } from "./util/chunk";
import { chunk, chunkBySize, mapInChunks } from "./util/chunk";
import {
getPlaywrightTracePath,
readMetadata,
Expand All @@ -24,6 +24,13 @@ import { skip } from "./skip";
*/
const CHUNK_SIZE = 10;

/**
* Maximum serialized size (in bytes) of the screenshots sent in a single
* `updateBuild` request. Kept below the API request body size limit (3MB) to
* leave room for the rest of the body (build metadata, etc.).
*/
const MAX_UPDATE_BUILD_SCREENSHOTS_BYTES = 2 * 1024 * 1024;

const PLAYWRIGHT_TRACE_CONTENT_TYPE = "application/zip";

type BuildMetadata = ArgosAPISchema.components["schemas"]["BuildMetadata"];
Expand Down Expand Up @@ -400,35 +407,71 @@ export async function upload(params: UploadParameters): Promise<{
// Update build
debug("Updating build");

const uploadBuildResponse = await apiClient.PUT("/builds/{buildId}", {
params: {
path: {
buildId: result.build.id,
const screenshotInputs = snapshots.map((snapshot) => ({
key: snapshot.hash,
name: snapshot.name,
metadata: snapshot.metadata,
pwTraceKey: snapshot.pwTrace?.hash ?? null,
threshold: snapshot.threshold ?? config?.threshold ?? null,
baseName: snapshot.baseName,
parentName: snapshot.parentName,
contentType: snapshot.contentType,
}));

// Split the screenshots across several requests to stay under the API request
// body size limit, since the metadata of a single screenshot can be large.
const screenshotChunks = chunkBySize(
screenshotInputs,
MAX_UPDATE_BUILD_SCREENSHOTS_BYTES,
);
// Always send at least one request so a build with no screenshots is
// finalized.
if (screenshotChunks.length === 0) {
screenshotChunks.push([]);
}

// A parallel shard with an index groups its requests into a single shard, so
// only its last request is `final`. Without an index (manual finalization),
// requests can't be grouped, so each is its own complete shard — that's fine
// since the build is finalized manually.
const groupsRequests = !config.parallel || config.parallelIndex != null;

let build: ArgosAPISchema.components["schemas"]["Build"] | null = null;
for (let index = 0; index < screenshotChunks.length; index++) {
const isLast = index === screenshotChunks.length - 1;
debug(`Updating build (request ${index + 1}/${screenshotChunks.length})`);
const uploadBuildResponse = await apiClient.PUT("/builds/{buildId}", {
params: {
path: {
buildId: result.build.id,
},
},
},
body: {
screenshots: snapshots.map((snapshot) => ({
key: snapshot.hash,
name: snapshot.name,
metadata: snapshot.metadata,
pwTraceKey: snapshot.pwTrace?.hash ?? null,
threshold: snapshot.threshold ?? config?.threshold ?? null,
baseName: snapshot.baseName,
parentName: snapshot.parentName,
contentType: snapshot.contentType,
})),
parallel: config.parallel,
parallelTotal: config.parallelTotal,
parallelIndex: config.parallelIndex,
metadata: params.metadata,
},
});
body: {
screenshots: screenshotChunks[index] ?? [],
parallel: config.parallel,
parallelTotal: config.parallelTotal,
parallelIndex: config.parallelIndex,
// The build/shard is finalized by its last request, which also carries
// the build metadata. Ungrouped requests are each final on their own.
final: groupsRequests ? isLast : true,
metadata: isLast ? params.metadata : undefined,
},
});

if (uploadBuildResponse.error) {
throwAPIError(uploadBuildResponse.error, uploadBuildResponse.response);
}

if (isLast) {
build = uploadBuildResponse.data.build;
}
}

if (uploadBuildResponse.error) {
throwAPIError(uploadBuildResponse.error, uploadBuildResponse.response);
if (!build) {
throw new Error("Invariant: the build was not updated");
}

return { build: uploadBuildResponse.data.build, screenshots: snapshots };
return { build, screenshots: snapshots };
}

async function uploadFilesToS3(files: SecureUploadFileInput[]) {
Expand Down
29 changes: 29 additions & 0 deletions packages/core/src/util/chunk.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import { describe, it, expect } from "vitest";
import { chunkBySize } from "./chunk";

describe("chunkBySize", () => {
it("returns no chunk for an empty array", () => {
expect(chunkBySize([], 100)).toEqual([]);
});

it("keeps items in a single chunk when they fit", () => {
const items = [{ a: 1 }, { a: 2 }, { a: 3 }];
expect(chunkBySize(items, 1000)).toEqual([items]);
});

it("splits items into several chunks when they exceed the size", () => {
// Each item serializes to `{"a":"xxxx"}` (11 bytes).
const items = Array.from({ length: 5 }, () => ({ a: "xxxx" }));
const chunks = chunkBySize(items, 25);
// 25 bytes fits 2 items (22 bytes) but not 3 (33 bytes).
expect(chunks.map((chunk) => chunk.length)).toEqual([2, 2, 1]);
expect(chunks.flat()).toEqual(items);
});

it("gives an oversized item its own chunk", () => {
const items = [{ a: "x" }, { a: "x".repeat(100) }, { a: "x" }];
const chunks = chunkBySize(items, 20);
expect(chunks.map((chunk) => chunk.length)).toEqual([1, 1, 1]);
expect(chunks.flat()).toEqual(items);
});
});
24 changes: 24 additions & 0 deletions packages/core/src/util/chunk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,30 @@ export const chunk = <T>(collection: T[], size: number) => {
return result;
};

/**
* Split items into chunks whose serialized JSON size stays under `maxBytes`.
* An item larger than `maxBytes` on its own gets its own chunk.
*/
export const chunkBySize = <T>(items: T[], maxBytes: number): T[][] => {
const chunks: T[][] = [];
let current: T[] = [];
let currentBytes = 0;
for (const item of items) {
const size = Buffer.byteLength(JSON.stringify(item));
if (current.length > 0 && currentBytes + size > maxBytes) {
chunks.push(current);
current = [];
currentBytes = 0;
}
current.push(item);
currentBytes += size;
}
if (current.length > 0) {
chunks.push(current);
}
return chunks;
};

/**
* Map over a collection asynchronously, processing at most `size` items
* concurrently to limit memory pressure.
Expand Down
Loading