feat: build caching in eigen (experiement)

app.json

@@ -3,10 +3,15 @@
   "version": "8.91.0",
   "slug": "eigen",
   "expo": {
-    "platforms": [
-      "ios",
-      "android"
-    ],
+    "remoteBuildCache": {
+      "provider": "./buildCacheProviderPlugin.js",
+      "plugin": "./provider.plugin.js",
+      "options": {
+        "owner": "artsy",
+        "repo": "eigen"
+      }
+    },
+    "platforms": ["ios", "android"],
     "runtimeVersion": "257035bdc63217c4adf6e7c7f0bfeea9ca986f22",
     "owner": "artsy_org",
     "extra": {

build-cache-provider/.gitigonore

@@ -0,0 +1 @@
+build/

build-cache-provider/src/download.ts

@@ -0,0 +1,107 @@
+import path from "path"
+import { pipeline } from "stream/promises"
+import spawnAsync from "@expo/spawn-async"
+import glob from "fast-glob"
+import fs from "fs-extra"
+import { extract } from "tar"
+import { v4 as uuidv4 } from "uuid"
+import { getTmpDirectory } from "./helpers"
+
+async function downloadFileAsync(url: string, outputPath: string): Promise<void> {
+  try {
+    const response = await fetch(url)
+
+    if (!response.ok || !response.body) {
+      throw new Error(`Failed to download file from ${url}`)
+    }
+
+    await pipeline(response.body, fs.createWriteStream(outputPath))
+  } catch (error: any) {
+    if (await fs.pathExists(outputPath)) {
+      await fs.remove(outputPath)
+    }
+    throw error
+  }
+}
+
+async function maybeCacheAppAsync(appPath: string, cachedAppPath?: string): Promise<string> {
+  if (cachedAppPath) {
+    await fs.ensureDir(path.dirname(cachedAppPath))
+    await fs.move(appPath, cachedAppPath)
+    return cachedAppPath
+  }
+  return appPath
+}
+
+export async function downloadAndMaybeExtractAppAsync(
+  url: string,
+  platform: "ios" | "android",
+  cachedAppPath?: string
+): Promise<string> {
+  const outputDir = path.join(getTmpDirectory(), uuidv4())
+  await fs.promises.mkdir(outputDir, { recursive: true })
+
+  if (url.endsWith("apk")) {
+    const apkFilePath = path.join(outputDir, `${uuidv4()}.apk`)
+    await downloadFileAsync(url, apkFilePath)
+    console.log("Successfully downloaded app")
+    return await maybeCacheAppAsync(apkFilePath, cachedAppPath)
+  } else {
+    const tmpArchivePathDir = path.join(getTmpDirectory(), uuidv4())
+    await fs.mkdir(tmpArchivePathDir, { recursive: true })
+
+    const tmpArchivePath = path.join(tmpArchivePathDir, `${uuidv4()}.tar.gz`)
+
+    await downloadFileAsync(url, tmpArchivePath)
+    console.log("Successfully downloaded app archive")
+    await tarExtractAsync(tmpArchivePath, outputDir)
+
+    const appPath = await getAppPathAsync(outputDir, platform === "ios" ? "app" : "apk")
+
+    return await maybeCacheAppAsync(appPath, cachedAppPath)
+  }
+}
+
+export async function extractAppFromLocalArchiveAsync(
+  appArchivePath: string,
+  platform: "ios" | "android"
+): Promise<string> {
+  const outputDir = path.join(getTmpDirectory(), uuidv4())
+  await fs.promises.mkdir(outputDir, { recursive: true })
+
+  await tarExtractAsync(appArchivePath, outputDir)
+
+  return await getAppPathAsync(outputDir, platform === "android" ? "apk" : "app")
+}
+
+async function getAppPathAsync(outputDir: string, applicationExtension: string): Promise<string> {
+  const appFilePaths = await glob(`./**/*.${applicationExtension}`, {
+    cwd: outputDir,
+    onlyFiles: false,
+  })
+
+  if (appFilePaths.length === 0) {
+    throw Error("Did not find any installable apps inside tarball.")
+  }
+
+  return path.join(outputDir, appFilePaths[0])
+}
+
+async function tarExtractAsync(input: string, output: string): Promise<void> {
+  try {
+    if (process.platform !== "win32") {
+      await spawnAsync("tar", ["-xf", input, "-C", output], {
+        stdio: "inherit",
+      })
+      return
+    }
+  } catch (error: any) {
+    console.warn(
+      `Failed to extract tar using native tools, falling back on JS tar module. ${error.message}`
+    )
+  }
+  console.debug(`Extracting ${input} to ${output} using JS tar module`)
+  // tar node module has previously had problems with big files, and seems to
+  // be slower, so only use it as a backup.
+  await extract({ file: input, cwd: output })
+}

build-cache-provider/src/github.ts

@@ -0,0 +1,181 @@
+import path from "path"
+import { type RestEndpointMethodTypes } from "@octokit/plugin-rest-endpoint-methods"
+import { Octokit } from "@octokit/rest"
+import fs from "fs-extra"
+import { create as createTar } from "tar"
+import { v4 as uuidv4 } from "uuid"
+
+import { getTmpDirectory } from "./helpers"
+
+interface GithubProviderOptions {
+  token: string
+  owner: string
+  repo: string
+  tagName: string
+  binaryPath: string
+}
+
+export async function createReleaseAndUploadAsset({
+  token,
+  owner,
+  repo,
+  tagName,
+  binaryPath,
+}: GithubProviderOptions) {
+  const octokit = new Octokit({ auth: token })
+
+  try {
+    const commitSha = await getBranchShaWithFallback(octokit, owner, repo)
+
+    const tagSha = await ensureAnnotatedTag(octokit, {
+      owner,
+      repo,
+      tag: tagName,
+      message: tagName,
+      object: commitSha,
+      type: "commit",
+      tagger: {
+        name: "Release Bot",
+        email: "[email protected]",
+        date: new Date().toISOString(),
+      },
+    })
+
+    const release = await octokit.rest.repos.createRelease({
+      owner,
+      repo,
+      tag_name: tagName,
+      name: tagName,
+      draft: false,
+      prerelease: true,
+    })
+
+    await uploadReleaseAsset(octokit, {
+      owner,
+      repo,
+      releaseId: release.data.id,
+      binaryPath,
+    })
+
+    return release.url
+  } catch (error) {
+    throw new Error(
+      `GitHub release failed: ${error instanceof Error ? error.message : String(error)}`
+    )
+  }
+}
+
+async function getBranchShaWithFallback(
+  octokit: Octokit,
+  owner: string,
+  repo: string
+): Promise<string> {
+  const branchesToTry = ["main", "master"]
+
+  for (const branchName of branchesToTry) {
+    try {
+      const { data } = await octokit.rest.repos.getBranch({
+        owner,
+        repo,
+        branch: branchName,
+      })
+      return data.commit.sha
+    } catch (error) {
+      if (error instanceof Error && error.message.includes("Branch not found")) {
+        if (branchName === "master") throw new Error("No valid branch found")
+        continue
+      }
+      throw error
+    }
+  }
+  throw new Error("Branch fallback exhausted")
+}
+async function ensureAnnotatedTag(
+  octokit: Octokit,
+  params: RestEndpointMethodTypes["git"]["createTag"]["parameters"]
+): Promise<string> {
+  const { owner, repo, tag } = params
+  const refName = `refs/tags/${tag}`
+
+  try {
+    const { data: existingRef } = await octokit.rest.git.getRef({
+      owner,
+      repo,
+      ref: `tags/${tag}`,
+    })
+    // Return existing tag SHA
+    return existingRef.object.sha
+  } catch (err: any) {
+    if (err.status !== 404) {
+      throw err
+    }
+  }
+
+  // Create the annotated tag object
+  const { data: tagData } = await octokit.rest.git.createTag(params)
+
+  // Create the tag reference pointing to the new tag object
+  await octokit.rest.git.createRef({
+    owner,
+    repo,
+    ref: refName,
+    sha: tagData.sha,
+  })
+
+  return tagData.sha
+}
+
+async function uploadReleaseAsset(
+  octokit: Octokit,
+  params: {
+    owner: string
+    repo: string
+    releaseId: number
+    binaryPath: string
+  }
+) {
+  let filePath = params.binaryPath
+  let name = path.basename(filePath)
+  if ((await fs.stat(filePath)).isDirectory()) {
+    await fs.mkdirp(getTmpDirectory())
+    const tarPath = path.join(getTmpDirectory(), `${uuidv4()}.tar.gz`)
+    const parentPath = path.dirname(filePath)
+    await createTar({ cwd: parentPath, file: tarPath, gzip: true }, [name])
+    filePath = tarPath
+    name = name + ".tar.gz"
+  }
+
+  const fileData = await fs.readFile(filePath)
+
+  return octokit.rest.repos.uploadReleaseAsset({
+    owner: params.owner,
+    repo: params.repo,
+    release_id: params.releaseId,
+    name: name,
+    data: fileData as unknown as string, // Type workaround for binary data
+    headers: {
+      "content-type": "application/octet-stream",
+      "content-length": fileData.length.toString(),
+    },
+  })
+}
+
+export async function getReleaseAssetsByTag({
+  token,
+  owner,
+  repo,
+  tag,
+}: {
+  token: string
+  owner: string
+  repo: string
+  tag: string
+}) {
+  const octokit = new Octokit({ auth: token })
+  const release = await octokit.rest.repos.getReleaseByTag({
+    owner,
+    repo,
+    tag,
+  })
+  return release.data.assets
+}

build-cache-provider/src/helpers.ts

@@ -0,0 +1,35 @@
+import path from "path"
+import { getPackageJson, RunOptions } from "@expo/config"
+import envPaths from "env-paths"
+
+const { temp: TEMP_PATH } = envPaths("github-build-cache-provider")
+
+export function isDevClientBuild({
+  runOptions,
+  projectRoot,
+}: {
+  runOptions: RunOptions
+  projectRoot: string
+}): boolean {
+  if (!hasDirectDevClientDependency(projectRoot)) {
+    return false
+  }
+
+  if ("variant" in runOptions && runOptions.variant !== undefined) {
+    return runOptions.variant === "debug"
+  }
+  if ("configuration" in runOptions && runOptions.configuration !== undefined) {
+    return runOptions.configuration === "Debug"
+  }
+
+  return true
+}
+
+export function hasDirectDevClientDependency(projectRoot: string): boolean {
+  const { dependencies = {}, devDependencies = {} } = getPackageJson(projectRoot)
+  return !!dependencies["expo-dev-client"] || !!devDependencies["expo-dev-client"]
+}
+
+export const getTmpDirectory = (): string => TEMP_PATH
+export const getBuildRunCacheDirectoryPath = (): string =>
+  path.join(getTmpDirectory(), "build-run-cache")