feat: official support for custom remote cache providers (#331)

* feat: add missing methods to remote cache provider

* add test

* unify interface

* mark delete and upload as optional

* changeset

* simplify return to array only

* remove id

* simplify upload return

* refactor getLocalBinaryPath

* get rid of local artifact path as it's handled by RNEF; pass targetURL

* make download return Reponse and move logic to RNEF

* rework provider plugin to be configurable

* add throws to TSDoc

* implement RemoteBuildCache in example

* cleanup

* use Error

Author: thymikee

.changeset/large-mirrors-confess.md

@@ -0,0 +1,7 @@
+---
+'@rnef/config': patch
+'@rnef/tools': patch
+'rnef-docs': patch
+---
+
+feat: official support for custom remote cache providers

packages/config/src/lib/schema.ts

@@ -40,9 +40,11 @@ const ConfigTypeSchema = Joi.object({
   plugins: Joi.array().items(Joi.function()).optional(),
   platforms: Joi.object().pattern(Joi.string(), Joi.function()).optional(),
   commands: Joi.array().items(CommandTypeSchema).optional(),
-  remoteCacheProvider: Joi.string()
-    .valid('github-actions', null, Joi.function())
-    .optional(),
+  remoteCacheProvider: Joi.alternatives().try(
+    Joi.string().valid('github-actions'),
+    Joi.function(),
+    Joi.any().valid(null)
+  ).optional(),
   fingerprint: Joi.object({
     extraSources: Joi.array().items(Joi.string()).default([]),
     ignorePaths: Joi.array().items(Joi.string()).default([]),

packages/tools/src/lib/__tests__/remoteBuildCache.test.ts

@@ -0,0 +1,90 @@
+import { expect, test } from 'vitest';
+import type { RemoteBuildCache } from '../build-cache/common.js';
+import { createRemoteBuildCache } from '../build-cache/remoteBuildCache.js';
+
+const uploadMock = vi.fn();
+
+class DummyRemoteCacheProvider implements RemoteBuildCache {
+  name = 'dummy';
+  constructor(options?: { name: string }) {
+    if (options) {
+      this.name = options.name;
+    }
+  }
+  async list({ artifactName }: { artifactName: string }) {
+    return [{ name: artifactName, url: '/path/to/dummy' }];
+  }
+  async download({ artifactName }: { artifactName: string }) {
+    const resBody = new TextEncoder().encode(artifactName);
+    return new Response(resBody);
+  }
+  async delete({ artifactName }: { artifactName: string }) {
+    if (artifactName === 'dummy') {
+      return [{ name: artifactName, url: '/path/to/dummy' }];
+    }
+    return [];
+  }
+  async upload({ artifactName }: { artifactName: string }) {
+    uploadMock(artifactName);
+    return { name: artifactName, url: '/path/to/dummy' };
+  }
+}
+
+test('dummy remote cache provider lists artifacts', async () => {
+  const pluginDummyRemoteCacheProvider = (options?: { name: string }) => () =>
+    new DummyRemoteCacheProvider(options);
+  const remoteBuildCache = await createRemoteBuildCache(
+    pluginDummyRemoteCacheProvider()
+  );
+  const artifacts = await remoteBuildCache?.list({
+    artifactName: 'rnef-android-debug-7af554b93cd696ca95308fdebe3a4484001bb7b4',
+  });
+  expect(artifacts).toEqual([
+    {
+      name: 'rnef-android-debug-7af554b93cd696ca95308fdebe3a4484001bb7b4',
+      url: '/path/to/dummy',
+    },
+  ]);
+});
+
+test('dummy remote cache provider downloads artifacts', async () => {
+  const pluginDummyRemoteCacheProvider = (options?: { name: string }) => () =>
+    new DummyRemoteCacheProvider(options);
+  const remoteBuildCache = await createRemoteBuildCache(
+    pluginDummyRemoteCacheProvider()
+  );
+  const artifact = await remoteBuildCache?.download({
+    artifactName: 'rnef-android-debug-7af554b93cd696ca95308fdebe3a4484001bb7b4',
+  });
+  const response = await artifact?.text();
+  expect(response).toEqual(
+    'rnef-android-debug-7af554b93cd696ca95308fdebe3a4484001bb7b4'
+  );
+});
+
+test('dummy remote cache provider deletes artifacts', async () => {
+  const pluginDummyRemoteCacheProvider = (options?: { name: string }) => () =>
+    new DummyRemoteCacheProvider(options);
+  const remoteBuildCache = await createRemoteBuildCache(
+    pluginDummyRemoteCacheProvider()
+  );
+  const result = await remoteBuildCache?.delete({ artifactName: 'dummy' });
+  expect(result).toEqual([
+    {
+      name: 'dummy',
+      url: '/path/to/dummy',
+    },
+  ]);
+  const result2 = await remoteBuildCache?.delete({ artifactName: 'dummy2' });
+  expect(result2).toEqual([]);
+});
+
+test('dummy remote cache provider uploads artifacts', async () => {
+  const pluginDummyRemoteCacheProvider = (options?: { name: string }) => () =>
+    new DummyRemoteCacheProvider(options);
+  const remoteBuildCache = await createRemoteBuildCache(
+    pluginDummyRemoteCacheProvider()
+  );
+  await remoteBuildCache?.upload({ artifactName: 'dummy' });
+  expect(uploadMock).toHaveBeenCalledWith('dummy');
+});

packages/tools/src/lib/build-cache/common.ts

@@ -2,36 +2,65 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { nativeFingerprint } from '../fingerprint/index.js';
 import { getCacheRootPath } from '../project.js';
-import type { spinner } from '../prompts.js';
 
 export const BUILD_CACHE_DIR = 'remote-build';
 
 export type SupportedRemoteCacheProviders = 'github-actions';
 
 export type RemoteArtifact = {
   name: string;
-  downloadUrl: string;
+  url: string;
 };
 
 export type LocalArtifact = {
   name: string;
-  path: string;
 };
 
+/**
+ * Interface for implementing remote build cache providers.
+ * Remote cache providers allow storing and retrieving native build artifacts (e.g. APK, IPA)
+ * from remote storage like S3, GitHub Artifacts etc.
+ */
 export interface RemoteBuildCache {
+  /** Unique identifier for this cache provider, will be displayed in logs */
   name: string;
-  query({
+
+  /**
+   * List available artifacts matching the given name pattern
+   * @param artifactName - Passed after fingerprinting the build, e.g. `rnef-android-debug-1234567890` for android in debug variant
+   * @param limit - Optional maximum number of artifacts to return
+   * @returns Array of matching remote artifacts, or empty array if none found
+   */
+  list({
     artifactName,
+    limit,
   }: {
-    artifactName: string;
-  }): Promise<RemoteArtifact | null>;
-  download({
-    artifact,
-    loader,
-  }: {
-    artifact: RemoteArtifact;
-    loader: ReturnType<typeof spinner>;
-  }): Promise<LocalArtifact>;
+    artifactName: string | undefined;
+    limit?: number;
+  }): Promise<RemoteArtifact[]>;
+
+  /**
+   * Download a remote artifact to local storage
+   * @param artifactName - Name of the artifact to download, e.g. `rnef-android-debug-1234567890` for android in debug variant
+   * @returns Response object from fetch, which will be used to download the artifact
+   */
+  download({ artifactName }: { artifactName: string }): Promise<Response>;
+
+  /**
+   * Delete a remote artifact
+   * @param artifact - Remote artifact to delete, as returned by `list` method
+   * @returns Array of deleted artifacts
+   * @throws {Error} Throws if artifact is not found or deletion fails
+   */
+  delete({ artifactName }: { artifactName: string }): Promise<RemoteArtifact[]>;
+
+  /**
+   * Upload a local artifact stored in build cache to remote storage
+   * @param artifactName - Name of the artifact to upload, e.g. `rnef-android-debug-1234567890` for android in debug variant
+   * @returns Remote artifact info if upload successful
+   * @throws {Error} Throws if upload fails
+   */
+  upload({ artifactName }: { artifactName: string }): Promise<RemoteArtifact>;
 }
 
 /**
@@ -63,16 +92,8 @@ export function getLocalArtifactPath(artifactName: string) {
 }
 
 export function getLocalBinaryPath(artifactPath: string) {
-  let binaryPath: string | null = null;
   const files = fs.readdirSync(artifactPath);
-
-  // assume there is only one binary in the artifact
-  for (const file of files) {
-    if (file) {
-      binaryPath = path.join(artifactPath, file);
-    }
-    break;
-  }
-
-  return binaryPath;
+  // Get the first non-hidden, non-directory file as the binary
+  const binaryName = files.find((file) => file && !file.startsWith('.'));
+  return binaryName ? path.join(artifactPath, binaryName) : null;
 }

packages/tools/src/lib/build-cache/fetchCachedBuild.ts

@@ -1,9 +1,14 @@
+import fs from 'node:fs';
 import path from 'node:path';
+import AdmZip from 'adm-zip';
+import * as tar from 'tar';
 import { color } from '../color.js';
+import { RnefError } from '../error.js';
 import logger from '../logger.js';
 import { getProjectRoot } from '../project.js';
 import { spinner } from '../prompts.js';
 import {
+  getLocalArtifactPath,
   getLocalBinaryPath,
   type RemoteBuildCache,
   type SupportedRemoteCacheProviders,
@@ -20,7 +25,7 @@ type FetchCachedBuildOptions = {
     | SupportedRemoteCacheProviders
     | undefined
     | null
-    | { new (): RemoteBuildCache };
+    | { (): RemoteBuildCache };
 };
 
 export async function fetchCachedBuild({
@@ -61,32 +66,103 @@ Proceeding with local build.`);
 
   loader.stop(`No local build cached. Checking ${remoteBuildCache.name}.`);
 
-  const remoteBuild = await remoteBuildCache.query({ artifactName });
-  if (!remoteBuild) {
-    loader.start('');
-    loader.stop(`No cached build found for "${artifactName}".`);
-    return null;
-  }
-
+  const localArtifactPath = getLocalArtifactPath(artifactName);
+  const response = await remoteBuildCache.download({ artifactName });
   loader.start(`Downloading cached build from ${remoteBuildCache.name}`);
-  const fetchedBuild = await remoteBuildCache.download({
-    artifact: remoteBuild,
-    loader,
-  });
-
-  const binaryPath = getLocalBinaryPath(fetchedBuild.path);
+  await handleDownloadResponse(
+    response,
+    localArtifactPath,
+    remoteBuildCache.name,
+    loader
+  );
+  await extractArtifactTarballIfNeeded(localArtifactPath);
+  const binaryPath = getLocalBinaryPath(localArtifactPath);
   if (!binaryPath) {
     loader.stop(`No binary found for "${artifactName}".`);
     return null;
   }
-
   loader.stop(
     `Downloaded cached build: ${color.cyan(path.relative(root, binaryPath))}.`
   );
 
   return {
-    name: fetchedBuild.name,
-    artifactPath: fetchedBuild.path,
+    name: artifactName,
+    artifactPath: localArtifactPath,
     binaryPath,
   };
 }
+
+async function handleDownloadResponse(
+  response: Response,
+  localArtifactPath: string,
+  name: string,
+  loader: ReturnType<typeof spinner>
+) {
+  try {
+    fs.mkdirSync(localArtifactPath, { recursive: true });
+    if (!response.ok || !response.body) {
+      throw new Error(`Failed to download artifact: ${response.statusText}`);
+    }
+    let responseData = response;
+    const contentLength = response.headers.get('content-length');
+
+    if (contentLength) {
+      const totalBytes = parseInt(contentLength, 10);
+      const totalMB = (totalBytes / 1024 / 1024).toFixed(2);
+      let downloadedBytes = 0;
+
+      const reader = response.body.getReader();
+      const stream = new ReadableStream({
+        async start(controller) {
+          while (true) {
+            const { done, value } = await reader.read();
+            if (done) {
+              break;
+            }
+            downloadedBytes += value.length;
+            const progress = ((downloadedBytes / totalBytes) * 100).toFixed(0);
+            loader?.message(
+              `Downloading cached build from ${name} (${progress}% of ${totalMB} MB)`
+            );
+            controller.enqueue(value);
+          }
+          controller.close();
+        },
+      });
+      responseData = new Response(stream);
+    }
+
+    const zipPath = localArtifactPath + '.zip';
+    const buffer = await responseData.arrayBuffer();
+    fs.writeFileSync(zipPath, new Uint8Array(buffer));
+    unzipFile(zipPath, localArtifactPath);
+    fs.unlinkSync(zipPath);
+  } catch (error) {
+    loader?.stop(`Failed: Downloading cached build from ${name}`);
+    throw new RnefError(`Unexpected error`, { cause: error });
+  }
+}
+
+function unzipFile(zipPath: string, targetPath: string): void {
+  const zip = new AdmZip(zipPath);
+  zip.extractAllTo(targetPath, true);
+}
+
+async function extractArtifactTarballIfNeeded(artifactPath: string) {
+  const tarPath = path.join(artifactPath, 'app.tar.gz');
+
+  // If the tarball is not found, it means the artifact is already unpacked.
+  if (!fs.existsSync(tarPath)) {
+    return;
+  }
+
+  // iOS simulator build artifact (*.app directory) is packed in .tar.gz file to
+  // preserve execute file permission.
+  // See: https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
+  await tar.extract({
+    file: tarPath,
+    cwd: artifactPath,
+    gzip: true,
+  });
+  fs.unlinkSync(tarPath);
+}