feat: use a git cache to speed up clones (#275)

* use a git cache to speed up clones

* drop unused git config

Author: nornagon

package.json

@@ -16,6 +16,7 @@
     "prepare": "husky install"
   },
   "dependencies": {
+    "async-mutex": "^0.5.0",
     "fs-extra": "^11.1.1",
     "global-agent": "^3.0.0",
     "node-fetch": "^2.6.7",

src/operations/init-repo.ts

@@ -2,54 +2,68 @@ import { parse } from 'yaml';
 import * as fs from 'fs-extra';
 import * as os from 'os';
 import * as path from 'path';
-import simpleGit from 'simple-git';
+import simpleGit, { CheckRepoActions } from 'simple-git';
 import { InitRepoOptions } from '../interfaces';
 import { LogLevel } from '../enums';
 import { log } from '../utils/log-util';
-import { ResetMode } from 'simple-git';
+import { Mutex } from 'async-mutex';
 
-const baseDir = path.resolve(os.tmpdir(), 'trop-working');
+const baseDir =
+  process.env.WORKING_DIR ?? path.resolve(os.tmpdir(), 'trop-working');
+
+function githubUrl({ slug, accessToken }: InitRepoOptions): string {
+  return `https://x-access-token:${accessToken}@github.com/${slug}.git`;
+}
+
+const repoMutex = new Map<string, Mutex>();
+function mutexForRepoCache(slug: string) {
+  if (!repoMutex.has(slug)) repoMutex.set(slug, new Mutex());
+  return repoMutex.get(slug)!;
+}
+
+async function updateRepoCache({ slug, accessToken }: InitRepoOptions) {
+  const cacheDir = path.resolve(baseDir, slug, 'git-cache');
+
+  await fs.mkdirp(cacheDir);
+  const git = simpleGit(cacheDir);
+  if (!(await git.checkIsRepo(CheckRepoActions.BARE))) {
+    // The repo might be missing, or otherwise somehow corrupt. Re-clone it.
+    log(
+      'updateRepoCache',
+      LogLevel.INFO,
+      `${cacheDir} was not a git repo, cloning...`,
+    );
+    await fs.remove(cacheDir);
+    await fs.mkdirp(cacheDir);
+    await git.clone(githubUrl({ slug, accessToken }), '.', ['--bare']);
+  }
+  await git.fetch();
+
+  return cacheDir;
+}
 
 /**
  * Initializes the cloned repo trop will use to run backports.
  *
  * @param {InitRepoOptions} options - repo and payload for repo initialization
- * @returns {Object} - an object containing the repo initialization directory
+ * @returns {{dir: string}} - an object containing the repo initialization directory
  */
-export const initRepo = async ({ slug, accessToken }: InitRepoOptions) => {
+export const initRepo = async ({
+  slug,
+  accessToken,
+}: InitRepoOptions): Promise<{ dir: string }> => {
   log('initRepo', LogLevel.INFO, 'Setting up local repository');
 
   await fs.mkdirp(path.resolve(baseDir, slug));
   const prefix = path.resolve(baseDir, slug, 'job-');
   const dir = await fs.mkdtemp(prefix);
-
-  // Ensure that this directory is empty.
-  await fs.mkdirp(dir);
-  await fs.remove(dir);
-  await fs.mkdirp(dir);
-
   const git = simpleGit(dir);
 
-  await git.clone(
-    `https://x-access-token:${accessToken}@github.com/${slug}.git`,
-    '.',
-  );
-
-  // Clean up just in case.
-  await git.reset(ResetMode.HARD);
-  const status = await git.status();
-
-  for (const file of status.not_added) {
-    await fs.remove(path.resolve(dir, file));
-  }
-
-  await git.pull();
-
-  const config = fs.readFileSync('./config.yml', 'utf8');
-  const { tropEmail, tropName } = parse(config);
-  await git.addConfig('user.email', tropEmail || '[email protected]');
-  await git.addConfig('user.name', tropName || 'Trop Bot');
+  // Concurrent access to the repo cache has the potential to mess things up.
+  await mutexForRepoCache(slug).runExclusive(async () => {
+    const cacheDir = await updateRepoCache({ slug, accessToken });
+    await git.clone(cacheDir, '.');
+  });
 
-  await git.addConfig('commit.gpgsign', 'false');
   return { dir };
 };

yarn.lock

@@ -1704,6 +1704,13 @@ astral-regex@^2.0.0:
   resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31"
   integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==
 
+async-mutex@^0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/async-mutex/-/async-mutex-0.5.0.tgz#353c69a0b9e75250971a64ac203b0ebfddd75482"
+  integrity sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==
+  dependencies:
+    tslib "^2.4.0"
+
 asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@@ -5602,6 +5609,11 @@ tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
   integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
 
+tslib@^2.4.0:
+  version "2.6.2"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae"
+  integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==
+
 tsutils@^3.21.0:
   version "3.21.0"
   resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"