Merge pull request #346 from mook-as/robust-startup

mattfarina · web-flow · commit b5d552522975 · 2021-06-15T13:57:35.000-04:00
Make startup more robust
diff --git a/src/k8s-engine/client.ts b/src/k8s-engine/client.ts
@@ -218,19 +218,8 @@ export class KubeClient extends events.EventEmitter {
         console.log(`Waited more than ${ maxWaitTime / 1000 } secs for kubernetes to fully start up. Giving up.`);
         break;
       }
-      try {
-        if (await this.getServiceListWatch()) {
-          break;
-        }
-      } catch (ex) {
-        if (ex?.code === 'ERR_TLS_CERT_ALTNAME_INVALID') {
-          // If the VM restarted and the IP address changed, the certificate
-          // will need to be regenerated (to include the new IP address).
-          // Wait for K3s to do so.
-          console.log('Incorrect TLS cert when waitin for services; retrying.');
-        } else {
-          throw ex;
-        }
+      if (await this.getServiceListWatch()) {
+        break;
       }
       await util.promisify(setTimeout)(waitTime);
     }
diff --git a/src/k8s-engine/hyperkit.ts b/src/k8s-engine/hyperkit.ts
@@ -6,10 +6,8 @@ import fs from 'fs';
 import os from 'os';
 import path from 'path';
 import timers from 'timers';
-import util from 'util';
 
 import Electron from 'electron';
-import fetch from 'node-fetch';
 import semver from 'semver';
 import XDGAppPaths from 'xdg-app-paths';
 import { exec as sudo } from 'sudo-prompt';
@@ -446,34 +444,9 @@ export default class HyperkitBackend extends events.EventEmitter implements K8s.
         }
       });
 
-      // Wait for k3s server; note that we're delibrately sending a HTTP request
-      // to an HTTPS server, and expecting an error response back.
-      while (true) {
-        try {
-          const resp = await fetch(`http://${ await this.ipAddress }:6443`);
-
-          if (resp.status === 400) {
-            break;
-          }
-        } catch (e) {
-          if (e.code !== 'ECONNREFUSED') {
-            throw e;
-          }
-        }
-        await util.promisify(setTimeout)(500);
-      }
-
-      try {
-        await this.k3sHelper.updateKubeconfig(
-          resources.executable('docker-machine-driver-hyperkit'),
-          '--storage-path', path.join(paths.state(), 'driver'),
-          'ssh', '--', 'sudo', `${ cacheDir }/kubeconfig`,
-        );
-      } catch (e) {
-        console.error(e);
-        console.error(e.stack);
-        throw e;
-      }
+      await this.k3sHelper.waitForServerReady(() => this.ipAddress);
+      await this.k3sHelper.updateKubeconfig(
+        () => this.hyperkitWithCapture('ssh', '--', 'sudo', `${ cacheDir }/kubeconfig`));
       this.setState(K8s.State.STARTED);
       this.setProgress(Progress.DONE);
       this.client = new K8s.Client();
diff --git a/src/k8s-engine/k3sHelper.ts b/src/k8s-engine/k3sHelper.ts
@@ -1,11 +1,11 @@
-import childProcess from 'child_process';
 import { Console } from 'console';
 import crypto from 'crypto';
 import events from 'events';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
 import stream from 'stream';
+import tls from 'tls';
 import util from 'util';
 
 import fetch from 'node-fetch';
@@ -14,6 +14,7 @@ import XDGAppPaths from 'xdg-app-paths';
 import { KubeConfig } from '@kubernetes/client-node';
 import yaml from 'yaml';
 
+import * as childProcess from '../utils/childProcess';
 import Logging from '../utils/logging';
 import resources from '../resources';
 import DownloadProgressListener from '../utils/DownloadProgressListener';
@@ -356,6 +357,69 @@ export default class K3sHelper extends events.EventEmitter {
     }
   }
 
+  /**
+   * Wait the K3s server to be ready after starting up.
+   *
+   * This will check that the proper TLS certificate is generated by K3s; this
+   * is required as if the VM IP address changes, K3s will use a certificate
+   * that is only valid for the old address for a short while.  If we attempt to
+   * communicate with the cluster at this point, things will fail.
+   *
+   * @param getHost A function to return the IP address that K3s will listen on
+   *                internally.  This may be called multiple times, as the
+   *                address may not be ready yet.
+   * @param port The port that K3s will listen on.
+   */
+  async waitForServerReady(getHost: () => Promise<string|undefined>, port = 6443): Promise<void> {
+    let host: string | undefined;
+
+    console.log(`Waiting for K3s server to be ready on port ${ port }...`);
+    while (true) {
+      try {
+        host = await getHost();
+
+        if (typeof host === 'undefined') {
+          await util.promisify(setTimeout)(500);
+          continue;
+        }
+
+        await new Promise<void>((resolve, reject) => {
+          const socket = tls.connect(
+            {
+              host, port, rejectUnauthorized: false
+            },
+            () => {
+              const { subjectaltname } = socket.getPeerCertificate();
+              const names = subjectaltname.split(',').map( s => s.trim());
+              const acceptable = [`IP Address:${ host }`, `DNS:${ host }`];
+
+              if (names.some(name => acceptable.includes(name))) {
+                // We got a certificate with a SubjectAltName that includes the
+                // host we're looking for.
+                resolve();
+              }
+              reject({ code: 'ENOHOST' });
+            });
+
+          socket.on('error', reject);
+        });
+        break;
+      } catch (error) {
+        switch (error.code) {
+        case 'ENOHOST':
+        case 'ECONNREFUSED':
+        case 'ECONNRESET':
+          break;
+        default:
+          // Unrecognized error; log but continue waiting.
+          console.error(error);
+        }
+        await util.promisify(setTimeout)(1_000);
+      }
+    }
+    console.log(`The K3s server is ready on ${ host }:${ port }.`);
+  }
+
   /**
    * Find the home directory, in a way that is compatible with the
    * @kubernetes/client-node package.
@@ -426,58 +490,23 @@ export default class K3sHelper extends events.EventEmitter {
   /**
    * Update the user's kubeconfig such that the K3s context is available and
    * set as the current context.  This assumes that K3s is already running.
+   *
+   * @param configReader A function that returns the kubeconfig from the K3s VM.
    */
-  async updateKubeconfig(spawnExecutable: string, ...spawnArgs: string[]): Promise<void> {
+  async updateKubeconfig(configReader: () => Promise<string>): Promise<void> {
     const contextName = 'rancher-desktop';
     const workDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'rancher-desktop-kubeconfig-'));
 
     try {
       const workPath = path.join(workDir, 'kubeconfig');
-      const workFile = await fs.promises.open(workPath, 'w+', 0o600);
-
-      try {
-        const k3sOptions: childProcess.SpawnOptions = { stdio: ['ignore', workFile.fd, 'inherit'] };
-        const k3sChild = childProcess.spawn(spawnExecutable, spawnArgs, k3sOptions);
-
-        console.log('Fetching K3s kubeconfig...');
-        await new Promise<void>((resolve, reject) => {
-          k3sChild.on('error', reject);
-          k3sChild.on('exit', (status, signal) => {
-            if (status === 0) {
-              return resolve();
-            }
-            const message = status ? `status ${ status }` : `signal ${ signal }`;
-
-            reject(new Error(`Error getting kubeconfig: exited with ${ message }`));
-          });
-        });
-      } finally {
-        await workFile.close();
-      }
-
-      // On Windows repeat until the kubeconfig file is readable
-      let delay = 0; // msec
-      const delayIncrement = 200;
-      const maxDelay = 10_000;
-
-      while (delay < maxDelay) {
-        try {
-          await fs.promises.readFile(workPath, { encoding: 'utf-8' });
-          break;
-        } catch (err) {
-          console.log(`Error reading ${ workPath }: ${ err }`);
-          console.log(`Waiting for ${ delay / 1000.0 } sec`);
-          delay += delayIncrement;
-          await util.promisify(setTimeout)(delay);
-        }
-      }
 
       // For some reason, using KubeConfig.loadFromFile presents permissions
       // errors; doing the same ourselves seems to work better.  Since the file
       // comes from the WSL container, it must not contain any paths, so there
-      // is no need to fix it up.
+      // is no need to fix it up.  This also lets us use an external function to
+      // read the kubeconfig.
       const workConfig = new KubeConfig();
-      const workContents = await fs.promises.readFile(workPath, { encoding: 'utf-8' });
+      const workContents = await configReader();
 
       workConfig.loadFromString(workContents);
       // @kubernetes/client-node deosn't have an API to modify the configs...
@@ -534,20 +563,8 @@ export default class K3sHelper extends events.EventEmitter {
       // The config file we modified might not be the top level one.
       // Update the current context.
       console.log('Setting default context...');
-      await new Promise<void>((resolve, reject) => {
-        const child = childProcess.spawn(
-          resources.executable('kubectl'),
-          ['config', 'use-context', contextName],
-          { stdio: 'inherit' });
-
-        child.on('error', reject);
-        child.on('exit', (status, signal) => {
-          if (status !== 0 || signal !== null) {
-            reject(new Error(`kubectl set-context returned with ${ [status, signal] }`));
-          }
-          resolve();
-        });
-      });
+      await childProcess.spawnFile(
+        resources.executable('kubectl'), ['config', 'use-context', contextName]);
     } finally {
       await fs.promises.rmdir(workDir, { recursive: true, maxRetries: 10 });
     }
diff --git a/src/k8s-engine/wsl.ts b/src/k8s-engine/wsl.ts
@@ -215,6 +215,53 @@ export default class WSLBackend extends events.EventEmitter implements K8s.Kuber
     }
   }
 
+  /**
+   * execCommand runs the given command in the K3s WSL environment and returns
+   * the standard output.
+   * @param command The command to execute.
+   * @returns The output of the command.
+   */
+  protected async execCommand(...command: string[]): Promise<string> {
+    const args = ['--distribution', INSTANCE_NAME, '--exec'].concat(command);
+    const { stdout } = await childProcess.spawnFile('wsl.exe', args,
+      {
+        stdio:       ['ignore', 'pipe', await Logging.wsl.fdStream],
+        windowsHide: true
+      });
+
+    return stdout;
+  }
+
+  /** Get the IPv4 address of the VM, assuming it's already up. */
+  protected get ipAddress(): Promise<string | undefined> {
+    return (async() => {
+      // Get the routing map structure
+      const state = await this.execCommand('cat', '/proc/net/fib_trie');
+
+      // We look for the IP address by:
+      // 1. Convert the structure (text) into lines.
+      // 2. Look for lines followed by "/32 host LOCAL".
+      //    This gives interface addresses.
+      const lines = state
+        .split(/\r?\n+/)
+        .filter((_, i, array) => (array[i + 1] || '').includes('/32 host LOCAL'));
+      // 3. Filter for lines with the shortest prefix; this is needed to reject
+      //    the CNI interfaces.
+      const lengths: [number, string][] = lines.map(line => [line.length - line.trimStart().length, line]);
+      const minLength = Math.min(...lengths.map(([length]) => length));
+      // 4. Drop the tree formatting ("    |-- ").  The result are IP addresses.
+      // 5. Reject loopback addresses.
+      const addresses = lengths
+        .filter(([length]) => length === minLength)
+        .map(([_, address]) => address.replace(/^\s+\|--/, '').trim())
+        .filter(address => !address.startsWith('127.'));
+
+      // Assume the first address is what we want, as the WSL VM only has one
+      // (non-loopback, non-CNI) interface.
+      return addresses[0];
+    })();
+  }
+
   async getBackendInvalidReason(): Promise<K8s.KubernetesError | null> {
     // Check if wsl.exe is available
     try {
@@ -274,6 +321,22 @@ export default class WSLBackend extends events.EventEmitter implements K8s.Kuber
 
       // If we were previously running, stop it now.
       this.process?.kill('SIGTERM');
+      await childProcess.spawnFile('wsl.exe', ['--terminate', INSTANCE_NAME],
+        {
+          stdio:       ['ignore', await Logging.wsl.fdStream, await Logging.wsl.fdStream],
+          windowsHide: true
+        });
+
+      // Temporary workaround: ensure root is mounted as shared -- this will be done later
+      // Right now the builder pod needs to be restarted after the remount
+      // TODO: When this code is removed, make `client.getActivePod` protected again.
+      await childProcess.spawnFile(
+        'wsl.exe',
+        ['--user', 'root', '--distribution', INSTANCE_NAME, 'mount', '--make-shared', '/'],
+        {
+          stdio:       ['ignore', await Logging.wsl.fdStream, await Logging.wsl.fdStream],
+          windowsHide: true,
+        });
 
       // Run run-k3s with NORUN, to set up the environment.
       await childProcess.spawnFile('wsl.exe',
@@ -316,30 +379,9 @@ export default class WSLBackend extends events.EventEmitter implements K8s.Kuber
         }
       });
 
-      // Wait for k3s server; note that we're deliberately sending an HTTP request
-      // to an HTTPS server, and expecting an error response back.
-      while (true) {
-        try {
-          const resp = await fetch('http://localhost:6444');
-
-          if (resp.status === 400) {
-            break;
-          }
-        } catch (e) {
-          if (!['ECONNREFUSED', 'ECONNRESET'].includes(e.code)) {
-            throw e;
-          }
-        }
-        await util.promisify(setTimeout)(500);
-      }
-
-      try {
-        await this.k3sHelper.updateKubeconfig(
-          'wsl.exe', '--distribution', INSTANCE_NAME, '--exec', '/usr/local/bin/kubeconfig');
-      } catch (err) {
-        console.log(`k3sHelper.updateKubeconfig failed: ${ err }. Will retry...`);
-        throw err;
-      }
+      await this.k3sHelper.waitForServerReady(() => this.ipAddress);
+      await this.k3sHelper.updateKubeconfig(
+        () => this.execCommand('/usr/local/bin/kubeconfig'));
 
       this.client = new K8s.Client();
       await this.client.waitForServiceWatcher();
@@ -352,15 +394,6 @@ export default class WSLBackend extends events.EventEmitter implements K8s.Kuber
       // Right now the builder pod needs to be restarted after the remount
       // TODO: When this code is removed, make `client.getActivePod` protected again.
       try {
-        await childProcess.spawnFile(
-          'wsl.exe',
-          ['--user', 'root', '--distribution', INSTANCE_NAME, 'mount', '--make-shared', '/'],
-          {
-            stdio:       ['ignore', await Logging.wsl.fdStream, await Logging.wsl.fdStream],
-            windowsHide: true,
-          });
-        console.log('Waiting for ensuring root is shared');
-        await util.promisify(setTimeout)(60_000);
         await childProcess.spawnFile(
           resources.executable('kim'),
           ['builder', 'install', '--force', '--no-wait'],
