When core crashes, restart workers and show a UI

quadratic-client/src/app/gridGL/loadAssets.ts

@@ -8,7 +8,7 @@ export const bitmapFonts = ['OpenSans', 'OpenSans-Bold', 'OpenSans-Italic', 'Ope
 
 const TIMEOUT = 10000;
 
-let assetsLoaded = false;
+export let assetsLoaded = false;
 
 async function loadFont(fontName: string): Promise<void> {
   const font = new FontFaceObserver(fontName);

quadratic-client/src/app/gridGL/pixiApp/PixiApp.ts

@@ -1,6 +1,6 @@
+import { defaultEditorInteractionState } from '@/app/atoms/editorInteractionStateAtom';
 import './pixiApp.css';
 
-import { defaultEditorInteractionState } from '@/app/atoms/editorInteractionStateAtom';
 import { events } from '@/app/events/events';
 import {
   copyToClipboardEvent,
@@ -48,7 +48,7 @@ export class PixiApp {
   // Used to track whether we're done with the first render (either before or
   // after init is called, depending on timing).
   private waitingForFirstRender?: Function;
-  private alreadyRendered = false;
+  alreadyRendered = false;
 
   // todo: UI should be pulled out and separated into its own class
 
@@ -143,6 +143,16 @@ export class PixiApp {
     });
   };
 
+  refresh = (): Promise<void> => {
+    return new Promise((resolve) => {
+      this.rebuild();
+      renderWebWorker.sendBitmapFonts();
+      urlParams.init();
+      this.alreadyRendered = false;
+      this.waitingForFirstRender = resolve;
+    });
+  };
+
   // called after RenderText has no more updates to send
   firstRenderComplete = () => {
     if (this.waitingForFirstRender) {

quadratic-client/src/app/ui/QuadraticUI.tsx

@@ -7,7 +7,6 @@ import {
   editorInteractionStateShowShareFileMenuAtom,
 } from '@/app/atoms/editorInteractionStateAtom';
 import { presentationModeAtom } from '@/app/atoms/gridSettingsAtom';
-import { events } from '@/app/events/events';
 import { pixiAppSettings } from '@/app/gridGL/pixiApp/PixiAppSettings';
 import QuadraticGrid from '@/app/gridGL/QuadraticGrid';
 import { isEmbed } from '@/app/helpers/isEmbed';
@@ -30,14 +29,11 @@ import { QuadraticSidebar } from '@/app/ui/QuadraticSidebar';
 import { UpdateAlertVersion } from '@/app/ui/UpdateAlertVersion';
 import { useRootRouteLoaderData } from '@/routes/_root';
 import { DialogRenameItem } from '@/shared/components/DialogRenameItem';
-import { EmptyPage } from '@/shared/components/EmptyPage';
 import { ShareFileDialog } from '@/shared/components/ShareDialog';
 import { UserMessage } from '@/shared/components/UserMessage';
 import { COMMUNITY_A1_FILE_UPDATE_URL } from '@/shared/constants/urls';
 import { useRemoveInitialLoadingUI } from '@/shared/hooks/useRemoveInitialLoadingUI';
-import { Button } from '@/shared/shadcn/ui/button';
-import { CrossCircledIcon } from '@radix-ui/react-icons';
-import { useEffect, useMemo, useState } from 'react';
+import { useEffect, useMemo } from 'react';
 import { useNavigation, useParams } from 'react-router';
 import { useRecoilState, useRecoilValue } from 'recoil';
 
@@ -54,15 +50,6 @@ export default function QuadraticUI() {
   const permissions = useRecoilValue(editorInteractionStatePermissionsAtom);
   const canEditFile = useMemo(() => hasPermissionToEditFile(permissions), [permissions]);
 
-  const [error, setError] = useState<{ from: string; error: Error | unknown } | null>(null);
-  useEffect(() => {
-    const handleError = (from: string, error: Error | unknown) => setError({ from, error });
-    events.on('coreError', handleError);
-    return () => {
-      events.off('coreError', handleError);
-    };
-  }, []);
-
   useRemoveInitialLoadingUI();
 
   // Show negative_offsets warning if present in URL (the result of an imported
@@ -84,19 +71,6 @@ export default function QuadraticUI() {
     }
   }, []);
 
-  if (error) {
-    return (
-      <EmptyPage
-        title="Quadratic crashed"
-        description="Something went wrong. Our team has been notified of this issue. Please reload the application to continue."
-        Icon={CrossCircledIcon}
-        actions={<Button onClick={() => window.location.reload()}>Reload</Button>}
-        error={error.error}
-        source={error.from}
-      />
-    );
-  }
-
   return (
     <div
       id="quadratic-ui"

quadratic-client/src/app/web-workers/Singleton.ts

@@ -0,0 +1,30 @@
+import { debugFlag } from '@/app/debugFlags/debugFlags';
+
+/**
+ * Singleton class to ensure only one instance of a class is created.
+ * Allows for easy refreshing of the singleton instance.
+ * Intended to be used for web worker messengers, but can be used for any class.
+ * Meant to be extended by the class it is managing.
+ */
+export class Singleton {
+  private static instances = new Map<Function, any>();
+
+  constructor() {
+    const constructor = this.constructor;
+
+    if (Singleton.instances.has(constructor)) {
+      return Singleton.instances.get(constructor);
+    }
+
+    Singleton.instances.set(constructor, this);
+
+    if (debugFlag('debugWebWorkers')) console.log(`[Singleton] ${constructor.name} initialized`);
+  }
+
+  refreshSingleton() {
+    const constructor = this.constructor;
+    Singleton.instances.delete(constructor);
+
+    if (debugFlag('debugWebWorkers')) console.log(`[Singleton] ${constructor.name} refreshed`);
+  }
+}

quadratic-client/src/app/web-workers/javascriptWebWorker/javascriptWebWorker.ts

@@ -58,6 +58,11 @@ class JavascriptWebWorker {
     quadraticCore.sendJavascriptInit(JavascriptCoreChannel.port2);
   }
 
+  terminate() {
+    this.worker?.terminate();
+    this.worker = undefined;
+  }
+
   cancelExecution = () => {
     mixpanel.track('[JavascriptWebWorker].restartFromUser');
     this.init();

quadratic-client/src/app/web-workers/multiplayerWebWorker/multiplayer.ts

@@ -18,6 +18,9 @@ import type {
   MultiplayerState,
 } from '@/app/web-workers/multiplayerWebWorker/multiplayerClientMessages';
 import type { MultiplayerUser, ReceiveRoom } from '@/app/web-workers/multiplayerWebWorker/multiplayerTypes';
+import { multiplayerClient } from '@/app/web-workers/multiplayerWebWorker/worker/multiplayerClient';
+import { multiplayerCore } from '@/app/web-workers/multiplayerWebWorker/worker/multiplayerCore';
+import { multiplayerServer } from '@/app/web-workers/multiplayerWebWorker/worker/multiplayerServer';
 import { quadraticCore } from '@/app/web-workers/quadraticCore/quadraticCore';
 import type { User } from '@/auth/auth';
 import { authClient } from '@/auth/auth';
@@ -39,6 +42,7 @@ export class Multiplayer {
 
   private anonymous?: boolean;
   private fileId?: string;
+  private user?: User;
   private jwt?: string | void;
 
   private codeRunning?: SheetPosTS[];
@@ -92,6 +96,25 @@ export class Multiplayer {
     };
   }
 
+  isInitialized() {
+    return this.worker !== undefined;
+  }
+
+  terminate() {
+    this.worker?.terminate();
+    this.worker = undefined;
+    this.refreshMessengers();
+    this.state = 'startup';
+
+    if (debugFlag('debugWebWorkers')) console.log('[Multiplayer] terminated');
+  }
+
+  refreshMessengers() {
+    multiplayerClient.refreshSingleton();
+    multiplayerCore.refreshSingleton();
+    multiplayerServer.refreshSingleton();
+  }
+
   private pythonState = (_state: LanguageState, current?: CodeRun, awaitingExecution?: CodeRun[]) => {
     const codeRunning: SheetPosTS[] = [];
     if (current) {
@@ -173,6 +196,7 @@ export class Multiplayer {
     const channel = new MessageChannel();
 
     this.fileId = fileId;
+    this.user = user;
     this.anonymous = anonymous;
     if (!this.anonymous) {
       await this.addJwtCookie();
@@ -200,6 +224,16 @@ export class Multiplayer {
       channel.port1
     );
     quadraticCore.initMultiplayer(channel.port2);
+
+    if (debugFlag('debugWebWorkers')) console.log('[Multiplayer] Initialized');
+  }
+
+  // used to re-initialize the multiplayer web worker after a core error,
+  // using existing fileId, user, and anonymous state
+  async reInit() {
+    if (this.fileId && this.user && this.anonymous !== undefined) {
+      await this.init(this.fileId, this.user, this.anonymous);
+    }
   }
 
   // used to pre-populate useMultiplayerUsers.tsx

quadratic-client/src/app/web-workers/multiplayerWebWorker/worker/multiplayerClient.ts

@@ -11,15 +11,17 @@ import type {
 import type { MessageUserUpdate, ReceiveRoom } from '@/app/web-workers/multiplayerWebWorker/multiplayerTypes';
 import { multiplayerCore } from '@/app/web-workers/multiplayerWebWorker/worker/multiplayerCore';
 import { cellEditDefault, multiplayerServer } from '@/app/web-workers/multiplayerWebWorker/worker/multiplayerServer';
+import { Singleton } from '@/app/web-workers/Singleton';
 
 declare var self: WorkerGlobalScope & typeof globalThis;
 
-class MultiplayerClient {
+class MultiplayerClient extends Singleton {
   // messages pending a reconnect
   private waitingForConnection: Record<number, Function> = {};
   private id = 0;
 
   constructor() {
+    super();
     self.onmessage = this.handleMessage;
   }
 

quadratic-client/src/app/web-workers/multiplayerWebWorker/worker/multiplayerCore.ts

@@ -9,8 +9,9 @@ import type {
 } from '@/app/web-workers/multiplayerWebWorker/multiplayerCoreMessages';
 import type { ReceiveTransaction, ReceiveTransactions } from '@/app/web-workers/multiplayerWebWorker/multiplayerTypes';
 import { multiplayerServer } from '@/app/web-workers/multiplayerWebWorker/worker/multiplayerServer';
+import { Singleton } from '@/app/web-workers/Singleton';
 
-class MultiplayerCore {
+class MultiplayerCore extends Singleton {
   private coreMessagePort?: MessagePort;
 
   init(coreMessagePort: MessagePort) {

quadratic-client/src/app/web-workers/multiplayerWebWorker/worker/multiplayerServer.ts

@@ -27,6 +27,7 @@ import {
 } from '@/app/web-workers/multiplayerWebWorker/proto/transaction';
 import { multiplayerClient } from '@/app/web-workers/multiplayerWebWorker/worker/multiplayerClient';
 import { multiplayerCore } from '@/app/web-workers/multiplayerWebWorker/worker/multiplayerCore';
+import { Singleton } from '@/app/web-workers/Singleton';
 import type { User } from '@/auth/auth';
 import * as Sentry from '@sentry/react';
 
@@ -55,7 +56,7 @@ export const cellEditDefault = (): CellEdit => ({
   inline_code_editor: false,
 });
 
-export class MultiplayerServer {
+export class MultiplayerServer extends Singleton {
   private websocket?: WebSocket;
 
   private _state: MultiplayerState = 'startup';

quadratic-client/src/app/web-workers/pythonWebWorker/pythonWebWorker.ts

@@ -58,6 +58,11 @@ class PythonWebWorker {
     quadraticCore.sendPythonInit(pythonCoreChannel.port2);
   }
 
+  terminate() {
+    this.worker?.terminate();
+    this.worker = undefined;
+  }
+
   cancelExecution = () => {
     mixpanel.track('[PythonWebWorker].restartFromUser');
     this.init();

quadratic-client/src/app/web-workers/quadraticCore/quadraticCore.ts

@@ -43,7 +43,6 @@ import type {
   SheetRect,
   Validation,
 } from '@/app/quadratic-core-types';
-import { SheetContentCache, SheetDataTablesCache } from '@/app/quadratic-core/quadratic_core';
 import { fromUint8Array } from '@/app/shared/utils/Uint8Array';
 import type {
   ClientCoreGetCellFormatSummary,
@@ -76,7 +75,14 @@ import type {
   CoreClientSummarizeSelection,
   CoreClientValidateInput,
 } from '@/app/web-workers/quadraticCore/coreClientMessages';
+import { coreClient } from '@/app/web-workers/quadraticCore/worker/coreClient';
+import { coreConnection } from '@/app/web-workers/quadraticCore/worker/coreConnection';
+import { coreJavascript } from '@/app/web-workers/quadraticCore/worker/coreJavascript';
+import { coreMultiplayer } from '@/app/web-workers/quadraticCore/worker/coreMultiplayer';
+import { corePython } from '@/app/web-workers/quadraticCore/worker/corePython';
+import { coreRender } from '@/app/web-workers/quadraticCore/worker/coreRender';
 import { renderWebWorker } from '@/app/web-workers/renderWebWorker/renderWebWorker';
+import { initWorkers, stopWorkers } from '@/app/web-workers/workers';
 import { authClient } from '@/auth/auth';
 
 class QuadraticCore {
@@ -94,10 +100,31 @@ class QuadraticCore {
       this.worker.onmessage = this.handleMessage;
       this.worker.onerror = (e) => console.warn(`[core.worker] error: ${e.message}`, e);
 
+      if (debugFlag('debugWebWorkers')) console.log('[quadraticCore] worker initialized');
+
       this.sendInit();
     }
   }
 
+  isInitialized() {
+    return this.worker !== undefined;
+  }
+
+  terminateWorker() {
+    this.worker?.terminate();
+    this.worker = undefined;
+    this.refreshMessengers();
+  }
+
+  refreshMessengers() {
+    coreClient.refreshSingleton();
+    coreConnection.refreshSingleton();
+    coreJavascript.refreshSingleton();
+    coreMultiplayer.refreshSingleton();
+    corePython.refreshSingleton();
+    coreRender.refreshSingleton();
+  }
+
   private handleMessage = async (e: MessageEvent<CoreClientMessage>) => {
     if (debugFlag('debugWebWorkersMessages')) console.log(`[quadraticCore] message: ${e.data.type}`);
 
@@ -205,13 +232,15 @@ class QuadraticCore {
       if (debugFlag('debug')) {
         console.error('[quadraticCore] core error', e.data.from, e.data.error);
       }
+      stopWorkers();
+      initWorkers();
       events.emit('coreError', e.data.from, e.data.error);
       return;
     } else if (e.data.type === 'coreClientContentCache') {
-      events.emit('contentCache', e.data.sheetId, new SheetContentCache(e.data.contentCache));
+      // events.emit('contentCache', e.data.sheetId, new SheetContentCache(e.data.contentCache));
       return;
     } else if (e.data.type === 'coreClientDataTablesCache') {
-      events.emit('dataTablesCache', e.data.sheetId, new SheetDataTablesCache(e.data.dataTablesCache));
+      // events.emit('dataTablesCache', e.data.sheetId, new SheetDataTablesCache(e.data.dataTablesCache));
       return;
     }
 

quadratic-client/src/app/web-workers/quadraticCore/worker/coreClient.ts

@@ -20,6 +20,7 @@ import { coreJavascript } from '@/app/web-workers/quadraticCore/worker/coreJavas
 import { coreMultiplayer } from '@/app/web-workers/quadraticCore/worker/coreMultiplayer';
 import { corePython } from '@/app/web-workers/quadraticCore/worker/corePython';
 import { offline } from '@/app/web-workers/quadraticCore/worker/offline';
+import { Singleton } from '@/app/web-workers/Singleton';
 
 declare var self: WorkerGlobalScope &
   typeof globalThis & {
@@ -61,7 +62,7 @@ declare var self: WorkerGlobalScope &
     sendContentCache: (sheetId: string, contentCache: Uint8Array) => void;
   };
 
-class CoreClient {
+class CoreClient extends Singleton {
   private id = 0;
   private waitingForResponse: Record<number, Function> = {};
   env: Record<string, string> = {};

quadratic-client/src/app/web-workers/quadraticCore/worker/coreConnection.ts

@@ -4,6 +4,7 @@ import type { CodeRun } from '@/app/web-workers/CodeRun';
 import type { LanguageState } from '@/app/web-workers/languageTypes';
 import { core } from '@/app/web-workers/quadraticCore/worker/core';
 import { coreClient } from '@/app/web-workers/quadraticCore/worker/coreClient';
+import { Singleton } from '@/app/web-workers/Singleton';
 
 declare var self: WorkerGlobalScope &
   typeof globalThis & {
@@ -18,7 +19,7 @@ declare var self: WorkerGlobalScope &
     ) => void;
   };
 
-class CoreConnection {
+class CoreConnection extends Singleton {
   controller: AbortController = new AbortController();
 
   lastTransactionId?: string;

quadratic-client/src/app/web-workers/quadraticCore/worker/coreJavascript.ts

@@ -6,13 +6,14 @@ import type {
   JavascriptCoreMessage,
 } from '@/app/web-workers/javascriptWebWorker/javascriptCoreMessages';
 import { core } from '@/app/web-workers/quadraticCore/worker/core';
+import { Singleton } from '@/app/web-workers/Singleton';
 
 declare var self: WorkerGlobalScope &
   typeof globalThis & {
     sendRunJavascript: (transactionId: string, x: number, y: number, sheetId: string, code: string) => void;
   };
 
-class CoreJavascript {
+class CoreJavascript extends Singleton {
   private coreJavascriptPort?: MessagePort;
 
   // last running transaction (used to cancel execution)