perf: speed up web startup and add progress

.github/workflows/test.yaml

@@ -1,4 +1,4 @@
-name: Test Python package
+name: Test package and web app
 
 on:
   push:
@@ -24,19 +24,67 @@ jobs:
         run: python -m uv pip install --system nox[uv]
       - name: Test with pytest
         run: nox -db uv
-      - name: Upload coverage reports to Codecov
+      - name: Upload Python coverage to Codecov
         uses: codecov/codecov-action@v7
         with:
           use_oidc: true
+          flags: python
+          name: python-${{ matrix.python }}
+
+  web:
+    name: Web tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 22
+      - name: Install web test dependencies
+        run: npm install
+      - name: Run web unit tests with coverage
+        run: npm run test:web
+      - name: Upload web coverage to Codecov
+        uses: codecov/codecov-action@v7
+        with:
+          use_oidc: true
+          files: ./coverage/lcov.info
+          flags: web
+          name: web
+          fail_ci_if_error: true
+      - name: Install Chromium
+        run: npx playwright install --with-deps chromium
+      - name: Run browser end-to-end smoke test
+        run: |
+          python3 -m http.server 8000 --directory docs >/tmp/wenxian-web.log 2>&1 &
+          server_pid=$!
+          trap 'kill "$server_pid"' EXIT
+          for _ in {1..20}; do
+            if curl --fail --silent http://127.0.0.1:8000/ >/dev/null; then
+              break
+            fi
+            sleep 0.25
+          done
+          npm run test:web:e2e
+      - name: Upload browser failure screenshot
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: web-e2e-failure
+          path: web-e2e-failure.png
+          if-no-files-found: ignore
+
   pass:
     name: Pass testing
-    needs: [test]
+    needs: [test, web]
     runs-on: ubuntu-latest
     if: always()
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         uses: re-actors/alls-green@release/v1
         with:
           jobs: ${{ toJSON(needs) }}
+
 permissions:
+  contents: read
   id-token: write

.gitignore

@@ -45,11 +45,16 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
+coverage/
 *.cover
 *.py,cover
 .hypothesis/
 .pytest_cache/
 cover/
+web-e2e-failure.png
+
+# JavaScript dependencies
+node_modules/
 
 # Translations
 *.mo

docs/index.html

@@ -16,6 +16,8 @@
     <meta name="theme-color" content="#03A89E" />
     <link rel="icon" type="image/svg" href="logo.svg" />
     <link rel="preconnect" href="https://cdn.jsdelivr.net" />
+    <link rel="preconnect" href="https://pypi.org" />
+    <link rel="preconnect" href="https://files.pythonhosted.org" />
     <link
       href="
 https://cdn.jsdelivr.net/npm/github-fork-ribbon-css@0.2.3/gh-fork-ribbon.min.css
@@ -48,7 +50,7 @@ <h1 style="margin-bottom: 50px; color: #03a89e">
       />
       wenxian
     </h1>
-    <form style="padding-bottom: 40px" target="_blank" class="flex">
+    <form style="padding-bottom: 20px" target="_blank" class="flex">
       <label class="block" style="width: fit-content">
         <input
           class="block identifier"
@@ -66,6 +68,10 @@ <h1 style="margin-bottom: 50px; color: #03a89e">
         value="Generate BibTeX"
       />
     </form>
+    <div id="progress-container" class="progress-container" hidden>
+      <progress id="progress-bar" max="100" value="0"></progress>
+      <div id="progress-text" class="progress-text" aria-live="polite"></div>
+    </div>
     <div class="examples" id="message">
       <p><b>Examples:</b></p>
       <ul>
@@ -90,7 +96,6 @@ <h1 style="margin-bottom: 50px; color: #03a89e">
         >njzjz/wenxian</a
       >
     </p>
-    <script src="https://cdn.jsdelivr.net/pyodide/v0.25.1/full/pyodide.js"></script>
     <script src="https://cdn.jsdelivr.net/npm/prismjs@1.29.0/prism.min.js"></script>
     <script
       src="https://cdn.jsdelivr.net/npm/prismjs-bibtex@2.1.0/prism-bibtex.js"

docs/pyworker.js

@@ -1,21 +1,39 @@
 const pyodideWorker = new Worker("./webworker.js");
 
 const callbacks = {};
+let workerProgress = {
+  progress: 0,
+  message: "Starting…",
+};
 
 pyodideWorker.onmessage = (event) => {
-  const { id, ...data } = event.data;
-  const onSuccess = callbacks[id];
+  const { id, type, ...data } = event.data;
+
+  if (type === "progress") {
+    if (id === null) {
+      workerProgress = data;
+      for (const callback of Object.values(callbacks)) {
+        callback.onProgress?.(data);
+      }
+    } else {
+      callbacks[id]?.onProgress?.(data);
+    }
+    return;
+  }
+
+  const callback = callbacks[id];
+  if (!callback) return;
   delete callbacks[id];
-  onSuccess(data);
+  callback.onSuccess(data);
 };
 
 const asyncRun = (() => {
   let id = 0; // identify a Promise
-  return (script) => {
-    // the id could be generated more carefully
+  return (script, { onProgress } = {}) => {
     id = (id + 1) % Number.MAX_SAFE_INTEGER;
     return new Promise((onSuccess) => {
-      callbacks[id] = onSuccess;
+      callbacks[id] = { onSuccess, onProgress };
+      onProgress?.(workerProgress);
       pyodideWorker.postMessage({
         python: script,
         id,

docs/style.css

@@ -81,6 +81,34 @@ input:focus {
   margin: 30px 0;
 }
 
+.btn:disabled {
+  cursor: wait;
+  opacity: 0.65;
+}
+
+.progress-container {
+  width: min(360px, 90vw);
+  min-height: 42px;
+  margin: 0 0 18px;
+}
+
+.progress-container[hidden] {
+  display: none;
+}
+
+.progress-container progress {
+  width: 100%;
+  height: 10px;
+  accent-color: #00bfb3;
+}
+
+.progress-text {
+  margin-top: 6px;
+  color: #666;
+  font-size: 13px;
+  text-align: center;
+}
+
 .tips,
 .example {
   color: #7b7b7b;

docs/webworker.js

@@ -1,10 +1,13 @@
 // webworker.js
 
-// Setup your project to serve `py-worker.js`. You should also serve
-// `pyodide.js`, and all its associated `.asm.js`, `.json`,
-// and `.wasm` files as well:
+// Pyodide only runs inside this worker. Keeping it off the main thread avoids
+// downloading and parsing the runtime twice.
 importScripts("https://cdn.jsdelivr.net/pyodide/v0.25.1/full/pyodide.js");
 
+function reportProgress(progress, message, id = null) {
+  self.postMessage({ type: "progress", progress, message, id });
+}
+
 function installLegacyWenxianBrowserShims() {
   self.pyodide.runPython(`
 from importlib.metadata import version
@@ -42,23 +45,32 @@ if Version(version("wenxian")) < Version("0.3.4"):
 }
 
 async function loadPyodideAndPackages() {
+  reportProgress(8, "Loading Python runtime…");
   self.pyodide = await loadPyodide();
+
+  reportProgress(32, "Loading package installer…");
   await self.pyodide.loadPackage("micropip");
   const micropip = self.pyodide.pyimport("micropip");
+
+  reportProgress(45, "Loading wenxian…");
   await micropip.install(["wenxian", "pylatexenc==3.0a21"]);
   installLegacyWenxianBrowserShims();
-  await self.pyodide.loadPackage("sqlite3");
+
+  // sqlite3 used to be loaded here even though the browser code does not use
+  // it. Avoiding that extra package download shortens cold starts.
+  reportProgress(65, "Ready");
 }
-let pyodideReadyPromise = loadPyodideAndPackages();
+const pyodideReadyPromise = loadPyodideAndPackages();
 
 self.onmessage = async (event) => {
   const { id, python } = event.data;
   try {
-    // Initialization errors must be returned to the caller too; otherwise the
-    // page remains stuck on "Fetching..." forever.
     await pyodideReadyPromise;
+    reportProgress(72, "Preparing query…", id);
     await self.pyodide.loadPackagesFromImports(python);
-    let results = await self.pyodide.runPythonAsync(python);
+    reportProgress(82, "Querying literature sources…", id);
+    const results = await self.pyodide.runPythonAsync(python);
+    reportProgress(100, "Done", id);
     self.postMessage({ results, id });
   } catch (error) {
     self.postMessage({ error: error.message || String(error), id });

docs/wenxian.js

@@ -1,8 +1,23 @@
 import { asyncRun } from "./pyworker.js";
 
-async function from_identifier(identifier) {
+const progressContainer = document.getElementById("progress-container");
+const progressBar = document.getElementById("progress-bar");
+const progressText = document.getElementById("progress-text");
+
+function setProgress({ progress, message }) {
+  progressContainer.hidden = false;
+  progressBar.value = progress;
+  progressText.textContent = `${message} ${progress}%`;
+}
+
+function hideProgress() {
+  progressContainer.hidden = true;
+}
+
+async function from_identifier(identifier, onProgress) {
   const pythonIdentifier = JSON.stringify(identifier);
-  const { results, error } = await asyncRun(`
+  const { results, error } = await asyncRun(
+    `
     try:
         from wenxian.from_identifier import async_from_identifier
     except ImportError:
@@ -11,18 +26,25 @@ async function from_identifier(identifier) {
     else:
         reference = await async_from_identifier(${pythonIdentifier})
     reference.bibtex if reference is not None and not reference.is_empty() else None
-    `);
+    `,
+    { onProgress },
+  );
   return { results, error };
 }
 
 document.getElementById("submit").addEventListener("click", function (event) {
   event.preventDefault();
   const message = document.getElementById("message");
-  message.textContent = "Fetching...";
+  message.textContent = "";
   const output_text = document.getElementById("bibtex");
   const output = document.getElementById("output");
+  const submit = document.getElementById("submit");
   const identifier = document.getElementById("identifier").value;
-  from_identifier(identifier).then(({ results, error }) => {
+
+  submit.disabled = true;
+  progressContainer.hidden = false;
+  from_identifier(identifier, setProgress).then(({ results, error }) => {
+    submit.disabled = false;
     if (results) {
       output_text.textContent = results;
       Prism.highlightElement(output_text);
@@ -36,6 +58,7 @@ document.getElementById("submit").addEventListener("click", function (event) {
       output.style.display = "none";
       message.textContent = error;
     }
+    setTimeout(hideProgress, 400);
   });
 });
 

package.json

@@ -0,0 +1,13 @@
+{
+  "name": "wenxian-web-tests",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test:web": "c8 --reporter=text --reporter=lcov --include=docs/*.js node --test tests/web/*.test.mjs",
+    "test:web:e2e": "node tests/web/e2e.mjs"
+  },
+  "devDependencies": {
+    "c8": "^10.1.3",
+    "playwright": "^1.55.0"
+  }
+}

tests/web/e2e.mjs

@@ -0,0 +1,37 @@
+import assert from "node:assert/strict";
+import { chromium } from "playwright";
+
+const target = process.env.WENXIAN_WEB_URL ?? "http://127.0.0.1:8000/";
+const browser = await chromium.launch({ headless: true });
+const page = await browser.newPage();
+const pageErrors = [];
+
+page.on("pageerror", (error) => pageErrors.push(error));
+page.on("console", (message) => {
+  if (message.type() === "error") console.error(`[browser] ${message.text()}`);
+});
+
+try {
+  await page.goto(target, { waitUntil: "domcontentloaded", timeout: 60_000 });
+  await page.locator("#identifier").fill("10.1063/5.0155600");
+  await page.locator("#submit").click();
+
+  await page.locator("#progress-container").waitFor({ state: "visible" });
+  await page.waitForFunction(
+    () => Number(document.querySelector("#progress-bar")?.value ?? 0) >= 45,
+    undefined,
+    { timeout: 120_000 },
+  );
+
+  await page.locator("#output").waitFor({ state: "visible", timeout: 180_000 });
+  const bibtex = (await page.locator("#bibtex").textContent()) ?? "";
+  assert.match(bibtex, /^@/);
+  assert.match(bibtex.toLowerCase(), /10\.1063\/5\.0155600/);
+  assert.equal(await page.locator("#submit").isDisabled(), false);
+  assert.deepEqual(pageErrors, []);
+} catch (error) {
+  await page.screenshot({ path: "web-e2e-failure.png", fullPage: true });
+  throw error;
+} finally {
+  await browser.close();
+}