fix(Crowdin): whitespace loss after translation

Crowdin's Website Translator strips whitespace between text nodes and adjacent inline elements (e.g. <a>, <em>, <strong>) during translation. This commit captures those whitespace boundaries before translation and restores them after mutations settle, using a MutationObserver with debounced timer. Also re-hooks on i18next languageChanged events for subsequent language switches.

Also fixes example asset paths to reference installed node_modules instead of a relative dist/ directory, adds a build.js asset-copy step for the Jekyll example, and scopes the sphinx rstcheck lint to the source/ directory.

Author: ReenigneArcher

examples/jekyll/_config.yml

@@ -24,9 +24,9 @@ page-col: "#303436"
 text-col: "#e4e4e4"
 mobile-theme-col: "#05FF3B"
 site-css:
-  - "../dist/crowdin-bootstrap-css.css"
+  - "/assets/shared-web/crowdin-bootstrap-css.css"
 site-js:
-  - "../dist/crowdin.js"
+  - "/assets/shared-web/crowdin.js"
   - "/assets/js/crowdin-init.js"
 
 # Advanced settings

examples/jekyll/build.js

@@ -0,0 +1,17 @@
+const fs = require('node:fs');
+const path = require('node:path');
+
+const exampleDir = __dirname;
+const outputRoot = process.env.READTHEDOCS_OUTPUT || path.join(exampleDir, 'build');
+const assetDir = path.join(outputRoot, 'html', 'jekyll', 'assets', 'shared-web');
+const sharedWebDist = path.join(exampleDir, 'node_modules', '@lizardbyte', 'shared-web', 'dist');
+const sharedWebAssets = [
+    'crowdin.js',
+    'crowdin-bootstrap-css.css',
+];
+
+fs.mkdirSync(assetDir, { recursive: true });
+
+sharedWebAssets.forEach((asset) => {
+    fs.copyFileSync(path.join(sharedWebDist, asset), path.join(assetDir, asset));
+});

examples/jekyll/package.json

@@ -12,6 +12,8 @@
     "postinstall": "npm-run-all postinstall:*",
     "postinstall:bundler": "echo 'Installing bundler...' && gem install bundler",
     "postinstall:bundle": "echo 'Installing Jekyll dependencies...' && bundle install",
-    "build": "bundle exec jekyll build --verbose --destination ${READTHEDOCS_OUTPUT:-build/}html/jekyll"
+    "build": "npm-run-all build:site build:assets",
+    "build:assets": "node build.js",
+    "build:site": "bundle exec jekyll build --verbose --destination ${READTHEDOCS_OUTPUT:-build/}html/jekyll"
   }
 }

examples/sphinx/package.json

@@ -8,6 +8,6 @@
   "scripts": {
     "postinstall": "echo 'Installing Python dependencies...' && python -m pip install -r requirements.txt --no-cache-dir",
     "build": "python -m sphinx -b html source ${READTHEDOCS_OUTPUT:-build/}html/sphinx/html",
-    "lint": "python -m rstcheck -r ."
+    "lint": "python -m rstcheck -r source"
   }
 }

examples/sphinx/source/conf.py

@@ -60,19 +60,22 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = [
+    '_static',
+    '../node_modules/@lizardbyte/shared-web/dist',
+]
 
 # These paths are either relative to html_static_path
 # or fully qualified paths (eg. https://...)
 html_css_files = [
     # use jsdelivr for an easy way to include the css
     # 'https://cdn.jsdelivr.net/npm/@lizardbyte/shared-web@latest/dist/crowdin-furo-css.css',
-    '../../../dist/crowdin-furo-css.css',  # crowdin style from the readthedocs build
+    'crowdin-furo-css.css',  # crowdin style from the installed shared-web package
 ]
 html_js_files = [
     # use jsdelivr for an easy way to include the script
     # 'https://cdn.jsdelivr.net/npm/@lizardbyte/shared-web@latest/dist/crowdin.js',
-    '../../../dist/crowdin.js',  # crowdin language selector from the readthedocs build
+    'crowdin.js',  # crowdin language selector from the installed shared-web package
     'js/crowdin.js',  # initialize crowdin language selector
 ]
 

src/js/crowdin.js

@@ -9,8 +9,120 @@ const loadScript = require('./load-script');
  * @type {string}
  */
 const CROWDIN_DIST_MIRROR = 'https://cdn.jsdelivr.net/gh/LizardByte/i18n@dist';
+const CROWDIN_TRANSLATION_SETTLE_DELAY_MS = 50;
 const CROWDIN_PLATFORM_STYLING_MAX_ATTEMPTS = 100;
 const CROWDIN_PLATFORM_STYLING_RETRY_DELAY_MS = 50;
+const CROWDIN_INLINE_ELEMENT_SELECTOR = [
+    'a',
+    'abbr',
+    'b',
+    'cite',
+    'code',
+    'del',
+    'em',
+    'i',
+    'ins',
+    'kbd',
+    'mark',
+    'q',
+    's',
+    'samp',
+    'small',
+    'span',
+    'strong',
+    'sub',
+    'sup',
+    'time',
+    'u',
+    'var',
+].join(',');
+
+/**
+ * Records whitespace that separates text from inline elements before Crowdin translates the page.
+ * @returns {Array<{node: Text, leading: boolean, trailing: boolean}>} Recorded text-node boundaries.
+ */
+function _captureCrowdinWhitespaceBoundaries() {
+    const boundaries = [];
+    const walker = document.createTreeWalker(document.body, globalThis.NodeFilter.SHOW_TEXT);
+    let node = walker.nextNode();
+
+    while (node !== null) {
+        const previousIsInline = node.previousSibling instanceof globalThis.Element &&
+            node.previousSibling.matches(CROWDIN_INLINE_ELEMENT_SELECTOR);
+        const nextIsInline = node.nextSibling instanceof globalThis.Element &&
+            node.nextSibling.matches(CROWDIN_INLINE_ELEMENT_SELECTOR);
+        const leading = previousIsInline && /^\s/.test(node.data);
+        const trailing = nextIsInline && /\s$/.test(node.data);
+
+        if (leading || trailing) {
+            boundaries.push({ node, leading, trailing });
+        }
+
+        node = walker.nextNode();
+    }
+
+    return boundaries;
+}
+
+/**
+ * Restores whitespace that Crowdin removed from translated text-node boundaries.
+ * @param {Array<{node: Text, leading: boolean, trailing: boolean}>} boundaries Recorded text-node boundaries.
+ */
+function _restoreCrowdinWhitespaceBoundaries(boundaries) {
+    boundaries.forEach((boundary) => {
+        if (!boundary.node.isConnected) return;
+
+        if (boundary.leading && !/^\s/.test(boundary.node.data)) {
+            boundary.node.data = ' ' + boundary.node.data;
+        }
+        if (boundary.trailing && !/\s$/.test(boundary.node.data)) {
+            boundary.node.data += ' ';
+        }
+    });
+}
+
+/**
+ * Creates the Website Translator callback that repairs whitespace after translation mutations settle.
+ * @param {Array<{node: Text, leading: boolean, trailing: boolean}>} boundaries Recorded text-node boundaries.
+ * @returns {Function} Website Translator callback.
+ */
+function _createCrowdinTranslationCallback(boundaries) {
+    let listeningForLanguageChanges = false;
+    let translationObserver;
+    let restoreTimer;
+
+    function restoreAfterTranslationSettles() {
+        globalThis.clearTimeout(restoreTimer);
+        restoreTimer = globalThis.setTimeout(function() {
+            translationObserver.disconnect();
+            translationObserver = undefined;
+            restoreTimer = undefined;
+            _restoreCrowdinWhitespaceBoundaries(boundaries);
+        }, CROWDIN_TRANSLATION_SETTLE_DELAY_MS);
+    }
+
+    function observeTranslationMutations() {
+        translationObserver?.disconnect();
+        translationObserver = new globalThis.MutationObserver(restoreAfterTranslationSettles);
+        translationObserver.observe(document.body, {
+            characterData: true,
+            childList: true,
+            subtree: true,
+        });
+        restoreAfterTranslationSettles();
+    }
+
+    return function waitForCrowdinTranslation() {
+        const i18next = globalThis.i18nextify?.i18next;
+
+        if (!listeningForLanguageChanges && typeof i18next?.on === 'function') {
+            i18next.on('languageChanged', observeTranslationMutations);
+            listeningForLanguageChanges = true;
+        }
+
+        observeTranslationMutations();
+    };
+}
 
 /**
  * Monkey-patches globalThis.fetch to redirect Crowdin distribution requests to
@@ -171,8 +283,11 @@ function initCrowdIn(project = 'LizardByte', platform = null) {
         let currentBaseUrl = globalThis.location.origin;
 
         // Initialize Crowdin translator
+        const whitespaceBoundaries = _captureCrowdinWhitespaceBoundaries();
+
         globalThis.proxyTranslator.init({
             baseUrl: currentBaseUrl,
+            callback: _createCrowdinTranslationCallback(whitespaceBoundaries),
             distribution: projectSettings[project].distribution,
             defaultLanguage: "en",
             languageTitles: languageTitles,

tests/crowdin.test.js

@@ -86,6 +86,7 @@ describe('initCrowdIn', () => {
         jest.clearAllMocks();
         jest.useRealTimers();
         delete globalThis.window.proxyTranslator;
+        delete globalThis.window.i18nextify;
         delete globalThis._crowdinMirrorInstalled;
     });
 
@@ -114,6 +115,97 @@ describe('initCrowdIn', () => {
         );
     });
 
+    it('should only pass supported Website Translator options', () => {
+        initCrowdIn();
+
+        // Simulate script loading
+        jest.runAllTimers();
+
+        const options = globalThis.proxyTranslator.init.mock.calls[0][0];
+        expect(Object.keys(options).sort()).toEqual([
+            'baseUrl',
+            'callback',
+            'defaultLanguage',
+            'distribution',
+            'languageRoutingMethod',
+            'languageTitles',
+            'position',
+            'poweredBy',
+            'showDefaultLanguageInUrl',
+            'submenuPosition',
+        ]);
+    });
+
+    it('should restore whitespace around translated inline elements', () => {
+        globalThis.document.body.innerHTML = `
+            <p id="translated">
+                GitHub <em>Discussions</em> are available. <strong>Yearly:</strong> <strong>$14.99</strong>, billed.
+            </p>
+            <p id="detached">Before <i>removed</i></p>
+        `;
+
+        initCrowdIn();
+        jest.runAllTimers();
+
+        const options = globalThis.proxyTranslator.init.mock.calls[0][0];
+        const translated = globalThis.document.getElementById('translated');
+        const emphasis = translated.querySelector('em');
+        const strongElements = translated.querySelectorAll('strong');
+        const detachedBoundary = globalThis.document.querySelector('#detached i').previousSibling;
+
+        emphasis.previousSibling.data = emphasis.previousSibling.data.trimEnd();
+        emphasis.nextSibling.data = emphasis.nextSibling.data.trim();
+        strongElements[0].nextSibling.data = '';
+        detachedBoundary.remove();
+
+        options.callback();
+        options.callback();
+        jest.runOnlyPendingTimers();
+
+        expect(translated.textContent.trim()).toBe(
+            'GitHub Discussions are available. Yearly: $14.99, billed.'
+        );
+    });
+
+    it('should restore whitespace after asynchronous language changes', async () => {
+        globalThis.document.body.innerHTML = '<p id="translated">Use <a href="#">this link</a> here.</p>';
+        const languageChangedHandlers = [];
+        globalThis.window.i18nextify = {
+            i18next: {
+                on: jest.fn((event, handler) => {
+                    expect(event).toBe('languageChanged');
+                    languageChangedHandlers.push(handler);
+                })
+            }
+        };
+
+        initCrowdIn();
+        jest.runAllTimers();
+
+        const options = globalThis.proxyTranslator.init.mock.calls[0][0];
+        const translated = globalThis.document.getElementById('translated');
+        const link = translated.querySelector('a');
+
+        options.callback();
+        options.callback();
+        expect(globalThis.i18nextify.i18next.on).toHaveBeenCalledTimes(1);
+
+        languageChangedHandlers[0]();
+        link.previousSibling.data = link.previousSibling.data.trimEnd();
+        await Promise.resolve();
+        jest.advanceTimersByTime(25);
+
+        link.nextSibling.data = link.nextSibling.data.trimStart();
+        await Promise.resolve();
+        jest.advanceTimersByTime(25);
+
+        expect(translated.textContent).toBe('Usethis linkhere.');
+
+        jest.advanceTimersByTime(25);
+
+        expect(translated.textContent).toBe('Use this link here.');
+    });
+
     it('should initialize proxyTranslator with LizardByte-docs settings', () => {
         initCrowdIn('LizardByte-docs');