Skip to content

Commit f876240

Browse files
BigJkBigJk
committed
Added multi-threaded rendering of templates via web-workers.
1 parent 2f65895 commit f876240

File tree

3 files changed

+159
-106
lines changed

3 files changed

+159
-106
lines changed

‎frontend/src/js/core/templating.js‎

Lines changed: 44 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -1,106 +1,45 @@
1-
import MarkdownIt from 'markdown-it';
2-
import * as nunjucks from 'nunjucks';
31
import hash from 'object-hash';
42

5-
import EvalWorker from '/js/workers/eval?worker';
3+
import TemplatingWorker from '/js/workers/templating-worker?worker';
64

7-
import api from '/js/core/api';
85
import store from '/js/core/store';
96

10-
// DataImportExtension
7+
// Worker Pool
118
//
12-
// This nunjucks extension makes it possible to parse
13-
// and add JSON to the context. This can help to embed
14-
// static data into the templates.
15-
function DataImportExtension() {
16-
this.tags = ['data'];
17-
18-
this.parse = function (parser, nodes) {
19-
let tok = parser.nextToken();
20-
21-
let args = parser.parseSignature(null, true);
22-
parser.advanceAfterBlockEnd(tok.value);
23-
24-
let body = parser.parseUntilBlocks('enddata');
25-
parser.advanceAfterBlockEnd();
26-
27-
return new nodes.CallExtension(this, 'run', args, [body]);
28-
};
29-
30-
this.run = function (context, name, body) {
31-
try {
32-
context.ctx[name] = JSON.parse(body());
33-
} catch (e) {
34-
console.log(e);
9+
//
10+
let cache = {};
11+
let workerSelect = 0;
12+
let workerPromises = {};
13+
let workers = new Array(navigator.hardwareConcurrency || 4).fill(null).map((_, i) => {
14+
let worker = new TemplatingWorker();
15+
16+
// when rendered response is received call the related resolve or reject.
17+
worker.onmessage = (e) => {
18+
if (e.data.log) {
19+
console.log(`Template Web-Worker ${i + 1}: ${e.data.log}`);
20+
return;
3521
}
36-
return '';
37-
};
38-
}
39-
40-
function JavascriptExecuteExtension() {
41-
this.tags = ['js'];
42-
43-
this.parse = function (parser, nodes) {
44-
let tok = parser.nextToken();
45-
46-
let args = parser.parseSignature(null, true);
47-
parser.advanceAfterBlockEnd(tok.value);
48-
49-
let body = parser.parseUntilBlocks('endjs');
50-
parser.advanceAfterBlockEnd();
5122

52-
return new nodes.CallExtensionAsync(this, 'run', args, [body]);
53-
};
54-
55-
this.run = function (context, name, fn, callback) {
56-
let worker = new EvalWorker();
23+
let { resolve, reject, timeout, hashed } = workerPromises[e.data.id];
24+
let res = e.data;
5725

58-
// Kill worker after timeout. This is important if
59-
// the code has a infinite loop.
60-
let timeout = setTimeout(() => {
61-
worker.terminate();
62-
callback('eval worker: timeout');
63-
}, 1000);
26+
// stop timeout handler
27+
clearTimeout(timeout);
6428

65-
// Wait for response.
66-
worker.onmessage = (e) => {
67-
clearTimeout(timeout);
68-
worker.terminate();
69-
context.ctx[name] = e.data;
70-
callback(null, '');
71-
};
29+
if (res.err) {
30+
let parsedErr = parseError(res.err);
31+
cache[hashed] = parsedErr;
32+
reject(parsedErr);
33+
} else {
34+
cache[hashed] = res.res;
35+
resolve(res.res);
36+
}
7237

73-
// Send request.
74-
worker.postMessage([context.ctx, fn()]);
38+
delete workerPromises[res.id];
7539
};
76-
}
77-
78-
let env = new nunjucks.Environment();
79-
let markdown = new MarkdownIt();
80-
81-
env.addExtension('DataImportExtension', new DataImportExtension());
82-
env.addExtension('JavascriptExecuteExtension', new JavascriptExecuteExtension());
83-
84-
env.addFilter('markdown', (md) => new nunjucks.runtime.SafeString(markdown.render(md)));
85-
env.addFilter('markdowni', (md) => new nunjucks.runtime.SafeString(markdown.renderInline(md)));
86-
env.addFilter('json', (data) => new nunjucks.runtime.SafeString(JSON.stringify(data)));
87-
env.addFilter(
88-
'source',
89-
(source, cb) => {
90-
let found = store.data.sources.find((s) => `ds:${s.author}+${s.slug}` === source);
9140

92-
if (!found) {
93-
cb(null, 'not found');
94-
return;
95-
}
96-
97-
api
98-
.getEntries(source)
99-
.then((res) => cb(null, res))
100-
.catch((err) => cb(err, null));
101-
},
102-
true
103-
);
41+
return worker;
42+
});
10443

10544
// Exports
10645
//
@@ -117,28 +56,28 @@ export const parseError = (e) => {
11756
return null;
11857
};
11958

120-
let cache = {};
121-
12259
export const render = (template, state) => {
12360
state.settings = store.data.settings;
12461

12562
return new Promise((resolve, reject) => {
126-
let id = hash(template) + hash(state);
127-
if (cache[id]) {
63+
// check if data is present in cache
64+
let hashed = hash(template) + hash(state);
65+
if (cache[hashed]) {
12866
console.log('templating: cache hit');
129-
resolve(cache[id]);
67+
resolve(cache[hashed]);
13068
return;
13169
}
13270

133-
env.renderString(template, state, (err, res) => {
134-
if (err) {
135-
let parsedErr = parseError(err);
136-
cache[id] = parsedErr;
137-
reject(parsedErr);
138-
} else {
139-
cache[id] = res;
140-
resolve(res);
141-
}
142-
});
71+
// setup promises for response
72+
let id = hash + '-' + Math.ceil(Math.random() * 10000000).toString();
73+
workerPromises[id] = {
74+
hashed,
75+
resolve,
76+
reject,
77+
timeout: setTimeout(() => reject('timeout'), 2000),
78+
};
79+
80+
// post message (round-robin style) to some worker
81+
workers[workerSelect++ % workers.length].postMessage({ id, template, state });
14382
});
14483
};

‎frontend/src/js/ui/views/generators/index.jsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ export default () => {
207207
<PreviewBox
208208
className={`w-50 ${(i & 1) === 0 ? 'pr2' : ''}`}
209209
value={g}
210-
previewContent={state.rendered[id]}
210+
previewContent={state.rendered[id] ?? 'Rendering...'}
211211
loading={state.rendered[id] === undefined}
212212
bottomRight={
213213
<div className='btn' onclick={() => m.route.set(`/generators/gen:${g.author}+${g.slug}`)}>

‎frontend/src/js/workers/templating-worker.js‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import MarkdownIt from 'markdown-it';
2+
import * as nunjucks from 'nunjucks';
3+
4+
import EvalWorker from '/js/workers/eval?worker';
5+
6+
// DataImportExtension
7+
//
8+
// This nunjucks extension makes it possible to parse
9+
// and add JSON to the context. This can help to embed
10+
// static data into the templates.
11+
function DataImportExtension() {
12+
this.tags = ['data'];
13+
14+
this.parse = function (parser, nodes) {
15+
let tok = parser.nextToken();
16+
17+
let args = parser.parseSignature(null, true);
18+
parser.advanceAfterBlockEnd(tok.value);
19+
20+
let body = parser.parseUntilBlocks('enddata');
21+
parser.advanceAfterBlockEnd();
22+
23+
return new nodes.CallExtension(this, 'run', args, [body]);
24+
};
25+
26+
this.run = function (context, name, body) {
27+
try {
28+
context.ctx[name] = JSON.parse(body());
29+
} catch (e) {
30+
console.log(e);
31+
}
32+
return '';
33+
};
34+
}
35+
36+
function JavascriptExecuteExtension() {
37+
this.tags = ['js'];
38+
39+
this.parse = function (parser, nodes) {
40+
let tok = parser.nextToken();
41+
42+
let args = parser.parseSignature(null, true);
43+
parser.advanceAfterBlockEnd(tok.value);
44+
45+
let body = parser.parseUntilBlocks('endjs');
46+
parser.advanceAfterBlockEnd();
47+
48+
return new nodes.CallExtensionAsync(this, 'run', args, [body]);
49+
};
50+
51+
this.run = function (context, name, fn, callback) {
52+
let worker = new EvalWorker();
53+
54+
// Kill worker after timeout. This is important if
55+
// the code has a infinite loop.
56+
let timeout = setTimeout(() => {
57+
worker.terminate();
58+
callback('eval worker: timeout');
59+
}, 1000);
60+
61+
// Wait for response.
62+
worker.onmessage = (e) => {
63+
clearTimeout(timeout);
64+
worker.terminate();
65+
context.ctx[name] = e.data;
66+
callback(null, '');
67+
};
68+
69+
// Send request.
70+
worker.postMessage([context.ctx, fn()]);
71+
};
72+
}
73+
74+
let env = new nunjucks.Environment();
75+
let markdown = new MarkdownIt();
76+
77+
env.addExtension('DataImportExtension', new DataImportExtension());
78+
env.addExtension('JavascriptExecuteExtension', new JavascriptExecuteExtension());
79+
80+
env.addFilter('markdown', (md) => new nunjucks.runtime.SafeString(markdown.render(md)));
81+
env.addFilter('markdowni', (md) => new nunjucks.runtime.SafeString(markdown.renderInline(md)));
82+
env.addFilter('json', (data) => new nunjucks.runtime.SafeString(JSON.stringify(data)));
83+
env.addFilter(
84+
'source',
85+
(source, cb) => {
86+
// we can't use api here as mithrils request functions don't work in web-workers,
87+
// so we just use a simple fetch.
88+
fetch('/api/getEntries', {
89+
method: 'POST',
90+
headers: {
91+
'Content-Type': 'application/json',
92+
},
93+
body: JSON.stringify([source]),
94+
})
95+
.then((resp) => resp.json())
96+
.then((data) => cb(null, data))
97+
.catch((err) => cb(err, null));
98+
},
99+
true
100+
);
101+
102+
onmessage = (e) => {
103+
env.renderString(e.data.template, e.data.state, (err, res) => {
104+
postMessage({
105+
id: e.data.id,
106+
res,
107+
err,
108+
});
109+
});
110+
};
111+
112+
postMessage({
113+
log: 'started',
114+
});

0 commit comments

Comments
 (0)