benchmark.ts

/******************************************************************************
 * Copyright 2021 TypeFox GmbH
 * This program and the accompanying materials are made available under the
 * terms of the MIT License, which is available in the project root.
 ******************************************************************************/

import { EmptyFileSystem, URI } from 'langium';
// Use relative import, because we need to import a specific file
import { createStatemachineServices } from '../../node_modules/langium-statemachine-dsl/out/language-server/statemachine-module.js';

// generateStatemachineContent generates a syntactically valid statemachine
// document for the given index. Each document contains:
//   - 4 events
//   - 3 commands
//   - 50 states, each with transitions that cycle through events/states
function generateStatemachineContent(index: number): string {
    const numEvents = 4;
    const numCommands = 3;
    const numStates = 50;

    const parts: string[] = [];

    parts.push(`statemachine sm${index}\n\n`);

    // Events block
    parts.push('events\n');
    for (let e = 0; e < numEvents; e++) {
        parts.push(`  evt${index}_${e}\n`);
    }

    // Commands block
    parts.push('commands\n');
    for (let c = 0; c < numCommands; c++) {
        parts.push(`  cmd${index}_${c}\n`);
    }

    // Initial state
    parts.push(`initialState S${index}_0\n\n`);

    // States
    for (let s = 0; s < numStates; s++) {
        parts.push(`state S${index}_${s}\n`);
        if (s === 0) {
            parts.push(`  actions { cmd${index}_0 cmd${index}_1 }\n`);
        }
        for (let e = 0; e < numEvents; e++) {
            const target = (s + e + 1) % numStates;
            parts.push(`  evt${index}_${e} => S${index}_${target}\n`);
        }
        parts.push('end\n\n');
    }

    return parts.join('');
}

const { shared } = createStatemachineServices(EmptyFileSystem);
const numDocuments = 50;
const totalBytes = Array.from({ length: numDocuments }, (_, i) => Buffer.byteLength(generateStatemachineContent(i), 'utf8')).reduce((a, b) => a + b, 0);

async function main() {
    const oldDocs = shared.workspace.LangiumDocuments.all.toArray();
    for (const doc of oldDocs) {
        shared.workspace.LangiumDocuments.deleteDocument(doc.uri);
    }
    const documents = [];

    for (let i = 0; i < numDocuments; i++) {
        const content = generateStatemachineContent(i);
        const doc = shared.workspace.LangiumDocumentFactory.fromString(content, URI.parse(`file:///sm${i}.statemachine`));
        shared.workspace.LangiumDocuments.addDocument(doc);
        documents.push(doc);
    }

    await shared.workspace.DocumentBuilder.build(documents, {
        validation: true,
    });

}

function computeStats(samples: number[]) {
    const sorted = [...samples].sort((a, b) => a - b);
    const n = sorted.length;
    const mean = samples.reduce((a, b) => a + b, 0) / n;
    const median = n % 2 === 0
        ? (sorted[n / 2 - 1] + sorted[n / 2]) / 2
        : sorted[Math.floor(n / 2)];
    const q1 = sorted[Math.floor(n / 4)];
    const q3 = sorted[Math.floor((3 * n) / 4)];
    const iqr = q3 - q1;
    const lowerFence = q1 - 1.5 * iqr;
    const upperFence = q3 + 1.5 * iqr;
    const outliers = sorted.filter(v => v < lowerFence || v > upperFence);
    const whiskerLow = sorted.find(v => v >= lowerFence) ?? sorted[0];
    const whiskerHigh = [...sorted].reverse().find(v => v <= upperFence) ?? sorted[n - 1];
    const variance = samples.reduce((acc, v) => acc + (v - mean) ** 2, 0) / n;
    const stdDev = Math.sqrt(variance);

    return { n, mean, median, stdDev, min: sorted[0], max: sorted[n - 1], q1, q3, iqr, whiskerLow, whiskerHigh, lowerFence, upperFence, outliers };
}

const warmupRuns = 5;
const times = 10;
(async () => {
    for (let i = 0; i < warmupRuns; i++) {
        process.stdout.write(`Warmup ${i + 1}/${warmupRuns}...\r`);
        await main();
    }
    console.log('Warmup complete, starting measurements...');

    const durations: number[] = [];
    for (let i = 0; i < times; i++) {
        process.stdout.write(`Run ${i + 1}/${times}...\r`);
        const t0 = process.hrtime.bigint();
        await main();
        const t1 = process.hrtime.bigint();
        durations.push(Number(t1 - t0) / 1_000_000);
    }
    console.log('');

    const s = computeStats(durations);
    console.log('\n=== Benchmark Results (box plot data) ===');
    console.log(`Runs:         ${s.n}`);
    console.log(`Mean:         ${s.mean.toFixed(2)} ms  (${(s.mean / numDocuments).toFixed(3)} ms/file, ${(totalBytes / 1024 / 1024 / (s.mean / 1000)).toFixed(2)} MB/s)`);
    console.log(`Median (Q2):  ${s.median.toFixed(2)} ms`);
    console.log(`Std Dev:      ${s.stdDev.toFixed(2)} ms`);
    console.log(`Min:          ${s.min.toFixed(2)} ms`);
    console.log(`Max:          ${s.max.toFixed(2)} ms`);
    console.log(`Q1:           ${s.q1.toFixed(2)} ms`);
    console.log(`Q3:           ${s.q3.toFixed(2)} ms`);
    console.log(`IQR:          ${s.iqr.toFixed(2)} ms`);
    console.log(`Whisker low:  ${s.whiskerLow.toFixed(2)} ms  (Q1 - 1.5×IQR = ${s.lowerFence.toFixed(2)})`);
    console.log(`Whisker high: ${s.whiskerHigh.toFixed(2)} ms  (Q3 + 1.5×IQR = ${s.upperFence.toFixed(2)})`);
    if (s.outliers.length > 0) {
        console.log(`Outliers:     ${s.outliers.map(v => v.toFixed(2)).join(', ')} ms`);
    } else {
        console.log('Outliers:     none');
    }
    console.log('\nRaw samples (ms):');
    console.log(durations.map((v, i) => `  ${i + 1}: ${v.toFixed(2)}`).join('\n'));
})();