test: write benchmark

Author: XiaochenCui

.gitignore

@@ -29,3 +29,18 @@ dist/
 # Local Netlify folder
 .netlify
 src/emulator/release/
+
+# ======================================================================
+# vscode
+# ======================================================================
+# vscode configuration
+.vscode/
+
+# JS language server, ref: https://code.visualstudio.com/docs/languages/jsconfig
+jsconfig.json
+
+# ======================================================================
+# node js
+# ======================================================================
+# the exact tree installed in the node_modules folder
+package-lock.json

src/backend/src/filesystem/hl_operations/hl_mkdir.js

@@ -493,7 +493,7 @@ class HLMkdir extends HLFilesystemOperation {
             throw APIError.create('dest_does_not_exist');
         }
 
-        let path = maybe_path_selector.value;
+        const path = maybe_path_selector.value;
 
         const fs = this.context.get('services').get('filesystem');
 

tools/api-tester/README.md

@@ -4,13 +4,12 @@ A test framework for testing the backend API of puter.
 
 ## Table of Contents
 
-- [API Tester](#api-tester)
 - [How to use](#how-to-use)
   - [Workflow](#workflow)
   - [Shorthands](#shorthands)
 - [Basic Concepts](#basic-concepts)
 - [Behaviors](#behaviors)
-  - [Isolation of `t.cwd`](#isolation-of-t-cwd)
+  - [Working directory (`t.cwd`)](#working-directory-t-cwd)
 - [Implementation](#implementation)
 - [TODO](#todo)
 
@@ -52,6 +51,12 @@ All commands below should be run from the root directory of puter.
     node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml --unit
     ```
 
+- Run benchmarks only:
+
+    ```bash
+    node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml --bench
+    ```
+
 - Filter tests by suite name:
 
     ```bash

tools/api-tester/apitest.js

@@ -61,27 +61,49 @@ const conf = YAML.parse(fs.readFileSync(config).toString());
 
 
 const main = async () => {
-    const results = [];
+    const unit_test_results = [];
+    const benchmark_results = [];
     for (const mountpoint of conf.mountpoints) {
-        const result = await test({ mountpoint });
-        results.push(...result);
+        const { unit_test_results: results, benchmark_results: benchs } = await test({ mountpoint });
+        unit_test_results.push(...results);
+        benchmark_results.push(...benchs);
     }
 
+    // hard-coded identifier for ci script
+    console.log("==================== nightly build results begin ====================")
+
+    // print unit test results
     let tbl = {};
-    for ( const result of results ) {
+    for ( const result of unit_test_results ) {
         tbl[result.name + ' - ' + result.settings] = {
             passed: result.caseCount - result.failCount,
             failed: result.failCount,
             total: result.caseCount,
             'duration (s)': result.duration ? result.duration.toFixed(2) : 'N/A',
         }
     }
-
-    // hard-coded identifier for ci script
-    console.log("==================== nightly build results begin ====================")
-
     console.table(tbl);
 
+    // print benchmark results
+    if (benchmark_results.length > 0) {
+        tbl = {};
+        for ( const result of benchmark_results ) {
+            const fs_provider = result.fs_provider || 'unknown';
+            tbl[result.name + ' - ' + fs_provider] = {
+                'duration (s)': result.duration ? (result.duration / 1000).toFixed(2) : 'N/A',
+            }
+        }
+        console.table(tbl);
+
+        // print description of each benchmark since it's too long to fit in the table
+        const seen = new Set();
+        for ( const result of benchmark_results ) {
+            if ( seen.has(result.name) ) continue;
+            seen.add(result.name);
+            console.log(result.name + ' - ' + result.description);
+        }
+    }
+
     // hard-coded identifier for ci script
     console.log("==================== nightly build results end ====================")
 }
@@ -108,7 +130,7 @@ async function test({ mountpoint }) {
     }));
 
     require('./tests/__entry__.js')(registry);
-    require('./benches/simple.js')(registry);
+    require('./benches/__entry__.js')(registry);
 
     if ( id ) {
         if ( unit ) {
@@ -129,11 +151,13 @@ async function test({ mountpoint }) {
         await registry.run_all();
     }
 
-    const all = unit && bench;
-    if ( all || unit ) ts.printTestResults();
-    if ( all || bench ) ts.printBenchmarkResults();
+    if ( unit ) ts.printTestResults();
+    if ( bench ) ts.printBenchmarkResults();
 
-    return ts.packageResults;
+    return {
+        unit_test_results: ts.packageResults,
+        benchmark_results: ts.benchmarkResults,
+    };
 }
 
 const main_e = async () => {

tools/api-tester/benches/__entry__.js

@@ -0,0 +1,3 @@
+module.exports = registry => {
+    registry.add_bench('write-intensive-1', require('./write_intensive_1.js'));
+};

tools/api-tester/benches/write_intensive_1.js

@@ -0,0 +1,32 @@
+const chai = require('chai');
+chai.use(require('chai-as-promised'))
+const expect = chai.expect;
+
+module.exports = {
+    name: 'write intensive 1',
+    description: 'create 100 new directories and write 10 files in each, then check integrity by stat/readdir/read api',
+    do: async t => {
+        console.log('write intensive 1');
+
+        const dir_count = 100;
+        const file_count = 10;
+
+        for ( let i=0 ; i < dir_count ; i++ ) {
+            await t.mkdir(`dir_${i}`);
+            for ( let j=0 ; j < file_count ; j++ ) {
+                const content = `example ${i} ${j}`;
+                await t.write(`dir_${i}/file_${j}.txt`, content, { overwrite: true });
+            }
+        }
+
+        for ( let i=0 ; i < dir_count ; i++ ) {
+            const dir = await t.stat(`dir_${i}`);
+            const files = await t.readdir(dir.path);
+            expect(files.length).equal(file_count);
+            for ( let j=0 ; j < file_count ; j++ ) {
+                const content = await t.read(`dir_${i}/file_${j}.txt`);
+                expect(content).equal(`example ${i} ${j}`);
+            }
+        }
+    }
+};

tools/api-tester/lib/TestRegistry.js

@@ -46,9 +46,9 @@ module.exports = class TestRegistry {
     // and I think that's pretty cool
 
     async run_all_benches () {
-        for ( const id in this.benches ) {
-            const benchDefinition = this.benches[id];
-            await this.t.runBenchmark(benchDefinition);
+        for ( const [id, bench_definition] of Object.entries(this.benches) ) {
+            console.log(`running bench: ${id}`);
+            await this.t.runBenchmark(bench_definition);
         }
     }
 

tools/api-tester/lib/TestSDK.js

@@ -46,6 +46,7 @@ module.exports = class TestSDK {
         try {
             await this.delete(this.default_cwd, { recursive: true });
         } catch (e) {
+            // ignore
         }
         await this.mkdir(this.default_cwd, { overwrite: true, create_missing_parents: true });
         this.cd(this.default_cwd);
@@ -90,20 +91,27 @@ module.exports = class TestSDK {
         process.stdout.write(strid + ' ... \n');
 
         this.nameStack.push(benchDefinition.name);
-        let results;
-        this.benchmarkResults.push(results = {
-            name: benchDefinition.name,
-            start: Date.now(),
-        });
+        const start = Date.now();
         try {
             await benchDefinition.do(this);
         } catch (e) {
-            results.error = e;
-        } finally {
-            results.end = Date.now();
-            const dur = results.end - results.start;
-            process.stdout.write(`...\x1B[32;1m[${dur}]\x1B[0m\n`);
+            // we don't tolerate errors at the moment
+            console.error(e);
+            throw e;
         }
+
+        const results = {
+            name: benchDefinition.name,
+            description: benchDefinition.description,
+            duration: Date.now() - start,
+            fs_provider: this.context.mountpoint?.provider || 'unknown',
+        };
+
+        console.log(`duration: ${(results.duration / 1000).toFixed(2)}s`);
+
+        this.benchmarkResults.push(results);
+
+        this.nameStack.pop();
     }
 
     recordResult (result) {
@@ -193,8 +201,7 @@ module.exports = class TestSDK {
         let tbl = {};
         for ( const bench of this.benchmarkResults ) {
             tbl[bench.name] = {
-                time: bench.end - bench.start,
-                error: bench.error ? bench.error.message : '',
+                'duration (ms)': bench.duration,
             }
         }
         console.table(tbl);