test: fix apitest, add new unit tests and benchmarks, normalize output

Author: XiaochenCui

.gitignore

@@ -29,3 +29,18 @@ dist/
 # Local Netlify folder
 .netlify
 src/emulator/release/
+
+# ======================================================================
+# vscode
+# ======================================================================
+# vscode configuration
+.vscode/
+
+# JS language server, ref: https://code.visualstudio.com/docs/languages/jsconfig
+jsconfig.json
+
+# ======================================================================
+# node js
+# ======================================================================
+# the exact tree installed in the node_modules folder
+package-lock.json

tools/api-tester/README.md

@@ -4,13 +4,12 @@ A test framework for testing the backend API of puter.
 
 ## Table of Contents
 
-- [API Tester](#api-tester)
 - [How to use](#how-to-use)
   - [Workflow](#workflow)
   - [Shorthands](#shorthands)
 - [Basic Concepts](#basic-concepts)
 - [Behaviors](#behaviors)
-  - [Isolation of `t.cwd`](#isolation-of-t-cwd)
+  - [Working directory (`t.cwd`)](#working-directory-t-cwd)
 - [Implementation](#implementation)
 - [TODO](#todo)
 
@@ -34,30 +33,65 @@ All commands below should be run from the root directory of puter.
 
     Fields:
     - url: The endpoint of the backend server. (default: http://api.puter.localhost:4100/)
-    - username: The username of the admin user. (e.g. admin)
-    - token: The token of the user. (can be obtained by typing `puter.authToken` in Developer Tools's console)
+    - username: The username of the user to test. (e.g. `admin`)
+    - token: The token of the user. (can be obtained by logging in on the webpage and typing `puter.authToken` in Developer Tools's console)
+    - mountpoints: The mountpoints to test. (default config includes 2 mountpoints: `/` for "puter fs provider" and `/admin/tmp` for "memory fs provider")
 
-3. Run the tests:
+3. Run all tests (unit tests and benchmarks):
 
     ```bash
     node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml
     ```
 
 ### Shorthands
 
-- Run unit tests only:
+- Run all tests (unit tests and benchmarks):
+
+    ```bash
+    node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml
+    ```
+
+- Run all unit tests:
 
     ```bash
     node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml --unit
     ```
 
+- Run all benchmarks:
+
+    ```bash
+    node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml --bench
+    ```
+
 - Filter tests by suite name:
 
     ```bash
     node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml --unit --suite=mkdir
     ```
 
-- Rerun failed tests in the last run:
+- Filter benchmarks by name:
+
+    ```bash
+    node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml --bench --suite=stat_intensive_1
+    ```
+
+- Stop on first failure:
+
+    ```bash
+    node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml --unit --stop-on-failure
+    ```
+
+- (unimplemented) Filter tests by test name:
+
+    ```bash
+    # (wildcard matching) Run tests containing "memoryfs" in the name
+    node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml --unit --test='*memoryfs*'
+
+    # (exact matching) Run the test "mkdir in memoryfs"
+    node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml --unit --test='mkdir in memoryfs'
+    ```
+
+- (unimplemented) Rerun failed tests in the last run:
 
     ```bash
     node ./tools/api-tester/apitest.js --config=./tools/api-tester/config.yml --rerun-failed
@@ -98,10 +132,13 @@ module.exports = {
 
 ## Behaviors
 
-### Isolation of `t.cwd`
+### Working directory (`t.cwd`)
 
-- `t.cwd` is reset at the beginning of each test suite, since a test suite usually doesn't want to be affected by other test suites.
-- `t.cwd` will be inherited from the cases in the same test suite, since a leaf case might want to share the context with its parent/sibling cases.
+- The working directory is stored in `t.cwd`.
+- All filesystem operations are performed relative to the working directory, if the given path is not absolute. (e.g., `t.mkdir('foo')`, `t.cd('foo')`, `t.stat('foo')`, etc.)
+- Tests will be run under all mountpoints. The default working directory for a mountpoint is `${mountpoint.path}/{username}/api_test`. (This is subject to change in the future, the reason we include `admin` in the path is to ensure the test user `admin` has write access, see [Permission Documentation](https://github.com/HeyPuter/puter/blob/3290440f4bf7a263f37bc5233565f8fec146f17b/src/backend/doc/A-and-A/permission.md#permission-options) for details.)
+- The working directory is reset at the beginning of each test suite, since a test suite usually doesn't want to be affected by other test suites.
+- The working directory will be inherited from the cases in the same test suite, since a leaf case might want to share the context with its parent/sibling cases.
 
 ```js
 module.exports = {
@@ -126,5 +163,5 @@ module.exports = {
 
 ## TODO
 
-- [ ] Update usage of apitest.js. (Is it possible to generate the usage automatically?)
-- [ ] Integrate it into CI, optionally running it only in specific scenarios (e.g., when backend code changes).
+- [ ] Reset `t.cwd` if a test case fails. Currently, `t.cwd` is not reset if a test case fails.
+- [ ] Integrate apitest into CI, optionally running it only in specific scenarios (e.g., when backend code changes).

tools/api-tester/apitest.js

@@ -9,13 +9,14 @@ const { parseArgs } = require('node:util');
 
 const args = process.argv.slice(2);
 
-let config, report, suiteName;
+let config, report, suiteName, onlycase, bench, unit, stopOnFailure, id;
 
 try {
     const parsed = parseArgs({
         options: {
             config: {
                 type: 'string',
+                default: './tools/api-tester/config.yml',
             },
             report: {
                 type: 'string',
@@ -24,6 +25,7 @@ try {
             bench: { type: 'boolean' },
             unit: { type: 'boolean' },
             suite: { type: 'string' },
+            'stop-on-failure': { type: 'boolean' },
         },
         allowPositionals: true,
     });
@@ -35,6 +37,7 @@ try {
         bench,
         unit,
         suite: suiteName,
+        'stop-on-failure': stopOnFailure,
     }, positionals: [id] } = parsed);
 
     onlycase = onlycase !== undefined ? Number.parseInt(onlycase) : undefined;
@@ -49,6 +52,7 @@ try {
         '  --config=<path>  (required)  Path to configuration file\n' +
         '  --report=<path>  (optional)  Output file for full test results\n' +
         '  --suite=<name>   (optional)  Run only tests with matching suite name\n' +
+        '  --stop-on-failure (optional)  Stop execution on first test failure\n' +
         ''
     );
     process.exit(1);
@@ -58,29 +62,83 @@ const conf = YAML.parse(fs.readFileSync(config).toString());
 
 
 const main = async () => {
-    const context = {
-        options: {
-            onlycase,
-            suite: suiteName,
+    const unit_test_results = [];
+    const benchmark_results = [];
+    for (const mountpoint of conf.mountpoints) {
+        const { unit_test_results: results, benchmark_results: benchs } = await test({ mountpoint });
+        unit_test_results.push(...results);
+        benchmark_results.push(...benchs);
+    }
+
+    // hard-coded identifier for ci script
+    console.log("==================== nightly build results begin ====================")
+
+    // print unit test results
+    let tbl = {};
+    for ( const result of unit_test_results ) {
+        tbl[result.name + ' - ' + result.settings] = {
+            passed: result.caseCount - result.failCount,
+            failed: result.failCount,
+            total: result.caseCount,
+            'duration (s)': result.duration ? result.duration.toFixed(2) : 'N/A',
+        }
+    }
+    console.table(tbl);
+
+    // print benchmark results
+    if (benchmark_results.length > 0) {
+        tbl = {};
+        for ( const result of benchmark_results ) {
+            const fs_provider = result.fs_provider || 'unknown';
+            tbl[result.name + ' - ' + fs_provider] = {
+                'duration (s)': result.duration ? (result.duration / 1000).toFixed(2) : 'N/A',
+            }
+        }
+        console.table(tbl);
+
+        // print description of each benchmark since it's too long to fit in the table
+        const seen = new Set();
+        for ( const result of benchmark_results ) {
+            if ( seen.has(result.name) ) continue;
+            seen.add(result.name);
+
+            if ( result.description ) {
+                console.log(result.name + ': ' + result.description);
+            }
         }
-    };
-    const ts = new TestSDK(conf, context);
-    try {
-        await ts.delete('api_test', { recursive: true });
-    } catch (e) {
     }
-    await ts.mkdir('api_test', { overwrite: true });
-    ts.cd('api_test');
+
+    // hard-coded identifier for ci script
+    console.log("==================== nightly build results end ====================")
+}
+
+/**
+ * Run test using the given config, and return the test results
+ * 
+ * @param {Object} options
+ * @param {Object} options.mountpoint
+ * @returns {Promise<Object>}
+ */
+async function test({ mountpoint }) {
+    const context = {
+        mountpoint
+    };
+
+    const ts = new TestSDK(conf, context, { stopOnFailure });
+    await ts.init_working_directory();
 
     const registry = new TestRegistry(ts);
 
     registry.add_test_sdk('puter-rest.v1', require('./test_sdks/puter-rest')({
         config: conf,
     }));
 
-    require('./tests/__entry__.js')(registry);
+    // TODO: merge it into the entry point
     require('./benches/simple.js')(registry);
 
+    require('./tests/__entry__.js')(registry);
+    require('./benches/__entry__.js')(registry);
+
     if ( id ) {
         if ( unit ) {
             await registry.run_test(id);
@@ -95,25 +153,18 @@ const main = async () => {
     if ( unit ) {
         await registry.run_all_tests(suiteName);
     } else if ( bench ) {
-        await registry.run_all_benches();
+        await registry.run_all_benches(suiteName);
     } else {
         await registry.run_all();
     }
 
+    if ( unit ) ts.printTestResults();
+    if ( bench ) ts.printBenchmarkResults();
 
-    // await ts.runTestPackage(require('./tests/write_cart'));
-    // await ts.runTestPackage(require('./tests/move_cart'));
-    // await ts.runTestPackage(require('./tests/copy_cart'));
-    // await ts.runTestPackage(require('./tests/write_and_read'));
-    // await ts.runTestPackage(require('./tests/move'));
-    // await ts.runTestPackage(require('./tests/stat'));
-    // await ts.runTestPackage(require('./tests/readdir'));
-    // await ts.runTestPackage(require('./tests/mkdir'));
-    // await ts.runTestPackage(require('./tests/batch'));
-    // await ts.runTestPackage(require('./tests/delete'));
-    const all = unit && bench;
-    if ( all || unit ) ts.printTestResults();
-    if ( all || bench ) ts.printBenchmarkResults();
+    return {
+        unit_test_results: ts.packageResults,
+        benchmark_results: ts.benchmarkResults,
+    };
 }
 
 const main_e = async () => {

tools/api-tester/benches/__entry__.js

@@ -0,0 +1,4 @@
+module.exports = registry => {
+    registry.add_bench('write_intensive_1', require('./write_intensive_1.js'));
+    registry.add_bench('stat_intensive_1', require('./stat_intensive_1.js'));
+};

tools/api-tester/benches/stat_intensive_1.js

@@ -0,0 +1,37 @@
+const chai = require('chai');
+chai.use(require('chai-as-promised'))
+const expect = chai.expect;
+
+module.exports = {
+    name: 'stat intensive 1',
+    description: 'create 10 directories and 100 subdirectories in each, then stat them over and over',
+    do: async t => {
+        console.log('stat intensive 1');
+
+        const dir_count = 10;
+        const subdir_count = 100;
+
+        // key: uuid
+        // value: path
+        const dirs = {};
+
+        for (let i = 0; i < dir_count; i++) {
+            await t.mkdir(`dir_${i}`);
+            for (let j = 0; j < subdir_count; j++) {
+                const subdir = await t.mkdir(`dir_${i}/subdir_${j}`);
+                dirs[subdir.uid] = subdir.path;
+            }
+        }
+
+        const start = Date.now();
+        for (let i = 0; i < 10; i++) {
+            for (const [uuid, path] of Object.entries(dirs)) {
+                const stat = await t.stat_uuid(uuid);
+                expect(stat.is_dir).equal(true);
+                expect(stat.path).equal(path);
+            }
+        }
+        const duration = Date.now() - start;
+        return { duration };
+    }
+};

tools/api-tester/benches/write_intensive_1.js

@@ -0,0 +1,32 @@
+const chai = require('chai');
+chai.use(require('chai-as-promised'))
+const expect = chai.expect;
+
+module.exports = {
+    name: 'write intensive 1',
+    description: 'create 100 new directories and write 10 files in each, then check integrity by stat/readdir/read api',
+    do: async t => {
+        console.log('write intensive 1');
+
+        const dir_count = 100;
+        const file_count = 10;
+
+        for ( let i=0 ; i < dir_count ; i++ ) {
+            await t.mkdir(`dir_${i}`);
+            for ( let j=0 ; j < file_count ; j++ ) {
+                const content = `example ${i} ${j}`;
+                await t.write(`dir_${i}/file_${j}.txt`, content, { overwrite: true });
+            }
+        }
+
+        for ( let i=0 ; i < dir_count ; i++ ) {
+            const dir = await t.stat(`dir_${i}`);
+            const files = await t.readdir(dir.path);
+            expect(files.length).equal(file_count);
+            for ( let j=0 ; j < file_count ; j++ ) {
+                const content = await t.read(`dir_${i}/file_${j}.txt`);
+                expect(content).equal(`example ${i} ${j}`);
+            }
+        }
+    }
+};

tools/api-tester/example_config.yml

@@ -1,3 +1,8 @@
-url: http://puter.localhost:4001/
-username: lavender_hat_6100
+url: http://api.puter.localhost:4100/
+username: admin
 token: ---
+mountpoints:
+  - path: /
+    provider: puterfs
+  - path: /admin/tmp
+    provider: memoryfs

tools/api-tester/lib/TestFactory.js

@@ -12,7 +12,7 @@ module.exports = class TestFactory {
             for ( let i=0 ; i < states.length ; i++ ) {
                 const state = states[i];
 
-                if ( t.context.options.onlycase !== undefined ) {
+                if ( t.context.options?.onlycase !== undefined ) {
                     if ( i !== t.context.options.onlycase ) {
                         continue;
                     }