✨Add pls test command and initial test framework

cowboyd · cowboyd · commit f5d625c59733 · 2023-02-16T11:44:04.000-06:00
Since we'll be writing more and more of PlatformScript in
PlatformScript, we need a way to test PlatformScript in
PlatformScript.

This adds a `pls test` command that will find all files ending in the
`.test.yaml` extension in the `test` directory, and runs them as
PlatformScript tests.

Added to stdlib is the "testing.yaml" module where you can find a test
constructor, as well as matchers. The structure of a test is really,
really simple for now. It is just a list of expectations:

```yaml
$test:
  - expect:
    - subject
    - matcher
    - matcher
    - matcher
  - expect:
    - subject
    - matcher
    - matcher
```

Matchers are just functions that run on the subject and return a
result for it. The following functions are shipped initially to create
matchers:

- `toEqual: expected`: takes an expected value and returns a matcher
that passes when the subject is equal to the expected.
- `not: matcher`: takes another matcher and passes only if that
matcher fails.

This is not the end all be all of testing frameworks, but what it
_is_, is something that we can use to write tests for our
PlatformScript as we begin to use it.
diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml
@@ -41,6 +41,10 @@ jobs:
           deno task test
           cd www && deno task test
 
+      - name: test stdlib
+        run: |
+          deno task pls test
+
       - name: check-npm-build
         run: |
           cat version.json | xargs deno task build:npm
diff --git a/builtin.ts b/builtin.ts
@@ -0,0 +1,6 @@
+import * as data from "./data.ts";
+import { testing } from "./builtin/testing.ts";
+
+export const builtin = data.map({
+  "testing": testing,
+});
diff --git a/builtin/testing.ts b/builtin/testing.ts
@@ -0,0 +1,130 @@
+import type { PSEnv, PSFnCallContext, PSList, PSValue } from "../types.ts";
+import type { Operation } from "../deps.ts";
+import { equal } from "../equal.ts";
+import * as data from "../data.ts";
+
+export interface TestResult {
+  type: "pass" | "fail";
+  message: PSValue;
+}
+
+export function isTestResult(value: unknown): value is TestResult {
+  return !!value && typeof value === "object" &&
+    ["pass", "fail"].includes((value as TestResult).type);
+}
+
+export const testing = data.map({
+  "test": data.fn(function* (cxt) {
+    return data.external(yield* test(cxt));
+  }, { name: "definition" }),
+  "toEqual": data.fn(function* (expected) {
+    return data.fn(function* (actual) {
+      if (equal(expected.arg, actual.arg).value) {
+        return data.external({
+          type: "pass",
+          message: data.map({
+            "equal to": expected.arg,
+          }),
+        });
+      } else {
+        return data.external({
+          type: "fail",
+          message: data.map({
+            "equal to": expected.arg,
+          }),
+        });
+      }
+    }, { name: "actual" });
+  }, { name: "expected" }),
+  "not": data.fn(function* (matcher) {
+    return data.fn(function* (actual) {
+      if (matcher.arg.type === "fn") {
+        let result = yield* actual.env.call(matcher.arg, actual.arg);
+        if (result.type === "external" && result.value.type == "fail") {
+          return data.external({
+            type: "pass",
+            message: data.map({
+              "not": result.value.message,
+            }),
+          });
+        } else {
+          return data.external({
+            type: "fail",
+            message: data.map({
+              "not": result.value.message,
+            }),
+          });
+        }
+      } else {
+        return data.external({
+          type: "fail",
+          message: data.map({
+            "not": actual.arg,
+          }),
+        });
+      }
+    }, { name: "actual" });
+  }, { name: "matcher" }),
+});
+
+function* test({ arg, env }: PSFnCallContext): Operation<TestResult[]> {
+  if (arg.type !== "list") {
+    return [yield* step(arg, env)];
+  } else {
+    let results: TestResult[] = [];
+    for (let item of arg.value) {
+      results.push(yield* step(item, env));
+    }
+    return results;
+  }
+}
+
+function* step(arg: PSValue, env: PSEnv): Operation<TestResult> {
+  if (arg.type === "map") {
+    for (let [key, value] of arg.value.entries()) {
+      if (key.value === "expect") {
+        if (value.type === "list") {
+          let [first, ...rest] = value.value ?? data.string("null");
+          let subject = yield* env.eval(first ?? data.string(""));
+          let results: PSValue[] = [];
+          let pass = true;
+          let matchers = (yield* env.eval(data.list(rest))) as PSList;
+          for (let matcher of matchers.value) {
+            if (matcher.type === "fn") {
+              let result = yield* env.call(matcher, subject);
+              if (
+                result.type === "external" && result.value &&
+                result.value.type && result.value.message
+              ) {
+                if (result.value.type === "fail") {
+                  pass = false;
+                }
+                results.push(result.value.message);
+              } else {
+                results.push(result);
+              }
+            } else {
+              results.push(matcher);
+            }
+          }
+          return {
+            type: pass ? "pass" : "fail",
+            message: data.list([
+              first,
+              ...results,
+            ]),
+          };
+        } else {
+          return {
+            type: "pass",
+            message: value,
+          };
+        }
+      }
+    }
+  }
+  return {
+    type: "pass",
+    message: arg,
+  };
+}
diff --git a/cli/pls.ts b/cli/pls.ts
@@ -2,11 +2,13 @@ import { main } from "./main.ts";
 import { dispatch } from "./router.ts";
 import { PlsCommand } from "./pls-command.ts";
 import { RunCommand } from "./run-command.ts";
+import { TestCommand } from "./test-command.ts";
 
 await main(function* (args) {
   yield* dispatch(["pls", ...args], {
     "pls": [PlsCommand, {
       "run :MODULE": RunCommand,
+      "test :PATH": TestCommand,
     }],
   });
 });
diff --git a/cli/test-command.ts b/cli/test-command.ts
@@ -0,0 +1,77 @@
+import type { Route } from "./router.ts";
+import { walk } from "https://deno.land/std@0.177.0/fs/walk.ts";
+import { Operation, resolve, subscribe, Subscription } from "../deps.ts";
+import { load, map, print } from "../mod.ts";
+import type { TestResult } from "../builtin/testing.ts";
+import type { PSValue } from "../types.ts";
+
+export const TestCommand: Route = {
+  options: [],
+  help: {
+    HEAD: "Run a suite of PlatformScript tests",
+    USAGE: "pls test PATH",
+  },
+  *handle({ segments }) {
+    let path = segments.PATH ?? "test";
+
+    let pass = true;
+
+    let options = {
+      match: [new RegExp(`^${path}`)],
+      exts: [".test.yaml"],
+    };
+
+    let foundSomeTests = false;
+
+    yield* forEach(subscribe(walk(".", options)), function* (entry) {
+      foundSomeTests = true;
+      if (entry.isFile) {
+        console.log(`${entry.path}:`);
+
+        let location = new URL(`file://${resolve(entry.path)}`).toString();
+        let mod = yield* load({ location });
+
+        if (mod.value.type !== "external") {
+          throw new Error(
+            `test file should return a test object see https://pls.pub/docs/testing.html for details`,
+          );
+        } else {
+          let results: TestResult[] = mod.value.value;
+          for (let result of results) {
+            let message: PSValue;
+            if (result.type === "fail") {
+              pass = false;
+              message = map({
+                "❌": result.message,
+              });
+            } else {
+              message = map({
+                "✅": result.message,
+              });
+            }
+            console.log(print(message).value);
+          }
+        }
+      }
+    });
+
+    if (!foundSomeTests) {
+      throw new Error(`no tests found corresponding to ${path}`);
+    }
+
+    if (!pass) {
+      throw new Error("test failure");
+    }
+  },
+};
+
+function* forEach<T, R>(
+  subscription: Subscription<T, R>,
+  block: (value: T) => Operation<void>,
+): Operation<R> {
+  let next = yield* subscription;
+  for (; !next.done; next = yield* subscription) {
+    yield* block(next.value);
+  }
+  return next.value;
+}
diff --git a/equal.ts b/equal.ts
@@ -0,0 +1,33 @@
+import type { PSBoolean, PSValue } from "./types.ts";
+import * as data from "./data.ts";
+
+export function equal(a: PSValue, b: PSValue): PSBoolean {
+  const t = data.boolean(true);
+  const f = data.boolean(false);
+  if (a.type === "map" && b.type === "map") {
+    let _a = [...a.value.entries()];
+    let _b = [...b.value.entries()];
+    if (_a.length !== _b.length) {
+      return t;
+    }
+    for (let i = 0; i < _a.length; i++) {
+      let [a_key, a_val] = _a[i];
+      let [b_key, b_val] = _b[i];
+      if (!equal(a_key, b_key) || !equal(a_val, b_val)) {
+        return f;
+      }
+    }
+    return t;
+  } else if (a.type === "list" && b.type === "list") {
+    if (a.value.length !== b.value.length) {
+      return f;
+    }
+    for (let i = 0; i < a.value.length; i++) {
+      if (!equal(a.value[i], b.value[i])) {
+        return f;
+      }
+    }
+    return t;
+  }
+  return data.boolean(a.value === b.value);
+}
diff --git a/load.ts b/load.ts
@@ -2,9 +2,10 @@ import type { PSEnv, PSMap, PSModule, PSValue } from "./types.ts";
 import type { Operation } from "./deps.ts";
 
 import { expect, useAbortSignal } from "./deps.ts";
-import { exclude, lookup } from "./psmap.ts";
+import { concat, exclude, lookup } from "./psmap.ts";
 import { createYSEnv, parse } from "./evaluate.ts";
 import { recognize } from "./recognize.ts";
+import { builtin } from "./builtin.ts";
 import * as data from "./data.ts";
 
 export interface LoadOptions {
@@ -18,10 +19,17 @@ export function* load(options: LoadOptions): Operation<PSModule> {
   let { location, base, env, canon } = options;
   let url = typeof location === "string" ? new URL(location, base) : location;
 
-  let content = yield* read(url);
-  let source = parse(content);
-
-  return yield* moduleEval({ source, location: url, env, canon });
+  try {
+    let content = yield* read(url);
+    let source = parse(content);
+    return yield* moduleEval({ source, location: url, env, canon });
+  } catch (error) {
+    if (error.name === "NotFound") {
+      throw new Error(`module not found: ${location}`);
+    } else {
+      throw error;
+    }
+  }
 }
 
 export interface ModuleEvalOptions {
@@ -55,7 +63,7 @@ export function* moduleEval(options: ModuleEvalOptions): Operation<PSModule> {
   if (imports.type === "just") {
     if (imports.value.type !== "map") {
       throw new Error(
-        `imports must be specified as a mapping of names: URL but was ${imports.value.type}`,
+        `imports must be specified as a mapping of names: URL but was '${imports.value.type}'`,
       );
     }
     for (let [names, loc] of imports.value.value.entries()) {
@@ -74,8 +82,8 @@ export function* moduleEval(options: ModuleEvalOptions): Operation<PSModule> {
       let dep = loc.value === "--canon--"
         ? ({
           location: loc.value,
-          source: canon,
-          value: canon,
+          source: concat(builtin, canon),
+          value: concat(builtin, canon),
           imports: [],
         })
         : yield* load({
diff --git a/stdlib/testing.yaml b/stdlib/testing.yaml
@@ -0,0 +1,6 @@
+$import:
+  testing: --canon--
+
+not: $testing.not
+test: $testing.test
+toEqual: $testing.toEqual
diff --git a/test/stdlib/testing.test.yaml b/test/stdlib/testing.test.yaml
@@ -0,0 +1,22 @@
+$import:
+  test, not, toEqual: ../../stdlib/testing.yaml
+
+$test:
+  - expect:
+    - Hello %(World)
+    - $toEqual: Hello World
+  - expect:
+    - {hello: world}
+    - $toEqual: {hello: world}
+  - expect:
+    - [hello, world]
+    - $toEqual: [hello, world]
+  - expect:
+    - hello
+    - $not:
+        $toEqual: world
+  - expect:
+    - hello
+    - $not:
+        $not:
+          $toEqual: hello
