PrintBoundary: Handle recursive types (#8786)

Before, we infinitely recursed.

Add this to fuzzing to find further issues.

Drive-by: fix a `python` to `python3` in a script I just noticed.

Author: kripken

scripts/bundle_clusterfuzz.py

@@ -184,5 +184,5 @@
 print('Done.')
 print('To run the tests on this bundle, do:')
 print()
-print(f'BINARYEN_CLUSTER_FUZZ_BUNDLE={output_file} python -m unittest test/unit/test_cluster_fuzz.py')
+print(f'BINARYEN_CLUSTER_FUZZ_BUNDLE={output_file} python3 -m unittest test/unit/test_cluster_fuzz.py')
 print()

scripts/fuzz_opt.py

@@ -2698,6 +2698,7 @@ def write_commands(commands, filename):
     ("--precompute",),
     ("--precompute-propagate",),
     ("--print",),
+    ("--print-boundary",),
     ("--remove-unused-brs",),
     ("--remove-unused-nonfunction-module-elements",),
     ("--remove-unused-module-elements",),

src/passes/PrintBoundary.cpp

@@ -104,15 +104,17 @@ struct PrintBoundary : public Pass {
   // results.
   //
   // We emit an array only when needed, unless forceArray is set.
-  json::Value::Ref getTypes(Type type, bool forceArray = false) {
-    if (type.isRef()) {
+  json::Value::Ref
+  getTypes(Type type, bool forceArray = false, size_t depth = 0) {
+    // Avoid infinite recursion.
+    if (type.isRef() && depth == 0) {
       auto heapType = type.getHeapType();
       if (heapType.isSignature()) {
         auto sig = heapType.getSignature();
         auto ret = json::Value::makeObject();
         // Always emit arrays for params and results.
-        ret["params"] = getTypes(sig.params, true);
-        ret["results"] = getTypes(sig.results, true);
+        ret["params"] = getTypes(sig.params, true, depth + 1);
+        ret["results"] = getTypes(sig.results, true, depth + 1);
         return ret;
       }
     }

test/lit/passes/print-boundary.wast

@@ -1,6 +1,8 @@
 (module
   (type $struct (struct))
 
+  (type $recurse (func (param $x (ref $recurse))))
+
   (import "module" "base" (func $foo (param i32) (param f64) (result anyref)))
 
   (import "module2" "other" (func $bar (result i32 f32)))
@@ -26,6 +28,10 @@
   (func $one (param $x (ref $struct)) (result i32 i32 i32)
     (unreachable)
   )
+
+  (func $recurse (export "recurse") (param $x (ref $recurse))
+    ;; We should not error on printing a recursive type.
+  )
 )
 
 ;; RUN: wasm-opt %s -all --print-boundary -S -o - | filecheck %s
@@ -62,6 +68,17 @@
 ;; CHECK-NEXT:  ],
 ;; CHECK-NEXT:  "exports": [
 ;; CHECK-NEXT:   {
+;; CHECK-NEXT:    "name": "recurse",
+;; CHECK-NEXT:    "kind": "func",
+;; CHECK-NEXT:    "type": {
+;; CHECK-NEXT:     "params": [
+;; CHECK-NEXT:      "(ref $func.0)"
+;; CHECK-NEXT:     ],
+;; CHECK-NEXT:     "results": [
+;; CHECK-NEXT:     ]
+;; CHECK-NEXT:    }
+;; CHECK-NEXT:   },
+;; CHECK-NEXT:   {
 ;; CHECK-NEXT:    "name": "one",
 ;; CHECK-NEXT:    "kind": "func",
 ;; CHECK-NEXT:    "type": {