Refactor std.native functions out of Std

bench/src/main/scala/sjsonnet/MainBenchmark.scala

@@ -33,7 +33,7 @@ object MainBenchmark {
       Map.empty[String, String],
       Map.empty[String, String],
       OsPath(wd),
-      importer = SjsonnetMain
+      importer = SjsonnetMainBase
         .resolveImport(config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)), None),
       parseCache = parseCache
     )
@@ -65,7 +65,7 @@ class MainBenchmark {
   @Benchmark
   def main(bh: Blackhole): Unit = {
     bh.consume(
-      SjsonnetMain.main0(
+      SjsonnetMainBase.main0(
         MainBenchmark.mainArgs,
         new DefaultParseCache,
         System.in,
@@ -102,7 +102,7 @@ object MemoryBenchmark {
     } else {
       false
     }
-    SjsonnetMain.main0(
+    SjsonnetMainBase.main0(
       MainBenchmark.mainArgs,
       cache,
       System.in,

bench/src/main/scala/sjsonnet/MaterializerBenchmark.scala

@@ -32,7 +32,7 @@ class MaterializerBenchmark {
       Map.empty[String, String],
       Map.empty[String, String],
       OsPath(wd),
-      importer = SjsonnetMain
+      importer = SjsonnetMainBase
         .resolveImport(
           config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)).toIndexedSeq,
           None

bench/src/main/scala/sjsonnet/MultiThreadedBenchmark.scala

@@ -39,7 +39,7 @@ class MultiThreadedBenchmark {
       pool.submit {
         (() =>
           if (
-            SjsonnetMain.main0(
+            SjsonnetMainBase.main0(
               MainBenchmark.mainArgs,
               cache, // new DefaultParseCache
               System.in,

bench/src/main/scala/sjsonnet/RunProfiler.scala

@@ -15,7 +15,7 @@ object RunProfiler extends App {
     Map.empty[String, String],
     Map.empty[String, String],
     OsPath(wd),
-    importer = SjsonnetMain
+    importer = SjsonnetMainBase
       .resolveImport(config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)).toIndexedSeq, None),
     parseCache = parseCache
   ) {

build.mill

@@ -102,7 +102,8 @@ object sjsonnet extends VersionFileModule {
       def jsEnvConfig = JsEnvConfig.NodeJs(args = List("--stack-size=" + stackSizekBytes))
       def resources = T.sources(
         this.millSourcePath / "resources" / "test_suite",
-        this.millSourcePath / "resources" / "go_test_suite"
+        this.millSourcePath / "resources" / "go_test_suite",
+        this.millSourcePath / "resources" / "new_test_suite",
       )
       def generatedSources = T {
         resources().map(_.path).flatMap { testSuite =>

sjsonnet/server/src/sjsonnet/SjsonnetServerMain.scala

@@ -80,7 +80,7 @@ object SjsonnetServerMain extends SjsonnetServerMain[DefaultParseCache] {
         scala.Console.withIn(stdin) {
           scala.Console.withOut(stdout) {
             scala.Console.withErr(stderr) {
-              sjsonnet.SjsonnetMain.main0(args, stateCache2, stdin, stdout, stderr, wd) == 0
+              sjsonnet.SjsonnetMainBase.main0(args, stateCache2, stdin, stdout, stderr, wd) == 0
             }
           }
         }

sjsonnet/src-js/sjsonnet/Platform.scala

@@ -2,25 +2,11 @@ package sjsonnet
 
 import org.virtuslab.yaml.*
 
-import java.io.File
 import java.util
 import java.util.regex.Pattern
 import scala.collection.mutable
 
 object Platform {
-  def gzipBytes(s: Array[Byte]): String = {
-    throw new Exception("GZip not implemented in Scala.js")
-  }
-  def gzipString(s: String): String = {
-    throw new Exception("GZip not implemented in Scala.js")
-  }
-  def xzBytes(s: Array[Byte], compressionLevel: Option[Int]): String = {
-    throw new Exception("XZ not implemented in Scala.js")
-  }
-  def xzString(s: String, compressionLevel: Option[Int]): String = {
-    throw new Exception("XZ not implemented in Scala.js")
-  }
-
   private def nodeToJson(node: Node): ujson.Value = node match {
     case _: Node.ScalarNode =>
       YamlDecoder.forAny.construct(node).getOrElse("") match {
@@ -98,9 +84,6 @@ object Platform {
   def sha3(s: String): String = {
     throw new Exception("SHA3 not implemented in Scala.js")
   }
-  def hashFile(file: File): String = {
-    throw new Exception("hashFile not implemented in Scala.js")
-  }
 
   private val regexCache = new util.concurrent.ConcurrentHashMap[String, Pattern]
   private val namedGroupPattern = Pattern.compile("\\(\\?<(.+?)>.*?\\)")

sjsonnet/src-js/sjsonnet/SjsonnetMain.scala

@@ -1,5 +1,7 @@
 package sjsonnet
 
+import sjsonnet.stdlib.NativeRegex
+
 import scala.collection.mutable
 import scala.scalajs.js
 import scala.scalajs.js.annotation.{JSExport, JSExportTopLevel}
@@ -39,7 +41,8 @@ object SjsonnetMain {
           }
       },
       parseCache = new DefaultParseCache,
-      new Settings(preserveOrder = preserveOrder)
+      settings = new Settings(preserveOrder = preserveOrder),
+      std = new Std(nativeFunctions = NativeRegex.functions).Std
     )
     interp.interpret0(text, JsVirtualPath("(memory)"), ujson.WebJson.Builder) match {
       case Left(msg) => throw new js.JavaScriptException(msg)

sjsonnet/src-jvm-native/sjsonnet/SjsonnetMain.scala → sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala

@@ -13,7 +13,7 @@ import java.nio.file.NoSuchFileException
 import scala.annotation.unused
 import scala.util.Try
 
-object SjsonnetMain {
+object SjsonnetMainBase {
   def resolveImport(
       searchRoots0: Seq[Path], // Evaluated in order, first occurrence wins
       allowedInputs: Option[Set[os.Path]] = None,
@@ -60,19 +60,6 @@ object SjsonnetMain {
       }
     }
 
-  def main(args: Array[String]): Unit = {
-    val exitCode = main0(
-      args,
-      new DefaultParseCache,
-      System.in,
-      System.out,
-      System.err,
-      os.pwd,
-      None
-    )
-    System.exit(exitCode)
-  }
-
   def main0(
       args: Array[String],
       parseCache: ParseCache,

sjsonnet/src-jvm-native/sjsonnet/stdlib/NativeGzip.scala

@@ -0,0 +1,16 @@
+package sjsonnet.stdlib
+
+import sjsonnet.{Error, EvalScope, Lazy, Platform, Position, Val}
+
+object NativeGzip {
+  def functions: Map[String, Val.Builtin] = Map(
+    "gzip" -> new Val.Builtin1("gzip", "v") {
+      override def evalRhs(v: Lazy, ev: EvalScope, pos: Position): Val = v.force match {
+        case Val.Str(_, value) => Val.Str(pos, Platform.gzipString(value))
+        case arr: Val.Arr      =>
+          Val.Str(pos, Platform.gzipBytes(arr.iterator.map(_.cast[Val.Num].asInt.toByte).toArray))
+        case x => Error.fail("Cannot gzip encode " + x.prettyName)
+      }
+    }
+  )
+}

sjsonnet/src-jvm/sjsonnet/SjsonnetMain.scala

@@ -0,0 +1,21 @@
+package sjsonnet
+
+import sjsonnet.stdlib.{NativeGzip, NativeRegex, NativeXz}
+
+object SjsonnetMain {
+  def main(args: Array[String]): Unit = {
+    val exitCode = SjsonnetMainBase.main0(
+      args,
+      new DefaultParseCache,
+      System.in,
+      System.out,
+      System.err,
+      os.pwd,
+      None,
+      std = new Std(nativeFunctions =
+        NativeXz.functions ++ NativeGzip.functions ++ NativeRegex.functions
+      ).Std
+    )
+    System.exit(exitCode)
+  }
+}

sjsonnet/src-jvm/sjsonnet/stdlib/NativeXz.scala

@@ -0,0 +1,40 @@
+package sjsonnet.stdlib
+
+import sjsonnet.{Error, EvalScope, Lazy, Platform, Position, Val}
+
+object NativeXz {
+  private val dummyPos: Position = new Position(null, 0)
+
+  def functions: Map[String, Val.Builtin] = Map(
+    "xz" -> new Val.Builtin2(
+      "xz",
+      "v",
+      "compressionLevel",
+      Array(Val.Null(dummyPos), Val.Null(dummyPos))
+    ) {
+      override def evalRhs(arg1: Lazy, arg2: Lazy, ev: EvalScope, pos: Position): Val = {
+        val compressionLevel: Option[Int] = arg2.force match {
+          case Val.Null(_) =>
+            // Use default compression level if the user didn't set one
+            None
+          case Val.Num(_, n) =>
+            Some(n.toInt)
+          case x =>
+            Error.fail("Cannot xz encode with compression level " + x.prettyName)
+        }
+        arg1.force match {
+          case Val.Str(_, value) => Val.Str(pos, Platform.xzString(value, compressionLevel))
+          case arr: Val.Arr      =>
+            Val.Str(
+              pos,
+              Platform.xzBytes(
+                arr.iterator.map(_.cast[Val.Num].asInt.toByte).toArray,
+                compressionLevel
+              )
+            )
+          case x => Error.fail("Cannot xz encode " + x.prettyName)
+        }
+      }
+    }
+  )
+}

sjsonnet/src-native/sjsonnet/SjsonnetMain.scala

@@ -0,0 +1,19 @@
+package sjsonnet
+
+import sjsonnet.stdlib.{NativeGzip, NativeRegex}
+
+object SjsonnetMain {
+  def main(args: Array[String]): Unit = {
+    val exitCode = SjsonnetMainBase.main0(
+      args,
+      new DefaultParseCache,
+      System.in,
+      System.out,
+      System.err,
+      os.pwd,
+      None,
+      std = new Std(nativeFunctions = NativeGzip.functions ++ NativeRegex.functions).Std
+    )
+    System.exit(exitCode)
+  }
+}

sjsonnet/src/sjsonnet/Std.scala

@@ -15,53 +15,17 @@ import scala.collection.mutable
  * `builtin` and other helpers to handle the common wrapper logic automatically
  */
 class Std(
-    private val additionalNativeFunctions: Map[String, Val.Func] = Map.empty,
+    private val nativeFunctions: Map[String, Val.Func] = Map.empty,
     private val additionalStdFunctions: Map[String, Val.Func] = Map.empty)
     extends FunctionBuilder {
   // keep for binary compatibility
-  def this(additionalNativeFunctions: Map[String, Val.Func]) =
-    this(additionalNativeFunctions, Map.empty)
+  def this(nativeFunctions: Map[String, Val.Func]) =
+    this(nativeFunctions, Map.empty)
 
   private val dummyPos: Position = new Position(null, 0)
   private val emptyLazyArray = new Array[Lazy](0)
   private val leadingWhiteSpacePattern = Platform.getPatternFromCache("^[ \t\n\f\r\u0085\u00A0']+")
   private val trailingWhiteSpacePattern = Platform.getPatternFromCache("[ \t\n\f\r\u0085\u00A0']+$")
-  private val builtinNativeFunctions = Map(
-    builtin("gzip", "v") { (_, _, v: Val) =>
-      v match {
-        case Val.Str(_, value) => Platform.gzipString(value)
-        case arr: Val.Arr      =>
-          Platform.gzipBytes(arr.iterator.map(_.cast[Val.Num].asInt.toByte).toArray)
-        case x => Error.fail("Cannot gzip encode " + x.prettyName)
-      }
-    },
-    builtinWithDefaults("xz", "v" -> null, "compressionLevel" -> Val.Null(dummyPos)) {
-      (args, _, _) =>
-        val compressionLevel: Option[Int] = args(1) match {
-          case Val.Null(_) =>
-            // Use default compression level if the user didn't set one
-            None
-          case Val.Num(_, n) =>
-            Some(n.toInt)
-          case x =>
-            Error.fail("Cannot xz encode with compression level " + x.prettyName)
-        }
-        args(0) match {
-          case Val.Str(_, value) => Platform.xzString(value, compressionLevel)
-          case arr: Val.Arr      =>
-            Platform.xzBytes(
-              arr.iterator.map(_.cast[Val.Num].asInt.toByte).toArray,
-              compressionLevel
-            )
-          case x => Error.fail("Cannot xz encode " + x.prettyName)
-        }
-    }
-  ) ++ StdRegex.functions
-  require(
-    builtinNativeFunctions.forall(k => !additionalNativeFunctions.contains(k._1)),
-    "Conflicting native functions"
-  )
-  private val nativeFunctions = builtinNativeFunctions ++ additionalNativeFunctions
 
   private object AssertEqual extends Val.Builtin2("assertEqual", "a", "b") {
     def evalRhs(v1: Lazy, v2: Lazy, ev: EvalScope, pos: Position): Val = {
@@ -2000,11 +1964,6 @@ class Std(
       }
     },
     builtin(Native)
-  ) ++ builtinNativeFunctions
-
-  require(
-    functions.forall(k => !additionalStdFunctions.contains(k._1)),
-    "Conflicting std functions"
   )
 
   private def toSetArr(args: Array[Val], idx: Int, pos: Position, ev: EvalScope) = {

sjsonnet/src/sjsonnet/StdRegex.scala → sjsonnet/src/sjsonnet/stdlib/NativeRegex.scala

@@ -1,9 +1,10 @@
-package sjsonnet
+package sjsonnet.stdlib
 
 import sjsonnet.Expr.Member.Visibility
 import sjsonnet.Val.Obj
+import sjsonnet._
 
-object StdRegex {
+object NativeRegex {
   private final def regexPartialMatch(pos: Position, pattern: String, str: String): Val = {
     val compiledPattern = Platform.getPatternFromCache(pattern)
     val matcher = compiledPattern.matcher(str)

sjsonnet/test/src-js/sjsonnet/BaseFileTests.scala

@@ -1,13 +1,15 @@
 package sjsonnet
 
+import sjsonnet.stdlib.NativeRegex
+
 import java.nio.charset.StandardCharsets
 import scala.scalajs.js
 import utest._
 
 abstract class BaseFileTests extends TestSuite {
   private val stderr = new StringBuffer()
   private val std = new Std(
-    additionalNativeFunctions = Map(
+    nativeFunctions = Map(
       "jsonToString" -> new Val.Builtin1("jsonToString", "x") {
         override def evalRhs(arg1: Lazy, ev: EvalScope, pos: Position): Val = {
           Val.Str(
@@ -29,7 +31,7 @@ abstract class BaseFileTests extends TestSuite {
         override def evalRhs(ev: EvalScope, pos: Position): Val =
           throw new RuntimeException("native function panic")
       }
-    )
+    ) ++ NativeRegex.functions
   )
 
   def importResolver(
@@ -103,7 +105,7 @@ abstract class BaseFileTests extends TestSuite {
   }
 
   def check(files: Map[String, () => Array[Byte]], fileName: String, testSuite: String): Unit = {
-    println(s"Checking $fileName")
+    println(s"Checking $testSuite/$fileName")
     val goldenContent = if (files.contains(fileName + ".golden_js")) {
       new String(files(fileName + ".golden_js")(), StandardCharsets.UTF_8)
     } else {

sjsonnet/test/src-js/sjsonnet/FileTests.scala

@@ -5,7 +5,6 @@ import utest._
 object FileTests extends BaseFileTests {
   val skippedTests = Set(
     "stdlib.jsonnet",
-    "regex.jsonnet",
 
     // Stack size issues with the JS runner
     "recursive_function.jsonnet",
@@ -55,5 +54,15 @@ object FileTests extends BaseFileTests {
         check(TestResources_go_test_suite.files, file, "go_test_suite")
       }
     }
+
+    test("new_test_suite") - {
+      val t = TestResources_new_test_suite.files.keys.toSeq
+        .filter(f => f.matches("[^/]+-js\\.jsonnet"))
+        .sorted
+      assert(t.nonEmpty)
+      t.foreach { file =>
+        check(TestResources_new_test_suite.files, file, "new_test_suite")
+      }
+    }
   }
 }