Change loading of external readers (#1394)

This introduces breaking changes for external readers are loaded:

1. In PklProject, relative paths are resolved relative to the enclosing
PklProject file (make behavior consistent with how other settings work)
2. Make CLI flags blow away any settings set on a PklProject
3. Introduce a new `workingDir` property, which defaults to the
PklProject dir

The overall goal is to make this behavior consistent with how other
settings work.
For example, relative paths for other evaluator settings are already
relative to the project directory.
Additionally, in every other case, CLI flags will overwrite any setting
set within PklProject.

Author: bioball

.idea/inspectionProfiles/Project_Default.xml

@@ -78,8 +78,10 @@ endif::[]
 :uri-stdlib-pklbinaryModule: {uri-pkl-stdlib-docs}/pklbinary
 :uri-stdlib-yamlModule: {uri-pkl-stdlib-docs}/yaml
 :uri-stdlib-YamlParser: {uri-stdlib-yamlModule}/Parser
+:uri-stdlib-projectModule: {uri-pkl-stdlib-docs}/Project
 :uri-stdlib-evaluatorSettingsModule: {uri-pkl-stdlib-docs}/EvaluatorSettings
 :uri-stdlib-evaluatorSettingsHttpClass: {uri-stdlib-evaluatorSettingsModule}/Http
+:uri-stdlib-evaluatorSettingsExternalReaderClass: {uri-stdlib-evaluatorSettingsModule}/ExternalReader
 :uri-stdlib-Boolean: {uri-stdlib-baseModule}/Boolean
 :uri-stdlib-xor: {uri-stdlib-baseModule}/Boolean#xor()
 :uri-stdlib-implies: {uri-stdlib-baseModule}/Boolean#implies()

docs/modules/ROOT/partials/component-attributes.adoc

@@ -21,17 +21,31 @@ XXX
 
 Ready when you need them.
 
-.XXX
-[%collapsible]
-====
-XXX
-====
+=== Standard Library Changes
+
+==== `pkl:EvaluatorSettings`
+
+**Additions**
+
+* New method: link:{uri-stdlib-evaluatorSettingsModule}#resolve()[`EvaluatorSettings.resolve()`]
+* New method: link:{uri-stdlib-evaluatorSettingsModule}#resolveForOs()[`EvaluatorSettings.resolveForOs()`]
+* New property: link:{uri-stdlib-evaluatorSettingsExternalReaderClass}#workingDir[`EvaluatorSettings#ExternalReader.workingDir`]
+
+==== `pkl:Project`
+
+**Additions**
+
+* New property: link:{uri-stdlib-projectModule}#resolvedEvaluatorSettings[`Project.resolvedEvaluatorSettings`]
 
 == Breaking Changes [small]#💔#
 
 Things to watch out for when upgrading.
 
-=== Removed Java APIs
+=== Java API changes
+
+Changes have been made to the Java API.
+
+==== Removals and deprecations
 
 The following APIs have been removed without replacement.
 
@@ -41,11 +55,26 @@ The following APIs have been deprecated for removal.
 
 * `org.pkl.config.java.mapper.NonNull` (https://github.com/apple/pkl/pull/1607[#1607]).
 
-.XXX
-[%collapsible]
-====
-XXX
-====
+==== Changes
+
+* `org.pkl.core.evaluatorSettings.PklEvaluatorSettings.parse` no longer accepts a `pathNormalizer` argument.
+
+=== Loading rule changes in `pkl:EvaluatorSettings`
+
+Breaking changes have been made to how evaluator settings are loaded (https://github.com/apple/pkl/pull/1394[#1394]).
+
+==== Loading rule changes for the external reader executable
+
+The following changes have been made for the `executable` property in an external reader:
+
+* If the executable does not contain a slash (`/` on POSIX, `\` on Windows) character, it is always resolved against the `PATH` environment variable.
+* If it does contain a slash, it is resolved relative to the enclosing PklProject directory, instead of the current working directory.
+
+=== Changes to `--external-module-reader` and `--external-resource-reader` CLI flags 
+
+The `--external-module-reader` and `--external-resource-reader` CLI flags will _replace_ any external readers otherwise configured within a PklProject, instead of add to it (https://github.com/apple/pkl/pull/1394[#1394]).
+
+This makes this behavior consistent with how other settings work.
 
 == Work In Progress [small]#🚆#
 

docs/modules/release-notes/pages/0.32.adoc

@@ -148,10 +148,10 @@ data class CliBaseOptions(
   val httpHeaders: Map<String, Map<String, List<String>>>? = null,
 
   /** External module reader process specs */
-  val externalModuleReaders: Map<String, ExternalReader> = mapOf(),
+  val externalModuleReaders: Map<String, ExternalReader>? = null,
 
   /** External resource reader process specs */
-  val externalResourceReaders: Map<String, ExternalReader> = mapOf(),
+  val externalResourceReaders: Map<String, ExternalReader>? = null,
 
   /** Defines options for the formatting of calls to the trace() method. */
   val traceMode: TraceMode? = null,

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -110,7 +110,7 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
   private val evaluatorSettings: PklEvaluatorSettings? by lazy {
     @Suppress("PklCliDirectProjectEvaluatorSettingsAccess")
-    if (cliOptions.omitProjectSettings) null else project?.evaluatorSettings
+    if (cliOptions.omitProjectSettings) null else project?.resolvedEvaluatorSettings
   }
 
   protected val allowedModules: List<Pattern> by lazy {
@@ -193,11 +193,11 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
   }
 
   protected val externalModuleReaders: Map<String, PklEvaluatorSettings.ExternalReader> by lazy {
-    (evaluatorSettings?.externalModuleReaders ?: emptyMap()) + cliOptions.externalModuleReaders
+    cliOptions.externalModuleReaders ?: evaluatorSettings?.externalModuleReaders ?: mapOf()
   }
 
   protected val externalResourceReaders: Map<String, PklEvaluatorSettings.ExternalReader> by lazy {
-    (evaluatorSettings?.externalResourceReaders ?: emptyMap()) + cliOptions.externalResourceReaders
+    cliOptions.externalResourceReaders ?: evaluatorSettings?.externalResourceReaders ?: mapOf()
   }
 
   private val externalProcesses:

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -55,7 +55,8 @@ class BaseOptions : OptionGroup() {
             if (IoUtils.isUriLike(moduleName)) URI(moduleName)
             // Can't just use URI constructor, because URI(null, null, "C:/foo/bar", null) turns
             // into `URI("C", null, "/foo/bar", null)`.
-            else if (IoUtils.isWindowsAbsolutePath(moduleName)) Path.of(moduleName).toUri()
+            else if (IoUtils.isWindows() && IoUtils.isWindowsAbsolutePath(moduleName))
+              Path.of(moduleName).toUri()
             else URI(null, null, IoUtils.toNormalizedPathString(Path.of(moduleName)), null)
           } catch (e: URISyntaxException) {
             val message = buildString {
@@ -91,7 +92,7 @@ class BaseOptions : OptionGroup() {
     > {
       return splitPair(delimiter).convert {
         val cmd = shlex(it.second)
-        Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1)))
+        Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1), null))
       }
     }
 
@@ -386,8 +387,8 @@ class BaseOptions : OptionGroup() {
       httpNoProxy = noProxy,
       httpRewrites = httpRewrites.ifEmpty { null },
       httpHeaders = httpHeaders.ifEmpty { null },
-      externalModuleReaders = externalModuleReaders,
-      externalResourceReaders = externalResourceReaders,
+      externalModuleReaders = externalModuleReaders.ifEmpty { null },
+      externalResourceReaders = externalResourceReaders.ifEmpty { null },
       traceMode = traceMode,
       powerAssertionsEnabled = powerAssertionsEnabled,
     )

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -98,17 +98,17 @@ class BaseCommandTest {
     assertThat(cmd.baseOptions.externalModuleReaders)
       .isEqualTo(
         mapOf(
-          "scheme3" to ExternalReader("reader3", emptyList()),
-          "scheme4" to ExternalReader("reader4", listOf("with", "args")),
-          "scheme+ext" to ExternalReader("reader5", listOf("with", "args")),
+          "scheme3" to ExternalReader("reader3", emptyList(), null),
+          "scheme4" to ExternalReader("reader4", listOf("with", "args"), null),
+          "scheme+ext" to ExternalReader("reader5", listOf("with", "args"), null),
         )
       )
     assertThat(cmd.baseOptions.externalResourceReaders)
       .isEqualTo(
         mapOf(
-          "scheme1" to ExternalReader("reader1", emptyList()),
-          "scheme2" to ExternalReader("reader2", listOf("with", "args")),
-          "scheme+ext" to ExternalReader("reader5", listOf("with", "args")),
+          "scheme1" to ExternalReader("reader1", emptyList(), null),
+          "scheme2" to ExternalReader("reader2", listOf("with", "args"), null),
+          "scheme+ext" to ExternalReader("reader5", listOf("with", "args"), null),
         )
       )
   }

pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt

@@ -28,12 +28,12 @@ import org.pkl.commons.cli.commands.BaseCommand
 import org.pkl.commons.cli.commands.ProjectOptions
 import org.pkl.commons.writeString
 import org.pkl.core.SecurityManagers
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
 import org.pkl.core.evaluatorSettings.TraceMode
 import org.pkl.core.util.IoUtils
 
 @OptIn(ExperimentalPathApi::class)
 class CliCommandTest {
-
   private val cmd =
     object : BaseCommand("test", "") {
       val projectOptions: ProjectOptions by ProjectOptions()
@@ -172,4 +172,32 @@ class CliCommandTest {
           .isNotNull
       }
   }
+
+  @Test
+  fun `--external-module-reader blows away PklProject externalModuleReaders`(
+    @TempDir tempDir: Path
+  ) {
+    tempDir
+      .resolve("PklProject")
+      .writeString(
+        // language=pkl
+        """
+        amends "pkl:Project"
+
+        evaluatorSettings {
+          externalModuleReaders {
+            ["foo"] {
+              executable = "foo"
+            }
+          }
+        }
+        """
+          .trimIndent()
+      )
+    cmd.parse(arrayOf("--external-module-reader", "bar=bar"))
+    val opts = cmd.baseOptions.baseOptions(emptyList(), null, true)
+    val cliTest = CliTest(opts)
+    assertThat(cliTest.myExternalModuleReaders)
+      .isEqualTo(mapOf("bar" to PklEvaluatorSettings.ExternalReader("bar", listOf(), null)))
+  }
 }

pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/CliCommandTest.kt

@@ -27,7 +27,17 @@ plugins {
   idea
 }
 
-val generatorSourceSet = sourceSets.register("generator")
+val generatorSourceSet: NamedDomainObjectProvider<SourceSet> = sourceSets.register("generator")
+
+val externalReaderFixtureSourceSet: NamedDomainObjectProvider<SourceSet> =
+  sourceSets.register("externalReaderFixture") {
+    compileClasspath += sourceSets.test.get().output + sourceSets.test.get().compileClasspath
+    runtimeClasspath += sourceSets.test.get().output + sourceSets.test.get().runtimeClasspath
+  }
+
+val externalReaderFixtureImplementation: Configuration by configurations.getting {
+  extendsFrom(configurations.testImplementation.get())
+}
 
 idea {
   module {
@@ -126,8 +136,45 @@ tasks.withType<JavaCompile>().configureEach {
 
 tasks.compileKotlin { enabled = false }
 
+val externalReaderFixture by tasks.registering {
+  group = "build"
+  dependsOn(tasks.named("compileExternalReaderFixtureJava"))
+  inputs.files(externalReaderFixtureSourceSet.map { it.output })
+  val fileName = if (buildInfo.os.isWindows) "externalreader.bat" else "externalreader"
+  val outputFile = layout.buildDirectory.file("fixtures/$fileName")
+  outputs.file(outputFile)
+  doLast {
+    val classpath = externalReaderFixtureSourceSet.get().runtimeClasspath.asPath
+    val scriptContent =
+      if (buildInfo.os.isWindows) {
+        """
+          @echo off
+          java -cp $classpath org.pkl.core.externalreaderfixture.Main
+        """
+          .trimIndent()
+      } else {
+        """
+          #!/usr/bin/env bash
+
+          java -cp $classpath org.pkl.core.externalreaderfixture.Main
+        """
+          .trimIndent()
+      }
+
+    outputFile.get().asFile.writeText(scriptContent)
+    outputFile.get().asFile.setExecutable(true)
+    println("Created external reader ${outputFile.get().asFile.absolutePath}")
+  }
+}
+
 tasks.test {
   configureTest()
+  dependsOn(externalReaderFixture)
+  environment(
+    "PATH",
+    listOf(System.getenv("PATH"), layout.buildDirectory.dir("fixtures/").get())
+      .joinToString(File.pathSeparator),
+  )
   useJUnitPlatform {
     excludeEngines("MacAmd64LanguageSnippetTestsEngine")
     excludeEngines("MacAarch64LanguageSnippetTestsEngine")
@@ -139,6 +186,11 @@ tasks.test {
 
   // testing very large lists requires more memory than the default 512m!
   maxHeapSize = "1g"
+
+  systemProperty(
+    "org.pkl.core.testExternalReaderPath",
+    externalReaderFixture.map { it.outputs.files.singleFile.absolutePath },
+  )
 }
 
 val generateBaseModuleMembers by

pkl-core/pkl-core.gradle.kts

@@ -0,0 +1,24 @@
+/*
+ * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.externalreader
+
+import org.pkl.core.messaging.MessageTransports
+
+private val systemInDecoder = ExternalReaderMessagePackDecoder(System.`in`)
+
+private val systemOutEncoder = ExternalReaderMessagePackEncoder(System.out)
+
+val stdioTransport = MessageTransports.stream(systemInDecoder, systemOutEncoder) {}