Change loading of external readers

docs/modules/release-notes/pages/0.31.adoc

@@ -46,7 +46,14 @@ The following APIs have been added:
 
 Things to watch out for when upgrading.
 
-=== XXX
+=== Rule changes when loading external readers
+
+The following changes have been made when loading external readers:
+
+1. Executable paths declared inside a PklProject file are resolved relative to the enclosing directory, instead of the current working directory.
+2. The `--external-module-reader` and `--external-resource-reader` CLI flags will _replace_ any external readers otherwise configured within a PklProject, instead of add to it. +
++
+This makes this behavior consistent with how other settings work.
 
 == Miscellaneous [small]#🐸#
 

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -145,10 +145,10 @@ data class CliBaseOptions(
   val httpRewrites: Map<URI, URI>? = null,
 
   /** External module reader process specs */
-  val externalModuleReaders: Map<String, ExternalReader> = mapOf(),
+  val externalModuleReaders: Map<String, ExternalReader>? = null,
 
   /** External resource reader process specs */
-  val externalResourceReaders: Map<String, ExternalReader> = mapOf(),
+  val externalResourceReaders: Map<String, ExternalReader>? = null,
 
   /** Defines options for the formatting of calls to the trace() method. */
   val traceMode: TraceMode? = null,

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -106,7 +106,7 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
   }
 
   private val evaluatorSettings: PklEvaluatorSettings? by lazy {
-    if (cliOptions.omitProjectSettings) null else project?.evaluatorSettings
+    if (cliOptions.omitProjectSettings) null else project?.resolvedEvaluatorSettings
   }
 
   protected val allowedModules: List<Pattern> by lazy {
@@ -169,31 +169,25 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
   protected val useColor: Boolean by lazy { cliOptions.color?.hasColor() ?: false }
 
   private val proxyAddress: URI? by lazy {
-    cliOptions.httpProxy
-      ?: project?.evaluatorSettings?.http?.proxy?.address
-      ?: settings.http?.proxy?.address
+    cliOptions.httpProxy ?: evaluatorSettings?.http?.proxy?.address ?: settings.http?.proxy?.address
   }
 
   private val noProxy: List<String>? by lazy {
     cliOptions.httpNoProxy
-      ?: project?.evaluatorSettings?.http?.proxy?.noProxy
+      ?: evaluatorSettings?.http?.proxy?.noProxy
       ?: settings.http?.proxy?.noProxy
   }
 
   private val httpRewrites: Map<URI, URI>? by lazy {
-    cliOptions.httpRewrites
-      ?: project?.evaluatorSettings?.http?.rewrites
-      ?: settings.http?.rewrites()
+    cliOptions.httpRewrites ?: evaluatorSettings?.http?.rewrites ?: settings.http?.rewrites()
   }
 
-  private val externalModuleReaders: Map<String, PklEvaluatorSettings.ExternalReader> by lazy {
-    (project?.evaluatorSettings?.externalModuleReaders ?: emptyMap()) +
-      cliOptions.externalModuleReaders
+  protected val externalModuleReaders: Map<String, PklEvaluatorSettings.ExternalReader> by lazy {
+    cliOptions.externalModuleReaders ?: evaluatorSettings?.externalModuleReaders ?: mapOf()
   }
 
-  private val externalResourceReaders: Map<String, PklEvaluatorSettings.ExternalReader> by lazy {
-    (project?.evaluatorSettings?.externalResourceReaders ?: emptyMap()) +
-      cliOptions.externalResourceReaders
+  protected val externalResourceReaders: Map<String, PklEvaluatorSettings.ExternalReader> by lazy {
+    cliOptions.externalResourceReaders ?: evaluatorSettings?.externalResourceReaders ?: mapOf()
   }
 
   private val externalProcesses:
@@ -207,7 +201,7 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
   }
 
   private val traceMode: TraceMode by lazy {
-    cliOptions.traceMode ?: project?.evaluatorSettings?.traceMode ?: TraceMode.COMPACT
+    cliOptions.traceMode ?: evaluatorSettings?.traceMode ?: TraceMode.COMPACT
   }
 
   private fun HttpClient.Builder.addDefaultCliCertificates() {

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -340,8 +340,8 @@ class BaseOptions : OptionGroup() {
       httpProxy = proxy,
       httpNoProxy = noProxy,
       httpRewrites = httpRewrites.ifEmpty { null },
-      externalModuleReaders = externalModuleReaders,
-      externalResourceReaders = externalResourceReaders,
+      externalModuleReaders = externalModuleReaders.ifEmpty { null },
+      externalResourceReaders = externalResourceReaders.ifEmpty { null },
       traceMode = traceMode,
     )
   }

pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/CliCommandTest.kt

@@ -16,10 +16,15 @@
 package org.pkl.commons.cli
 
 import com.github.ajalt.clikt.core.parse
+import java.nio.file.Path
+import kotlin.collections.mapOf
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.io.TempDir
 import org.pkl.commons.cli.commands.BaseCommand
+import org.pkl.commons.writeString
 import org.pkl.core.SecurityManagers
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
 
 class CliCommandTest {
 
@@ -28,6 +33,8 @@ class CliCommandTest {
 
     val myAllowedResources = allowedResources
     val myAllowedModules = allowedModules
+    val myExternalModuleReaders = externalModuleReaders
+    val myExternalResourceReaders = externalResourceReaders
   }
 
   private val cmd =
@@ -68,4 +75,32 @@ class CliCommandTest {
           listOf("\\Qscheme1:\\E", "\\Qscheme2:\\E", "\\Qscheme+ext:\\E")
       )
   }
+
+  @Test
+  fun `--external-module-reader blows away PklProject externalModuleReaders`(
+    @TempDir tempDir: Path
+  ) {
+    tempDir
+      .resolve("PklProject")
+      .writeString(
+        // language=pkl
+        """
+      amends "pkl:Project"
+
+      evaluatorSettings {
+        externalModuleReaders {
+          ["foo"] {
+            executable = "foo"
+          }
+        }
+      }
+      """
+          .trimIndent()
+      )
+    cmd.parse(arrayOf("--external-module-reader", "bar=bar"))
+    val opts = cmd.baseOptions.baseOptions(emptyList(), null, true)
+    val cliTest = CliTest(opts)
+    assertThat(cliTest.myExternalModuleReaders)
+      .isEqualTo(mapOf("bar" to PklEvaluatorSettings.ExternalReader("bar", listOf())))
+  }
 }

pkl-core/pkl-core.gradle.kts

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,16 @@ plugins {
   idea
 }
 
-val generatorSourceSet = sourceSets.register("generator")
+val generatorSourceSet: NamedDomainObjectProvider<SourceSet> = sourceSets.register("generator")
+
+val externalReaderFixtureSourceSet: NamedDomainObjectProvider<SourceSet> =
+  sourceSets.register("externalReaderFixture") {
+    compileClasspath += sourceSets.test.get().output + sourceSets.test.get().compileClasspath
+    runtimeClasspath += sourceSets.test.get().output + sourceSets.test.get().runtimeClasspath
+  }
+
+val externalReaderFixtureImplementation: Configuration by
+  configurations.getting { extendsFrom(configurations.testImplementation.get()) }
 
 idea {
   module {
@@ -110,8 +119,46 @@ tasks.compileJava { options.generatedSourceOutputDirectory.set(file("generated/t
 
 tasks.compileKotlin { enabled = false }
 
+val externalReaderFixture by
+  tasks.registering {
+    group = "build"
+    dependsOn(tasks.named("compileExternalReaderFixtureJava"))
+    inputs.files(externalReaderFixtureSourceSet.map { it.output })
+    val fileName = if (buildInfo.os.isWindows) "externalreader.bat" else "externalreader"
+    val outputFile = layout.buildDirectory.file("fixtures/$fileName")
+    outputs.file(outputFile)
+    doLast {
+      val classpath = externalReaderFixtureSourceSet.get().runtimeClasspath.asPath
+      val scriptContent =
+        if (buildInfo.os.isWindows) {
+          """
+          @echo off
+          java -cp $classpath org.pkl.core.externalreaderfixture.Main
+        """
+            .trimIndent()
+        } else {
+          """
+          #!/usr/bin/env bash
+
+          java -cp $classpath org.pkl.core.externalreaderfixture.Main
+        """
+            .trimIndent()
+        }
+
+      outputFile.get().asFile.writeText(scriptContent)
+      outputFile.get().asFile.setExecutable(true)
+      println("Created external reader ${outputFile.get().asFile.absolutePath}")
+    }
+  }
+
 tasks.test {
   configureTest()
+  dependsOn(externalReaderFixture)
+  environment(
+    "PATH",
+    listOf(System.getenv("PATH"), layout.buildDirectory.dir("fixtures/").get())
+      .joinToString(File.pathSeparator),
+  )
   useJUnitPlatform {
     excludeEngines("MacAmd64LanguageSnippetTestsEngine")
     excludeEngines("MacAarch64LanguageSnippetTestsEngine")
@@ -123,6 +170,12 @@ tasks.test {
 
   // testing very large lists requires more memory than the default 512m!
   maxHeapSize = "1g"
+
+  dependsOn(externalReaderFixture)
+  systemProperty(
+    "org.pkl.core.testExternalReaderPath",
+    externalReaderFixture.map { it.outputs.files.singleFile.absolutePath },
+  )
 }
 
 val testJavaExecutable by

pkl-core/src/externalReaderFixture/kotlin/org/pkl/core/externalreaderfixture/Main.kt

@@ -0,0 +1,67 @@
+/*
+ * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@file:JvmName("Main")
+
+package org.pkl.core.externalreaderfixture
+
+import java.net.URI
+import org.pkl.core.externalreader.ExternalModuleReader
+import org.pkl.core.externalreader.ExternalReaderClient
+import org.pkl.core.externalreader.ExternalReaderMessagePackDecoder
+import org.pkl.core.externalreader.ExternalReaderMessagePackEncoder
+import org.pkl.core.externalreader.ExternalResourceReader
+import org.pkl.core.messaging.MessageTransports
+import org.pkl.core.module.PathElement
+
+object ModuleReader : ExternalModuleReader {
+  override val isLocal: Boolean = true
+
+  override fun read(uri: URI): String = "hello"
+
+  override val scheme: String = "foo"
+
+  override val hasHierarchicalUris: Boolean = false
+
+  override val isGlobbable: Boolean = false
+
+  override fun listElements(uri: URI): List<PathElement> {
+    throw NotImplementedError()
+  }
+}
+
+object ResourceReader : ExternalResourceReader {
+  override fun read(uri: URI): ByteArray = "hello".toByteArray()
+
+  override val scheme: String = "foo"
+
+  override val hasHierarchicalUris: Boolean = false
+
+  override val isGlobbable: Boolean = false
+
+  override fun listElements(uri: URI): List<PathElement> {
+    throw NotImplementedError()
+  }
+}
+
+fun main() {
+  val transport =
+    MessageTransports.stream(
+      ExternalReaderMessagePackDecoder(System.`in`),
+      ExternalReaderMessagePackEncoder(System.out),
+    ) {}
+  val client = ExternalReaderClient(listOf(ModuleReader), listOf(ResourceReader), transport)
+  client.run()
+}

pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -475,7 +475,7 @@ public TraceMode getTraceMode() {
    */
   public EvaluatorBuilder applyFromProject(Project project) {
     this.dependencies = project.getDependencies();
-    var settings = project.getEvaluatorSettings();
+    var settings = project.getResolvedEvaluatorSettings();
     if (securityManager != null) {
       throw new IllegalStateException(
           "Cannot call both `setSecurityManager` and `setProject`, because both define security manager settings. Call `setProjectOnly` if the security manager is desired.");

pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,6 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
-import java.util.function.BiFunction;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import org.pkl.core.Duration;
@@ -55,17 +54,13 @@ public record PklEvaluatorSettings(
 
   /** Initializes a {@link PklEvaluatorSettings} from a raw object representation. */
   @SuppressWarnings("unchecked")
-  public static PklEvaluatorSettings parse(
-      Value input, BiFunction<? super String, ? super String, Path> pathNormalizer) {
+  public static PklEvaluatorSettings parse(Value input) {
     if (!(input instanceof PObject pSettings)) {
       throw PklBugException.unreachableCode();
     }
 
     var moduleCacheDirStr = (String) pSettings.get("moduleCacheDir");
-    var moduleCacheDir =
-        moduleCacheDirStr == null
-            ? null
-            : pathNormalizer.apply(moduleCacheDirStr, "moduleCacheDir");
+    var moduleCacheDir = moduleCacheDirStr == null ? null : Path.of(moduleCacheDirStr).normalize();
 
     var allowedModulesStrs = (List<String>) pSettings.get("allowedModules");
     var allowedModules =
@@ -80,13 +75,10 @@ public static PklEvaluatorSettings parse(
             : allowedResourcesStrs.stream().map(Pattern::compile).toList();
 
     var modulePathStrs = (List<String>) pSettings.get("modulePath");
-    var modulePath =
-        modulePathStrs == null
-            ? null
-            : modulePathStrs.stream().map(it -> pathNormalizer.apply(it, "modulePath")).toList();
+    var modulePath = modulePathStrs == null ? null : modulePathStrs.stream().map(Path::of).toList();
 
     var rootDirStr = (String) pSettings.get("rootDir");
-    var rootDir = rootDirStr == null ? null : pathNormalizer.apply(rootDirStr, "rootDir");
+    var rootDir = rootDirStr == null ? null : Path.of(rootDirStr).normalize();
 
     var externalModuleReadersRaw = (Map<String, Value>) pSettings.get("externalModuleReaders");
     var externalModuleReaders =