Use logs dir as working directory (#124966)

In the unexpected case that Elasticsearch dies due to a segfault or
other similar native issue, a core dump is useful in diagnosing the
problem. Yet core dumps are written to the working directory, which is
read-only for most installations of Elasticsearch. This commit changes
the working directory to the logs dir which should always be writeable.

Author: rjernst

Diff for: build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy

@@ -179,9 +179,9 @@ echo "Running elasticsearch \$0"
 
         file(distProjectFolder, 'src/config/elasticsearch.properties') << "some propes"
         file(distProjectFolder, 'src/config/jvm.options') << """
--Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m
--XX:ErrorFile=logs/hs_err_pid%p.log
--XX:HeapDumpPath=data
+-Xlog:gc*,gc+age=trace,safepoint:file=gc.log:utctime,level,pid,tags:filecount=32,filesize=64m
+-XX:ErrorFile=hs_err_pid%p.log
+# -XX:HeapDumpPath=/heap/dump/path
 """
         file(distProjectFolder, 'build.gradle') << """
             import org.gradle.api.internal.artifacts.ArtifactAttributes;

Diff for: build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -1435,30 +1435,57 @@ private void tweakJvmOptions(Path configFileRoot) {
         Path jvmOptions = configFileRoot.resolve("jvm.options");
         try {
             String content = new String(Files.readAllBytes(jvmOptions));
-            Map<String, String> expansions = jvmOptionExpansions();
-            for (String origin : expansions.keySet()) {
-                if (content.contains(origin) == false) {
-                    throw new IOException("template property " + origin + " not found in template.");
+            Map<ReplacementKey, String> expansions = jvmOptionExpansions();
+            for (var entry : expansions.entrySet()) {
+                ReplacementKey replacement = entry.getKey();
+                String key = replacement.key();
+                if (content.contains(key) == false) {
+                    key = replacement.fallback();
+                    if (content.contains(key) == false) {
+                        throw new IOException("Template property '" + replacement + "' not found in template:\n" + content);
+                    }
                 }
-                content = content.replace(origin, expansions.get(origin));
+                content = content.replace(key, entry.getValue());
             }
             Files.write(jvmOptions, content.getBytes());
         } catch (IOException ioException) {
             throw new UncheckedIOException(ioException);
         }
     }
 
-    private Map<String, String> jvmOptionExpansions() {
-        Map<String, String> expansions = new HashMap<>();
+    private record ReplacementKey(String key, String fallback) {}
+
+    private Map<ReplacementKey, String> jvmOptionExpansions() {
+        Map<ReplacementKey, String> expansions = new HashMap<>();
         Version version = getVersion();
-        String heapDumpOrigin = getVersion().onOrAfter("6.3.0") ? "-XX:HeapDumpPath=data" : "-XX:HeapDumpPath=/heap/dump/path";
-        expansions.put(heapDumpOrigin, "-XX:HeapDumpPath=" + confPathLogs);
-        if (version.onOrAfter("6.2.0")) {
-            expansions.put("logs/gc.log", confPathLogs.resolve("gc.log").toString());
+
+        ReplacementKey heapDumpPathSub;
+        if (version.before("8.19.0") && version.onOrAfter("6.3.0")) {
+            heapDumpPathSub = new ReplacementKey("-XX:HeapDumpPath=data", null);
+        } else {
+            // temporarily fall back to the old substitution so both old and new work during backport
+            heapDumpPathSub = new ReplacementKey("# -XX:HeapDumpPath=/heap/dump/path", "-XX:HeapDumpPath=data");
         }
-        if (getVersion().getMajor() >= 7) {
-            expansions.put("-XX:ErrorFile=logs/hs_err_pid%p.log", "-XX:ErrorFile=" + confPathLogs.resolve("hs_err_pid%p.log"));
+        expansions.put(heapDumpPathSub, "-XX:HeapDumpPath=" + confPathLogs);
+
+        ReplacementKey gcLogSub;
+        if (version.before("8.19.0") && version.onOrAfter("6.2.0")) {
+            gcLogSub = new ReplacementKey("logs/gc.log", null);
+        } else {
+            // temporarily check the old substitution first so both old and new work during backport
+            gcLogSub = new ReplacementKey("logs/gc.log", "gc.log");
         }
+        expansions.put(gcLogSub, confPathLogs.resolve("gc.log").toString());
+
+        ReplacementKey errorFileSub;
+        if (version.before("8.19.0") && version.getMajor() >= 7) {
+            errorFileSub = new ReplacementKey("-XX:ErrorFile=logs/hs_err_pid%p.log", null);
+        } else {
+            // temporarily check the old substitution first so both old and new work during backport
+            errorFileSub = new ReplacementKey("-XX:ErrorFile=logs/hs_err_pid%p.log", "-XX:ErrorFile=hs_err_pid%p.log");
+        }
+        expansions.put(errorFileSub, "-XX:ErrorFile=" + confPathLogs.resolve("hs_err_pid%p.log"));
+
         return expansions;
     }
 

Diff for: distribution/build.gradle

@@ -531,7 +531,6 @@ subprojects {
     final String packagingPathData = "path.data: /var/lib/elasticsearch"
     final String pathLogs = "/var/log/elasticsearch"
     final String packagingPathLogs = "path.logs: ${pathLogs}"
-    final String packagingLoggc = "${pathLogs}/gc.log"
 
     String licenseText
     if (isTestDistro) {
@@ -576,23 +575,6 @@ subprojects {
         'rpm': packagingPathLogs,
         'def': '#path.logs: /path/to/logs'
       ],
-      'loggc': [
-        'deb': packagingLoggc,
-        'rpm': packagingLoggc,
-        'def': 'logs/gc.log'
-      ],
-
-      'heap.dump.path': [
-        'deb': "-XX:HeapDumpPath=/var/lib/elasticsearch",
-        'rpm': "-XX:HeapDumpPath=/var/lib/elasticsearch",
-        'def': "-XX:HeapDumpPath=data"
-      ],
-
-      'error.file': [
-        'deb': "-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
-        'rpm': "-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
-        'def': "-XX:ErrorFile=logs/hs_err_pid%p.log"
-      ],
 
       'scripts.footer': [
         /* Debian needs exit 0 on these scripts so we add it here and preserve

Diff for: distribution/src/config/jvm.options

@@ -77,10 +77,10 @@
 
 # specify an alternative path for heap dumps; ensure the directory exists and
 # has sufficient space
-@heap.dump.path@
+# -XX:HeapDumpPath=/heap/dump/path
 
 # specify an alternative path for JVM fatal error logs
-@error.file@
+-XX:ErrorFile=hs_err_pid%p.log
 
 ## GC logging
--Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,level,pid,tags:filecount=32,filesize=64m
+-Xlog:gc*,gc+age=trace,safepoint:file=gc.log:utctime,level,pid,tags:filecount=32,filesize=64m

Diff for: distribution/tools/cli-launcher/src/main/java/org/elasticsearch/launcher/CliToolLauncher.java

@@ -58,7 +58,7 @@ public static void main(String[] args) throws Exception {
         String toolname = getToolName(pinfo.sysprops());
         String libs = pinfo.sysprops().getOrDefault("cli.libs", "");
 
-        command = CliToolProvider.load(toolname, libs).create();
+        command = CliToolProvider.load(pinfo.sysprops(), toolname, libs).create();
         Terminal terminal = Terminal.DEFAULT;
         Runtime.getRuntime().addShutdownHook(createShutdownHook(terminal, command));
 

Diff for: distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java

@@ -10,12 +10,14 @@
 package org.elasticsearch.server.cli;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.SuppressForbidden;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Locale;
@@ -106,7 +108,9 @@ private static List<String> flagsFinal(final List<String> userDefinedJvmOptions)
             userDefinedJvmOptions.stream(),
             Stream.of("-XX:+PrintFlagsFinal", "-version")
         ).flatMap(Function.identity()).toList();
-        final Process process = new ProcessBuilder().command(command).start();
+        final ProcessBuilder builder = new ProcessBuilder().command(command);
+        setWorkingDir(builder);
+        final Process process = builder.start();
         final List<String> output = readLinesFromInputStream(process.getInputStream());
         final List<String> error = readLinesFromInputStream(process.getErrorStream());
         final int status = process.waitFor();
@@ -124,6 +128,14 @@ private static List<String> flagsFinal(final List<String> userDefinedJvmOptions)
         }
     }
 
+    @SuppressForbidden(reason = "ProcessBuilder takes File")
+    private static void setWorkingDir(ProcessBuilder builder) throws IOException {
+        // The real ES process uses the logs dir as the working directory. Since we don't
+        // have the logs dir yet, here we use a temp directory for calculating jvm options.
+        final Path tmpDir = Files.createTempDirectory("final-flags");
+        builder.directory(tmpDir.toFile());
+    }
+
     private static List<String> readLinesFromInputStream(final InputStream is) throws IOException {
         try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) {
             return br.lines().toList();

Diff for: distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java

@@ -269,13 +269,13 @@ interface InvalidLineConsumer {
      * and the following JVM options will not be accepted:
      * <ul>
      *     <li>
-     *         {@code 18:-Xlog:age*=trace,gc*,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m}
+     *         {@code 18:-Xlog:age*=trace,gc*,safepoint:file=gc.log:utctime,pid,tags:filecount=32,filesize=64m}
      *     </li>
      *     <li>
-     *         {@code 18-:-Xlog:age*=trace,gc*,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m}
+     *         {@code 18-:-Xlog:age*=trace,gc*,safepoint:file=gc.log:utctime,pid,tags:filecount=32,filesize=64m}
      *     </li>
      *     <li>
-     *         {@code 18-19:-Xlog:age*=trace,gc*,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m}
+     *         {@code 18-19:-Xlog:age*=trace,gc*,safepoint:file=gc.log:utctime,pid,tags:filecount=32,filesize=64m}
      *     </li>
      * </ul>
      *

Diff for: distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java

@@ -33,6 +33,7 @@
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Locale;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -168,7 +169,7 @@ Environment autoConfigureSecurity(
         assert secureSettingsLoader(env) instanceof KeyStoreLoader;
 
         String autoConfigLibs = "modules/x-pack-core,modules/x-pack-security,lib/tools/security-cli";
-        Command cmd = loadTool("auto-configure-node", autoConfigLibs);
+        Command cmd = loadTool(processInfo.sysprops(), "auto-configure-node", autoConfigLibs);
         assert cmd instanceof EnvironmentAwareCommand;
         @SuppressWarnings("raw")
         var autoConfigNode = (EnvironmentAwareCommand) cmd;
@@ -210,7 +211,7 @@ Environment autoConfigureSecurity(
     // package private for testing
     void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception {
         String pluginCliLibs = "lib/tools/plugin-cli";
-        Command cmd = loadTool("sync-plugins", pluginCliLibs);
+        Command cmd = loadTool(processInfo.sysprops(), "sync-plugins", pluginCliLibs);
         assert cmd instanceof EnvironmentAwareCommand;
         @SuppressWarnings("raw")
         var syncPlugins = (EnvironmentAwareCommand) cmd;
@@ -258,8 +259,8 @@ protected ServerProcess getServer() {
     }
 
     // protected to allow tests to override
-    protected Command loadTool(String toolname, String libs) {
-        return CliToolProvider.load(toolname, libs).create();
+    protected Command loadTool(Map<String, String> sysprops, String toolname, String libs) {
+        return CliToolProvider.load(sysprops, toolname, libs).create();
     }
 
     // protected to allow tests to override
@@ -270,7 +271,8 @@ protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo,
             .withProcessInfo(processInfo)
             .withServerArgs(args)
             .withTempDir(tempDir)
-            .withJvmOptions(jvmOptions);
+            .withJvmOptions(jvmOptions)
+            .withWorkingDir(args.logsDir());
         return serverProcessBuilder.start();
     }
 

Diff for: distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.core.SuppressForbidden;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -43,6 +44,7 @@ public class ServerProcessBuilder {
     private ServerArgs serverArgs;
     private ProcessInfo processInfo;
     private List<String> jvmOptions;
+    private Path workingDir;
     private Terminal terminal;
 
     // this allows mocking the process building by tests
@@ -82,6 +84,11 @@ public ServerProcessBuilder withJvmOptions(List<String> jvmOptions) {
         return this;
     }
 
+    public ServerProcessBuilder withWorkingDir(Path workingDir) {
+        this.workingDir = workingDir;
+        return this;
+    }
+
     /**
      * Specifies the {@link Terminal} to use for reading input and writing output from/to the cli console
      */
@@ -155,7 +162,7 @@ ServerProcess start(ProcessStarter processStarter) throws UserException {
 
         boolean success = false;
         try {
-            jvmProcess = createProcess(getCommand(), getJvmArgs(), jvmOptions, getEnvironment(), processStarter);
+            jvmProcess = createProcess(getCommand(), getJvmArgs(), jvmOptions, getEnvironment(), workingDir, processStarter);
             errorPump = new ErrorPumpThread(terminal, jvmProcess.getErrorStream());
             errorPump.start();
             sendArgs(serverArgs, jvmProcess.getOutputStream());
@@ -185,16 +192,23 @@ private static Process createProcess(
         List<String> jvmArgs,
         List<String> jvmOptions,
         Map<String, String> environment,
+        Path workingDir,
         ProcessStarter processStarter
     ) throws InterruptedException, IOException {
 
         var builder = new ProcessBuilder(Stream.concat(Stream.of(command), Stream.concat(jvmOptions.stream(), jvmArgs.stream())).toList());
         builder.environment().putAll(environment);
+        setWorkingDir(builder, workingDir);
         builder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
 
         return processStarter.start(builder);
     }
 
+    @SuppressForbidden(reason = "ProcessBuilder takes File")
+    private static void setWorkingDir(ProcessBuilder builder, Path path) {
+        builder.directory(path.toFile());
+    }
+
     private static void sendArgs(ServerArgs args, OutputStream processStdin) {
         // DO NOT close the underlying process stdin, since we need to be able to write to it to signal exit
         var out = new OutputStreamStreamOutput(processStdin);

Diff for: distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -24,6 +24,7 @@
 final class SystemJvmOptions {
 
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
+        Path esHome = Path.of(sysprops.get("es.path.home"));
         String distroType = sysprops.get("es.distribution.type");
         String javaType = sysprops.get("es.java.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
@@ -67,7 +68,8 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Djava.locale.providers=CLDR",
                 // Enable vectorization for whatever version we are running. This ensures we use vectorization even when running EA builds.
                 "-Dorg.apache.lucene.vectorization.upperJavaFeatureVersion=" + Runtime.version().feature(),
-                // Pass through distribution type and java type
+                // Pass through some properties
+                "-Des.path.home=" + esHome,
                 "-Des.distribution.type=" + distroType,
                 "-Des.java.type=" + javaType
             ),
@@ -77,7 +79,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
             maybeSetReplayFile(distroType, isHotspot),
             maybeWorkaroundG1Bug(),
             maybeAllowSecurityManager(useEntitlements),
-            maybeAttachEntitlementAgent(useEntitlements)
+            maybeAttachEntitlementAgent(esHome, useEntitlements)
         ).flatMap(s -> s).toList();
     }
 
@@ -159,12 +161,12 @@ private static Stream<String> maybeAllowSecurityManager(boolean useEntitlements)
         return Stream.of();
     }
 
-    private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlements) {
+    private static Stream<String> maybeAttachEntitlementAgent(Path esHome, boolean useEntitlements) {
         if (useEntitlements == false) {
             return Stream.empty();
         }
 
-        Path dir = Path.of("lib", "entitlement-bridge");
+        Path dir = esHome.resolve("lib/entitlement-bridge");
         if (Files.exists(dir) == false) {
             throw new IllegalStateException("Directory for entitlement bridge jar does not exist: " + dir);
         }