diff --git a/.editorconfig b/.editorconfig index a683ed60802..2f2d99c5dd2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,7 +5,7 @@ end_of_line = lf insert_final_newline = true charset = utf-8 indent_style = space -indent_size = 4 +indent_size = 2 [*.scala] end_of_line = lf diff --git a/build.mill b/build.mill index ddc503914c1..44d91349add 100644 --- a/build.mill +++ b/build.mill @@ -429,7 +429,7 @@ trait MillPublishJavaModule extends MillJavaModule with PublishModule { "info.releaseNotesURL" -> Settings.changelogUrl ) def pomSettings = MillPublishJavaModule.commonPomSettings(artifactName()) - def javacOptions = Seq("-source", "1.8", "-target", "1.8", "-encoding", "UTF-8") + def javacOptions = Seq("-source", "11", "-target", "11", "-encoding", "UTF-8") } object MillPublishJavaModule { @@ -446,7 +446,6 @@ object MillPublishJavaModule { ) ) } - } /** diff --git a/docs/modules/ROOT/pages/cli/flags.adoc b/docs/modules/ROOT/pages/cli/flags.adoc index 025972be24d..ef8ad18f0d0 100644 --- a/docs/modules/ROOT/pages/cli/flags.adoc +++ b/docs/modules/ROOT/pages/cli/flags.adoc @@ -131,6 +131,13 @@ forcefully terminating the previous process even though it may be still alive: $ mill -w foo.runBackground ---- +Note that even if you interrupt mill watch via CTRL+C, the server spawned by `runBackground` still runs in the +background. To actually stop the background server use: + +[source,console] +---- +> mill clean foo.runBackground +---- === `--jobs`/`-j` diff --git a/integration/invalidation/run-background/src/RunBackgroundTests.scala b/integration/invalidation/run-background/src/RunBackgroundTests.scala index b3d8a467d0e..d238bf0c57a 100644 --- a/integration/invalidation/run-background/src/RunBackgroundTests.scala +++ b/integration/invalidation/run-background/src/RunBackgroundTests.scala @@ -37,6 +37,28 @@ object RunBackgroundTests extends UtestIntegrationTestSuite { os.write(stop, "") eventually { probeLockAvailable(lock) } } + + test("sequential") - integrationTest { tester => + import tester._ + val lock1 = os.temp() + val lock2 = os.temp() + val stop = os.temp() + os.remove(stop) + eval(("foo.runBackground", lock1, stop)) + eventually { !probeLockAvailable(lock1) } + eval(("foo.runBackground", lock2, stop)) + eventually { !probeLockAvailable(lock2) } + Predef.assert( + probeLockAvailable(lock1), + "first process should be exited after second process is running" + ) + + if (tester.clientServerMode) eval("shutdown") + continually { !probeLockAvailable(lock2) } + os.write(stop, "") + eventually { probeLockAvailable(lock2) } + } + test("clean") - integrationTest { tester => import tester._ val lock = os.temp() @@ -45,6 +67,7 @@ object RunBackgroundTests extends UtestIntegrationTestSuite { eval(("foo.runBackground", lock, stop)) eventually { !probeLockAvailable(lock) + } eval(("clean", "foo.runBackground")) diff --git a/main/define/src/mill/define/Task.scala b/main/define/src/mill/define/Task.scala index 55ec1a281b1..c1c46fc2ee6 100644 --- a/main/define/src/mill/define/Task.scala +++ b/main/define/src/mill/define/Task.scala @@ -122,6 +122,12 @@ object Task extends TaskBase { cls: EnclosingClass ): Command[T] = macro Target.Internal.commandImpl[T] + /** Binary compatibility forwarder. */ + def Command( + t: NamedParameterOnlyDummy, + exclusive: Boolean + ): CommandFactory = new CommandFactory(exclusive = exclusive, persistent = false) + /** * @param exclusive Exclusive commands run serially at the end of an evaluation, * without any other tasks running parallel, and without the @@ -129,12 +135,20 @@ object Task extends TaskBase { * These are normally used for "top level" commands which are * run directly to perform some action or display some output * to the user. + * @param persistent Persistent comands do not erase the `Task.dest` folder + * between runs. */ def Command( t: NamedParameterOnlyDummy = new NamedParameterOnlyDummy, - exclusive: Boolean = false - ): CommandFactory = new CommandFactory(exclusive) - class CommandFactory private[mill] (val exclusive: Boolean) extends TaskBase.TraverseCtxHolder { + exclusive: Boolean = false, + persistent: Boolean = false + ): CommandFactory = new CommandFactory(exclusive = exclusive, persistent = persistent) + class CommandFactory private[mill] (val exclusive: Boolean, val persistent: Boolean) + extends TaskBase.TraverseCtxHolder { + + /** Binary compatibility forwarder */ + private[mill] def this(exclusive: Boolean) = this(exclusive, persistent = false) + def apply[T](t: Result[T])(implicit w: W[T], ctx: mill.define.Ctx, @@ -662,7 +676,8 @@ object Target extends TaskBase { w.splice, cls.splice.value, taskIsPrivate.splice, - exclusive = c.prefix.splice.asInstanceOf[Task.CommandFactory].exclusive + exclusive = c.prefix.splice.asInstanceOf[Task.CommandFactory].exclusive, + persistent = c.prefix.splice.asInstanceOf[Task.CommandFactory].persistent ) ) } @@ -871,15 +886,30 @@ class Command[+T]( val writer: W[_], val cls: Class[_], val isPrivate: Option[Boolean], - val exclusive: Boolean + val exclusive: Boolean, + val persistent: Boolean ) extends NamedTask[T] { + override def flushDest: Boolean = !persistent + + /** Binary compatibility forwarder. */ def this( t: Task[T], ctx0: mill.define.Ctx, writer: W[_], cls: Class[_], isPrivate: Option[Boolean] - ) = this(t, ctx0, writer, cls, isPrivate, false) + ) = this(t, ctx0, writer, cls, isPrivate, exclusive = false, persistent = false) + + /** Binary compatibility forwarder. */ + def this( + t: Task[T], + ctx0: mill.define.Ctx, + writer: W[_], + cls: Class[_], + isPrivate: Option[Boolean], + exclusive: Boolean + ) = this(t, ctx0, writer, cls, isPrivate, exclusive = exclusive, persistent = false) + override def asCommand: Some[Command[T]] = Some(this) // FIXME: deprecated return type: Change to Option override def writerOpt: Some[W[_]] = Some(writer) diff --git a/pythonlib/src/mill/pythonlib/PythonModule.scala b/pythonlib/src/mill/pythonlib/PythonModule.scala index 3c905750ca9..7a68d10a103 100644 --- a/pythonlib/src/mill/pythonlib/PythonModule.scala +++ b/pythonlib/src/mill/pythonlib/PythonModule.scala @@ -175,8 +175,8 @@ trait PythonModule extends PipModule with TaskModule { outer => * * @see [[mainScript]] */ - def runBackground(args: mill.define.Args) = Task.Command { - val (procUuidPath, procLockfile, procUuid) = mill.scalalib.RunModule.backgroundSetup(Task.dest) + def runBackground(args: mill.define.Args) = Task.Command(persistent = true) { + val backgroundPaths = new mill.scalalib.RunModule.BackgroundPaths(destDir = Task.dest) val pwd0 = os.Path(java.nio.file.Paths.get(".").toAbsolutePath) os.checker.withValue(os.Checker.Nop) { @@ -185,11 +185,7 @@ trait PythonModule extends PipModule with TaskModule { outer => classPath = mill.scalalib.JvmWorkerModule.backgroundWrapperClasspath().map(_.path).toSeq, jvmArgs = Nil, env = runnerEnvTask(), - mainArgs = Seq( - procUuidPath.toString, - procLockfile.toString, - procUuid, - "500", + mainArgs = backgroundPaths.toArgs ++ Seq( "", pythonExe().path.toString, mainScript().path.toString diff --git a/scalalib/backgroundwrapper/src/mill/scalalib/backgroundwrapper/MillBackgroundWrapper.java b/scalalib/backgroundwrapper/src/mill/scalalib/backgroundwrapper/MillBackgroundWrapper.java index c7d9947c884..5b747fe867c 100644 --- a/scalalib/backgroundwrapper/src/mill/scalalib/backgroundwrapper/MillBackgroundWrapper.java +++ b/scalalib/backgroundwrapper/src/mill/scalalib/backgroundwrapper/MillBackgroundWrapper.java @@ -1,78 +1,213 @@ package mill.scalalib.backgroundwrapper; +import java.io.IOException; import java.io.RandomAccessFile; import java.nio.channels.FileChannel; -import java.nio.file.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Optional; +@SuppressWarnings("BusyWait") public class MillBackgroundWrapper { + private static final long NS_IN_S = 1_000_000_000; + public static void main(String[] args) throws Exception { - Path procUuidPath = Paths.get(args[0]); - Path procLockfile = Paths.get(args[1]); - String procUuid = args[2]; - int lockDelay = Integer.parseInt(args[3]); + var newestProcessIdPath = Paths.get(args[0]); + var currentlyRunningProccesIdPath = Paths.get(args[1]); + var procLockfile = Paths.get(args[2]); + var logPath = Paths.get(args[3]); + var realMain = args[4]; + var realArgs = java.util.Arrays.copyOfRange(args, 5, args.length); + + // The following code should handle this scenario, when we have 2 processes contending for the + // lock. + // + // This can happen if a new MillBackgroundWrapper is launched while the previous one is still + // running due to rapid + // changes in the source code. + // + // Process 1 starts, writes newest_pid=1, claims lock, writes currently_running_pid = 1 + // Process 2 starts, writes newest_pid=2, tries to claim lock but is blocked + // Process 3 starts at the same time as process 2, writes newest_pid=3, tries to claim lock but + // is blocked + // + // Process 1 reads newest_pid=3, terminates, releases lock + // Process 2 claims lock, reads currently_running_pid = 1, waits for process 1 to die, writes + // currently_running_pid = 2 + // Process 2 reads newest_pid=3, terminates, releases lock + // Process 3 claims lock, reads currently_running_pid = 2, waits for process 2 to die, writes + // currently_running_pid = 3, then starts + // Process 3 reads newest_pid=3, continues running - Files.writeString(procUuidPath, procUuid, StandardOpenOption.CREATE); + // Indicate to the previous process that we want to take over. + var myPid = ProcessHandle.current().pid(); + var myPidStr = "" + myPid; + log(logPath, myPid, "Starting. Writing my PID to " + newestProcessIdPath); + Files.writeString( + newestProcessIdPath, + myPidStr, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); // Take a lock on `procLockfile` to ensure that only one // `runBackground` process is running at any point in time. + log(logPath, myPid, "Acquiring lock on " + procLockfile); + + //noinspection resource - this is intentional, file is released when process dies. RandomAccessFile raf = new RandomAccessFile(procLockfile.toFile(), "rw"); FileChannel chan = raf.getChannel(); if (chan.tryLock() == null) { - System.err.println("Waiting for runBackground lock to be available"); + var startTimeNanos = System.nanoTime(); + System.err.println("[mill:runBackground] Waiting for runBackground lock to be available"); + // this is intentional, lock is released when process dies. + //noinspection ResultOfMethodCallIgnored chan.lock(); + var delta = (System.nanoTime() - startTimeNanos) / NS_IN_S; + System.err.println("[mill:runBackground] Lock acquired after " + delta + "s"); } + log(logPath, myPid, "Lock acquired"); - // For some reason even after the previous process exits things like sockets - // may still take time to free, so sleep for a configurable duration before proceeding - Thread.sleep(lockDelay); + var oldProcessPid = readPreviousPid(currentlyRunningProccesIdPath); + log(logPath, myPid, "Old process PID: " + oldProcessPid); + oldProcessPid.ifPresent((oldPid) -> { + var startTimeNanos = System.nanoTime(); + log(logPath, myPid, "Waiting for old process to terminate"); + var processExisted = waitForPreviousProcessToTerminate(oldPid); + if (processExisted) { + log( + logPath, + myPid, + "Old process terminated in " + (System.nanoTime() - startTimeNanos) / NS_IN_S + "s"); + } else { + log(logPath, myPid, "Old process was already terminated."); + } + }); + + log(logPath, myPid, "Writing my PID to " + currentlyRunningProccesIdPath); + Files.writeString( + currentlyRunningProccesIdPath, + myPidStr, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); // Start the thread to watch for updates on the process marker file, // so we can exit if it is deleted or replaced - long startTime = System.currentTimeMillis(); - Thread watcher = new Thread(() -> { - while (true) { - long delta = (System.currentTimeMillis() - startTime) / 1000; - try { - Thread.sleep(1); - String token = Files.readString(procUuidPath); - if (!token.equals(procUuid)) { - System.err.println("runBackground exiting after " + delta + "s"); - System.exit(0); - } - } catch (Exception e) { - System.err.println("runBackground exiting after " + delta + "s"); - System.exit(0); - } - } + var startTimeNanos = System.nanoTime(); // use nanoTime as it is monotonic + checkIfWeStillNeedToBeRunning(logPath, startTimeNanos, newestProcessIdPath, myPid); + var watcher = new Thread(() -> { + while (true) + checkIfWeStillNeedToBeRunning(logPath, startTimeNanos, newestProcessIdPath, myPid); }); - watcher.setDaemon(true); watcher.start(); // Actually start the Java main method we wanted to run in the background - String realMain = args[4]; - String[] realArgs = java.util.Arrays.copyOfRange(args, 5, args.length); if (!realMain.equals("")) { + log( + logPath, + myPid, + "Running main method " + realMain + " with args " + Arrays.toString(realArgs)); Class.forName(realMain).getMethod("main", String[].class).invoke(null, (Object) realArgs); } else { + log(logPath, myPid, "Running subprocess with args " + Arrays.toString(realArgs)); Process subprocess = new ProcessBuilder().command(realArgs).inheritIO().start(); + log( + logPath, + myPid, + "Subprocess started with PID " + subprocess.pid() + ", waiting for it to exit"); Runtime.getRuntime().addShutdownHook(new Thread(() -> { + log(logPath, myPid, "Shutdown hook called, terminating subprocess"); subprocess.destroy(); long now = System.currentTimeMillis(); + // If the process does not shut down withing 100ms kill it forcibly. while (subprocess.isAlive() && System.currentTimeMillis() - now < 100) { try { Thread.sleep(1); } catch (InterruptedException e) { - } - if (subprocess.isAlive()) { - subprocess.destroyForcibly(); + // do nothing } } + if (subprocess.isAlive()) { + log(logPath, myPid, "Forcing subprocess termination"); + subprocess.destroyForcibly(); + } })); - System.exit(subprocess.waitFor()); + + log(logPath, myPid, "Waiting for subprocess to terminate"); + var exitCode = subprocess.waitFor(); + log(logPath, myPid, "Subprocess terminated with exit code " + exitCode); + System.exit(exitCode); + } + } + + private static void checkIfWeStillNeedToBeRunning( + Path logPath, long startTimeNanos, Path newestProcessIdPath, long myPid) { + var delta = (System.nanoTime() - startTimeNanos) / NS_IN_S; + var myPidStr = "" + myPid; + try { + Thread.sleep(50); + var token = Files.readString(newestProcessIdPath); + if (!myPidStr.equals(token)) { + log( + logPath, + myPid, + "New process started, exiting. Token file (" + newestProcessIdPath + ") contents: \"" + + token + "\""); + System.err.println( + "[mill:runBackground] Background process has been replaced with a new process (PID: " + + token + "), exiting after " + delta + "s"); + System.exit(0); + } + } catch (Exception e) { + log(logPath, myPid, "Check if we still need to be running failed, exiting. Exception: " + e); + System.err.println("[mill:runBackground] Background process exiting after " + delta + "s"); + System.exit(0); + } + } + + static Optional readPreviousPid(Path pidFilePath) { + try { + var pidStr = Files.readString(pidFilePath); + return Optional.of(Long.parseLong(pidStr)); + } catch (IOException | NumberFormatException e) { + return Optional.empty(); + } + } + + /** Returns true if the process with the given PID has terminated, false if the process did not exist. */ + static boolean waitForPreviousProcessToTerminate(long pid) { + var maybeOldProcess = ProcessHandle.of(pid); + if (maybeOldProcess.isEmpty()) return false; + var oldProcess = maybeOldProcess.get(); + + try { + while (oldProcess.isAlive()) { + Thread.sleep(50); + } + return true; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + static void log(Path logFile, long myPid, String message) { + var timestamp = DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now()); + try { + Files.writeString( + logFile, + "[" + timestamp + "] " + myPid + ": " + message + "\n", + StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (IOException e) { + // do nothing } } } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index abb01d5bc7b..92dc3148fe6 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -1397,7 +1397,7 @@ trait JavaModule */ def runBackground(args: String*): Command[Unit] = { val task = runBackgroundTask(finalMainClass, Task.Anon { Args(args) }) - Task.Command { task() } + Task.Command(persistent = true) { task() } } /** diff --git a/scalalib/src/mill/scalalib/RunModule.scala b/scalalib/src/mill/scalalib/RunModule.scala index 9419d07fe79..c1d08722f08 100644 --- a/scalalib/src/mill/scalalib/RunModule.scala +++ b/scalalib/src/mill/scalalib/RunModule.scala @@ -1,18 +1,17 @@ package mill.scalalib -import java.lang.reflect.Modifier - import mainargs.arg import mill.api.JsonFormatters.pathReadWrite import mill.api.{Ctx, PathRef, Result} import mill.define.{Command, ModuleRef, Task} +import mill.main.client.ServerFiles +import mill.scalalib.classgraph.ClassgraphWorkerModule import mill.util.Jvm import mill.{Agg, Args, T} -import mill.main.client.ServerFiles import os.{Path, ProcessOutput} -import scala.util.control.NonFatal -import mill.scalalib.classgraph.ClassgraphWorkerModule +import java.lang.reflect.Modifier +import scala.util.control.NonFatal trait RunModule extends WithJvmWorker { @@ -115,7 +114,7 @@ trait RunModule extends WithJvmWorker { */ def runMainBackground(@arg(positional = true) mainClass: String, args: String*): Command[Unit] = { val task = runBackgroundTask(Task.Anon { mainClass }, Task.Anon { Args(args) }) - Task.Command { task() } + Task.Command(persistent = true) { task() } } /** @@ -162,13 +161,10 @@ trait RunModule extends WithJvmWorker { def runBackgroundTask(mainClass: Task[String], args: Task[Args] = Task.Anon(Args())): Task[Unit] = Task.Anon { - val (procUuidPath, procLockfile, procUuid) = RunModule.backgroundSetup(Task.dest) + val backgroundPaths = new RunModule.BackgroundPaths(destDir = Task.dest) + runner().run( - args = Seq( - procUuidPath.toString, - procLockfile.toString, - procUuid, - runBackgroundRestartDelayMillis().toString, + args = backgroundPaths.toArgs ++ Seq( mainClass() ) ++ args().value, mainClass = "mill.scalalib.backgroundwrapper.MillBackgroundWrapper", @@ -189,6 +185,8 @@ trait RunModule extends WithJvmWorker { */ // TODO: make this a task, to be more dynamic def runBackgroundLogToConsole: Boolean = true + + @deprecated("Binary compat shim, no longer used.`", "Mill 0.12.11") def runBackgroundRestartDelayMillis: T[Int] = 500 @deprecated("Binary compat shim, use `.runner().run(..., background=true)`", "Mill 0.12.0") @@ -203,18 +201,15 @@ trait RunModule extends WithJvmWorker { runUseArgsFile: Boolean, backgroundOutputs: Option[Tuple2[ProcessOutput, ProcessOutput]] )(args: String*): Ctx => Result[Unit] = ctx => { - val (procUuidPath, procLockfile, procUuid) = RunModule.backgroundSetup(taskDest) + val backgroundPaths = new RunModule.BackgroundPaths(destDir = taskDest) + try Result.Success( Jvm.runSubprocessWithBackgroundOutputs( "mill.scalalib.backgroundwrapper.MillBackgroundWrapper", (runClasspath ++ zwBackgroundWrapperClasspath).map(_.path), forkArgs, forkEnv, - Seq( - procUuidPath.toString, - procLockfile.toString, - procUuid, - 500.toString, + backgroundPaths.toArgs ++ Seq( finalMainClass ) ++ args, workingDir = forkWorkingDir, @@ -251,13 +246,6 @@ trait RunModule extends WithJvmWorker { object RunModule { - private[mill] def backgroundSetup(dest: os.Path): (Path, Path, String) = { - val procUuid = java.util.UUID.randomUUID().toString - val procUuidPath = dest / ".mill-background-process-uuid" - val procLockfile = dest / ".mill-background-process-lock" - (procUuidPath, procLockfile, procUuid) - } - private[mill] def getMainMethod(mainClassName: String, cl: ClassLoader) = { val mainClass = cl.loadClass(mainClassName) val method = mainClass.getMethod("main", classOf[Array[String]]) @@ -361,4 +349,19 @@ object RunModule { } } + /** @param destDir The `Task.dest`. Needs to be persistent for it to work properly. */ + private[mill] class BackgroundPaths(val destDir: os.Path) { + def newestPidPath: os.Path = destDir / "newest-pid" + def currentlyRunningPidPath: os.Path = destDir / "currently-running-pid" + def lockPath: os.Path = destDir / "lock" + def logPath: os.Path = destDir / "log" + + def toArgs: Seq[String] = + Seq( + newestPidPath.toString, + currentlyRunningPidPath.toString, + lockPath.toString, + logPath.toString + ) + } }