Skip to content

Commit 2c6de27

Browse files
committed
Backport of com-lihaoyi#5115
1 parent f9f243b commit 2c6de27

File tree

7 files changed

+141
-57
lines changed

7 files changed

+141
-57
lines changed

.editorconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ end_of_line = lf
55
insert_final_newline = true
66
charset = utf-8
77
indent_style = space
8-
indent_size = 4
8+
indent_size = 2
99

1010
[*.scala]
1111
end_of_line = lf

build.mill

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ trait MillPublishJavaModule extends MillJavaModule with PublishModule {
442442
"info.releaseNotesURL" -> Settings.changelogUrl
443443
)
444444
def pomSettings = commonPomSettings(artifactName())
445-
def javacOptions = Seq("-source", "1.8", "-target", "1.8", "-encoding", "UTF-8")
445+
def javacOptions = Seq("-source", "11", "-target", "11", "-encoding", "UTF-8")
446446
}
447447

448448
/**

docs/modules/ROOT/pages/cli/flags.adoc

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,13 @@ forcefully terminating the previous process even though it may be still alive:
131131
$ mill -w foo.runBackground
132132
----
133133

134+
Note that even if you interrupt mill watch via CTRL+C, the server spawned by `runBackground` still runs in the
135+
background. To actually stop the background server use:
136+
137+
[source,console]
138+
----
139+
> mill clean foo.runBackground
140+
----
134141

135142
=== `--jobs`/`-j`
136143

integration/invalidation/run-background/src/RunBackgroundTests.scala

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,28 @@ object RunBackgroundTests extends UtestIntegrationTestSuite {
3737
os.write(stop, "")
3838
eventually { probeLockAvailable(lock) }
3939
}
40+
41+
test("sequential") - integrationTest { tester =>
42+
import tester._
43+
val lock1 = os.temp()
44+
val lock2 = os.temp()
45+
val stop = os.temp()
46+
os.remove(stop)
47+
eval(("foo.runBackground", lock1, stop))
48+
eventually { !probeLockAvailable(lock1) }
49+
eval(("foo.runBackground", lock2, stop))
50+
eventually { !probeLockAvailable(lock2) }
51+
Predef.assert(
52+
probeLockAvailable(lock1),
53+
"first process should be exited after second process is running"
54+
)
55+
56+
if (tester.clientServerMode) eval("shutdown")
57+
continually { !probeLockAvailable(lock2) }
58+
os.write(stop, "")
59+
eventually { probeLockAvailable(lock2) }
60+
}
61+
4062
test("clean") - integrationTest { tester =>
4163
import tester._
4264
val lock = os.temp()

pythonlib/src/mill/pythonlib/PythonModule.scala

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ trait PythonModule extends PipModule with TaskModule { outer =>
176176
* @see [[mainScript]]
177177
*/
178178
def runBackground(args: mill.define.Args) = Task.Command {
179-
val (procUuidPath, procLockfile, procUuid) = mill.scalalib.RunModule.backgroundSetup(Task.dest)
179+
val backgroundPaths = mill.scalalib.RunModule.BackgroundPaths(Task.dest)
180180
val pwd0 = os.Path(java.nio.file.Paths.get(".").toAbsolutePath)
181181

182182
os.checker.withValue(os.Checker.Nop) {
@@ -185,11 +185,7 @@ trait PythonModule extends PipModule with TaskModule { outer =>
185185
classPath = mill.scalalib.JvmWorkerModule.backgroundWrapperClasspath().map(_.path).toSeq,
186186
jvmArgs = Nil,
187187
env = runnerEnvTask(),
188-
mainArgs = Seq(
189-
procUuidPath.toString,
190-
procLockfile.toString,
191-
procUuid,
192-
"500",
188+
mainArgs = backgroundPaths.toArgs ++ Seq(
193189
"<subprocess>",
194190
pythonExe().path.toString,
195191
mainScript().path.toString
Lines changed: 85 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,71 @@
11
package mill.scalalib.backgroundwrapper;
22

3+
import java.io.IOException;
34
import java.io.RandomAccessFile;
45
import java.nio.channels.FileChannel;
56
import java.nio.file.*;
7+
import java.util.Optional;
68

9+
@SuppressWarnings("BusyWait")
710
public class MillBackgroundWrapper {
811
public static void main(String[] args) throws Exception {
9-
Path procUuidPath = Paths.get(args[0]);
10-
Path procLockfile = Paths.get(args[1]);
11-
String procUuid = args[2];
12-
int lockDelay = Integer.parseInt(args[3]);
12+
var newestProcessIdPath = Paths.get(args[0]);
13+
var currentlyRunningProccesIdPath = Paths.get(args[1]);
14+
var procLockfile = Paths.get(args[2]);
15+
var realMain = args[3];
16+
var realArgs = java.util.Arrays.copyOfRange(args, 4, args.length);
1317

14-
Files.writeString(procUuidPath, procUuid, StandardOpenOption.CREATE);
18+
// The following code should handle this scenario, when we have 2 processes contending for the
19+
// lock.
20+
//
21+
// This can happen if a new MillBackgroundWrapper is launched while the previous one is still
22+
// running due to rapid
23+
// changes in the source code.
24+
//
25+
// Process 1 starts, writes newest_pid=1, claims lock, writes currently_running_pid = 1
26+
// Process 2 starts, writes newest_pid=2, tries to claim lock but is blocked
27+
// Process 3 starts at the same time as process 2, writes newest_pid=3, tries to claim lock but
28+
// is blocked
29+
//
30+
// Process 1 reads newest_pid=3, terminates, releases lock
31+
// Process 2 claims lock, reads currently_running_pid = 1, waits for process 1 to die, writes
32+
// currently_running_pid = 2
33+
// Process 2 reads newest_pid=3, terminates, releases lock
34+
// Process 3 claims lock, reads currently_running_pid = 2, waits for process 2 to die, writes
35+
// currently_running_pid = 3, then starts
36+
// Process 3 reads newest_pid=3, continues running
37+
38+
// Indicate to the previous process that we want to take over.
39+
var myPid = ProcessHandle.current().pid();
40+
var myPidStr = "" + myPid;
41+
Files.writeString(newestProcessIdPath, myPidStr, StandardOpenOption.CREATE);
1542

1643
// Take a lock on `procLockfile` to ensure that only one
1744
// `runBackground` process is running at any point in time.
45+
//noinspection resource - this is intentional, file is released when process dies.
1846
RandomAccessFile raf = new RandomAccessFile(procLockfile.toFile(), "rw");
1947
FileChannel chan = raf.getChannel();
2048
if (chan.tryLock() == null) {
2149
System.err.println("Waiting for runBackground lock to be available");
50+
//noinspection ResultOfMethodCallIgnored - this is intentional, lock is released when process dies.
2251
chan.lock();
2352
}
2453

25-
// For some reason even after the previous process exits things like sockets
26-
// may still take time to free, so sleep for a configurable duration before proceeding
27-
Thread.sleep(lockDelay);
54+
var oldProcessPid = readPreviousPid(currentlyRunningProccesIdPath);
55+
oldProcessPid.ifPresent(MillBackgroundWrapper::waitForPreviousProcessToTerminate);
56+
Files.writeString(currentlyRunningProccesIdPath, myPidStr, StandardOpenOption.CREATE);
2857

2958
// Start the thread to watch for updates on the process marker file,
3059
// so we can exit if it is deleted or replaced
31-
long startTime = System.currentTimeMillis();
32-
Thread watcher = new Thread(() -> {
33-
while (true) {
34-
long delta = (System.currentTimeMillis() - startTime) / 1000;
35-
try {
36-
Thread.sleep(1);
37-
String token = Files.readString(procUuidPath);
38-
if (!token.equals(procUuid)) {
39-
System.err.println("runBackground exiting after " + delta + "s");
40-
System.exit(0);
41-
}
42-
} catch (Exception e) {
43-
System.err.println("runBackground exiting after " + delta + "s");
44-
System.exit(0);
45-
}
46-
}
60+
var startTime = System.currentTimeMillis();
61+
checkIfWeStillNeedToBeRunning(startTime, newestProcessIdPath, myPidStr);
62+
var watcher = new Thread(() -> {
63+
while (true) checkIfWeStillNeedToBeRunning(startTime, newestProcessIdPath, myPidStr);
4764
});
48-
4965
watcher.setDaemon(true);
5066
watcher.start();
5167

5268
// Actually start the Java main method we wanted to run in the background
53-
String realMain = args[4];
54-
String[] realArgs = java.util.Arrays.copyOfRange(args, 5, args.length);
5569
if (!realMain.equals("<subprocess>")) {
5670
Class.forName(realMain).getMethod("main", String[].class).invoke(null, (Object) realArgs);
5771
} else {
@@ -62,17 +76,58 @@ public static void main(String[] args) throws Exception {
6276

6377
long now = System.currentTimeMillis();
6478

79+
// If the process does not shut down withing 100ms kill it forcibly.
6580
while (subprocess.isAlive() && System.currentTimeMillis() - now < 100) {
6681
try {
6782
Thread.sleep(1);
6883
} catch (InterruptedException e) {
84+
// do nothing
6985
}
70-
if (subprocess.isAlive()) {
71-
subprocess.destroyForcibly();
72-
}
86+
}
87+
if (subprocess.isAlive()) {
88+
subprocess.destroyForcibly();
7389
}
7490
}));
7591
System.exit(subprocess.waitFor());
7692
}
7793
}
94+
95+
private static void checkIfWeStillNeedToBeRunning(
96+
long startTime, Path newestProcessIdPath, String myPidStr) {
97+
long delta = (System.currentTimeMillis() - startTime) / 1000;
98+
try {
99+
Thread.sleep(50);
100+
String token = Files.readString(newestProcessIdPath);
101+
if (!token.equals(myPidStr)) {
102+
System.err.println("runBackground exiting after " + delta + "s");
103+
System.exit(0);
104+
}
105+
} catch (Exception e) {
106+
System.err.println("runBackground exiting after " + delta + "s");
107+
System.exit(0);
108+
}
109+
}
110+
111+
static Optional<Long> readPreviousPid(Path pidFilePath) {
112+
try {
113+
var pidStr = Files.readString(pidFilePath);
114+
return Optional.of(Long.parseLong(pidStr));
115+
} catch (IOException | NumberFormatException e) {
116+
return Optional.empty();
117+
}
118+
}
119+
120+
static void waitForPreviousProcessToTerminate(long pid) {
121+
var maybeOldProcess = ProcessHandle.of(pid);
122+
if (maybeOldProcess.isEmpty()) return;
123+
var oldProcess = maybeOldProcess.get();
124+
125+
try {
126+
while (oldProcess.isAlive()) {
127+
Thread.sleep(50);
128+
}
129+
} catch (InterruptedException e) {
130+
throw new RuntimeException(e);
131+
}
132+
}
78133
}

scalalib/src/mill/scalalib/RunModule.scala

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -162,13 +162,9 @@ trait RunModule extends WithJvmWorker {
162162

163163
def runBackgroundTask(mainClass: Task[String], args: Task[Args] = Task.Anon(Args())): Task[Unit] =
164164
Task.Anon {
165-
val (procUuidPath, procLockfile, procUuid) = RunModule.backgroundSetup(Task.dest)
165+
val dest = Task.dest
166166
runner().run(
167-
args = Seq(
168-
procUuidPath.toString,
169-
procLockfile.toString,
170-
procUuid,
171-
runBackgroundRestartDelayMillis().toString,
167+
args = RunModule.BackgroundPaths(dest).toArgs ++ Seq(
172168
mainClass()
173169
) ++ args().value,
174170
mainClass = "mill.scalalib.backgroundwrapper.MillBackgroundWrapper",
@@ -189,6 +185,8 @@ trait RunModule extends WithJvmWorker {
189185
*/
190186
// TODO: make this a task, to be more dynamic
191187
def runBackgroundLogToConsole: Boolean = true
188+
189+
@deprecated("Binary compat shim, no longer used.`", "Mill 0.12.11")
192190
def runBackgroundRestartDelayMillis: T[Int] = 500
193191

194192
@deprecated("Binary compat shim, use `.runner().run(..., background=true)`", "Mill 0.12.0")
@@ -203,18 +201,14 @@ trait RunModule extends WithJvmWorker {
203201
runUseArgsFile: Boolean,
204202
backgroundOutputs: Option[Tuple2[ProcessOutput, ProcessOutput]]
205203
)(args: String*): Ctx => Result[Unit] = ctx => {
206-
val (procUuidPath, procLockfile, procUuid) = RunModule.backgroundSetup(taskDest)
204+
val backgroundPaths = RunModule.BackgroundPaths(taskDest)
207205
try Result.Success(
208206
Jvm.runSubprocessWithBackgroundOutputs(
209207
"mill.scalalib.backgroundwrapper.MillBackgroundWrapper",
210208
(runClasspath ++ zwBackgroundWrapperClasspath).map(_.path),
211209
forkArgs,
212210
forkEnv,
213-
Seq(
214-
procUuidPath.toString,
215-
procLockfile.toString,
216-
procUuid,
217-
500.toString,
211+
backgroundPaths.toArgs ++ Seq(
218212
finalMainClass
219213
) ++ args,
220214
workingDir = forkWorkingDir,
@@ -251,13 +245,6 @@ trait RunModule extends WithJvmWorker {
251245

252246
object RunModule {
253247

254-
private[mill] def backgroundSetup(dest: os.Path): (Path, Path, String) = {
255-
val procUuid = java.util.UUID.randomUUID().toString
256-
val procUuidPath = dest / ".mill-background-process-uuid"
257-
val procLockfile = dest / ".mill-background-process-lock"
258-
(procUuidPath, procLockfile, procUuid)
259-
}
260-
261248
private[mill] def getMainMethod(mainClassName: String, cl: ClassLoader) = {
262249
val mainClass = cl.loadClass(mainClassName)
263250
val method = mainClass.getMethod("main", classOf[Array[String]])
@@ -361,4 +348,21 @@ object RunModule {
361348
}
362349
}
363350

351+
case class BackgroundPaths(
352+
newestPidPath: os.Path,
353+
currentlyRunningPidPath: os.Path,
354+
lockPath: os.Path
355+
) {
356+
def toArgs: Seq[String] =
357+
Seq(newestPidPath.toString, currentlyRunningPidPath.toString, lockPath.toString)
358+
}
359+
object BackgroundPaths {
360+
def apply(dest: os.Path): BackgroundPaths = {
361+
BackgroundPaths(
362+
newestPidPath = (dest / ".mill-background-process-newest-pid"),
363+
currentlyRunningPidPath = (dest / ".mill-background-process-currently-running-pid"),
364+
lockPath = (dest / ".mill-background-process-lock")
365+
)
366+
}
367+
}
364368
}

0 commit comments

Comments
 (0)