Skip to content

Commit 1000593

Browse files
committed
Merge remote-tracking branch 'upstream/main' into improvement/fs-watching
2 parents e7faaa8 + 7578c20 commit 1000593

File tree

21 files changed

+267
-167
lines changed

21 files changed

+267
-167
lines changed

core/constants/src/mill/constants/OutFiles.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public class OutFiles {
6060
/**
6161
* Lock file used for exclusive access to the Mill output directory
6262
*/
63-
public static final String millLock = "mill-lock";
63+
public static final String millOutLock = "mill-out-lock";
6464

6565
/**
6666
* Any active Mill command that is currently run, for debugging purposes

core/constants/src/mill/constants/ProxyStream.java

+11-2
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public class ProxyStream {
3535
public static final int OUT = 1;
3636
public static final int ERR = -1;
3737
public static final int END = 0;
38+
public static final int HEARTBEAT = 127;
3839

3940
public static void sendEnd(OutputStream out) throws IOException {
4041
synchronized (out) {
@@ -43,6 +44,13 @@ public static void sendEnd(OutputStream out) throws IOException {
4344
}
4445
}
4546

47+
public static void sendHeartbeat(OutputStream out) throws IOException {
48+
synchronized (out) {
49+
out.write(ProxyStream.HEARTBEAT);
50+
out.flush();
51+
}
52+
}
53+
4654
public static class Output extends java.io.OutputStream {
4755
private final java.io.OutputStream destination;
4856
private final int key;
@@ -75,7 +83,7 @@ public void write(byte[] b, int off, int len) throws IOException {
7583
synchronized (destination) {
7684
int i = 0;
7785
while (i < len && i + off < b.length) {
78-
int chunkLength = Math.min(len - i, 127);
86+
int chunkLength = Math.min(len - i, 126);
7987
if (chunkLength > 0) {
8088
destination.write(chunkLength * key);
8189
destination.write(b, off + i, Math.min(b.length - off - i, chunkLength));
@@ -136,7 +144,8 @@ public void run() {
136144
// that only header values > 0 represent actual data to read:
137145
// - sign((byte)header) represents which stream the data should be sent to
138146
// - abs((byte)header) represents the length of the data to read and send
139-
if (header == -1 || header == 0) break;
147+
if (header == -1 || header == END) break;
148+
else if (header == HEARTBEAT) continue;
140149
else {
141150
int stream = (byte) header > 0 ? 1 : -1;
142151
int quantity0 = (byte) header;

core/constants/src/mill/constants/ServerFiles.java

+1-7
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ public class ServerFiles {
2020
* folder. If multiple servers are spawned in the same folder, only one takes
2121
* the lock and the others fail to do so and terminate immediately.
2222
*/
23-
public static final String processLock = "processLock";
23+
public static final String serverLock = "serverLock";
2424

2525
/**
2626
* The port used to connect between server and client
@@ -48,12 +48,6 @@ public static String pipe(String base) {
4848
*/
4949
public static final String serverLog = "server.log";
5050

51-
/**
52-
* File that the client writes to pass the arguments, environment variables,
53-
* and other necessary metadata to the Mill server to kick off a run
54-
*/
55-
public static final String runArgs = "runArgs";
56-
5751
/**
5852
* File the server writes to pass the exit code of a completed run back to the
5953
* client

dist/package.mill

+3-3
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ trait DistModule extends Module {
2525

2626
def localBinName: String
2727

28-
def cacheBinarySuffix: String = ""
28+
def cacheBinarySuffix: Task[String] = Task.Anon("")
2929

3030
/**
3131
* Build and install Mill locally.
@@ -43,7 +43,7 @@ trait DistModule extends Module {
4343
def installLocalCache() = Task.Command {
4444
val path = installLocalTask(
4545
Task.Anon(
46-
(os.home / ".cache/mill/download" / (build.millVersion() + cacheBinarySuffix + batExt)).toString()
46+
(os.home / ".cache/mill/download" / (build.millVersion() + cacheBinarySuffix() + batExt)).toString()
4747
)
4848
)()
4949
Task.log.streams.out.println(path.toString())
@@ -261,7 +261,7 @@ object `package` extends MillJavaModule with DistModule {
261261

262262
def localBinName = "mill-native"
263263

264-
def cacheBinarySuffix = "-native"
264+
def cacheBinarySuffix = Task.Anon { s"-native-${artifactOsSuffix()}-${artifactCpuSuffix()}" }
265265

266266
def executableRaw = nativeImage()
267267

example/fundamentals/out-dir/1-out-files/build.mill

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ out/mill-build/...
2929
out/mill-profile.json
3030
out/mill-runner-state.json
3131
out/mill-dependency-tree.json
32-
out/mill-lock
32+
out/mill-out-lock
3333
out/mill-invalidation-tree.json
3434
out/mill-chrome-profile.json
3535
out/mill-server/...

runner/bsp/worker/src/mill/bsp/worker/BspWorkerImpl.scala

+5-2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import mill.bsp.BuildInfo
55
import mill.api.internal.{BspServerHandle, BspServerResult, EvaluatorApi}
66
import mill.bsp.Constants
77
import mill.api.{Result, SystemStreams}
8+
import mill.client.lock.Lock
89
import org.eclipse.lsp4j.jsonrpc.Launcher
910

1011
import java.io.PrintWriter
@@ -17,7 +18,8 @@ object BspWorkerImpl {
1718
topLevelBuildRoot: os.Path,
1819
streams: SystemStreams,
1920
logDir: os.Path,
20-
canReload: Boolean
21+
canReload: Boolean,
22+
outLock: Lock
2123
): mill.api.Result[BspServerHandle] = {
2224

2325
try {
@@ -33,7 +35,8 @@ object BspWorkerImpl {
3335
logStream = streams.err,
3436
canReload = canReload,
3537
debugMessages = Option(System.getenv("MILL_BSP_DEBUG")).contains("true"),
36-
onShutdown = () => listening.cancel(true)
38+
onShutdown = () => listening.cancel(true),
39+
outLock = outLock
3740
) with MillJvmBuildServer with MillJavaBuildServer with MillScalaBuildServer
3841

3942
lazy val launcher = new Launcher.Builder[BuildClient]()

runner/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala

+7-3
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,14 @@ import com.google.gson.JsonObject
66
import mill.api.*
77
import mill.api.internal.{JvmBuildTarget, ScalaBuildTarget, *}
88
import mill.api.Segment.Label
9-
import mill.bsp.{Constants}
9+
import mill.bsp.Constants
1010
import mill.bsp.worker.Utils.{makeBuildTarget, outputPaths, sanitizeUri}
11+
import mill.client.lock.Lock
1112
import mill.server.Server
1213

1314
import java.io.PrintStream
1415
import java.util.concurrent.CompletableFuture
16+
import java.util.concurrent.locks.ReentrantLock
1517
import scala.collection.mutable
1618
import scala.concurrent.Promise
1719
import scala.jdk.CollectionConverters.*
@@ -28,7 +30,8 @@ private class MillBuildServer(
2830
logStream: PrintStream,
2931
canReload: Boolean,
3032
debugMessages: Boolean,
31-
onShutdown: () => Unit
33+
onShutdown: () => Unit,
34+
outLock: Lock
3235
)(implicit ec: scala.concurrent.ExecutionContext) extends BuildServer {
3336

3437
import MillBuildServer._
@@ -724,7 +727,8 @@ private class MillBuildServer(
724727
case n: NamedTaskApi[_] => n.label
725728
case t => t.toString
726729
},
727-
streams = logger0.streams
730+
streams = logger0.streams,
731+
outLock = outLock
728732
) {
729733
evaluator.executeApi(
730734
goals,

runner/client/src/mill/client/ServerCouldNotBeStarted.java

-7
This file was deleted.
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package mill.client;
22

3+
import java.io.IOException;
34
import java.io.InputStream;
45
import java.io.OutputStream;
56
import java.io.PrintStream;
@@ -9,7 +10,6 @@
910
import java.nio.file.Path;
1011
import java.util.Map;
1112
import mill.client.lock.Locks;
12-
import mill.client.lock.TryLocked;
1313
import mill.constants.InputPumper;
1414
import mill.constants.ProxyStream;
1515
import mill.constants.ServerFiles;
@@ -23,10 +23,10 @@
2323
*
2424
* - Client:
2525
* - Take clientLock
26-
* - If processLock is not yet taken, it means server is not running, so spawn a server
26+
* - If serverLock is not yet taken, it means server is not running, so spawn a server
2727
* - Wait for server socket to be available for connection
2828
* - Server:
29-
* - Take processLock.
29+
* - Take serverLock.
3030
* - If already taken, it means another server was running
3131
* (e.g. spawned by a different client) so exit immediately
3232
* - Server: loop:
@@ -46,19 +46,18 @@ public static class Result {
4646
public Path serverDir;
4747
}
4848

49-
final int serverProcessesLimit = 5;
5049
final int serverInitWaitMillis = 10000;
5150

52-
public abstract void initServer(Path serverDir, boolean b, Locks locks) throws Exception;
51+
public abstract void initServer(Path serverDir, Locks locks) throws Exception;
5352

54-
public abstract void preRun(Path serverDir) throws Exception;
53+
public abstract void prepareServerDir(Path serverDir) throws Exception;
5554

5655
InputStream stdin;
5756
PrintStream stdout;
5857
PrintStream stderr;
5958
Map<String, String> env;
6059
String[] args;
61-
Locks[] memoryLocks;
60+
Locks memoryLock;
6261
int forceFailureForTestingMillisDelay;
6362

6463
public ServerLauncher(
@@ -67,7 +66,7 @@ public ServerLauncher(
6766
PrintStream stderr,
6867
Map<String, String> env,
6968
String[] args,
70-
Locks[] memoryLocks,
69+
Locks memoryLock,
7170
int forceFailureForTestingMillisDelay) {
7271
this.stdin = stdin;
7372
this.stdout = stdout;
@@ -78,56 +77,41 @@ public ServerLauncher(
7877
// For testing in memory, we need to pass in the locks separately, so that the
7978
// locks can be shared between the different instances of `ServerLauncher` the
8079
// same way file locks are shared between different Mill client/server processes
81-
this.memoryLocks = memoryLocks;
80+
this.memoryLock = memoryLock;
8281

8382
this.forceFailureForTestingMillisDelay = forceFailureForTestingMillisDelay;
8483
}
8584

86-
public Result acquireLocksAndRun(Path serverDir0) throws Exception {
85+
public Result run(Path serverDir) throws Exception {
8786

88-
final boolean setJnaNoSys = System.getProperty("jna.nosys") == null;
89-
if (setJnaNoSys) {
90-
System.setProperty("jna.nosys", "true");
91-
}
87+
Files.createDirectories(serverDir);
9288

93-
int serverIndex = 0;
94-
while (serverIndex < serverProcessesLimit) { // Try each possible server process (-1 to -5)
95-
serverIndex++;
96-
final Path serverDir =
97-
serverDir0.getParent().resolve(serverDir0.getFileName() + "-" + serverIndex);
98-
99-
Files.createDirectories(serverDir);
100-
101-
try (Locks locks = memoryLocks != null
102-
? memoryLocks[serverIndex - 1]
103-
: Locks.files(serverDir.toString());
104-
TryLocked clientLocked = locks.clientLock.tryLock()) {
105-
if (clientLocked.isLocked()) {
106-
Result result = new Result();
107-
preRun(serverDir);
108-
result.exitCode = run(serverDir, setJnaNoSys, locks);
109-
result.serverDir = serverDir;
110-
return result;
111-
}
112-
}
113-
}
114-
throw new ServerCouldNotBeStarted(
115-
"Reached max server processes limit: " + serverProcessesLimit);
116-
}
89+
prepareServerDir(serverDir);
11790

118-
int run(Path serverDir, boolean setJnaNoSys, Locks locks) throws Exception {
91+
Socket ioSocket = launchConnectToServer(serverDir);
11992

120-
try (OutputStream f = Files.newOutputStream(serverDir.resolve(ServerFiles.runArgs))) {
121-
f.write(Util.hasConsole() ? 1 : 0);
122-
ClientUtil.writeString(f, BuildInfo.millVersion);
123-
ClientUtil.writeArgs(args, f);
124-
ClientUtil.writeMap(env, f);
93+
try {
94+
Thread outPumperThread = startStreamPumpers(ioSocket);
95+
forceTestFailure(serverDir);
96+
outPumperThread.join();
97+
} finally {
98+
ioSocket.close();
12599
}
126100

127-
if (locks.processLock.probe()) initServer(serverDir, setJnaNoSys, locks);
101+
Result result = new Result();
102+
result.exitCode = readExitCode(serverDir);
103+
result.serverDir = serverDir;
104+
return result;
105+
}
106+
107+
Socket launchConnectToServer(Path serverDir) throws Exception {
128108

129-
while (locks.processLock.probe()) Thread.sleep(1);
109+
try (Locks locks = memoryLock != null ? memoryLock : Locks.files(serverDir.toString());
110+
mill.client.lock.Locked locked = locks.clientLock.lock()) {
130111

112+
if (locks.serverLock.probe()) initServer(serverDir, locks);
113+
while (locks.serverLock.probe()) Thread.sleep(1);
114+
}
131115
long retryStart = System.currentTimeMillis();
132116
Socket ioSocket = null;
133117
Throwable socketThrowable = null;
@@ -140,13 +124,26 @@ int run(Path serverDir, boolean setJnaNoSys, Locks locks) throws Exception {
140124
Thread.sleep(1);
141125
}
142126
}
143-
144127
if (ioSocket == null) {
145128
throw new Exception("Failed to connect to server", socketThrowable);
146129
}
130+
return ioSocket;
131+
}
132+
133+
private void forceTestFailure(Path serverDir) throws Exception {
134+
if (forceFailureForTestingMillisDelay > 0) {
135+
Thread.sleep(forceFailureForTestingMillisDelay);
136+
throw new Exception("Force failure for testing: " + serverDir);
137+
}
138+
}
147139

140+
Thread startStreamPumpers(Socket ioSocket) throws Exception {
148141
InputStream outErr = ioSocket.getInputStream();
149142
OutputStream in = ioSocket.getOutputStream();
143+
in.write(Util.hasConsole() ? 1 : 0);
144+
ClientUtil.writeString(in, BuildInfo.millVersion);
145+
ClientUtil.writeArgs(args, in);
146+
ClientUtil.writeMap(env, in);
150147
ProxyStream.Pumper outPumper = new ProxyStream.Pumper(outErr, stdout, stderr);
151148
InputPumper inPump = new InputPumper(() -> stdin, () -> in, true);
152149
Thread outPumperThread = new Thread(outPumper, "outPump");
@@ -155,23 +152,16 @@ int run(Path serverDir, boolean setJnaNoSys, Locks locks) throws Exception {
155152
inThread.setDaemon(true);
156153
outPumperThread.start();
157154
inThread.start();
155+
return outPumperThread;
156+
}
158157

159-
if (forceFailureForTestingMillisDelay > 0) {
160-
Thread.sleep(forceFailureForTestingMillisDelay);
161-
throw new Exception("Force failure for testing: " + serverDir);
162-
}
163-
outPumperThread.join();
164-
165-
try {
166-
Path exitCodeFile = serverDir.resolve(ServerFiles.exitCode);
167-
if (Files.exists(exitCodeFile)) {
168-
return Integer.parseInt(Files.readAllLines(exitCodeFile).get(0));
169-
} else {
170-
System.err.println("mill-server/ exitCode file not found");
171-
return 1;
172-
}
173-
} finally {
174-
ioSocket.close();
158+
int readExitCode(Path serverDir) throws IOException {
159+
Path exitCodeFile = serverDir.resolve(ServerFiles.exitCode);
160+
if (Files.exists(exitCodeFile)) {
161+
return Integer.parseInt(Files.readAllLines(exitCodeFile).get(0));
162+
} else {
163+
System.err.println("mill-server/ exitCode file not found");
164+
return 1;
175165
}
176166
}
177167
}

0 commit comments

Comments
 (0)