Skip to content

Commit a1a6cd4

Browse files
jgloganjglogan
committed
Fix CI builds, add observability.
- Localhost file monitor was taking down the API server at startup if it can't read the /etc/resolver directory, which apparently it can't on CI. Adapted the monitor to log an error and proceed to file monitoring which doesn't look like it'll throw. - Currently we don't collect logs on CI builds, and we don't have permission to run the log command there. This PR adds a `--log-root` parameter for `container system start`, which gets propagated everywhere via the `CONTAINER_LOG_ROOT` variable, similarly to what we do for `--app-root` and `--install-root`. - Use FilePath from swift-system for the log root. Foundation URL is a bit of a footgun for filesystem paths, so unless we identify a showstopper, we should incrementally transition to this type everywhere except where we really need network URLs. - Set log root for the CI test phase, and archive/upload logs when tests fail. - Output the hostname of the CI runner at the start of the test phase so we can identify runner-specific issues where they exist. - Breaking change to CLI output - plumb log root into container system status, rework the command for consistency with resource list/inspect commands (i.e. table and JSON output), and add a unit test. - Adds command reference documentation for `--log-root`.
1 parent c9f81ca commit a1a6cd4

26 files changed

Lines changed: 602 additions & 112 deletions

‎.github/workflows/common.yml‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,16 +71,34 @@ jobs:
7171
- name: Test the container project
7272
run: |
7373
APP_ROOT=$(mktemp -d -p "${RUNNER_TEMP}")
74+
LOG_ROOT="${APP_ROOT}/logs"
7475
trap 'rm -rf "${APP_ROOT}"; echo Removing data directory ${APP_ROOT}' EXIT
75-
echo "Created data directory ${APP_ROOT}"
76+
echo "created data directory: ${APP_ROOT}"
77+
echo "hostname: $(hostname)"
7678
export NO_PROXY="${NO_PROXY},192.168.0.0/16,fe80::/10"
7779
echo NO_PROXY=${NO_PROXY}
7880
export no_proxy="${no_proxy},192.168.0.0/16,fe80::/10"
7981
echo no_proxy=${no_proxy}
80-
make APP_ROOT="${APP_ROOT}" test install-kernel integration
82+
make APP_ROOT="${APP_ROOT}" LOG_ROOT="${LOG_ROOT}" test install-kernel integration || status=$?
83+
if [ "${status:-0}" -ne "0" ] ; then
84+
echo tests failed with status: ${status}
85+
if [ -d "${LOG_ROOT}" ] ; then
86+
tar czf container-logs.tar.gz -C "$(dirname "${LOG_ROOT}")" "$(basename "${LOG_ROOT}")"
87+
fi
88+
exit ${status}
89+
fi
8190
env:
8291
DEVELOPER_DIR: "/Applications/Xcode-latest.app/Contents/Developer"
8392

93+
- name: Upload logs if present
94+
if: always()
95+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
96+
with:
97+
name: container-test-logs
98+
path: container-logs.tar.gz
99+
retention-days: 14
100+
if-no-files-found: ignore
101+
84102
- name: Save documentation artifact
85103
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
86104
with:

‎Makefile‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,12 @@ DSYM_PATH := bin/$(BUILD_CONFIGURATION)/bundle/container-dSYM.zip
3333
CODESIGN_OPTS ?= --force --sign - --timestamp=none
3434

3535
# Conditionally use a temporary data directory for integration tests
36-
ifeq ($(strip $(APP_ROOT)),)
37-
SYSTEM_START_OPTS :=
38-
else
39-
SYSTEM_START_OPTS := --app-root "$(strip $(APP_ROOT))"
36+
SYSTEM_START_OPTS :=
37+
ifneq ($(strip $(APP_ROOT)),)
38+
SYSTEM_START_OPTS += --app-root "$(strip $(APP_ROOT))"
39+
endif
40+
ifneq ($(strip $(LOG_ROOT)),)
41+
SYSTEM_START_OPTS += --log-root "$(strip $(LOG_ROOT))"
4042
endif
4143

4244
MACOS_VERSION := $(shell sw_vers -productVersion)
@@ -77,6 +79,7 @@ release: all
7779

7880
.PHONY: init-block
7981
init-block:
82+
@echo Building initfs if containerization is in edit mode
8083
@scripts/install-init.sh
8184

8285
.PHONY: install
@@ -144,15 +147,17 @@ test:
144147

145148
.PHONY: install-kernel
146149
install-kernel:
150+
@echo Stopping system before installing kernel
147151
@bin/container system stop || true
152+
@echo Starting system to install kernel
148153
@bin/container system start --enable-kernel-install $(SYSTEM_START_OPTS)
149154

150155
.PHONY: coverage
151156
coverage: init-block
152-
@echo Ensuring apiserver stopped before the CLI integration tests...
157+
@echo Ensuring apiserver stopped before the coverage analysis...
153158
@bin/container system stop && sleep 3 && scripts/ensure-container-stopped.sh
154159
@bin/container system start $(SYSTEM_START_OPTS) && \
155-
echo "Starting unit tests" && \
160+
echo "Starting coverage analysis" && \
156161
{ \
157162
exit_code=0; \
158163
$(SWIFT) test --no-parallel --enable-code-coverage -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) || exit_code=1 ; \

‎Package.swift‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -46,17 +46,18 @@ let package = Package(
4646
.library(name: "TerminalProgress", targets: ["TerminalProgress"]),
4747
],
4848
dependencies: [
49-
.package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"),
49+
.package(url: "https://github.com/Bouke/DNS.git", from: "1.2.0"),
50+
.package(url: "https://github.com/apple/containerization.git", exact: Version(stringLiteral: scVersion)),
5051
.package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.3.0"),
5152
.package(url: "https://github.com/apple/swift-collections.git", from: "1.2.0"),
52-
.package(url: "https://github.com/grpc/grpc-swift.git", from: "1.26.0"),
53-
.package(url: "https://github.com/apple/swift-protobuf.git", from: "1.29.0"),
53+
.package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"),
5454
.package(url: "https://github.com/apple/swift-nio.git", from: "2.80.0"),
55-
.package(url: "https://github.com/swiftlang/swift-docc-plugin.git", from: "1.1.0"),
56-
.package(url: "https://github.com/swift-server/async-http-client.git", from: "1.20.1"),
55+
.package(url: "https://github.com/apple/swift-protobuf.git", from: "1.29.0"),
56+
.package(url: "https://github.com/apple/swift-system.git", from: "1.4.0"),
57+
.package(url: "https://github.com/grpc/grpc-swift.git", from: "1.26.0"),
5758
.package(url: "https://github.com/orlandos-nl/DNSClient.git", from: "2.4.1"),
58-
.package(url: "https://github.com/Bouke/DNS.git", from: "1.2.0"),
59-
.package(url: "https://github.com/apple/containerization.git", exact: Version(stringLiteral: scVersion)),
59+
.package(url: "https://github.com/swift-server/async-http-client.git", from: "1.20.1"),
60+
.package(url: "https://github.com/swiftlang/swift-docc-plugin.git", from: "1.1.0"),
6061
],
6162
targets: [
6263
.executableTarget(
@@ -151,6 +152,7 @@ let package = Package(
151152
.product(name: "ContainerizationExtras", package: "containerization"),
152153
.product(name: "ContainerizationOS", package: "containerization"),
153154
.product(name: "Logging", package: "swift-log"),
155+
.product(name: "SystemPackage", package: "swift-system"),
154156
"CVersion",
155157
"ContainerAPIClient",
156158
"ContainerNetworkServiceClient",
@@ -168,13 +170,14 @@ let package = Package(
168170
name: "ContainerAPIClient",
169171
dependencies: [
170172
.product(name: "ArgumentParser", package: "swift-argument-parser"),
171-
.product(name: "Logging", package: "swift-log"),
172-
.product(name: "NIOCore", package: "swift-nio"),
173-
.product(name: "NIOPosix", package: "swift-nio"),
174173
.product(name: "Containerization", package: "containerization"),
175174
.product(name: "ContainerizationArchive", package: "containerization"),
176175
.product(name: "ContainerizationOCI", package: "containerization"),
177176
.product(name: "ContainerizationOS", package: "containerization"),
177+
.product(name: "Logging", package: "swift-log"),
178+
.product(name: "NIOCore", package: "swift-nio"),
179+
.product(name: "NIOPosix", package: "swift-nio"),
180+
.product(name: "SystemPackage", package: "swift-system"),
178181
"ContainerImagesServiceClient",
179182
"ContainerPersistence",
180183
"ContainerPlugin",
@@ -198,6 +201,7 @@ let package = Package(
198201
.product(name: "ArgumentParser", package: "swift-argument-parser"),
199202
.product(name: "Logging", package: "swift-log"),
200203
.product(name: "Containerization", package: "containerization"),
204+
.product(name: "SystemPackage", package: "swift-system"),
201205
"ContainerImagesService",
202206
"ContainerLog",
203207
"ContainerPlugin",
@@ -247,6 +251,7 @@ let package = Package(
247251
"ContainerLog",
248252
"ContainerNetworkService",
249253
"ContainerNetworkServiceClient",
254+
"ContainerPlugin",
250255
"ContainerResource",
251256
"ContainerVersion",
252257
"ContainerXPC",
@@ -293,6 +298,7 @@ let package = Package(
293298
.product(name: "GRPC", package: "grpc-swift"),
294299
.product(name: "Containerization", package: "containerization"),
295300
"ContainerLog",
301+
"ContainerPlugin",
296302
"ContainerResource",
297303
"ContainerSandboxService",
298304
"ContainerSandboxServiceClient",
@@ -344,7 +350,8 @@ let package = Package(
344350
.target(
345351
name: "ContainerLog",
346352
dependencies: [
347-
.product(name: "Logging", package: "swift-log")
353+
.product(name: "Logging", package: "swift-log"),
354+
.product(name: "SystemPackage", package: "swift-system"),
348355
]
349356
),
350357
.target(
@@ -361,6 +368,7 @@ let package = Package(
361368
dependencies: [
362369
.product(name: "Logging", package: "swift-log"),
363370
.product(name: "ContainerizationOS", package: "containerization"),
371+
.product(name: "SystemPackage", package: "swift-system"),
364372
"ContainerVersion",
365373
]
366374
),

‎Sources/ContainerCommands/Application.swift‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ public struct Application: AsyncLoggableCommand {
171171
return try PluginLoader(
172172
appRoot: systemHealth.appRoot,
173173
installRoot: systemHealth.installRoot,
174+
logRoot: systemHealth.logRoot,
174175
pluginDirectories: pluginDirectories,
175176
pluginFactories: pluginFactories,
176177
log: bootstrapLogger

‎Sources/ContainerCommands/System/SystemStart.swift‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import ContainerPersistence
2020
import ContainerPlugin
2121
import ContainerizationError
2222
import Foundation
23+
import SystemPackage
2324
import TerminalProgress
2425

2526
extension Application {
@@ -41,6 +42,12 @@ extension Application {
4142
transform: { URL(filePath: $0) })
4243
var installRoot = InstallRoot.defaultURL
4344

45+
@Option(
46+
name: .long,
47+
help: "Path to the root directory for log data, using macOS log facility if not set",
48+
transform: { FilePath($0) })
49+
var logRoot: FilePath? = nil
50+
4451
@Flag(
4552
name: .long,
4653
inversion: .prefixedEnableDisable,
@@ -73,7 +80,12 @@ extension Application {
7380
var env = PluginLoader.filterEnvironment()
7481
env[ApplicationRoot.environmentName] = appRoot.path(percentEncoded: false)
7582
env[InstallRoot.environmentName] = installRoot.path(percentEncoded: false)
76-
83+
if let logRoot {
84+
env[LogRoot.environmentName] =
85+
logRoot.isAbsolute
86+
? logRoot.string
87+
: FilePath(FileManager.default.currentDirectoryPath).appending(logRoot.components).string
88+
}
7789
let plist = LaunchPlist(
7890
label: "com.apple.container.apiserver",
7991
arguments: args,

‎Sources/ContainerCommands/System/SystemStatus.swift‎

Lines changed: 75 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,28 +31,96 @@ extension Application {
3131
@Option(name: .shortAndLong, help: "Launchd prefix for services")
3232
var prefix: String = "com.apple.container."
3333

34+
@Option(name: .long, help: "Format of the output")
35+
var format: ListFormat = .table
36+
3437
@OptionGroup
3538
public var logOptions: Flags.Logging
3639

3740
public init() {}
3841

42+
struct PrintableStatus: Codable {
43+
let status: String
44+
let appRoot: String
45+
let installRoot: String
46+
let logRoot: String?
47+
let apiServerVersion: String
48+
let apiServerCommit: String
49+
let apiServerBuild: String
50+
let apiServerAppName: String
51+
}
52+
3953
public func run() async throws {
4054
let isRegistered = try ServiceManager.isRegistered(fullServiceLabel: "\(prefix)apiserver")
4155
if !isRegistered {
42-
print("apiserver is not running and not registered with launchd")
56+
if format == .json {
57+
let status = PrintableStatus(
58+
status: "unregistered",
59+
appRoot: "",
60+
installRoot: "",
61+
logRoot: nil,
62+
apiServerVersion: "",
63+
apiServerCommit: "",
64+
apiServerBuild: "",
65+
apiServerAppName: ""
66+
)
67+
let data = try JSONEncoder().encode(status)
68+
print(String(decoding: data, as: UTF8.self))
69+
} else {
70+
print("apiserver is not running and not registered with launchd")
71+
}
4372
Application.exit(withError: ExitCode(1))
4473
}
4574

4675
// Now ping our friendly daemon. Fail after 10 seconds with no response.
4776
do {
4877
let systemHealth = try await ClientHealthCheck.ping(timeout: .seconds(10))
49-
print("apiserver is running")
50-
print("application data root: \(systemHealth.appRoot.path(percentEncoded: false))")
51-
print("application install root: \(systemHealth.installRoot.path(percentEncoded: false))")
52-
print("container-apiserver version: \(systemHealth.apiServerVersion)")
53-
print("container-apiserver commit: \(systemHealth.apiServerCommit)")
78+
79+
if format == .json {
80+
let status = PrintableStatus(
81+
status: "running",
82+
appRoot: systemHealth.appRoot.path(percentEncoded: false),
83+
installRoot: systemHealth.installRoot.path(percentEncoded: false),
84+
logRoot: systemHealth.logRoot?.string,
85+
apiServerVersion: systemHealth.apiServerVersion,
86+
apiServerCommit: systemHealth.apiServerCommit,
87+
apiServerBuild: systemHealth.apiServerBuild,
88+
apiServerAppName: systemHealth.apiServerAppName
89+
)
90+
let data = try JSONEncoder().encode(status)
91+
print(String(decoding: data, as: UTF8.self))
92+
} else {
93+
let rows: [[String]] = [
94+
["FIELD", "VALUE"],
95+
["status", "running"],
96+
["app-root", systemHealth.appRoot.path(percentEncoded: false)],
97+
["install-root", systemHealth.installRoot.path(percentEncoded: false)],
98+
["log-root", systemHealth.logRoot?.string ?? ""],
99+
["apiserver-version", systemHealth.apiServerVersion],
100+
["apiserver-commit", systemHealth.apiServerCommit],
101+
["apiserver-build", systemHealth.apiServerBuild],
102+
["apiserver-appname", systemHealth.apiServerAppName],
103+
]
104+
let formatter = TableOutput(rows: rows)
105+
print(formatter.format())
106+
}
54107
} catch {
55-
print("apiserver is not running")
108+
if format == .json {
109+
let status = PrintableStatus(
110+
status: "not running",
111+
appRoot: "",
112+
installRoot: "",
113+
logRoot: nil,
114+
apiServerVersion: "",
115+
apiServerCommit: "",
116+
apiServerBuild: "",
117+
apiServerAppName: ""
118+
)
119+
let data = try JSONEncoder().encode(status)
120+
print(String(decoding: data, as: UTF8.self))
121+
} else {
122+
print("apiserver is not running")
123+
}
56124
Application.exit(withError: ExitCode(1))
57125
}
58126
}

0 commit comments

Comments
 (0)