Skip to content

Commit f25b244

Browse files
authored
fix(e2e): make sshd setup idempotent across reboots (AD domain-controller flakiness) (#54681)
### What does this PR do? Makes setup-ssh.ps1 idempotent across reboots: a marker file skips the OpenSSH stop/MSI-reinstall once already done ### Motivation This script is persisted as EC2 user data and reruns on every boot, not just the first (see DataDog/test-infra-definitions#1178, which relies on that for authorized_keys reset). It wasn't written with that in mind: on every boot of a Windows Server 2025 host it unconditionally stopped sshd and reinstalled it via MSI. That means every reboot -- including the one triggered by domain controller promotion -- created an extended sshd outage racing whatever tries to SSH in right after, which is the likely actual cause of the WINA-2095 flakiness this was meant to address, not some AD-side instability. https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html ### Describe how you validated your changes Windows e2e tests should pass. Co-authored-by: branden.clark <branden.clark@datadoghq.com>
1 parent a055712 commit f25b244

2 files changed

Lines changed: 36 additions & 1 deletion

File tree

.gitlab-ci.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,9 @@
121121

122122
# E2E framework
123123
- test/e2e-framework/components/activedirectory/**/*
124+
- test/e2e-framework/components/os/scripts/setup-ssh.ps1
125+
- test/e2e-framework/components/os/windows*
126+
- test/e2e-framework/scenarios/aws/ec2/os_win.go
124127

125128
# Process manager
126129
.windows_path_4: &windows_path_4

test/e2e-framework/components/os/scripts/setup-ssh.ps1

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,12 @@
1+
# This script is embedded as EC2 user data with <persist>true</persist>, so it reruns on every
2+
# boot, not just the first -- see https://github.com/DataDog/test-infra-definitions/pull/1178,
3+
# which relies on that to let authorized_keys be reset on every boot for reused/custom AMIs. Every
4+
# step below must therefore either be safe to repeat unconditionally, or be guarded to only do real
5+
# work once. In particular, replacing OpenSSH via MSI stops sshd and takes a while; doing that on
6+
# every boot -- including the one triggered by domain controller promotion -- creates an extended
7+
# sshd outage that can overlap with whatever is trying to SSH in right after reboot. See WINA-2095.
8+
$sshInstallMarkerPath = "$env:ProgramData\ssh\.dd-openssh-installed"
9+
110
# function to test if the OS is Windows Server 2025
211
function Is-WindowsServer2025 {
312
$osInfo = Get-CimInstance -ClassName Win32_OperatingSystem | Select-Object Caption, Version, BuildNumber
@@ -7,13 +16,19 @@ function Is-WindowsServer2025 {
716

817
# function to test if the sshd service is running and if it needs to be replaced
918
function Test-SshInstallationNeeded {
19+
if (Test-Path $sshInstallMarkerPath) {
20+
# Already installed/replaced on a previous boot -- nothing to do.
21+
return $false
22+
}
23+
1024
$service = Get-Service -Name sshd -ErrorAction SilentlyContinue
1125

1226
if ($service -ne $null) {
1327
Write-Host "Stop sshd service"
1428
Stop-Service sshd
1529
if (Is-WindowsServer2025) {
16-
# for Windows Server 2025, replace the service
30+
# Windows Server 2025 ships a preinstalled OpenSSH that's a different, inconsistent version;
31+
# replace it with our pinned MSI version below (only happens once, per $sshInstallMarkerPath).
1732
return $true
1833
}
1934
} else {
@@ -134,6 +149,22 @@ if (Test-SshInstallationNeeded) {
134149
Set-Service -Name sshd -StartupType Automatic
135150
$retries++
136151
}
152+
153+
# Write sshd_config so LogLevel DEBUG3 is set so OpenSSH/Operational captures more detail if
154+
# sshd fails to stay up after a later reboot (e.g. domain controller promotion, see WINA-2095).
155+
Write-Host "Writing sshd_config"
156+
$sshdConfigLines = @(
157+
"LogLevel DEBUG3",
158+
"AuthorizedKeysFile`t.ssh/authorized_keys",
159+
"Subsystem`tsftp`tsftp-server.exe",
160+
"",
161+
"Match Group administrators",
162+
" AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys"
163+
)
164+
Set-Content -Path "$env:ProgramData\ssh\sshd_config" -Value $sshdConfigLines
165+
Restart-Service sshd -ErrorAction SilentlyContinue
166+
167+
New-Item -Path $sshInstallMarkerPath -ItemType File -Force | Out-Null
137168
}
138169

139170
Restore-AutoInheritedFlag
@@ -176,6 +207,7 @@ while (-not (Test-Path $env:ProgramData\ssh\administrators_authorized_keys)) {
176207
}
177208
Add-Content -Path $env:ProgramData\ssh\administrators_authorized_keys -Value $authorizedKey
178209
icacls.exe ""$env:ProgramData\ssh\administrators_authorized_keys"" /inheritance:r /grant ""Administrators:F"" /grant ""SYSTEM:F""
210+
179211
# Start sshd service
180212
$retries = 0
181213
while ((Get-Service -Name sshd -ErrorAction SilentlyContinue).Status -ne "Running") {

0 commit comments

Comments
 (0)