From 2f7512614634fa72616b5a1562b4f50810d0637a Mon Sep 17 00:00:00 2001 From: usamasaqib Date: Thu, 12 Jun 2025 14:34:40 +0200 Subject: [PATCH 1/7] setup gdb server if port specified --- scenarios/aws/microVMs/config/config.go | 1 + scenarios/aws/microVMs/microvms/domain.go | 36 ++++++++++++++--------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/scenarios/aws/microVMs/config/config.go b/scenarios/aws/microVMs/config/config.go index e855ad7d8..2527aa9ba 100644 --- a/scenarios/aws/microVMs/config/config.go +++ b/scenarios/aws/microVMs/config/config.go @@ -19,6 +19,7 @@ const ( DDMicroVMLocalWorkingDirectory = "localWorkingDir" DDMicroVMRemoteWorkingDirectory = "remoteWorkingDir" DDMicroVMShutdownPeriod = "shutdownPeriod" + DDMicroVMGDBServerPort = "gdbPort" ) var SSHKeyConfigNames = map[string]string{ diff --git a/scenarios/aws/microVMs/microvms/domain.go b/scenarios/aws/microVMs/microvms/domain.go index 0a3559029..806dc3423 100644 --- a/scenarios/aws/microVMs/microvms/domain.go +++ b/scenarios/aws/microVMs/microvms/domain.go @@ -13,6 +13,7 @@ import ( "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/common/namer" "github.com/DataDog/test-infra-definitions/common/utils" + microVMConfig "github.com/DataDog/test-infra-definitions/scenarios/aws/microVMs/config" "github.com/DataDog/test-infra-definitions/scenarios/aws/microVMs/microvms/resources" "github.com/DataDog/test-infra-definitions/scenarios/aws/microVMs/vmconfig" ) @@ -146,6 +147,13 @@ func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory int, hostOS = "linux" // Remote VMs are always on Linux hosts } + m := microVMConfig.NewMicroVMConfig(e) + gdbPort, err = m.TryInt(microVMConfig.DDMicroVMGDBServerPort) + qemuArgs := make(map[string]pulumi.StringInput) + if err == nil { + qemuArgs["-gdb"] = pulumi.Sprintf("tcp:127.0.0.1:%d", gdbPort) + } + if hostOS == "linux" { hypervisor = "kvm" } else if hostOS == "darwin" { @@ -153,21 +161,21 @@ func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory int, // We have to use QEMU network devices because libvirt does not support the macOS // network devices. netID := libvirtResourceName(domainName, "netdev") - qemuArgs := map[string]pulumi.StringInput{ - "-netdev": pulumi.Sprintf("vmnet-shared,id=%s", netID), - // Important: use virtio-net-pci instead of virtio-net-device so that the guest has a PCI - // device and that information can be used by udev to rename the device, instead of having eth0. - // This makes the naming consistent across different execution environments and avoids - // problems (for example, DHCP is configured for interfaces starting with en*, so - // if we had eth0 we wouldn't have a network connection) - // Also, configure the PCI address as 17 so that we don't have conflicts with other libvirt controlled devices - "-device": pulumi.Sprintf("virtio-net-pci,netdev=%s,mac=%s,addr=17", netID, domain.mac), - } - for k, v := range qemuArgs { - commandLine = pulumi.Sprintf("%s\n", commandLine, k) - commandLine = pulumi.Sprintf("%s\n", commandLine, v) - } + qemuArgs["-netdev"] = pulumi.Sprintf("vmnet-shared,id=%s", netID) + // Important: use virtio-net-pci instead of virtio-net-device so that the guest has a PCI + // device and that information can be used by udev to rename the device, instead of having eth0. + // This makes the naming consistent across different execution environments and avoids + // problems (for example, DHCP is configured for interfaces starting with en*, so + // if we had eth0 we wouldn't have a network connection) + // Also, configure the PCI address as 17 so that we don't have conflicts with other libvirt controlled devices + qemuArgs["-device"] = pulumi.Sprintf("virtio-net-pci,netdev=%s,mac=%s,addr=17", netID, domain.mac) + + } + + for k, v := range qemuArgs { + commandLine = pulumi.Sprintf("%s\n", commandLine, k) + commandLine = pulumi.Sprintf("%s\n", commandLine, v) } domain.RecipeLibvirtDomainArgs.Xls = rc.GetDomainXLS( From fe604cfad85dce139e33f3884b78087083dd6e99 Mon Sep 17 00:00:00 2001 From: usamasaqib Date: Thu, 12 Jun 2025 15:54:21 +0200 Subject: [PATCH 2/7] fix gdb port setup --- scenarios/aws/microVMs/microvms/domain.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scenarios/aws/microVMs/microvms/domain.go b/scenarios/aws/microVMs/microvms/domain.go index 806dc3423..ca3b99aee 100644 --- a/scenarios/aws/microVMs/microvms/domain.go +++ b/scenarios/aws/microVMs/microvms/domain.go @@ -13,7 +13,7 @@ import ( "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/common/namer" "github.com/DataDog/test-infra-definitions/common/utils" - microVMConfig "github.com/DataDog/test-infra-definitions/scenarios/aws/microVMs/config" + microvmConfig "github.com/DataDog/test-infra-definitions/scenarios/aws/microVMs/config" "github.com/DataDog/test-infra-definitions/scenarios/aws/microVMs/microvms/resources" "github.com/DataDog/test-infra-definitions/scenarios/aws/microVMs/vmconfig" ) @@ -147,10 +147,15 @@ func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory int, hostOS = "linux" // Remote VMs are always on Linux hosts } - m := microVMConfig.NewMicroVMConfig(e) - gdbPort, err = m.TryInt(microVMConfig.DDMicroVMGDBServerPort) + commonEnv, err := config.NewCommonEnvironment(e.Ctx()) + if err != nil { + return nil, err + } + + m := microvmConfig.NewMicroVMConfig(commonEnv) + gdbPort := m.GetIntWithDefault(m.MicroVMConfig, microvmConfig.DDMicroVMGDBServerPort, 0) qemuArgs := make(map[string]pulumi.StringInput) - if err == nil { + if gdbPort != 0 { qemuArgs["-gdb"] = pulumi.Sprintf("tcp:127.0.0.1:%d", gdbPort) } From f0565862c1ccabe8ebd96e6a55a317d2ea57377b Mon Sep 17 00:00:00 2001 From: Usama Saqib Date: Fri, 4 Jul 2025 14:45:55 +0200 Subject: [PATCH 3/7] if setupGDB bool is true then discover free ports and pass them to QEMU --- scenarios/aws/microVMs/config/config.go | 2 +- scenarios/aws/microVMs/microvms/domain.go | 44 ++++++++++++++++++++--- scenarios/aws/microVMs/microvms/run.go | 1 + 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/scenarios/aws/microVMs/config/config.go b/scenarios/aws/microVMs/config/config.go index 2527aa9ba..125712b35 100644 --- a/scenarios/aws/microVMs/config/config.go +++ b/scenarios/aws/microVMs/config/config.go @@ -19,7 +19,7 @@ const ( DDMicroVMLocalWorkingDirectory = "localWorkingDir" DDMicroVMRemoteWorkingDirectory = "remoteWorkingDir" DDMicroVMShutdownPeriod = "shutdownPeriod" - DDMicroVMGDBServerPort = "gdbPort" + DDMicroVMSetupGDB = "setupGDB" ) var SSHKeyConfigNames = map[string]string{ diff --git a/scenarios/aws/microVMs/microvms/domain.go b/scenarios/aws/microVMs/microvms/domain.go index ca3b99aee..e8da9ecc5 100644 --- a/scenarios/aws/microVMs/microvms/domain.go +++ b/scenarios/aws/microVMs/microvms/domain.go @@ -6,6 +6,7 @@ import ( "path/filepath" "runtime" "strings" + "time" "github.com/pulumi/pulumi-libvirt/sdk/go/libvirt" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -22,6 +23,8 @@ const ( dhcpEntriesTemplate = "" sharedFSMountPoint = "/opt/kernel-version-testing" maxDomainIDLength = 64 + gdbPortRangeStart = 4321 + gdbPortRangeEnd = 4421 ) func getNextVMIP(ip *net.IP) net.IP { @@ -42,6 +45,7 @@ type Domain struct { lvDomain *libvirt.Domain tag string vmset vmconfig.VMSet + gdbPort int } func generateDomainIdentifier(vcpu, memory int, vmsetTags, tag, arch string) string { @@ -102,7 +106,7 @@ func getCPUTuneXML(vmcpus, hostCPUSet, cpuCount int) (string, int) { return fmt.Sprintf("%s", strings.Join(vcpuMap, "\n")), hostCPUSet } -func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory int, kernel vmconfig.Kernel, cputune string) (*Domain, error) { +func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory, gdbPort int, kernel vmconfig.Kernel, cputune string) (*Domain, error) { var err error domain := new(Domain) @@ -152,11 +156,10 @@ func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory int, return nil, err } - m := microvmConfig.NewMicroVMConfig(commonEnv) - gdbPort := m.GetIntWithDefault(m.MicroVMConfig, microvmConfig.DDMicroVMGDBServerPort, 0) qemuArgs := make(map[string]pulumi.StringInput) if gdbPort != 0 { qemuArgs["-gdb"] = pulumi.Sprintf("tcp:127.0.0.1:%d", gdbPort) + domain.gdbPort = gdbPort } if hostOS == "linux" { @@ -235,15 +238,48 @@ func getVolumeDiskTarget(isRootVolume bool, lastDisk string) string { return fmt.Sprintf("/dev/vd%c", rune(int(lastDisk[len(lastDisk)-1])+1)) } +// isPortFree checks if a given TCP port on localhost in free +func isPortFree(port int) bool { + address := fmt.Sprintf("127.0.0.1:%d", host, port) + conn, err := net.DialTimeout("tcp", address, 1*time.Second) + if err != nil { + // If there's an error connecting, we assume the port is free + return true + } + conn.Close() + // If connection was successful, port is in use + return false +} + func GenerateDomainConfigurationsForVMSet(e config.Env, providerFn LibvirtProviderFn, depends []pulumi.Resource, set *vmconfig.VMSet, fs *LibvirtFilesystem, cpuSetStart int) ([]*Domain, int, error) { var domains []*Domain var cpuTuneXML string + m := microvmConfig.NewMicroVMConfig(commonEnv) + setupGDB := m.GetBoolWithDefault(m.MicroVMConfig, microvmConfig.DDMicroVMSetupGDB, false) && set.Arch == LocalVMSet + for _, vcpu := range set.VCpu { for _, memory := range set.Memory { for _, kernel := range set.Kernels { cpuTuneXML, cpuSetStart = getCPUTuneXML(vcpu, cpuSetStart, set.VMHost.AvailableCPUs) - domain, err := newDomainConfiguration(e, set, vcpu, memory, kernel, cpuTuneXML) + + domainPort := 0 + if setupGDB { + for port := gdbPort; port < gdbPortRangeEnd; port++ { + if isPortFree(port) { + domainPort = port + break + } + } + + if domainPort == 0 { + return nil, 0, fmt.Errorf("could not find free port in range [%d,%d] for gdb server", gdbPortRangeStart, gdbPortRangeEnd) + } + + gdbPort = domainPort + } + + domain, err := newDomainConfiguration(e, set, vcpu, memory, domainPort, kernel, cpuTuneXML) if err != nil { return []*Domain{}, 0, err } diff --git a/scenarios/aws/microVMs/microvms/run.go b/scenarios/aws/microVMs/microvms/run.go index a24020baf..cebf57245 100644 --- a/scenarios/aws/microVMs/microvms/run.go +++ b/scenarios/aws/microVMs/microvms/run.go @@ -270,6 +270,7 @@ func exportVMInformation(ctx *pulumi.Context, instances map[string]*Instance, vm "tag": pulumi.ToOutput(domain.tag), "vmset-tags": pulumi.ToArrayOutput(tags), "ssh-key-path": pulumi.ToOutput(filepath.Join(GetWorkingDirectory(domain.vmset.Arch), "ddvm_rsa")), + "gdb-port": pulumi.ToOutput(domain.gdbPort), })) } } From 845e54952448caddbd0236ea482e81c90a30deb4 Mon Sep 17 00:00:00 2001 From: Usama Saqib Date: Fri, 4 Jul 2025 14:55:45 +0200 Subject: [PATCH 4/7] fix build errors --- scenarios/aws/microVMs/microvms/domain.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scenarios/aws/microVMs/microvms/domain.go b/scenarios/aws/microVMs/microvms/domain.go index e8da9ecc5..55c1e2227 100644 --- a/scenarios/aws/microVMs/microvms/domain.go +++ b/scenarios/aws/microVMs/microvms/domain.go @@ -151,11 +151,6 @@ func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory, gdb hostOS = "linux" // Remote VMs are always on Linux hosts } - commonEnv, err := config.NewCommonEnvironment(e.Ctx()) - if err != nil { - return nil, err - } - qemuArgs := make(map[string]pulumi.StringInput) if gdbPort != 0 { qemuArgs["-gdb"] = pulumi.Sprintf("tcp:127.0.0.1:%d", gdbPort) @@ -240,12 +235,13 @@ func getVolumeDiskTarget(isRootVolume bool, lastDisk string) string { // isPortFree checks if a given TCP port on localhost in free func isPortFree(port int) bool { - address := fmt.Sprintf("127.0.0.1:%d", host, port) + address := fmt.Sprintf("127.0.0.1:%d", port) conn, err := net.DialTimeout("tcp", address, 1*time.Second) if err != nil { // If there's an error connecting, we assume the port is free return true } + conn.Close() // If connection was successful, port is in use return false @@ -255,8 +251,13 @@ func GenerateDomainConfigurationsForVMSet(e config.Env, providerFn LibvirtProvid var domains []*Domain var cpuTuneXML string - m := microvmConfig.NewMicroVMConfig(commonEnv) + commonEnv, err := config.NewCommonEnvironment(e.Ctx()) + if err != nil { + return nil, err + } + m := microvmConfig.NewMicroVMConfig(OcommonEnv) setupGDB := m.GetBoolWithDefault(m.MicroVMConfig, microvmConfig.DDMicroVMSetupGDB, false) && set.Arch == LocalVMSet + gdbPort := gdbPortRangeStart for _, vcpu := range set.VCpu { for _, memory := range set.Memory { From b8e6daece8153a83da5b53203ef90503ae581017 Mon Sep 17 00:00:00 2001 From: Usama Saqib Date: Fri, 4 Jul 2025 16:48:45 +0200 Subject: [PATCH 5/7] fix build errors --- scenarios/aws/microVMs/microvms/domain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scenarios/aws/microVMs/microvms/domain.go b/scenarios/aws/microVMs/microvms/domain.go index 55c1e2227..7d5b78536 100644 --- a/scenarios/aws/microVMs/microvms/domain.go +++ b/scenarios/aws/microVMs/microvms/domain.go @@ -253,9 +253,9 @@ func GenerateDomainConfigurationsForVMSet(e config.Env, providerFn LibvirtProvid commonEnv, err := config.NewCommonEnvironment(e.Ctx()) if err != nil { - return nil, err + return nil, 0, err } - m := microvmConfig.NewMicroVMConfig(OcommonEnv) + m := microvmConfig.NewMicroVMConfig(commonEnv) setupGDB := m.GetBoolWithDefault(m.MicroVMConfig, microvmConfig.DDMicroVMSetupGDB, false) && set.Arch == LocalVMSet gdbPort := gdbPortRangeStart From 2741ae5a645d9b446182f6508897e529224c88b3 Mon Sep 17 00:00:00 2001 From: Usama Saqib Date: Fri, 4 Jul 2025 16:50:12 +0200 Subject: [PATCH 6/7] add commandline passthrough to xsl template --- .../aws/microVMs/microvms/resources/distro/domain-amd64.xsl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scenarios/aws/microVMs/microvms/resources/distro/domain-amd64.xsl b/scenarios/aws/microVMs/microvms/resources/distro/domain-amd64.xsl index 63671da56..6ef430122 100644 --- a/scenarios/aws/microVMs/microvms/resources/distro/domain-amd64.xsl +++ b/scenarios/aws/microVMs/microvms/resources/distro/domain-amd64.xsl @@ -69,6 +69,9 @@ + + {commandLine} + From 1928b39c25df04a8c62cd5c313acf414e5071939 Mon Sep 17 00:00:00 2001 From: Usama Saqib Date: Mon, 7 Jul 2025 14:56:06 +0200 Subject: [PATCH 7/7] increment port number so unique ports are selected --- scenarios/aws/microVMs/microvms/domain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/aws/microVMs/microvms/domain.go b/scenarios/aws/microVMs/microvms/domain.go index 7d5b78536..e9ec6bc9f 100644 --- a/scenarios/aws/microVMs/microvms/domain.go +++ b/scenarios/aws/microVMs/microvms/domain.go @@ -277,7 +277,7 @@ func GenerateDomainConfigurationsForVMSet(e config.Env, providerFn LibvirtProvid return nil, 0, fmt.Errorf("could not find free port in range [%d,%d] for gdb server", gdbPortRangeStart, gdbPortRangeEnd) } - gdbPort = domainPort + gdbPort = domainPort + 1 // evaluate another port in the next iteration } domain, err := newDomainConfiguration(e, set, vcpu, memory, domainPort, kernel, cpuTuneXML)