diff --git a/scenarios/aws/microVMs/config/config.go b/scenarios/aws/microVMs/config/config.go index e855ad7d8..125712b35 100644 --- a/scenarios/aws/microVMs/config/config.go +++ b/scenarios/aws/microVMs/config/config.go @@ -19,6 +19,7 @@ const ( DDMicroVMLocalWorkingDirectory = "localWorkingDir" DDMicroVMRemoteWorkingDirectory = "remoteWorkingDir" DDMicroVMShutdownPeriod = "shutdownPeriod" + DDMicroVMSetupGDB = "setupGDB" ) var SSHKeyConfigNames = map[string]string{ diff --git a/scenarios/aws/microVMs/microvms/domain.go b/scenarios/aws/microVMs/microvms/domain.go index 4443558bd..95295509d 100644 --- a/scenarios/aws/microVMs/microvms/domain.go +++ b/scenarios/aws/microVMs/microvms/domain.go @@ -6,6 +6,7 @@ import ( "path/filepath" "runtime" "strings" + "time" "github.com/pulumi/pulumi-libvirt/sdk/go/libvirt" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -13,6 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/common/namer" "github.com/DataDog/test-infra-definitions/common/utils" + microvmConfig "github.com/DataDog/test-infra-definitions/scenarios/aws/microVMs/config" "github.com/DataDog/test-infra-definitions/scenarios/aws/microVMs/microvms/resources" "github.com/DataDog/test-infra-definitions/scenarios/aws/microVMs/vmconfig" ) @@ -21,6 +23,8 @@ const ( dhcpEntriesTemplate = "" sharedFSMountPoint = "/opt/kernel-version-testing" maxDomainIDLength = 64 + gdbPortRangeStart = 4321 + gdbPortRangeEnd = 4421 ) func getNextVMIP(ip *net.IP) net.IP { @@ -41,6 +45,7 @@ type Domain struct { lvDomain *libvirt.Domain tag string vmset vmconfig.VMSet + gdbPort int } func generateDomainIdentifier(vcpu, memory int, vmsetTags, tag, arch string) string { @@ -101,7 +106,7 @@ func getCPUTuneXML(vmcpus, hostCPUSet, cpuCount int) (string, int) { return fmt.Sprintf("%s", strings.Join(vcpuMap, "\n")), hostCPUSet } -func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory int, kernel vmconfig.Kernel, cputune string) (*Domain, error) { +func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory, gdbPort int, kernel vmconfig.Kernel, cputune string) (*Domain, error) { var err error domain := new(Domain) @@ -146,6 +151,12 @@ func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory int, hostOS = "linux" // Remote VMs are always on Linux hosts } + qemuArgs := make(map[string]pulumi.StringInput) + if gdbPort != 0 { + qemuArgs["-gdb"] = pulumi.Sprintf("tcp:127.0.0.1:%d", gdbPort) + domain.gdbPort = gdbPort + } + var driver string if hostOS == "linux" { hypervisor = "kvm" @@ -155,25 +166,23 @@ func newDomainConfiguration(e config.Env, set *vmconfig.VMSet, vcpu, memory int, // We have to use QEMU network devices because libvirt does not support the macOS // network devices. netID := libvirtResourceName(domainName, "netdev") - qemuArgs := map[string]pulumi.StringInput{ - "-netdev": pulumi.Sprintf("vmnet-shared,id=%s", netID), - // Important: use virtio-net-pci instead of virtio-net-device so that the guest has a PCI - // device and that information can be used by udev to rename the device, instead of having eth0. - // This makes the naming consistent across different execution environments and avoids - // problems (for example, DHCP is configured for interfaces starting with en*, so - // if we had eth0 we wouldn't have a network connection) - // Also, configure the PCI address as 17 so that we don't have conflicts with other libvirt controlled devices - "-device": pulumi.Sprintf("virtio-net-pci,netdev=%s,mac=%s,addr=17", netID, domain.mac), - } - - for k, v := range qemuArgs { - commandLine = pulumi.Sprintf("%s\n", commandLine, k) - commandLine = pulumi.Sprintf("%s\n", commandLine, v) - } + qemuArgs["-netdev"] = pulumi.Sprintf("vmnet-shared,id=%s", netID) + // Important: use virtio-net-pci instead of virtio-net-device so that the guest has a PCI + // device and that information can be used by udev to rename the device, instead of having eth0. + // This makes the naming consistent across different execution environments and avoids + // problems (for example, DHCP is configured for interfaces starting with en*, so + // if we had eth0 we wouldn't have a network connection) + // Also, configure the PCI address as 17 so that we don't have conflicts with other libvirt controlled devices + qemuArgs["-device"] = pulumi.Sprintf("virtio-net-pci,netdev=%s,mac=%s,addr=17", netID, domain.mac) driver = "" } + for k, v := range qemuArgs { + commandLine = pulumi.Sprintf("%s\n", commandLine, k) + commandLine = pulumi.Sprintf("%s\n", commandLine, v) + } + domain.RecipeLibvirtDomainArgs.Xls = rc.GetDomainXLS( map[string]pulumi.StringInput{ resources.SharedFSMount: pulumi.String(sharedFSMountPoint), @@ -227,15 +236,54 @@ func getVolumeDiskTarget(isRootVolume bool, lastDisk string) string { return fmt.Sprintf("/dev/vd%c", rune(int(lastDisk[len(lastDisk)-1])+1)) } +// isPortFree checks if a given TCP port on localhost in free +func isPortFree(port int) bool { + address := fmt.Sprintf("127.0.0.1:%d", port) + conn, err := net.DialTimeout("tcp", address, 1*time.Second) + if err != nil { + // If there's an error connecting, we assume the port is free + return true + } + + conn.Close() + // If connection was successful, port is in use + return false +} + func GenerateDomainConfigurationsForVMSet(e config.Env, providerFn LibvirtProviderFn, depends []pulumi.Resource, set *vmconfig.VMSet, fs *LibvirtFilesystem, cpuSetStart int) ([]*Domain, int, error) { var domains []*Domain var cpuTuneXML string + commonEnv, err := config.NewCommonEnvironment(e.Ctx()) + if err != nil { + return nil, 0, err + } + m := microvmConfig.NewMicroVMConfig(commonEnv) + setupGDB := m.GetBoolWithDefault(m.MicroVMConfig, microvmConfig.DDMicroVMSetupGDB, false) && set.Arch == LocalVMSet + gdbPort := gdbPortRangeStart + for _, vcpu := range set.VCpu { for _, memory := range set.Memory { for _, kernel := range set.Kernels { cpuTuneXML, cpuSetStart = getCPUTuneXML(vcpu, cpuSetStart, set.VMHost.AvailableCPUs) - domain, err := newDomainConfiguration(e, set, vcpu, memory, kernel, cpuTuneXML) + + domainPort := 0 + if setupGDB { + for port := gdbPort; port < gdbPortRangeEnd; port++ { + if isPortFree(port) { + domainPort = port + break + } + } + + if domainPort == 0 { + return nil, 0, fmt.Errorf("could not find free port in range [%d,%d] for gdb server", gdbPortRangeStart, gdbPortRangeEnd) + } + + gdbPort = domainPort + 1 // evaluate another port in the next iteration + } + + domain, err := newDomainConfiguration(e, set, vcpu, memory, domainPort, kernel, cpuTuneXML) if err != nil { return []*Domain{}, 0, err } diff --git a/scenarios/aws/microVMs/microvms/resources/distro/domain-amd64.xsl b/scenarios/aws/microVMs/microvms/resources/distro/domain-amd64.xsl index e02225995..3570bb3f8 100644 --- a/scenarios/aws/microVMs/microvms/resources/distro/domain-amd64.xsl +++ b/scenarios/aws/microVMs/microvms/resources/distro/domain-amd64.xsl @@ -69,6 +69,9 @@ + + {commandLine} + diff --git a/scenarios/aws/microVMs/microvms/run.go b/scenarios/aws/microVMs/microvms/run.go index a24020baf..cebf57245 100644 --- a/scenarios/aws/microVMs/microvms/run.go +++ b/scenarios/aws/microVMs/microvms/run.go @@ -270,6 +270,7 @@ func exportVMInformation(ctx *pulumi.Context, instances map[string]*Instance, vm "tag": pulumi.ToOutput(domain.tag), "vmset-tags": pulumi.ToArrayOutput(tags), "ssh-key-path": pulumi.ToOutput(filepath.Join(GetWorkingDirectory(domain.vmset.Arch), "ddvm_rsa")), + "gdb-port": pulumi.ToOutput(domain.gdbPort), })) } }