e2e: verify podvm console logs

src/cloud-api-adaptor/test/e2e/assessment_runner.go

@@ -64,6 +64,7 @@ type TestCase struct {
 	expectedCaaPodLogStrings    []string
 	expectedPodLogString        string
 	expectedPodEventErrorString string
+	expectedPodvmConsoleLog     string
 	podState                    v1.PodPhase
 	imagePullTimer              bool
 	saImagePullSecret           string
@@ -132,6 +133,11 @@ func (pod *ExtraPod) WithTestCommands(TestCommands []TestCommand) *ExtraPod {
 	return pod
 }
 
+func (tc *TestCase) WithExpectedPodvmConsoleLog(expectedPodvmConsoleLog string) *TestCase {
+	tc.expectedPodvmConsoleLog = expectedPodvmConsoleLog
+	return tc
+}
+
 func (tc *TestCase) WithExpectedCaaPodLogStrings(expectedCaaPodLogStrings ...string) *TestCase {
 	tc.expectedCaaPodLogStrings = expectedCaaPodLogStrings
 	return tc
@@ -224,6 +230,24 @@ func (tc *TestCase) Run() {
 				if err = client.Resources().Create(ctx, tc.pod); err != nil {
 					t.Fatal(err)
 				}
+				var podvmName string
+				if tc.expectedPodvmConsoleLog != "" {
+					for i := range 10 {
+						podvmName, err = getPodvmName(ctx, client, tc.pod)
+						if err != nil {
+							t.Logf("%d attempt: to getPodvmName failed: %v", i, err)
+							time.Sleep(2 * time.Second)
+						} else {
+							break
+						}
+					}
+					if podvmName != "" {
+						t.Log("Verifying PodVM console log")
+						tc.assert.VerifyPodvmConsole(t, podvmName, tc.expectedPodvmConsoleLog)
+					} else {
+						t.Logf("Warning: Failed to validated as podvmName is failed")
+					}
+				}
 				if err = wait.For(conditions.New(client.Resources()).PodPhaseMatch(tc.pod, tc.podState), wait.WithTimeout(WAIT_POD_RUNNING_TIMEOUT)); err != nil {
 					t.Error(err)
 				}

src/cloud-api-adaptor/test/e2e/aws_common.go

@@ -70,3 +70,10 @@ func (aa AWSAssert) GetInstanceType(t *testing.T, podName string) (string, error
 	// Get Instance Type of PodVM
 	return "", nil
 }
+
+func (aa AWSAssert) VerifyPodvmConsole(t *testing.T, podvmName, expectedString string) {
+	// Verify PodVM console output with provided expectedString
+	// This is not implemented for AWS as of now.
+	// So skipping this test.
+	t.Log("Warning: console verification is not added for AWS")
+}

src/cloud-api-adaptor/test/e2e/azure_common.go

@@ -83,3 +83,10 @@ func (c AzureCloudAssert) GetInstanceType(t *testing.T, podName string) (string,
 	}
 
 }
+
+func (c AzureCloudAssert) VerifyPodvmConsole(t *testing.T, podvmName, expectedString string) {
+	// Verify PodVM console output with provided expectedString
+	// This is not implemented for Azure as of now.
+	// So skipping this test.
+	t.Log("Warning: console verification is not added for Azure")
+}

src/cloud-api-adaptor/test/e2e/byom_common.go

@@ -27,3 +27,10 @@ func (b ByomAssert) GetInstanceType(t *testing.T, podName string) (string, error
 	// Get Instance Type of PodVM
 	return "", nil
 }
+
+func (b ByomAssert) VerifyPodvmConsole(t *testing.T, podvmName, expectedString string) {
+	// Verify PodVM console output with provided expectedString
+	// This is not implemented for Byom as of now.
+	// So skipping this test.
+	t.Log("Warning: console verification is not added for Byom")
+}

src/cloud-api-adaptor/test/e2e/common.go

@@ -598,9 +598,10 @@ func CreateSealedSecretValue(resourceURI string) string {
 
 // CloudAssert defines assertions to perform on the cloud provider.
 type CloudAssert interface {
-	HasPodVM(t *testing.T, id string)                             // Assert there is a PodVM with `id`.
-	GetInstanceType(t *testing.T, podName string) (string, error) // Get Instance Type of PodVM
-	DefaultTimeout() time.Duration                                // Default timeout for cloud operations
+	HasPodVM(t *testing.T, id string)                                  // Assert there is a PodVM with `id`.
+	GetInstanceType(t *testing.T, podName string) (string, error)      // Get Instance Type of PodVM
+	DefaultTimeout() time.Duration                                     // Default timeout for cloud operations
+	VerifyPodvmConsole(t *testing.T, podvmNamw, expectedString string) // Verify PodVM console output
 }
 
 // RollingUpdateAssert defines assertions for rolling update test

src/cloud-api-adaptor/test/e2e/common_suite.go

@@ -29,6 +29,7 @@ var E2eNamespace = envconf.RandomName("coco-pp-e2e-test", 25)
 func DoTestCreateSimplePod(t *testing.T, e env.Environment, assert CloudAssert) {
 	pod := NewBusyboxPodWithName(E2eNamespace, "simple-test").GetPodOrFatal(t)
 	if isTestOnCrio() {
+		t.Log("crio busybox error")
 		NewTestCase(t, e, "SimplePeerPod", assert, "PodVM is created").WithPod(pod).Run()
 	} else {
 		NewTestCase(t, e, "SimplePeerPod", assert, "PodVM is created").WithPod(pod).WithNydusSnapshotter().Run()
@@ -612,7 +613,7 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert, kb
 		},
 	}
 
-	NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release is successful").WithPod(pod).WithTestCommands(testCommands).Run()
+	NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release is successful").WithPod(pod).WithTestCommands(testCommands).WithExpectedPodvmConsoleLog("error").Run()
 }
 
 // DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaining opa in kbs

src/cloud-api-adaptor/test/e2e/docker_common.go

@@ -51,3 +51,10 @@ func (l DockerAssert) GetInstanceType(t *testing.T, podName string) (string, err
 	// Get Instance Type of PodVM
 	return "", nil
 }
+
+func (l DockerAssert) VerifyPodvmConsole(t *testing.T, podvmName, expectedString string) {
+	// Verify PodVM console output with provided expectedString
+	// This is not implemented for Docker as of now.
+	// So skipping this test.
+	t.Log("Warning: console verification is not added for Docker")
+}

src/cloud-api-adaptor/test/e2e/gcp_common.go

@@ -74,3 +74,10 @@ func (c GCPCloudAssert) GetInstanceType(t *testing.T, podName string) (string, e
 
 	return vm.GetMachineType(), nil
 }
+
+func (l GCPCloudAssert) VerifyPodvmConsole(t *testing.T, podvmName, expectedString string) {
+	// Verify PodVM console output with provided expectedString
+	// This is not implemented for GCP as of now.
+	// So skipping this test.
+	t.Log("Warning: console verification is not added for GCP")
+}

src/cloud-api-adaptor/test/e2e/ibmcloud_common.go

@@ -386,3 +386,75 @@ func (c *IBMRollingUpdateAssert) VerifyOldVmDeleted(t *testing.T) {
 		}
 	}
 }
+
+func (c IBMCloudAssert) VerifyPodvmConsole(t *testing.T, podvmName, expectedString string) {
+
+	// vpcClient := c.VPC
+
+	// // Find the instance by name
+	// listOpts := &vpcv1.ListInstancesOptions{}
+	// instanceList, _, err := vpcClient.ListInstances(listOpts)
+	// require.NoError(t, err, "Failed to list instances from IBM Cloud VPC")
+
+	// var instanceID string
+	// for _, inst := range instanceList.Instances {
+	// 	if inst.Name != nil && *inst.Name == podvmName {
+	// 		instanceID = *inst.ID
+	// 		break
+	// 	}
+	// }
+
+	// // Create console access token
+	// opts := &vpcv1.CreateInstanceConsoleAccessTokenOptions{
+	// 	InstanceID:  core.StringPtr(instanceID),
+	// 	ConsoleType: core.StringPtr("serial"),
+	// }
+	// tokenResp, _, err := c.VPC.CreateInstanceConsoleAccessToken(opts)
+	// require.NoError(t, err, "Failed to create console access token")
+	// accessToken := *tokenResp.AccessToken
+	// href := *tokenResp.Href // WebSocket URL to connect to console
+
+	// t.Logf("Console access token: %s\nWebsocket URL: %s", accessToken, href)
+
+	// // Connect to websocket
+	// u, err := url.Parse(href)
+	// require.NoError(t, err, "Invalid console websocket URL")
+	// q := u.Query()
+	// q.Set("access_token", accessToken)
+	// u.RawQuery = q.Encode()
+	// t.Logf("Full WS URL: %s", u.String())
+
+	// wsDialer := websocket.Dialer{
+	// 	HandshakeTimeout: 10 * time.Second,
+	// }
+	// wsConn, _, err := wsDialer.Dial(u.String(), nil)
+	// require.NoError(t, err, "WebSocket dial failed")
+	// defer wsConn.Close()
+
+	// // Read console output frames for some time and error
+	// timeout := time.After(3 * time.Minute)
+	// for {
+	// 	select {
+	// 	case <-timeout:
+	// 		// Not erroring out here as i am not sure in implementation part
+	// 		t.Logf("Expected string %q not found on console of instance %s", expectedString, instanceID)
+	// 	default:
+	// 		_, message, err := wsConn.ReadMessage()
+	// 		if err != nil {
+	// 			t.Logf("WebSocket read error: %v", err)
+	// 			time.Sleep(20 * time.Second)
+	// 			continue
+	// 		}
+	// 		msgStr := string(message)
+	// 		t.Logf("Console output chunk: %s", msgStr)
+	// 		if strings.Contains(msgStr, expectedString) {
+	// 			t.Logf("Found expected string %s on console of instance %s", expectedString, instanceID)
+	// 		}
+	// 	}
+	// }
+
+	// Verify PodVM console output with provided expectedString
+	// This is not implemented for Docker as of now.
+	// So skipping this test.
+	t.Log("Warning: console verification is not added for Docker")
+}

src/cloud-api-adaptor/test/e2e/libvirt_common.go

@@ -81,3 +81,95 @@ func CreateInstanceProfileFromCPUMemory(cpu uint, memory uint) string {
 	memStr := strconv.FormatUint(uint64(memory), 10)
 	return cpuStr + "x" + memStr
 }
+
+func (l LibvirtAssert) VerifyPodvmConsole(t *testing.T, podvmName, expectedString string) {
+
+	var dom *libvirt.Domain
+	var err error
+	t.Logf("Looking for PodVM %s", podvmName)
+	domains, err := l.conn.ListAllDomains(libvirt.CONNECT_LIST_DOMAINS_ACTIVE)
+	for _, dom := range domains {
+		name, _ := dom.GetName()
+		t.Logf("Found PodVM %s", name)
+	}
+
+	for range 10 {
+		t.Logf("Checking for PodVM %s", podvmName)
+		dom, err = l.conn.LookupDomainByName(podvmName)
+		t.Logf("LookupDomainByName returned err: %v", err)
+		if err == nil {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	if dom == nil {
+		t.Error("PodVM was not created")
+	}
+
+	state, _, err := dom.GetState()
+	if err != nil {
+		t.Error("Failed to get domain state")
+	}
+
+	if state == libvirt.DOMAIN_SHUTOFF {
+		t.Log("starting podvm")
+		err = dom.Create()
+		if err != nil {
+			t.Error("Failed to start domain")
+		}
+
+	}
+
+	stream, err := l.conn.NewStream(0)
+	for err != nil {
+		t.Logf("Warning: Failed to create stream : %v", err)
+	}
+
+	defer stream.Free()
+
+	err = dom.OpenConsole("", stream, libvirt.DOMAIN_CONSOLE_FORCE)
+	start := time.Now()
+	duration := 1 * time.Minute
+	for err != nil && time.Since(start) < duration {
+		t.Logf("Warning: Failed to open console : %v", err)
+		err = dom.OpenConsole("", stream, libvirt.DOMAIN_CONSOLE_FORCE)
+		time.Sleep(2 * time.Second)
+	}
+
+	if err != nil {
+		t.Logf("Warning: Failed to open console after retries : %v", err)
+		return
+	}
+
+	buf := make([]byte, 4096)
+	var output strings.Builder
+	var LibvirtLog = ""
+
+	start = time.Now()
+	duration = 6 * time.Minute
+	maxBytes := 0
+
+	for time.Since(start) < duration {
+		n, err := stream.Recv(buf)
+		if maxBytes < n {
+			maxBytes = n
+			output.Write(buf[:n])
+			if len(output.String()) > len(LibvirtLog) {
+				LibvirtLog = output.String()
+			}
+			if strings.Contains(LibvirtLog, expectedString) {
+				t.Logf("Found expected String :%s in \n console :%s", expectedString, LibvirtLog)
+				return
+			}
+		}
+		if err != nil && LibvirtLog != "" {
+			t.Logf("Warning: Did not find expected String :%s in \n console :%s", expectedString, LibvirtLog)
+		} else if err != nil {
+			t.Logf("Warning: Did not receive any data from console yet, err: %v", err)
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	t.Logf("Warning: Timed out waiting for expected String :%s in \n console :%s", expectedString, LibvirtLog)
+}