From 6d53cc7a71cce3c80cfe1d539cf0c5b5ecc1b016 Mon Sep 17 00:00:00 2001 From: s-gatti Date: Wed, 9 Apr 2025 00:50:15 +0530 Subject: [PATCH 1/2] Adds support to run camera app yaml test cases using camera-controller Signed-off-by: Sathvik K Gatti Signed-off-by: Suyambulingam Rathinasamy Muthupandi Signed-off-by: Charles Kim --- app/api/api_v1/sockets/web_sockets.py | 20 +++++++ app/constants/websockets_constants.py | 1 + app/test_engine/models/manual_test_case.py | 45 ++++++++++++++++ app/user_prompt_support/__init__.py | 1 + app/user_prompt_support/prompt_request.py | 5 ++ .../sdk_tests/support/chip/chip_server.py | 2 + .../support/yaml_tests/models/chip_test.py | 53 +++++++++++++++---- .../support/yaml_tests/models/test_case.py | 4 +- .../yaml_tests/models/yaml_test_parser.py | 4 +- 9 files changed, 122 insertions(+), 13 deletions(-) diff --git a/app/api/api_v1/sockets/web_sockets.py b/app/api/api_v1/sockets/web_sockets.py index dd9bf724..6e854416 100644 --- a/app/api/api_v1/sockets/web_sockets.py +++ b/app/api/api_v1/sockets/web_sockets.py @@ -17,6 +17,8 @@ from fastapi import APIRouter, WebSocket from fastapi.websockets import WebSocketDisconnect +from loguru import logger +import socket from app.socket_connection_manager import SocketConnectionManager @@ -41,3 +43,21 @@ async def websocket_endpoint(websocket: WebSocket) -> None: except WebSocketDisconnect: socket_connection_manager.disconnect(websocket) + +@router.websocket("/ws/video") +async def websocket_video_endpoint(websocket: WebSocket) -> None: + try: + await websocket.accept() + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) + sock.bind(('0.0.0.0', 5000)) + logger.info("UDP socket bound successfully") + loop = asyncio.get_event_loop() + while True: + try: + data, addr = await loop.run_in_executor(None, sock.recvfrom, 65536) + #send data to ws + await websocket.send_bytes(data) + except Exception as e: + break + except Exception as e: + logger.info(f"Failed with exception {e}") diff --git a/app/constants/websockets_constants.py b/app/constants/websockets_constants.py index 54502c47..fe984b4c 100644 --- a/app/constants/websockets_constants.py +++ b/app/constants/websockets_constants.py @@ -33,6 +33,7 @@ class MessageTypeEnum(str, Enum): TIME_OUT_NOTIFICATION = "time_out_notification" TEST_LOG_RECORDS = "test_log_records" INVALID_MESSAGE = "invalid_message" + STREAM_VERIFICATION_REQUEST = "stream_verification_request" # Enum keys used with messages at the top level diff --git a/app/test_engine/models/manual_test_case.py b/app/test_engine/models/manual_test_case.py index 91a4b467..4df055cd 100644 --- a/app/test_engine/models/manual_test_case.py +++ b/app/test_engine/models/manual_test_case.py @@ -25,7 +25,9 @@ UploadFile, UploadFilePromptRequest, UserPromptSupport, + TextInputPromptRequest, UserResponseStatusEnum, + StreamVerificationPromptRequest ) from .test_case import TestCase @@ -51,6 +53,49 @@ class ManualVerificationTestStep(TestStep, UserPromptSupport): def __init__(self, name: str, verification: Optional[str] = None) -> None: super().__init__(name=name) self.verification = verification + + async def prompt_verification_step_with_response(self, textInputPrompt: TextInputPromptRequest)->str: + """Prompt user to verify the video stream. + + Returns: + str: string response received as user input + """ + prompt_response = await self.invoke_prompt_and_get_str_response(textInputPrompt) + return prompt_response + + async def prompt_stream_verification_step(self) -> bool: + """Prompt user to verify the video stream. + + Raises: + ValueError: When receiving an unexpected response + + Returns: + bool: False if user responds Failed + """ + prompt = self.name + if self.verification is not None: + prompt += f"\n\n{self.verification}" + + options = { + "PASS": PromptOptions.PASS, + "FAIL": PromptOptions.FAIL, + } + prompt_request = StreamVerificationPromptRequest( + prompt=prompt, options=options, timeout=OUTCOME_TIMEOUT_S + ) + prompt_response = await self.send_prompt_request(prompt_request) + self.__evaluate_user_response_for_errors(prompt_response) + + if prompt_response.response == PromptOptions.FAIL: + self.append_failure("User stated manual step FAILED.") + return False + elif prompt_response.response == PromptOptions.PASS: + logger.info("User stated this manual step PASSED.") + return True + else: + raise ValueError( + f"Received unknown prompt option: {prompt_response.response}" + ) async def prompt_verification_step(self) -> bool: """Sends a prompt request to present instructions and get outcome from user. diff --git a/app/user_prompt_support/__init__.py b/app/user_prompt_support/__init__.py index 9f0f77d2..4b12e05b 100644 --- a/app/user_prompt_support/__init__.py +++ b/app/user_prompt_support/__init__.py @@ -19,6 +19,7 @@ PromptRequest, TextInputPromptRequest, UploadFilePromptRequest, + StreamVerificationPromptRequest ) from .prompt_response import PromptResponse from .uploaded_file_support import UploadedFileSupport, UploadFile diff --git a/app/user_prompt_support/prompt_request.py b/app/user_prompt_support/prompt_request.py index be96b2db..6f38a489 100644 --- a/app/user_prompt_support/prompt_request.py +++ b/app/user_prompt_support/prompt_request.py @@ -61,3 +61,8 @@ class MessagePromptRequest(PromptRequest): @property def messageType(self) -> MessageTypeEnum: return MessageTypeEnum.MESSAGE_REQUEST + +class StreamVerificationPromptRequest(OptionsSelectPromptRequest): + @property + def messageType(self) -> MessageTypeEnum: + return MessageTypeEnum.STREAM_VERIFICATION_REQUEST diff --git a/test_collections/matter/sdk_tests/support/chip/chip_server.py b/test_collections/matter/sdk_tests/support/chip/chip_server.py index 7c8082a8..44a7795f 100644 --- a/test_collections/matter/sdk_tests/support/chip/chip_server.py +++ b/test_collections/matter/sdk_tests/support/chip/chip_server.py @@ -34,6 +34,8 @@ CHIP_TOOL_EXE = "./chip-tool" CHIP_TOOL_ARG_PAA_CERTS_PATH = "--paa-trust-store-path" +#TODO: Use chip-camera-controller for camera tests. + # Chip App Parameters CHIP_APP_EXE = "./chip-app1" diff --git a/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py b/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py index 95d4cc46..7b05803f 100644 --- a/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py +++ b/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py @@ -30,17 +30,17 @@ ManualLogUploadStep, ManualVerificationTestStep, ) -from app.user_prompt_support.prompt_request import MessagePromptRequest +from app.user_prompt_support.prompt_request import MessagePromptRequest, PromptRequest, TextInputPromptRequest from app.user_prompt_support.uploaded_file_support import UploadFile from app.user_prompt_support.user_prompt_manager import user_prompt_manager from app.user_prompt_support.user_prompt_support import UserPromptSupport - from ...chip.chip_server import ChipServerType from ...sdk_container import SDKContainer from ...yaml_tests.matter_yaml_runner import MatterYAMLRunner CHIP_TOOL_DEFAULT_PROMPT_TIMEOUT_S = 60 # seconds OUTCOME_TIMEOUT_S = 60 * 10 # Seconds +EXTENDED_PROMPT_TIMEOUT_S = 300 class ChipPromptTypeEnum(str, Enum): @@ -170,28 +170,57 @@ def step_failure( def step_unknown(self) -> None: self.__runned += 1 - async def step_manual(self) -> None: + async def step_manual(self, request) -> None: step = self.current_test_step if not isinstance(step, ManualVerificationTestStep): raise TestError(f"Unexpected user prompt found in test step: {step.name}") try: - await asyncio.wait_for( - self.__prompt_user_manual_step(step), OUTCOME_TIMEOUT_S - ) + if request and request.command == "VerifyVideoStream": + await asyncio.wait_for( + self.__prompt_stream_verification_manual_step(step), OUTCOME_TIMEOUT_S + ) + else: + await asyncio.wait_for( + self.__prompt_user_manual_step(step), OUTCOME_TIMEOUT_S + ) except asyncio.TimeoutError: self.current_test_step.append_failure("Prompt timed out.") self.next_step() - def show_prompt( + async def show_prompt( self, msg: str, placeholder: Optional[str] = None, default_value: Optional[str] = None, - ) -> None: - pass + ) -> str: + step = self.current_test_step + if not isinstance(step, ManualVerificationTestStep): + raise TestError(f"Unexpected user prompt found in test step: {step.name}") + + if placeholder is None: + placeholder = "Enter value.." + + userPrompt = TextInputPromptRequest( + prompt=msg, + timeout=EXTENDED_PROMPT_TIMEOUT_S, + default_value=default_value, + placeholder_text=placeholder, + regex_pattern=None + ) + response = await self.__prompt_user_manual_step_with_response(step, userPrompt) + if response is None: + raise ValueError("Failed to receive user input") + return response # Other methods + async def __prompt_user_manual_step_with_response(self, step: ManualVerificationTestStep, prompt: PromptRequest) -> str: + result = await step.prompt_verification_step_with_response(prompt) + + if not result: + self.current_test_step.append_failure("Manual Test Step Failure.") + result = "Failed getting response" + return result async def __prompt_user_manual_step(self, step: ManualVerificationTestStep) -> None: result = await step.prompt_verification_step() @@ -199,6 +228,12 @@ async def __prompt_user_manual_step(self, step: ManualVerificationTestStep) -> N if not result: self.current_test_step.append_failure("Manual Test Step Failure.") + async def __prompt_stream_verification_manual_step(self, step: ManualVerificationTestStep) -> None: + result = await step.prompt_stream_verification_step() + + if not result: + self.current_test_step.append_failure("Video Verification Test Step Failure.") + def __report_failures(self, logger: Any, request: TestStep, received: Any) -> None: """ The logger from runner contains all logs entries for the test step, this method diff --git a/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py b/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py index 027403b0..918e93b7 100644 --- a/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py +++ b/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py @@ -165,7 +165,7 @@ def _append_automated_test_step(self, yaml_step: MatterTestStep) -> None: Disabled steps are ignored. (Such tests will be marked as 'Steps Disabled' elsewhere) - UserPrompt are special cases that will prompt test operator for input. + UserPrompt, PromptWithResponse or VerifyVideoStream are special cases that will prompt test operator for input. """ if yaml_step.disabled: test_engine_logger.info( @@ -174,7 +174,7 @@ def _append_automated_test_step(self, yaml_step: MatterTestStep) -> None: return step = TestStep(yaml_step.label) - if yaml_step.command == "UserPrompt": + if yaml_step.command in ["UserPrompt", "PromptWithResponse", "VerifyVideoStream"]: step = ManualVerificationTestStep( name=yaml_step.label, verification=yaml_step.verification, diff --git a/test_collections/matter/sdk_tests/support/yaml_tests/models/yaml_test_parser.py b/test_collections/matter/sdk_tests/support/yaml_tests/models/yaml_test_parser.py index 04688589..a979464d 100644 --- a/test_collections/matter/sdk_tests/support/yaml_tests/models/yaml_test_parser.py +++ b/test_collections/matter/sdk_tests/support/yaml_tests/models/yaml_test_parser.py @@ -50,8 +50,8 @@ def _test_type(test: YamlTest) -> MatterTestType: if all(s.disabled is True for s in steps): return MatterTestType.MANUAL - # if any step has a UserPrompt, categorize as semi-automated - if any(s.command == "UserPrompt" for s in steps): + # if any step has a UserPrompt, PromptWithResponse or VerifyVideoStream command, categorize as semi-automated + if any(s.command in ["UserPrompt", "PromptWithResponse", "VerifyVideoStream"] for s in steps): return MatterTestType.SEMI_AUTOMATED # Otherwise Automated From f238435878b2f6e49b68dc726cf6b01c8c56cb2e Mon Sep 17 00:00:00 2001 From: s-gatti Date: Wed, 16 Apr 2025 05:59:18 +0530 Subject: [PATCH 2/2] Apply formatter and fix action workflow failure Signed-off-by: Sathvik K Gatti Signed-off-by: Suyambulingam Rathinasamy Muthupandi Signed-off-by: Charles Kim --- app/api/api_v1/sockets/web_sockets.py | 27 ++++++++++++------- app/constants/websockets_constants.py | 3 +++ app/test_engine/models/manual_test_case.py | 10 ++++--- app/user_prompt_support/__init__.py | 2 +- app/user_prompt_support/prompt_request.py | 1 + .../sdk_tests/support/chip/chip_server.py | 2 +- .../support/yaml_tests/models/chip_test.py | 25 ++++++++++++----- .../support/yaml_tests/models/test_case.py | 9 +++++-- .../yaml_tests/models/yaml_test_parser.py | 8 ++++-- 9 files changed, 61 insertions(+), 26 deletions(-) diff --git a/app/api/api_v1/sockets/web_sockets.py b/app/api/api_v1/sockets/web_sockets.py index 6e854416..c90c0cdc 100644 --- a/app/api/api_v1/sockets/web_sockets.py +++ b/app/api/api_v1/sockets/web_sockets.py @@ -14,12 +14,13 @@ # limitations under the License. # import asyncio +import socket from fastapi import APIRouter, WebSocket from fastapi.websockets import WebSocketDisconnect from loguru import logger -import socket +from app.constants.websockets_constants import UDP_SOCKET_INTERFACE, UDP_SOCKET_PORT from app.socket_connection_manager import SocketConnectionManager router = APIRouter() @@ -44,20 +45,28 @@ async def websocket_endpoint(websocket: WebSocket) -> None: except WebSocketDisconnect: socket_connection_manager.disconnect(websocket) + @router.websocket("/ws/video") async def websocket_video_endpoint(websocket: WebSocket) -> None: try: await websocket.accept() + logger.info(f'Websocket connected: "{websocket}".') + except RuntimeError as e: + logger.info(f'Failed to connect with error: "{e}".') + raise e + + try: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) - sock.bind(('0.0.0.0', 5000)) + sock.bind((UDP_SOCKET_INTERFACE, UDP_SOCKET_PORT)) logger.info("UDP socket bound successfully") loop = asyncio.get_event_loop() while True: - try: - data, addr = await loop.run_in_executor(None, sock.recvfrom, 65536) - #send data to ws - await websocket.send_bytes(data) - except Exception as e: - break + data, _ = await loop.run_in_executor(None, sock.recvfrom, 65536) + # send data to ws + await websocket.send_bytes(data) + except WebSocketDisconnect: + logger.info(f'Websocket for video stream disconnected: "{websocket}".') except Exception as e: - logger.info(f"Failed with exception {e}") + logger.info(f"Failed with {e}") + finally: + await websocket.close() diff --git a/app/constants/websockets_constants.py b/app/constants/websockets_constants.py index fe984b4c..44b59856 100644 --- a/app/constants/websockets_constants.py +++ b/app/constants/websockets_constants.py @@ -21,6 +21,9 @@ MISSING_TYPE_ERROR_STR = "The message is missing a type key" NO_HANDLER_FOR_MSG_ERROR_STR = "There is no handler registered for this message type" +UDP_SOCKET_PORT = 5000 +UDP_SOCKET_INTERFACE = "0.0.0.0" + # Enum Keys for different types of messages currently supported by the tool class MessageTypeEnum(str, Enum): diff --git a/app/test_engine/models/manual_test_case.py b/app/test_engine/models/manual_test_case.py index 4df055cd..b78d34bf 100644 --- a/app/test_engine/models/manual_test_case.py +++ b/app/test_engine/models/manual_test_case.py @@ -22,12 +22,12 @@ from app.user_prompt_support import ( OptionsSelectPromptRequest, PromptResponse, + StreamVerificationPromptRequest, + TextInputPromptRequest, UploadFile, UploadFilePromptRequest, UserPromptSupport, - TextInputPromptRequest, UserResponseStatusEnum, - StreamVerificationPromptRequest ) from .test_case import TestCase @@ -53,8 +53,10 @@ class ManualVerificationTestStep(TestStep, UserPromptSupport): def __init__(self, name: str, verification: Optional[str] = None) -> None: super().__init__(name=name) self.verification = verification - - async def prompt_verification_step_with_response(self, textInputPrompt: TextInputPromptRequest)->str: + + async def prompt_verification_step_with_response( + self, textInputPrompt: TextInputPromptRequest + ) -> str: """Prompt user to verify the video stream. Returns: diff --git a/app/user_prompt_support/__init__.py b/app/user_prompt_support/__init__.py index 4b12e05b..88cb6299 100644 --- a/app/user_prompt_support/__init__.py +++ b/app/user_prompt_support/__init__.py @@ -17,9 +17,9 @@ from .prompt_request import ( OptionsSelectPromptRequest, PromptRequest, + StreamVerificationPromptRequest, TextInputPromptRequest, UploadFilePromptRequest, - StreamVerificationPromptRequest ) from .prompt_response import PromptResponse from .uploaded_file_support import UploadedFileSupport, UploadFile diff --git a/app/user_prompt_support/prompt_request.py b/app/user_prompt_support/prompt_request.py index 6f38a489..51d9104b 100644 --- a/app/user_prompt_support/prompt_request.py +++ b/app/user_prompt_support/prompt_request.py @@ -62,6 +62,7 @@ class MessagePromptRequest(PromptRequest): def messageType(self) -> MessageTypeEnum: return MessageTypeEnum.MESSAGE_REQUEST + class StreamVerificationPromptRequest(OptionsSelectPromptRequest): @property def messageType(self) -> MessageTypeEnum: diff --git a/test_collections/matter/sdk_tests/support/chip/chip_server.py b/test_collections/matter/sdk_tests/support/chip/chip_server.py index 44a7795f..22d08bf1 100644 --- a/test_collections/matter/sdk_tests/support/chip/chip_server.py +++ b/test_collections/matter/sdk_tests/support/chip/chip_server.py @@ -34,7 +34,7 @@ CHIP_TOOL_EXE = "./chip-tool" CHIP_TOOL_ARG_PAA_CERTS_PATH = "--paa-trust-store-path" -#TODO: Use chip-camera-controller for camera tests. +# TODO: Use chip-camera-controller for camera tests. # Chip App Parameters CHIP_APP_EXE = "./chip-app1" diff --git a/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py b/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py index 7b05803f..aa13ba9e 100644 --- a/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py +++ b/test_collections/matter/sdk_tests/support/yaml_tests/models/chip_test.py @@ -30,10 +30,14 @@ ManualLogUploadStep, ManualVerificationTestStep, ) -from app.user_prompt_support.prompt_request import MessagePromptRequest, PromptRequest, TextInputPromptRequest +from app.user_prompt_support.prompt_request import ( + MessagePromptRequest, + TextInputPromptRequest, +) from app.user_prompt_support.uploaded_file_support import UploadFile from app.user_prompt_support.user_prompt_manager import user_prompt_manager from app.user_prompt_support.user_prompt_support import UserPromptSupport + from ...chip.chip_server import ChipServerType from ...sdk_container import SDKContainer from ...yaml_tests.matter_yaml_runner import MatterYAMLRunner @@ -170,7 +174,7 @@ def step_failure( def step_unknown(self) -> None: self.__runned += 1 - async def step_manual(self, request) -> None: + async def step_manual(self, request: TestStep) -> None: step = self.current_test_step if not isinstance(step, ManualVerificationTestStep): raise TestError(f"Unexpected user prompt found in test step: {step.name}") @@ -178,7 +182,8 @@ async def step_manual(self, request) -> None: try: if request and request.command == "VerifyVideoStream": await asyncio.wait_for( - self.__prompt_stream_verification_manual_step(step), OUTCOME_TIMEOUT_S + self.__prompt_stream_verification_manual_step(step), + OUTCOME_TIMEOUT_S, ) else: await asyncio.wait_for( @@ -206,7 +211,7 @@ async def show_prompt( timeout=EXTENDED_PROMPT_TIMEOUT_S, default_value=default_value, placeholder_text=placeholder, - regex_pattern=None + regex_pattern=None, ) response = await self.__prompt_user_manual_step_with_response(step, userPrompt) if response is None: @@ -214,7 +219,9 @@ async def show_prompt( return response # Other methods - async def __prompt_user_manual_step_with_response(self, step: ManualVerificationTestStep, prompt: PromptRequest) -> str: + async def __prompt_user_manual_step_with_response( + self, step: ManualVerificationTestStep, prompt: TextInputPromptRequest + ) -> str: result = await step.prompt_verification_step_with_response(prompt) if not result: @@ -228,11 +235,15 @@ async def __prompt_user_manual_step(self, step: ManualVerificationTestStep) -> N if not result: self.current_test_step.append_failure("Manual Test Step Failure.") - async def __prompt_stream_verification_manual_step(self, step: ManualVerificationTestStep) -> None: + async def __prompt_stream_verification_manual_step( + self, step: ManualVerificationTestStep + ) -> None: result = await step.prompt_stream_verification_step() if not result: - self.current_test_step.append_failure("Video Verification Test Step Failure.") + self.current_test_step.append_failure( + "Video Verification Test Step Failure." + ) def __report_failures(self, logger: Any, request: TestStep, received: Any) -> None: """ diff --git a/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py b/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py index 918e93b7..cfcb7843 100644 --- a/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py +++ b/test_collections/matter/sdk_tests/support/yaml_tests/models/test_case.py @@ -165,7 +165,8 @@ def _append_automated_test_step(self, yaml_step: MatterTestStep) -> None: Disabled steps are ignored. (Such tests will be marked as 'Steps Disabled' elsewhere) - UserPrompt, PromptWithResponse or VerifyVideoStream are special cases that will prompt test operator for input. + UserPrompt, PromptWithResponse or VerifyVideoStream are special cases that will + prompt test operator for input. """ if yaml_step.disabled: test_engine_logger.info( @@ -174,7 +175,11 @@ def _append_automated_test_step(self, yaml_step: MatterTestStep) -> None: return step = TestStep(yaml_step.label) - if yaml_step.command in ["UserPrompt", "PromptWithResponse", "VerifyVideoStream"]: + if yaml_step.command in [ + "UserPrompt", + "PromptWithResponse", + "VerifyVideoStream", + ]: step = ManualVerificationTestStep( name=yaml_step.label, verification=yaml_step.verification, diff --git a/test_collections/matter/sdk_tests/support/yaml_tests/models/yaml_test_parser.py b/test_collections/matter/sdk_tests/support/yaml_tests/models/yaml_test_parser.py index a979464d..a247e3c5 100644 --- a/test_collections/matter/sdk_tests/support/yaml_tests/models/yaml_test_parser.py +++ b/test_collections/matter/sdk_tests/support/yaml_tests/models/yaml_test_parser.py @@ -50,8 +50,12 @@ def _test_type(test: YamlTest) -> MatterTestType: if all(s.disabled is True for s in steps): return MatterTestType.MANUAL - # if any step has a UserPrompt, PromptWithResponse or VerifyVideoStream command, categorize as semi-automated - if any(s.command in ["UserPrompt", "PromptWithResponse", "VerifyVideoStream"] for s in steps): + # if any step has a UserPrompt, PromptWithResponse or VerifyVideoStream command, + # categorize as semi-automated + if any( + s.command in ["UserPrompt", "PromptWithResponse", "VerifyVideoStream"] + for s in steps + ): return MatterTestType.SEMI_AUTOMATED # Otherwise Automated