BCP-008-01/02 Status Monitoring test suites (#869)

Co-authored-by: Simon Lo <[email protected]>
Co-authored-by: Gareth Sylvester-Bradley <[email protected]>
Co-authored-by: Gareth Sylvester-Bradley <[email protected]>
Co-authored-by: alabou <[email protected]>
Co-authored-by: cristian-recoseanu <[email protected]>

Author: 5 people

README.md

@@ -34,7 +34,7 @@ The following test suites are currently supported.
 | IS-10-01 | IS-10 Authorization API | | | | Authorization Server |
 | IS-11-01 | IS-11 Stream Compatibility Management API | X | | | |
 | IS-12-01 | IS-12 Control Protocol API | X | | | See [Invasive Device Model Testing](docs/2.10%20Usage%20-%20Invasive%20Device%20Model%20Testing.md) |
-| IS-14-01 | IS-14 Device Configuration API | X | | | |
+| IS-14-01 | IS-14 Device Configuration API | X | | | See [Invasive Device Model Testing](docs/2.10%20Usage%20-%20Invasive%20Device%20Model%20Testing.md) |
 | - | BCP-002-01 Natural Grouping | X | | | Included in IS-04 Node API suite |
 | - | BCP-002-02 Asset Distinguishing Information | X | | | Included in IS-04 Node API suite |
 | BCP-003-01 | BCP-003-01 Secure Communication | X | X | | See [Testing TLS](docs/2.2.%20Usage%20-%20Testing%20BCP-003-01%20TLS.md) |
@@ -43,6 +43,8 @@ The following test suites are currently supported.
 | BCP-006-01-01 | BCP-006-01 NMOS With JPEG XS | X | | | |
 | BCP-006-01-02 | BCP-006-01 Controller | | | X | See [Testing Controllers](docs/2.8.%20Usage%20-%20Testing%20Controllers.md) |
 | BCP-006-04 | BCP-006-04 NMOS With MPEG-TS | X | | | |
+| BCP-008-01-01 | BCP-008-01 Receiver Status Monitoring | X | | | |
+| BCP-008-02-01 | BCP-008-02 Sender Status Monitoring | X | | | |
 
 When testing any of the above APIs it is important that they contain representative data. The test results will generate 'Could Not Test' results if no testable entities can be located. In addition, if devices support many modes of operation (including multiple video/audio formats) it is strongly recommended to re-test them in multiple modes.
 

docs/2.10 Usage - Invasive Device Model Testing.md

@@ -5,7 +5,7 @@
 Invasive tests are those that attempt to change the values of properties, and execute methods on the Device Model of a Node under Test (NuT).
 Non-invasive tests are limited to changing only the value of User Labels in the NuT's Device Model.
 
-By default, the `IS-12-01` test suite will __NOT__ execute invasive tests; these can be enabled by setting `MS05_INVASIVE_TESTING` to `True` in your `UserConfig.py` file as described in [Installation](1.0.%20Installation.md).
+By default, the `IS-12-01` and `IS-14-01` test suites will __NOT__ execute invasive tests; these can be enabled by setting `MS05_INVASIVE_TESTING` to `True` in your `UserConfig.py` file as described in [Installation](1.0.%20Installation.md).
 
 ## Running Interactive Invasive Tests
 
@@ -21,4 +21,4 @@ python3 nmos-testing-facade.py
 
 By default the Testing Façade will run on localhost port 5001. This can be altered by adding `TESTING_FACADE_PORT` to your `UserConfig.py` file.
 
-In the NMOS Testing Tool, when running the `IS-12-01` test suite, provide the IP/hostname of the Testing Façade (e.g. localhost) and the port of the Testing Façade (use 5001 by default).
+In the NMOS Testing Tool, when running the `IS-12-01` or `IS-14-01` test suites, provide the IP/hostname of the Testing Façade (e.g. localhost) and the port of the Testing Façade (use 5001 by default).

nmostesting/Config.py

@@ -386,6 +386,26 @@
             }
         }
     },
+    "bcp-008-01": {
+        "repo": "bcp-008-01",
+        "versions": ["v1.0"],
+        "default_version": "v1.0",
+        "apis": {
+            "receivermonitor": {
+                "name": "Receiver Monitor"
+            }
+        }
+    },
+    "bcp-008-02": {
+        "repo": "bcp-008-02",
+        "versions": ["v1.0"],
+        "default_version": "v1.0",
+        "apis": {
+            "sendermonitor": {
+                "name": "Sender Monitor"
+            }
+        }
+    },
     "bcp-005-01": {
         "repo": "bcp-005-01",
         "versions": ["v1.0"],

nmostesting/IS12Utils.py

@@ -58,10 +58,11 @@ def __str__(self):
 
 
 class IS12Notification():
-    def __init__(self, notification_json):
+    def __init__(self, notification_json, received_time):
         self.oid = notification_json["oid"]
         self.eventId = NcEventId(notification_json["eventId"])
         self.eventData = NcPropertyChangedEventData(notification_json["eventData"])
+        self.received_time = received_time
 
 
 class IS12Utils(MS05Utils):
@@ -167,8 +168,8 @@ def send_command(self, test, command_json):
                 continue
 
             # find the response to our request
-            for message in self.ncp_websocket.get_messages():
-                parsed_message = json.loads(message)
+            for tm in self.ncp_websocket.get_timestamped_messages():
+                parsed_message = json.loads(tm.message)
 
                 if self.message_type_to_schema_name(parsed_message.get("messageType")):
                     self._validate_is12_schema(
@@ -190,7 +191,8 @@ def send_command(self, test, command_json):
                 if parsed_message["messageType"] == MessageTypes.SubscriptionResponse:
                     results.append(parsed_message["subscriptions"])
                 if parsed_message["messageType"] == MessageTypes.Notification:
-                    self.notifications += [IS12Notification(n) for n in parsed_message["notifications"]]
+                    self.notifications += [IS12Notification(n, tm.received_time)
+                                           for n in parsed_message["notifications"]]
                 if parsed_message["messageType"] == MessageTypes.Error:
                     raise NMOSTestException(test.FAIL(  # Append the IS12Error so it can be used in negative tests
                         f"IS-I2 Error: {str(parsed_message)} for command: {str(command_json)}",
@@ -225,8 +227,17 @@ def send_command(self, test, command_json):
         return results[0]
 
     def get_notifications(self):
+        # Get any timestamped messages that have arrived in the interim period
+        for tm in self.ncp_websocket.get_timestamped_messages():
+            parsed_message = json.loads(tm.message)
+            if parsed_message["messageType"] == MessageTypes.Notification:
+                self.notifications += [IS12Notification(n, tm.received_time)
+                                       for n in parsed_message["notifications"]]
         return self.notifications
 
+    def reset_notifications(self):
+        self.notifications = []
+
     def start_logging_notifications(self, oid, property):
         self.expect_notifications = True
         self.expect_notifications_oid = oid

nmostesting/MS05Utils.py

@@ -171,6 +171,10 @@ class NcDeviceManagerProperties(Enum):
     NCVERSION = NcPropertyId({"level": 3, "index": 1})
 
 
+class NcWorkerProperties(Enum):
+    ENABLED = NcPropertyId({"level": 2, "index": 1})
+
+
 class NcObjectMethods(Enum):
     GENERIC_GET = NcMethodId({"level": 1, "index": 1})
     GENERIC_SET = NcMethodId({"level": 1, "index": 2})
@@ -515,6 +519,9 @@ def __init__(self, class_id: List[int], oid: int, owner: Optional[int], role: st
         self.runtime_constraints = runtime_constraints
         self.member_descriptor = member_descriptor
 
+    def __str__(self):
+        return f"[oid={self.oid}, role_path={self.role_path}]"
+
 
 class NcBlock(NcObject):
     def __init__(self, class_id: List[int], oid: int, owner: Optional[int], role: str, role_path: List[str],

nmostesting/NMOSTesting.py

@@ -90,6 +90,8 @@
 from .suites import BCP0060101Test
 from .suites import BCP0060102Test
 from .suites import BCP00604Test
+from .suites import BCP0080101Test
+from .suites import BCP0080201Test
 
 
 FLASK_APPS = []
@@ -457,6 +459,66 @@
         }],
         "class": BCP0060102Test.BCP0060102Test
     },
+    "BCP-008-01": {
+        "name": "BCP-008-01 Receiver Status Monitoring",
+        "specs": [{
+            "spec_key": "is-04",
+            "api_key": "node",
+            "disable_fields": ["urlpath"]
+        }, {
+            "spec_key": "is-05",
+            "api_key": "connection",
+            "disable_fields": ["urlpath"]
+        }, {
+            "spec_key": "is-12",
+            "api_key": "ncp",
+            "websocket": True,
+        }, {
+            "spec_key": "ms-05-02",
+            "api_key": "controlframework",
+            "disable_fields": ["host", "port", "urlpath"]
+        }],
+        "extra_specs": [{
+            "spec_key": "nmos-control-feature-sets",
+            "api_key": "featuresets"
+        }, {
+            "spec_key": "bcp-008-01",
+            "api_key": "receivermonitor",
+            "disable_fields": ["host", "port", "urlpath"]
+        }],
+        "class": BCP0080101Test.BCP0080101Test,
+        "urlpath": True
+    },
+    "BCP-008-02": {
+        "name": "BCP-008-02 Sender Status Monitoring",
+        "specs": [{
+            "spec_key": "is-04",
+            "api_key": "node",
+            "disable_fields": ["urlpath"]
+        }, {
+            "spec_key": "is-05",
+            "api_key": "connection",
+            "disable_fields": ["urlpath"]
+        }, {
+            "spec_key": "is-12",
+            "api_key": "ncp",
+            "websocket": True,
+        }, {
+            "spec_key": "ms-05-02",
+            "api_key": "controlframework",
+            "disable_fields": ["host", "port", "urlpath"]
+        }],
+        "extra_specs": [{
+            "spec_key": "nmos-control-feature-sets",
+            "api_key": "featuresets"
+        }, {
+            "spec_key": "bcp-008-02",
+            "api_key": "sendermonitor",
+            "disable_fields": ["host", "port", "urlpath"]
+        }],
+        "class": BCP0080201Test.BCP0080201Test,
+        "urlpath": True
+    },
     "BCP-006-04": {
         "name": "BCP-006-04 NMOS With MPEG TS",
         "specs": [{
@@ -471,7 +533,7 @@
             "api_key": "sender-register"
         }],
         "class": BCP00604Test.BCP00604Test
-    },
+    }
 }
 
 

nmostesting/TestHelper.py

@@ -25,12 +25,14 @@
 import jsonref
 import netifaces
 import paho.mqtt.client as mqtt
-from copy import copy
+from copy import deepcopy
 from pathlib import Path
 from enum import IntEnum
 from numbers import Number
 from functools import cmp_to_key
 from collections.abc import KeysView
+from threading import Lock
+from time import time
 from urllib.parse import urlparse
 
 from . import Config as CONFIG
@@ -287,6 +289,11 @@ def check_content_type(headers, expected_type="application/json"):
 class WebsocketWorker(threading.Thread):
     """Websocket Client Worker Thread"""
 
+    class Message():
+        def __init__(self, message):
+            self.received_time = time()
+            self.message = message
+
     def __init__(self, ws_href):
         """
         Initializer
@@ -313,6 +320,7 @@ def __init__(self, ws_href):
         self.error_occurred = False
         self.connected = False
         self.error_message = ""
+        self.mutex = Lock()
 
     def run(self):
         url = urlparse(self.ws.url)
@@ -326,7 +334,8 @@ def on_open(self, ws):
         self.connected = True
 
     def on_message(self, ws, message):
-        self.messages.append(message)
+        with self.mutex:
+            self.messages.append(WebsocketWorker.Message(message))
 
     def on_close(self, ws, close_status, close_message):
         self.connected = False
@@ -347,8 +356,15 @@ def is_open(self):
         return self.connected
 
     def get_messages(self):
-        msg_cpy = copy(self.messages)
-        self.clear_messages()  # Reset message list after reading
+        with self.mutex:
+            msg_cpy = [m.message for m in self.messages]
+            self.messages.clear()  # Reset message list after reading
+        return msg_cpy
+
+    def get_timestamped_messages(self):
+        with self.mutex:
+            msg_cpy = deepcopy(self.messages)
+            self.messages.clear()  # Reset message list after reading
         return msg_cpy
 
     def did_error_occur(self):
@@ -361,7 +377,8 @@ def is_messages_received(self):
         return len(self.messages) > 0
 
     def clear_messages(self):
-        self.messages.clear()
+        with self.mutex:
+            self.messages.clear()
 
 
 class MQTTClientWorker: