Exception: Failed to acquire started lock
self = <tests.unit.builtin_monitors.container_checker_test.ContainerCheckerMultiAccountTest testMethod=test_check_containers_with_api_key_annotations>

    def test_check_containers_with_api_key_annotations(self):
        WORKLOAD_POD_1_CONTAINER_3 = "workload-pod-1-container-3"
        WORKLOAD_POD_1_CONTAINER_2 = "workload-pod-1-container-2"
        WORKLOAD_POD_1_CONTAINER_1 = "workload-pod-1-container-1"
        WORKLOAD_POD_2_CONTAINER_1 = "workload-pod-2-container-1"
        WORKLOAD_POD_3_CONTAINER_1 = "workload-pod-3-container-1"
    
        k8s_namespace_1 = MockNamespace(
            name="namespace-1",
            annotations=process_annotations(
                {"log.config.scalyr.com/teams.1.secret": "scalyr-api-key-team-2"}
            ),
        )
    
        k8s_pod_1_namespace_1 = MockPod(
            namespace=k8s_namespace_1.name,
            name="k8s_pod_1_namespace_1",
            node_name="test_node",
            uid="lkjsefr",
            annotations=process_annotations(
                {
                    "log.config.scalyr.com/teams.1.secret": "scalyr-api-key-team-3",
                    "log.config.scalyr.com/teams.5.secret": "scalyr-api-key-team-4",
                    "log.config.scalyr.com/workload-pod-1-container-1.teams.1.secret": "scalyr-api-key-team-5",
                    "log.config.scalyr.com/workload-pod-1-container-2.teams.1.secret": "scalyr-api-key-team-6",
                    "log.config.scalyr.com/workload-pod-1-container-2.teams.2.secret": "scalyr-api-key-team-7",
                }
            ),
            container_names=[
                WORKLOAD_POD_1_CONTAINER_1,
                WORKLOAD_POD_1_CONTAINER_2,
                WORKLOAD_POD_1_CONTAINER_3,
            ],
            controller=Controller(
                name="controller_name_k8s_pod_1_namespace_1",
                kind="controller_kind_k8s_pod_1_namespace_1",
                labels={"controller_labels_k8s_pod_1_namespace_1": "value"},
            ),
        )
    
        k8s_pod_2_namespace_1 = MockPod(
            namespace=k8s_namespace_1.name,
            name="k8s_pod_2_namespace_1",
            node_name="test_node",
            uid="opimon",
            annotations=process_annotations({}),
            container_names=[
                WORKLOAD_POD_2_CONTAINER_1,
            ],
            controller=Controller(
                name="controller_name_k8s_pod_2_namespace_1",
                kind="controller_kind_k8s_pod_2_namespace_1",
                labels={"controller_labels_k8s_pod_2_namespace_1": "value"},
            ),
        )
    
        k8s_namespace_2 = MockNamespace(
            name="namespace-2", annotations=process_annotations({})
        )
        k8s_pod_3_namespace_2 = MockPod(
            namespace=k8s_namespace_2.name,
            name="k8s_pod_3_namespace_2",
            node_name="test_node",
            uid="fsdjlkh",
            annotations=process_annotations({}),
            container_names=[WORKLOAD_POD_3_CONTAINER_1],
            controller=Controller(
                name="controller_name_k8s_pod_3_namespace_2",
                kind="controller_kind_k8s_pod_3_namespace_2",
                labels={"controller_labels_k8s_pod_3_namespace_2": "value"},
            ),
        )
    
        get_containers_value = self._set_mocked_k8s_objects(
            [k8s_pod_1_namespace_1, k8s_pod_2_namespace_1, k8s_pod_3_namespace_2],
            [k8s_namespace_1, k8s_namespace_2],
        )
    
        run_state, exception_holder = self._start_check_containers_thread()
    
        # Wait for ContainerChecker to complete at least one loop
        run_state.wait_for_next_loop_start()
        run_state.wait_for_next_loop_start()
    
        assert len(exception_holder) == 0, "Expected no exceptions"
        assert len(self.mock_logger.exception.mock_calls) == 0, (
            "Some exceptions were logged: %s" % self.mock_logger.exception.mock_calls
        )
        assert len(self.mock_logger.warn.mock_calls) == 0, (
            "Some warnings were logged: %s" % self.mock_logger.warn.mock_calls
        )
        assert len(self.mock_logger.error.mock_calls) == 0, (
            "Some errors were logged: %s" % self.mock_logger.error.mock_calls
        )
    
        def expected_log_config(
            pod: MockPod, container_name: str, api_key: str, teams: List[str]
        ):
            log_config = {
                "parser": "docker",
                "path": f"/var/log/scalyr-agent2/containers/{container_name}.log",
                "parse_format": "test_parse_format",
                "rename_logfile": f"/unknown/{container_name}.log",
                "rename_no_original": None,
                "attributes": JsonObject(
                    {
                        "container_name": container_name,
                        "monitor": "agentKubernetes",
                        "container_id": container_name,
                        "pod_name": pod.name,
                        "pod_namespace": pod.namespace,
                        "_k8s_cn": "the-cluster-name",
                        "scalyr-category": "log",
                        "pod_uid": pod.uid,
                        "k8s_node": "test_node",
                        "_k8s_dn": pod.controller.name,
                        "_k8s_dl": self._flatten_labels(pod.controller.labels),
                        "_k8s_ck": pod.controller.kind,
                    }
                ),
                "k8s_pod_glob": "*",
                "k8s_namespace_glob": "*",
                "k8s_container_glob": "*",
                "lineGroupers": JsonArray(),
                "sampling_rules": JsonArray(),
                "redaction_rules": JsonArray(),
            }
            if teams:
                log_config["teams"] = JsonArray(
                    *[JsonObject({"secret": team}) for team in teams]
                )
            if api_key:
                log_config["api_key"] = api_key
    
            return log_config
    
        assert self.log_watcher.add_log_config.call_args_list == [
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_1_namespace_1,
                    WORKLOAD_POD_1_CONTAINER_1,
                    "scalyr-api-key-team-5-value",
                    ["scalyr-api-key-team-5"],
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_1_namespace_1,
                    WORKLOAD_POD_1_CONTAINER_2,
                    "scalyr-api-key-team-6-value",
                    ["scalyr-api-key-team-6", "scalyr-api-key-team-7"],
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_1_namespace_1,
                    WORKLOAD_POD_1_CONTAINER_2,
                    "scalyr-api-key-team-7-value",
                    ["scalyr-api-key-team-6", "scalyr-api-key-team-7"],
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_1_namespace_1,
                    WORKLOAD_POD_1_CONTAINER_3,
                    "scalyr-api-key-team-3-value",
                    ["scalyr-api-key-team-3", "scalyr-api-key-team-4"],
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_1_namespace_1,
                    WORKLOAD_POD_1_CONTAINER_3,
                    "scalyr-api-key-team-4-value",
                    ["scalyr-api-key-team-3", "scalyr-api-key-team-4"],
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_2_namespace_1,
                    WORKLOAD_POD_2_CONTAINER_1,
                    "scalyr-api-key-team-2-value",
                    None,
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_3_namespace_2, WORKLOAD_POD_3_CONTAINER_1, None, None
                ),
                force_add=True,
            ),
        ]
    
        self.log_watcher.add_log_config.reset_mock()
    
        # Wait for the loop to complete and pause it to change pod annotations.
        with run_state.running_lock:
>           run_state.wait_for_next_loop_start()

#x1B[1m#x1B[31mtests/unit/builtin_monitors/container_checker_test.py#x1B[0m:548: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tests.unit.builtin_monitors.container_checker_test.MockRunState object at 0x7feff6630bb0>

    def wait_for_next_loop_start(self):
        with self.next_loop_condition:
            if not self.next_loop_condition.wait(timeout=15):
>               raise Exception("Failed to acquire started lock")
#x1B[1m#x1B[31mE               Exception: Failed to acquire started lock#x1B[0m

#x1B[1m#x1B[31mtests/unit/builtin_monitors/container_checker_test.py#x1B[0m:40: Exception