MINIFICPP-2711 Docker tests, missing logs if container fails to start

behave_framework/src/minifi_test_framework/containers/container.py

@@ -14,6 +14,9 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
 import json
 import logging
 import os
@@ -28,6 +31,9 @@
 from minifi_test_framework.containers.file import File
 from minifi_test_framework.containers.host_file import HostFile
 
+if TYPE_CHECKING:
+    from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
 
 class Container:
     def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None):
@@ -74,7 +80,7 @@ def _configure_volumes_of_container_dirs(self):
                 self._write_content_to_file(file_path, None, content)
             self.volumes[temp_path] = {"bind": directory.path, "mode": directory.mode}
 
-    def deploy(self) -> bool:
+    def deploy(self, context: MinifiTestContext | None) -> bool:
         self._temp_dir = tempfile.TemporaryDirectory()
         self._configure_volumes_of_container_files()
         self._configure_volumes_of_container_dirs()

behave_framework/src/minifi_test_framework/containers/http_proxy_container.py

@@ -46,14 +46,14 @@ def __init__(self, test_context: MinifiTestContext):
 
         super().__init__("minifi-http-proxy:latest", f"http-proxy-{test_context.scenario_id}", test_context.network)
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "Accepting HTTP Socket connections at"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=5,
             bail_condition=lambda: self.exited,
-            context=None
+            context=context
         )
 
     def check_http_proxy_access(self, url):

behave_framework/src/minifi_test_framework/containers/minifi_container.py

@@ -59,7 +59,7 @@ def __init__(self, container_name: str, test_context: MinifiTestContext):
         self._fill_default_properties()
         self._fill_default_log_properties()
 
-    def deploy(self) -> bool:
+    def deploy(self, context: MinifiTestContext | None) -> bool:
         flow_config = self.flow_definition.to_yaml()
         logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}")
         if self.is_fhs:
@@ -77,15 +77,15 @@ def deploy(self) -> bool:
         resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller"
         self.host_files.append(HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml")))
 
-        if not super().deploy():
+        if not super().deploy(context):
             return False
 
         finished_str = "MiNiFi started"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=15,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)
 
     def set_property(self, key: str, value: str):
         self.properties[key] = value

behave_framework/src/minifi_test_framework/containers/nifi_container.py

@@ -70,7 +70,7 @@ def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]]
 
         super().__init__("apache/nifi:" + self.NIFI_VERSION, name, test_context.network, entrypoint=command)
 
-    def deploy(self):
+    def deploy(self, context: MinifiTestContext | None) -> bool:
         flow_config = self.flow_definition.to_json()
         buffer = io.BytesIO()
 
@@ -80,10 +80,10 @@ def deploy(self):
         gzipped_bytes = buffer.getvalue()
         self.files.append(File("/tmp/nifi_config/flow.json.gz", gzipped_bytes))
 
-        super().deploy()
+        super().deploy(context)
         finished_str = "Started Application in"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=300,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)

behave_framework/src/minifi_test_framework/steps/core_steps.py

@@ -36,13 +36,13 @@
 @when("all instances start up")
 def step_impl(context: MinifiTestContext):
     for container in context.containers.values():
-        assert container.deploy() or container.log_app_output()
+        assert container.deploy(context)
     logging.debug("All instances started up")
 
 
 @when("the MiNiFi instance starts up")
 def step_impl(context: MinifiTestContext):
-    assert context.get_or_create_default_minifi_container().deploy()
+    assert context.get_or_create_default_minifi_container().deploy(context)
     logging.debug("MiNiFi instance started up")
 
 

extensions/aws/tests/features/steps/kinesis_server_container.py

@@ -34,14 +34,14 @@ def __init__(self, test_context: MinifiTestContext):
         self.environment.append("INITIALIZE_STREAMS=test_stream:3")
         self.environment.append("LOG_LEVEL=DEBUG")
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "Starting Kinesis Plain Mock Service on port 4568"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=300,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)
 
     @retry_check()
     def check_kinesis_server_record_data(self, record_data):

extensions/aws/tests/features/steps/s3_server_container.py

@@ -26,14 +26,14 @@ def __init__(self, test_context: MinifiTestContext):
         super().__init__("adobe/s3mock:3.12.0", f"s3-server-{test_context.scenario_id}", test_context.network)
         self.environment.append("initialBuckets=test_bucket")
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "Started S3MockApplication"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=15,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)
 
     def check_s3_server_object_data(self, test_data):
         (code, output) = self.exec_run(["find", "/s3mockroot/test_bucket", "-mindepth", "1", "-maxdepth", "1", "-type", "d"])

extensions/azure/tests/features/steps/azure_server_container.py

@@ -33,13 +33,13 @@ def __init__(self, test_context: MinifiTestContext):
         self.azure_connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" \
                                        f"BlobEndpoint=http://{azure_storage_hostname}:10000/devstoreaccount1;QueueEndpoint=http://{azure_storage_hostname}:10001/devstoreaccount1;"
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "Azurite Queue service is successfully listening at"
         return wait_for_condition(condition=lambda: finished_str in self.get_logs(),
                                   timeout_seconds=15,
                                   bail_condition=lambda: self.exited,
-                                  context=None)
+                                  context=context)
 
     def check_azure_storage_server_data(self, test_data):
         (code, output) = self.exec_run(["find", "/data/__blobstorage__", "-type", "f"])

extensions/azure/tests/features/steps/steps.py

@@ -43,7 +43,7 @@ def step_impl(context: MinifiTestContext, processor_name: str):
 @step("an Azure storage server is set up")
 def step_impl(context):
     context.containers["azure-storage-server"] = AzureServerContainer(context)
-    assert context.containers["azure-storage-server"].deploy()
+    assert context.containers["azure-storage-server"].deploy(context)
 
 
 @then('the object on the Azure storage server is "{object_data}"')

extensions/couchbase/tests/features/steps/couchbase_server_container.py

@@ -37,14 +37,14 @@ def __init__(self, test_context: MinifiTestContext):
         couchbase_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
         self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666))
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "logs available in"
         assert wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=15,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)
         return self.run_post_startup_commands()
 
     def run_post_startup_commands(self):

extensions/couchbase/tests/features/steps/steps.py

@@ -28,7 +28,7 @@
 @step("a Couchbase server is started")
 def step_impl(context: MinifiTestContext):
     context.containers["couchbase-server"] = CouchbaseServerContainer(context)
-    assert context.containers["couchbase-server"].deploy()
+    assert context.containers["couchbase-server"].deploy(context)
 
 
 @step("a CouchbaseClusterService controller service is set up to communicate with the Couchbase server")

extensions/elasticsearch/tests/features/steps/elastic_base_container.py

@@ -23,13 +23,13 @@ def __init__(self, test_context: MinifiTestContext, image: str, container_name:
         super().__init__(image, container_name, test_context.network)
         self.user = None
 
-    def deploy(self, finished_str: str):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None, finished_str: str):
+        super().deploy(context)
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=300,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)
 
     def create_doc_elasticsearch(self, index_name: str, doc_id: str) -> bool:
         (code, output) = self.exec_run(["/bin/bash", "-c",

extensions/elasticsearch/tests/features/steps/elasticsearch_container.py

@@ -53,8 +53,8 @@ def __init__(self, test_context: MinifiTestContext):
 
         self.environment.append("ELASTIC_PASSWORD=password")
 
-    def deploy(self):
-        return super().deploy('"current.health":"GREEN"')
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        return super().deploy(context, '"current.health":"GREEN"')
 
     def elastic_generate_apikey(self):
         api_url = "https://localhost:9200/_security/api_key"

extensions/elasticsearch/tests/features/steps/opensearch_container.py

@@ -43,8 +43,8 @@ def __init__(self, test_context: MinifiTestContext):
         features_dir = Path(__file__).resolve().parent.parent
         self.host_files.append(HostFile('/usr/share/opensearch/config/opensearch.yml', os.path.join(features_dir, "resources", "opensearch.yml")))
 
-    def deploy(self):
-        return super().deploy('Hot-reloading of audit configuration is enabled')
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        return super().deploy(context, 'Hot-reloading of audit configuration is enabled')
 
     def add_elastic_user_to_opensearch(self):
         curl_cmd = [

extensions/elasticsearch/tests/features/steps/steps.py

@@ -30,7 +30,7 @@
 @step('an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"')
 def step_impl(context: MinifiTestContext):
     context.containers["elasticsearch"] = ElasticsearchContainer(context)
-    assert context.containers["elasticsearch"].deploy()
+    assert context.containers["elasticsearch"].deploy(context)
     assert context.containers["elasticsearch"].create_doc_elasticsearch("my_index", "preloaded_id") or context.containers["elasticsearch"].log_app_output()
 
 
@@ -65,7 +65,7 @@ def step_impl(context):
 @given('an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"')
 def step_impl(context):
     context.containers["opensearch"] = OpensearchContainer(context)
-    context.containers["opensearch"].deploy()
+    context.containers["opensearch"].deploy(context)
     context.containers["opensearch"].add_elastic_user_to_opensearch()
     context.containers["opensearch"].create_doc_elasticsearch("my_index", "preloaded_id")
 

extensions/gcp/tests/features/steps/fake_gcs_server_container.py

@@ -26,14 +26,14 @@ def __init__(self, test_context: MinifiTestContext):
                          command=f'-scheme http -host fake-gcs-server-{test_context.scenario_id}')
         self.dirs.append(Directory(path="/data/test-bucket", files={"test-file": "preloaded data\n"}))
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "server started at http"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=30,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)
 
     @retry_check()
     def check_google_cloud_storage(self, content):

extensions/gcp/tests/features/steps/steps.py

@@ -28,7 +28,7 @@
 @step('a Google Cloud storage server is set up and a single object with contents "preloaded data" is present')
 def step_impl(context: MinifiTestContext):
     context.containers["fake-gcs-server"] = FakeGcsServerContainer(context)
-    assert context.containers["fake-gcs-server"].deploy()
+    assert context.containers["fake-gcs-server"].deploy(context)
 
 
 @then('an object with the content \"{content}\" is present in the Google Cloud storage')

extensions/grafana-loki/tests/features/steps/grafana_loki_container.py

@@ -126,14 +126,14 @@ def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions)
 
         self.files.append(File("/etc/loki/local-config.yaml", grafana_loki_yml_content.encode(), permissions=0o644))
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "Loki started"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=120,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)
 
     @retry_check()
     def are_lines_present(self, lines: str, timeout: int, ssl: bool, tenant_id: str = "") -> bool:

extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py

@@ -28,11 +28,11 @@ def __init__(self, test_context: MinifiTestContext):
             "FORWARD_PORT=3100",
         ]
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "start worker process"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=60,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)

extensions/kafka/tests/features/steps/kafka_server_container.py

@@ -88,14 +88,14 @@ def __init__(self, test_context: MinifiTestContext):
 """
         self.files.append(File("/opt/kafka/config/kafka_jaas.conf", jaas_config_file_content, permissions=0o644))
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "Kafka Server started"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=60,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)
 
     def create_topic(self, topic_name: str):
         (code, output) = self.exec_run(["/bin/sh", "-c", f"/opt/kafka/bin/kafka-topics.sh --create --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092"])

extensions/kafka/tests/features/steps/steps.py

@@ -66,7 +66,7 @@ def step_impl(context):
 def step_impl(context: MinifiTestContext):
     kafka_server_container = context.containers["kafka-server"]
     assert isinstance(kafka_server_container, KafkaServer)
-    assert kafka_server_container.deploy()
+    assert kafka_server_container.deploy(context)
 
 
 @step('the topic "{topic_name}" is initialized on the kafka broker')

extensions/opc/tests/features/steps/opc_ua_server_container.py

@@ -23,11 +23,11 @@ class OPCUAServerContainer(Container):
     def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None):
         super().__init__("lordgamez/open62541:1.4.10", f"opcua-server-{test_context.scenario_id}", test_context.network, command=command)
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "New DiscoveryUrl added: opc.tcp://"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=15,
             bail_condition=lambda: self.exited,
-            context=None)
+            context=context)

extensions/splunk/tests/features/steps/splunk_container.py

@@ -49,14 +49,14 @@ def __init__(self, test_context: MinifiTestContext):
         self.files.append(File("/opt/splunk/etc/auth/splunk_cert.pem", splunk_cert_content.decode() + splunk_key_content.decode() + root_ca_content.decode(), permissions=0o644))
         self.files.append(File("/opt/splunk/etc/auth/root_ca.pem", root_ca_content.decode(), permissions=0o644))
 
-    def deploy(self):
-        super().deploy()
+    def deploy(self, context: MinifiTestContext | None) -> bool:
+        super().deploy(context)
         finished_str = "Ansible playbook complete, will begin streaming splunkd_stderr.log"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
             timeout_seconds=300,
             bail_condition=lambda: False,
-            context=None)
+            context=context)
 
     @retry_check()
     def check_splunk_event(self, query: str) -> bool:

extensions/splunk/tests/features/steps/steps.py

@@ -26,7 +26,7 @@
 @step("a Splunk HEC is set up and running")
 def step_impl(context: MinifiTestContext):
     context.containers["splunk"] = SplunkContainer(context)
-    assert context.containers["splunk"].deploy()
+    assert context.containers["splunk"].deploy(context)
     assert context.containers["splunk"].enable_splunk_hec_indexer('splunk_hec_token') or context.containers["splunk"].log_app_output()