MINIFICPP-2716 Refactor and stabilize modular docker tests

behave_framework/src/minifi_test_framework/containers/http_proxy_container.py

@@ -51,7 +51,7 @@ def deploy(self):
         finished_str = "Accepting HTTP Socket connections at"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
-            timeout_seconds=5,
+            timeout_seconds=30,
             bail_condition=lambda: self.exited,
             context=None
         )

behave_framework/src/minifi_test_framework/steps/core_steps.py

@@ -51,9 +51,41 @@ def step_impl(context: MinifiTestContext):
 def step_impl(context: MinifiTestContext, directory: str, size: str):
     size = humanfriendly.parse_size(size)
     content = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size))
+    dirs = context.get_or_create_default_minifi_container().dirs
+    if directory in dirs:
+        dirs[directory].files[str(uuid.uuid4())] = content
+        return
     new_dir = Directory(directory)
-    new_dir.files["input.txt"] = content
-    context.get_or_create_default_minifi_container().dirs.append(new_dir)
+    new_dir.files[str(uuid.uuid4())] = content
+    dirs.append(new_dir)
+
+
+def __add_directory_with_file_to_container(context: MinifiTestContext, directory: str, file_name: str, content: str, container_name: str):
+    dirs = context.get_or_create_minifi_container(container_name).dirs
+    new_content = content.replace("\\n", "\n")
+    if directory in dirs:
+        dirs[directory].files[file_name] = new_content
+        return
+    new_dir = Directory(directory)
+    new_dir.files[file_name] = new_content
+    dirs.append(new_dir)
+
+
+@step('a directory at "{directory}" has a file with the content "{content}" in the "{flow_name}" flow')
+@step("a directory at '{directory}' has a file with the content '{content}' in the '{flow_name}' flow")
+def step_impl(context: MinifiTestContext, directory: str, content: str, flow_name: str):
+    __add_directory_with_file_to_container(context, directory, str(uuid.uuid4()), content, flow_name)
+
+
+@step('a directory at "{directory}" has a file with the content "{content}"')
+@step("a directory at '{directory}' has a file with the content '{content}'")
+def step_impl(context: MinifiTestContext, directory: str, content: str):
+    context.execute_steps(f'given a directory at "{directory}" has a file with the content "{content}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow')
+
+
+@step('a directory at "{directory}" has a file "{file_name}" with the content "{content}"')
+def step_impl(context: MinifiTestContext, directory: str, file_name: str, content: str):
+    __add_directory_with_file_to_container(context, directory, file_name, content, DEFAULT_MINIFI_CONTAINER_NAME)
 
 
 @step('a file with filename "{file_name}" and content "{content}" is present in "{path}"')

behave_framework/src/minifi_test_framework/steps/flow_building_steps.py

@@ -19,7 +19,6 @@
 import uuid
 from behave import given, step
 
-from minifi_test_framework.containers.directory import Directory
 from minifi_test_framework.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext
 from minifi_test_framework.minifi.connection import Connection
 from minifi_test_framework.minifi.controller_service import ControllerService
@@ -227,29 +226,6 @@ def step_impl(context: MinifiTestContext, parameter_name: str, parameter_value:
     parameter_context.parameters.append(Parameter(parameter_name, parameter_value, False))
 
 
-@step('a directory at "{directory}" has a file with the content "{content}" in the "{flow_name}" flow')
-@step("a directory at '{directory}' has a file with the content '{content}' in the '{flow_name}' flow")
-def step_impl(context: MinifiTestContext, directory: str, content: str, flow_name: str):
-    new_content = content.replace("\\n", "\n")
-    new_dir = Directory(directory)
-    new_dir.files["input.txt"] = new_content
-    context.get_or_create_minifi_container(flow_name).dirs.append(new_dir)
-
-
-@step('a directory at "{directory}" has a file with the content "{content}"')
-@step("a directory at '{directory}' has a file with the content '{content}'")
-def step_impl(context: MinifiTestContext, directory: str, content: str):
-    context.execute_steps(f'given a directory at "{directory}" has a file with the content "{content}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow')
-
-
-@step('a directory at "{directory}" has a file ("{file_name}") with the content "{content}"')
-def step_impl(context: MinifiTestContext, directory: str, file_name: str, content: str):
-    new_content = content.replace("\\n", "\n")
-    new_dir = Directory(directory)
-    new_dir.files[file_name] = new_content
-    context.get_or_create_default_minifi_container().dirs.append(new_dir)
-
-
 @given("these processor properties are set in the \"{minifi_container_name}\" flow")
 def step_impl(context: MinifiTestContext, minifi_container_name: str):
     for row in context.table:

docker/RunBehaveTests.sh

@@ -192,22 +192,6 @@ fi
 
 echo "${BEHAVE_OPTS[@]}"
 
-exec \
-  behavex "${BEHAVE_OPTS[@]}" \
-    "${docker_dir}/../extensions/standard-processors/tests/features" \
-    "${docker_dir}/../extensions/aws/tests/features" \
-    "${docker_dir}/../extensions/azure/tests/features" \
-    "${docker_dir}/../extensions/sql/tests/features" \
-    "${docker_dir}/../extensions/llamacpp/tests/features" \
-    "${docker_dir}/../extensions/opc/tests/features" \
-    "${docker_dir}/../extensions/kafka/tests/features" \
-    "${docker_dir}/../extensions/couchbase/tests/features" \
-    "${docker_dir}/../extensions/elasticsearch/tests/features" \
-    "${docker_dir}/../extensions/splunk/tests/features" \
-    "${docker_dir}/../extensions/gcp/tests/features" \
-    "${docker_dir}/../extensions/grafana-loki/tests/features" \
-    "${docker_dir}/../extensions/lua/tests/features/" \
-    "${docker_dir}/../extensions/civetweb/tests/features/" \
-    "${docker_dir}/../extensions/mqtt/tests/features/" \
-    "${docker_dir}/../extensions/prometheus/tests/features/" \
-    "${docker_dir}/../extensions/python/tests/features/"
+mapfile -t FEATURE_FILES < <(find "${docker_dir}/../extensions" -type f -name '*.feature')
+
+behavex "${BEHAVE_OPTS[@]}" "${FEATURE_FILES[@]}"

extensions/aws/tests/features/steps/s3_server_container.py → extensions/aws/tests/features/containers/s3_server_container.py

@@ -31,7 +31,7 @@ def deploy(self):
         finished_str = "Started S3MockApplication"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
-            timeout_seconds=15,
+            timeout_seconds=60,
             bail_condition=lambda: self.exited,
             context=None)
 

extensions/aws/tests/features/kinesis.feature

@@ -22,7 +22,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS Kinesis server
   Scenario: A MiNiFi instance can send data to AWS Kinesis
     Given a kinesis server is set up in correspondence with the PutKinesisStream
     And a GetFile processor with the "Input Directory" property set to "/tmp/input"
-    And a file with the content "Schnappi, das kleine Krokodil" is present in "/tmp/input"
+    And a directory at "/tmp/input" has a file with the content "Schnappi, das kleine Krokodil"
     And a PutKinesisStream processor
     And these processor properties are set
       | processor name     | property name              | property value                           |

extensions/aws/tests/features/s3.feature

@@ -29,9 +29,9 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
     And the "failure" relationship of the PutS3Object processor is connected to the PutS3Object
     And PutFile's success relationship is auto-terminated
 
-    And an s3 server is set up in correspondence with the PutS3Object
+    And the s3 server starts up
 
-    When both instances start up
+    When the MiNiFi instance starts up
 
     Then a single file with the content "LH_O#L|FD<FASD{FO#@$#$%^ \"#\"$L%:\"@#$L\":test_data#$#%#$%?{\"F{" is placed in the "/tmp/output" directory in less than 20 seconds
     And the object on the s3 server is "LH_O#L|FD<FASD{FO#@$#$%^ \"#\"$L%:\"@#$L\":test_data#$#%#$%?{\"F{"
@@ -54,7 +54,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
     And the "failure" relationship of the PutS3Object processor is connected to the PutS3Object
     And PutFile's success relationship is auto-terminated
 
-    And a s3 server is set up in correspondence with the PutS3Object
+    And the s3 server starts up
     And the http proxy server is set up
     When all instances start up
 
@@ -77,9 +77,9 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
       | DeleteS3Object | success           | PutFile          |
       | PutFile        | success           | auto-terminated  |
 
-    And a s3 server is set up in correspondence with the PutS3Object
+    And the s3 server starts up
 
-    When both instances start up
+    When the MiNiFi instance starts up
 
     Then a single file with the content "LH_O#L|FD<FASD{FO#@$#$%^ \"#\"$L%:\"@#$L\":test_data#$#%#$%?{\"F{" is placed in the "/tmp/output" directory in less than 20 seconds
     And the object bucket on the s3 server is empty in less than 10 seconds
@@ -92,9 +92,9 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
     And the "success" relationship of the GetFile processor is connected to the DeleteS3Object
     And the "success" relationship of the DeleteS3Object processor is connected to the PutFile
 
-    And a s3 server is set up in correspondence with the DeleteS3Object
+    And the s3 server starts up
 
-    When both instances start up
+    When the MiNiFi instance starts up
 
     Then a single file with the content "test" is placed in the "/tmp/output" directory in less than 20 seconds
     And the object bucket on the s3 server is empty in less than 10 seconds
@@ -119,7 +119,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
       | DeleteS3Object | success           | PutFile          |
       | PutFile        | success           | auto-terminated  |
 
-    And a s3 server is set up in correspondence with the PutS3Object
+    And the s3 server starts up
     And the http proxy server is set up
 
     When all instances start up
@@ -145,9 +145,9 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
       | PutS3Object      | success           | auto-terminated  |
       | PutS3Object      | failure           | PutS3Object      |
 
-    And a s3 server is set up in correspondence with the PutS3Object
+    And the s3 server starts up
 
-    When all instances start up
+    When the MiNiFi instance starts up
 
     Then a single file with the content "test" is placed in the "/tmp/output" directory in less than 20 seconds
 
@@ -171,7 +171,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
       | GenerateFlowFile | success           | FetchS3Object    |
       | FetchS3Object    | success           | PutFile          |
 
-    And a s3 server is set up in correspondence with the PutS3Object
+    And the s3 server starts up
     And the http proxy server is set up
 
     When all instances start up
@@ -183,8 +183,8 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
     Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
     And the "Batch Size" property of the GetFile processor is set to "1"
     And the scheduling period of the GetFile processor is set to "5 sec"
-    And a file with filename "test_file_1.log" and content "test_data1" is present in "/tmp/input"
-    And a file with filename "test_file_2.log" and content "test_data2" is present in "/tmp/input"
+    And a directory at "/tmp/input" has a file "test_file_1.log" with the content "test_data1"
+    And a directory at "/tmp/input" has a file "test_file_2.log" with the content "test_data2"
     And a PutS3Object processor set up to communicate with an s3 server
     And the "Object Key" property of the PutS3Object processor is set to "${filename}"
     And the "success" relationship of the GetFile processor is connected to the PutS3Object
@@ -193,9 +193,9 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
     And a PutFile processor with the "Directory" property set to "/tmp/output"
     And the "success" relationship of the ListS3 processor is connected to the PutFile
 
-    And a s3 server is set up in correspondence with the PutS3Object
+    And the s3 server starts up
 
-    When all instances start up
+    When the MiNiFi instance starts up
 
     Then 2 files are placed in the "/tmp/output" directory in less than 20 seconds
 
@@ -215,7 +215,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
     And a PutFile processor with the "Directory" property set to "/tmp/output"
     And the "success" relationship of the ListS3 processor is connected to the PutFile
 
-    And a s3 server is set up in correspondence with the PutS3Object
+    And the s3 server starts up
     And the http proxy server is set up
 
     When all instances start up
@@ -232,9 +232,9 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
     And the "success" relationship of the GetFile processor is connected to the PutS3Object
     And a PutFile processor with the "Directory" property set to "/tmp/output"
     And the "success" relationship of the PutS3Object processor is connected to the PutFile
-    And a s3 server is set up in correspondence with the PutS3Object
+    And the s3 server starts up
 
-    When all instances start up
+    When the MiNiFi instance starts up
 
     Then 1 file is placed in the "/tmp/output" directory in less than 20 seconds
     And the object on the s3 server is present and matches the original hash
@@ -256,7 +256,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
     And a PutFile processor with the "Directory" property set to "/tmp/output"
     And the "success" relationship of the PutS3Object processor is connected to the PutFile
 
-    And a s3 server is set up in correspondence with the PutS3Object
+    And the s3 server starts up
     And the http proxy server is set up
     When all instances start up
 

extensions/aws/tests/features/steps/steps.py

@@ -29,8 +29,8 @@
 from minifi_test_framework.minifi.processor import Processor
 from minifi_test_framework.core.helpers import wait_for_condition, log_due_to_failure
 
-from s3_server_container import S3ServerContainer
-from kinesis_server_container import KinesisServerContainer
+from containers.s3_server_container import S3ServerContainer
+from containers.kinesis_server_container import KinesisServerContainer
 
 
 @step('a {processor_name} processor set up to communicate with an s3 server')
@@ -50,10 +50,10 @@ def step_impl(context: MinifiTestContext, processor_name: str):
     context.get_or_create_default_minifi_container().flow_definition.add_processor(processor)
 
 
-@step('a s3 server is set up in correspondence with the {processor_name}')
-@step('an s3 server is set up in correspondence with the {processor_name}')
-def step_impl(context: MinifiTestContext, processor_name: str):
+@step('the s3 server starts up')
+def step_impl(context: MinifiTestContext):
     context.containers["s3-server"] = S3ServerContainer(context)
+    assert context.containers["s3-server"].deploy()
 
 
 @step('the object on the s3 server is "{object_data}"')

extensions/azure/tests/features/steps/steps.py

@@ -25,7 +25,7 @@
 from minifi_test_framework.steps import core_steps  # noqa: F401
 from minifi_test_framework.steps import flow_building_steps  # noqa: F401
 
-from azure_server_container import AzureServerContainer
+from containers.azure_server_container import AzureServerContainer
 
 
 @step("a {processor_name} processor set up to communicate with an Azure blob storage")

extensions/civetweb/tests/features/http.feature

@@ -22,7 +22,7 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
   Scenario: A MiNiFi instance transfers data to another MiNiFi instance with message body
     Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
     And the "Keep Source File" property of the GetFile processor is set to "true"
-    And a file with the content "test" is present in "/tmp/input"
+    And a directory at "/tmp/input" has a file with the content "test"
     And a InvokeHTTP processor with the "Remote URL" property set to "http://secondary-${scenario_id}:8080/contentListener"
     And InvokeHTTP is EVENT_DRIVEN
     And the "HTTP Method" property of the InvokeHTTP processor is set to "POST"
@@ -43,7 +43,7 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
     Given the http proxy server is set up
     And a GetFile processor with the "Input Directory" property set to "/tmp/input"
     And the "Keep Source File" property of the GetFile processor is set to "true"
-    And a file with the content "test" is present in "/tmp/input"
+    And a directory at "/tmp/input" has a file with the content "test"
     And a InvokeHTTP processor with the "Remote URL" property set to "http://minifi-listen-${scenario_id}:8080/contentListener"
     And InvokeHTTP is EVENT_DRIVEN
     And these processor properties are set
@@ -70,7 +70,7 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
   Scenario: A MiNiFi instance and transfers hashed data to another MiNiFi instance
     Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
     And the "Keep Source File" property of the GetFile processor is set to "true"
-    And a file with the content "test" is present in "/tmp/input"
+    And a directory at "/tmp/input" has a file with the content "test"
     And a HashContent processor with the "Hash Attribute" property set to "hash"
     And HashContent is EVENT_DRIVEN
     And a InvokeHTTP processor with the "Remote URL" property set to "http://secondary-${scenario_id}:8080/contentListener"
@@ -93,7 +93,7 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
   Scenario: A MiNiFi instance transfers data to another MiNiFi instance without message body
     Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
     And the "Keep Source File" property of the GetFile processor is set to "true"
-    And a file with the content "test" is present in "/tmp/input"
+    And a directory at "/tmp/input" has a file with the content "test"
     And a InvokeHTTP processor with the "Remote URL" property set to "http://secondary-${scenario_id}:8080/contentListener"
     And InvokeHTTP is EVENT_DRIVEN
     And the "HTTP Method" property of the InvokeHTTP processor is set to "POST"
@@ -114,7 +114,7 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
   Scenario: A MiNiFi instance transfers data to a NiFi instance with message body
     Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
     And the "Keep Source File" property of the GetFile processor is set to "true"
-    And a file with the content "test" is present in "/tmp/input"
+    And a directory at "/tmp/input" has a file with the content "test"
     And a InvokeHTTP processor with the "Remote URL" property set to "http://nifi-${scenario_id}:8081/contentListener"
     And the "HTTP Method" property of the InvokeHTTP processor is set to "POST"
     And the "success" relationship of the GetFile processor is connected to the InvokeHTTP

extensions/couchbase/tests/features/steps/couchbase_server_container.py → extensions/couchbase/tests/features/containers/couchbase_server_container.py

@@ -42,7 +42,7 @@ def deploy(self):
         finished_str = "logs available in"
         assert wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
-            timeout_seconds=15,
+            timeout_seconds=30,
             bail_condition=lambda: self.exited,
             context=None)
         return self.run_post_startup_commands()

extensions/couchbase/tests/features/steps/steps.py

@@ -22,7 +22,7 @@
 from minifi_test_framework.core.minifi_test_context import MinifiTestContext
 from minifi_test_framework.core.helpers import log_due_to_failure
 from minifi_test_framework.minifi.controller_service import ControllerService
-from couchbase_server_container import CouchbaseServerContainer
+from containers.couchbase_server_container import CouchbaseServerContainer
 
 
 @step("a Couchbase server is started")