MINIFICPP-2687 Move Python extension docker tests to modular docker tests

Author: lordgamez

behave_framework/src/minifi_test_framework/containers/minifi_container.py

@@ -85,7 +85,7 @@ def deploy(self) -> bool:
         finished_str = "MiNiFi started"
         return wait_for_condition(
             condition=lambda: finished_str in self.get_logs(),
-            timeout_seconds=15,
+            timeout_seconds=300,
             bail_condition=lambda: self.exited,
             context=None)
 
@@ -147,9 +147,11 @@ def _fill_default_properties(self):
         if self.is_fhs:
             self.properties["nifi.flow.configuration.file"] = "/etc/nifi-minifi-cpp/config.yml"
             self.properties["nifi.extension.path"] = "/usr/lib64/nifi-minifi-cpp/extensions/*"
+            self.properties["nifi.python.processor.dir"] = '/var/lib/nifi-minifi-cpp/minifi-python'
         else:
             self.properties["nifi.flow.configuration.file"] = "./conf/config.yml"
             self.properties["nifi.extension.path"] = "../extensions/*"
+            self.properties["nifi.python.processor.dir"] = '/opt/minifi/minifi-current/minifi-python'
         self.properties["nifi.administrative.yield.duration"] = "1 sec"
         self.properties["nifi.bored.yield.duration"] = "100 millis"
         self.properties["nifi.openssl.fips.support.enable"] = "false"
@@ -248,3 +250,9 @@ def create_debug_bundle(self) -> bool:
 
         (code, _) = self.exec_run(["test", "-f", "/tmp/debug.tar.gz"])
         return code == 0
+
+    def add_example_python_processors(self):
+        run_minifi_cmd = '/opt/minifi/minifi-current/bin/minifi.sh run' if not self.is_fhs else '/usr/bin/minifi'
+        minifi_python_dir_path = '/opt/minifi/minifi-current/minifi-python' if not self.is_fhs else '/var/lib/nifi-minifi-cpp/minifi-python'
+        minifi_python_examples = '/opt/minifi/minifi-current/minifi-python-examples' if not self.is_fhs else '/usr/share/doc/nifi-minifi-cpp/pythonprocessor-examples'
+        self.command = f'sh -c "cp -r {minifi_python_examples} {minifi_python_dir_path}/examples && {run_minifi_cmd}"'

behave_framework/src/minifi_test_framework/steps/checking_steps.py

@@ -44,6 +44,7 @@ def step_impl(context: MinifiTestContext, container_name: str, content: str, dir
 
 
 @then('a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}')
+@then("a single file with the content '{content}' is placed in the '{directory}' directory in less than {duration}")
 def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str):
     context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}')
 
@@ -57,6 +58,7 @@ def step_impl(context: MinifiTestContext, container_name: str, content: str, dir
 
 
 @then('at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}')
+@then("at least one file with the content '{content}' is placed in the '{directory}' directory in less than {duration}")
 def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str):
     context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}')
 
@@ -171,6 +173,16 @@ def step_impl(context: MinifiTestContext, directory: str, timeout: str, contents
                               context=context)
 
 
+@then('files with at least these contents "{contents}" are placed in the "{directory}" directory in less than {timeout}')
+def step_impl(context: MinifiTestContext, directory: str, timeout: str, contents: str):
+    timeout_seconds = humanfriendly.parse_timespan(timeout)
+    new_contents = contents.replace("\\n", "\n")
+    contents_arr = new_contents.split(",")
+    assert wait_for_condition(condition=lambda: all([context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_content(directory, content) for content in contents_arr]),
+                              timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited,
+                              context=context)
+
+
 @then("a file with the JSON content \"{content}\" is placed in the \"{directory}\" directory in less than {duration}")
 @then("a file with the JSON content '{content}' is placed in the '{directory}' directory in less than {duration}")
 def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str):

docker/RunBehaveTests.sh

@@ -209,4 +209,5 @@ exec \
     "${docker_dir}/../extensions/lua/tests/features/" \
     "${docker_dir}/../extensions/civetweb/tests/features/" \
     "${docker_dir}/../extensions/mqtt/tests/features/" \
-    "${docker_dir}/../extensions/prometheus/tests/features/"
+    "${docker_dir}/../extensions/prometheus/tests/features/" \
+    "${docker_dir}/../extensions/python/tests/features/"

docker/test/integration/cluster/ContainerStore.py

@@ -191,21 +191,6 @@ def set_ssl_context_properties_in_minifi(self):
     def enable_sql_in_minifi(self):
         self.minifi_options.enable_sql = True
 
-    def use_nifi_python_processors_with_system_python_packages_installed_in_minifi(self):
-        self.minifi_options.use_nifi_python_processors_with_system_python_packages_installed = True
-
-    def use_nifi_python_processors_with_virtualenv_in_minifi(self):
-        self.minifi_options.use_nifi_python_processors_with_virtualenv = True
-
-    def use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi(self):
-        self.minifi_options.use_nifi_python_processors_with_virtualenv_packages_installed = True
-
-    def remove_python_requirements_txt_in_minifi(self):
-        self.minifi_options.remove_python_requirements_txt = True
-
-    def use_nifi_python_processors_without_dependencies_in_minifi(self):
-        self.minifi_options.use_nifi_python_processors_without_dependencies = True
-
     def set_yaml_in_minifi(self):
         self.minifi_options.config_format = "yaml"
 
@@ -215,9 +200,6 @@ def set_json_in_minifi(self):
     def enable_log_metrics_publisher_in_minifi(self):
         self.minifi_options.enable_log_metrics_publisher = True
 
-    def enable_example_minifi_python_processors(self):
-        self.minifi_options.enable_example_minifi_python_processors = True
-
     def enable_openssl_fips_mode_in_minifi(self):
         self.minifi_options.enable_openssl_fips_mode = True
 

docker/test/integration/cluster/DockerTestCluster.py

@@ -86,21 +86,6 @@ def disable_openssl_fips_mode_in_minifi(self):
     def enable_sql_in_minifi(self):
         self.container_store.enable_sql_in_minifi()
 
-    def use_nifi_python_processors_with_system_python_packages_installed_in_minifi(self):
-        self.container_store.use_nifi_python_processors_with_system_python_packages_installed_in_minifi()
-
-    def use_nifi_python_processors_with_virtualenv_in_minifi(self):
-        self.container_store.use_nifi_python_processors_with_virtualenv_in_minifi()
-
-    def use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi(self):
-        self.container_store.use_nifi_python_processors_with_virtualenv_packages_installed_in_minifi()
-
-    def remove_python_requirements_txt_in_minifi(self):
-        self.container_store.remove_python_requirements_txt_in_minifi()
-
-    def use_nifi_python_processors_without_dependencies_in_minifi(self):
-        self.container_store.use_nifi_python_processors_without_dependencies_in_minifi()
-
     def set_yaml_in_minifi(self):
         self.container_store.set_yaml_in_minifi()
 
@@ -110,9 +95,6 @@ def set_json_in_minifi(self):
     def enable_log_metrics_publisher_in_minifi(self):
         self.container_store.enable_log_metrics_publisher_in_minifi()
 
-    def enable_example_minifi_python_processors(self):
-        self.container_store.enable_example_minifi_python_processors()
-
     def llama_model_is_downloaded_in_minifi(self):
         self.container_store.llama_model_is_downloaded_in_minifi()
 

docker/test/integration/cluster/DockerTestDirectoryBindings.py

@@ -54,7 +54,6 @@ def create_new_data_directories(self):
 
         # Add resources
         test_dir = os.environ['TEST_DIRECTORY']  # Based on DockerVerify.sh
-        shutil.copytree(test_dir + "/resources/python", self.data_directories[self.feature_id]["resources_dir"] + "/python")
         shutil.copytree(test_dir + "/resources/minifi", self.data_directories[self.feature_id]["minifi_config_dir"], dirs_exist_ok=True)
 
     def get_data_directories(self):