Fix resource processor porter install logs truncation (#4048)

Author: marrobi

CHANGELOG.md

@@ -6,6 +6,7 @@
 FEATURES:
 
 ENHANCEMENTS:
+* Split log entries with [Log chunk X of Y] for better readability. ([[#3992](https://github.com/microsoft/AzureTRE/issues/3992)
 * Expose APP_SERVICE_SKU build variable to allow enablement of App Gateway WAF ([#4111](https://github.com/microsoft/AzureTRE/pull/4111))
 * Update Terraform to use Azure AD authentication rather than storage account keys ([#4103](https://github.com/microsoft/AzureTRE/issues/4103))
 

resource_processor/_version.py

@@ -1 +1 @@
-__version__ = "0.9.2"
+__version__ = "0.9.3"

resource_processor/shared/logging.py

@@ -95,6 +95,17 @@ def initialize_logging() -> logging.Logger:
     return logger
 
 
+def chunk_log_output(output: str, chunk_size: int = 30000):
+    """
+    Split the log output into smaller chunks and prefix each chunk with [Log chunk X of Y].
+    """
+    total_chunks = (len(output) + chunk_size - 1) // chunk_size  # Calculate total number of chunks
+    for i in range(0, len(output), chunk_size):
+        current_chunk = i // chunk_size + 1
+        prefix = f"[Log chunk {current_chunk} of {total_chunks}] "
+        yield prefix + output[i:i + chunk_size]
+
+
 def shell_output_logger(console_output: str, prefix_item: str, logging_level: int):
     if not console_output:
         logger.debug("shell console output is empty.")
@@ -118,4 +129,5 @@ def shell_output_logger(console_output: str, prefix_item: str, logging_level: in
     console_output = console_output.strip()
 
     if console_output:
-        logger.log(logging_level, f"{prefix_item} {console_output}")
+        for chunk in chunk_log_output(console_output):
+            logger.log(logging_level, f"{prefix_item} {chunk}")

resource_processor/tests_rp/test_logging.py

@@ -1,6 +1,6 @@
 from mock import patch
 import logging
-from shared.logging import shell_output_logger
+from shared.logging import shell_output_logger, chunk_log_output
 
 
 @patch("shared.logging.logger")
@@ -29,3 +29,20 @@ def test_shell_output_logger_normal_case(mock_logger):
     console_output = "Some logs"
     shell_output_logger(console_output, "prefix", logging.DEBUG)
     mock_logger.log.assert_called_with(logging.DEBUG, "prefix Some logs")
+
+
+@patch("shared.logging.logger")
+def test_shell_output_logger_chunked_logging(mock_logger):
+    console_output = "A" * 60000  # 60,000 characters long
+    shell_output_logger(console_output, "prefix", logging.DEBUG)
+    assert mock_logger.log.call_count == 2
+    mock_logger.log.assert_any_call(logging.DEBUG, f"prefix [Log chunk 1 of 2] {'A' * 30000}")
+    mock_logger.log.assert_any_call(logging.DEBUG, f"prefix [Log chunk 2 of 2] {'A' * 30000}")
+
+
+def test_chunk_log_output():
+    output = "A" * 60000  # 60,000 characters long
+    chunks = list(chunk_log_output(output, chunk_size=30000))
+    assert len(chunks) == 2
+    assert chunks[0] == "[Log chunk 1 of 2] " + "A" * 30000
+    assert chunks[1] == "[Log chunk 2 of 2] " + "A" * 30000