Add compression and decompression logic to file transfer

Signed-off-by: Simone Orru <simone.orru@secomind.com>

Author: sorru94

.gitignore

@@ -35,4 +35,5 @@ e2e/http_server_certs/192.0.2.2.pem
 e2e/http_server_certs/192.0.2.2.key
 
 # All data in the http_server_data folder
-e2e/http_server_data/*_test_data.txt
+e2e/http_server_data/*.txt
+e2e/http_server_data/*.lz4

e2e/prj.conf

@@ -6,7 +6,7 @@
 # in the application. See the README for more details.
 
 # Increase size for heap allocation space
-CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=32768
+CONFIG_COMMON_LIBC_MALLOC_ARENA_SIZE=262144
 
 # Increase the UART RX ring buffer to handle long automated commands
 CONFIG_SHELL_BACKEND_SERIAL_RX_RING_BUFFER_SIZE=1024
@@ -193,3 +193,7 @@ CONFIG_UUID=y
 CONFIG_UUID_V4=y
 CONFIG_UUID_V5=y
 CONFIG_UUID_BASE64=y
+
+# Enable LZ4 compression
+CONFIG_LZ4=y
+CONFIG_LZ4_FRAME_SUPPORT=y

e2e/pytest/file_transfer.py

@@ -5,8 +5,8 @@
 import logging
 import hashlib
 import time
-
 import uuid
+import lz4.frame
 from pathlib import Path
 from datetime import datetime, timezone
 
@@ -78,41 +78,53 @@ def _execute_and_wait_for_transfer(e2e_cfg: Configuration, interface: str, trans
     assert progress_received, f"No progress update received for transfer {target_id}"
 
 
-def _run_full_transfer_cycle(e2e_cfg: Configuration, transfer_type: str, device_path: str):
+def _run_full_transfer_cycle(e2e_cfg: Configuration, transfer_type: str, device_path: str, encoding: str = None):
     """
-    Executes a complete Server -> Device -> Server test loop.
+    Executes a complete Server -> Device -> Server test loop with optional encoding.
     Generates unique content, downloads to the device, uploads back to the server,
     and verifies the data integrity.
     """
-    logger.info(f"Testing full file transfer loop for type: {transfer_type}")
+    logger.info(f"Testing full file transfer loop for type: '{transfer_type}', encoding: '{encoding}'")
+
+    # Generate test data unique to the transfer type and encoding
+    test_payload = f"Zephyr {transfer_type.capitalize()} Transfer Test Payload! (Encoding: {encoding}) " * 200
 
-    # Generate test data unique to the transfer type
-    test_payload = f"Zephyr {transfer_type.capitalize()} Transfer Test Payload! " * 200
-    test_bytes = test_payload.encode('utf-8')
-    file_digest = f"sha256:{hashlib.sha256(test_bytes).hexdigest()}"
+    # We hash the UNCOMPRESSED bytes, as device code calculates digest of decompressed chunks
+    raw_bytes = test_payload.encode('utf-8')
+    file_digest = f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}"
 
-    download_filename = f"{transfer_type}_download.txt"
-    upload_filename = f"{transfer_type}_upload.txt"
+    enc_suffix = f"{encoding}" if encoding else "txt"
+    download_filename = f"{transfer_type}_download.{enc_suffix}"
+    upload_filename = f"{transfer_type}_upload.{enc_suffix}"
 
     dl_id = str(uuid.uuid4())
     ul_id = str(uuid.uuid4())
 
+    # If encoding is lz4, create an lz4 compressed file for the server to serve
+    server_download_bytes = raw_bytes
+    if encoding == "lz4":
+        server_download_bytes = lz4.frame.compress(raw_bytes)
+
     # Write the initial file to the HTTP server's data directory
     download_file_path = Path(e2e_cfg.http_server_data_dir) / download_filename
-    download_file_path.write_bytes(test_bytes)
+    download_file_path.write_bytes(server_download_bytes)
 
     # --- PHASE 1: DOWNLOAD (Server to Device) ---
     dl_data = {
         "url": f"https://192.0.2.2:8443/{download_filename}",
         "id": dl_id,
         "progress": True,
         "digest": file_digest,
-        "fileSizeBytes": len(test_bytes),
+        "fileSizeBytes": len(raw_bytes),
         "httpHeaderKeys": ["Content-Type"],
-        "httpHeaderValues": ["text/plain"],
+        "httpHeaderValues": ["application/octet-stream" if encoding else "text/plain"],
         "destinationType": transfer_type,
         "destination": device_path,
     }
+
+    if encoding:
+        dl_data["encoding"] = encoding
+
     logger.info(f"Initiating {transfer_type} download...")
     _execute_and_wait_for_transfer(e2e_cfg, interface_ft_server_to_device, dl_data)
 
@@ -122,21 +134,35 @@ def _run_full_transfer_cycle(e2e_cfg: Configuration, transfer_type: str, device_
         "id": ul_id,
         "progress": True,
         "httpHeaderKeys": ["Content-Type"],
-        "httpHeaderValues": ["text/plain"],
+        "httpHeaderValues": ["application/octet-stream" if encoding else "text/plain"],
         "sourceType": transfer_type,
         "source": device_path,
     }
+
+    if encoding:
+        ul_data["encoding"] = encoding
+
     logger.info(f"Initiating {transfer_type} upload...")
     _execute_and_wait_for_transfer(e2e_cfg, interface_ft_device_to_server, ul_data)
 
     # --- PHASE 3: VERIFICATION ---
     uploaded_file_path = Path(e2e_cfg.http_server_data_dir) / upload_filename
     assert uploaded_file_path.exists(), "The device failed to upload the file back to the server"
 
-    actual_payload = uploaded_file_path.read_bytes()
-    assert actual_payload == test_bytes, f"Data corruption! Uploaded file does not match downloaded file for {transfer_type}."
+    actual_uploaded_bytes = uploaded_file_path.read_bytes()
 
-    logger.info(f"Full transfer loop for '{transfer_type}' completed.")
+    # If encoding is lz4, the device compressed the file. We need to decompress to verify.
+    if encoding == "lz4":
+        try:
+            actual_payload = lz4.frame.decompress(actual_uploaded_bytes)
+        except Exception as e:
+            assert False, f"Failed to decompress the uploaded LZ4 file: {e}"
+    else:
+        actual_payload = actual_uploaded_bytes
+
+    assert actual_payload == raw_bytes, f"Data corruption! Uploaded file does not match downloaded file for {transfer_type} with encoding {encoding}."
+
+    logger.info(f"Full transfer loop for '{transfer_type}' (encoding: '{encoding}') completed.")
 
 
 def validate_file_transfer_capabilities(e2e_cfg: Configuration):
@@ -145,16 +171,22 @@ def validate_file_transfer_capabilities(e2e_cfg: Configuration):
     ft_res = http_get_server_data(e2e_cfg, interface_ft_capabilities)
 
     assert (i in ft_res for i in ["encodings", "targets", "unixPermissions"]), "Wrong capabilities"
-    assert ft_res.get("encodings") is None, "Wrong encodings"
+    assert ft_res.get("encodings") == ["lz4"], "Incorrect capability encodings"
     assert ft_res.get("targets") == ["streaming", "filesystem"], "Wrong targets"
     assert not ft_res.get("unixPermissions"), "Wrong unix permissions"
 
     logger.info("File transfer test (capabilities) completed successfully")
 
-
+# Raw Transfer Tests
 def validate_file_transfer_stream(e2e_cfg: Configuration):
-    _run_full_transfer_cycle(e2e_cfg, transfer_type="stream", device_path="loopback")
-
+    _run_full_transfer_cycle(e2e_cfg, transfer_type="stream", device_path="loopback", encoding=None)
 
 def validate_file_transfer_filesystem(e2e_cfg: Configuration):
-    _run_full_transfer_cycle(e2e_cfg, transfer_type="filesystem", device_path="/lfs1/test_fs_transfer.txt")
+    _run_full_transfer_cycle(e2e_cfg, transfer_type="filesystem", device_path="/lfs1/test_fs_transfer.txt", encoding=None)
+
+# LZ4 Transfer Tests
+def validate_file_transfer_stream_lz4(e2e_cfg: Configuration):
+    _run_full_transfer_cycle(e2e_cfg, transfer_type="stream", device_path="loopback", encoding="lz4")
+
+def validate_file_transfer_filesystem_lz4(e2e_cfg: Configuration):
+    _run_full_transfer_cycle(e2e_cfg, transfer_type="filesystem", device_path="/lfs1/test_fs_transfer_lz4.txt", encoding="lz4")

e2e/pytest/test_e2e.py

@@ -10,7 +10,10 @@
 from configuration import Configuration
 from http_server import start_server, stop_server
 from telemetry import validate_initial_telemetry, validate_telemetry_frequency
-from file_transfer import is_file_transfer_enabled, validate_file_transfer_stream, validate_file_transfer_capabilities, validate_file_transfer_filesystem
+from file_transfer import (
+    is_file_transfer_enabled, validate_file_transfer_stream, validate_file_transfer_capabilities,
+    validate_file_transfer_filesystem, validate_file_transfer_stream_lz4, validate_file_transfer_filesystem_lz4
+    )
 
 logger = logging.getLogger(__name__)
 logging.getLogger("urllib3").setLevel(logging.INFO)
@@ -69,5 +72,7 @@ def test_file_transfer(e2e_device_env):
     validate_file_transfer_capabilities(cfg)
     validate_file_transfer_stream(cfg)
     validate_file_transfer_filesystem(cfg)
+    validate_file_transfer_stream_lz4(cfg)
+    validate_file_transfer_filesystem_lz4(cfg)
 
     time.sleep(1)

e2e/testcase.yaml

@@ -33,3 +33,4 @@ tests:
       - ft
     extra_configs:
       - CONFIG_EDGEHOG_DEVICE_FILE_TRANSFER=y
+      - CONFIG_EDGEHOG_DEVICE_FILE_TRANSFER_COMPRESSION=y

lib/edgehog_device/CMakeLists.txt

@@ -4,7 +4,7 @@
 
 zephyr_library()
 
-# TODO: remove this as it's a workaroung to make available the mbedTLS crypto headers in zephyr 4.3
+# TODO: remove this as it's a workaround to make available the mbedTLS crypto headers in zephyr 4.3
 zephyr_library_link_libraries(mbedTLS)
 
 zephyr_library_include_directories(include)
@@ -18,5 +18,12 @@ zephyr_library_sources(${lib_sources})
 
 if(CONFIG_EDGEHOG_DEVICE_FILE_TRANSFER)
     FILE(GLOB ft_sources file_transfer/*.c)
+
+    # Remove the compression source file if the config is not enabled
+    if(NOT CONFIG_EDGEHOG_DEVICE_FILE_TRANSFER_COMPRESSION)
+        list(REMOVE_ITEM ft_sources "${CMAKE_CURRENT_SOURCE_DIR}/file_transfer/compression.c")
+        list(REMOVE_ITEM ft_sources "${CMAKE_CURRENT_SOURCE_DIR}/file_transfer/decompression.c")
+    endif()
+
     zephyr_library_sources(${ft_sources})
 endif()

lib/edgehog_device/Kconfig

@@ -97,6 +97,7 @@ config EDGEHOG_DEVICE_FILE_TRANSFER_COMPRESSION
 	bool "Enable file transfer compression functionality"
 	depends on EDGEHOG_DEVICE
 	depends on EDGEHOG_DEVICE_FILE_TRANSFER
+	depends on LZ4
 	default false
 	help
 	  Enable the possibility to compress and decompress files
@@ -227,4 +228,14 @@ module-str = Log level for Edgehog device file transfer storage utilities.
 module-help = Sets log level for Edgehog device file transfer storage utilities.
 source "subsys/logging/Kconfig.template.log_config"
 
+module = EDGEHOG_DEVICE_FILE_TRANSFER_DECOMPRESSION
+module-str = Log level for the file transfer decompression
+module-help = Sets log level for zephyr file transfer decompression
+source "subsys/logging/Kconfig.template.log_config"
+
+module = EDGEHOG_DEVICE_FILE_TRANSFER_COMPRESSION
+module-str = Log level for the file transfer compression
+module-help = Sets log level for zephyr file transfer compression
+source "subsys/logging/Kconfig.template.log_config"
+
 endmenu

lib/edgehog_device/base_image.c

@@ -20,7 +20,7 @@
 EDGEHOG_LOG_MODULE_REGISTER(base_image, CONFIG_EDGEHOG_DEVICE_BASE_IMAGE_LOG_LEVEL);
 
 /************************************************
- * Static functions declaration
+ *         Static functions declarations        *
  ***********************************************/
 
 static void publish_fingerprint(edgehog_device_handle_t edgehog_device);
@@ -32,7 +32,7 @@ static void publish_version(edgehog_device_handle_t edgehog_device);
 static void publish_build_id(edgehog_device_handle_t edgehog_device);
 
 /************************************************
- * Constants and defines
+ *             Constants and defines            *
  ***********************************************/
 
 #define FINGERPRINT_PROP "/fingerprint"
@@ -41,7 +41,7 @@ static void publish_build_id(edgehog_device_handle_t edgehog_device);
 #define BUILD_ID_PROP "/buildId"
 
 /************************************************
- * Global functions definition
+ *         Global functions definition          *
  ***********************************************/
 
 void publish_base_image(edgehog_device_handle_t edgehog_device)
@@ -54,7 +54,7 @@ void publish_base_image(edgehog_device_handle_t edgehog_device)
 }
 
 /************************************************
- * Static functions definition
+ *         Static functions definitions         *
  ***********************************************/
 
 static void publish_fingerprint(edgehog_device_handle_t edgehog_device)

lib/edgehog_device/device.c

@@ -44,7 +44,7 @@ typedef edgehog_result_t (*datastream_obj_event_handler_cb_t)(
     edgehog_device_handle_t, astarte_device_datastream_object_event_t *);
 
 /************************************************
- * Static functions declaration
+ *         Static functions declarations        *
  ***********************************************/
 
 static edgehog_result_t add_interfaces(astarte_device_handle_t astarte_device);
@@ -223,7 +223,7 @@ static void astarte_property_unset_cbk(astarte_device_data_event_t event)
 }
 
 /************************************************
- * Global functions definition
+ *         Global functions definition          *
  ***********************************************/
 
 edgehog_result_t edgehog_device_new(
@@ -468,7 +468,7 @@ astarte_result_t edgehog_device_get_astarte_error(edgehog_device_handle_t edgeho
 }
 
 /************************************************
- * Static functions definition
+ *         Static functions definitions         *
  ***********************************************/
 
 static edgehog_result_t add_interfaces(astarte_device_handle_t astarte_device)