modules: Pull in sending of modem traces to Memfault

Pull in sending of modem traces to Memfault
Verify and test in CI

Signed-off-by: Simen S. Røstad <simen.rostad@nordicsemi.no>

Author: simensrostad

.github/actions/build-step/action.yml

@@ -52,7 +52,7 @@ runs:
           FW_TYPE=${{ inputs.memfault_fw_type }}
         fi
         cd ${{ inputs.path }}
-        cp overlay-memfault.conf overlay-memfault-att.conf
+        cat overlay-memfault.conf overlay-upload-modem-traces-to-memfault.conf > overlay-memfault-att.conf
         echo "CONFIG_MEMFAULT_NCS_PROJECT_KEY=\"${{ inputs.memfault_project_key }}\"" >> overlay-memfault-att.conf
         echo CONFIG_MEMFAULT_NCS_FW_TYPE=\"$FW_TYPE\" >> overlay-memfault-att.conf
         echo CONFIG_MEMFAULT_NCS_FW_VERSION_AUTO=y >> overlay-memfault-att.conf

app/overlay-upload-modem-traces-to-memfault.conf

@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2025 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+#
+
+# Continuously store modem traces to flash and upload them to Memfault upon a coredump
+
+# Memfault
+CONFIG_MEMFAULT_NCS_POST_MODEM_TRACE_ON_COREDUMP=y
+
+# nRF Modem lib trace
+CONFIG_FCB=y
+CONFIG_FLASH_MAP=y
+CONFIG_NRF_MODEM_LIB_TRACE=y
+# Set the default trace level to off preventing overwriting the trace leading up to the crash after
+# boot. The Memfault LTE coredump modem trace layer will ensure that traces are enabled once
+# the coredump/modem trace has been uploaded, or if there is no application coredump.
+CONFIG_NRF_MODEM_LIB_TRACE_LEVEL_OFF=y
+CONFIG_NRF_MODEM_LIB_SHELL_TRACE=y
+CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_FLASH=y
+CONFIG_NRF_MODEM_TRACE_FLASH_NOSPACE_ERASE_OLDEST=y
+CONFIG_NRF_MODEM_LIB_TRACE_STACK_SIZE=1024
+# Maximum number of sectors that the trace backend can handle
+CONFIG_NRF_MODEM_LIB_TRACE_FLASH_SECTORS=255
+# Modem trace flash partition size with 255 sectors of 4kB each
+CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_FLASH_PARTITION_SIZE=0xFF000

docs/common/advanced.md

@@ -17,6 +17,9 @@
 ### Keep it simple - small functions, pass by reference, readability and maintainability
 - Explanation of keeping it simple with small functions, passing by reference, readability, and maintainability.
 
+###
+- Document uploading modem traces to memfault and how the device can be setup with a custom data recording upload and setup with sec tags to dump TLS traffic.
+
 ## Useful links
 
 [nRF Cloud CoAP](https://docs.nordicsemi.com/bundle/ncs-latest/page/nrf/libraries/networking/nrf_cloud_coap.html)

docs/common/getting_started.md

@@ -58,5 +58,10 @@ west build -b thingy91x/nrf9151/ns -- -DEXTRA_CONF_FILE="overlay-memfault.conf"
 
 ### Build with memfault and etb traces overlay
 ```shell
-west build -b thingy91x/nrf9151/ns -- -DEXTRA_CONF_FILE="overlay-memfault-debug.conf;overlay-etb.conf"
+west build -b thingy91x/nrf9151/ns -- -DEXTRA_CONF_FILE="overlay-memfault.conf;overlay-etb.conf"
+```
+
+### Build with memfault and uploading of modem traces to memfault on coredumps
+```shell
+west build -b thingy91x/nrf9151/ns -- -DEXTRA_CONF_FILE="overlay-memfault.conf;overlay-upload-modem-traces-to-memfault.conf"
 ```

tests/on_target/tests/test_functional/test_memfault.py

@@ -6,10 +6,10 @@
 import pytest
 import os
 import requests
-import datetime
 import time
 from utils.flash_tools import flash_device, reset_device
 from utils.logger import get_logger
+from datetime import datetime, timezone
 
 MEMFAULT_ORG_TOKEN = os.getenv('MEMFAULT_ORGANIZATION_TOKEN')
 MEMFAULT_ORG = os.getenv('MEMFAULT_ORGANIZATION_SLUG')
@@ -20,8 +20,36 @@
 logger = get_logger()
 
 url = "https://api.memfault.com/api/v0"
+url_cdr = "https://app.memfault.com/api/v0"
 auth = ("", MEMFAULT_ORG_TOKEN)
 
+def fetch_recent_modem_trace(device_id, start_time, end_time):
+    r = requests.get(
+        f"{url_cdr}/organizations/{MEMFAULT_ORG}/projects/{MEMFAULT_PROJ}/devices/{device_id}/custom-data-recordings?end_time={end_time}&page=1&per_page=1&start_time={start_time}",
+        auth=auth
+    )
+
+    # Print the request URL for debugging
+    logger.debug(f"Request URL: {r.url}")
+    # Print the response status code for debugging
+    logger.debug(f"Response status code: {r.status_code}")
+    # Print the response content for debugging
+    logger.debug(f"Response content: {r.content}")
+
+    r.raise_for_status()
+    response = r.json()
+
+    # Extract relevant data from the response
+    data = response.get("data", [])
+
+    if data:
+        recording = data[0]
+        reason = recording.get("reason")
+
+        if reason == "modem traces":
+                return recording.get("id")
+
+    return None
 
 def get_traces(family, device_id):
     r = requests.get(
@@ -39,11 +67,10 @@ def get_latest_coredump_traces(device_id):
     return get_traces("coredump", device_id)
 
 def timestamp(event):
-    return datetime.datetime.strptime(
+    return datetime.strptime(
         event["captured_date"], "%Y-%m-%dT%H:%M:%S.%f%z"
     )
 
-
 def test_memfault(dut_board, hex_file):
     # Save timestamp of latest coredump
     coredumps = get_latest_coredump_traces(IMEI)
@@ -81,3 +108,21 @@ def test_memfault(dut_board, hex_file):
                 break
     else:
         raise RuntimeError("No new coredump observed")
+
+    # Wait for modem trace to be reported to memfault api
+    start = time.time()
+    now = datetime.now(timezone.utc)
+    start_time = now.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    while time.time() - start < MEMFAULT_TIMEOUT:
+
+        now = datetime.now(timezone.utc)
+        end_time = now.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        modem_trace_id = fetch_recent_modem_trace(IMEI, start_time, end_time)
+        if modem_trace_id:
+            print(f"Found modem trace with ID {modem_trace_id}")
+            break
+        time.sleep(5)
+    else:
+        raise RuntimeError("No modem trace observed")

west.yml

@@ -14,5 +14,5 @@ manifest:
     - name: nrf
       remote: ncs
       repo-path: sdk-nrf
-      revision: 52fa7533833478ecbf796c44bc5dc92e79fcbf9b
+      revision: 76347af2dcd98467b26f28dfecb3b7bc1f57e68a
       import: true