ci: add memory history plot to readme

Save history of ram and flash usage and link to readme.

Signed-off-by: Giacomo Dematteis <giacomo.dematteis@nordicsemi.no>

Author: DematteisGiacomo

README.md

@@ -8,8 +8,8 @@
 [![Power Consumption Badge](https://img.shields.io/endpoint?url=https://nrfconnect.github.io/Asset-Tracker-Template/power_badge.json)](https://nrfconnect.github.io/Asset-Tracker-Template/power_measurements_plot.html)
 
 
-[![RAM Usage thingy91x](https://img.shields.io/endpoint?url=https://nrfconnect.github.io/Asset-Tracker-Template/ram_badge.json)](https://github.com/nrfconnect/Asset-Tracker-Template/actions/workflows/build-and-target-test.yml)
-[![Flash Usage thingy91x](https://img.shields.io/endpoint?url=https://nrfconnect.github.io/Asset-Tracker-Template/flash_badge.json)](https://github.com/nrfconnect/Asset-Tracker-Template/actions/workflows/build-and-target-test.yml)
+[![RAM Usage thingy91x](https://img.shields.io/endpoint?url=https://nrfconnect.github.io/Asset-Tracker-Template/ram_badge.json)](https://nrfconnect.github.io/Asset-Tracker-Template/ram_history_plot.html)
+[![Flash Usage thingy91x](https://img.shields.io/endpoint?url=https://nrfconnect.github.io/Asset-Tracker-Template/flash_badge.json)](https://nrfconnect.github.io/Asset-Tracker-Template/flash_history_plot.html)
 
 ## Overview
 

tests/on_target/scripts/memory_plot_generator.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+import plotly.express as px
+from datetime import datetime
+import os
+import sys
+
+def append_to_memory_csv(filename, used, total, current_date, output_dir):
+    """Append memory data to a CSV file."""
+    data = {'Date': current_date, 'Used (B)': used, 'Total (B)': total, 'Usage (%)': (used/total)*100}
+    df = pd.DataFrame([data])
+
+    csv_path = os.path.join(output_dir, filename)
+
+    if os.path.exists(csv_path):
+        print(f"CSV exists at {csv_path}, appending data")
+        # Read existing data
+        existing_df = pd.read_csv(csv_path)
+        print(f"Existing data:\n{existing_df}")
+
+        # Combine existing and new data
+        combined_df = pd.concat([existing_df, df], ignore_index=True)
+        print(f"Combined data:\n{combined_df}")
+
+        # Write back the complete dataset
+        combined_df.to_csv(csv_path, index=False)
+    else:
+        print(f"Creating new CSV at {csv_path}")
+        df.to_csv(csv_path, index=False)
+
+def append_to_csv(ram_used, ram_total, flash_used, flash_total, output_dir):
+    """Append memory usage data to CSV files with timestamps."""
+    current_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+    # Ensure output directory exists
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Append to RAM and Flash CSVs
+    append_to_memory_csv("ram_history.csv", ram_used, ram_total, current_date, output_dir)
+    append_to_memory_csv("flash_history.csv", flash_used, flash_total, current_date, output_dir)
+
+def generate_memory_plots(output_dir):
+    """Generate HTML plots from the CSV data."""
+    # Ensure output directory exists
+    os.makedirs(output_dir, exist_ok=True)
+
+    # RAM Plot
+    ram_csv = os.path.join(output_dir, "ram_history.csv")
+    if os.path.exists(ram_csv):
+        df_ram = pd.read_csv(ram_csv)
+        df_ram['Date'] = pd.to_datetime(df_ram['Date'])
+
+        # Calculate y-axis range for RAM
+        y_min_ram = 0  # Start from 0
+        y_max_ram = df_ram['Total (B)'].max() * 1.1  # 10% margin above max total
+
+        fig_ram = px.scatter(df_ram, x='Date', y=['Used (B)', 'Total (B)'],
+                           title='RAM Usage History - Asset Tracker Template',
+                           labels={'value': 'Bytes', 'variable': 'Metric'})
+        # Add markers
+        fig_ram.update_traces(mode='markers', marker=dict(size=10))
+        # Set y-axis range
+        fig_ram.update_layout(yaxis=dict(range=[y_min_ram, y_max_ram]))
+        fig_ram.write_html(os.path.join(output_dir, "ram_history_plot.html"))
+
+    # Flash Plot
+    flash_csv = os.path.join(output_dir, "flash_history.csv")
+    if os.path.exists(flash_csv):
+        df_flash = pd.read_csv(flash_csv)
+        df_flash['Date'] = pd.to_datetime(df_flash['Date'])
+
+        # Calculate y-axis range for Flash
+        y_min_flash = 0  # Start from 0
+        y_max_flash = df_flash['Total (B)'].max() * 1.1  # 10% margin above max total
+
+        fig_flash = px.scatter(df_flash, x='Date', y=['Used (B)', 'Total (B)'],
+                             title='Flash Usage History - Asset Tracker Template',
+                             labels={'value': 'Bytes', 'variable': 'Metric'})
+        # Add markers
+        fig_flash.update_traces(mode='markers', marker=dict(size=10))
+        # Set y-axis range
+        fig_flash.update_layout(yaxis=dict(range=[y_min_flash, y_max_flash]))
+        fig_flash.write_html(os.path.join(output_dir, "flash_history_plot.html"))
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <output_directory>")
+        sys.exit(1)
+    output_dir = sys.argv[1]
+    generate_memory_plots(output_dir)

tests/on_target/scripts/parse_memory_stats.py

@@ -4,6 +4,7 @@
 import sys
 import json
 import os
+from memory_plot_generator import append_to_csv, generate_memory_plots
 
 def format_size(size_in_bytes):
     """Format size in bytes to a human-readable string with appropriate unit."""
@@ -26,7 +27,7 @@ def create_badge_json(label, used, total, percent, output_file):
     with open(output_file, 'w') as f:
         json.dump(badge_data, f, indent=4)
 
-def parse_memory_stats(log_file):
+def parse_memory_stats(log_file, output_dir=None):
     with open(log_file, 'r') as f:
         content = f.read()
 
@@ -51,23 +52,30 @@ def parse_memory_stats(log_file):
     ram_percent = float(match.group(6))
 
     # Create badge JSON files
-    badge_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-    ram_badge_file = os.path.join(badge_dir, "ram_badge.json")
-    flash_badge_file = os.path.join(badge_dir, "flash_badge.json")
+    if output_dir is None:
+        output_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    ram_badge_file = os.path.join(output_dir, "ram_badge.json")
+    flash_badge_file = os.path.join(output_dir, "flash_badge.json")
 
     create_badge_json("RAM Usage thingy91x", ram_used, ram_total, ram_percent, ram_badge_file)
     create_badge_json("Flash Usage thingy91x", flash_used, flash_total, flash_percent, flash_badge_file)
 
     print("Memory Usage Statistics:")
     print(f"FLASH: {flash_used:,} B / {flash_total:,} B ({flash_percent:.2f}%)")
     print(f"RAM:   {ram_used:,} B / {ram_total:,} B ({ram_percent:.2f}%)")
+    # Append data to CSV files and generate plots
+    append_to_csv(ram_used, ram_total, flash_used, flash_total, output_dir)
+    generate_memory_plots(output_dir)
+
     print(f"\nBadge files created:")
     print(f"RAM Badge: {ram_badge_file}")
     print(f"Flash Badge: {flash_badge_file}")
+    print(f"Memory history plots generated")
 
 if __name__ == "__main__":
-    if len(sys.argv) != 2:
-        print(f"Usage: {sys.argv[0]} <build_output_log>")
+    if len(sys.argv) not in [2, 3]:
+        print(f"Usage: {sys.argv[0]} <build_output_log> [output_directory]")
         sys.exit(1)
 
-    parse_memory_stats(sys.argv[1])
+    output_dir = sys.argv[2] if len(sys.argv) == 3 else None
+    parse_memory_stats(sys.argv[1], output_dir)

tests/on_target/scripts/update_memory_badges.sh

@@ -3,12 +3,12 @@
 # Define paths
 BUILD_LOG=$1
 
-# Define paths to files to be committed
-FLASH_BADGE_FILE=tests/on_target/flash_badge.json
-RAM_BADGE_FILE=tests/on_target/ram_badge.json
-
 FLASH_BADGE_FILE_DEST=docs/flash_badge.json
 RAM_BADGE_FILE_DEST=docs/ram_badge.json
+FLASH_HISTORY_CSV_DEST=docs/flash_history.csv
+RAM_HISTORY_CSV_DEST=docs/ram_history.csv
+FLASH_PLOT_HTML_DEST=docs/flash_history_plot.html
+RAM_PLOT_HTML_DEST=docs/ram_history_plot.html
 
 if [ -z "$BUILD_LOG" ]; then
     echo "Error: Build log file path not provided"
@@ -21,24 +21,53 @@ handle_error() {
     exit 0
 }
 
-# Generate badge files using Python script
-echo "Generating badge files..."
-./tests/on_target/scripts/parse_memory_stats.py "$BUILD_LOG" || handle_error "Failed to generate badge files"
-
-
 # Configure Git
 git config --global --add safe.directory "$(pwd)"
 git config --global user.email "github-actions@github.com"
 git config --global user.name "GitHub Actions"
 
-# Ensure the gh-pages branch exists and switch to it
+# Create a temporary directory for gh-pages content
+TEMP_DIR=$(mktemp -d)
+
+# Fetch existing CSV files from gh-pages branch
 git fetch origin gh-pages
+
+echo "Fetching existing CSV files..."
+if git show origin/gh-pages:docs/ram_history.csv > "$TEMP_DIR/ram_history.csv" 2>/dev/null; then
+    echo "Found existing RAM history:"
+    cat "$TEMP_DIR/ram_history.csv"
+else
+    echo "No existing RAM history, creating new file"
+    touch "$TEMP_DIR/ram_history.csv"
+fi
+
+if git show origin/gh-pages:docs/flash_history.csv > "$TEMP_DIR/flash_history.csv" 2>/dev/null; then
+    echo "Found existing Flash history:"
+    cat "$TEMP_DIR/flash_history.csv"
+else
+    echo "No existing Flash history, creating new file"
+    touch "$TEMP_DIR/flash_history.csv"
+fi
+
+# Generate badge files and append to existing CSV files
+echo "Parsing build log and generating badge files, updating csv history files and html plots..."
+./tests/on_target/scripts/parse_memory_stats.py "$BUILD_LOG" "$TEMP_DIR" || handle_error "Failed to update badge, csv and html files"
+
+# Switch to gh-pages branch and update files
 git checkout gh-pages || handle_error "Not able to checkout gh-pages"
 git pull origin gh-pages || handle_error "Failed to pull latest gh-pages"
 
-# Stage, commit, and push changes to the branch
-cp $FLASH_BADGE_FILE $FLASH_BADGE_FILE_DEST
-cp $RAM_BADGE_FILE $RAM_BADGE_FILE_DEST
-git add $FLASH_BADGE_FILE_DEST $RAM_BADGE_FILE_DEST
+# Copy all files to docs/
+mkdir -p docs
+cp "$TEMP_DIR"/*.json docs/
+cp "$TEMP_DIR"/*.csv docs/
+cp "$TEMP_DIR"/*.html docs/
+
+git add $FLASH_BADGE_FILE_DEST $RAM_BADGE_FILE_DEST \
+    $FLASH_HISTORY_CSV_DEST $RAM_HISTORY_CSV_DEST \
+    $FLASH_PLOT_HTML_DEST $RAM_PLOT_HTML_DEST
 git commit -m "Update memory usage badges"
 git push origin gh-pages
+
+# Clean up
+rm -rf "$TEMP_DIR"