From 6118e2d1e53319a119485d51e9c556abbb42ee8f Mon Sep 17 00:00:00 2001 From: Adrian Cojocaru Date: Fri, 1 Aug 2025 13:38:54 +0300 Subject: [PATCH 1/2] Add Android metrics --- .../activity/benchmark/BenchmarkActivity.kt | 13 +- .../android/testapp/utils/BenchmarkUtils.kt | 201 +++++++++++++++++- 2 files changed, 209 insertions(+), 5 deletions(-) diff --git a/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt b/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt index c7f4f1ca202a..751f9a19119c 100644 --- a/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt +++ b/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt @@ -2,9 +2,11 @@ package org.maplibre.android.testapp.activity.benchmark import android.annotation.SuppressLint import android.app.Activity +import android.app.ActivityManager import android.content.Context import android.os.Build import android.os.Bundle +import android.os.Debug import android.os.Environment import android.os.Handler import android.os.PowerManager @@ -29,7 +31,7 @@ import org.maplibre.android.maps.MapLibreMap.CancelableCallback import org.maplibre.android.maps.MapView import org.maplibre.android.maps.RenderingStats import org.maplibre.android.testapp.R -import org.maplibre.android.testapp.styles.TestStyles +import org.maplibre.android.testapp.utils.BenchmarkAdvancedMetrics import org.maplibre.android.testapp.utils.BenchmarkInputData import org.maplibre.android.testapp.utils.BenchmarkResult import org.maplibre.android.testapp.utils.BenchmarkRun @@ -37,7 +39,6 @@ import org.maplibre.android.testapp.utils.BenchmarkRunResult import org.maplibre.android.testapp.utils.FrameTimeStore import org.maplibre.android.testapp.utils.jsonPayload import java.io.File -import java.util.ArrayList import kotlin.collections.flatMap import kotlin.collections.toTypedArray import kotlin.coroutines.resume @@ -88,6 +89,7 @@ suspend fun MapView.setStyleSuspend(styleUrl: String): Unit = */ class BenchmarkActivity : AppCompatActivity() { private val TAG = "BenchmarkActivity" + private val useAdvancedMetrics = false private lateinit var mapView: MapView private var handler: Handler? = null @@ -248,6 +250,9 @@ class BenchmarkActivity : AppCompatActivity() { val encodingTimeStore = FrameTimeStore() val renderingTimeStore = FrameTimeStore() + val metrics = if (useAdvancedMetrics) BenchmarkAdvancedMetrics() else null + + metrics?.start() maplibreMap.setSwapBehaviorFlush(benchmarkRun.syncRendering) @@ -256,6 +261,7 @@ class BenchmarkActivity : AppCompatActivity() { renderingTimeStore.add(stats.renderingTime * 1e3) numFrames++; } + mapView.addOnDidFinishRenderingFrameListener(listener) mapView.setStyleSuspend(benchmarkRun.styleURL) numFrames = 0 @@ -272,8 +278,9 @@ class BenchmarkActivity : AppCompatActivity() { val fps = (numFrames * 1E9) / (endTime - startTime) mapView.removeOnDidFinishRenderingFrameListener(listener) + metrics?.stop() - return BenchmarkRunResult(fps, encodingTimeStore, renderingTimeStore, getThermalStatus()) + return BenchmarkRunResult(fps, encodingTimeStore, renderingTimeStore, getThermalStatus(), metrics) } override fun onStart() { diff --git a/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/BenchmarkUtils.kt b/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/BenchmarkUtils.kt index 6f07bf49e802..f9d663c827bc 100644 --- a/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/BenchmarkUtils.kt +++ b/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/BenchmarkUtils.kt @@ -1,15 +1,23 @@ package org.maplibre.android.testapp.utils +import android.app.ActivityManager +import android.content.Context +import android.net.TrafficStats import android.os.Build +import android.os.Debug +import android.os.StrictMode import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.addJsonObject -import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.buildJsonObject import kotlinx.serialization.json.putJsonArray import org.maplibre.android.BuildConfig.GIT_REVISION import org.maplibre.android.testapp.BuildConfig +import java.io.BufferedReader +import java.io.InputStreamReader import java.util.ArrayList +import java.util.Timer +import kotlin.concurrent.fixedRateTimer data class BenchmarkInputData( val styleNames: List, @@ -32,7 +40,8 @@ data class BenchmarkRunResult( val fps: Double, val encodingTimeStore: FrameTimeStore, val renderingTimeStore: FrameTimeStore, - val thermalState: Int + val thermalState: Int, + val advancedMetrics: BenchmarkAdvancedMetrics? ) data class BenchmarkResult ( @@ -53,6 +62,24 @@ fun jsonPayload(benchmarkResult: BenchmarkResult): JsonObject { put("avgRenderingTime", JsonPrimitive(run.second.renderingTimeStore.average())) put("low1pEncodingTime", JsonPrimitive(run.second.encodingTimeStore.low1p())) put("low1pRenderingTime", JsonPrimitive(run.second.renderingTimeStore.low1p())) + + run.second.advancedMetrics?.let { metrics -> + put("cpu", buildJsonObject { + put("min", JsonPrimitive(metrics.min.cpu.value)) + put("max", JsonPrimitive(metrics.max.cpu.value)) + put("avg", JsonPrimitive(metrics.avg.cpu.value)) + }) + + put("memory", buildJsonObject { + // convert to MB + put("min", JsonPrimitive(metrics.min.memory.value / 1024) ) + put("max", JsonPrimitive(metrics.max.memory.value / 1024)) + put("avg", JsonPrimitive(metrics.avg.memory.value / 1024)) + }) + + // convert to MB + put("traffic", JsonPrimitive(metrics.traffic / 1024)) + } } } } @@ -85,3 +112,173 @@ class FrameTimeStore { return timeValues.average() } } + +class BenchmarkAdvancedMetrics { + class Metric>(var value: T) { + + fun min(newValue: Metric) { + if (value > newValue.value) { + value = newValue.value + } + } + + fun max(newValue: Metric) { + if (value < newValue.value) { + value = newValue.value + } + } + + @Suppress("UNCHECKED_CAST") + operator fun plusAssign(newValue: Metric) { + when (value) { + is Int -> value = ((value as Int) + (newValue.value as Int)) as T + is Long -> value = ((value as Long) + (newValue.value as Long)) as T + is Float -> value = ((value as Float) + (newValue.value as Float)) as T + is Double -> value = ((value as Double) + (newValue.value as Double)) as T + } + } + + @Suppress("UNCHECKED_CAST") + operator fun div(newValue: Int): Metric { + return when (value) { + is Int -> Metric(((value as Int) / newValue) as T) + is Long -> Metric(((value as Long) / newValue) as T) + is Float -> Metric(((value as Float) / newValue) as T) + is Double -> Metric(((value as Double) / newValue) as T) + else -> this + } + } + } + + class Snapshot ( + var cpu: Metric, + var memory: Metric, + ) { + fun min(snapshot: Snapshot) { + cpu.min(snapshot.cpu) + memory.min(snapshot.memory) + } + + fun max(snapshot: Snapshot) { + cpu.max(snapshot.cpu) + memory.max(snapshot.memory) + } + + operator fun plusAssign(snapshot: Snapshot) { + cpu += snapshot.cpu + memory += snapshot.memory + } + + operator fun div(value: Int): Snapshot { + return Snapshot( + cpu / value, + memory / value, + ) + } + } + + private var startTime = 0L + private var stopTime = 0L + private var trafficStart = 0L + private var trafficStop = 0L + private var timer: Timer? = null + + public var min = Snapshot(Metric(Float.MAX_VALUE), Metric(Long.MAX_VALUE)) + public var max = Snapshot(Metric(Float.MIN_VALUE), Metric(Long.MIN_VALUE)) + // track total/avg or keep values for median/low/high? + public var total = Snapshot(Metric(0.0f), Metric(0)) + public var frameCount = 0 + public val avg: Snapshot get() { return total / frameCount } + public val time: Long get() { return stopTime - startTime } + public val traffic: Long get() { return trafficStop - trafficStart } + public val enabled: Boolean get() { return time > 0.0 } + + @Synchronized public fun start(collectInterval: Long = 1000L) { + reset() + + startTime = System.currentTimeMillis() + trafficStart = TrafficStats.getTotalRxBytes() + + timer = fixedRateTimer( + name = "BenchmarkAdvancedMetrics", + period = collectInterval + ) { + collect() + } + } + + @Synchronized public fun stop() { + timer?.cancel() + timer = null + + stopTime = System.currentTimeMillis() + trafficStop = TrafficStats.getTotalRxBytes() + } + + private fun collect() { + val snapshot = Snapshot(Metric(getCPU()), Metric(getMemory())) + + min.min(snapshot) + max.max(snapshot) + total += snapshot + + ++frameCount + } + + private fun reset() { + min = Snapshot(Metric(Float.MAX_VALUE), Metric(Long.MAX_VALUE)) + max = Snapshot(Metric(Float.MIN_VALUE), Metric(Long.MIN_VALUE)) + total = Snapshot(Metric(0.0f), Metric(0)) + frameCount = 0 + } + + public fun toJson(): JsonObject { + return buildJsonObject { + put("cpu", buildJsonObject { + put("min", JsonPrimitive(min.cpu.value)) + put("max", JsonPrimitive(max.cpu.value)) + put("avg", JsonPrimitive(avg.cpu.value)) + }) + + put("memory", buildJsonObject { + put("min", JsonPrimitive(min.memory.value)) + put("max", JsonPrimitive(max.memory.value)) + put("avg", JsonPrimitive(avg.memory.value)) + }) + + put("traffic", JsonPrimitive(traffic)) + } + } + + companion object { + public fun getCPU(): Float { + val currentPolicy = StrictMode.allowThreadDiskReads() + + try { + val pid = android.os.Process.myPid().toString() + val cores = Runtime.getRuntime().availableProcessors() + val process = Runtime.getRuntime().exec("top -n 1 -o PID,%CPU") + val bufferedReader = BufferedReader(InputStreamReader(process.inputStream)) + var line = bufferedReader.readLine() + while (line != null) { + if (line.contains(pid)) { + val rawCpu = line.split(" ").last().toFloat() + return rawCpu / cores + } + line = bufferedReader.readLine() + } + } catch (e: Exception) { + return 0.0f + } finally { + StrictMode.setThreadPolicy(currentPolicy) + } + return 0.0f + } + + public fun getMemory(): Long { + val debugMemInfo = Debug.MemoryInfo() + Debug.getMemoryInfo(debugMemInfo) + return debugMemInfo.totalPss.toLong() + } + } +} From 693bffc8a4e24d2f71dcdd304b7a0cdea4b4577a Mon Sep 17 00:00:00 2001 From: Adrian Cojocaru Date: Fri, 1 Aug 2025 17:52:18 +0300 Subject: [PATCH 2/2] Add iOS metrics --- platform/ios/benchmark/BUILD.bazel | 2 + .../ios/benchmark/BenchmarkAdvancedMetrics.h | 45 +++++++++ .../ios/benchmark/BenchmarkAdvancedMetrics.mm | 99 +++++++++++++++++++ .../ios/benchmark/MBXBenchViewController.mm | 14 +++ 4 files changed, 160 insertions(+) create mode 100644 platform/ios/benchmark/BenchmarkAdvancedMetrics.h create mode 100644 platform/ios/benchmark/BenchmarkAdvancedMetrics.mm diff --git a/platform/ios/benchmark/BUILD.bazel b/platform/ios/benchmark/BUILD.bazel index 8949bdd49a91..2c12c2319fe8 100644 --- a/platform/ios/benchmark/BUILD.bazel +++ b/platform/ios/benchmark/BUILD.bazel @@ -3,6 +3,7 @@ load("//platform/ios/bazel:macros.bzl", "info_plist") filegroup( name = "ios_benchmark_srcs", srcs = [ + "BenchmarkAdvancedMetrics.mm", "MBXBenchAppDelegate.mm", "MBXBenchViewController.mm", "locations.cpp", @@ -14,6 +15,7 @@ filegroup( filegroup( name = "ios_benchmark_hdrs", srcs = [ + "BenchmarkAdvancedMetrics.h", "MBXBenchAppDelegate.h", "MBXBenchViewController.h", "locations.hpp", diff --git a/platform/ios/benchmark/BenchmarkAdvancedMetrics.h b/platform/ios/benchmark/BenchmarkAdvancedMetrics.h new file mode 100644 index 000000000000..f15718d031e6 --- /dev/null +++ b/platform/ios/benchmark/BenchmarkAdvancedMetrics.h @@ -0,0 +1,45 @@ +#import + +struct BenchmarkSnapshot { + float cpu = 0.0f; + unsigned long long memory = 0; + + void min(BenchmarkSnapshot s) { + if (cpu > s.cpu) { + cpu = s.cpu; + } + + if (memory > s.memory) { + memory = s.memory; + } + } + + void max(BenchmarkSnapshot s) { + if (cpu < s.cpu) { + cpu = s.cpu; + } + + if (memory < s.memory) { + memory = s.memory; + } + } + + void add(BenchmarkSnapshot s) { + cpu += s.cpu; + memory += s.memory; + } +}; + +@interface BenchmarkAdvancedMetrics : NSObject + +@property (nonatomic, readonly) BenchmarkSnapshot min; +@property (nonatomic, readonly) BenchmarkSnapshot max; +@property (nonatomic, readonly) BenchmarkSnapshot avg; + +- (void)start:(NSTimeInterval)collectInterval; +- (void)stop; + ++ (float)getCpuUsage; ++ (unsigned long long)getMemoryUsage; + +@end diff --git a/platform/ios/benchmark/BenchmarkAdvancedMetrics.mm b/platform/ios/benchmark/BenchmarkAdvancedMetrics.mm new file mode 100644 index 000000000000..ed53848be44c --- /dev/null +++ b/platform/ios/benchmark/BenchmarkAdvancedMetrics.mm @@ -0,0 +1,99 @@ +#import "BenchmarkAdvancedMetrics.h" +#import + +@implementation BenchmarkAdvancedMetrics { + NSTimer* timer; + unsigned int snapshotCount; + BenchmarkSnapshot total; +} + +- (BenchmarkSnapshot)avg { + BenchmarkSnapshot avg; + + avg.cpu = total.cpu / snapshotCount; + avg.memory = total.memory / snapshotCount; + + return avg; +} + +- (void)start:(NSTimeInterval)collectInterval { + if (timer) { + [self stop]; + } + + [self reset]; + + timer = [NSTimer scheduledTimerWithTimeInterval:collectInterval repeats:YES block:^(NSTimer * _Nonnull timer) { + [self collect]; + }]; +} + +- (void)stop { + if (timer) { + [timer invalidate]; + timer = nil; + } +} + +- (void)collect { + BenchmarkSnapshot snapshot{ + [BenchmarkAdvancedMetrics getCpuUsage], + [BenchmarkAdvancedMetrics getMemoryUsage] + }; + + _min.min(snapshot); + _max.max(snapshot); + total.add(snapshot); + + ++snapshotCount; +} + +- (void)reset { + _min = {FLT_MAX, ULLONG_MAX}; + _max = {-FLT_MAX, 0ull}; + total = {0.0f, 0ull}; + + snapshotCount = 0; +} + ++ (unsigned long long)getMemoryUsage { + mach_task_basic_info_data_t info; + mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT; + + kern_return_t result = task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &count); + return result == KERN_SUCCESS ? info.resident_size : 0ull; +} + ++ (float)getCpuUsage { + mach_port_t* threadList = NULL; + mach_msg_type_number_t threadCount = 0; + thread_basic_info_data_t threadInfo; + + kern_return_t result = task_threads(mach_task_self(), &threadList, &threadCount); + if (result != KERN_SUCCESS) { + return 0.0f; + } + + float cpuUsage = 0.0f; + + for (unsigned int i = 0; i < threadCount; ++i) { + thread_info_data_t threadInfoData; + mach_msg_type_number_t threadInfoCount = THREAD_INFO_MAX; + + result = thread_info(threadList[i], THREAD_BASIC_INFO, threadInfoData, &threadInfoCount); + if (result != KERN_SUCCESS) { + continue; + } + + thread_basic_info_t basicInfo = (thread_basic_info_t)threadInfoData; + if (basicInfo->flags & TH_FLAGS_IDLE) { + continue; + } + + cpuUsage += ((float)basicInfo->cpu_usage / TH_USAGE_SCALE) * 100.0f; + } + + return cpuUsage; +} + +@end diff --git a/platform/ios/benchmark/MBXBenchViewController.mm b/platform/ios/benchmark/MBXBenchViewController.mm index 21451a5e6b43..caf3ce31409e 100644 --- a/platform/ios/benchmark/MBXBenchViewController.mm +++ b/platform/ios/benchmark/MBXBenchViewController.mm @@ -5,6 +5,7 @@ #import "MLNSettings_Private.h" #include "locations.hpp" +#include "BenchmarkAdvancedMetrics.h" #include @@ -180,6 +181,7 @@ - (void)redirectLogToDocuments: (NSString*) fileName int frames = 0; double totalFrameEncodingTime = 0; double totalFrameRenderingTime = 0; +BenchmarkAdvancedMetrics* advancedMetrics = nil; std::chrono::steady_clock::time_point started; std::vector> > result; @@ -218,6 +220,11 @@ - (void)renderFrame - (void)startBenchmarkIteration { if (mbgl::bench::locations.size() > idx) { + if (advancedMetrics == nil) { + advancedMetrics = [[BenchmarkAdvancedMetrics alloc] init]; + [advancedMetrics start:1]; + } + const auto& location = mbgl::bench::locations[idx]; mbgl::CameraOptions cameraOptions; @@ -261,6 +268,13 @@ - (void)startBenchmarkIteration // NSLog(@"Total uploads with invalid segs: %zu", mbgl::uploadInvalidSegments); // NSLog(@"Total uploads with build: %zu", mbgl::uploadBuildCount); + [advancedMetrics stop]; + + NSLog(@"Cpu usage: (min %.2f%%) (max %.2f%%) (avg %.2f%%)", advancedMetrics.min.cpu, advancedMetrics.max.cpu, advancedMetrics.avg.cpu); + NSLog(@"Memory usage: (min %llu MB) (max %llu MB) (avg %llu MB)", advancedMetrics.min.memory / 0x100000, advancedMetrics.max.memory / 0x100000, advancedMetrics.avg.memory / 0x100000); + + advancedMetrics = nil; + #if !defined(NDEBUG) // Clean up and show rendering stats, as in `destroyCoreObjects` from tests. // TODO: This doesn't clean up everything, what are we missing?