src/benchmarks/gc/GC.Infrastructure/Notebooks/ASPNetBenchmarkAnalysis.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ASP.NET Benchmark Analysis \n",
    "\n",
    "This notebook highlights the steps associated with analyzing data from the ASP.NET benchmarks obtained using crank."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "#r \"nuget: Microsoft.Diagnostics.Tracing.TraceEvent, 3.1.13\"\n",
    "#r \"nuget: YamlDotnet\"\n",
    "#r \"nuget: XPlot.Plotly\"\n",
    "#r \"nuget: XPlot.Plotly.Interactive\"\n",
    "#r \"nuget: Microsoft.Data.Analysis, 0.19.1\"\n",
    "#r \"nuget: Newtonsoft.Json\"\n",
    "\n",
    "using Etlx = Microsoft.Diagnostics.Tracing.Etlx;\n",
    "using Microsoft.Data.Analysis;\n",
    "using Microsoft.Diagnostics.Tracing.Analysis.GC;\n",
    "using Microsoft.Diagnostics.Tracing.Analysis;\n",
    "using Microsoft.Diagnostics.Tracing.Parsers.Clr;\n",
    "using Microsoft.Diagnostics.Tracing;\n",
    "using System.Diagnostics;\n",
    "using XPlot.Plotly;\n",
    "\n",
    "using System.IO;\n",
    "using Newtonsoft.Json;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building and Using The GC Analysis API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "pwsh"
    },
    "polyglot_notebook": {
     "kernelName": "pwsh"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "dotnet build -c Release \"..\\GC.Analysis.API\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "// TODO: Ensure you are pointing to the right artifacts folder.\n",
    "#r \"..\\..\\..\\..\\..\\artifacts\\bin\\GC.Analysis.API\\Release\\net6.0\\GC.Analysis.API.dll\"\n",
    "\n",
    "using GC.Analysis.API;"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Acquisition\n",
    "\n",
    "The next few cells detail how to retrieve the data from a base path. The run name below is the name of the folder generated from running the ``aspnetbenchmarks`` command from the GC.Infrastructure API. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "// The LoadInfo class consists of all the pertinent fields needed to represent both the result from a particular benchmark\n",
    "// as well as the the comparison between two runs where the Data2 represents the GCProcessData of the comparand.\n",
    "public sealed class LoadInfo\n",
    "{\n",
    "    public double MaxWorkingSetMB {get;set;} = double.NaN;\n",
    "    public double P99WorkingSetMB {get;set;} = double.NaN;\n",
    "    public double P95WorkingSetMB {get;set;} = double.NaN;\n",
    "    public double P90WorkingSetMB {get;set;} = double.NaN;\n",
    "    public double P75WorkingSetMB {get;set;} = double.NaN;\n",
    "    public double P50WorkingSetMB {get;set;} = double.NaN;\n",
    "\n",
    "    public double MaxPrivateMemoryMB {get;set;} = double.NaN;\n",
    "    public double P99PrivateMemoryMB {get;set;} = double.NaN;\n",
    "    public double P95PrivateMemoryMB {get;set;} = double.NaN;\n",
    "    public double P90PrivateMemoryMB {get;set;} = double.NaN;\n",
    "    public double P75PrivateMemoryMB {get;set;} = double.NaN;\n",
    "    public double P50PrivateMemoryMB {get;set;} = double.NaN;\n",
    "\n",
    "    public double Latency50thMS {get; set;} = double.NaN;\n",
    "    public double Latency75thMS {get; set;} = double.NaN;\n",
    "    public double Latency90thMS {get; set;} = double.NaN;\n",
    "    public double Latency99thMS {get; set;} = double.NaN;\n",
    "    public double MeanLatencyMS {get; set;} = double.NaN;\n",
    "    public int ProcessId {get;set;}\n",
    "    public double RequestsPerMSec {get; set;} = double.NaN;\n",
    "    public string Run {get; set;}\n",
    "    public GCProcessData Data {get;set;}\n",
    "    public GCProcessData? Data2 {get;set;}\n",
    "    public string CommandLine {get;set;}\n",
    "    public double NumberOfHeapCountSwitches {get;set;} = 0;\n",
    "    public string Benchmark {get; set;}\n",
    "    public string Id {get; set;}\n",
    "    public double TotalSuspensionTimeMSec {get;set;} = double.NaN;\n",
    "    public double PercentPauseTimeInGC {get; set;} = double.NaN;\n",
    "    public double PercentTimeInGC {get; set;} = double.NaN;\n",
    "    public double MeanHeapSizeBeforeMB {get; set;} = double.NaN;\n",
    "    public double MaxHeapSizeMB {get; set;} = double.NaN;\n",
    "    public double TotalAllocationsMB {get;set;} = double.NaN;\n",
    "    public double GCScore {get;set;} = double.NaN;\n",
    "    public string TracePath {get; set;}\n",
    "    public string ProcessName {get;set;}\n",
    "}\n",
    "\n",
    "public class BenchmarkVolatilityData\n",
    "{\n",
    "    public string Benchmark {get; set;}\n",
    "    public double MaxWorkingSetMB {get;set;} = double.NaN;\n",
    "    public double P99WorkingSetMB {get;set;} = double.NaN;\n",
    "    public double P95WorkingSetMB {get;set;} = double.NaN;\n",
    "    public double P90WorkingSetMB {get;set;} = double.NaN;\n",
    "    public double P75WorkingSetMB {get;set;} = double.NaN;\n",
    "    public double P50WorkingSetMB {get;set;} = double.NaN;\n",
    "\n",
    "    public double MaxPrivateMemoryMB {get;set;} = double.NaN;\n",
    "    public double P99PrivateMemoryMB {get;set;} = double.NaN;\n",
    "    public double P95PrivateMemoryMB {get;set;} = double.NaN;\n",
    "    public double P90PrivateMemoryMB {get;set;} = double.NaN;\n",
    "    public double P75PrivateMemoryMB {get;set;} = double.NaN;\n",
    "    public double P50PrivateMemoryMB {get;set;} = double.NaN;\n",
    "\n",
    "    public double RequestsPerMSec {get;set;} = double.NaN;\n",
    "    public double MeanLatencyMS {get; set;} = double.NaN;\n",
    "    public double Latency50thMS {get; set;} = double.NaN;\n",
    "    public double Latency75thMS {get; set;} = double.NaN;\n",
    "    public double Latency90thMS {get; set;} = double.NaN;\n",
    "    public double Latency99thMS {get; set;} = double.NaN;\n",
    "    public double HeapCount { get; set; }= double.NaN;\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "// The DataManager is responsible for parsing all the data from ASP.NET results from a basepath.\n",
    "public class DataManager \n",
    "{\n",
    "    private readonly Dictionary<string, Dictionary<string, LoadInfo>> _benchmarkToRunData = new();\n",
    "    private readonly Dictionary<string, Dictionary<string, LoadInfo>> _runToBenchmarkData = new();\n",
    "    private readonly Dictionary<string, LoadInfo> _data; \n",
    "    private readonly string _basePath;\n",
    "\n",
    "    public DataManager(string basePath)\n",
    "    {\n",
    "        _basePath = basePath;\n",
    "        _data = GetLoadInfoFromBasePath(basePath);\n",
    "        foreach (var d in _data)\n",
    "        {\n",
    "            if (!_benchmarkToRunData.TryGetValue(d.Value.Benchmark, out var runData))\n",
    "            {\n",
    "                _benchmarkToRunData[d.Value.Benchmark] = runData = new();\n",
    "            }\n",
    "            runData[d.Value.Run] = d.Value;\n",
    "\n",
    "            if (!_runToBenchmarkData.TryGetValue(d.Value.Run, out var benchmarkData))\n",
    "            {\n",
    "                _runToBenchmarkData[d.Value.Run] = benchmarkData = new();\n",
    "            }\n",
    "\n",
    "            benchmarkData[d.Value.Benchmark] = d.Value;\n",
    "        }\n",
    "    }\n",
    "\n",
    "    public static double DeltaPercent (double baseline, double comparand) => Math.Round((comparand - baseline) / baseline * 100, 2);\n",
    "\n",
    "    public LoadInfo GetComparison(LoadInfo baseline, LoadInfo comparand)\n",
    "    {\n",
    "        return new LoadInfo\n",
    "        {\n",
    "            MaxWorkingSetMB    = DeltaPercent(baseline.MaxWorkingSetMB, comparand.MaxWorkingSetMB),\n",
    "            P99WorkingSetMB = DeltaPercent(baseline.P99WorkingSetMB, comparand.P99WorkingSetMB),\n",
    "            P95WorkingSetMB    = DeltaPercent(baseline.P95WorkingSetMB, comparand.P95WorkingSetMB),\n",
    "            P90WorkingSetMB    = DeltaPercent(baseline.P90WorkingSetMB, comparand.P90WorkingSetMB),\n",
    "            P75WorkingSetMB    = DeltaPercent(baseline.P75WorkingSetMB, comparand.P75WorkingSetMB),\n",
    "            P50WorkingSetMB    = DeltaPercent(baseline.P50WorkingSetMB, comparand.P50WorkingSetMB),\n",
    "\n",
    "            MaxPrivateMemoryMB = DeltaPercent(baseline.MaxPrivateMemoryMB, comparand.MaxPrivateMemoryMB),\n",
    "            P99PrivateMemoryMB = DeltaPercent(baseline.P99PrivateMemoryMB, comparand.P99PrivateMemoryMB),\n",
    "            P95PrivateMemoryMB    = DeltaPercent(baseline.P95PrivateMemoryMB, comparand.P95PrivateMemoryMB),\n",
    "            P90PrivateMemoryMB    = DeltaPercent(baseline.P90PrivateMemoryMB, comparand.P90PrivateMemoryMB),\n",
    "            P75PrivateMemoryMB    = DeltaPercent(baseline.P75PrivateMemoryMB, comparand.P75PrivateMemoryMB),\n",
    "            P50PrivateMemoryMB    = DeltaPercent(baseline.P50PrivateMemoryMB, comparand.P50PrivateMemoryMB),\n",
    "            \n",
    "            Latency50thMS   = DeltaPercent(baseline.Latency50thMS, comparand.Latency50thMS),\n",
    "            Latency75thMS   = DeltaPercent(baseline.Latency75thMS, comparand.Latency75thMS),\n",
    "            Latency90thMS   = DeltaPercent(baseline.Latency90thMS, comparand.Latency90thMS), \n",
    "            Latency99thMS   = DeltaPercent(baseline.Latency99thMS, comparand.Latency99thMS),  \n",
    "            MeanLatencyMS   = DeltaPercent(baseline.MeanLatencyMS, comparand.MeanLatencyMS),\n",
    "            RequestsPerMSec = DeltaPercent(baseline.RequestsPerMSec, comparand.RequestsPerMSec),\n",
    "            GCScore         = DeltaPercent(baseline.GCScore, comparand.GCScore),\n",
    "            NumberOfHeapCountSwitches = DeltaPercent(baseline.NumberOfHeapCountSwitches, comparand.NumberOfHeapCountSwitches),\n",
    "            Data = baseline.Data,\n",
    "            Data2 = comparand.Data,\n",
    "            Run = $\"{baseline.Run} vs. {comparand.Run}\",\n",
    "            Benchmark = baseline.Benchmark,\n",
    "            Id = $\"{baseline.Run} vs. {comparand.Run} for {baseline.Benchmark}\"\n",
    "        };\n",
    "    }\n",
    "\n",
    "    public void SummarizeResults(Dictionary<string, LoadInfo> info = null)\n",
    "    {\n",
    "        if (info == null)\n",
    "        {\n",
    "            info = _data;\n",
    "        }\n",
    "\n",
    "        Console.WriteLine(\"{0,10} | {1,35} | {2, 5} | {3, 5:0.00} | {4, 5} | {5, 5:0.00} | {6, 5} | {7, 5:0.00} | {8, 5} | {9, 5:0.00} | {10, 10:0.00} | {11, 10:0.00} | {12, 5:0.00} | {13, 10:0.00} | {14, 10:0.00} |\", \n",
    "                        \"run\", \"benchmark\", \"gen0\", \"pause\", \"gen1\", \"pause\", \"ngc2\", \"pause\", \"bgc\", \"pause\", \"allocMB\", \"alloc/gc\", \"pct\", \"peakMB\", \"meanMB\");\n",
    "        Console.WriteLine(\"{0}\", new String('-', 174));\n",
    "\n",
    "        foreach (var kvp in info)\n",
    "        {\n",
    "            List<TraceGC> gcs = kvp.Value?.Data?.GCs;\n",
    "            if (gcs == null || gcs.Count == 0)\n",
    "            {\n",
    "                continue;\n",
    "            }\n",
    "\n",
    "            int[] gc_counts = new int[4];\n",
    "            double[] gc_pauses = new double[4];\n",
    "            for (int i = 0; i < gcs.Count; i++)\n",
    "            {\n",
    "                TraceGC gc = gcs[i];\n",
    "                if (gc.Generation < 2)\n",
    "                {\n",
    "                    gc_counts[gc.Generation]++;\n",
    "                    gc_pauses[gc.Generation] += gc.PauseDurationMSec;\n",
    "                }\n",
    "                else\n",
    "                {\n",
    "                    if (gc.Type == GCType.BackgroundGC)\n",
    "                    {\n",
    "                        gc_counts[3]++;\n",
    "                        gc_pauses[3] += gc.PauseDurationMSec;\n",
    "                    }\n",
    "                    else\n",
    "                    {\n",
    "                        gc_counts[2]++;\n",
    "                        gc_pauses[2] += gc.PauseDurationMSec;\n",
    "                    }\n",
    "                }\n",
    "            }\n",
    "            \n",
    "            for (int i = 0; i < 4; i++)\n",
    "            {\n",
    "                if (gc_counts[i] > 0)\n",
    "                {\n",
    "                    gc_pauses[i] /= gc_counts[i];\n",
    "                }\n",
    "            }\n",
    "            \n",
    "            Console.WriteLine(\"{0,10} | {1,35} | {2, 5} | {3, 5:0.00} | {4, 5} | {5, 5:0.00} | {6, 5} | {7, 5:0.00} | {8, 5} | {9, 5:0.00} | {10, 10:0.00} | {11, 10:0.00} | {12, 5:0.00} | {13, 10:0.00} | {14, 10:0.00} |\",\n",
    "                kvp.Value.Run, kvp.Value.Benchmark, gc_counts[0], gc_pauses[0], gc_counts[1], gc_pauses[1], gc_counts[2], gc_pauses[2], gc_counts[3], gc_pauses[3],\n",
    "                kvp.Value.Data.Stats.TotalAllocatedMB, (kvp.Value.Data.Stats.TotalAllocatedMB / gcs.Count), kvp.Value.Data.Stats.GetGCPauseTimePercentage(), kvp.Value.Data.Stats.MaxSizePeakMB, kvp.Value.Data.Stats.MeanSizePeakMB);\n",
    "        }\n",
    "    }\n",
    "\n",
    "    public Dictionary<string, LoadInfo>? GetAllBenchmarksForRun(string run)\n",
    "    {\n",
    "        if (!_runToBenchmarkData.TryGetValue(run, out var benchmarksForRun))\n",
    "        {\n",
    "            Console.WriteLine($\"No benchmarks found for run: {run}\");\n",
    "            return null;\n",
    "        }\n",
    "\n",
    "        return benchmarksForRun;\n",
    "    }\n",
    "\n",
    "    public void SaveBenchmarkData(string outputPath = \"\")\n",
    "    {\n",
    "        if (string.IsNullOrEmpty(outputPath))\n",
    "        {\n",
    "            outputPath = _basePath;\n",
    "        }\n",
    "\n",
    "        StringBuilder sb = new();\n",
    "        sb.AppendLine($\"Run,Benchmark,Max Working Set (MB), Max Private Memory (MB), Request/MSec, Mean Latency (MSec), Latency 50th Percentile MSec, Latency 75th Percentile MSec, Latency 90th Percentile MSec, Latency 99th Percentile MSec\");\n",
    "        foreach (var b in _data)\n",
    "        {\n",
    "            var val = b.Value; \n",
    "            sb.AppendLine($\"{val.Run},{val.Benchmark},{val.MaxWorkingSetMB},{val.MaxPrivateMemoryMB},{val.RequestsPerMSec},{val.MeanLatencyMS},{val.Latency50thMS},{val.Latency75thMS},{val.Latency90thMS},{val.Latency99thMS}\");\n",
    "        }\n",
    "\n",
    "        File.WriteAllText(Path.Combine(outputPath, \"AllBenchmarks.csv\"), sb.ToString());\n",
    "    }\n",
    "\n",
    "    public Dictionary<string, LoadInfo>? GetAllRunsForBenchmark(string benchmark)\n",
    "    {\n",
    "        if (!_benchmarkToRunData.TryGetValue(benchmark, out var runsForBenchmark))\n",
    "        {\n",
    "            Console.WriteLine($\"No runs found for benchmark: {benchmark}\");\n",
    "            return null;\n",
    "        }\n",
    "\n",
    "        return runsForBenchmark;\n",
    "    }\n",
    "\n",
    "    public LoadInfo? GetBenchmarkData(string benchmark, string run)\n",
    "    {\n",
    "        if (!_benchmarkToRunData.TryGetValue(benchmark, out var runData))\n",
    "        {\n",
    "            Console.WriteLine($\"Benchmark: {benchmark} not found!\");\n",
    "            return null;\n",
    "        }\n",
    "\n",
    "        if (!runData.TryGetValue(run, out var loadInfo))\n",
    "        {\n",
    "            Console.WriteLine($\"Run: {run} not found!\");\n",
    "            return null;\n",
    "        }\n",
    "\n",
    "        return loadInfo;\n",
    "    }\n",
    "\n",
    "    public Dictionary<string, LoadInfo> Data => _data; \n",
    "\n",
    "    private Dictionary<string, LoadInfo> GetLoadInfoFromBasePath(string basePath)\n",
    "    {\n",
    "        Dictionary<string, LoadInfo> flatLoadMap = new();\n",
    "        var files = Directory.GetFiles(basePath, \"*.log\", SearchOption.AllDirectories);\n",
    "\n",
    "        foreach (var f in files)\n",
    "        {\n",
    "            if (f.Contains(\"build.log\") || f.Contains(\"output.log\") || f.Contains(\"_GCLog\"))\n",
    "            {\n",
    "                continue;\n",
    "            }\n",
    "\n",
    "            LoadInfo info = new();\n",
    "\n",
    "            string[] lines = File.ReadAllLines(f);\n",
    "            int idxOfApplication = Int32.MaxValue;\n",
    "            int idxOfLoad = Int32.MaxValue;\n",
    "            int idx = 0;\n",
    "\n",
    "            foreach (var line in lines)\n",
    "            {\n",
    "                string[] sp = line.Split(\"|\", StringSplitOptions.TrimEntries);\n",
    "                if (line.Contains(\"| application\"))\n",
    "                {\n",
    "                    idxOfApplication = idx;\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"| load\"))\n",
    "                {\n",
    "                    idxOfLoad = idx;\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"| Latency 50th\"))\n",
    "                {\n",
    "                    info.Latency50thMS = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"| Latency 75th\"))\n",
    "                {\n",
    "                    info.Latency75thMS = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"| Latency 90th\"))\n",
    "                {\n",
    "                    info.Latency90thMS = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"| Latency 99th\"))\n",
    "                {\n",
    "                    info.Latency99thMS = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Requests/sec\"))\n",
    "                {\n",
    "                    info.RequestsPerMSec = double.Parse(sp[2]) / 1000;\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Mean latency\"))\n",
    "                {\n",
    "                    info.MeanLatencyMS = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Max Working Set\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.MaxWorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Working Set P99\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.P99WorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Working Set P95\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.P95WorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Working Set P90\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.P90WorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "                \n",
    "                else if (line.Contains(\"Working Set P75\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.P75WorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Working Set P50\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.P50WorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "                \n",
    "                else if (line.Contains(\"Max Private Memory\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.MaxPrivateMemoryMB  = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Private Memory P99\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.P99WorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Private Memory P95\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.P95WorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Private Memory P90\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.P90WorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "                \n",
    "                else if (line.Contains(\"Private Memory P75\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.P75WorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                else if (line.Contains(\"Private Memory P50\") && (idxOfApplication < idx && idx < idxOfLoad)) \n",
    "                {\n",
    "                    info.P50WorkingSetMB = double.Parse(sp[2]);\n",
    "                }\n",
    "\n",
    "                ++idx;\n",
    "            }\n",
    "\n",
    "            string[] split =  f.Replace(\".5\", \"5\").Split(\".\");\n",
    "            string run = split[1];\n",
    "            string benchmark = Path.GetFileName( split[0] ).Replace(\"_Windows\", \"\").Replace(\"_Linux\", \"\").Replace(\".gc\", \"\").Replace(\".nettrace\", \"\");\n",
    "\n",
    "            string key = $\"{run} | {benchmark}\";\n",
    "            info.Benchmark = benchmark;\n",
    "            info.Run = run;\n",
    "            info.Id = key;\n",
    "\n",
    "            flatLoadMap[key] = info;\n",
    "        }\n",
    "\n",
    "        var traceFiles = Directory.GetFiles(basePath, \"*.etl.zip\", SearchOption.AllDirectories).ToList();\n",
    "        var nettraceFiles = Directory.GetFiles(basePath, \"*.nettrace\", SearchOption.AllDirectories);\n",
    "        traceFiles.AddRange(nettraceFiles);\n",
    "\n",
    "        HashSet<string> pertinentProcesses = new HashSet<string>\n",
    "        {\n",
    "            \"PlatformBenchmarks\",\n",
    "            \"Benchmarks\",\n",
    "            \"MapAction\",\n",
    "            \"TodosApi\",\n",
    "            \"BasicGrpc\",\n",
    "            \"BasicMinimalApi\",\n",
    "        };\n",
    "\n",
    "        Parallel.ForEach(traceFiles, (t) => {\n",
    "            string[] sp = t.Split(\"\\\\\");\n",
    "            string benchmark = Path.GetFileNameWithoutExtension(sp[sp.Length - 1]).Replace(\"_Windows\", \"\").Replace(\".gc.etl\", \"\").Replace(\"_Linux\", \"\").Replace(\".nettrace\", \"\").Replace(\".gc\", \"\");\n",
    "            string name = sp[sp.Length - 2].Replace(\".5\", \"5\");\n",
    "            string key = $\"{name} | {benchmark}\";\n",
    "\n",
    "            Analyzer analyzer = AnalyzerManager.GetAnalyzer(t);\n",
    "            GCProcessData? data = null;\n",
    "\n",
    "            if (t.Contains(\".nettrace\"))\n",
    "            {\n",
    "                data = analyzer.AllGCProcessData.First().Value.First();\n",
    "            }\n",
    "\n",
    "            else\n",
    "            {\n",
    "                foreach (var p in pertinentProcesses)\n",
    "                {\n",
    "                    data = analyzer.GetProcessGCData(p).FirstOrDefault();\n",
    "                    if (data != null)\n",
    "                    {\n",
    "                        break;\n",
    "                    }\n",
    "                }\n",
    "            }\n",
    "\n",
    "            if (data == null)\n",
    "            {\n",
    "                Console.WriteLine($\"The following key doesn't have the pertinent process {key} - {t}: {string.Join(\" , \", analyzer.TraceLog.Processes.Select(p => p.Name))}\");\n",
    "            }\n",
    "\n",
    "            else\n",
    "            {\n",
    "                lock (flatLoadMap)\n",
    "                {\n",
    "                    if (flatLoadMap.TryGetValue(key, out var f))\n",
    "                    {\n",
    "                        f.MeanHeapSizeBeforeMB = data.Stats.MeanSizePeakMB;\n",
    "                        f.MaxHeapSizeMB = data.Stats.MaxSizePeakMB;\n",
    "                        f.PercentTimeInGC = (data.GCs.Sum(gc => gc.PauseDurationMSec - gc.SuspendDurationMSec) / (data.Stats.ProcessDuration) ) * 100;\n",
    "                        f.TracePath = data.Parent.TraceLogPath;\n",
    "                        f.TotalAllocationsMB = data.Stats.TotalAllocatedMB;\n",
    "                        f.CommandLine = data.CommandLine;\n",
    "                        f.PercentPauseTimeInGC = data.Stats.GetGCPauseTimePercentage();\n",
    "                        f.GCScore = (f.MaxHeapSizeMB / f.PercentPauseTimeInGC);\n",
    "                        f.ProcessId = data.ProcessID;\n",
    "                        f.Data = data;\n",
    "                        f.ProcessName = data.ProcessName;\n",
    "                        f.TotalSuspensionTimeMSec = data.GCs.Sum(gc => gc.SuspendDurationMSec);\n",
    "\n",
    "                        for (int i = 0; i < data.GCs.Count - 1; i++)\n",
    "                        {\n",
    "                            if ( data.GCs[i].GlobalHeapHistory?.NumHeaps != data.GCs[i + 1].GlobalHeapHistory?.NumHeaps)\n",
    "                            {\n",
    "                                ++f.NumberOfHeapCountSwitches;\n",
    "                            }\n",
    "                        }\n",
    "\n",
    "                    }\n",
    "\n",
    "                    else\n",
    "                    {\n",
    "                        Console.WriteLine($\"{key} not found - Check if the trace has any elements: {t}\");\n",
    "                    }\n",
    "                }\n",
    "            }\n",
    "        });\n",
    "\n",
    "        return flatLoadMap;\n",
    "    }\n",
    "\n",
    "    public Dictionary<string, LoadInfo> GetBenchmarkToComparison(string baselineRun, string comparandRun)\n",
    "    {\n",
    "        Dictionary<string, LoadInfo> comparisons = new();\n",
    "\n",
    "        Dictionary<string, LoadInfo> baselineData = new();\n",
    "        Dictionary<string, LoadInfo> comparandData = new();\n",
    "        HashSet<string> allBenchmarks = new();\n",
    "\n",
    "        foreach (var d in _data)\n",
    "        {\n",
    "            allBenchmarks.Add(d.Value.Benchmark);\n",
    "\n",
    "            string run = d.Key.Split(\"|\", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)[0];\n",
    "\n",
    "            if (string.CompareOrdinal(run, baselineRun) == 0 && !baselineData.TryGetValue(d.Key, out var baselineInfo))\n",
    "            {\n",
    "                baselineInfo = baselineData[d.Value.Benchmark] = d.Value;\n",
    "            }\n",
    "\n",
    "            else if (string.CompareOrdinal(run, comparandRun) == 0 && !comparandData.TryGetValue(d.Key, out var comparandInfo))\n",
    "            {\n",
    "                comparandInfo = comparandData[d.Value.Benchmark] = d.Value;\n",
    "            }\n",
    "        }\n",
    "\n",
    "        foreach (var benchmark in allBenchmarks)\n",
    "        {\n",
    "            if (!baselineData.TryGetValue(benchmark, out var baselineBenchmarkInfo))\n",
    "            {\n",
    "                Console.WriteLine($\"Benchmark: {benchmark} not found on the baseline: {baselineRun}\");\n",
    "                continue;\n",
    "            }\n",
    "\n",
    "            if (!comparandData.TryGetValue(benchmark, out var comparandBenchmarkInfo))\n",
    "            {\n",
    "                Console.WriteLine($\"Benchmark: {benchmark} not found on the comparand: {comparandRun}\");\n",
    "                continue;\n",
    "            }\n",
    "\n",
    "            LoadInfo comparison = GetComparison(baselineBenchmarkInfo, comparandBenchmarkInfo);\n",
    "            comparisons[benchmark] = comparison;\n",
    "        }\n",
    "        \n",
    "        return comparisons;\n",
    "    }\n",
    "\n",
    "    private static double ComputeVolatility(List<double> data)\n",
    "    {\n",
    "        var max = data.Max();\n",
    "        var min = data.Min();\n",
    "        return Math.Round(((max - min) / min) * 100, 2);\n",
    "    }\n",
    "\n",
    "    public void SaveVolatilityData(List<string> namesOfBuilds, List<string> sortingCriteria = null)\n",
    "    {\n",
    "        // Build Parent -> < Build Name -> < Benchmark -> Data >>>\n",
    "        Dictionary<string, Dictionary<string, Dictionary<string, LoadInfo>>> listOfData = new();\n",
    "\n",
    "        foreach (var build in namesOfBuilds)\n",
    "        {\n",
    "            if (!listOfData.TryGetValue(build, out var b))\n",
    "            {\n",
    "                listOfData[build] = b = new();\n",
    "            }\n",
    "\n",
    "            foreach (var run in _runToBenchmarkData)\n",
    "            {\n",
    "                if (run.Key.Contains(build))\n",
    "                {\n",
    "                    b.Add(run.Key, run.Value);\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "\n",
    "        // At this point all the data has been categorized.\n",
    "        Dictionary<string, Dictionary<string, BenchmarkVolatilityData>> buildToBenchmarkVolatilityData = new();\n",
    "\n",
    "        // Get the Volatility Data Per Build.\n",
    "        foreach (var b in listOfData)\n",
    "        {\n",
    "            if (!buildToBenchmarkVolatilityData.TryGetValue(b.Key, out var volData))\n",
    "            {\n",
    "                buildToBenchmarkVolatilityData[b.Key] = volData = new();\n",
    "            }\n",
    "\n",
    "            foreach (var br in _benchmarkToRunData)\n",
    "            {\n",
    "                volData[br.Key] = new();\n",
    "            }\n",
    "\n",
    "            Dictionary<string, List<LoadInfo>> benchmarkToData = new();\n",
    "            foreach (var run in b.Value)\n",
    "            {\n",
    "                foreach (var benchmark in run.Value)\n",
    "                {\n",
    "                    if (!benchmarkToData.TryGetValue(benchmark.Key, out var d))\n",
    "                    {\n",
    "                        benchmarkToData[benchmark.Key] = d = new();\n",
    "                    }\n",
    "\n",
    "                    d.Add(benchmark.Value);\n",
    "                }\n",
    "            }\n",
    "\n",
    "            foreach (var benchmark in benchmarkToData)\n",
    "            {\n",
    "                volData[benchmark.Key] = new BenchmarkVolatilityData\n",
    "                {\n",
    "                    MaxWorkingSetMB = ComputeVolatility( benchmark.Value.Select(v => v.MaxWorkingSetMB) ),\n",
    "                    P99WorkingSetMB = ComputeVolatility( benchmark.Value.Select(v => v.P99WorkingSetMB) ),\n",
    "                    P95WorkingSetMB = ComputeVolatility( benchmark.Value.Select(v => v.P95WorkingSetMB) ),\n",
    "                    P90WorkingSetMB = ComputeVolatility( benchmark.Value.Select(v => v.P90WorkingSetMB) ),\n",
    "                    P75WorkingSetMB = ComputeVolatility( benchmark.Value.Select(v => v.P75WorkingSetMB) ),\n",
    "                    P50WorkingSetMB = ComputeVolatility( benchmark.Value.Select(v => v.P50WorkingSetMB) ),\n",
    "\n",
    "                    MaxPrivateMemoryMB = ComputeVolatility( benchmark.Value.Select(v => v.MaxPrivateMemoryMB) ),\n",
    "                    P99PrivateMemoryMB = ComputeVolatility( benchmark.Value.Select(v => v.P99PrivateMemoryMB) ),\n",
    "                    P95PrivateMemoryMB = ComputeVolatility( benchmark.Value.Select(v => v.P95PrivateMemoryMB) ),\n",
    "                    P90PrivateMemoryMB = ComputeVolatility( benchmark.Value.Select(v => v.P90PrivateMemoryMB) ),\n",
    "                    P75PrivateMemoryMB = ComputeVolatility( benchmark.Value.Select(v => v.P75PrivateMemoryMB) ),\n",
    "                    P50PrivateMemoryMB = ComputeVolatility( benchmark.Value.Select(v => v.P50PrivateMemoryMB) ),\n",
    "\n",
    "                    RequestsPerMSec = ComputeVolatility( benchmark.Value.Select(v => v.RequestsPerMSec) ),\n",
    "                    MeanLatencyMS = ComputeVolatility( benchmark.Value.Select(v => v.MeanLatencyMS) ),\n",
    "                    Latency50thMS = ComputeVolatility( benchmark.Value.Select(v => v.Latency50thMS) ),\n",
    "                    Latency75thMS = ComputeVolatility( benchmark.Value.Select(v => v.Latency75thMS) ),\n",
    "                    Latency90thMS = ComputeVolatility( benchmark.Value.Select(v => v.Latency90thMS) ),\n",
    "                    Latency99thMS = ComputeVolatility( benchmark.Value.Select(v => v.Latency99thMS) ),\n",
    "                    HeapCount     = ComputeVolatility( benchmark.Value.Select(v => (double)v.NumberOfHeapCountSwitches )),\n",
    "                    Benchmark     = benchmark.Key,\n",
    "                };\n",
    "            }\n",
    "        }\n",
    "       \n",
    "        Dictionary<string, List<BenchmarkVolatilityData>> sortedPerBuildVolatility = new();\n",
    "\n",
    "        string DisplayDetailsForABenchmark(BenchmarkVolatilityData val) =>\n",
    "            $\"{val.Benchmark},{val.MaxWorkingSetMB},{val.MaxPrivateMemoryMB},{val.RequestsPerMSec},{val.MeanLatencyMS},{val.Latency50thMS},{val.Latency75thMS},{val.Latency90thMS},{val.Latency99thMS},{val.HeapCount}\";\n",
    "        if (sortingCriteria == null)\n",
    "        {\n",
    "            sortingCriteria = new() { nameof(LoadInfo.MaxPrivateMemoryMB) };\n",
    "        }\n",
    "\n",
    "        foreach (var s in sortingCriteria)\n",
    "        {\n",
    "            Func<KeyValuePair<string, BenchmarkVolatilityData>, double> sortingFunctor = null;\n",
    "            Func<BenchmarkVolatilityData, double> selectionFunctor = null;\n",
    "\n",
    "            switch (s)\n",
    "            {\n",
    "                case nameof(BenchmarkVolatilityData.MaxWorkingSetMB):\n",
    "                    sortingFunctor = (data) => data.Value.MaxWorkingSetMB;\n",
    "                    selectionFunctor = (data) => data.MaxWorkingSetMB;\n",
    "                    break;\n",
    "                case nameof(BenchmarkVolatilityData.MaxPrivateMemoryMB):\n",
    "                    sortingFunctor = (data) => data.Value.MaxPrivateMemoryMB;\n",
    "                    selectionFunctor = (data) => data.MaxPrivateMemoryMB;\n",
    "                    break;\n",
    "                case nameof(BenchmarkVolatilityData.RequestsPerMSec):\n",
    "                    sortingFunctor = (data) => data.Value.RequestsPerMSec;\n",
    "                    selectionFunctor = (data) => data.RequestsPerMSec;\n",
    "                    break;\n",
    "                case nameof(BenchmarkVolatilityData.MeanLatencyMS):\n",
    "                    sortingFunctor = (data) => data.Value.MeanLatencyMS;\n",
    "                    selectionFunctor = (data) => data.MeanLatencyMS;\n",
    "                    break;\n",
    "                case nameof(BenchmarkVolatilityData.Latency50thMS):\n",
    "                    sortingFunctor = (data) => data.Value.Latency50thMS;\n",
    "                    selectionFunctor = (data) => data.Latency50thMS;\n",
    "                    break;\n",
    "                case nameof(BenchmarkVolatilityData.Latency75thMS):\n",
    "                    sortingFunctor = (data) => data.Value.Latency75thMS;\n",
    "                    selectionFunctor = (data) => data.Latency75thMS;\n",
    "                    break;\n",
    "                case nameof(BenchmarkVolatilityData.Latency90thMS):\n",
    "                    sortingFunctor = (data) => data.Value.Latency90thMS;\n",
    "                    selectionFunctor = (data) => data.Latency90thMS;\n",
    "                    break;\n",
    "                case nameof(BenchmarkVolatilityData.Latency99thMS):\n",
    "                    sortingFunctor = (data) => data.Value.Latency99thMS;\n",
    "                    selectionFunctor = (data) => data.Latency99thMS;\n",
    "                    break;\n",
    "                default:\n",
    "                    sortingFunctor = (data) => data.Value.MaxPrivateMemoryMB;\n",
    "                    selectionFunctor = (data) => data.MaxPrivateMemoryMB;\n",
    "                    break;\n",
    "            }\n",
    "\n",
    "            foreach (var b in buildToBenchmarkVolatilityData)\n",
    "            {\n",
    "                sortedPerBuildVolatility[b.Key] = b.Value.OrderByDescending(sortingFunctor).Select(k => k.Value).ToList();\n",
    "            }\n",
    "\n",
    "            // Create CSV.\n",
    "            StringBuilder top = new();\n",
    "\n",
    "            // Iterate over each of the runs.\n",
    "            const int singleBuildColumnSize = 10;\n",
    "            int numberOfBuilds = buildToBenchmarkVolatilityData.Count;\n",
    "            string columnHeader = \"Benchmark Name,MaxWorkingSetMB,MaxPrivateMemoryMB,RequestsPerMSec,MeanLatencyMS,Latency50thMS,Latency75thMS,Latency90thMS,Latency99thMS,HeapCount\";\n",
    "\n",
    "            // Assumption: the same benchmarks are present for all runs.\n",
    "            int totalCountOfBenchmarks = buildToBenchmarkVolatilityData.First().Value.Count;\n",
    "\n",
    "            string first = string.Join(\",\", namesOfBuilds.Select(build => build + string.Join(\"\", Enumerable.Repeat(\",\", singleBuildColumnSize))));\n",
    "            string second = string.Join(\",,\", Enumerable.Repeat(columnHeader, numberOfBuilds));\n",
    "\n",
    "            top.AppendLine(first);\n",
    "            top.AppendLine(second);\n",
    "\n",
    "            for (int benchmarkIdx = 0; benchmarkIdx < totalCountOfBenchmarks; benchmarkIdx++)\n",
    "            {\n",
    "                top.AppendLine(string.Join(\",,\", namesOfBuilds.Select(buildName => DisplayDetailsForABenchmark(sortedPerBuildVolatility[buildName][benchmarkIdx]))));\n",
    "            }\n",
    "\n",
    "            File.WriteAllText(Path.Combine(_basePath, $\"Volatility_{s}.csv\"), top.ToString());\n",
    "\n",
    "            // Chart the sorted % Vol Results.\n",
    "\n",
    "            List<Scatter> scatters = new();\n",
    "\n",
    "            var layout = new Layout.Layout\n",
    "            {\n",
    "                xaxis = new Xaxis { title = \"Benchmark Name\" },\n",
    "                yaxis = new Yaxis { title = \"Metric Volatility Score\" },\n",
    "                width = 1500,\n",
    "                title = $\"Volatility Scores Sorted by {s} for {sortedPerBuildVolatility.First().Key}\"\n",
    "            };\n",
    "\n",
    "            foreach (var b in sortedPerBuildVolatility)\n",
    "            {\n",
    "                var scatter = new Scatter\n",
    "                {\n",
    "                    x = b.Value.Select(s => s.Benchmark),\n",
    "                    y = b.Value.Select(selectionFunctor),\n",
    "                    mode = \"markers\",\n",
    "                    name = b.Key,\n",
    "                };\n",
    "\n",
    "                scatters.Add(scatter);\n",
    "            }\n",
    "\n",
    "            Chart.Plot(scatters, layout).Display();\n",
    "\n",
    "            scatters.Clear();\n",
    "            layout = new Layout.Layout\n",
    "            {\n",
    "                xaxis = new Xaxis { title = \"Volatility Index\" },\n",
    "                yaxis = new Yaxis { title = \"Metric Volatility Score\" },\n",
    "                width = 1500,\n",
    "                title = $\"Volatility Index Sorted by {s} for {sortedPerBuildVolatility.First().Key}\"\n",
    "            };\n",
    "\n",
    "            foreach (var b in sortedPerBuildVolatility)\n",
    "            {\n",
    "                var sorted = b.Value.OrderByDescending(selectionFunctor);\n",
    "                var scatter = new Scatter\n",
    "                {\n",
    "                    x = Enumerable.Range(0, sorted.Count()),\n",
    "                    y = sorted.Select(selectionFunctor),\n",
    "                    mode = \"markers\",\n",
    "                    name = b.Key,\n",
    "                    text = sorted.Select(ss => ss.Benchmark),\n",
    "                };\n",
    "\n",
    "                scatters.Add(scatter);\n",
    "            }\n",
    "            \n",
    "            Chart.Plot(scatters, layout).Display();\n",
    "        }\n",
    "    }\n",
    "\n",
    "    public void SaveDifferences(string baseline, string comparand, List<string> sortingCriteria = null)\n",
    "    {\n",
    "        // This function assumes the runs are all in:\n",
    "        // {build}_{iteration} form.\n",
    "        // Else, it will except.\n",
    "\n",
    "        // Iteration -> LoadInfos\n",
    "        Dictionary<int, List<LoadInfo>> iterationData = new();\n",
    "\n",
    "        // Get the max iteration.\n",
    "        int maxIteration = -1;\n",
    "        foreach (var run in _runToBenchmarkData)\n",
    "        {\n",
    "            string runName = run.Key;\n",
    "            string[] split = run.Key.Split(\"_\");\n",
    "            Debug.Assert(split.Length == 2);\n",
    "            string build = split[0];\n",
    "            string iterationAsString = split[1];\n",
    "            int iteration = Convert.ToInt32(iterationAsString);\n",
    "            maxIteration = System.Math.Max(iteration, maxIteration);\n",
    "        }\n",
    "\n",
    "        // Compute Average Diff\n",
    "        // Build to Benchmark -> Data\n",
    "        Dictionary<string, Dictionary<string, LoadInfo>> averageData = new();\n",
    "\n",
    "        for (int i = 0; i <= maxIteration; i++)\n",
    "        {\n",
    "            string baselineIteration  = baseline  + \"_\" + i.ToString();\n",
    "            string comparandIteration = comparand + \"_\" + i.ToString();\n",
    "\n",
    "            Dictionary<string, LoadInfo> baselineIterationRuns  = _runToBenchmarkData[baselineIteration];\n",
    "            Dictionary<string, LoadInfo> comparandIterationRuns = _runToBenchmarkData[comparandIteration];\n",
    "\n",
    "            foreach (var b in baselineIterationRuns)\n",
    "            {\n",
    "                if (!iterationData.TryGetValue(i, out var benchmarks))\n",
    "                {\n",
    "                    iterationData[i] = benchmarks = new();\n",
    "                }\n",
    "\n",
    "                benchmarks.Add(GetComparison(baselineIterationRuns[b.Key], comparandIterationRuns[b.Key]));\n",
    "            }\n",
    "\n",
    "            if (!averageData.TryGetValue(baseline, out var bVal))\n",
    "            {\n",
    "                averageData[baseline] = bVal = new();\n",
    "                foreach (var benchmark in baselineIterationRuns)\n",
    "                {\n",
    "                    bVal[benchmark.Key] = new LoadInfo\n",
    "                    {\n",
    "                        Benchmark = benchmark.Key,\n",
    "                        MaxWorkingSetMB    = benchmark.Value.MaxWorkingSetMB,\n",
    "                        MaxPrivateMemoryMB = benchmark.Value.MaxPrivateMemoryMB,\n",
    "                        RequestsPerMSec = benchmark.Value.RequestsPerMSec,\n",
    "                        MeanLatencyMS   = benchmark.Value.MeanLatencyMS,\n",
    "                        Latency50thMS   = benchmark.Value.Latency50thMS, \n",
    "                        Latency75thMS   = benchmark.Value.Latency75thMS,\n",
    "                        Latency90thMS   = benchmark.Value.Latency90thMS,\n",
    "                        Latency99thMS   = benchmark.Value.Latency99thMS,\n",
    "                        NumberOfHeapCountSwitches = benchmark.Value.NumberOfHeapCountSwitches,\n",
    "                    };\n",
    "                }\n",
    "            }\n",
    "\n",
    "            else\n",
    "            {\n",
    "                foreach (var benchmark in baselineIterationRuns)\n",
    "                {\n",
    "                    var data = bVal[benchmark.Key];\n",
    "                    data.Benchmark = benchmark.Key;\n",
    "                    data.MaxWorkingSetMB    += benchmark.Value.MaxWorkingSetMB;\n",
    "                    data.MaxPrivateMemoryMB += benchmark.Value.MaxPrivateMemoryMB;\n",
    "                    data.RequestsPerMSec += benchmark.Value.RequestsPerMSec;\n",
    "                    data.MeanLatencyMS   += benchmark.Value.MeanLatencyMS;\n",
    "                    data.Latency50thMS   += benchmark.Value.Latency50thMS; \n",
    "                    data.Latency75thMS   += benchmark.Value.Latency75thMS;\n",
    "                    data.Latency90thMS   += benchmark.Value.Latency90thMS;\n",
    "                    data.Latency99thMS   += benchmark.Value.Latency99thMS;\n",
    "                    data.NumberOfHeapCountSwitches += benchmark.Value.NumberOfHeapCountSwitches;\n",
    "                }\n",
    "            }\n",
    "\n",
    "            if (!averageData.TryGetValue(comparand, out var cVal))\n",
    "            {\n",
    "                averageData[comparand] = cVal = new();\n",
    "                foreach (var benchmark in comparandIterationRuns)\n",
    "                {\n",
    "                    cVal[benchmark.Key] = new LoadInfo\n",
    "                    {\n",
    "                        Benchmark = benchmark.Key,\n",
    "                        MaxWorkingSetMB    = benchmark.Value.MaxWorkingSetMB,\n",
    "                        MaxPrivateMemoryMB = benchmark.Value.MaxPrivateMemoryMB,\n",
    "                        RequestsPerMSec = benchmark.Value.RequestsPerMSec,\n",
    "                        MeanLatencyMS   = benchmark.Value.MeanLatencyMS,\n",
    "                        Latency50thMS   = benchmark.Value.Latency50thMS, \n",
    "                        Latency75thMS   = benchmark.Value.Latency75thMS,\n",
    "                        Latency90thMS   = benchmark.Value.Latency90thMS,\n",
    "                        Latency99thMS   = benchmark.Value.Latency99thMS,\n",
    "                        NumberOfHeapCountSwitches = benchmark.Value.NumberOfHeapCountSwitches,\n",
    "                    };\n",
    "                }\n",
    "            }\n",
    "\n",
    "            else\n",
    "            {\n",
    "                foreach (var benchmark in comparandIterationRuns)\n",
    "                {\n",
    "                    var data = cVal[benchmark.Key];\n",
    "                    data.Benchmark = benchmark.Key;\n",
    "                    data.MaxWorkingSetMB    += benchmark.Value.MaxWorkingSetMB;\n",
    "                    data.MaxPrivateMemoryMB += benchmark.Value.MaxPrivateMemoryMB;\n",
    "                    data.RequestsPerMSec += benchmark.Value.RequestsPerMSec;\n",
    "                    data.MeanLatencyMS   += benchmark.Value.MeanLatencyMS;\n",
    "                    data.Latency50thMS   += benchmark.Value.Latency50thMS; \n",
    "                    data.Latency75thMS   += benchmark.Value.Latency75thMS;\n",
    "                    data.Latency90thMS   += benchmark.Value.Latency90thMS;\n",
    "                    data.Latency99thMS   += benchmark.Value.Latency99thMS;\n",
    "                    data.NumberOfHeapCountSwitches += benchmark.Value.NumberOfHeapCountSwitches;\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "\n",
    "        foreach (var benchmark in _benchmarkToRunData)\n",
    "        {\n",
    "            foreach (var build in averageData)\n",
    "            {\n",
    "                var data = build.Value[benchmark.Key];\n",
    "                data.Benchmark = benchmark.Key;\n",
    "                data.MaxWorkingSetMB    /= (maxIteration + 1); \n",
    "                data.MaxPrivateMemoryMB /=  (maxIteration + 1);\n",
    "                data.RequestsPerMSec /=  (maxIteration + 1);\n",
    "                data.MeanLatencyMS   /= (maxIteration + 1);\n",
    "                data.Latency50thMS   /= (maxIteration + 1);\n",
    "                data.Latency75thMS   /= (maxIteration + 1);\n",
    "                data.Latency90thMS   /= (maxIteration + 1);\n",
    "                data.Latency99thMS   /= (maxIteration + 1);\n",
    "                data.NumberOfHeapCountSwitches /= (maxIteration + 1);\n",
    "            }\n",
    "        }\n",
    "\n",
    "        string DisplayDetailsForABenchmark(LoadInfo val) =>\n",
    "            $\"{val.Benchmark},{val.MaxWorkingSetMB},{val.MaxPrivateMemoryMB},{val.RequestsPerMSec},{val.MeanLatencyMS},{val.Latency50thMS},{val.Latency75thMS},{val.Latency90thMS},{val.Latency99thMS},{val.NumberOfHeapCountSwitches}\";\n",
    "\n",
    "        if (sortingCriteria == null)\n",
    "        {\n",
    "            sortingCriteria = new() { nameof(LoadInfo.MaxPrivateMemoryMB) };\n",
    "        }\n",
    "\n",
    "        foreach (var s in sortingCriteria)\n",
    "        {\n",
    "            Func<LoadInfo, double> sortingFunctor = null;\n",
    "            Func<KeyValuePair<string, LoadInfo>, double> selectionFunctor = null;\n",
    "\n",
    "            switch (s)\n",
    "            {\n",
    "                case nameof(LoadInfo.MaxWorkingSetMB):\n",
    "                    sortingFunctor = (data) => data.MaxWorkingSetMB;\n",
    "                    selectionFunctor = (data) => data.Value.MaxWorkingSetMB;\n",
    "                    break;\n",
    "                case nameof(LoadInfo.MaxPrivateMemoryMB):\n",
    "                    sortingFunctor = (data) => data.MaxPrivateMemoryMB;\n",
    "                    selectionFunctor = (data) => data.Value.MaxPrivateMemoryMB;\n",
    "                    break;\n",
    "                case nameof(LoadInfo.RequestsPerMSec):\n",
    "                    sortingFunctor = (data) => data.RequestsPerMSec;\n",
    "                    selectionFunctor = (data) => data.Value.RequestsPerMSec;\n",
    "                    break;\n",
    "                case nameof(LoadInfo.MeanLatencyMS):\n",
    "                    sortingFunctor = (data) => data.MeanLatencyMS;\n",
    "                    selectionFunctor = (data) => data.Value.MeanLatencyMS;\n",
    "                    break;\n",
    "                case nameof(LoadInfo.Latency50thMS):\n",
    "                    sortingFunctor = (data) => data.Latency50thMS;\n",
    "                    selectionFunctor = (data) => data.Value.Latency50thMS;\n",
    "                    break;\n",
    "                case nameof(LoadInfo.Latency75thMS):\n",
    "                    sortingFunctor = (data) => data.Latency75thMS;\n",
    "                    selectionFunctor = (data) => data.Value.Latency75thMS;\n",
    "                    break;\n",
    "                case nameof(LoadInfo.Latency90thMS):\n",
    "                    sortingFunctor = (data) => data.Latency90thMS;\n",
    "                    selectionFunctor = (data) => data.Value.Latency90thMS;\n",
    "                    break;\n",
    "                case nameof(LoadInfo.Latency99thMS):\n",
    "                    sortingFunctor = (data) => data.Latency99thMS;\n",
    "                    selectionFunctor = (data) => data.Value.Latency99thMS;\n",
    "                    break;\n",
    "                case nameof(LoadInfo.NumberOfHeapCountSwitches):\n",
    "                    sortingFunctor = (data) => data.NumberOfHeapCountSwitches;\n",
    "                    selectionFunctor = (data) => data.Value.NumberOfHeapCountSwitches;\n",
    "                    break;\n",
    "                default:\n",
    "                    sortingFunctor = (data) => data.MaxPrivateMemoryMB;\n",
    "                    selectionFunctor = (data) => data.Value.MaxPrivateMemoryMB;\n",
    "                    break;\n",
    "            }\n",
    "\n",
    "            List<List<LoadInfo>> sortedLoadInfo = new(); \n",
    "            foreach (var iteration in iterationData)\n",
    "            {\n",
    "                sortedLoadInfo.Add(iteration.Value.OrderByDescending(sortingFunctor).ToList());\n",
    "            }\n",
    "\n",
    "            List<LoadInfo> sortedAverages = new();\n",
    "\n",
    "            foreach (var benchmark in averageData[baseline])\n",
    "            {\n",
    "                LoadInfo baselineInfo   = benchmark.Value;\n",
    "                LoadInfo comparandInfo  = averageData[comparand][benchmark.Key];\n",
    "                LoadInfo comparisonInfo = GetComparison(baselineInfo, comparandInfo);\n",
    "                sortedAverages.Add(comparisonInfo);\n",
    "            }\n",
    "            sortedAverages = sortedAverages.OrderByDescending(sortingFunctor).ToList();\n",
    "\n",
    "            // Create CSV.\n",
    "            StringBuilder top = new();\n",
    "\n",
    "            // Iterate over each of the runs.\n",
    "            const int singleBuildColumnSize = 11;\n",
    "            int numberOfIterations = maxIteration + 1;\n",
    "            string columnHeader = \"Benchmark Name,WorkingSetMB,PrivateMemoryMB,RequestsPerMSec,MeanLatencyMS,Latency50thMS,Latency75thMS,Latency90thMS,Latency99thMS,HeapCount\";\n",
    "\n",
    "            int totalCountOfBenchmarks = sortedLoadInfo.First().Count;\n",
    "\n",
    "            string first = string.Join(\"\", Enumerable.Range(0, numberOfIterations).Select(build => build + string.Join(\"\", Enumerable.Repeat(\",\", singleBuildColumnSize))));\n",
    "            string second = string.Join(\",,\", Enumerable.Repeat(columnHeader, numberOfIterations));\n",
    "\n",
    "            // Add the average diff.\n",
    "            first  += \"Average Diff %\" + string.Join(\"\", Enumerable.Repeat(\",\", singleBuildColumnSize));\n",
    "            second += \",,\" + string.Join(\",,\", columnHeader);\n",
    "\n",
    "            top.AppendLine(first);\n",
    "            top.AppendLine(second);\n",
    "\n",
    "            for (int benchmarkIdx = 0; benchmarkIdx < totalCountOfBenchmarks; benchmarkIdx++)\n",
    "            {\n",
    "                string benchmarkData = string.Join(\",,\", Enumerable.Range(0, numberOfIterations).Select(iteration => DisplayDetailsForABenchmark(sortedLoadInfo[iteration][benchmarkIdx])));\n",
    "                benchmarkData += $\",,{DisplayDetailsForABenchmark(sortedAverages[benchmarkIdx])}\";\n",
    "\n",
    "                top.AppendLine(benchmarkData);\n",
    "            }\n",
    "\n",
    "            File.WriteAllText(Path.Combine(_basePath, $\"Difference_{s}.csv\"), top.ToString());\n",
    "\n",
    "            var layout = new Layout.Layout\n",
    "            {\n",
    "                xaxis = new Xaxis { title = \"Benchmark Name\" },\n",
    "                yaxis = new Yaxis { title = $\"{s}\" },\n",
    "                width = 1500,\n",
    "                title = $\"Raw values of {s} for Runs\"\n",
    "            };\n",
    "\n",
    "            List<Scatter> scatters = new();\n",
    "\n",
    "            const int baseColor = 150;\n",
    "\n",
    "            for (int iterationIdx = 0; iterationIdx <= maxIteration; iterationIdx++)\n",
    "            {\n",
    "                string baselineIteration  = baseline  + \"_\" + iterationIdx.ToString();\n",
    "                string comparandIteration = comparand + \"_\" + iterationIdx.ToString();\n",
    "\n",
    "                Dictionary<string, LoadInfo> baselineData  = _runToBenchmarkData[baselineIteration];\n",
    "                Dictionary<string, LoadInfo> comparandData = _runToBenchmarkData[comparandIteration];\n",
    "\n",
    "                if (iterationIdx == 0)\n",
    "                {\n",
    "                    var sortedBaseline = baselineData.Values.OrderByDescending(sortingFunctor);\n",
    "                    baselineData = sortedBaseline.ToDictionary(d => d.Benchmark);\n",
    "                }\n",
    "\n",
    "                Scatter baselineScatter = new()\n",
    "                {\n",
    "                    x = baselineData.Select(b => b.Key),\n",
    "                    y = baselineData.Select(selectionFunctor),\n",
    "                    name = $\"{baselineIteration} - {s}\",\n",
    "                    mode = \"markers\",\n",
    "                    marker = new Marker { color = $\"rgb({baseColor + iterationIdx * 50}, 0, 0)\" } \n",
    "                };\n",
    "\n",
    "                Scatter comparandScatter = new()\n",
    "                {\n",
    "                    x = comparandData.Select(b => b.Key),\n",
    "                    y = comparandData.Select(selectionFunctor),\n",
    "                    name = $\"{comparandIteration} - {s}\",\n",
    "                    mode = \"markers\",\n",
    "                    marker = new Marker { color = $\"rgb(0, 0, {baseColor + iterationIdx * 50})\" } \n",
    "                };\n",
    "\n",
    "                scatters.Add(baselineScatter);\n",
    "                scatters.Add(comparandScatter);\n",
    "            }\n",
    "\n",
    "            Chart.Plot(scatters, layout).Display();\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "string basePath = @\"C:\\Traces\\GCTraces\\DATAS_5_Fixed\";\n",
    "var dataManager = new DataManager(basePath);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "dataManager.SummarizeResults()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "dataManager.SaveVolatilityData(new List<string> { \"baseline\", \"run\" }, new List<string> { nameof(LoadInfo.PrivateMemoryMB), nameof(LoadInfo.RequestsPerMSec )});"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "dataManager.SaveDifferences(\"baseline\", \"run\", new List<string> { nameof(LoadInfo.PrivateMemoryMB), nameof(LoadInfo.RequestsPerMSec) });"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "dataManager.Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using the DataManager\n",
    "\n",
    "The following cells demonstrates how to make use of the ``DataManager``. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "// The name of the run from the yaml file for which the ASP.NET run is created for.\n",
    "string runName = \"run\";\n",
    "\n",
    "Dictionary<string, LoadInfo> run = dataManager.GetAllBenchmarksForRun(runName);\n",
    "dataManager.Data.Display();\n",
    "List<KeyValuePair<string, LoadInfo>> runsWithGCData = dataManager.GetAllBenchmarksForRun(runName).Where(gc => gc.Value.Data != null);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "string benchmarkName = \"Name of the specific benchmark\";\n",
    "LoadInfo benchmarkData = dataManager.GetBenchmarkData(benchmark: benchmarkName, run: runName);\n",
    "benchmarkData.Id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "Dictionary<string, LoadInfo> allRunsForBenchmark = dataManager.GetAllRunsForBenchmark(benchmark: benchmarkName);\n",
    "allRunsForBenchmark.Keys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saving The Benchmark Results\n",
    "\n",
    "The following call will persist a flat list of all the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "dataManager.SaveBenchmarkData()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build to Build Comparison and Volatility Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "var run1_vs_run2 = dataManager.GetBenchmarkToComparison(\"datas_2\", \"datas_3\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "static bool IsNotInvalidDouble(double val) => \n",
    "    !double.IsNaN(val) && \n",
    "    !double.IsInfinity(val) && \n",
    "    !double.IsPositiveInfinity(val) && \n",
    "    !double.IsNegativeInfinity(val);\n",
    "\n",
    "public class SummaryTable\n",
    "{\n",
    "    public SummaryTable(Dictionary<string, Dictionary<string, LoadInfo>> comparisons)\n",
    "    {\n",
    "        Comparisons = comparisons;\n",
    "    }\n",
    "\n",
    "    private string GenerateSummaryForComparison(string comparisonKey, Dictionary<string, LoadInfo> comparison)\n",
    "    {\n",
    "        double averageWorkingSet = comparison.Where(a => IsNotInvalidDouble(a.Value.MaxWorkingSetMB)).Average(a => a.Value.MaxWorkingSetMB);\n",
    "        double privateMemory = comparison.Where(a => IsNotInvalidDouble(a.Value.MaxPrivateMemoryMB)).Average(a => a.Value.MaxPrivateMemoryMB);\n",
    "        double throughput = comparison.Where(a => IsNotInvalidDouble(a.Value.RequestsPerMSec)).Average(a => a.Value.RequestsPerMSec);\n",
    "        double meanLatency = comparison.Where(a => IsNotInvalidDouble(a.Value.MeanLatencyMS)).Average(a => a.Value.MeanLatencyMS);\n",
    "\n",
    "        double p50Latency = comparison.Where(a => IsNotInvalidDouble(a.Value.Latency50thMS)).Average(a => a.Value.Latency50thMS);\n",
    "        double p75Latency = comparison.Where(a => IsNotInvalidDouble(a.Value.Latency75thMS)).Average(a => a.Value.Latency75thMS);\n",
    "        double p90Latency = comparison.Where(a => IsNotInvalidDouble(a.Value.Latency90thMS)).Average(a => a.Value.Latency90thMS);\n",
    "        double p99Latency = comparison.Where(a => IsNotInvalidDouble(a.Value.Latency99thMS)).Average(a => a.Value.Latency99thMS);\n",
    "\n",
    "        return $\"{comparisonKey},{averageWorkingSet},{privateMemory},{throughput},{meanLatency},{p50Latency},{p75Latency},{p90Latency},{p99Latency}\";\n",
    "    }\n",
    "\n",
    "    public string GenerateSummaryForComparisons()\n",
    "    {\n",
    "        StringBuilder sb = new();\n",
    "        sb.AppendLine(\"Build to Build,Average Max Working Set (MB) %, Average Max Private Memory (MB) %, Average Request/MSec %, Average Mean Latency (MSec), Average P50 Latency (MSec) %, Average P75 Latency (MSec) %, Average P90 Latency (MSec) %, Average P99 Latency (MSec) %\");\n",
    "        foreach (var comparison in Comparisons)\n",
    "        {\n",
    "            sb.AppendLine(GenerateSummaryForComparison(comparison.Key, comparison.Value));\n",
    "        }\n",
    "\n",
    "        return sb.ToString();\n",
    "    }\n",
    "\n",
    "    private int GetCountOfRegressions(List<double> selected, double thresholdPercentage, bool lessIsBetter = true)\n",
    "    {\n",
    "        // If throughput, less is worse => threshold <= -5%.\n",
    "        var comparison = selected.Where(d => IsNotInvalidDouble(d) && ( (lessIsBetter) ? (d >= thresholdPercentage) : (d <= -thresholdPercentage)));\n",
    "        return comparison.Count;\n",
    "    }\n",
    "\n",
    "    private int GetCountOfAbsRegressions(List<double> selected, double thresholdPercentage)\n",
    "    {\n",
    "        var comparison = selected.Where(d => IsNotInvalidDouble(d) && Math.Abs(d) >= thresholdPercentage);\n",
    "        return comparison.Count;\n",
    "    }\n",
    "\n",
    "    // # of benchmarks with throughput regressed by >= 5% and 10%\n",
    "    private string GenerateRegressionSummary(string comparisonKey, Dictionary<string, LoadInfo> comparison)\n",
    "    {\n",
    "        List<double> workingSet  = comparison.Select(c => c.Value.MaxWorkingSetMB);\n",
    "        int workingSetCountGT_5  = GetCountOfRegressions(workingSet, 5);\n",
    "        int workingSetCountGT_10 = GetCountOfRegressions(workingSet, 10);\n",
    "\n",
    "        List<double> privateMemory  = comparison.Select(c => c.Value.MaxPrivateMemoryMB);\n",
    "        int privateMemoryCountGT_5  = GetCountOfRegressions(privateMemory, 5);\n",
    "        int privateMemoryCountGT_10 = GetCountOfRegressions(privateMemory, 10);\n",
    "\n",
    "        List<double> throughput  = comparison.Select(a => a.Value.RequestsPerMSec);\n",
    "        int throughputCountGT_5  = GetCountOfRegressions(throughput, 5, false);\n",
    "        int throughputCountGT_10 = GetCountOfRegressions(throughput, 10, false);\n",
    "\n",
    "        List<double> meanLatency  = comparison.Select(a => a.Value.MeanLatencyMS);\n",
    "        int meanLatencyCountGT_5  = GetCountOfRegressions(meanLatency, 5);\n",
    "        int meanLatencyCountGT_10 = GetCountOfRegressions(meanLatency, 10);\n",
    "\n",
    "        List<double> p50Latency  = comparison.Select(a => a.Value.Latency50thMS);\n",
    "        int p50LatencyCountGT_5  = GetCountOfRegressions(p50Latency, 5);\n",
    "        int p50LatencyCountGT_10 = GetCountOfRegressions(p50Latency, 10);\n",
    "\n",
    "        List<double> p75Latency  = comparison.Select(a => a.Value.Latency75thMS);\n",
    "        int p75LatencyCountGT_5  = GetCountOfRegressions(p75Latency, 5);\n",
    "        int p75LatencyCountGT_10 = GetCountOfRegressions(p75Latency, 10);\n",
    "\n",
    "        List<double> p90Latency  = comparison.Select(a => a.Value.Latency90thMS);\n",
    "        int p90LatencyCountGT_5  = GetCountOfRegressions(p90Latency, 5);\n",
    "        int p90LatencyCountGT_10 = GetCountOfRegressions(p90Latency, 10);\n",
    "        \n",
    "        List<double> p99Latency  = comparison.Select(a => a.Value.Latency99thMS);\n",
    "        int p99LatencyCountGT_5  = GetCountOfRegressions(p99Latency, 5);\n",
    "        int p99LatencyCountGT_10 = GetCountOfRegressions(p99Latency, 10);\n",
    "\n",
    "        return $\"{comparisonKey},{workingSetCountGT_5},{workingSetCountGT_10},{privateMemoryCountGT_5},{privateMemoryCountGT_10},{throughputCountGT_5},{throughputCountGT_10},{meanLatencyCountGT_5},{meanLatencyCountGT_10},{p50LatencyCountGT_5},{p50LatencyCountGT_10},{p75LatencyCountGT_5},{p75LatencyCountGT_10},{p90LatencyCountGT_5},{p90LatencyCountGT_10},{p99LatencyCountGT_5},{p99LatencyCountGT_10}\";\n",
    "    }\n",
    "\n",
    "    public string GenerateRegressionSummaryForComparisons()\n",
    "    {\n",
    "        StringBuilder sb = new();\n",
    "        sb.AppendLine(\"Build to Build,Reg. Count - Working Set (MB),Large Reg. Count - Working Set (MB),Reg. Count - Max Private Memory (MB),Large Reg. Count - Max Private Memory (MB),Reg. Count - Throughput, Large Reg. Count - Throughput,Reg. Count - Mean Latency,Large Reg. Count - Mean Latency,Reg. Count - P50 Latency, Large Reg. Count - P50 Latency, Reg. Count - P75 Latency, Large Reg. Count - P75 Latency,Reg. Count - P90 Latency, Large Reg. Count - P90 Latency,Reg. Count - P99 Latency, Large Reg. Count - P99 Latency\");\n",
    "        foreach (var comparison in Comparisons)\n",
    "        {\n",
    "            sb.AppendLine(GenerateRegressionSummary(comparison.Key, comparison.Value));\n",
    "        }\n",
    "\n",
    "        return sb.ToString();\n",
    "    }\n",
    "\n",
    "    public Dictionary<string, string> GenerateRegressionAnalysisForComparison(string comparisonKey)\n",
    "    {\n",
    "        StringBuilder sb = new();\n",
    "        Dictionary<string, string> csvData = new();\n",
    "        Dictionary<string, LoadInfo> comparison = Comparisons[comparisonKey];\n",
    "\n",
    "        string header = \"Benchmark,MaxWorkingSetMB,MaxPrivateMemoryMB,RequestsPerMSec,MeanLatencyMS,Latency50thMS,Latency75thMS,Latency90thMS,Latency99thMS\";\n",
    "\n",
    "        // Generate Memory Regressions.\n",
    "        StringBuilder memRegressions = new();\n",
    "        memRegressions.AppendLine(header);\n",
    "        foreach (var benchmark in comparison.Where(c => c.Value.MaxWorkingSetMB >= 10 || c.Value.MaxPrivateMemoryMB >= 10 ))\n",
    "        {\n",
    "            memRegressions.AppendLine($\"{benchmark.Key},{benchmark.Value.MaxWorkingSetMB},{benchmark.Value.MaxPrivateMemoryMB},{benchmark.Value.RequestsPerMSec},{benchmark.Value.MeanLatencyMS},{benchmark.Value.Latency50thMS},{benchmark.Value.Latency75thMS},{benchmark.Value.Latency90thMS},{benchmark.Value.Latency99thMS}\");\n",
    "        }\n",
    "        csvData[\"memory\"] = memRegressions.ToString();\n",
    "\n",
    "        // Generate Throughput Regressions.\n",
    "        StringBuilder throughputRegressions = new();\n",
    "        throughputRegressions.AppendLine(header);\n",
    "        foreach (var benchmark in comparison.Where(c => c.Value.RequestsPerMSec <= -10))\n",
    "        {\n",
    "            throughputRegressions.AppendLine($\"{benchmark.Key},{benchmark.Value.MaxWorkingSetMB},{benchmark.Value.MaxPrivateMemoryMB},{benchmark.Value.RequestsPerMSec},{benchmark.Value.MeanLatencyMS},{benchmark.Value.Latency50thMS},{benchmark.Value.Latency75thMS},{benchmark.Value.Latency90thMS},{benchmark.Value.Latency99thMS}\");\n",
    "        }\n",
    "        csvData[\"throughput\"] = throughputRegressions.ToString();\n",
    "\n",
    "        // Generate Latency Regressions.\n",
    "        StringBuilder latencyRegressions = new();\n",
    "        latencyRegressions.AppendLine(header);\n",
    "        foreach (var benchmark in comparison.Where(c => c.Value.MeanLatencyMS >= 10 || \n",
    "                                                        c.Value.Latency50thMS >= 10 || \n",
    "                                                        c.Value.Latency75thMS >= 10 || \n",
    "                                                        c.Value.Latency90thMS >= 10 || \n",
    "                                                        c.Value.Latency99thMS >= 10 ))\n",
    "        {\n",
    "            latencyRegressions.AppendLine($\"{benchmark.Key},{benchmark.Value.MaxWorkingSetMB},{benchmark.Value.MaxPrivateMemoryMB},{benchmark.Value.RequestsPerMSec},{benchmark.Value.MeanLatencyMS},{benchmark.Value.Latency50thMS},{benchmark.Value.Latency75thMS},{benchmark.Value.Latency90thMS},{benchmark.Value.Latency99thMS}\");\n",
    "        }\n",
    "        csvData[\"latency\"] = latencyRegressions.ToString();\n",
    "\n",
    "        // All.\n",
    "        StringBuilder all = new();\n",
    "        all.AppendLine(header);\n",
    "        foreach (var benchmark in comparison)\n",
    "        {\n",
    "            all.AppendLine($\"{benchmark.Key},{benchmark.Value.MaxWorkingSetMB},{benchmark.Value.MaxPrivateMemoryMB},{benchmark.Value.RequestsPerMSec},{benchmark.Value.MeanLatencyMS},{benchmark.Value.Latency50thMS},{benchmark.Value.Latency75thMS},{benchmark.Value.Latency90thMS},{benchmark.Value.Latency99thMS}\");\n",
    "        }\n",
    "        csvData[\"all\"] = all.ToString();\n",
    "\n",
    "        return csvData;\n",
    "    }\n",
    "\n",
    "    public void SaveComparisons(string basePath)\n",
    "    {\n",
    "        // Add Summary for Comparisons.\n",
    "        string summaryOfComparisons = GenerateSummaryForComparisons();\n",
    "        File.WriteAllText(Path.Combine(basePath, \"SummaryOfComparisons.csv\"), summaryOfComparisons);\n",
    "\n",
    "        // Add Regression Summary for Comparisons.\n",
    "        string regressionSummary = GenerateRegressionSummaryForComparisons();\n",
    "        File.WriteAllText(Path.Combine(basePath, \"RegressionSummary.csv\"), regressionSummary);\n",
    "\n",
    "        // Add Large Regression Analysis for Comparison.\n",
    "        string perComparisonDataPath = Path.Combine(basePath, \"PerComparisonData\");\n",
    "        if (!Directory.Exists(perComparisonDataPath))\n",
    "        {\n",
    "            Directory.CreateDirectory(perComparisonDataPath);\n",
    "        }\n",
    "\n",
    "        foreach (var comparison in Comparisons)\n",
    "        {\n",
    "            string comparisonPath = Path.Combine(perComparisonDataPath, comparison.Key);\n",
    "            Directory.CreateDirectory(comparisonPath);\n",
    "\n",
    "            Dictionary<string, string> regressionComparisons = GenerateRegressionAnalysisForComparison(comparison.Key);\n",
    "\n",
    "            // Memory\n",
    "            File.WriteAllText(Path.Combine(comparisonPath, \"MemoryRegressions.csv\"), regressionComparisons[\"memory\"]);\n",
    "\n",
    "            // Throughput\n",
    "            File.WriteAllText(Path.Combine(comparisonPath, \"ThroughputRegressions.csv\"), regressionComparisons[\"throughput\"]);\n",
    "\n",
    "            // Latency\n",
    "            File.WriteAllText(Path.Combine(comparisonPath, \"LatencyRegressions.csv\"), regressionComparisons[\"latency\"]);\n",
    "\n",
    "            // All\n",
    "            File.WriteAllText(Path.Combine(comparisonPath, \"All.csv\"), regressionComparisons[\"all\"]);\n",
    "        }\n",
    "    }\n",
    "\n",
    "    public Dictionary<string, Dictionary<string, LoadInfo>> Comparisons { get; }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "var datas3_vs_datas_4 = dataManager.GetBenchmarkToComparison(\"datas_3\", \"datas_4\");\n",
    "\n",
    "Dictionary<string, Dictionary<string, LoadInfo>> comparisons = new()\n",
    "{\n",
    "    { nameof(run1_vs_run2), run1_vs_run2 },\n",
    "};\n",
    "\n",
    "SummaryTable summaryTable = new(comparisons);\n",
    "summaryTable.SaveComparisons(\"./\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Charting Helpers\n",
    "\n",
    "The following cells highlight how to chart certain properties of the LoadInfo class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "void ChartProperty(LoadInfo baseline, LoadInfo comparand, string nameOfProperty)\n",
    "{\n",
    "    GCProcessData baselineGC = baseline.Data;\n",
    "    GCProcessData comparandGC = comparand.Data;\n",
    "\n",
    "    List<(string scatterName, List<TraceGC> gcs)> gcData = \n",
    "        new()\n",
    "        {\n",
    "            { ( scatterName :  $\"{nameOfProperty} for {baseline.Id}\" , gcs : baselineGC.GCs )},\n",
    "            { ( scatterName :  $\"{nameOfProperty} for {comparand.Id}\" , gcs : comparandGC.GCs )}\n",
    "        };\n",
    "\n",
    "    GCCharting.ChartGCData(gcData          : gcData, \n",
    "                           title           : $\"{nameOfProperty} Comparison Between {baseline.Run} and {comparand.Run}\", \n",
    "                           isXAxisRelative : false,\n",
    "                           fieldName       : nameOfProperty).Display();\n",
    "\n",
    "}\n",
    "\n",
    "void ChartProperty(LoadInfo comparison, string nameOfProperty)\n",
    "{\n",
    "    GCProcessData baselineGC = comparison.Data;\n",
    "    GCProcessData comparandGC = comparison.Data2;\n",
    "\n",
    "    List<(string scatterName, List<TraceGC> gcs)> gcData = \n",
    "        new()\n",
    "        {\n",
    "            { ( scatterName :  $\"{nameOfProperty} for Baseline\" , gcs : baselineGC.GCs )},\n",
    "            { ( scatterName :  $\"{nameOfProperty} for Comparand\" , gcs : comparandGC.GCs )}\n",
    "        };\n",
    "\n",
    "    GCCharting.ChartGCData(gcData          : gcData, \n",
    "                           title           : $\"{nameOfProperty} Comparison\", \n",
    "                           isXAxisRelative : false,\n",
    "                           fieldName       : nameOfProperty).Display();\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "dataManager.Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "var run1_Benchmark = dataManager.GetBenchmarkData(benchmark: \"CachingPlatform\", \"datas_2\");\n",
    "var run2_Benchmark = dataManager.GetBenchmarkData(benchmark: \"CachingPlatform\", \"datas_3\");\n",
    "\n",
    "// Chart the PauseDurationMSec for the run1 vs. run2.\n",
    "ChartProperty(baseline: run1_Benchmark, comparand: run2_Benchmark, nameof(TraceGC.PauseDurationMSec))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Debugging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "System.Diagnostics.Process.GetCurrentProcess().Id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "#!about"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".NET (C#)",
   "language": "C#",
   "name": ".net-csharp"
  },
  "language_info": {
   "name": "python"
  },
  "orig_nbformat": 4,
  "polyglot_notebook": {
   "kernelInfo": {
    "defaultKernelName": "csharp",
    "items": [
     {
      "aliases": [],
      "name": "csharp"
     }
    ]
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}