honey-comb/docs/statistical_validation.json at master · DJLougen/honey-comb · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
{
  "accuracy": {
    "metric": "Classification Accuracy",
    "n_samples": 273,
    "mean": 84.24908424908425,
    "std": 36.494916158395505,
    "ci_lower": 79.85347985347985,
    "ci_upper": 88.27838827838828,
    "p_value": 0.0,
    "baseline": 25.0,
    "effect_size": 1.4161960742638573,
    "significant": true
  },
  "compression": {
    "metric": "Compression Ratio",
    "n_samples": 100,
    "mean": 13.68739845779851,
    "std": 7.7117213846741,
    "ci_lower": 12.246868783862004,
    "ci_upper": 15.266635301512332,
    "p_value": 0.0,
    "baseline": 1.0,
    "effect_size": 2.3266778032085775,
    "significant": true
  },
  "throughput_rule": {
    "metric": "Rule-based Throughput",
    "n_samples": 100,
    "mean": 16162.517472062165,
    "std": 572.2563615787285,
    "ci_lower": 16048.21771920757,
    "ci_upper": 16274.576227734375,
    "p_value": null,
    "baseline": null,
    "effect_size": null,
    "significant": null
  },
  "token_savings": {
    "metric": "Token Savings",
    "n_samples": 100,
    "mean": 3103.44,
    "std": 1487.3050084911392,
    "ci_lower": 2814.93975,
    "ci_upper": 3398.2217499999997,
    "p_value": 0.0,
    "baseline": 0,
    "effect_size": null,
    "significant": true
  },
  "throughput_ml": {
    "metric": "ML-based Throughput",
    "n_samples": 100,
    "mean": 1190.463488485791,
    "std": 33.96065233782735,
    "ci_lower": 1183.843214349166,
    "ci_upper": 1196.9286021460362,
    "p_value": null,
    "baseline": null,
    "effect_size": null,
    "significant": null
  }
}