assignment2-systems/benchmark_results.json at main · yinlin09/assignment2-systems · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
[
  {
    "size": "small",
    "context_length": 128,
    "n_params": 128625408,
    "forward_mean_ms": 14.730034400054137,
    "forward_std_ms": 5.260893373690511,
    "backward_mean_ms": 11.966589100302372,
    "backward_std_ms": 0.3510218398399944
  },
  {
    "size": "small",
    "context_length": 256,
    "n_params": 128625408,
    "forward_mean_ms": 13.864526900033525,
    "forward_std_ms": 7.36269535423986,
    "backward_mean_ms": 21.425170300062746,
    "backward_std_ms": 0.6114865790766699
  },
  {
    "size": "small",
    "context_length": 512,
    "n_params": 128625408,
    "forward_mean_ms": 19.587802000023657,
    "forward_std_ms": 13.197953055577223,
    "backward_mean_ms": 42.82014420023188,
    "backward_std_ms": 0.33003835457292346
  },
  {
    "size": "medium",
    "context_length": 128,
    "n_params": 423183360,
    "forward_mean_ms": 10.994495600243681,
    "forward_std_ms": 0.561863625382259,
    "backward_mean_ms": 29.848582099839405,
    "backward_std_ms": 0.6621328540110509
  },
  {
    "size": "medium",
    "context_length": 256,
    "n_params": 423183360,
    "forward_mean_ms": 18.74238369982777,
    "forward_std_ms": 0.7976030221138545,
    "backward_mean_ms": 53.74204790023214,
    "backward_std_ms": 0.5951216536762237
  },
  {
    "size": "medium",
    "context_length": 512,
    "n_params": 423183360,
    "forward_mean_ms": 39.09303490017919,
    "forward_std_ms": 0.9515135131390929,
    "backward_mean_ms": 121.90063360030763,
    "backward_std_ms": 0.5736189103102263
  },
  {
    "size": "large",
    "context_length": 128,
    "n_params": 969411840,
    "forward_mean_ms": 21.692813099798514,
    "forward_std_ms": 0.5967471673542315,
    "backward_mean_ms": 69.39642349971109,
    "backward_std_ms": 0.9387306883261599
  },
  {
    "size": "large",
    "context_length": 256,
    "n_params": 969411840,
    "forward_mean_ms": 39.98736789999384,
    "forward_std_ms": 1.801182523024521,
    "backward_mean_ms": 131.77685640002892,
    "backward_std_ms": 0.5136215215612505
  },
  {
    "size": "large",
    "context_length": 512,
    "n_params": 969411840,
    "forward_mean_ms": 79.80461690021912,
    "forward_std_ms": 0.6922161013029009,
    "backward_mean_ms": 288.0667237997841,
    "backward_std_ms": 0.3927526787499487
  },
  {
    "size": "xl",
    "context_length": 128,
    "n_params": 1998235200,
    "forward_mean_ms": 40.58821269954933,
    "forward_std_ms": 1.107729008861389,
    "backward_mean_ms": 171.55337910007802,
    "backward_std_ms": 0.2987124306700213
  },
  {
    "size": "xl",
    "context_length": 256,
    "error": "RESOURCE_EXHAUSTED: Out of memory while trying to allocate 3.38GiB."
  },
  {
    "size": "xl",
    "context_length": 512,
    "error": "RESOURCE_EXHAUSTED: Out of memory while trying to allocate 39.06MiB."
  },
  {
    "size": "2.7B",
    "context_length": 128,
    "error": "RESOURCE_EXHAUSTED: Out of memory while trying to allocate 100.00MiB."
  },
  {
    "size": "2.7B",
    "context_length": 256,
    "error": "RESOURCE_EXHAUSTED: Out of memory while trying to allocate 100.00MiB."
  },
  {
    "size": "2.7B",
    "context_length": 512,
    "error": "RESOURCE_EXHAUSTED: Out of memory while trying to allocate 100.00MiB."
  }
]