all_benchmark_results_plus.json

{"CodeLlama 7B": {"HumanEval Plus": {"4 bit": 0.036585365853658534, "8 bit": 0.024390243902439025, "unquantized": 0.036585365853658534}, "MBPP Plus": {"4 bit": 0.12030075187969924, "8 bit": 0.11278195488721804, "unquantized": 0.11027568922305764}}, "Mistral 7B Instruct": {"HumanEval Plus": {"4 bit": 0.1402439024390244, "8 bit": 0.14634146341463414, "unquantized": 0.25609756097560976}, "MBPP Plus": {"4 bit": 0.3408521303258145, "8 bit": 0.3408521303258145, "unquantized": 0.2756892230576441}}, "WizardCoder Python 7B": {"HumanEval Plus": {"4 bit": 0.21341463414634146, "8 bit": 0.1951219512195122, "unquantized": 0.27439024390243905}, "MBPP Plus": {"4 bit": 0.2606516290726817, "8 bit": 0.24812030075187969, "unquantized": 0.2832080200501253}}, "starcoder2 7b": {"HumanEval Plus": {"4 bit": 0.012195121951219513, "8 bit": 0.0, "unquantized": 0.006097560975609756}, "MBPP Plus": {"4 bit": 0.010025062656641603, "8 bit": 0.007518796992481203, "unquantized": 0.010025062656641603}}}