codebleu_results.json

{"CodeLlama 7B": {"HumanEval Plus": {"4 bit": 0.24718391889230173, "8 bit": 0.2398257390575718, "unquantized": 0.23998698384844577}, "MBPP Plus": {"4 bit": 0.23834239909086147, "8 bit": 0.23639333221348105, "unquantized": 0.24125355404601825}}, "Mistral 7B Instruct": {"HumanEval Plus": {"4 bit": 0.24428412566846264, "8 bit": 0.24309682360519724, "unquantized": 0.2627925737727571}, "MBPP Plus": {"4 bit": 0.21338238380018784, "8 bit": 0.21340086053135737, "unquantized": 0.2442849065786985}}, "WizardCoder Python 7B": {"HumanEval Plus": {"4 bit": 0.2872375754841764, "8 bit": 0.2837118700741148, "unquantized": 0.2797887415094741}, "MBPP Plus": {"4 bit": 0.26260912133000835, "8 bit": 0.26033740202138417, "unquantized": 0.2593165745414837}}, "starcoder2 7b": {"HumanEval Plus": {"4 bit": 0.20360700816730792, "8 bit": 0.21270912264762704, "unquantized": 0.1882340670543336}, "MBPP Plus": {"4 bit": 0.257308277838058, "8 bit": 0.25871528061967686, "unquantized": 0.25768152469396155}}}