diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AFQMC.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AFQMC.json new file mode 100644 index 0000000000..e16ff9c68d --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AFQMC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b44c3b011063adb25877c13823db83bb193913c4", + "task_name": "AFQMC", + "mteb_version": "1.36.5", + "scores": { + "validation": [ + { + "pearson": 0.531772, + "spearman": 0.566464, + "cosine_pearson": 0.531772, + "cosine_spearman": 0.566464, + "manhattan_pearson": 0.54362, + "manhattan_spearman": 0.564367, + "euclidean_pearson": 0.546235, + "euclidean_spearman": 0.566623, + "main_score": 0.566464, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 11.063741445541382, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ATEC.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ATEC.json new file mode 100644 index 0000000000..f30f073255 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ATEC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865", + "task_name": "ATEC", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.513083, + "spearman": 0.542766, + "cosine_pearson": 0.513083, + "cosine_spearman": 0.542766, + "manhattan_pearson": 0.554734, + "manhattan_spearman": 0.541402, + "euclidean_pearson": 0.556633, + "euclidean_spearman": 0.542737, + "main_score": 0.542766, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 58.03469467163086, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AmazonCounterfactualClassification.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AmazonCounterfactualClassification.json new file mode 100644 index 0000000000..8ff7714ba8 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AmazonCounterfactualClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", + "task_name": "AmazonCounterfactualClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.921642, + "f1": 0.887001, + "f1_weighted": 0.925097, + "ap": 0.702104, + "ap_weighted": 0.702104, + "scores_per_experiment": [ + { + "accuracy": 0.91791, + "f1": 0.88217, + "f1_weighted": 0.921688, + "ap": 0.691285, + "ap_weighted": 0.691285 + }, + { + "accuracy": 0.925373, + "f1": 0.891594, + "f1_weighted": 0.928444, + "ap": 0.711871, + "ap_weighted": 0.711871 + }, + { + "accuracy": 0.928358, + "f1": 0.895423, + "f1_weighted": 0.931161, + "ap": 0.720459, + "ap_weighted": 0.720459 + }, + { + "accuracy": 0.91791, + "f1": 0.88217, + "f1_weighted": 0.921688, + "ap": 0.691285, + "ap_weighted": 0.691285 + }, + { + "accuracy": 0.91791, + "f1": 0.88217, + "f1_weighted": 0.921688, + "ap": 0.691285, + "ap_weighted": 0.691285 + }, + { + "accuracy": 0.928358, + "f1": 0.895423, + "f1_weighted": 0.931161, + "ap": 0.720459, + "ap_weighted": 0.720459 + }, + { + "accuracy": 0.928358, + "f1": 0.895423, + "f1_weighted": 0.931161, + "ap": 0.720459, + "ap_weighted": 0.720459 + }, + { + "accuracy": 0.928358, + "f1": 0.895423, + "f1_weighted": 0.931161, + "ap": 0.720459, + "ap_weighted": 0.720459 + }, + { + "accuracy": 0.916418, + "f1": 0.88031, + "f1_weighted": 0.920343, + "ap": 0.687312, + "ap_weighted": 0.687312 + }, + { + "accuracy": 0.907463, + "f1": 0.869903, + "f1_weighted": 0.912471, + "ap": 0.666169, + "ap_weighted": 0.666169 + } + ], + "main_score": 0.921642, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.491443872451782, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AmazonReviewsClassification.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AmazonReviewsClassification.json new file mode 100644 index 0000000000..cb268ef548 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AmazonReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", + "task_name": "AmazonReviewsClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.58168, + "f1": 0.565522, + "f1_weighted": 0.565522, + "scores_per_experiment": [ + { + "accuracy": 0.5866, + "f1": 0.579767, + "f1_weighted": 0.579767 + }, + { + "accuracy": 0.5576, + "f1": 0.528684, + "f1_weighted": 0.528684 + }, + { + "accuracy": 0.5782, + "f1": 0.555548, + "f1_weighted": 0.555548 + }, + { + "accuracy": 0.599, + "f1": 0.589227, + "f1_weighted": 0.589227 + }, + { + "accuracy": 0.5782, + "f1": 0.555265, + "f1_weighted": 0.555265 + }, + { + "accuracy": 0.5816, + "f1": 0.55776, + "f1_weighted": 0.55776 + }, + { + "accuracy": 0.5842, + "f1": 0.577233, + "f1_weighted": 0.577233 + }, + { + "accuracy": 0.5798, + "f1": 0.566306, + "f1_weighted": 0.566306 + }, + { + "accuracy": 0.591, + "f1": 0.581925, + "f1_weighted": 0.581925 + }, + { + "accuracy": 0.5806, + "f1": 0.563507, + "f1_weighted": 0.563507 + } + ], + "main_score": 0.58168, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 18.467613220214844, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArXivHierarchicalClusteringP2P.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArXivHierarchicalClusteringP2P.json new file mode 100644 index 0000000000..a6669e2e23 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArXivHierarchicalClusteringP2P.json @@ -0,0 +1,46 @@ +{ + "dataset_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", + "task_name": "ArXivHierarchicalClusteringP2P", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.670622, + 0.650928, + 0.68815, + 0.694634, + 0.697376, + 0.688693, + 0.701147, + 0.689221, + 0.714288, + 0.67648 + ], + "Level 1": [ + 0.587947, + 0.616887, + 0.597128, + 0.599237, + 0.60791, + 0.618341, + 0.608314, + 0.619125, + 0.588229, + 0.635191 + ] + }, + "v_measure": 0.647493, + "v_measure_std": 0.042601, + "main_score": 0.647493, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.10978651046753, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArXivHierarchicalClusteringS2S.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArXivHierarchicalClusteringS2S.json new file mode 100644 index 0000000000..fc6101a6a1 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArXivHierarchicalClusteringS2S.json @@ -0,0 +1,46 @@ +{ + "dataset_revision": "b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3", + "task_name": "ArXivHierarchicalClusteringS2S", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.654192, + 0.708699, + 0.736566, + 0.714094, + 0.702885, + 0.686076, + 0.678604, + 0.653956, + 0.677153, + 0.724612 + ], + "Level 1": [ + 0.573281, + 0.618247, + 0.590201, + 0.587791, + 0.580318, + 0.620178, + 0.58259, + 0.602671, + 0.595175, + 0.589441 + ] + }, + "v_measure": 0.643837, + "v_measure_std": 0.054352, + "main_score": 0.643837, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.936519145965576, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArguAna.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArguAna.json new file mode 100644 index 0000000000..e3a7c3f656 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArguAna.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a", + "task_name": "ArguAna", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.58393, + "ndcg_at_3": 0.72526, + "ndcg_at_5": 0.76031, + "ndcg_at_10": 0.782, + "ndcg_at_20": 0.78603, + "ndcg_at_100": 0.78717, + "ndcg_at_1000": 0.78717, + "map_at_1": 0.58393, + "map_at_3": 0.69097, + "map_at_5": 0.71053, + "map_at_10": 0.71966, + "map_at_20": 0.7208, + "map_at_100": 0.72099, + "map_at_1000": 0.72099, + "recall_at_1": 0.58393, + "recall_at_3": 0.82432, + "recall_at_5": 0.90896, + "recall_at_10": 0.97511, + "recall_at_20": 0.99075, + "recall_at_100": 0.99644, + "recall_at_1000": 0.99644, + "precision_at_1": 0.58393, + "precision_at_3": 0.27477, + "precision_at_5": 0.18179, + "precision_at_10": 0.09751, + "precision_at_20": 0.04954, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "mrr_at_1": 0.583926, + "mrr_at_3": 0.690967, + "mrr_at_5": 0.710526, + "mrr_at_10": 0.719657, + "mrr_at_20": 0.720803, + "mrr_at_100": 0.72099, + "mrr_at_1000": 0.72099, + "nauc_ndcg_at_1_max": -0.106041, + "nauc_ndcg_at_1_std": -0.243721, + "nauc_ndcg_at_1_diff1": 0.233751, + "nauc_ndcg_at_3_max": -0.084297, + "nauc_ndcg_at_3_std": -0.245363, + "nauc_ndcg_at_3_diff1": 0.195642, + "nauc_ndcg_at_5_max": -0.079444, + "nauc_ndcg_at_5_std": -0.243597, + "nauc_ndcg_at_5_diff1": 0.192894, + "nauc_ndcg_at_10_max": -0.087187, + "nauc_ndcg_at_10_std": -0.239662, + "nauc_ndcg_at_10_diff1": 0.201041, + "nauc_ndcg_at_20_max": -0.089242, + "nauc_ndcg_at_20_std": -0.232867, + "nauc_ndcg_at_20_diff1": 0.200873, + "nauc_ndcg_at_100_max": -0.091846, + "nauc_ndcg_at_100_std": -0.238979, + "nauc_ndcg_at_100_diff1": 0.201345, + "nauc_ndcg_at_1000_max": -0.091846, + "nauc_ndcg_at_1000_std": -0.238979, + "nauc_ndcg_at_1000_diff1": 0.201345, + "nauc_map_at_1_max": -0.106041, + "nauc_map_at_1_std": -0.243721, + "nauc_map_at_1_diff1": 0.233751, + "nauc_map_at_3_max": -0.093325, + "nauc_map_at_3_std": -0.245767, + "nauc_map_at_3_diff1": 0.201947, + "nauc_map_at_5_max": -0.091879, + "nauc_map_at_5_std": -0.244779, + "nauc_map_at_5_diff1": 0.200793, + "nauc_map_at_10_max": -0.095518, + "nauc_map_at_10_std": -0.244173, + "nauc_map_at_10_diff1": 0.20364, + "nauc_map_at_20_max": -0.095974, + "nauc_map_at_20_std": -0.242624, + "nauc_map_at_20_diff1": 0.203519, + "nauc_map_at_100_max": -0.096244, + "nauc_map_at_100_std": -0.243379, + "nauc_map_at_100_diff1": 0.203621, + "nauc_map_at_1000_max": -0.096244, + "nauc_map_at_1000_std": -0.243379, + "nauc_map_at_1000_diff1": 0.203621, + "nauc_recall_at_1_max": -0.106041, + "nauc_recall_at_1_std": -0.243721, + "nauc_recall_at_1_diff1": 0.233751, + "nauc_recall_at_3_max": -0.043013, + "nauc_recall_at_3_std": -0.243011, + "nauc_recall_at_3_diff1": 0.171221, + "nauc_recall_at_5_max": 0.018641, + "nauc_recall_at_5_std": -0.233678, + "nauc_recall_at_5_diff1": 0.139987, + "nauc_recall_at_10_max": 0.138292, + "nauc_recall_at_10_std": -0.089752, + "nauc_recall_at_10_diff1": 0.185019, + "nauc_recall_at_20_max": 0.361786, + "nauc_recall_at_20_std": 0.644488, + "nauc_recall_at_20_diff1": 0.162924, + "nauc_recall_at_100_max": 0.396703, + "nauc_recall_at_100_std": 0.539252, + "nauc_recall_at_100_diff1": 0.179148, + "nauc_recall_at_1000_max": 0.396703, + "nauc_recall_at_1000_std": 0.539252, + "nauc_recall_at_1000_diff1": 0.179148, + "nauc_precision_at_1_max": -0.106041, + "nauc_precision_at_1_std": -0.243721, + "nauc_precision_at_1_diff1": 0.233751, + "nauc_precision_at_3_max": -0.043013, + "nauc_precision_at_3_std": -0.243011, + "nauc_precision_at_3_diff1": 0.171221, + "nauc_precision_at_5_max": 0.018641, + "nauc_precision_at_5_std": -0.233678, + "nauc_precision_at_5_diff1": 0.139987, + "nauc_precision_at_10_max": 0.138292, + "nauc_precision_at_10_std": -0.089752, + "nauc_precision_at_10_diff1": 0.185019, + "nauc_precision_at_20_max": 0.361786, + "nauc_precision_at_20_std": 0.644488, + "nauc_precision_at_20_diff1": 0.162924, + "nauc_precision_at_100_max": 0.396703, + "nauc_precision_at_100_std": 0.539252, + "nauc_precision_at_100_diff1": 0.179148, + "nauc_precision_at_1000_max": 0.396703, + "nauc_precision_at_1000_std": 0.539252, + "nauc_precision_at_1000_diff1": 0.179148, + "nauc_mrr_at_1_max": -0.106041, + "nauc_mrr_at_1_std": -0.243721, + "nauc_mrr_at_1_diff1": 0.233751, + "nauc_mrr_at_3_max": -0.093325, + "nauc_mrr_at_3_std": -0.245767, + "nauc_mrr_at_3_diff1": 0.201947, + "nauc_mrr_at_5_max": -0.091879, + "nauc_mrr_at_5_std": -0.244779, + "nauc_mrr_at_5_diff1": 0.200793, + "nauc_mrr_at_10_max": -0.095518, + "nauc_mrr_at_10_std": -0.244173, + "nauc_mrr_at_10_diff1": 0.20364, + "nauc_mrr_at_20_max": -0.095974, + "nauc_mrr_at_20_std": -0.242624, + "nauc_mrr_at_20_diff1": 0.203519, + "nauc_mrr_at_100_max": -0.096244, + "nauc_mrr_at_100_std": -0.243379, + "nauc_mrr_at_100_diff1": 0.203621, + "nauc_mrr_at_1000_max": -0.096244, + "nauc_mrr_at_1000_std": -0.243379, + "nauc_mrr_at_1000_diff1": 0.203621, + "main_score": 0.782, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 592.4430773258209, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AskUbuntuDupQuestions.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AskUbuntuDupQuestions.json new file mode 100644 index 0000000000..c081b2097e --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AskUbuntuDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2000358ca161889fa9c082cb41daa8dcfb161a54", + "task_name": "AskUbuntuDupQuestions", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "map": 0.686923, + "mrr": 0.816614, + "nAUC_map_max": 0.2913, + "nAUC_map_std": 0.195983, + "nAUC_map_diff1": 0.074254, + "nAUC_mrr_max": 0.446855, + "nAUC_mrr_std": 0.296643, + "nAUC_mrr_diff1": 0.101196, + "main_score": 0.686923, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 281.7872676849365, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BIOSSES.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BIOSSES.json new file mode 100644 index 0000000000..e98785ff22 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BIOSSES.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a", + "task_name": "BIOSSES", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.849657, + "spearman": 0.848007, + "cosine_pearson": 0.849657, + "cosine_spearman": 0.848007, + "manhattan_pearson": 0.840541, + "manhattan_spearman": 0.851447, + "euclidean_pearson": 0.840205, + "euclidean_spearman": 0.849682, + "main_score": 0.848007, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.3601527214050293, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BQ.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BQ.json new file mode 100644 index 0000000000..a5848864cd --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BQ.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55", + "task_name": "BQ", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.674468, + "spearman": 0.697282, + "cosine_pearson": 0.674468, + "cosine_spearman": 0.697282, + "manhattan_pearson": 0.67003, + "manhattan_spearman": 0.69487, + "euclidean_pearson": 0.673158, + "euclidean_spearman": 0.69745, + "main_score": 0.697282, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 22.85646677017212, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Banking77Classification.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Banking77Classification.json new file mode 100644 index 0000000000..f29c32e94c --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Banking77Classification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "0fd18e25b25c072e09e0d92ab615fda904d66300", + "task_name": "Banking77Classification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.907597, + "f1": 0.90586, + "f1_weighted": 0.90586, + "scores_per_experiment": [ + { + "accuracy": 0.91461, + "f1": 0.91306, + "f1_weighted": 0.91306 + }, + { + "accuracy": 0.909416, + "f1": 0.908145, + "f1_weighted": 0.908145 + }, + { + "accuracy": 0.911688, + "f1": 0.909924, + "f1_weighted": 0.909924 + }, + { + "accuracy": 0.902597, + "f1": 0.901141, + "f1_weighted": 0.901141 + }, + { + "accuracy": 0.909091, + "f1": 0.90795, + "f1_weighted": 0.90795 + }, + { + "accuracy": 0.908442, + "f1": 0.907355, + "f1_weighted": 0.907355 + }, + { + "accuracy": 0.908442, + "f1": 0.907267, + "f1_weighted": 0.907267 + }, + { + "accuracy": 0.896429, + "f1": 0.893008, + "f1_weighted": 0.893008 + }, + { + "accuracy": 0.903896, + "f1": 0.900223, + "f1_weighted": 0.900223 + }, + { + "accuracy": 0.911364, + "f1": 0.910529, + "f1_weighted": 0.910529 + } + ], + "main_score": 0.907597, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 35.1525981426239, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BiorxivClusteringP2P.v2.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BiorxivClusteringP2P.v2.json new file mode 100644 index 0000000000..9f845e5243 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BiorxivClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "f5dbc242e11dd8e24def4c4268607a49e02946dc", + "task_name": "BiorxivClusteringP2P.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.557888, + 0.54971, + 0.556983, + 0.555511, + 0.557293, + 0.556872, + 0.552489, + 0.548273, + 0.55031, + 0.544084 + ] + }, + "v_measure": 0.552941, + "v_measure_std": 0.004468, + "main_score": 0.552941, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 17.046616077423096, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CLSClusteringP2P.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CLSClusteringP2P.json new file mode 100644 index 0000000000..8ad8bd58f5 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CLSClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476", + "task_name": "CLSClusteringP2P", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measure": 0.544964, + "v_measure_std": 0.012841, + "v_measures": [ + 0.53674, + 0.540984, + 0.557885, + 0.557414, + 0.569621, + 0.546623, + 0.54454, + 0.528693, + 0.526219, + 0.540917 + ], + "main_score": 0.544964, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 243.35979628562927, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CLSClusteringS2S.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CLSClusteringS2S.json new file mode 100644 index 0000000000..a29b6df283 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CLSClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f", + "task_name": "CLSClusteringS2S", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measure": 0.615191, + "v_measure_std": 0.007345, + "v_measures": [ + 0.615635, + 0.61215, + 0.621993, + 0.620515, + 0.620063, + 0.599862, + 0.621689, + 0.605452, + 0.622321, + 0.612228 + ], + "main_score": 0.615191, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 114.00036978721619, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CMedQAv1-reranking.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CMedQAv1-reranking.json new file mode 100644 index 0000000000..488ec866e0 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CMedQAv1-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "8d7f1e942507dac42dc58017c1a001c3717da7df", + "task_name": "CMedQAv1-reranking", + "mteb_version": "1.38.1", + "scores": { + "test": [ + { + "map": 0.822074, + "mrr": 0.848177, + "nAUC_map_max": 0.24092, + "nAUC_map_std": -0.181179, + "nAUC_map_diff1": 0.641521, + "nAUC_mrr_max": 0.292732, + "nAUC_mrr_std": -0.155614, + "nAUC_mrr_diff1": 0.719898, + "main_score": 0.822074, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 345.6980080604553, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CMedQAv2-reranking.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CMedQAv2-reranking.json new file mode 100644 index 0000000000..70129e50da --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CMedQAv2-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "23d186750531a14a0357ca22cd92d712fd512ea0", + "task_name": "CMedQAv2-reranking", + "mteb_version": "1.38.1", + "scores": { + "test": [ + { + "map": 0.840939, + "mrr": 0.867324, + "nAUC_map_max": 0.351412, + "nAUC_map_std": -0.080669, + "nAUC_map_diff1": 0.637344, + "nAUC_mrr_max": 0.408913, + "nAUC_mrr_std": -0.046197, + "nAUC_mrr_diff1": 0.715298, + "main_score": 0.840939, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 378.6483790874481, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CQADupstackGamingRetrieval.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CQADupstackGamingRetrieval.json new file mode 100644 index 0000000000..7391bfff8e --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CQADupstackGamingRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340", + "task_name": "CQADupstackGamingRetrieval", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.54796, + "ndcg_at_3": 0.63348, + "ndcg_at_5": 0.67035, + "ndcg_at_10": 0.69751, + "ndcg_at_20": 0.71081, + "ndcg_at_100": 0.72546, + "ndcg_at_1000": 0.72546, + "map_at_1": 0.47958, + "map_at_3": 0.59018, + "map_at_5": 0.61571, + "map_at_10": 0.63116, + "map_at_20": 0.63682, + "map_at_100": 0.64006, + "map_at_1000": 0.64006, + "recall_at_1": 0.47958, + "recall_at_3": 0.69113, + "recall_at_5": 0.77987, + "recall_at_10": 0.85468, + "recall_at_20": 0.9024, + "recall_at_100": 0.96799, + "recall_at_1000": 0.96799, + "precision_at_1": 0.54796, + "precision_at_3": 0.28067, + "precision_at_5": 0.19549, + "precision_at_10": 0.11179, + "precision_at_20": 0.06085, + "precision_at_100": 0.01347, + "precision_at_1000": 0.00135, + "mrr_at_1": 0.547962, + "mrr_at_3": 0.639603, + "mrr_at_5": 0.658036, + "mrr_at_10": 0.666344, + "mrr_at_20": 0.668769, + "mrr_at_100": 0.670302, + "mrr_at_1000": 0.670302, + "nauc_ndcg_at_1_max": 0.21042, + "nauc_ndcg_at_1_std": 0.030806, + "nauc_ndcg_at_1_diff1": 0.534727, + "nauc_ndcg_at_3_max": 0.219436, + "nauc_ndcg_at_3_std": 0.060287, + "nauc_ndcg_at_3_diff1": 0.500827, + "nauc_ndcg_at_5_max": 0.224689, + "nauc_ndcg_at_5_std": 0.082649, + "nauc_ndcg_at_5_diff1": 0.496132, + "nauc_ndcg_at_10_max": 0.245953, + "nauc_ndcg_at_10_std": 0.115787, + "nauc_ndcg_at_10_diff1": 0.502159, + "nauc_ndcg_at_20_max": 0.254329, + "nauc_ndcg_at_20_std": 0.122176, + "nauc_ndcg_at_20_diff1": 0.504281, + "nauc_ndcg_at_100_max": 0.250172, + "nauc_ndcg_at_100_std": 0.106842, + "nauc_ndcg_at_100_diff1": 0.508157, + "nauc_ndcg_at_1000_max": 0.250172, + "nauc_ndcg_at_1000_std": 0.106842, + "nauc_ndcg_at_1000_diff1": 0.508157, + "nauc_map_at_1_max": 0.141554, + "nauc_map_at_1_std": -0.045338, + "nauc_map_at_1_diff1": 0.525085, + "nauc_map_at_3_max": 0.189041, + "nauc_map_at_3_std": 0.01421, + "nauc_map_at_3_diff1": 0.5061, + "nauc_map_at_5_max": 0.197894, + "nauc_map_at_5_std": 0.03546, + "nauc_map_at_5_diff1": 0.503988, + "nauc_map_at_10_max": 0.21285, + "nauc_map_at_10_std": 0.060162, + "nauc_map_at_10_diff1": 0.509061, + "nauc_map_at_20_max": 0.219306, + "nauc_map_at_20_std": 0.066896, + "nauc_map_at_20_diff1": 0.509873, + "nauc_map_at_100_max": 0.220402, + "nauc_map_at_100_std": 0.066328, + "nauc_map_at_100_diff1": 0.51077, + "nauc_map_at_1000_max": 0.220402, + "nauc_map_at_1000_std": 0.066328, + "nauc_map_at_1000_diff1": 0.51077, + "nauc_recall_at_1_max": 0.141554, + "nauc_recall_at_1_std": -0.045338, + "nauc_recall_at_1_diff1": 0.525085, + "nauc_recall_at_3_max": 0.21366, + "nauc_recall_at_3_std": 0.071187, + "nauc_recall_at_3_diff1": 0.448102, + "nauc_recall_at_5_max": 0.215882, + "nauc_recall_at_5_std": 0.14354, + "nauc_recall_at_5_diff1": 0.413902, + "nauc_recall_at_10_max": 0.301824, + "nauc_recall_at_10_std": 0.309663, + "nauc_recall_at_10_diff1": 0.409397, + "nauc_recall_at_20_max": 0.416686, + "nauc_recall_at_20_std": 0.4538, + "nauc_recall_at_20_diff1": 0.416682, + "nauc_recall_at_100_max": 0.575432, + "nauc_recall_at_100_std": 0.575278, + "nauc_recall_at_100_diff1": 0.430533, + "nauc_recall_at_1000_max": 0.575432, + "nauc_recall_at_1000_std": 0.575278, + "nauc_recall_at_1000_diff1": 0.430533, + "nauc_precision_at_1_max": 0.21042, + "nauc_precision_at_1_std": 0.030806, + "nauc_precision_at_1_diff1": 0.534727, + "nauc_precision_at_3_max": 0.267977, + "nauc_precision_at_3_std": 0.225413, + "nauc_precision_at_3_diff1": 0.259536, + "nauc_precision_at_5_max": 0.267852, + "nauc_precision_at_5_std": 0.317417, + "nauc_precision_at_5_diff1": 0.136212, + "nauc_precision_at_10_max": 0.286151, + "nauc_precision_at_10_std": 0.398831, + "nauc_precision_at_10_diff1": 0.046915, + "nauc_precision_at_20_max": 0.290475, + "nauc_precision_at_20_std": 0.412712, + "nauc_precision_at_20_diff1": -0.021936, + "nauc_precision_at_100_max": 0.234755, + "nauc_precision_at_100_std": 0.330061, + "nauc_precision_at_100_diff1": -0.084131, + "nauc_precision_at_1000_max": 0.234755, + "nauc_precision_at_1000_std": 0.330061, + "nauc_precision_at_1000_diff1": -0.084131, + "nauc_mrr_at_1_max": 0.21042, + "nauc_mrr_at_1_std": 0.030806, + "nauc_mrr_at_1_diff1": 0.534727, + "nauc_mrr_at_3_max": 0.244082, + "nauc_mrr_at_3_std": 0.08031, + "nauc_mrr_at_3_diff1": 0.515221, + "nauc_mrr_at_5_max": 0.241443, + "nauc_mrr_at_5_std": 0.086204, + "nauc_mrr_at_5_diff1": 0.513512, + "nauc_mrr_at_10_max": 0.24533, + "nauc_mrr_at_10_std": 0.089881, + "nauc_mrr_at_10_diff1": 0.514876, + "nauc_mrr_at_20_max": 0.246475, + "nauc_mrr_at_20_std": 0.09015, + "nauc_mrr_at_20_diff1": 0.516043, + "nauc_mrr_at_100_max": 0.245526, + "nauc_mrr_at_100_std": 0.088203, + "nauc_mrr_at_100_diff1": 0.516382, + "nauc_mrr_at_1000_max": 0.245526, + "nauc_mrr_at_1000_std": 0.088203, + "nauc_mrr_at_1000_diff1": 0.516382, + "main_score": 0.69751, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 590.2733745574951, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CQADupstackUnixRetrieval.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CQADupstackUnixRetrieval.json new file mode 100644 index 0000000000..f4f0b9cd15 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CQADupstackUnixRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53", + "task_name": "CQADupstackUnixRetrieval", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.43563, + "ndcg_at_3": 0.50878, + "ndcg_at_5": 0.54107, + "ndcg_at_10": 0.57298, + "ndcg_at_20": 0.59323, + "ndcg_at_100": 0.61812, + "ndcg_at_1000": 0.61812, + "map_at_1": 0.36484, + "map_at_3": 0.46522, + "map_at_5": 0.48938, + "map_at_10": 0.50643, + "map_at_20": 0.51357, + "map_at_100": 0.51803, + "map_at_1000": 0.51803, + "recall_at_1": 0.36484, + "recall_at_3": 0.55522, + "recall_at_5": 0.6383, + "recall_at_10": 0.72995, + "recall_at_20": 0.7995, + "recall_at_100": 0.91536, + "recall_at_1000": 0.91536, + "precision_at_1": 0.43563, + "precision_at_3": 0.2357, + "precision_at_5": 0.16772, + "precision_at_10": 0.09916, + "precision_at_20": 0.05606, + "precision_at_100": 0.01344, + "precision_at_1000": 0.00134, + "mrr_at_1": 0.435634, + "mrr_at_3": 0.5199, + "mrr_at_5": 0.536831, + "mrr_at_10": 0.548351, + "mrr_at_20": 0.552825, + "mrr_at_100": 0.555265, + "mrr_at_1000": 0.555265, + "nauc_ndcg_at_1_max": 0.366397, + "nauc_ndcg_at_1_std": 0.196525, + "nauc_ndcg_at_1_diff1": 0.517216, + "nauc_ndcg_at_3_max": 0.366037, + "nauc_ndcg_at_3_std": 0.231565, + "nauc_ndcg_at_3_diff1": 0.462567, + "nauc_ndcg_at_5_max": 0.369889, + "nauc_ndcg_at_5_std": 0.256293, + "nauc_ndcg_at_5_diff1": 0.455727, + "nauc_ndcg_at_10_max": 0.373558, + "nauc_ndcg_at_10_std": 0.278856, + "nauc_ndcg_at_10_diff1": 0.456382, + "nauc_ndcg_at_20_max": 0.379656, + "nauc_ndcg_at_20_std": 0.284576, + "nauc_ndcg_at_20_diff1": 0.458756, + "nauc_ndcg_at_100_max": 0.37799, + "nauc_ndcg_at_100_std": 0.271785, + "nauc_ndcg_at_100_diff1": 0.466101, + "nauc_ndcg_at_1000_max": 0.37799, + "nauc_ndcg_at_1000_std": 0.271785, + "nauc_ndcg_at_1000_diff1": 0.466101, + "nauc_map_at_1_max": 0.305693, + "nauc_map_at_1_std": 0.106968, + "nauc_map_at_1_diff1": 0.540979, + "nauc_map_at_3_max": 0.346619, + "nauc_map_at_3_std": 0.1933, + "nauc_map_at_3_diff1": 0.483107, + "nauc_map_at_5_max": 0.358913, + "nauc_map_at_5_std": 0.220745, + "nauc_map_at_5_diff1": 0.479098, + "nauc_map_at_10_max": 0.360624, + "nauc_map_at_10_std": 0.23605, + "nauc_map_at_10_diff1": 0.476703, + "nauc_map_at_20_max": 0.363255, + "nauc_map_at_20_std": 0.238909, + "nauc_map_at_20_diff1": 0.477793, + "nauc_map_at_100_max": 0.363807, + "nauc_map_at_100_std": 0.237726, + "nauc_map_at_100_diff1": 0.478757, + "nauc_map_at_1000_max": 0.363807, + "nauc_map_at_1000_std": 0.237726, + "nauc_map_at_1000_diff1": 0.478757, + "nauc_recall_at_1_max": 0.305693, + "nauc_recall_at_1_std": 0.106968, + "nauc_recall_at_1_diff1": 0.540979, + "nauc_recall_at_3_max": 0.344358, + "nauc_recall_at_3_std": 0.236532, + "nauc_recall_at_3_diff1": 0.410083, + "nauc_recall_at_5_max": 0.357717, + "nauc_recall_at_5_std": 0.305597, + "nauc_recall_at_5_diff1": 0.380656, + "nauc_recall_at_10_max": 0.367307, + "nauc_recall_at_10_std": 0.395418, + "nauc_recall_at_10_diff1": 0.366679, + "nauc_recall_at_20_max": 0.397878, + "nauc_recall_at_20_std": 0.452719, + "nauc_recall_at_20_diff1": 0.361403, + "nauc_recall_at_100_max": 0.410063, + "nauc_recall_at_100_std": 0.468163, + "nauc_recall_at_100_diff1": 0.414555, + "nauc_recall_at_1000_max": 0.410063, + "nauc_recall_at_1000_std": 0.468163, + "nauc_recall_at_1000_diff1": 0.414555, + "nauc_precision_at_1_max": 0.366397, + "nauc_precision_at_1_std": 0.196525, + "nauc_precision_at_1_diff1": 0.517216, + "nauc_precision_at_3_max": 0.360804, + "nauc_precision_at_3_std": 0.345836, + "nauc_precision_at_3_diff1": 0.239239, + "nauc_precision_at_5_max": 0.337146, + "nauc_precision_at_5_std": 0.390121, + "nauc_precision_at_5_diff1": 0.149571, + "nauc_precision_at_10_max": 0.27459, + "nauc_precision_at_10_std": 0.39605, + "nauc_precision_at_10_diff1": 0.059894, + "nauc_precision_at_20_max": 0.230536, + "nauc_precision_at_20_std": 0.354387, + "nauc_precision_at_20_diff1": 0.004209, + "nauc_precision_at_100_max": 0.100958, + "nauc_precision_at_100_std": 0.194654, + "nauc_precision_at_100_diff1": -0.103794, + "nauc_precision_at_1000_max": 0.100958, + "nauc_precision_at_1000_std": 0.194654, + "nauc_precision_at_1000_diff1": -0.103794, + "nauc_mrr_at_1_max": 0.366397, + "nauc_mrr_at_1_std": 0.196525, + "nauc_mrr_at_1_diff1": 0.517216, + "nauc_mrr_at_3_max": 0.378097, + "nauc_mrr_at_3_std": 0.241562, + "nauc_mrr_at_3_diff1": 0.468154, + "nauc_mrr_at_5_max": 0.377484, + "nauc_mrr_at_5_std": 0.250107, + "nauc_mrr_at_5_diff1": 0.464558, + "nauc_mrr_at_10_max": 0.379104, + "nauc_mrr_at_10_std": 0.253787, + "nauc_mrr_at_10_diff1": 0.468621, + "nauc_mrr_at_20_max": 0.379894, + "nauc_mrr_at_20_std": 0.253806, + "nauc_mrr_at_20_diff1": 0.469539, + "nauc_mrr_at_100_max": 0.379184, + "nauc_mrr_at_100_std": 0.252289, + "nauc_mrr_at_100_diff1": 0.470175, + "nauc_mrr_at_1000_max": 0.379184, + "nauc_mrr_at_1000_std": 0.252289, + "nauc_mrr_at_1000_diff1": 0.470175, + "main_score": 0.57298, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 468.1757462024689, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ClimateFEVERHardNegatives.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ClimateFEVERHardNegatives.json new file mode 100644 index 0000000000..7562c203d4 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ClimateFEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21", + "task_name": "ClimateFEVERHardNegatives", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.49, + "ndcg_at_3": 0.40666, + "ndcg_at_5": 0.43072, + "ndcg_at_10": 0.46828, + "ndcg_at_20": 0.4965, + "ndcg_at_100": 0.53954, + "ndcg_at_1000": 0.53954, + "map_at_1": 0.21202, + "map_at_3": 0.30851, + "map_at_5": 0.3417, + "map_at_10": 0.36401, + "map_at_20": 0.37543, + "map_at_100": 0.38519, + "map_at_1000": 0.38519, + "recall_at_1": 0.21202, + "recall_at_3": 0.36212, + "recall_at_5": 0.43858, + "recall_at_10": 0.51797, + "recall_at_20": 0.59555, + "recall_at_100": 0.7563, + "recall_at_1000": 0.7563, + "precision_at_1": 0.49, + "precision_at_3": 0.30333, + "precision_at_5": 0.2284, + "precision_at_10": 0.141, + "precision_at_20": 0.08315, + "precision_at_100": 0.02203, + "precision_at_1000": 0.0022, + "mrr_at_1": 0.49, + "mrr_at_3": 0.585167, + "mrr_at_5": 0.598617, + "mrr_at_10": 0.607001, + "mrr_at_20": 0.611331, + "mrr_at_100": 0.613073, + "mrr_at_1000": 0.613073, + "nauc_ndcg_at_1_max": 0.360776, + "nauc_ndcg_at_1_std": 0.299758, + "nauc_ndcg_at_1_diff1": 0.318288, + "nauc_ndcg_at_3_max": 0.383317, + "nauc_ndcg_at_3_std": 0.371527, + "nauc_ndcg_at_3_diff1": 0.225245, + "nauc_ndcg_at_5_max": 0.410746, + "nauc_ndcg_at_5_std": 0.414798, + "nauc_ndcg_at_5_diff1": 0.226205, + "nauc_ndcg_at_10_max": 0.402117, + "nauc_ndcg_at_10_std": 0.42679, + "nauc_ndcg_at_10_diff1": 0.210798, + "nauc_ndcg_at_20_max": 0.402204, + "nauc_ndcg_at_20_std": 0.430579, + "nauc_ndcg_at_20_diff1": 0.206003, + "nauc_ndcg_at_100_max": 0.415045, + "nauc_ndcg_at_100_std": 0.436848, + "nauc_ndcg_at_100_diff1": 0.213161, + "nauc_ndcg_at_1000_max": 0.415045, + "nauc_ndcg_at_1000_std": 0.436848, + "nauc_ndcg_at_1000_diff1": 0.213161, + "nauc_map_at_1_max": 0.379244, + "nauc_map_at_1_std": 0.242106, + "nauc_map_at_1_diff1": 0.361144, + "nauc_map_at_3_max": 0.379026, + "nauc_map_at_3_std": 0.341172, + "nauc_map_at_3_diff1": 0.247943, + "nauc_map_at_5_max": 0.396058, + "nauc_map_at_5_std": 0.380692, + "nauc_map_at_5_diff1": 0.237548, + "nauc_map_at_10_max": 0.391384, + "nauc_map_at_10_std": 0.394155, + "nauc_map_at_10_diff1": 0.224872, + "nauc_map_at_20_max": 0.391881, + "nauc_map_at_20_std": 0.39672, + "nauc_map_at_20_diff1": 0.221791, + "nauc_map_at_100_max": 0.395205, + "nauc_map_at_100_std": 0.399989, + "nauc_map_at_100_diff1": 0.222694, + "nauc_map_at_1000_max": 0.395205, + "nauc_map_at_1000_std": 0.399989, + "nauc_map_at_1000_diff1": 0.222694, + "nauc_recall_at_1_max": 0.379244, + "nauc_recall_at_1_std": 0.242106, + "nauc_recall_at_1_diff1": 0.361144, + "nauc_recall_at_3_max": 0.361042, + "nauc_recall_at_3_std": 0.357188, + "nauc_recall_at_3_diff1": 0.186138, + "nauc_recall_at_5_max": 0.381033, + "nauc_recall_at_5_std": 0.421487, + "nauc_recall_at_5_diff1": 0.161346, + "nauc_recall_at_10_max": 0.346885, + "nauc_recall_at_10_std": 0.426779, + "nauc_recall_at_10_diff1": 0.123665, + "nauc_recall_at_20_max": 0.335428, + "nauc_recall_at_20_std": 0.427264, + "nauc_recall_at_20_diff1": 0.100071, + "nauc_recall_at_100_max": 0.379535, + "nauc_recall_at_100_std": 0.453819, + "nauc_recall_at_100_diff1": 0.101189, + "nauc_recall_at_1000_max": 0.379535, + "nauc_recall_at_1000_std": 0.453819, + "nauc_recall_at_1000_diff1": 0.101189, + "nauc_precision_at_1_max": 0.360776, + "nauc_precision_at_1_std": 0.299758, + "nauc_precision_at_1_diff1": 0.318288, + "nauc_precision_at_3_max": 0.298698, + "nauc_precision_at_3_std": 0.400863, + "nauc_precision_at_3_diff1": 0.062526, + "nauc_precision_at_5_max": 0.301122, + "nauc_precision_at_5_std": 0.444408, + "nauc_precision_at_5_diff1": 0.030899, + "nauc_precision_at_10_max": 0.216117, + "nauc_precision_at_10_std": 0.389619, + "nauc_precision_at_10_diff1": -0.025778, + "nauc_precision_at_20_max": 0.166439, + "nauc_precision_at_20_std": 0.341883, + "nauc_precision_at_20_diff1": -0.056654, + "nauc_precision_at_100_max": 0.085905, + "nauc_precision_at_100_std": 0.223807, + "nauc_precision_at_100_diff1": -0.0848, + "nauc_precision_at_1000_max": 0.085905, + "nauc_precision_at_1000_std": 0.223807, + "nauc_precision_at_1000_diff1": -0.0848, + "nauc_mrr_at_1_max": 0.360776, + "nauc_mrr_at_1_std": 0.299758, + "nauc_mrr_at_1_diff1": 0.318288, + "nauc_mrr_at_3_max": 0.396184, + "nauc_mrr_at_3_std": 0.360692, + "nauc_mrr_at_3_diff1": 0.272368, + "nauc_mrr_at_5_max": 0.403448, + "nauc_mrr_at_5_std": 0.375832, + "nauc_mrr_at_5_diff1": 0.2746, + "nauc_mrr_at_10_max": 0.402198, + "nauc_mrr_at_10_std": 0.372638, + "nauc_mrr_at_10_diff1": 0.276002, + "nauc_mrr_at_20_max": 0.401482, + "nauc_mrr_at_20_std": 0.371493, + "nauc_mrr_at_20_diff1": 0.277013, + "nauc_mrr_at_100_max": 0.40098, + "nauc_mrr_at_100_std": 0.370062, + "nauc_mrr_at_100_diff1": 0.277706, + "nauc_mrr_at_1000_max": 0.40098, + "nauc_mrr_at_1000_std": 0.370062, + "nauc_mrr_at_1000_diff1": 0.277706, + "main_score": 0.46828, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 332.00027537345886, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CmedqaRetrieval.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CmedqaRetrieval.json new file mode 100644 index 0000000000..523f75099b --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CmedqaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301", + "task_name": "CmedqaRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.47587, + "ndcg_at_3": 0.47514, + "ndcg_at_5": 0.49551, + "ndcg_at_10": 0.52422, + "ndcg_at_20": 0.55155, + "ndcg_at_100": 0.59138, + "ndcg_at_1000": 0.59138, + "map_at_1": 0.31647, + "map_at_3": 0.41467, + "map_at_5": 0.43936, + "map_at_10": 0.45792, + "map_at_20": 0.4682, + "map_at_100": 0.47595, + "map_at_1000": 0.47595, + "recall_at_1": 0.31647, + "recall_at_3": 0.47102, + "recall_at_5": 0.53897, + "recall_at_10": 0.62574, + "recall_at_20": 0.71883, + "recall_at_100": 0.90051, + "recall_at_1000": 0.90051, + "precision_at_1": 0.47587, + "precision_at_3": 0.26423, + "precision_at_5": 0.1879, + "precision_at_10": 0.11183, + "precision_at_20": 0.06517, + "precision_at_100": 0.01668, + "precision_at_1000": 0.00167, + "mrr_at_1": 0.475869, + "mrr_at_3": 0.530424, + "mrr_at_5": 0.54344, + "mrr_at_10": 0.552938, + "mrr_at_20": 0.558171, + "mrr_at_100": 0.561618, + "mrr_at_1000": 0.561618, + "nauc_ndcg_at_1_max": 0.656709, + "nauc_ndcg_at_1_std": 0.437066, + "nauc_ndcg_at_1_diff1": 0.699187, + "nauc_ndcg_at_3_max": 0.572755, + "nauc_ndcg_at_3_std": 0.434064, + "nauc_ndcg_at_3_diff1": 0.58235, + "nauc_ndcg_at_5_max": 0.567015, + "nauc_ndcg_at_5_std": 0.448788, + "nauc_ndcg_at_5_diff1": 0.574607, + "nauc_ndcg_at_10_max": 0.569985, + "nauc_ndcg_at_10_std": 0.472241, + "nauc_ndcg_at_10_diff1": 0.570179, + "nauc_ndcg_at_20_max": 0.577128, + "nauc_ndcg_at_20_std": 0.48847, + "nauc_ndcg_at_20_diff1": 0.570994, + "nauc_ndcg_at_100_max": 0.589363, + "nauc_ndcg_at_100_std": 0.484309, + "nauc_ndcg_at_100_diff1": 0.582862, + "nauc_ndcg_at_1000_max": 0.589363, + "nauc_ndcg_at_1000_std": 0.484309, + "nauc_ndcg_at_1000_diff1": 0.582862, + "nauc_map_at_1_max": 0.412917, + "nauc_map_at_1_std": 0.243238, + "nauc_map_at_1_diff1": 0.581721, + "nauc_map_at_3_max": 0.505882, + "nauc_map_at_3_std": 0.368507, + "nauc_map_at_3_diff1": 0.56694, + "nauc_map_at_5_max": 0.531955, + "nauc_map_at_5_std": 0.406672, + "nauc_map_at_5_diff1": 0.566398, + "nauc_map_at_10_max": 0.547736, + "nauc_map_at_10_std": 0.432545, + "nauc_map_at_10_diff1": 0.566982, + "nauc_map_at_20_max": 0.553398, + "nauc_map_at_20_std": 0.441324, + "nauc_map_at_20_diff1": 0.568423, + "nauc_map_at_100_max": 0.557027, + "nauc_map_at_100_std": 0.442199, + "nauc_map_at_100_diff1": 0.571372, + "nauc_map_at_1000_max": 0.557027, + "nauc_map_at_1000_std": 0.442199, + "nauc_map_at_1000_diff1": 0.571372, + "nauc_recall_at_1_max": 0.412917, + "nauc_recall_at_1_std": 0.243238, + "nauc_recall_at_1_diff1": 0.581721, + "nauc_recall_at_3_max": 0.454793, + "nauc_recall_at_3_std": 0.368259, + "nauc_recall_at_3_diff1": 0.493194, + "nauc_recall_at_5_max": 0.470277, + "nauc_recall_at_5_std": 0.428055, + "nauc_recall_at_5_diff1": 0.469078, + "nauc_recall_at_10_max": 0.474353, + "nauc_recall_at_10_std": 0.497149, + "nauc_recall_at_10_diff1": 0.437474, + "nauc_recall_at_20_max": 0.474334, + "nauc_recall_at_20_std": 0.563101, + "nauc_recall_at_20_diff1": 0.40686, + "nauc_recall_at_100_max": 0.506892, + "nauc_recall_at_100_std": 0.650958, + "nauc_recall_at_100_diff1": 0.368649, + "nauc_recall_at_1000_max": 0.506892, + "nauc_recall_at_1000_std": 0.650958, + "nauc_recall_at_1000_diff1": 0.368649, + "nauc_precision_at_1_max": 0.656709, + "nauc_precision_at_1_std": 0.437066, + "nauc_precision_at_1_diff1": 0.699187, + "nauc_precision_at_3_max": 0.580033, + "nauc_precision_at_3_std": 0.498072, + "nauc_precision_at_3_diff1": 0.39621, + "nauc_precision_at_5_max": 0.554118, + "nauc_precision_at_5_std": 0.520731, + "nauc_precision_at_5_diff1": 0.32047, + "nauc_precision_at_10_max": 0.515825, + "nauc_precision_at_10_std": 0.52785, + "nauc_precision_at_10_diff1": 0.250274, + "nauc_precision_at_20_max": 0.460042, + "nauc_precision_at_20_std": 0.493719, + "nauc_precision_at_20_diff1": 0.187243, + "nauc_precision_at_100_max": 0.337485, + "nauc_precision_at_100_std": 0.327432, + "nauc_precision_at_100_diff1": 0.084493, + "nauc_precision_at_1000_max": 0.337485, + "nauc_precision_at_1000_std": 0.327432, + "nauc_precision_at_1000_diff1": 0.084493, + "nauc_mrr_at_1_max": 0.656709, + "nauc_mrr_at_1_std": 0.437066, + "nauc_mrr_at_1_diff1": 0.699187, + "nauc_mrr_at_3_max": 0.64583, + "nauc_mrr_at_3_std": 0.4609, + "nauc_mrr_at_3_diff1": 0.664564, + "nauc_mrr_at_5_max": 0.644814, + "nauc_mrr_at_5_std": 0.466761, + "nauc_mrr_at_5_diff1": 0.660951, + "nauc_mrr_at_10_max": 0.643757, + "nauc_mrr_at_10_std": 0.470569, + "nauc_mrr_at_10_diff1": 0.658593, + "nauc_mrr_at_20_max": 0.64378, + "nauc_mrr_at_20_std": 0.471311, + "nauc_mrr_at_20_diff1": 0.658178, + "nauc_mrr_at_100_max": 0.644217, + "nauc_mrr_at_100_std": 0.470449, + "nauc_mrr_at_100_diff1": 0.659058, + "nauc_mrr_at_1000_max": 0.644217, + "nauc_mrr_at_1000_std": 0.470449, + "nauc_mrr_at_1000_diff1": 0.659058, + "main_score": 0.52422, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 410.8545677661896, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Cmnli.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Cmnli.json new file mode 100644 index 0000000000..5351eb0c18 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Cmnli.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "41bc36f332156f7adc9e38f53777c959b2ae9766", + "task_name": "Cmnli", + "mteb_version": "1.38.1", + "scores": { + "validation": [ + { + "similarity_accuracy": 0.84089, + "similarity_accuracy_threshold": 0.60406, + "similarity_f1": 0.846546, + "similarity_f1_threshold": 0.579322, + "similarity_precision": 0.822026, + "similarity_recall": 0.872574, + "similarity_ap": 0.914692, + "cosine_accuracy": 0.84089, + "cosine_accuracy_threshold": 0.60406, + "cosine_f1": 0.846546, + "cosine_f1_threshold": 0.579322, + "cosine_precision": 0.822026, + "cosine_recall": 0.872574, + "cosine_ap": 0.914692, + "manhattan_accuracy": 0.828382, + "manhattan_accuracy_threshold": 31.505726, + "manhattan_f1": 0.835195, + "manhattan_f1_threshold": 32.420506, + "manhattan_precision": 0.806888, + "manhattan_recall": 0.86556, + "manhattan_ap": 0.908846, + "euclidean_accuracy": 0.840289, + "euclidean_accuracy_threshold": 0.889985, + "euclidean_f1": 0.84645, + "euclidean_f1_threshold": 0.919001, + "euclidean_precision": 0.819575, + "euclidean_recall": 0.875146, + "euclidean_ap": 0.914695, + "dot_accuracy": 0.84113, + "dot_accuracy_threshold": 0.60376, + "dot_f1": 0.846939, + "dot_f1_threshold": 0.579089, + "dot_precision": 0.822144, + "dot_recall": 0.873276, + "dot_ap": 0.914513, + "max_accuracy": 0.84113, + "max_f1": 0.846939, + "max_precision": 0.822144, + "max_recall": 0.875146, + "max_ap": 0.914695, + "main_score": 0.914695, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 60.5124876499176, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CovidRetrieval.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CovidRetrieval.json new file mode 100644 index 0000000000..6c0278e382 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CovidRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "1271c7809071a13532e05f25fb53511ffce77117", + "task_name": "CovidRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.78082, + "ndcg_at_3": 0.85195, + "ndcg_at_5": 0.86762, + "ndcg_at_10": 0.87889, + "ndcg_at_20": 0.88134, + "ndcg_at_100": 0.88445, + "ndcg_at_1000": 0.88445, + "map_at_1": 0.77871, + "map_at_3": 0.83496, + "map_at_5": 0.84401, + "map_at_10": 0.84887, + "map_at_20": 0.84957, + "map_at_100": 0.85006, + "map_at_1000": 0.85006, + "recall_at_1": 0.77871, + "recall_at_3": 0.89989, + "recall_at_5": 0.93783, + "recall_at_10": 0.97155, + "recall_at_20": 0.98103, + "recall_at_100": 0.99684, + "recall_at_1000": 0.99684, + "precision_at_1": 0.78082, + "precision_at_3": 0.30242, + "precision_at_5": 0.18946, + "precision_at_10": 0.09821, + "precision_at_20": 0.04958, + "precision_at_100": 0.01007, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.780822, + "mrr_at_3": 0.835265, + "mrr_at_5": 0.843642, + "mrr_at_10": 0.848458, + "mrr_at_20": 0.849158, + "mrr_at_100": 0.84965, + "mrr_at_1000": 0.84965, + "nauc_ndcg_at_1_max": 0.444867, + "nauc_ndcg_at_1_std": -0.12969, + "nauc_ndcg_at_1_diff1": 0.882473, + "nauc_ndcg_at_3_max": 0.496143, + "nauc_ndcg_at_3_std": -0.105607, + "nauc_ndcg_at_3_diff1": 0.859459, + "nauc_ndcg_at_5_max": 0.506219, + "nauc_ndcg_at_5_std": -0.095122, + "nauc_ndcg_at_5_diff1": 0.862265, + "nauc_ndcg_at_10_max": 0.499748, + "nauc_ndcg_at_10_std": -0.083397, + "nauc_ndcg_at_10_diff1": 0.866903, + "nauc_ndcg_at_20_max": 0.495668, + "nauc_ndcg_at_20_std": -0.083474, + "nauc_ndcg_at_20_diff1": 0.866266, + "nauc_ndcg_at_100_max": 0.488509, + "nauc_ndcg_at_100_std": -0.090451, + "nauc_ndcg_at_100_diff1": 0.868791, + "nauc_ndcg_at_1000_max": 0.488509, + "nauc_ndcg_at_1000_std": -0.090451, + "nauc_ndcg_at_1000_diff1": 0.868791, + "nauc_map_at_1_max": 0.447823, + "nauc_map_at_1_std": -0.128308, + "nauc_map_at_1_diff1": 0.886251, + "nauc_map_at_3_max": 0.48052, + "nauc_map_at_3_std": -0.112466, + "nauc_map_at_3_diff1": 0.866825, + "nauc_map_at_5_max": 0.485226, + "nauc_map_at_5_std": -0.103582, + "nauc_map_at_5_diff1": 0.868159, + "nauc_map_at_10_max": 0.482277, + "nauc_map_at_10_std": -0.099624, + "nauc_map_at_10_diff1": 0.870094, + "nauc_map_at_20_max": 0.48132, + "nauc_map_at_20_std": -0.09938, + "nauc_map_at_20_diff1": 0.869932, + "nauc_map_at_100_max": 0.480403, + "nauc_map_at_100_std": -0.100273, + "nauc_map_at_100_diff1": 0.870253, + "nauc_map_at_1000_max": 0.480403, + "nauc_map_at_1000_std": -0.100273, + "nauc_map_at_1000_diff1": 0.870253, + "nauc_recall_at_1_max": 0.447823, + "nauc_recall_at_1_std": -0.128308, + "nauc_recall_at_1_diff1": 0.886251, + "nauc_recall_at_3_max": 0.561557, + "nauc_recall_at_3_std": -0.092765, + "nauc_recall_at_3_diff1": 0.827338, + "nauc_recall_at_5_max": 0.6512, + "nauc_recall_at_5_std": -0.039434, + "nauc_recall_at_5_diff1": 0.823403, + "nauc_recall_at_10_max": 0.745871, + "nauc_recall_at_10_std": 0.161167, + "nauc_recall_at_10_diff1": 0.830248, + "nauc_recall_at_20_max": 0.772812, + "nauc_recall_at_20_std": 0.262258, + "nauc_recall_at_20_diff1": 0.799201, + "nauc_recall_at_100_max": 0.907389, + "nauc_recall_at_100_std": 0.771201, + "nauc_recall_at_100_diff1": 0.907389, + "nauc_recall_at_1000_max": 0.907389, + "nauc_recall_at_1000_std": 0.771201, + "nauc_recall_at_1000_diff1": 0.907389, + "nauc_precision_at_1_max": 0.444867, + "nauc_precision_at_1_std": -0.12969, + "nauc_precision_at_1_diff1": 0.882473, + "nauc_precision_at_3_max": 0.485152, + "nauc_precision_at_3_std": -0.055966, + "nauc_precision_at_3_diff1": 0.687597, + "nauc_precision_at_5_max": 0.477861, + "nauc_precision_at_5_std": 0.055084, + "nauc_precision_at_5_diff1": 0.537279, + "nauc_precision_at_10_max": 0.375487, + "nauc_precision_at_10_std": 0.269042, + "nauc_precision_at_10_diff1": 0.291563, + "nauc_precision_at_20_max": 0.294975, + "nauc_precision_at_20_std": 0.347914, + "nauc_precision_at_20_diff1": 0.138737, + "nauc_precision_at_100_max": -0.036907, + "nauc_precision_at_100_std": 0.485874, + "nauc_precision_at_100_diff1": -0.331444, + "nauc_precision_at_1000_max": -0.036907, + "nauc_precision_at_1000_std": 0.485874, + "nauc_precision_at_1000_diff1": -0.331444, + "nauc_mrr_at_1_max": 0.444867, + "nauc_mrr_at_1_std": -0.12969, + "nauc_mrr_at_1_diff1": 0.882473, + "nauc_mrr_at_3_max": 0.480953, + "nauc_mrr_at_3_std": -0.108048, + "nauc_mrr_at_3_diff1": 0.86687, + "nauc_mrr_at_5_max": 0.484597, + "nauc_mrr_at_5_std": -0.106455, + "nauc_mrr_at_5_diff1": 0.868764, + "nauc_mrr_at_10_max": 0.481707, + "nauc_mrr_at_10_std": -0.102927, + "nauc_mrr_at_10_diff1": 0.870756, + "nauc_mrr_at_20_max": 0.48075, + "nauc_mrr_at_20_std": -0.1027, + "nauc_mrr_at_20_diff1": 0.870597, + "nauc_mrr_at_100_max": 0.479833, + "nauc_mrr_at_100_std": -0.103603, + "nauc_mrr_at_100_diff1": 0.87092, + "nauc_mrr_at_1000_max": 0.479833, + "nauc_mrr_at_1000_std": -0.103603, + "nauc_mrr_at_1000_diff1": 0.87092, + "main_score": 0.87889, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 313.91021943092346, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/DuRetrieval.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/DuRetrieval.json new file mode 100644 index 0000000000..42284ceab8 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/DuRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced", + "task_name": "DuRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.9595, + "ndcg_at_3": 0.93976, + "ndcg_at_5": 0.93018, + "ndcg_at_10": 0.94099, + "ndcg_at_20": 0.94921, + "ndcg_at_100": 0.95624, + "ndcg_at_1000": 0.95624, + "map_at_1": 0.29229, + "map_at_3": 0.63205, + "map_at_5": 0.79462, + "map_at_10": 0.89239, + "map_at_20": 0.91099, + "map_at_100": 0.91601, + "map_at_1000": 0.91601, + "recall_at_1": 0.29229, + "recall_at_3": 0.64549, + "recall_at_5": 0.82116, + "recall_at_10": 0.94051, + "recall_at_20": 0.97235, + "recall_at_100": 0.99403, + "recall_at_1000": 0.99403, + "precision_at_1": 0.9595, + "precision_at_3": 0.8375, + "precision_at_5": 0.706, + "precision_at_10": 0.4397, + "precision_at_20": 0.23565, + "precision_at_100": 0.04884, + "precision_at_1000": 0.00488, + "mrr_at_1": 0.9595, + "mrr_at_3": 0.97275, + "mrr_at_5": 0.973375, + "mrr_at_10": 0.973841, + "mrr_at_20": 0.973993, + "mrr_at_100": 0.974061, + "mrr_at_1000": 0.974061, + "nauc_ndcg_at_1_max": 0.863829, + "nauc_ndcg_at_1_std": 0.74783, + "nauc_ndcg_at_1_diff1": 0.536795, + "nauc_ndcg_at_3_max": 0.570196, + "nauc_ndcg_at_3_std": 0.640562, + "nauc_ndcg_at_3_diff1": 0.025336, + "nauc_ndcg_at_5_max": 0.463852, + "nauc_ndcg_at_5_std": 0.503717, + "nauc_ndcg_at_5_diff1": 0.090672, + "nauc_ndcg_at_10_max": 0.510803, + "nauc_ndcg_at_10_std": 0.510109, + "nauc_ndcg_at_10_diff1": 0.170441, + "nauc_ndcg_at_20_max": 0.559643, + "nauc_ndcg_at_20_std": 0.560445, + "nauc_ndcg_at_20_diff1": 0.188847, + "nauc_ndcg_at_100_max": 0.58531, + "nauc_ndcg_at_100_std": 0.599857, + "nauc_ndcg_at_100_diff1": 0.167416, + "nauc_ndcg_at_1000_max": 0.58531, + "nauc_ndcg_at_1000_std": 0.599857, + "nauc_ndcg_at_1000_diff1": 0.167416, + "nauc_map_at_1_max": -0.126209, + "nauc_map_at_1_std": -0.509793, + "nauc_map_at_1_diff1": 0.569346, + "nauc_map_at_3_max": -0.006571, + "nauc_map_at_3_std": -0.468325, + "nauc_map_at_3_diff1": 0.383444, + "nauc_map_at_5_max": 0.143315, + "nauc_map_at_5_std": -0.242347, + "nauc_map_at_5_diff1": 0.265368, + "nauc_map_at_10_max": 0.396641, + "nauc_map_at_10_std": 0.245479, + "nauc_map_at_10_diff1": 0.157123, + "nauc_map_at_20_max": 0.456895, + "nauc_map_at_20_std": 0.43785, + "nauc_map_at_20_diff1": 0.114849, + "nauc_map_at_100_max": 0.467016, + "nauc_map_at_100_std": 0.475982, + "nauc_map_at_100_diff1": 0.099126, + "nauc_map_at_1000_max": 0.467016, + "nauc_map_at_1000_std": 0.475982, + "nauc_map_at_1000_diff1": 0.099126, + "nauc_recall_at_1_max": -0.126209, + "nauc_recall_at_1_std": -0.509793, + "nauc_recall_at_1_diff1": 0.569346, + "nauc_recall_at_3_max": -0.052712, + "nauc_recall_at_3_std": -0.516843, + "nauc_recall_at_3_diff1": 0.371823, + "nauc_recall_at_5_max": 0.054516, + "nauc_recall_at_5_std": -0.37952, + "nauc_recall_at_5_diff1": 0.282004, + "nauc_recall_at_10_max": 0.333481, + "nauc_recall_at_10_std": 0.060358, + "nauc_recall_at_10_diff1": 0.256157, + "nauc_recall_at_20_max": 0.501232, + "nauc_recall_at_20_std": 0.41858, + "nauc_recall_at_20_diff1": 0.274928, + "nauc_recall_at_100_max": 0.708789, + "nauc_recall_at_100_std": 0.640945, + "nauc_recall_at_100_diff1": 0.173004, + "nauc_recall_at_1000_max": 0.708789, + "nauc_recall_at_1000_std": 0.640945, + "nauc_recall_at_1000_diff1": 0.173004, + "nauc_precision_at_1_max": 0.863829, + "nauc_precision_at_1_std": 0.74783, + "nauc_precision_at_1_diff1": 0.536795, + "nauc_precision_at_3_max": 0.431584, + "nauc_precision_at_3_std": 0.808999, + "nauc_precision_at_3_diff1": -0.578971, + "nauc_precision_at_5_max": 0.315178, + "nauc_precision_at_5_std": 0.78521, + "nauc_precision_at_5_diff1": -0.502566, + "nauc_precision_at_10_max": 0.227355, + "nauc_precision_at_10_std": 0.728151, + "nauc_precision_at_10_diff1": -0.404948, + "nauc_precision_at_20_max": 0.172614, + "nauc_precision_at_20_std": 0.687491, + "nauc_precision_at_20_diff1": -0.381291, + "nauc_precision_at_100_max": 0.138583, + "nauc_precision_at_100_std": 0.644898, + "nauc_precision_at_100_diff1": -0.379365, + "nauc_precision_at_1000_max": 0.138583, + "nauc_precision_at_1000_std": 0.644898, + "nauc_precision_at_1000_diff1": -0.379365, + "nauc_mrr_at_1_max": 0.863829, + "nauc_mrr_at_1_std": 0.74783, + "nauc_mrr_at_1_diff1": 0.536795, + "nauc_mrr_at_3_max": 0.892684, + "nauc_mrr_at_3_std": 0.80178, + "nauc_mrr_at_3_diff1": 0.573777, + "nauc_mrr_at_5_max": 0.890655, + "nauc_mrr_at_5_std": 0.798109, + "nauc_mrr_at_5_diff1": 0.573283, + "nauc_mrr_at_10_max": 0.888987, + "nauc_mrr_at_10_std": 0.795149, + "nauc_mrr_at_10_diff1": 0.570324, + "nauc_mrr_at_20_max": 0.888336, + "nauc_mrr_at_20_std": 0.793949, + "nauc_mrr_at_20_diff1": 0.569589, + "nauc_mrr_at_100_max": 0.888043, + "nauc_mrr_at_100_std": 0.793407, + "nauc_mrr_at_100_diff1": 0.56901, + "nauc_mrr_at_1000_max": 0.888043, + "nauc_mrr_at_1000_std": 0.793407, + "nauc_mrr_at_1000_diff1": 0.56901, + "main_score": 0.94099, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 347.83355379104614, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/EcomRetrieval.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/EcomRetrieval.json new file mode 100644 index 0000000000..a39a155d6c --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/EcomRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9", + "task_name": "EcomRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.578, + "ndcg_at_3": 0.68103, + "ndcg_at_5": 0.7066, + "ndcg_at_10": 0.73203, + "ndcg_at_20": 0.73989, + "ndcg_at_100": 0.74693, + "ndcg_at_1000": 0.74693, + "map_at_1": 0.578, + "map_at_3": 0.6555, + "map_at_5": 0.6697, + "map_at_10": 0.68033, + "map_at_20": 0.6825, + "map_at_100": 0.68354, + "map_at_1000": 0.68354, + "recall_at_1": 0.578, + "recall_at_3": 0.755, + "recall_at_5": 0.817, + "recall_at_10": 0.895, + "recall_at_20": 0.926, + "recall_at_100": 0.963, + "recall_at_1000": 0.963, + "precision_at_1": 0.578, + "precision_at_3": 0.25167, + "precision_at_5": 0.1634, + "precision_at_10": 0.0895, + "precision_at_20": 0.0463, + "precision_at_100": 0.00963, + "precision_at_1000": 0.00096, + "mrr_at_1": 0.578, + "mrr_at_3": 0.6555, + "mrr_at_5": 0.6697, + "mrr_at_10": 0.680331, + "mrr_at_20": 0.682499, + "mrr_at_100": 0.683536, + "mrr_at_1000": 0.683536, + "nauc_ndcg_at_1_max": 0.417576, + "nauc_ndcg_at_1_std": 0.058976, + "nauc_ndcg_at_1_diff1": 0.790364, + "nauc_ndcg_at_3_max": 0.425761, + "nauc_ndcg_at_3_std": 0.096229, + "nauc_ndcg_at_3_diff1": 0.751927, + "nauc_ndcg_at_5_max": 0.440973, + "nauc_ndcg_at_5_std": 0.12318, + "nauc_ndcg_at_5_diff1": 0.750429, + "nauc_ndcg_at_10_max": 0.458858, + "nauc_ndcg_at_10_std": 0.155603, + "nauc_ndcg_at_10_diff1": 0.751461, + "nauc_ndcg_at_20_max": 0.454504, + "nauc_ndcg_at_20_std": 0.152391, + "nauc_ndcg_at_20_diff1": 0.754141, + "nauc_ndcg_at_100_max": 0.457785, + "nauc_ndcg_at_100_std": 0.145772, + "nauc_ndcg_at_100_diff1": 0.758744, + "nauc_ndcg_at_1000_max": 0.457785, + "nauc_ndcg_at_1000_std": 0.145772, + "nauc_ndcg_at_1000_diff1": 0.758744, + "nauc_map_at_1_max": 0.417576, + "nauc_map_at_1_std": 0.058976, + "nauc_map_at_1_diff1": 0.790364, + "nauc_map_at_3_max": 0.424522, + "nauc_map_at_3_std": 0.084224, + "nauc_map_at_3_diff1": 0.762129, + "nauc_map_at_5_max": 0.432266, + "nauc_map_at_5_std": 0.097629, + "nauc_map_at_5_diff1": 0.762062, + "nauc_map_at_10_max": 0.438272, + "nauc_map_at_10_std": 0.108294, + "nauc_map_at_10_diff1": 0.762447, + "nauc_map_at_20_max": 0.437115, + "nauc_map_at_20_std": 0.10697, + "nauc_map_at_20_diff1": 0.763174, + "nauc_map_at_100_max": 0.437478, + "nauc_map_at_100_std": 0.10625, + "nauc_map_at_100_diff1": 0.763718, + "nauc_map_at_1000_max": 0.437478, + "nauc_map_at_1000_std": 0.10625, + "nauc_map_at_1000_diff1": 0.763718, + "nauc_recall_at_1_max": 0.417576, + "nauc_recall_at_1_std": 0.058976, + "nauc_recall_at_1_diff1": 0.790364, + "nauc_recall_at_3_max": 0.429752, + "nauc_recall_at_3_std": 0.141055, + "nauc_recall_at_3_diff1": 0.714828, + "nauc_recall_at_5_max": 0.482541, + "nauc_recall_at_5_std": 0.244431, + "nauc_recall_at_5_diff1": 0.697675, + "nauc_recall_at_10_max": 0.627605, + "nauc_recall_at_10_std": 0.535538, + "nauc_recall_at_10_diff1": 0.675317, + "nauc_recall_at_20_max": 0.646523, + "nauc_recall_at_20_std": 0.65955, + "nauc_recall_at_20_diff1": 0.674218, + "nauc_recall_at_100_max": 0.933328, + "nauc_recall_at_100_std": 0.949529, + "nauc_recall_at_100_diff1": 0.734095, + "nauc_recall_at_1000_max": 0.933328, + "nauc_recall_at_1000_std": 0.949529, + "nauc_recall_at_1000_diff1": 0.734095, + "nauc_precision_at_1_max": 0.417576, + "nauc_precision_at_1_std": 0.058976, + "nauc_precision_at_1_diff1": 0.790364, + "nauc_precision_at_3_max": 0.429752, + "nauc_precision_at_3_std": 0.141055, + "nauc_precision_at_3_diff1": 0.714828, + "nauc_precision_at_5_max": 0.482541, + "nauc_precision_at_5_std": 0.244431, + "nauc_precision_at_5_diff1": 0.697675, + "nauc_precision_at_10_max": 0.627605, + "nauc_precision_at_10_std": 0.535538, + "nauc_precision_at_10_diff1": 0.675317, + "nauc_precision_at_20_max": 0.646523, + "nauc_precision_at_20_std": 0.65955, + "nauc_precision_at_20_diff1": 0.674218, + "nauc_precision_at_100_max": 0.933328, + "nauc_precision_at_100_std": 0.949529, + "nauc_precision_at_100_diff1": 0.734095, + "nauc_precision_at_1000_max": 0.933328, + "nauc_precision_at_1000_std": 0.949529, + "nauc_precision_at_1000_diff1": 0.734095, + "nauc_mrr_at_1_max": 0.417576, + "nauc_mrr_at_1_std": 0.058976, + "nauc_mrr_at_1_diff1": 0.790364, + "nauc_mrr_at_3_max": 0.424522, + "nauc_mrr_at_3_std": 0.084224, + "nauc_mrr_at_3_diff1": 0.762129, + "nauc_mrr_at_5_max": 0.432266, + "nauc_mrr_at_5_std": 0.097629, + "nauc_mrr_at_5_diff1": 0.762062, + "nauc_mrr_at_10_max": 0.438272, + "nauc_mrr_at_10_std": 0.108294, + "nauc_mrr_at_10_diff1": 0.762447, + "nauc_mrr_at_20_max": 0.437115, + "nauc_mrr_at_20_std": 0.10697, + "nauc_mrr_at_20_diff1": 0.763174, + "nauc_mrr_at_100_max": 0.437478, + "nauc_mrr_at_100_std": 0.10625, + "nauc_mrr_at_100_diff1": 0.763718, + "nauc_mrr_at_1000_max": 0.437478, + "nauc_mrr_at_1000_std": 0.10625, + "nauc_mrr_at_1000_diff1": 0.763718, + "main_score": 0.73203, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 111.0213143825531, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/FEVERHardNegatives.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/FEVERHardNegatives.json new file mode 100644 index 0000000000..e68855eea1 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/FEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "080c9ed6267b65029207906e815d44a9240bafca", + "task_name": "FEVERHardNegatives", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.925, + "ndcg_at_3": 0.94329, + "ndcg_at_5": 0.94706, + "ndcg_at_10": 0.95113, + "ndcg_at_20": 0.95275, + "ndcg_at_100": 0.95487, + "ndcg_at_1000": 0.95487, + "map_at_1": 0.87005, + "map_at_3": 0.92371, + "map_at_5": 0.92848, + "map_at_10": 0.93169, + "map_at_20": 0.93258, + "map_at_100": 0.93315, + "map_at_1000": 0.93315, + "recall_at_1": 0.87005, + "recall_at_3": 0.95916, + "recall_at_5": 0.97046, + "recall_at_10": 0.98177, + "recall_at_20": 0.98669, + "recall_at_100": 0.99486, + "recall_at_1000": 0.99486, + "precision_at_1": 0.925, + "precision_at_3": 0.35067, + "precision_at_5": 0.2156, + "precision_at_10": 0.1104, + "precision_at_20": 0.056, + "precision_at_100": 0.01147, + "precision_at_1000": 0.00115, + "mrr_at_1": 0.925, + "mrr_at_3": 0.957167, + "mrr_at_5": 0.957817, + "mrr_at_10": 0.958293, + "mrr_at_20": 0.958293, + "mrr_at_100": 0.958306, + "mrr_at_1000": 0.958306, + "nauc_ndcg_at_1_max": 0.229549, + "nauc_ndcg_at_1_std": -0.397579, + "nauc_ndcg_at_1_diff1": 0.831279, + "nauc_ndcg_at_3_max": 0.345077, + "nauc_ndcg_at_3_std": -0.072668, + "nauc_ndcg_at_3_diff1": 0.623454, + "nauc_ndcg_at_5_max": 0.336361, + "nauc_ndcg_at_5_std": -0.060913, + "nauc_ndcg_at_5_diff1": 0.610684, + "nauc_ndcg_at_10_max": 0.348595, + "nauc_ndcg_at_10_std": -0.045944, + "nauc_ndcg_at_10_diff1": 0.618741, + "nauc_ndcg_at_20_max": 0.341925, + "nauc_ndcg_at_20_std": -0.068655, + "nauc_ndcg_at_20_diff1": 0.635124, + "nauc_ndcg_at_100_max": 0.327143, + "nauc_ndcg_at_100_std": -0.10994, + "nauc_ndcg_at_100_diff1": 0.65361, + "nauc_ndcg_at_1000_max": 0.327143, + "nauc_ndcg_at_1000_std": -0.10994, + "nauc_ndcg_at_1000_diff1": 0.65361, + "nauc_map_at_1_max": 0.277748, + "nauc_map_at_1_std": -0.159446, + "nauc_map_at_1_diff1": 0.664173, + "nauc_map_at_3_max": 0.32976, + "nauc_map_at_3_std": -0.056905, + "nauc_map_at_3_diff1": 0.607185, + "nauc_map_at_5_max": 0.32383, + "nauc_map_at_5_std": -0.062695, + "nauc_map_at_5_diff1": 0.607574, + "nauc_map_at_10_max": 0.331541, + "nauc_map_at_10_std": -0.057637, + "nauc_map_at_10_diff1": 0.613536, + "nauc_map_at_20_max": 0.329293, + "nauc_map_at_20_std": -0.065479, + "nauc_map_at_20_diff1": 0.619456, + "nauc_map_at_100_max": 0.327509, + "nauc_map_at_100_std": -0.072611, + "nauc_map_at_100_diff1": 0.62301, + "nauc_map_at_1000_max": 0.327509, + "nauc_map_at_1000_std": -0.072611, + "nauc_map_at_1000_diff1": 0.62301, + "nauc_recall_at_1_max": 0.277748, + "nauc_recall_at_1_std": -0.159446, + "nauc_recall_at_1_diff1": 0.664173, + "nauc_recall_at_3_max": 0.460841, + "nauc_recall_at_3_std": 0.289535, + "nauc_recall_at_3_diff1": 0.40414, + "nauc_recall_at_5_max": 0.461057, + "nauc_recall_at_5_std": 0.376735, + "nauc_recall_at_5_diff1": 0.332602, + "nauc_recall_at_10_max": 0.593936, + "nauc_recall_at_10_std": 0.70984, + "nauc_recall_at_10_diff1": 0.223391, + "nauc_recall_at_20_max": 0.63102, + "nauc_recall_at_20_std": 0.769572, + "nauc_recall_at_20_diff1": 0.241462, + "nauc_recall_at_100_max": 0.514529, + "nauc_recall_at_100_std": 0.700814, + "nauc_recall_at_100_diff1": 0.186988, + "nauc_recall_at_1000_max": 0.514529, + "nauc_recall_at_1000_std": 0.700814, + "nauc_recall_at_1000_diff1": 0.186988, + "nauc_precision_at_1_max": 0.229549, + "nauc_precision_at_1_std": -0.397579, + "nauc_precision_at_1_diff1": 0.831279, + "nauc_precision_at_3_max": -0.106541, + "nauc_precision_at_3_std": 0.007851, + "nauc_precision_at_3_diff1": -0.251206, + "nauc_precision_at_5_max": -0.145253, + "nauc_precision_at_5_std": -0.011083, + "nauc_precision_at_5_diff1": -0.276066, + "nauc_precision_at_10_max": -0.134234, + "nauc_precision_at_10_std": -0.000724, + "nauc_precision_at_10_diff1": -0.269767, + "nauc_precision_at_20_max": -0.155461, + "nauc_precision_at_20_std": -0.039035, + "nauc_precision_at_20_diff1": -0.257761, + "nauc_precision_at_100_max": -0.189907, + "nauc_precision_at_100_std": -0.111851, + "nauc_precision_at_100_diff1": -0.237226, + "nauc_precision_at_1000_max": -0.189907, + "nauc_precision_at_1000_std": -0.111851, + "nauc_precision_at_1000_diff1": -0.237226, + "nauc_mrr_at_1_max": 0.229549, + "nauc_mrr_at_1_std": -0.397579, + "nauc_mrr_at_1_diff1": 0.831279, + "nauc_mrr_at_3_max": 0.288032, + "nauc_mrr_at_3_std": -0.368318, + "nauc_mrr_at_3_diff1": 0.837628, + "nauc_mrr_at_5_max": 0.279328, + "nauc_mrr_at_5_std": -0.375607, + "nauc_mrr_at_5_diff1": 0.837392, + "nauc_mrr_at_10_max": 0.272209, + "nauc_mrr_at_10_std": -0.377581, + "nauc_mrr_at_10_diff1": 0.836645, + "nauc_mrr_at_20_max": 0.272209, + "nauc_mrr_at_20_std": -0.377581, + "nauc_mrr_at_20_diff1": 0.836645, + "nauc_mrr_at_100_max": 0.27198, + "nauc_mrr_at_100_std": -0.377928, + "nauc_mrr_at_100_diff1": 0.836594, + "nauc_mrr_at_1000_max": 0.27198, + "nauc_mrr_at_1000_std": -0.377928, + "nauc_mrr_at_1000_diff1": 0.836594, + "main_score": 0.95113, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 611.7832398414612, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/FiQA2018.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/FiQA2018.json new file mode 100644 index 0000000000..2b8d133cf0 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/FiQA2018.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", + "task_name": "FiQA2018", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.65741, + "ndcg_at_3": 0.61626, + "ndcg_at_5": 0.63112, + "ndcg_at_10": 0.66111, + "ndcg_at_20": 0.68652, + "ndcg_at_100": 0.71295, + "ndcg_at_1000": 0.71295, + "map_at_1": 0.34637, + "map_at_3": 0.50875, + "map_at_5": 0.54763, + "map_at_10": 0.5798, + "map_at_20": 0.59259, + "map_at_100": 0.59936, + "map_at_1000": 0.59936, + "recall_at_1": 0.34637, + "recall_at_3": 0.55781, + "recall_at_5": 0.63836, + "recall_at_10": 0.73285, + "recall_at_20": 0.80867, + "recall_at_100": 0.91285, + "recall_at_1000": 0.91285, + "precision_at_1": 0.65741, + "precision_at_3": 0.41204, + "precision_at_5": 0.29784, + "precision_at_10": 0.18071, + "precision_at_20": 0.10216, + "precision_at_100": 0.02367, + "precision_at_1000": 0.00237, + "mrr_at_1": 0.657407, + "mrr_at_3": 0.711934, + "mrr_at_5": 0.725823, + "mrr_at_10": 0.731834, + "mrr_at_20": 0.735029, + "mrr_at_100": 0.73618, + "mrr_at_1000": 0.73618, + "nauc_ndcg_at_1_max": 0.60133, + "nauc_ndcg_at_1_std": 0.373812, + "nauc_ndcg_at_1_diff1": 0.649189, + "nauc_ndcg_at_3_max": 0.512998, + "nauc_ndcg_at_3_std": 0.366042, + "nauc_ndcg_at_3_diff1": 0.466868, + "nauc_ndcg_at_5_max": 0.473621, + "nauc_ndcg_at_5_std": 0.340173, + "nauc_ndcg_at_5_diff1": 0.462476, + "nauc_ndcg_at_10_max": 0.501493, + "nauc_ndcg_at_10_std": 0.371637, + "nauc_ndcg_at_10_diff1": 0.493497, + "nauc_ndcg_at_20_max": 0.516934, + "nauc_ndcg_at_20_std": 0.393167, + "nauc_ndcg_at_20_diff1": 0.504597, + "nauc_ndcg_at_100_max": 0.527854, + "nauc_ndcg_at_100_std": 0.400531, + "nauc_ndcg_at_100_diff1": 0.500517, + "nauc_ndcg_at_1000_max": 0.527854, + "nauc_ndcg_at_1000_std": 0.400531, + "nauc_ndcg_at_1000_diff1": 0.500517, + "nauc_map_at_1_max": 0.181046, + "nauc_map_at_1_std": 0.022273, + "nauc_map_at_1_diff1": 0.536018, + "nauc_map_at_3_max": 0.367718, + "nauc_map_at_3_std": 0.198235, + "nauc_map_at_3_diff1": 0.486053, + "nauc_map_at_5_max": 0.406729, + "nauc_map_at_5_std": 0.255813, + "nauc_map_at_5_diff1": 0.46897, + "nauc_map_at_10_max": 0.454292, + "nauc_map_at_10_std": 0.317929, + "nauc_map_at_10_diff1": 0.474531, + "nauc_map_at_20_max": 0.465263, + "nauc_map_at_20_std": 0.334804, + "nauc_map_at_20_diff1": 0.476112, + "nauc_map_at_100_max": 0.469487, + "nauc_map_at_100_std": 0.339214, + "nauc_map_at_100_diff1": 0.474904, + "nauc_map_at_1000_max": 0.469487, + "nauc_map_at_1000_std": 0.339214, + "nauc_map_at_1000_diff1": 0.474904, + "nauc_recall_at_1_max": 0.181046, + "nauc_recall_at_1_std": 0.022273, + "nauc_recall_at_1_diff1": 0.536018, + "nauc_recall_at_3_max": 0.322155, + "nauc_recall_at_3_std": 0.185121, + "nauc_recall_at_3_diff1": 0.412036, + "nauc_recall_at_5_max": 0.320721, + "nauc_recall_at_5_std": 0.217148, + "nauc_recall_at_5_diff1": 0.373236, + "nauc_recall_at_10_max": 0.418821, + "nauc_recall_at_10_std": 0.350604, + "nauc_recall_at_10_diff1": 0.396705, + "nauc_recall_at_20_max": 0.427282, + "nauc_recall_at_20_std": 0.419752, + "nauc_recall_at_20_diff1": 0.403585, + "nauc_recall_at_100_max": 0.420303, + "nauc_recall_at_100_std": 0.487616, + "nauc_recall_at_100_diff1": 0.292163, + "nauc_recall_at_1000_max": 0.420303, + "nauc_recall_at_1000_std": 0.487616, + "nauc_recall_at_1000_diff1": 0.292163, + "nauc_precision_at_1_max": 0.60133, + "nauc_precision_at_1_std": 0.373812, + "nauc_precision_at_1_diff1": 0.649189, + "nauc_precision_at_3_max": 0.51054, + "nauc_precision_at_3_std": 0.466791, + "nauc_precision_at_3_diff1": 0.134252, + "nauc_precision_at_5_max": 0.458631, + "nauc_precision_at_5_std": 0.480277, + "nauc_precision_at_5_diff1": 0.023693, + "nauc_precision_at_10_max": 0.434295, + "nauc_precision_at_10_std": 0.508137, + "nauc_precision_at_10_diff1": -0.032679, + "nauc_precision_at_20_max": 0.396462, + "nauc_precision_at_20_std": 0.495029, + "nauc_precision_at_20_diff1": -0.079782, + "nauc_precision_at_100_max": 0.31047, + "nauc_precision_at_100_std": 0.400515, + "nauc_precision_at_100_diff1": -0.156443, + "nauc_precision_at_1000_max": 0.31047, + "nauc_precision_at_1000_std": 0.400515, + "nauc_precision_at_1000_diff1": -0.156443, + "nauc_mrr_at_1_max": 0.60133, + "nauc_mrr_at_1_std": 0.373812, + "nauc_mrr_at_1_diff1": 0.649189, + "nauc_mrr_at_3_max": 0.616896, + "nauc_mrr_at_3_std": 0.422952, + "nauc_mrr_at_3_diff1": 0.614689, + "nauc_mrr_at_5_max": 0.611709, + "nauc_mrr_at_5_std": 0.414266, + "nauc_mrr_at_5_diff1": 0.618661, + "nauc_mrr_at_10_max": 0.617626, + "nauc_mrr_at_10_std": 0.423826, + "nauc_mrr_at_10_diff1": 0.621655, + "nauc_mrr_at_20_max": 0.617488, + "nauc_mrr_at_20_std": 0.422414, + "nauc_mrr_at_20_diff1": 0.62263, + "nauc_mrr_at_100_max": 0.617824, + "nauc_mrr_at_100_std": 0.422786, + "nauc_mrr_at_100_diff1": 0.623004, + "nauc_mrr_at_1000_max": 0.617824, + "nauc_mrr_at_1000_std": 0.422786, + "nauc_mrr_at_1000_diff1": 0.623004, + "main_score": 0.66111, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 130.8485324382782, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/HotpotQAHardNegatives.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/HotpotQAHardNegatives.json new file mode 100644 index 0000000000..91389a4f74 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/HotpotQAHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "617612fa63afcb60e3b134bed8b7216a99707c37", + "task_name": "HotpotQAHardNegatives", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.903, + "ndcg_at_3": 0.83442, + "ndcg_at_5": 0.85656, + "ndcg_at_10": 0.87505, + "ndcg_at_20": 0.88143, + "ndcg_at_100": 0.89143, + "ndcg_at_1000": 0.89143, + "map_at_1": 0.4515, + "map_at_3": 0.79125, + "map_at_5": 0.8103, + "map_at_10": 0.82217, + "map_at_20": 0.82494, + "map_at_100": 0.82732, + "map_at_1000": 0.82732, + "recall_at_1": 0.4515, + "recall_at_3": 0.8265, + "recall_at_5": 0.87, + "recall_at_10": 0.9165, + "recall_at_20": 0.937, + "recall_at_100": 0.9795, + "recall_at_1000": 0.9795, + "precision_at_1": 0.903, + "precision_at_3": 0.551, + "precision_at_5": 0.348, + "precision_at_10": 0.1833, + "precision_at_20": 0.0937, + "precision_at_100": 0.01959, + "precision_at_1000": 0.00196, + "mrr_at_1": 0.903, + "mrr_at_3": 0.9345, + "mrr_at_5": 0.9368, + "mrr_at_10": 0.938056, + "mrr_at_20": 0.938253, + "mrr_at_100": 0.938417, + "mrr_at_1000": 0.938417, + "nauc_ndcg_at_1_max": 0.56767, + "nauc_ndcg_at_1_std": 0.354727, + "nauc_ndcg_at_1_diff1": 0.614634, + "nauc_ndcg_at_3_max": 0.490098, + "nauc_ndcg_at_3_std": 0.486246, + "nauc_ndcg_at_3_diff1": 0.086946, + "nauc_ndcg_at_5_max": 0.498261, + "nauc_ndcg_at_5_std": 0.513091, + "nauc_ndcg_at_5_diff1": 0.087253, + "nauc_ndcg_at_10_max": 0.495892, + "nauc_ndcg_at_10_std": 0.534143, + "nauc_ndcg_at_10_diff1": 0.096344, + "nauc_ndcg_at_20_max": 0.503389, + "nauc_ndcg_at_20_std": 0.530302, + "nauc_ndcg_at_20_diff1": 0.12041, + "nauc_ndcg_at_100_max": 0.496569, + "nauc_ndcg_at_100_std": 0.509378, + "nauc_ndcg_at_100_diff1": 0.124802, + "nauc_ndcg_at_1000_max": 0.496569, + "nauc_ndcg_at_1000_std": 0.509378, + "nauc_ndcg_at_1000_diff1": 0.124802, + "nauc_map_at_1_max": 0.56767, + "nauc_map_at_1_std": 0.354727, + "nauc_map_at_1_diff1": 0.614634, + "nauc_map_at_3_max": 0.462348, + "nauc_map_at_3_std": 0.472783, + "nauc_map_at_3_diff1": 0.038458, + "nauc_map_at_5_max": 0.468908, + "nauc_map_at_5_std": 0.491957, + "nauc_map_at_5_diff1": 0.040052, + "nauc_map_at_10_max": 0.46763, + "nauc_map_at_10_std": 0.50234, + "nauc_map_at_10_diff1": 0.044043, + "nauc_map_at_20_max": 0.470132, + "nauc_map_at_20_std": 0.500732, + "nauc_map_at_20_diff1": 0.05131, + "nauc_map_at_100_max": 0.468773, + "nauc_map_at_100_std": 0.496988, + "nauc_map_at_100_diff1": 0.050901, + "nauc_map_at_1000_max": 0.468773, + "nauc_map_at_1000_std": 0.496988, + "nauc_map_at_1000_diff1": 0.050901, + "nauc_recall_at_1_max": 0.56767, + "nauc_recall_at_1_std": 0.354727, + "nauc_recall_at_1_diff1": 0.614634, + "nauc_recall_at_3_max": 0.4847, + "nauc_recall_at_3_std": 0.523256, + "nauc_recall_at_3_diff1": -0.003814, + "nauc_recall_at_5_max": 0.494499, + "nauc_recall_at_5_std": 0.582966, + "nauc_recall_at_5_diff1": -0.029169, + "nauc_recall_at_10_max": 0.481673, + "nauc_recall_at_10_std": 0.67983, + "nauc_recall_at_10_diff1": -0.054239, + "nauc_recall_at_20_max": 0.51631, + "nauc_recall_at_20_std": 0.702847, + "nauc_recall_at_20_diff1": 0.029696, + "nauc_recall_at_100_max": 0.470178, + "nauc_recall_at_100_std": 0.712498, + "nauc_recall_at_100_diff1": -0.025631, + "nauc_recall_at_1000_max": 0.470178, + "nauc_recall_at_1000_std": 0.712498, + "nauc_recall_at_1000_diff1": -0.025631, + "nauc_precision_at_1_max": 0.56767, + "nauc_precision_at_1_std": 0.354727, + "nauc_precision_at_1_diff1": 0.614634, + "nauc_precision_at_3_max": 0.4847, + "nauc_precision_at_3_std": 0.523256, + "nauc_precision_at_3_diff1": -0.003814, + "nauc_precision_at_5_max": 0.494499, + "nauc_precision_at_5_std": 0.582966, + "nauc_precision_at_5_diff1": -0.029169, + "nauc_precision_at_10_max": 0.481673, + "nauc_precision_at_10_std": 0.67983, + "nauc_precision_at_10_diff1": -0.054239, + "nauc_precision_at_20_max": 0.51631, + "nauc_precision_at_20_std": 0.702847, + "nauc_precision_at_20_diff1": 0.029696, + "nauc_precision_at_100_max": 0.470178, + "nauc_precision_at_100_std": 0.712498, + "nauc_precision_at_100_diff1": -0.025631, + "nauc_precision_at_1000_max": 0.470178, + "nauc_precision_at_1000_std": 0.712498, + "nauc_precision_at_1000_diff1": -0.025631, + "nauc_mrr_at_1_max": 0.56767, + "nauc_mrr_at_1_std": 0.354727, + "nauc_mrr_at_1_diff1": 0.614634, + "nauc_mrr_at_3_max": 0.639309, + "nauc_mrr_at_3_std": 0.423909, + "nauc_mrr_at_3_diff1": 0.639046, + "nauc_mrr_at_5_max": 0.62836, + "nauc_mrr_at_5_std": 0.415235, + "nauc_mrr_at_5_diff1": 0.632263, + "nauc_mrr_at_10_max": 0.624561, + "nauc_mrr_at_10_std": 0.414827, + "nauc_mrr_at_10_diff1": 0.631704, + "nauc_mrr_at_20_max": 0.623714, + "nauc_mrr_at_20_std": 0.413835, + "nauc_mrr_at_20_diff1": 0.632245, + "nauc_mrr_at_100_max": 0.622805, + "nauc_mrr_at_100_std": 0.412435, + "nauc_mrr_at_100_diff1": 0.631493, + "nauc_mrr_at_1000_max": 0.622805, + "nauc_mrr_at_1000_std": 0.412435, + "nauc_mrr_at_1000_diff1": 0.631493, + "main_score": 0.87505, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 395.8876428604126, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/IFlyTek.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/IFlyTek.json new file mode 100644 index 0000000000..2a9fb05f04 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/IFlyTek.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "421605374b29664c5fc098418fe20ada9bd55f8a", + "task_name": "IFlyTek", + "mteb_version": "1.36.5", + "scores": { + "validation": [ + { + "accuracy": 0.556214, + "f1": 0.373724, + "f1_weighted": 0.544264, + "scores_per_experiment": [ + { + "accuracy": 0.5606, + "f1": 0.366592, + "f1_weighted": 0.548964 + }, + { + "accuracy": 0.553674, + "f1": 0.378529, + "f1_weighted": 0.540646 + }, + { + "accuracy": 0.554829, + "f1": 0.377535, + "f1_weighted": 0.544027 + }, + { + "accuracy": 0.555983, + "f1": 0.382325, + "f1_weighted": 0.543534 + }, + { + "accuracy": 0.555983, + "f1": 0.36364, + "f1_weighted": 0.544146 + } + ], + "main_score": 0.556214, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 80.71388745307922, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ImdbClassification.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ImdbClassification.json new file mode 100644 index 0000000000..a28aac37a2 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ImdbClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7", + "task_name": "ImdbClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.971348, + "f1": 0.971343, + "f1_weighted": 0.971343, + "ap": 0.956574, + "ap_weighted": 0.956574, + "scores_per_experiment": [ + { + "accuracy": 0.97, + "f1": 0.969994, + "f1_weighted": 0.969994, + "ap": 0.962376, + "ap_weighted": 0.962376 + }, + { + "accuracy": 0.96792, + "f1": 0.96791, + "f1_weighted": 0.96791, + "ap": 0.96101, + "ap_weighted": 0.96101 + }, + { + "accuracy": 0.97104, + "f1": 0.971035, + "f1_weighted": 0.971035, + "ap": 0.951658, + "ap_weighted": 0.951658 + }, + { + "accuracy": 0.97332, + "f1": 0.973318, + "f1_weighted": 0.973318, + "ap": 0.956454, + "ap_weighted": 0.956454 + }, + { + "accuracy": 0.97312, + "f1": 0.973119, + "f1_weighted": 0.973119, + "ap": 0.962426, + "ap_weighted": 0.962426 + }, + { + "accuracy": 0.97232, + "f1": 0.972319, + "f1_weighted": 0.972319, + "ap": 0.95709, + "ap_weighted": 0.95709 + }, + { + "accuracy": 0.97032, + "f1": 0.970312, + "f1_weighted": 0.970312, + "ap": 0.949369, + "ap_weighted": 0.949369 + }, + { + "accuracy": 0.96924, + "f1": 0.969231, + "f1_weighted": 0.969231, + "ap": 0.947434, + "ap_weighted": 0.947434 + }, + { + "accuracy": 0.97264, + "f1": 0.972637, + "f1_weighted": 0.972637, + "ap": 0.955088, + "ap_weighted": 0.955088 + }, + { + "accuracy": 0.97356, + "f1": 0.97356, + "f1_weighted": 0.97356, + "ap": 0.962829, + "ap_weighted": 0.962829 + } + ], + "main_score": 0.971348, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 114.73927187919617, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/JDReview.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/JDReview.json new file mode 100644 index 0000000000..46e78f61df --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/JDReview.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b", + "task_name": "JDReview", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.894371, + "f1": 0.847786, + "f1_weighted": 0.900697, + "ap": 0.615631, + "ap_weighted": 0.615631, + "scores_per_experiment": [ + { + "accuracy": 0.902439, + "f1": 0.854939, + "f1_weighted": 0.907111, + "ap": 0.627012, + "ap_weighted": 0.627012 + }, + { + "accuracy": 0.893058, + "f1": 0.845368, + "f1_weighted": 0.899342, + "ap": 0.609771, + "ap_weighted": 0.609771 + }, + { + "accuracy": 0.893058, + "f1": 0.846291, + "f1_weighted": 0.89958, + "ap": 0.612484, + "ap_weighted": 0.612484 + }, + { + "accuracy": 0.88743, + "f1": 0.83868, + "f1_weighted": 0.894418, + "ap": 0.597156, + "ap_weighted": 0.597156 + }, + { + "accuracy": 0.891182, + "f1": 0.845868, + "f1_weighted": 0.898395, + "ap": 0.613765, + "ap_weighted": 0.613765 + }, + { + "accuracy": 0.889306, + "f1": 0.844547, + "f1_weighted": 0.896974, + "ap": 0.612438, + "ap_weighted": 0.612438 + }, + { + "accuracy": 0.908068, + "f1": 0.862869, + "f1_weighted": 0.912351, + "ap": 0.643841, + "ap_weighted": 0.643841 + }, + { + "accuracy": 0.904315, + "f1": 0.85818, + "f1_weighted": 0.90902, + "ap": 0.634224, + "ap_weighted": 0.634224 + }, + { + "accuracy": 0.889306, + "f1": 0.841836, + "f1_weighted": 0.896296, + "ap": 0.604069, + "ap_weighted": 0.604069 + }, + { + "accuracy": 0.885553, + "f1": 0.839278, + "f1_weighted": 0.893482, + "ap": 0.601552, + "ap_weighted": 0.601552 + } + ], + "main_score": 0.894371, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 10.303980827331543, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/LCQMC.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/LCQMC.json new file mode 100644 index 0000000000..8458e8900b --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/LCQMC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "17f9b096f80380fce5ed12a9be8be7784b337daf", + "task_name": "LCQMC", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.784684, + "spearman": 0.807269, + "cosine_pearson": 0.784684, + "cosine_spearman": 0.807269, + "manhattan_pearson": 0.815972, + "manhattan_spearman": 0.806909, + "euclidean_pearson": 0.816827, + "euclidean_spearman": 0.807263, + "main_score": 0.807269, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 27.864264488220215, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MMarcoReranking.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MMarcoReranking.json new file mode 100644 index 0000000000..2e24c7c0a5 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MMarcoReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6", + "task_name": "MMarcoReranking", + "mteb_version": "1.38.1", + "scores": { + "dev": [ + { + "map": 0.364721, + "mrr": 0.352869, + "nAUC_map_max": 0.087113, + "nAUC_map_std": -0.311067, + "nAUC_map_diff1": 0.225131, + "nAUC_mrr_max": 0.097595, + "nAUC_mrr_std": -0.304086, + "nAUC_mrr_diff1": 0.223243, + "main_score": 0.364721, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 753.0943818092346, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MMarcoRetrieval.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MMarcoRetrieval.json new file mode 100644 index 0000000000..bef066690b --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MMarcoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "539bbde593d947e2a124ba72651aafc09eb33fc2", + "task_name": "MMarcoRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.80659, + "ndcg_at_3": 0.86358, + "ndcg_at_5": 0.87869, + "ndcg_at_10": 0.88771, + "ndcg_at_20": 0.89155, + "ndcg_at_100": 0.89389, + "ndcg_at_1000": 0.89389, + "map_at_1": 0.78172, + "map_at_3": 0.84745, + "map_at_5": 0.85666, + "map_at_10": 0.86077, + "map_at_20": 0.86193, + "map_at_100": 0.86231, + "map_at_1000": 0.86231, + "recall_at_1": 0.78172, + "recall_at_3": 0.90351, + "recall_at_5": 0.93923, + "recall_at_10": 0.96589, + "recall_at_20": 0.98049, + "recall_at_100": 0.99234, + "recall_at_1000": 0.99234, + "precision_at_1": 0.80659, + "precision_at_3": 0.31877, + "precision_at_5": 0.19954, + "precision_at_10": 0.10275, + "precision_at_20": 0.05221, + "precision_at_100": 0.01057, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.80659, + "mrr_at_3": 0.853558, + "mrr_at_5": 0.861051, + "mrr_at_10": 0.864547, + "mrr_at_20": 0.86551, + "mrr_at_100": 0.865831, + "mrr_at_1000": 0.865831, + "nauc_ndcg_at_1_max": 0.393267, + "nauc_ndcg_at_1_std": -0.291631, + "nauc_ndcg_at_1_diff1": 0.8522, + "nauc_ndcg_at_3_max": 0.455721, + "nauc_ndcg_at_3_std": -0.210388, + "nauc_ndcg_at_3_diff1": 0.832866, + "nauc_ndcg_at_5_max": 0.469954, + "nauc_ndcg_at_5_std": -0.171952, + "nauc_ndcg_at_5_diff1": 0.836739, + "nauc_ndcg_at_10_max": 0.471652, + "nauc_ndcg_at_10_std": -0.159934, + "nauc_ndcg_at_10_diff1": 0.839367, + "nauc_ndcg_at_20_max": 0.463681, + "nauc_ndcg_at_20_std": -0.172077, + "nauc_ndcg_at_20_diff1": 0.840854, + "nauc_ndcg_at_100_max": 0.455933, + "nauc_ndcg_at_100_std": -0.189382, + "nauc_ndcg_at_100_diff1": 0.841678, + "nauc_ndcg_at_1000_max": 0.455933, + "nauc_ndcg_at_1000_std": -0.189382, + "nauc_ndcg_at_1000_diff1": 0.841678, + "nauc_map_at_1_max": 0.346827, + "nauc_map_at_1_std": -0.319482, + "nauc_map_at_1_diff1": 0.84523, + "nauc_map_at_3_max": 0.433768, + "nauc_map_at_3_std": -0.240327, + "nauc_map_at_3_diff1": 0.836531, + "nauc_map_at_5_max": 0.441551, + "nauc_map_at_5_std": -0.221193, + "nauc_map_at_5_diff1": 0.838773, + "nauc_map_at_10_max": 0.441872, + "nauc_map_at_10_std": -0.217264, + "nauc_map_at_10_diff1": 0.840263, + "nauc_map_at_20_max": 0.440001, + "nauc_map_at_20_std": -0.220227, + "nauc_map_at_20_diff1": 0.840773, + "nauc_map_at_100_max": 0.43898, + "nauc_map_at_100_std": -0.222407, + "nauc_map_at_100_diff1": 0.840876, + "nauc_map_at_1000_max": 0.43898, + "nauc_map_at_1000_std": -0.222407, + "nauc_map_at_1000_diff1": 0.840876, + "nauc_recall_at_1_max": 0.346827, + "nauc_recall_at_1_std": -0.319482, + "nauc_recall_at_1_diff1": 0.84523, + "nauc_recall_at_3_max": 0.530468, + "nauc_recall_at_3_std": -0.099997, + "nauc_recall_at_3_diff1": 0.804443, + "nauc_recall_at_5_max": 0.638799, + "nauc_recall_at_5_std": 0.137938, + "nauc_recall_at_5_diff1": 0.808434, + "nauc_recall_at_10_max": 0.782754, + "nauc_recall_at_10_std": 0.478845, + "nauc_recall_at_10_diff1": 0.8115, + "nauc_recall_at_20_max": 0.84982, + "nauc_recall_at_20_std": 0.698682, + "nauc_recall_at_20_diff1": 0.821107, + "nauc_recall_at_100_max": 0.933035, + "nauc_recall_at_100_std": 0.86997, + "nauc_recall_at_100_diff1": 0.854237, + "nauc_recall_at_1000_max": 0.933035, + "nauc_recall_at_1000_std": 0.86997, + "nauc_recall_at_1000_diff1": 0.854237, + "nauc_precision_at_1_max": 0.393267, + "nauc_precision_at_1_std": -0.291631, + "nauc_precision_at_1_diff1": 0.8522, + "nauc_precision_at_3_max": 0.326107, + "nauc_precision_at_3_std": 0.087135, + "nauc_precision_at_3_diff1": 0.269403, + "nauc_precision_at_5_max": 0.279282, + "nauc_precision_at_5_std": 0.237818, + "nauc_precision_at_5_diff1": 0.089151, + "nauc_precision_at_10_max": 0.215291, + "nauc_precision_at_10_std": 0.33503, + "nauc_precision_at_10_diff1": -0.07293, + "nauc_precision_at_20_max": 0.141023, + "nauc_precision_at_20_std": 0.333521, + "nauc_precision_at_20_diff1": -0.179956, + "nauc_precision_at_100_max": 0.062475, + "nauc_precision_at_100_std": 0.291982, + "nauc_precision_at_100_diff1": -0.278737, + "nauc_precision_at_1000_max": 0.062475, + "nauc_precision_at_1000_std": 0.291982, + "nauc_precision_at_1000_diff1": -0.278737, + "nauc_mrr_at_1_max": 0.393267, + "nauc_mrr_at_1_std": -0.291631, + "nauc_mrr_at_1_diff1": 0.8522, + "nauc_mrr_at_3_max": 0.444134, + "nauc_mrr_at_3_std": -0.227309, + "nauc_mrr_at_3_diff1": 0.842415, + "nauc_mrr_at_5_max": 0.446611, + "nauc_mrr_at_5_std": -0.214289, + "nauc_mrr_at_5_diff1": 0.8441, + "nauc_mrr_at_10_max": 0.446376, + "nauc_mrr_at_10_std": -0.21216, + "nauc_mrr_at_10_diff1": 0.844841, + "nauc_mrr_at_20_max": 0.444459, + "nauc_mrr_at_20_std": -0.215075, + "nauc_mrr_at_20_diff1": 0.845113, + "nauc_mrr_at_100_max": 0.443522, + "nauc_mrr_at_100_std": -0.217186, + "nauc_mrr_at_100_diff1": 0.845197, + "nauc_mrr_at_1000_max": 0.443522, + "nauc_mrr_at_1000_std": -0.217186, + "nauc_mrr_at_1000_diff1": 0.845197, + "main_score": 0.88771, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 248.3955385684967, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MTOPDomainClassification.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MTOPDomainClassification.json new file mode 100644 index 0000000000..bcc55d9c86 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MTOPDomainClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", + "task_name": "MTOPDomainClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.993092, + "f1": 0.992212, + "f1_weighted": 0.993095, + "scores_per_experiment": [ + { + "accuracy": 0.993388, + "f1": 0.992559, + "f1_weighted": 0.993394 + }, + { + "accuracy": 0.992932, + "f1": 0.992012, + "f1_weighted": 0.992935 + }, + { + "accuracy": 0.992932, + "f1": 0.99193, + "f1_weighted": 0.992936 + }, + { + "accuracy": 0.992932, + "f1": 0.991917, + "f1_weighted": 0.992938 + }, + { + "accuracy": 0.992932, + "f1": 0.991948, + "f1_weighted": 0.992936 + }, + { + "accuracy": 0.99316, + "f1": 0.9923, + "f1_weighted": 0.993163 + }, + { + "accuracy": 0.993388, + "f1": 0.992702, + "f1_weighted": 0.99339 + }, + { + "accuracy": 0.992932, + "f1": 0.992035, + "f1_weighted": 0.992935 + }, + { + "accuracy": 0.992704, + "f1": 0.991811, + "f1_weighted": 0.992708 + }, + { + "accuracy": 0.993616, + "f1": 0.99291, + "f1_weighted": 0.993619 + } + ], + "main_score": 0.993092, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14.000364303588867, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MassiveIntentClassification.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MassiveIntentClassification.json new file mode 100644 index 0000000000..b3813a9a50 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MassiveIntentClassification.json @@ -0,0 +1,135 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "task_name": "MassiveIntentClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.870746, + "f1": 0.843495, + "f1_weighted": 0.862711, + "scores_per_experiment": [ + { + "accuracy": 0.868863, + "f1": 0.839854, + "f1_weighted": 0.859451 + }, + { + "accuracy": 0.869536, + "f1": 0.849327, + "f1_weighted": 0.862286 + }, + { + "accuracy": 0.863147, + "f1": 0.836531, + "f1_weighted": 0.850233 + }, + { + "accuracy": 0.870545, + "f1": 0.844792, + "f1_weighted": 0.862346 + }, + { + "accuracy": 0.85844, + "f1": 0.826055, + "f1_weighted": 0.847129 + }, + { + "accuracy": 0.878615, + "f1": 0.851093, + "f1_weighted": 0.875158 + }, + { + "accuracy": 0.864492, + "f1": 0.840487, + "f1_weighted": 0.853137 + }, + { + "accuracy": 0.871217, + "f1": 0.841511, + "f1_weighted": 0.862727 + }, + { + "accuracy": 0.879623, + "f1": 0.850761, + "f1_weighted": 0.87456 + }, + { + "accuracy": 0.882986, + "f1": 0.854543, + "f1_weighted": 0.880078 + } + ], + "main_score": 0.870746, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.849361, + "f1": 0.813451, + "f1_weighted": 0.840686, + "scores_per_experiment": [ + { + "accuracy": 0.860121, + "f1": 0.818744, + "f1_weighted": 0.854465 + }, + { + "accuracy": 0.848352, + "f1": 0.817926, + "f1_weighted": 0.837766 + }, + { + "accuracy": 0.8423, + "f1": 0.80507, + "f1_weighted": 0.829994 + }, + { + "accuracy": 0.847007, + "f1": 0.815023, + "f1_weighted": 0.838647 + }, + { + "accuracy": 0.83961, + "f1": 0.806622, + "f1_weighted": 0.828101 + }, + { + "accuracy": 0.855414, + "f1": 0.817438, + "f1_weighted": 0.851863 + }, + { + "accuracy": 0.84499, + "f1": 0.811117, + "f1_weighted": 0.831874 + }, + { + "accuracy": 0.848352, + "f1": 0.813196, + "f1_weighted": 0.839975 + }, + { + "accuracy": 0.854069, + "f1": 0.814282, + "f1_weighted": 0.848964 + }, + { + "accuracy": 0.853396, + "f1": 0.815088, + "f1_weighted": 0.84521 + } + ], + "main_score": 0.849361, + "hf_subset": "zh-CN", + "languages": [ + "cmo-Hans" + ] + } + ] + }, + "evaluation_time": 35.69178891181946, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MassiveScenarioClassification.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MassiveScenarioClassification.json new file mode 100644 index 0000000000..ff4ed976c3 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MassiveScenarioClassification.json @@ -0,0 +1,135 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "task_name": "MassiveScenarioClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.930531, + "f1": 0.923812, + "f1_weighted": 0.930335, + "scores_per_experiment": [ + { + "accuracy": 0.929724, + "f1": 0.924, + "f1_weighted": 0.929231 + }, + { + "accuracy": 0.931069, + "f1": 0.924416, + "f1_weighted": 0.931035 + }, + { + "accuracy": 0.931406, + "f1": 0.924144, + "f1_weighted": 0.931003 + }, + { + "accuracy": 0.930061, + "f1": 0.923352, + "f1_weighted": 0.929555 + }, + { + "accuracy": 0.930061, + "f1": 0.924079, + "f1_weighted": 0.929751 + }, + { + "accuracy": 0.930061, + "f1": 0.923063, + "f1_weighted": 0.929796 + }, + { + "accuracy": 0.930397, + "f1": 0.922911, + "f1_weighted": 0.930417 + }, + { + "accuracy": 0.930733, + "f1": 0.923113, + "f1_weighted": 0.930497 + }, + { + "accuracy": 0.930061, + "f1": 0.923304, + "f1_weighted": 0.929996 + }, + { + "accuracy": 0.931742, + "f1": 0.925735, + "f1_weighted": 0.932065 + } + ], + "main_score": 0.930531, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.916073, + "f1": 0.911465, + "f1_weighted": 0.915572, + "scores_per_experiment": [ + { + "accuracy": 0.917956, + "f1": 0.914664, + "f1_weighted": 0.917407 + }, + { + "accuracy": 0.916611, + "f1": 0.912294, + "f1_weighted": 0.916526 + }, + { + "accuracy": 0.918628, + "f1": 0.913962, + "f1_weighted": 0.917729 + }, + { + "accuracy": 0.916611, + "f1": 0.911481, + "f1_weighted": 0.915672 + }, + { + "accuracy": 0.913921, + "f1": 0.908905, + "f1_weighted": 0.913441 + }, + { + "accuracy": 0.914929, + "f1": 0.909861, + "f1_weighted": 0.914292 + }, + { + "accuracy": 0.916611, + "f1": 0.911714, + "f1_weighted": 0.915911 + }, + { + "accuracy": 0.914593, + "f1": 0.908262, + "f1_weighted": 0.91397 + }, + { + "accuracy": 0.914929, + "f1": 0.911247, + "f1_weighted": 0.914746 + }, + { + "accuracy": 0.915938, + "f1": 0.912264, + "f1_weighted": 0.916029 + } + ], + "main_score": 0.916073, + "hf_subset": "zh-CN", + "languages": [ + "cmo-Hans" + ] + } + ] + }, + "evaluation_time": 28.163952350616455, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedicalRetrieval.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedicalRetrieval.json new file mode 100644 index 0000000000..6a36da6f72 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedicalRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6", + "task_name": "MedicalRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.625, + "ndcg_at_3": 0.68667, + "ndcg_at_5": 0.69872, + "ndcg_at_10": 0.71032, + "ndcg_at_20": 0.72061, + "ndcg_at_100": 0.73618, + "ndcg_at_1000": 0.73618, + "map_at_1": 0.625, + "map_at_3": 0.67167, + "map_at_5": 0.67842, + "map_at_10": 0.68319, + "map_at_20": 0.68597, + "map_at_100": 0.6882, + "map_at_1000": 0.6882, + "recall_at_1": 0.625, + "recall_at_3": 0.73, + "recall_at_5": 0.759, + "recall_at_10": 0.795, + "recall_at_20": 0.836, + "recall_at_100": 0.919, + "recall_at_1000": 0.919, + "precision_at_1": 0.625, + "precision_at_3": 0.24333, + "precision_at_5": 0.1518, + "precision_at_10": 0.0795, + "precision_at_20": 0.0418, + "precision_at_100": 0.00919, + "precision_at_1000": 0.00092, + "mrr_at_1": 0.625, + "mrr_at_3": 0.671667, + "mrr_at_5": 0.678417, + "mrr_at_10": 0.683188, + "mrr_at_20": 0.68597, + "mrr_at_100": 0.688199, + "mrr_at_1000": 0.688199, + "nauc_ndcg_at_1_max": 0.80072, + "nauc_ndcg_at_1_std": 0.636977, + "nauc_ndcg_at_1_diff1": 0.875459, + "nauc_ndcg_at_3_max": 0.806118, + "nauc_ndcg_at_3_std": 0.684007, + "nauc_ndcg_at_3_diff1": 0.829828, + "nauc_ndcg_at_5_max": 0.800808, + "nauc_ndcg_at_5_std": 0.688087, + "nauc_ndcg_at_5_diff1": 0.823982, + "nauc_ndcg_at_10_max": 0.797286, + "nauc_ndcg_at_10_std": 0.693212, + "nauc_ndcg_at_10_diff1": 0.820978, + "nauc_ndcg_at_20_max": 0.796819, + "nauc_ndcg_at_20_std": 0.693453, + "nauc_ndcg_at_20_diff1": 0.822419, + "nauc_ndcg_at_100_max": 0.79652, + "nauc_ndcg_at_100_std": 0.689375, + "nauc_ndcg_at_100_diff1": 0.823915, + "nauc_ndcg_at_1000_max": 0.79652, + "nauc_ndcg_at_1000_std": 0.689375, + "nauc_ndcg_at_1000_diff1": 0.823915, + "nauc_map_at_1_max": 0.80072, + "nauc_map_at_1_std": 0.636977, + "nauc_map_at_1_diff1": 0.875459, + "nauc_map_at_3_max": 0.805287, + "nauc_map_at_3_std": 0.672239, + "nauc_map_at_3_diff1": 0.841009, + "nauc_map_at_5_max": 0.802456, + "nauc_map_at_5_std": 0.674199, + "nauc_map_at_5_diff1": 0.837977, + "nauc_map_at_10_max": 0.800941, + "nauc_map_at_10_std": 0.675967, + "nauc_map_at_10_diff1": 0.836955, + "nauc_map_at_20_max": 0.800777, + "nauc_map_at_20_std": 0.67587, + "nauc_map_at_20_diff1": 0.837403, + "nauc_map_at_100_max": 0.80068, + "nauc_map_at_100_std": 0.675277, + "nauc_map_at_100_diff1": 0.837684, + "nauc_map_at_1000_max": 0.80068, + "nauc_map_at_1000_std": 0.675277, + "nauc_map_at_1000_diff1": 0.837684, + "nauc_recall_at_1_max": 0.80072, + "nauc_recall_at_1_std": 0.636977, + "nauc_recall_at_1_diff1": 0.875459, + "nauc_recall_at_3_max": 0.808589, + "nauc_recall_at_3_std": 0.722729, + "nauc_recall_at_3_diff1": 0.793157, + "nauc_recall_at_5_max": 0.793692, + "nauc_recall_at_5_std": 0.738055, + "nauc_recall_at_5_diff1": 0.773635, + "nauc_recall_at_10_max": 0.779882, + "nauc_recall_at_10_std": 0.765602, + "nauc_recall_at_10_diff1": 0.755047, + "nauc_recall_at_20_max": 0.774165, + "nauc_recall_at_20_std": 0.783294, + "nauc_recall_at_20_diff1": 0.749682, + "nauc_recall_at_100_max": 0.752481, + "nauc_recall_at_100_std": 0.80255, + "nauc_recall_at_100_diff1": 0.70579, + "nauc_recall_at_1000_max": 0.752481, + "nauc_recall_at_1000_std": 0.80255, + "nauc_recall_at_1000_diff1": 0.70579, + "nauc_precision_at_1_max": 0.80072, + "nauc_precision_at_1_std": 0.636977, + "nauc_precision_at_1_diff1": 0.875459, + "nauc_precision_at_3_max": 0.808589, + "nauc_precision_at_3_std": 0.722729, + "nauc_precision_at_3_diff1": 0.793157, + "nauc_precision_at_5_max": 0.793692, + "nauc_precision_at_5_std": 0.738055, + "nauc_precision_at_5_diff1": 0.773635, + "nauc_precision_at_10_max": 0.779882, + "nauc_precision_at_10_std": 0.765602, + "nauc_precision_at_10_diff1": 0.755047, + "nauc_precision_at_20_max": 0.774165, + "nauc_precision_at_20_std": 0.783294, + "nauc_precision_at_20_diff1": 0.749682, + "nauc_precision_at_100_max": 0.752481, + "nauc_precision_at_100_std": 0.80255, + "nauc_precision_at_100_diff1": 0.70579, + "nauc_precision_at_1000_max": 0.752481, + "nauc_precision_at_1000_std": 0.80255, + "nauc_precision_at_1000_diff1": 0.70579, + "nauc_mrr_at_1_max": 0.80072, + "nauc_mrr_at_1_std": 0.636977, + "nauc_mrr_at_1_diff1": 0.875459, + "nauc_mrr_at_3_max": 0.805287, + "nauc_mrr_at_3_std": 0.672239, + "nauc_mrr_at_3_diff1": 0.841009, + "nauc_mrr_at_5_max": 0.802456, + "nauc_mrr_at_5_std": 0.674199, + "nauc_mrr_at_5_diff1": 0.837977, + "nauc_mrr_at_10_max": 0.800941, + "nauc_mrr_at_10_std": 0.675967, + "nauc_mrr_at_10_diff1": 0.836955, + "nauc_mrr_at_20_max": 0.800777, + "nauc_mrr_at_20_std": 0.67587, + "nauc_mrr_at_20_diff1": 0.837403, + "nauc_mrr_at_100_max": 0.80068, + "nauc_mrr_at_100_std": 0.675277, + "nauc_mrr_at_100_diff1": 0.837684, + "nauc_mrr_at_1000_max": 0.80068, + "nauc_mrr_at_1000_std": 0.675277, + "nauc_mrr_at_1000_diff1": 0.837684, + "main_score": 0.71032, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 155.00591444969177, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedrxivClusteringP2P.v2.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedrxivClusteringP2P.v2.json new file mode 100644 index 0000000000..52ca8d6c4a --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedrxivClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "task_name": "MedrxivClusteringP2P.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.517263, + 0.508769, + 0.516029, + 0.502497, + 0.514773, + 0.516316, + 0.505753, + 0.521569, + 0.523436, + 0.503615 + ] + }, + "v_measure": 0.513002, + "v_measure_std": 0.007019, + "main_score": 0.513002, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 16.72254991531372, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedrxivClusteringS2S.v2.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedrxivClusteringS2S.v2.json new file mode 100644 index 0000000000..7f3cb8de24 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedrxivClusteringS2S.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "task_name": "MedrxivClusteringS2S.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.51167, + 0.490931, + 0.498294, + 0.502563, + 0.524512, + 0.515661, + 0.507906, + 0.517739, + 0.518453, + 0.498826 + ] + }, + "v_measure": 0.508655, + "v_measure_std": 0.010217, + "main_score": 0.508655, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.596285343170166, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MindSmallReranking.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MindSmallReranking.json new file mode 100644 index 0000000000..43fc54c887 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MindSmallReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "59042f120c80e8afa9cdbb224f67076cec0fc9a7", + "task_name": "MindSmallReranking", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "map": 0.322369, + "mrr": 0.332378, + "nAUC_map_max": -0.288497, + "nAUC_map_std": -0.069576, + "nAUC_map_diff1": 0.160447, + "nAUC_mrr_max": -0.236891, + "nAUC_mrr_std": -0.050994, + "nAUC_mrr_diff1": 0.144051, + "main_score": 0.322369, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 201.48509311676025, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MultilingualSentiment.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MultilingualSentiment.json new file mode 100644 index 0000000000..cd6d042403 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MultilingualSentiment.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a", + "task_name": "MultilingualSentiment", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.8313, + "f1": 0.832331, + "f1_weighted": 0.832331, + "scores_per_experiment": [ + { + "accuracy": 0.830333, + "f1": 0.83178, + "f1_weighted": 0.83178 + }, + { + "accuracy": 0.83, + "f1": 0.829174, + "f1_weighted": 0.829174 + }, + { + "accuracy": 0.834333, + "f1": 0.834847, + "f1_weighted": 0.834847 + }, + { + "accuracy": 0.835333, + "f1": 0.837048, + "f1_weighted": 0.837048 + }, + { + "accuracy": 0.829333, + "f1": 0.830766, + "f1_weighted": 0.830766 + }, + { + "accuracy": 0.813667, + "f1": 0.815166, + "f1_weighted": 0.815166 + }, + { + "accuracy": 0.834333, + "f1": 0.835277, + "f1_weighted": 0.835277 + }, + { + "accuracy": 0.832333, + "f1": 0.832519, + "f1_weighted": 0.832519 + }, + { + "accuracy": 0.834333, + "f1": 0.835994, + "f1_weighted": 0.835994 + }, + { + "accuracy": 0.839, + "f1": 0.840739, + "f1_weighted": 0.840739 + } + ], + "main_score": 0.8313, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 15.6958327293396, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Ocnli.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Ocnli.json new file mode 100644 index 0000000000..e454069da0 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Ocnli.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "66e76a618a34d6d565d5538088562851e6daa7ec", + "task_name": "Ocnli", + "mteb_version": "1.38.1", + "scores": { + "validation": [ + { + "similarity_accuracy": 0.787764, + "similarity_accuracy_threshold": 0.594352, + "similarity_f1": 0.805408, + "similarity_f1_threshold": 0.570571, + "similarity_precision": 0.741993, + "similarity_recall": 0.880676, + "similarity_ap": 0.835234, + "cosine_accuracy": 0.787764, + "cosine_accuracy_threshold": 0.594352, + "cosine_f1": 0.805408, + "cosine_f1_threshold": 0.570571, + "cosine_precision": 0.741993, + "cosine_recall": 0.880676, + "cosine_ap": 0.835234, + "manhattan_accuracy": 0.780184, + "manhattan_accuracy_threshold": 31.54244, + "manhattan_f1": 0.797297, + "manhattan_f1_threshold": 32.603615, + "manhattan_precision": 0.734222, + "manhattan_recall": 0.872228, + "manhattan_ap": 0.829364, + "euclidean_accuracy": 0.788305, + "euclidean_accuracy_threshold": 0.90114, + "euclidean_f1": 0.805941, + "euclidean_f1_threshold": 0.914983, + "euclidean_precision": 0.758621, + "euclidean_recall": 0.859556, + "euclidean_ap": 0.835024, + "dot_accuracy": 0.787764, + "dot_accuracy_threshold": 0.596746, + "dot_f1": 0.806217, + "dot_f1_threshold": 0.574256, + "dot_precision": 0.746403, + "dot_recall": 0.876452, + "dot_ap": 0.835722, + "max_accuracy": 0.788305, + "max_f1": 0.806217, + "max_precision": 0.758621, + "max_recall": 0.880676, + "max_ap": 0.835722, + "main_score": 0.835722, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 36.054293394088745, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/OnlineShopping.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/OnlineShopping.json new file mode 100644 index 0000000000..32b80769f4 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/OnlineShopping.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e610f2ebd179a8fda30ae534c3878750a96db120", + "task_name": "OnlineShopping", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.9609, + "f1": 0.960843, + "f1_weighted": 0.960914, + "ap": 0.950686, + "ap_weighted": 0.950686, + "scores_per_experiment": [ + { + "accuracy": 0.96, + "f1": 0.959929, + "f1_weighted": 0.96001, + "ap": 0.947729, + "ap_weighted": 0.947729 + }, + { + "accuracy": 0.963, + "f1": 0.962949, + "f1_weighted": 0.963015, + "ap": 0.954018, + "ap_weighted": 0.954018 + }, + { + "accuracy": 0.963, + "f1": 0.962944, + "f1_weighted": 0.963013, + "ap": 0.953116, + "ap_weighted": 0.953116 + }, + { + "accuracy": 0.96, + "f1": 0.959929, + "f1_weighted": 0.96001, + "ap": 0.947729, + "ap_weighted": 0.947729 + }, + { + "accuracy": 0.962, + "f1": 0.961945, + "f1_weighted": 0.962014, + "ap": 0.952209, + "ap_weighted": 0.952209 + }, + { + "accuracy": 0.962, + "f1": 0.961956, + "f1_weighted": 0.962018, + "ap": 0.95402, + "ap_weighted": 0.95402 + }, + { + "accuracy": 0.958, + "f1": 0.957957, + "f1_weighted": 0.958022, + "ap": 0.949482, + "ap_weighted": 0.949482 + }, + { + "accuracy": 0.961, + "f1": 0.960952, + "f1_weighted": 0.961018, + "ap": 0.952204, + "ap_weighted": 0.952204 + }, + { + "accuracy": 0.96, + "f1": 0.959922, + "f1_weighted": 0.960007, + "ap": 0.946853, + "ap_weighted": 0.946853 + }, + { + "accuracy": 0.96, + "f1": 0.959942, + "f1_weighted": 0.960015, + "ap": 0.9495, + "ap_weighted": 0.9495 + } + ], + "main_score": 0.9609, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 10.69285535812378, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/PAWSX.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/PAWSX.json new file mode 100644 index 0000000000..4a356fbabb --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/PAWSX.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1", + "task_name": "PAWSX", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.632101, + "spearman": 0.678299, + "cosine_pearson": 0.632101, + "cosine_spearman": 0.678299, + "manhattan_pearson": 0.662306, + "manhattan_spearman": 0.67793, + "euclidean_pearson": 0.663557, + "euclidean_spearman": 0.678254, + "main_score": 0.678299, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 7.729218244552612, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/QBQTC.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/QBQTC.json new file mode 100644 index 0000000000..0911cb78e2 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/QBQTC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7", + "task_name": "QBQTC", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.516381, + "spearman": 0.522269, + "cosine_pearson": 0.516381, + "cosine_spearman": 0.522269, + "manhattan_pearson": 0.51874, + "manhattan_spearman": 0.522045, + "euclidean_pearson": 0.519086, + "euclidean_spearman": 0.52218, + "main_score": 0.522269, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 13.653958797454834, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SCIDOCS.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SCIDOCS.json new file mode 100644 index 0000000000..edb265c1b1 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SCIDOCS.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88", + "task_name": "SCIDOCS", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.264, + "ndcg_at_3": 0.2273, + "ndcg_at_5": 0.20357, + "ndcg_at_10": 0.25305, + "ndcg_at_20": 0.29528, + "ndcg_at_100": 0.36523, + "ndcg_at_1000": 0.36523, + "map_at_1": 0.05363, + "map_at_3": 0.10376, + "map_at_5": 0.12849, + "map_at_10": 0.1546, + "map_at_20": 0.17177, + "map_at_100": 0.18793, + "map_at_1000": 0.18793, + "recall_at_1": 0.05363, + "recall_at_3": 0.13193, + "recall_at_5": 0.18618, + "recall_at_10": 0.27652, + "recall_at_20": 0.37583, + "recall_at_100": 0.60217, + "recall_at_1000": 0.60217, + "precision_at_1": 0.264, + "precision_at_3": 0.21667, + "precision_at_5": 0.1836, + "precision_at_10": 0.1363, + "precision_at_20": 0.0926, + "precision_at_100": 0.02969, + "precision_at_1000": 0.00297, + "mrr_at_1": 0.264, + "mrr_at_3": 0.352833, + "mrr_at_5": 0.376983, + "mrr_at_10": 0.39639, + "mrr_at_20": 0.403958, + "mrr_at_100": 0.40753, + "mrr_at_1000": 0.40753, + "nauc_ndcg_at_1_max": 0.147775, + "nauc_ndcg_at_1_std": 0.201838, + "nauc_ndcg_at_1_diff1": 0.161577, + "nauc_ndcg_at_3_max": 0.193913, + "nauc_ndcg_at_3_std": 0.300612, + "nauc_ndcg_at_3_diff1": 0.148798, + "nauc_ndcg_at_5_max": 0.216859, + "nauc_ndcg_at_5_std": 0.343373, + "nauc_ndcg_at_5_diff1": 0.126749, + "nauc_ndcg_at_10_max": 0.237082, + "nauc_ndcg_at_10_std": 0.377149, + "nauc_ndcg_at_10_diff1": 0.112935, + "nauc_ndcg_at_20_max": 0.245425, + "nauc_ndcg_at_20_std": 0.41973, + "nauc_ndcg_at_20_diff1": 0.101868, + "nauc_ndcg_at_100_max": 0.249378, + "nauc_ndcg_at_100_std": 0.419384, + "nauc_ndcg_at_100_diff1": 0.095862, + "nauc_ndcg_at_1000_max": 0.249378, + "nauc_ndcg_at_1000_std": 0.419384, + "nauc_ndcg_at_1000_diff1": 0.095862, + "nauc_map_at_1_max": 0.150201, + "nauc_map_at_1_std": 0.202124, + "nauc_map_at_1_diff1": 0.163758, + "nauc_map_at_3_max": 0.201513, + "nauc_map_at_3_std": 0.301725, + "nauc_map_at_3_diff1": 0.156489, + "nauc_map_at_5_max": 0.225137, + "nauc_map_at_5_std": 0.347351, + "nauc_map_at_5_diff1": 0.134572, + "nauc_map_at_10_max": 0.240837, + "nauc_map_at_10_std": 0.376185, + "nauc_map_at_10_diff1": 0.121334, + "nauc_map_at_20_max": 0.249066, + "nauc_map_at_20_std": 0.405415, + "nauc_map_at_20_diff1": 0.115597, + "nauc_map_at_100_max": 0.250109, + "nauc_map_at_100_std": 0.410644, + "nauc_map_at_100_diff1": 0.111036, + "nauc_map_at_1000_max": 0.250109, + "nauc_map_at_1000_std": 0.410644, + "nauc_map_at_1000_diff1": 0.111036, + "nauc_recall_at_1_max": 0.150201, + "nauc_recall_at_1_std": 0.202124, + "nauc_recall_at_1_diff1": 0.163758, + "nauc_recall_at_3_max": 0.20974, + "nauc_recall_at_3_std": 0.337045, + "nauc_recall_at_3_diff1": 0.137174, + "nauc_recall_at_5_max": 0.232275, + "nauc_recall_at_5_std": 0.379378, + "nauc_recall_at_5_diff1": 0.093219, + "nauc_recall_at_10_max": 0.248647, + "nauc_recall_at_10_std": 0.40896, + "nauc_recall_at_10_diff1": 0.065683, + "nauc_recall_at_20_max": 0.247614, + "nauc_recall_at_20_std": 0.467394, + "nauc_recall_at_20_diff1": 0.040836, + "nauc_recall_at_100_max": 0.232125, + "nauc_recall_at_100_std": 0.410303, + "nauc_recall_at_100_diff1": 0.018137, + "nauc_recall_at_1000_max": 0.232125, + "nauc_recall_at_1000_std": 0.410303, + "nauc_recall_at_1000_diff1": 0.018137, + "nauc_precision_at_1_max": 0.147775, + "nauc_precision_at_1_std": 0.201838, + "nauc_precision_at_1_diff1": 0.161577, + "nauc_precision_at_3_max": 0.210451, + "nauc_precision_at_3_std": 0.337002, + "nauc_precision_at_3_diff1": 0.137081, + "nauc_precision_at_5_max": 0.233539, + "nauc_precision_at_5_std": 0.37949, + "nauc_precision_at_5_diff1": 0.092747, + "nauc_precision_at_10_max": 0.251047, + "nauc_precision_at_10_std": 0.40787, + "nauc_precision_at_10_diff1": 0.065779, + "nauc_precision_at_20_max": 0.249218, + "nauc_precision_at_20_std": 0.462136, + "nauc_precision_at_20_diff1": 0.039026, + "nauc_precision_at_100_max": 0.231989, + "nauc_precision_at_100_std": 0.399861, + "nauc_precision_at_100_diff1": 0.012526, + "nauc_precision_at_1000_max": 0.231989, + "nauc_precision_at_1000_std": 0.399861, + "nauc_precision_at_1000_diff1": 0.012526, + "nauc_mrr_at_1_max": 0.147775, + "nauc_mrr_at_1_std": 0.201838, + "nauc_mrr_at_1_diff1": 0.161577, + "nauc_mrr_at_3_max": 0.166217, + "nauc_mrr_at_3_std": 0.255581, + "nauc_mrr_at_3_diff1": 0.15415, + "nauc_mrr_at_5_max": 0.173104, + "nauc_mrr_at_5_std": 0.260243, + "nauc_mrr_at_5_diff1": 0.148081, + "nauc_mrr_at_10_max": 0.175671, + "nauc_mrr_at_10_std": 0.262729, + "nauc_mrr_at_10_diff1": 0.146584, + "nauc_mrr_at_20_max": 0.174594, + "nauc_mrr_at_20_std": 0.262841, + "nauc_mrr_at_20_diff1": 0.1461, + "nauc_mrr_at_100_max": 0.173462, + "nauc_mrr_at_100_std": 0.261072, + "nauc_mrr_at_100_diff1": 0.145977, + "nauc_mrr_at_1000_max": 0.173462, + "nauc_mrr_at_1000_std": 0.261072, + "nauc_mrr_at_1000_diff1": 0.145977, + "main_score": 0.25305, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 77.67848324775696, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SICK-R.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SICK-R.json new file mode 100644 index 0000000000..dcddfee831 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SICK-R.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "20a6d6f312dd54037fe07a32d58e5e168867909d", + "task_name": "SICK-R", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.875086, + "spearman": 0.843765, + "cosine_pearson": 0.875086, + "cosine_spearman": 0.843765, + "manhattan_pearson": 0.85472, + "manhattan_spearman": 0.844858, + "euclidean_pearson": 0.852475, + "euclidean_spearman": 0.843781, + "main_score": 0.843765, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 31.051287412643433, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS12.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS12.json new file mode 100644 index 0000000000..eb4e11cdcd --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS12.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", + "task_name": "STS12", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.903793, + "spearman": 0.84604, + "cosine_pearson": 0.903793, + "cosine_spearman": 0.84604, + "manhattan_pearson": 0.895735, + "manhattan_spearman": 0.848926, + "euclidean_pearson": 0.893902, + "euclidean_spearman": 0.846063, + "main_score": 0.84604, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.102168083190918, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS13.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS13.json new file mode 100644 index 0000000000..904360bea1 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS13.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca", + "task_name": "STS13", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.919942, + "spearman": 0.924564, + "cosine_pearson": 0.919942, + "cosine_spearman": 0.924564, + "manhattan_pearson": 0.917732, + "manhattan_spearman": 0.921685, + "euclidean_pearson": 0.920566, + "euclidean_spearman": 0.924562, + "main_score": 0.924564, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.6929774284362793, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS14.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS14.json new file mode 100644 index 0000000000..e28f5bb45e --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS14.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", + "task_name": "STS14", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.908146, + "spearman": 0.897071, + "cosine_pearson": 0.908146, + "cosine_spearman": 0.897071, + "manhattan_pearson": 0.903391, + "manhattan_spearman": 0.896457, + "euclidean_pearson": 0.905138, + "euclidean_spearman": 0.897041, + "main_score": 0.897071, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.813732385635376, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS15.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS15.json new file mode 100644 index 0000000000..0433f46497 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS15.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3", + "task_name": "STS15", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.907463, + "spearman": 0.918135, + "cosine_pearson": 0.907463, + "cosine_spearman": 0.918135, + "manhattan_pearson": 0.909195, + "manhattan_spearman": 0.9161, + "euclidean_pearson": 0.911217, + "euclidean_spearman": 0.917972, + "main_score": 0.918135, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.040424823760986, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS17.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS17.json new file mode 100644 index 0000000000..4d3f54a0d9 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS17.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "faeb762787bd10488a50c8b5be4a3b82e411949c", + "task_name": "STS17", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.929991, + "spearman": 0.926677, + "cosine_pearson": 0.929991, + "cosine_spearman": 0.926677, + "manhattan_pearson": 0.926164, + "manhattan_spearman": 0.926858, + "euclidean_pearson": 0.926333, + "euclidean_spearman": 0.926913, + "main_score": 0.926677, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.4333853721618652, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS22.v2.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS22.v2.json new file mode 100644 index 0000000000..eb5f33625f --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS22.v2.json @@ -0,0 +1,41 @@ +{ + "dataset_revision": "d31f33a128469b20e357535c39b82fb3c3f6f2bd", + "task_name": "STS22.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.701085, + "spearman": 0.709998, + "cosine_pearson": 0.701085, + "cosine_spearman": 0.709998, + "manhattan_pearson": 0.712211, + "manhattan_spearman": 0.709634, + "euclidean_pearson": 0.711794, + "euclidean_spearman": 0.710278, + "main_score": 0.709998, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "pearson": 0.733064, + "spearman": 0.732343, + "cosine_pearson": 0.733064, + "cosine_spearman": 0.732343, + "manhattan_pearson": 0.70666, + "manhattan_spearman": 0.733649, + "euclidean_pearson": 0.707428, + "euclidean_spearman": 0.732222, + "main_score": 0.732343, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 10.821203470230103, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STSB.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STSB.json new file mode 100644 index 0000000000..a889af55cb --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STSB.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0", + "task_name": "STSB", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.844738, + "spearman": 0.85886, + "cosine_pearson": 0.844738, + "cosine_spearman": 0.85886, + "manhattan_pearson": 0.844247, + "manhattan_spearman": 0.859067, + "euclidean_pearson": 0.844343, + "euclidean_spearman": 0.85879, + "main_score": 0.85886, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 6.177364826202393, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STSBenchmark.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STSBenchmark.json new file mode 100644 index 0000000000..bb187dfb01 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STSBenchmark.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831", + "task_name": "STSBenchmark", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.898551, + "spearman": 0.916252, + "cosine_pearson": 0.898551, + "cosine_spearman": 0.916252, + "manhattan_pearson": 0.908936, + "manhattan_spearman": 0.915676, + "euclidean_pearson": 0.910178, + "euclidean_spearman": 0.916217, + "main_score": 0.916252, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.4057884216308594, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SprintDuplicateQuestions.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SprintDuplicateQuestions.json new file mode 100644 index 0000000000..6c82397ab3 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SprintDuplicateQuestions.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "task_name": "SprintDuplicateQuestions", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "similarity_accuracy": 0.998752, + "similarity_accuracy_threshold": 0.918854, + "similarity_f1": 0.935714, + "similarity_f1_threshold": 0.918854, + "similarity_precision": 0.955208, + "similarity_recall": 0.917, + "similarity_ap": 0.970389, + "cosine_accuracy": 0.998752, + "cosine_accuracy_threshold": 0.918854, + "cosine_f1": 0.935714, + "cosine_f1_threshold": 0.918854, + "cosine_precision": 0.955208, + "cosine_recall": 0.917, + "cosine_ap": 0.970389, + "manhattan_accuracy": 0.998772, + "manhattan_accuracy_threshold": 14.450589, + "manhattan_f1": 0.936864, + "manhattan_f1_threshold": 14.492561, + "manhattan_precision": 0.954357, + "manhattan_recall": 0.92, + "manhattan_ap": 0.970427, + "euclidean_accuracy": 0.998762, + "euclidean_accuracy_threshold": 0.403939, + "euclidean_f1": 0.936322, + "euclidean_f1_threshold": 0.403939, + "euclidean_precision": 0.954309, + "euclidean_recall": 0.919, + "euclidean_ap": 0.970432, + "dot_accuracy": 0.998713, + "dot_accuracy_threshold": 0.918457, + "dot_f1": 0.934077, + "dot_f1_threshold": 0.917061, + "dot_precision": 0.947531, + "dot_recall": 0.921, + "dot_ap": 0.969719, + "max_accuracy": 0.998772, + "max_f1": 0.936864, + "max_precision": 0.955208, + "max_recall": 0.921, + "max_ap": 0.970432, + "main_score": 0.970432, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 15.470523834228516, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/StackExchangeClustering.v2.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/StackExchangeClustering.v2.json new file mode 100644 index 0000000000..9a7938cd0f --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/StackExchangeClustering.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "task_name": "StackExchangeClustering.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.795911, + 0.800232, + 0.796249, + 0.793007, + 0.793649, + 0.796365, + 0.794289, + 0.801969, + 0.78302, + 0.798507 + ] + }, + "v_measure": 0.79532, + "v_measure_std": 0.004912, + "main_score": 0.79532, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 16.17296075820923, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/StackExchangeClusteringP2P.v2.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/StackExchangeClusteringP2P.v2.json new file mode 100644 index 0000000000..774a665d56 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/StackExchangeClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "task_name": "StackExchangeClusteringP2P.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.517999, + 0.51444, + 0.518347, + 0.522482, + 0.518169, + 0.518051, + 0.512139, + 0.525898, + 0.522386, + 0.521748 + ] + }, + "v_measure": 0.519166, + "v_measure_std": 0.003864, + "main_score": 0.519166, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 35.140610218048096, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SummEvalSummarization.v2.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SummEvalSummarization.v2.json new file mode 100644 index 0000000000..30a7827c4b --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SummEvalSummarization.v2.json @@ -0,0 +1,24 @@ +{ + "dataset_revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c", + "task_name": "SummEvalSummarization.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.407111, + "spearman": 0.345063, + "cosine_spearman": 0.345063, + "cosine_pearson": 0.407111, + "dot_spearman": 0.328059, + "dot_pearson": 0.390692, + "main_score": 0.345063, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4.816890716552734, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/T2Reranking.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/T2Reranking.json new file mode 100644 index 0000000000..8e9fe8e0e6 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/T2Reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "76631901a18387f85eaa53e5450019b87ad58ef9", + "task_name": "T2Reranking", + "mteb_version": "1.38.1", + "scores": { + "dev": [ + { + "map": 0.66806, + "mrr": 0.761721, + "nAUC_map_max": 0.242251, + "nAUC_map_std": 0.028146, + "nAUC_map_diff1": -0.100297, + "nAUC_mrr_max": 0.154325, + "nAUC_mrr_std": -0.030553, + "nAUC_mrr_diff1": -0.09196, + "main_score": 0.66806, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 2881.4226155281067, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/T2Retrieval.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/T2Retrieval.json new file mode 100644 index 0000000000..5ace79d614 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/T2Retrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "8731a845f1bf500a4f111cf1070785c793d10e64", + "task_name": "T2Retrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.93569, + "ndcg_at_3": 0.90674, + "ndcg_at_5": 0.89657, + "ndcg_at_10": 0.89737, + "ndcg_at_20": 0.91073, + "ndcg_at_100": 0.923, + "ndcg_at_1000": 0.923, + "map_at_1": 0.29362, + "map_at_3": 0.58667, + "map_at_5": 0.72297, + "map_at_10": 0.83633, + "map_at_20": 0.86252, + "map_at_100": 0.86895, + "map_at_1000": 0.86895, + "recall_at_1": 0.29362, + "recall_at_3": 0.59917, + "recall_at_5": 0.74872, + "recall_at_10": 0.8862, + "recall_at_20": 0.93521, + "recall_at_100": 0.97352, + "recall_at_1000": 0.97352, + "precision_at_1": 0.93569, + "precision_at_3": 0.79182, + "precision_at_5": 0.66634, + "precision_at_10": 0.44392, + "precision_at_20": 0.24313, + "precision_at_100": 0.05104, + "precision_at_1000": 0.0051, + "mrr_at_1": 0.935692, + "mrr_at_3": 0.949427, + "mrr_at_5": 0.951148, + "mrr_at_10": 0.952255, + "mrr_at_20": 0.952607, + "mrr_at_100": 0.952788, + "mrr_at_1000": 0.952788, + "nauc_ndcg_at_1_max": 0.822029, + "nauc_ndcg_at_1_std": 0.696889, + "nauc_ndcg_at_1_diff1": 0.493752, + "nauc_ndcg_at_3_max": 0.680087, + "nauc_ndcg_at_3_std": 0.6541, + "nauc_ndcg_at_3_diff1": 0.097467, + "nauc_ndcg_at_5_max": 0.601332, + "nauc_ndcg_at_5_std": 0.593135, + "nauc_ndcg_at_5_diff1": 0.103248, + "nauc_ndcg_at_10_max": 0.520156, + "nauc_ndcg_at_10_std": 0.50553, + "nauc_ndcg_at_10_diff1": 0.146629, + "nauc_ndcg_at_20_max": 0.567671, + "nauc_ndcg_at_20_std": 0.565448, + "nauc_ndcg_at_20_diff1": 0.149226, + "nauc_ndcg_at_100_max": 0.619819, + "nauc_ndcg_at_100_std": 0.617163, + "nauc_ndcg_at_100_diff1": 0.149437, + "nauc_ndcg_at_1000_max": 0.619819, + "nauc_ndcg_at_1000_std": 0.617163, + "nauc_ndcg_at_1000_diff1": 0.149437, + "nauc_map_at_1_max": -0.282609, + "nauc_map_at_1_std": -0.489754, + "nauc_map_at_1_diff1": 0.5468, + "nauc_map_at_3_max": -0.183251, + "nauc_map_at_3_std": -0.438922, + "nauc_map_at_3_diff1": 0.395664, + "nauc_map_at_5_max": -0.024882, + "nauc_map_at_5_std": -0.252784, + "nauc_map_at_5_diff1": 0.309741, + "nauc_map_at_10_max": 0.340017, + "nauc_map_at_10_std": 0.261124, + "nauc_map_at_10_diff1": 0.156492, + "nauc_map_at_20_max": 0.479724, + "nauc_map_at_20_std": 0.469813, + "nauc_map_at_20_diff1": 0.109276, + "nauc_map_at_100_max": 0.50556, + "nauc_map_at_100_std": 0.503387, + "nauc_map_at_100_diff1": 0.102151, + "nauc_map_at_1000_max": 0.50556, + "nauc_map_at_1000_std": 0.503387, + "nauc_map_at_1000_diff1": 0.102151, + "nauc_recall_at_1_max": -0.282609, + "nauc_recall_at_1_std": -0.489754, + "nauc_recall_at_1_diff1": 0.5468, + "nauc_recall_at_3_max": -0.222146, + "nauc_recall_at_3_std": -0.475006, + "nauc_recall_at_3_diff1": 0.383284, + "nauc_recall_at_5_max": -0.112776, + "nauc_recall_at_5_std": -0.347423, + "nauc_recall_at_5_diff1": 0.30823, + "nauc_recall_at_10_max": 0.213917, + "nauc_recall_at_10_std": 0.13519, + "nauc_recall_at_10_diff1": 0.157431, + "nauc_recall_at_20_max": 0.422043, + "nauc_recall_at_20_std": 0.471109, + "nauc_recall_at_20_diff1": 0.084853, + "nauc_recall_at_100_max": 0.562273, + "nauc_recall_at_100_std": 0.649178, + "nauc_recall_at_100_diff1": 0.035994, + "nauc_recall_at_1000_max": 0.562273, + "nauc_recall_at_1000_std": 0.649178, + "nauc_recall_at_1000_diff1": 0.035994, + "nauc_precision_at_1_max": 0.822029, + "nauc_precision_at_1_std": 0.696889, + "nauc_precision_at_1_diff1": 0.493752, + "nauc_precision_at_3_max": 0.643855, + "nauc_precision_at_3_std": 0.789959, + "nauc_precision_at_3_diff1": -0.443947, + "nauc_precision_at_5_max": 0.579994, + "nauc_precision_at_5_std": 0.798513, + "nauc_precision_at_5_diff1": -0.438635, + "nauc_precision_at_10_max": 0.514781, + "nauc_precision_at_10_std": 0.782636, + "nauc_precision_at_10_diff1": -0.390987, + "nauc_precision_at_20_max": 0.486417, + "nauc_precision_at_20_std": 0.765981, + "nauc_precision_at_20_diff1": -0.373457, + "nauc_precision_at_100_max": 0.461821, + "nauc_precision_at_100_std": 0.73418, + "nauc_precision_at_100_diff1": -0.368993, + "nauc_precision_at_1000_max": 0.461821, + "nauc_precision_at_1000_std": 0.73418, + "nauc_precision_at_1000_diff1": -0.368993, + "nauc_mrr_at_1_max": 0.822029, + "nauc_mrr_at_1_std": 0.696889, + "nauc_mrr_at_1_diff1": 0.493752, + "nauc_mrr_at_3_max": 0.855867, + "nauc_mrr_at_3_std": 0.747754, + "nauc_mrr_at_3_diff1": 0.497988, + "nauc_mrr_at_5_max": 0.8562, + "nauc_mrr_at_5_std": 0.748672, + "nauc_mrr_at_5_diff1": 0.499119, + "nauc_mrr_at_10_max": 0.855039, + "nauc_mrr_at_10_std": 0.747494, + "nauc_mrr_at_10_diff1": 0.4997, + "nauc_mrr_at_20_max": 0.8544, + "nauc_mrr_at_20_std": 0.746768, + "nauc_mrr_at_20_diff1": 0.499448, + "nauc_mrr_at_100_max": 0.853974, + "nauc_mrr_at_100_std": 0.746015, + "nauc_mrr_at_100_diff1": 0.49938, + "nauc_mrr_at_1000_max": 0.853974, + "nauc_mrr_at_1000_std": 0.746015, + "nauc_mrr_at_1000_diff1": 0.49938, + "main_score": 0.89737, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 723.2841963768005, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TNews.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TNews.json new file mode 100644 index 0000000000..b32efe97b6 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TNews.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "317f262bf1e6126357bbe89e875451e4b0938fe4", + "task_name": "TNews", + "mteb_version": "1.36.5", + "scores": { + "validation": [ + { + "accuracy": 0.57099, + "f1": 0.549404, + "f1_weighted": 0.56562, + "scores_per_experiment": [ + { + "accuracy": 0.5685, + "f1": 0.54513, + "f1_weighted": 0.56093 + }, + { + "accuracy": 0.5763, + "f1": 0.552967, + "f1_weighted": 0.571146 + }, + { + "accuracy": 0.5787, + "f1": 0.556517, + "f1_weighted": 0.573184 + }, + { + "accuracy": 0.5738, + "f1": 0.554099, + "f1_weighted": 0.570082 + }, + { + "accuracy": 0.5795, + "f1": 0.561124, + "f1_weighted": 0.578429 + }, + { + "accuracy": 0.5715, + "f1": 0.549147, + "f1_weighted": 0.563936 + }, + { + "accuracy": 0.5631, + "f1": 0.543581, + "f1_weighted": 0.560894 + }, + { + "accuracy": 0.5647, + "f1": 0.541152, + "f1_weighted": 0.555012 + }, + { + "accuracy": 0.5673, + "f1": 0.542179, + "f1_weighted": 0.557314 + }, + { + "accuracy": 0.5665, + "f1": 0.548142, + "f1_weighted": 0.565276 + } + ], + "main_score": 0.57099, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 41.41187644004822, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TRECCOVID.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TRECCOVID.json new file mode 100644 index 0000000000..bb0fb4a7fd --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TRECCOVID.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e", + "task_name": "TRECCOVID", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.91, + "ndcg_at_3": 0.90754, + "ndcg_at_5": 0.90077, + "ndcg_at_10": 0.87826, + "ndcg_at_20": 0.84672, + "ndcg_at_100": 0.69157, + "ndcg_at_1000": 0.28034, + "map_at_1": 0.00236, + "map_at_3": 0.00713, + "map_at_5": 0.01184, + "map_at_10": 0.02292, + "map_at_20": 0.04357, + "map_at_100": 0.1493, + "map_at_1000": 0.1493, + "recall_at_1": 0.00236, + "recall_at_3": 0.00735, + "recall_at_5": 0.01238, + "recall_at_10": 0.02418, + "recall_at_20": 0.04647, + "recall_at_100": 0.17569, + "recall_at_1000": 0.17569, + "precision_at_1": 0.94, + "precision_at_3": 0.94667, + "precision_at_5": 0.948, + "precision_at_10": 0.926, + "precision_at_20": 0.893, + "precision_at_100": 0.7184, + "precision_at_1000": 0.07184, + "mrr_at_1": 0.94, + "mrr_at_3": 0.96, + "mrr_at_5": 0.965, + "mrr_at_10": 0.965, + "mrr_at_20": 0.965, + "mrr_at_100": 0.965, + "mrr_at_1000": 0.965, + "nauc_ndcg_at_1_max": 0.355301, + "nauc_ndcg_at_1_std": 0.51637, + "nauc_ndcg_at_1_diff1": -0.165017, + "nauc_ndcg_at_3_max": 0.356377, + "nauc_ndcg_at_3_std": 0.356167, + "nauc_ndcg_at_3_diff1": -0.26669, + "nauc_ndcg_at_5_max": 0.316415, + "nauc_ndcg_at_5_std": 0.389211, + "nauc_ndcg_at_5_diff1": -0.32258, + "nauc_ndcg_at_10_max": 0.272324, + "nauc_ndcg_at_10_std": 0.252685, + "nauc_ndcg_at_10_diff1": -0.325436, + "nauc_ndcg_at_20_max": 0.195685, + "nauc_ndcg_at_20_std": 0.277742, + "nauc_ndcg_at_20_diff1": -0.294667, + "nauc_ndcg_at_100_max": 0.236231, + "nauc_ndcg_at_100_std": -0.020196, + "nauc_ndcg_at_100_diff1": -0.174016, + "nauc_ndcg_at_1000_max": -0.033966, + "nauc_ndcg_at_1000_std": 0.503882, + "nauc_ndcg_at_1000_diff1": 0.064135, + "nauc_map_at_1_max": -0.005866, + "nauc_map_at_1_std": 0.502955, + "nauc_map_at_1_diff1": 0.115436, + "nauc_map_at_3_max": -0.008989, + "nauc_map_at_3_std": 0.488624, + "nauc_map_at_3_diff1": 0.07888, + "nauc_map_at_5_max": -0.040601, + "nauc_map_at_5_std": 0.505538, + "nauc_map_at_5_diff1": 0.078679, + "nauc_map_at_10_max": -0.084628, + "nauc_map_at_10_std": 0.526171, + "nauc_map_at_10_diff1": 0.029859, + "nauc_map_at_20_max": -0.100777, + "nauc_map_at_20_std": 0.567795, + "nauc_map_at_20_diff1": 0.043912, + "nauc_map_at_100_max": -0.004181, + "nauc_map_at_100_std": 0.455132, + "nauc_map_at_100_diff1": 0.078044, + "nauc_map_at_1000_max": -0.004181, + "nauc_map_at_1000_std": 0.455132, + "nauc_map_at_1000_diff1": 0.078044, + "nauc_recall_at_1_max": -0.005866, + "nauc_recall_at_1_std": 0.502955, + "nauc_recall_at_1_diff1": 0.115436, + "nauc_recall_at_3_max": -0.049479, + "nauc_recall_at_3_std": 0.465229, + "nauc_recall_at_3_diff1": 0.090739, + "nauc_recall_at_5_max": -0.105389, + "nauc_recall_at_5_std": 0.467105, + "nauc_recall_at_5_diff1": 0.081737, + "nauc_recall_at_10_max": -0.138155, + "nauc_recall_at_10_std": 0.478586, + "nauc_recall_at_10_diff1": 0.047471, + "nauc_recall_at_20_max": -0.147, + "nauc_recall_at_20_std": 0.523325, + "nauc_recall_at_20_diff1": 0.064437, + "nauc_recall_at_100_max": -0.081128, + "nauc_recall_at_100_std": 0.411566, + "nauc_recall_at_100_diff1": 0.109115, + "nauc_recall_at_1000_max": -0.081128, + "nauc_recall_at_1000_std": 0.411566, + "nauc_recall_at_1000_diff1": 0.109115, + "nauc_precision_at_1_max": 0.742453, + "nauc_precision_at_1_std": 0.466387, + "nauc_precision_at_1_diff1": 0.168534, + "nauc_precision_at_3_max": 0.797995, + "nauc_precision_at_3_std": 0.242288, + "nauc_precision_at_3_diff1": 0.039452, + "nauc_precision_at_5_max": 0.653861, + "nauc_precision_at_5_std": 0.336573, + "nauc_precision_at_5_diff1": -0.070067, + "nauc_precision_at_10_max": 0.402123, + "nauc_precision_at_10_std": 0.324081, + "nauc_precision_at_10_diff1": -0.312831, + "nauc_precision_at_20_max": 0.244881, + "nauc_precision_at_20_std": 0.437629, + "nauc_precision_at_20_diff1": -0.205382, + "nauc_precision_at_100_max": 0.294594, + "nauc_precision_at_100_std": -0.03951, + "nauc_precision_at_100_diff1": -0.081512, + "nauc_precision_at_1000_max": 0.294594, + "nauc_precision_at_1000_std": -0.03951, + "nauc_precision_at_1000_diff1": -0.081512, + "nauc_mrr_at_1_max": 0.742453, + "nauc_mrr_at_1_std": 0.466387, + "nauc_mrr_at_1_diff1": 0.168534, + "nauc_mrr_at_3_max": 0.77416, + "nauc_mrr_at_3_std": 0.488329, + "nauc_mrr_at_3_diff1": 0.264939, + "nauc_mrr_at_5_max": 0.760571, + "nauc_mrr_at_5_std": 0.478925, + "nauc_mrr_at_5_diff1": 0.223623, + "nauc_mrr_at_10_max": 0.760571, + "nauc_mrr_at_10_std": 0.478925, + "nauc_mrr_at_10_diff1": 0.223623, + "nauc_mrr_at_20_max": 0.760571, + "nauc_mrr_at_20_std": 0.478925, + "nauc_mrr_at_20_diff1": 0.223623, + "nauc_mrr_at_100_max": 0.760571, + "nauc_mrr_at_100_std": 0.478925, + "nauc_mrr_at_100_diff1": 0.223623, + "nauc_mrr_at_1000_max": 0.760571, + "nauc_mrr_at_1000_std": 0.478925, + "nauc_mrr_at_1000_diff1": 0.223623, + "main_score": 0.87826, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 502.69182991981506, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ThuNewsClusteringP2P.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ThuNewsClusteringP2P.json new file mode 100644 index 0000000000..75b5eeff0a --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ThuNewsClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "5798586b105c0434e4f0fe5e767abe619442cf93", + "task_name": "ThuNewsClusteringP2P", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measure": 0.82717, + "v_measure_std": 0.011715, + "v_measures": [ + 0.81519, + 0.826319, + 0.829972, + 0.823754, + 0.807998, + 0.849307, + 0.841698, + 0.829372, + 0.831645, + 0.816442 + ], + "main_score": 0.82717, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 181.41818141937256, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ThuNewsClusteringS2S.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ThuNewsClusteringS2S.json new file mode 100644 index 0000000000..2caa33263e --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ThuNewsClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d", + "task_name": "ThuNewsClusteringS2S", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measure": 0.845584, + "v_measure_std": 0.010142, + "v_measures": [ + 0.859946, + 0.858633, + 0.839342, + 0.837734, + 0.829878, + 0.84684, + 0.852203, + 0.843045, + 0.832956, + 0.855258 + ], + "main_score": 0.845584, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 120.26335215568542, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Touche2020Retrieval.v3.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Touche2020Retrieval.v3.json new file mode 100644 index 0000000000..bd6bee5597 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Touche2020Retrieval.v3.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "431886eaecc48f067a3975b70d0949ea2862463c", + "task_name": "Touche2020Retrieval.v3", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.73469, + "ndcg_at_3": 0.70925, + "ndcg_at_5": 0.70709, + "ndcg_at_10": 0.64158, + "ndcg_at_20": 0.56941, + "ndcg_at_100": 0.62322, + "ndcg_at_1000": 0.62322, + "map_at_1": 0.02631, + "map_at_3": 0.0704, + "map_at_5": 0.11432, + "map_at_10": 0.19364, + "map_at_20": 0.2801, + "map_at_100": 0.38612, + "map_at_1000": 0.38612, + "recall_at_1": 0.02631, + "recall_at_3": 0.0793, + "recall_at_5": 0.13173, + "recall_at_10": 0.23153, + "recall_at_20": 0.36454, + "recall_at_100": 0.63133, + "recall_at_1000": 0.63133, + "precision_at_1": 0.7551, + "precision_at_3": 0.7483, + "precision_at_5": 0.75918, + "precision_at_10": 0.66122, + "precision_at_20": 0.52551, + "precision_at_100": 0.20061, + "precision_at_1000": 0.02006, + "mrr_at_1": 0.755102, + "mrr_at_3": 0.867347, + "mrr_at_5": 0.867347, + "mrr_at_10": 0.867347, + "mrr_at_20": 0.867347, + "mrr_at_100": 0.867347, + "mrr_at_1000": 0.867347, + "nauc_ndcg_at_1_max": 0.40765, + "nauc_ndcg_at_1_std": 0.045931, + "nauc_ndcg_at_1_diff1": 0.222542, + "nauc_ndcg_at_3_max": 0.265356, + "nauc_ndcg_at_3_std": 0.245896, + "nauc_ndcg_at_3_diff1": 0.251934, + "nauc_ndcg_at_5_max": 0.222297, + "nauc_ndcg_at_5_std": 0.347132, + "nauc_ndcg_at_5_diff1": 0.293632, + "nauc_ndcg_at_10_max": 0.333994, + "nauc_ndcg_at_10_std": 0.545888, + "nauc_ndcg_at_10_diff1": 0.302297, + "nauc_ndcg_at_20_max": 0.23514, + "nauc_ndcg_at_20_std": 0.710261, + "nauc_ndcg_at_20_diff1": 0.418778, + "nauc_ndcg_at_100_max": 0.211507, + "nauc_ndcg_at_100_std": 0.848447, + "nauc_ndcg_at_100_diff1": 0.464868, + "nauc_ndcg_at_1000_max": 0.211507, + "nauc_ndcg_at_1000_std": 0.848447, + "nauc_ndcg_at_1000_diff1": 0.464868, + "nauc_map_at_1_max": 0.073378, + "nauc_map_at_1_std": 0.241373, + "nauc_map_at_1_diff1": 0.29529, + "nauc_map_at_3_max": 0.096809, + "nauc_map_at_3_std": 0.515835, + "nauc_map_at_3_diff1": 0.40909, + "nauc_map_at_5_max": 0.115905, + "nauc_map_at_5_std": 0.672859, + "nauc_map_at_5_diff1": 0.455626, + "nauc_map_at_10_max": 0.236225, + "nauc_map_at_10_std": 0.8104, + "nauc_map_at_10_diff1": 0.461905, + "nauc_map_at_20_max": 0.182128, + "nauc_map_at_20_std": 0.842177, + "nauc_map_at_20_diff1": 0.478173, + "nauc_map_at_100_max": 0.223091, + "nauc_map_at_100_std": 0.843336, + "nauc_map_at_100_diff1": 0.492811, + "nauc_map_at_1000_max": 0.223091, + "nauc_map_at_1000_std": 0.843336, + "nauc_map_at_1000_diff1": 0.492811, + "nauc_recall_at_1_max": 0.073378, + "nauc_recall_at_1_std": 0.241373, + "nauc_recall_at_1_diff1": 0.29529, + "nauc_recall_at_3_max": 0.005206, + "nauc_recall_at_3_std": 0.532647, + "nauc_recall_at_3_diff1": 0.388838, + "nauc_recall_at_5_max": 0.012405, + "nauc_recall_at_5_std": 0.702358, + "nauc_recall_at_5_diff1": 0.4219, + "nauc_recall_at_10_max": 0.139635, + "nauc_recall_at_10_std": 0.786541, + "nauc_recall_at_10_diff1": 0.376452, + "nauc_recall_at_20_max": 0.030936, + "nauc_recall_at_20_std": 0.782245, + "nauc_recall_at_20_diff1": 0.389714, + "nauc_recall_at_100_max": 0.068115, + "nauc_recall_at_100_std": 0.689512, + "nauc_recall_at_100_diff1": 0.357701, + "nauc_recall_at_1000_max": 0.068115, + "nauc_recall_at_1000_std": 0.689512, + "nauc_recall_at_1000_diff1": 0.357701, + "nauc_precision_at_1_max": 0.467499, + "nauc_precision_at_1_std": 0.041082, + "nauc_precision_at_1_diff1": 0.3107, + "nauc_precision_at_3_max": 0.199465, + "nauc_precision_at_3_std": 0.271313, + "nauc_precision_at_3_diff1": 0.418504, + "nauc_precision_at_5_max": 0.09253, + "nauc_precision_at_5_std": 0.477303, + "nauc_precision_at_5_diff1": 0.445367, + "nauc_precision_at_10_max": 0.34139, + "nauc_precision_at_10_std": 0.705388, + "nauc_precision_at_10_diff1": 0.330426, + "nauc_precision_at_20_max": 0.191755, + "nauc_precision_at_20_std": 0.63636, + "nauc_precision_at_20_diff1": 0.344872, + "nauc_precision_at_100_max": 0.212834, + "nauc_precision_at_100_std": -0.156268, + "nauc_precision_at_100_diff1": 0.060361, + "nauc_precision_at_1000_max": 0.212834, + "nauc_precision_at_1000_std": -0.156268, + "nauc_precision_at_1000_diff1": 0.060361, + "nauc_mrr_at_1_max": 0.467499, + "nauc_mrr_at_1_std": 0.041082, + "nauc_mrr_at_1_diff1": 0.3107, + "nauc_mrr_at_3_max": 0.465847, + "nauc_mrr_at_3_std": 0.010564, + "nauc_mrr_at_3_diff1": 0.342992, + "nauc_mrr_at_5_max": 0.465847, + "nauc_mrr_at_5_std": 0.010564, + "nauc_mrr_at_5_diff1": 0.342992, + "nauc_mrr_at_10_max": 0.465847, + "nauc_mrr_at_10_std": 0.010564, + "nauc_mrr_at_10_diff1": 0.342992, + "nauc_mrr_at_20_max": 0.465847, + "nauc_mrr_at_20_std": 0.010564, + "nauc_mrr_at_20_diff1": 0.342992, + "nauc_mrr_at_100_max": 0.465847, + "nauc_mrr_at_100_std": 0.010564, + "nauc_mrr_at_100_diff1": 0.342992, + "nauc_mrr_at_1000_max": 0.465847, + "nauc_mrr_at_1000_std": 0.010564, + "nauc_mrr_at_1000_diff1": 0.342992, + "main_score": 0.64158, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 988.0547540187836, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ToxicConversationsClassification.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ToxicConversationsClassification.json new file mode 100644 index 0000000000..e37af0ae63 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ToxicConversationsClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "edfaf9da55d3dd50d43143d90c1ac476895ae6de", + "task_name": "ToxicConversationsClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.856885, + "f1": 0.713175, + "f1_weighted": 0.884028, + "ap": 0.333754, + "ap_weighted": 0.333754, + "scores_per_experiment": [ + { + "accuracy": 0.851074, + "f1": 0.70425, + "f1_weighted": 0.879666, + "ap": 0.319378, + "ap_weighted": 0.319378 + }, + { + "accuracy": 0.873047, + "f1": 0.731092, + "f1_weighted": 0.895561, + "ap": 0.354826, + "ap_weighted": 0.354826 + }, + { + "accuracy": 0.851562, + "f1": 0.706422, + "f1_weighted": 0.880187, + "ap": 0.324653, + "ap_weighted": 0.324653 + }, + { + "accuracy": 0.831543, + "f1": 0.688051, + "f1_weighted": 0.866151, + "ap": 0.308587, + "ap_weighted": 0.308587 + }, + { + "accuracy": 0.875488, + "f1": 0.734263, + "f1_weighted": 0.897339, + "ap": 0.359266, + "ap_weighted": 0.359266 + }, + { + "accuracy": 0.844727, + "f1": 0.700185, + "f1_weighted": 0.875424, + "ap": 0.319413, + "ap_weighted": 0.319413 + }, + { + "accuracy": 0.865723, + "f1": 0.721818, + "f1_weighted": 0.890244, + "ap": 0.342152, + "ap_weighted": 0.342152 + }, + { + "accuracy": 0.836426, + "f1": 0.691773, + "f1_weighted": 0.869522, + "ap": 0.310462, + "ap_weighted": 0.310462 + }, + { + "accuracy": 0.87793, + "f1": 0.737476, + "f1_weighted": 0.89912, + "ap": 0.36382, + "ap_weighted": 0.36382 + }, + { + "accuracy": 0.861328, + "f1": 0.716417, + "f1_weighted": 0.887064, + "ap": 0.33498, + "ap_weighted": 0.33498 + } + ], + "main_score": 0.856885, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.903000593185425, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TweetSentimentExtractionClassification.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TweetSentimentExtractionClassification.json new file mode 100644 index 0000000000..564b403941 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TweetSentimentExtractionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a", + "task_name": "TweetSentimentExtractionClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.718166, + "f1": 0.721629, + "f1_weighted": 0.713889, + "scores_per_experiment": [ + { + "accuracy": 0.717317, + "f1": 0.718649, + "f1_weighted": 0.709045 + }, + { + "accuracy": 0.706565, + "f1": 0.710736, + "f1_weighted": 0.702202 + }, + { + "accuracy": 0.726372, + "f1": 0.730204, + "f1_weighted": 0.723507 + }, + { + "accuracy": 0.721562, + "f1": 0.725261, + "f1_weighted": 0.717349 + }, + { + "accuracy": 0.738257, + "f1": 0.741518, + "f1_weighted": 0.735374 + }, + { + "accuracy": 0.727504, + "f1": 0.731597, + "f1_weighted": 0.723719 + }, + { + "accuracy": 0.693265, + "f1": 0.696033, + "f1_weighted": 0.688593 + }, + { + "accuracy": 0.735144, + "f1": 0.738612, + "f1_weighted": 0.731133 + }, + { + "accuracy": 0.702603, + "f1": 0.705282, + "f1_weighted": 0.695852 + }, + { + "accuracy": 0.713073, + "f1": 0.718398, + "f1_weighted": 0.712116 + } + ], + "main_score": 0.718166, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 12.780118465423584, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwentyNewsgroupsClustering.v2.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwentyNewsgroupsClustering.v2.json new file mode 100644 index 0000000000..61ad8c0d20 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwentyNewsgroupsClustering.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "task_name": "TwentyNewsgroupsClustering.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.63079, + 0.616938, + 0.627545, + 0.617647, + 0.62197, + 0.612137, + 0.615339, + 0.622265, + 0.63969, + 0.646119 + ] + }, + "v_measure": 0.625044, + "v_measure_std": 0.01047, + "main_score": 0.625044, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.289347648620605, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwitterSemEval2015.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwitterSemEval2015.json new file mode 100644 index 0000000000..a4a31d3062 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwitterSemEval2015.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "task_name": "TwitterSemEval2015", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "similarity_accuracy": 0.869643, + "similarity_accuracy_threshold": 0.932409, + "similarity_f1": 0.702717, + "similarity_f1_threshold": 0.925067, + "similarity_precision": 0.686196, + "similarity_recall": 0.720053, + "similarity_ap": 0.768518, + "cosine_accuracy": 0.869643, + "cosine_accuracy_threshold": 0.932409, + "cosine_f1": 0.702717, + "cosine_f1_threshold": 0.925067, + "cosine_precision": 0.686196, + "cosine_recall": 0.720053, + "cosine_ap": 0.768518, + "manhattan_accuracy": 0.868987, + "manhattan_accuracy_threshold": 12.996463, + "manhattan_f1": 0.700813, + "manhattan_f1_threshold": 13.783428, + "manhattan_precision": 0.676067, + "manhattan_recall": 0.727441, + "manhattan_ap": 0.767642, + "euclidean_accuracy": 0.869703, + "euclidean_accuracy_threshold": 0.364296, + "euclidean_f1": 0.703123, + "euclidean_f1_threshold": 0.385803, + "euclidean_precision": 0.690858, + "euclidean_recall": 0.715831, + "euclidean_ap": 0.768585, + "dot_accuracy": 0.86857, + "dot_accuracy_threshold": 0.932518, + "dot_f1": 0.698898, + "dot_f1_threshold": 0.926966, + "dot_precision": 0.703664, + "dot_recall": 0.694195, + "dot_ap": 0.764109, + "max_accuracy": 0.869703, + "max_f1": 0.703123, + "max_precision": 0.703664, + "max_recall": 0.727441, + "max_ap": 0.768585, + "main_score": 0.768585, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 20.51383399963379, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwitterURLCorpus.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwitterURLCorpus.json new file mode 100644 index 0000000000..222cfb47ad --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwitterURLCorpus.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "task_name": "TwitterURLCorpus", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "similarity_accuracy": 0.897408, + "similarity_accuracy_threshold": 0.918904, + "similarity_f1": 0.797091, + "similarity_f1_threshold": 0.908807, + "similarity_precision": 0.74858, + "similarity_recall": 0.852325, + "similarity_ap": 0.873592, + "cosine_accuracy": 0.897408, + "cosine_accuracy_threshold": 0.918904, + "cosine_f1": 0.797091, + "cosine_f1_threshold": 0.908807, + "cosine_precision": 0.74858, + "cosine_recall": 0.852325, + "cosine_ap": 0.873592, + "manhattan_accuracy": 0.897039, + "manhattan_accuracy_threshold": 14.302161, + "manhattan_f1": 0.797484, + "manhattan_f1_threshold": 15.040656, + "manhattan_precision": 0.759562, + "manhattan_recall": 0.83939, + "manhattan_ap": 0.873564, + "euclidean_accuracy": 0.897272, + "euclidean_accuracy_threshold": 0.402168, + "euclidean_f1": 0.79711, + "euclidean_f1_threshold": 0.427664, + "euclidean_precision": 0.747489, + "euclidean_recall": 0.853788, + "euclidean_ap": 0.873571, + "dot_accuracy": 0.89636, + "dot_accuracy_threshold": 0.920283, + "dot_f1": 0.796197, + "dot_f1_threshold": 0.910412, + "dot_precision": 0.755548, + "dot_recall": 0.841469, + "dot_ap": 0.872947, + "max_accuracy": 0.897408, + "max_f1": 0.797484, + "max_precision": 0.759562, + "max_recall": 0.853788, + "max_ap": 0.873592, + "main_score": 0.873592, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 58.10989594459534, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/VideoRetrieval.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/VideoRetrieval.json new file mode 100644 index 0000000000..286129e2c0 --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/VideoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "58c2597a5943a2ba48f4668c3b90d796283c5639", + "task_name": "VideoRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.692, + "ndcg_at_3": 0.78616, + "ndcg_at_5": 0.79972, + "ndcg_at_10": 0.81272, + "ndcg_at_20": 0.82039, + "ndcg_at_100": 0.82646, + "ndcg_at_1000": 0.82646, + "map_at_1": 0.692, + "map_at_3": 0.76367, + "map_at_5": 0.77117, + "map_at_10": 0.77657, + "map_at_20": 0.77872, + "map_at_100": 0.7796, + "map_at_1000": 0.7796, + "recall_at_1": 0.692, + "recall_at_3": 0.851, + "recall_at_5": 0.884, + "recall_at_10": 0.924, + "recall_at_20": 0.954, + "recall_at_100": 0.986, + "recall_at_1000": 0.986, + "precision_at_1": 0.692, + "precision_at_3": 0.28367, + "precision_at_5": 0.1768, + "precision_at_10": 0.0924, + "precision_at_20": 0.0477, + "precision_at_100": 0.00986, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.692, + "mrr_at_3": 0.763667, + "mrr_at_5": 0.771167, + "mrr_at_10": 0.776569, + "mrr_at_20": 0.778721, + "mrr_at_100": 0.779599, + "mrr_at_1000": 0.779599, + "nauc_ndcg_at_1_max": 0.146445, + "nauc_ndcg_at_1_std": -0.371343, + "nauc_ndcg_at_1_diff1": 0.789644, + "nauc_ndcg_at_3_max": 0.188201, + "nauc_ndcg_at_3_std": -0.327342, + "nauc_ndcg_at_3_diff1": 0.777939, + "nauc_ndcg_at_5_max": 0.196619, + "nauc_ndcg_at_5_std": -0.300216, + "nauc_ndcg_at_5_diff1": 0.764522, + "nauc_ndcg_at_10_max": 0.185689, + "nauc_ndcg_at_10_std": -0.286634, + "nauc_ndcg_at_10_diff1": 0.765292, + "nauc_ndcg_at_20_max": 0.187561, + "nauc_ndcg_at_20_std": -0.278063, + "nauc_ndcg_at_20_diff1": 0.768551, + "nauc_ndcg_at_100_max": 0.185866, + "nauc_ndcg_at_100_std": -0.296851, + "nauc_ndcg_at_100_diff1": 0.772701, + "nauc_ndcg_at_1000_max": 0.185866, + "nauc_ndcg_at_1000_std": -0.296851, + "nauc_ndcg_at_1000_diff1": 0.772701, + "nauc_map_at_1_max": 0.146445, + "nauc_map_at_1_std": -0.371343, + "nauc_map_at_1_diff1": 0.789644, + "nauc_map_at_3_max": 0.174939, + "nauc_map_at_3_std": -0.34237, + "nauc_map_at_3_diff1": 0.781373, + "nauc_map_at_5_max": 0.178315, + "nauc_map_at_5_std": -0.329478, + "nauc_map_at_5_diff1": 0.774894, + "nauc_map_at_10_max": 0.173526, + "nauc_map_at_10_std": -0.325687, + "nauc_map_at_10_diff1": 0.774984, + "nauc_map_at_20_max": 0.173899, + "nauc_map_at_20_std": -0.324274, + "nauc_map_at_20_diff1": 0.776009, + "nauc_map_at_100_max": 0.173662, + "nauc_map_at_100_std": -0.326752, + "nauc_map_at_100_diff1": 0.776514, + "nauc_map_at_1000_max": 0.173662, + "nauc_map_at_1000_std": -0.326752, + "nauc_map_at_1000_diff1": 0.776514, + "nauc_recall_at_1_max": 0.146445, + "nauc_recall_at_1_std": -0.371343, + "nauc_recall_at_1_diff1": 0.789644, + "nauc_recall_at_3_max": 0.244907, + "nauc_recall_at_3_std": -0.262469, + "nauc_recall_at_3_diff1": 0.763418, + "nauc_recall_at_5_max": 0.294243, + "nauc_recall_at_5_std": -0.141525, + "nauc_recall_at_5_diff1": 0.707835, + "nauc_recall_at_10_max": 0.274633, + "nauc_recall_at_10_std": 0.034301, + "nauc_recall_at_10_diff1": 0.690832, + "nauc_recall_at_20_max": 0.356057, + "nauc_recall_at_20_std": 0.365424, + "nauc_recall_at_20_diff1": 0.681687, + "nauc_recall_at_100_max": 0.633954, + "nauc_recall_at_100_std": 0.751334, + "nauc_recall_at_100_diff1": 0.728458, + "nauc_recall_at_1000_max": 0.633954, + "nauc_recall_at_1000_std": 0.751334, + "nauc_recall_at_1000_diff1": 0.728458, + "nauc_precision_at_1_max": 0.146445, + "nauc_precision_at_1_std": -0.371343, + "nauc_precision_at_1_diff1": 0.789644, + "nauc_precision_at_3_max": 0.244907, + "nauc_precision_at_3_std": -0.262469, + "nauc_precision_at_3_diff1": 0.763418, + "nauc_precision_at_5_max": 0.294243, + "nauc_precision_at_5_std": -0.141525, + "nauc_precision_at_5_diff1": 0.707835, + "nauc_precision_at_10_max": 0.274633, + "nauc_precision_at_10_std": 0.034301, + "nauc_precision_at_10_diff1": 0.690832, + "nauc_precision_at_20_max": 0.356057, + "nauc_precision_at_20_std": 0.365424, + "nauc_precision_at_20_diff1": 0.681687, + "nauc_precision_at_100_max": 0.633954, + "nauc_precision_at_100_std": 0.751334, + "nauc_precision_at_100_diff1": 0.728458, + "nauc_precision_at_1000_max": 0.633954, + "nauc_precision_at_1000_std": 0.751334, + "nauc_precision_at_1000_diff1": 0.728458, + "nauc_mrr_at_1_max": 0.146445, + "nauc_mrr_at_1_std": -0.371343, + "nauc_mrr_at_1_diff1": 0.789644, + "nauc_mrr_at_3_max": 0.174939, + "nauc_mrr_at_3_std": -0.34237, + "nauc_mrr_at_3_diff1": 0.781373, + "nauc_mrr_at_5_max": 0.178315, + "nauc_mrr_at_5_std": -0.329478, + "nauc_mrr_at_5_diff1": 0.774894, + "nauc_mrr_at_10_max": 0.173526, + "nauc_mrr_at_10_std": -0.325687, + "nauc_mrr_at_10_diff1": 0.774984, + "nauc_mrr_at_20_max": 0.173899, + "nauc_mrr_at_20_std": -0.324274, + "nauc_mrr_at_20_diff1": 0.776009, + "nauc_mrr_at_100_max": 0.173662, + "nauc_mrr_at_100_std": -0.326752, + "nauc_mrr_at_100_diff1": 0.776514, + "nauc_mrr_at_1000_max": 0.173662, + "nauc_mrr_at_1000_std": -0.326752, + "nauc_mrr_at_1000_diff1": 0.776514, + "main_score": 0.81272, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 399.5703556537628, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Waimai.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Waimai.json new file mode 100644 index 0000000000..019db1979f --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Waimai.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "339287def212450dcaa9df8c22bf93e9980c7023", + "task_name": "Waimai", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.9192, + "f1": 0.908417, + "f1_weighted": 0.919409, + "ap": 0.806487, + "ap_weighted": 0.806487, + "scores_per_experiment": [ + { + "accuracy": 0.92, + "f1": 0.908977, + "f1_weighted": 0.920063, + "ap": 0.808664, + "ap_weighted": 0.808664 + }, + { + "accuracy": 0.921, + "f1": 0.909312, + "f1_weighted": 0.920707, + "ap": 0.812851, + "ap_weighted": 0.812851 + }, + { + "accuracy": 0.918, + "f1": 0.907979, + "f1_weighted": 0.918607, + "ap": 0.80189, + "ap_weighted": 0.80189 + }, + { + "accuracy": 0.923, + "f1": 0.912868, + "f1_weighted": 0.923267, + "ap": 0.814111, + "ap_weighted": 0.814111 + }, + { + "accuracy": 0.911, + "f1": 0.900776, + "f1_weighted": 0.911924, + "ap": 0.78641, + "ap_weighted": 0.78641 + }, + { + "accuracy": 0.925, + "f1": 0.914462, + "f1_weighted": 0.92497, + "ap": 0.820232, + "ap_weighted": 0.820232 + }, + { + "accuracy": 0.92, + "f1": 0.90824, + "f1_weighted": 0.919737, + "ap": 0.810426, + "ap_weighted": 0.810426 + }, + { + "accuracy": 0.917, + "f1": 0.906223, + "f1_weighted": 0.91735, + "ap": 0.800844, + "ap_weighted": 0.800844 + }, + { + "accuracy": 0.917, + "f1": 0.906648, + "f1_weighted": 0.917528, + "ap": 0.800108, + "ap_weighted": 0.800108 + }, + { + "accuracy": 0.92, + "f1": 0.908686, + "f1_weighted": 0.919936, + "ap": 0.809335, + "ap_weighted": 0.809335 + } + ], + "main_score": 0.9192, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 10.191343307495117, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/model_meta.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/model_meta.json new file mode 100644 index 0000000000..8264ccdc3f --- /dev/null +++ b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/model_meta.json @@ -0,0 +1 @@ +{"name": "ByteDance-Seed/Doubao-1.5-Embedding", "revision": "1", "release_date": "2025-04-25", "languages": ["eng-Latn","zho-Hans"], "n_parameters": null, "memory_usage_mb": null, "max_tokens": 32768, "embed_dim": 2048, "license": null, "open_weights": false, "public_training_code": null, "public_training_data": null, "framework": ["API"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": null, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": null} \ No newline at end of file