Skip to content

Commit 03dedd4

Browse files
elronbandeldafnapension
elronbandel
authored and
dafnapension
committed
Added test and fix
Signed-off-by: elronbandel <elronbandel@gmail.com>
1 parent 0127b32 commit 03dedd4

File tree

59 files changed

+1642
-468
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+1642
-468
lines changed

‎src/unitxt/artifact.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -446,6 +446,14 @@ class Artifact(Dataclass):
446446
default=None, required=False, also_positional=False
447447
)
448448

449+
def __init_subclass__(cls, **kwargs):
450+
super().__init_subclass__(**kwargs)
451+
module = inspect.getmodule(cls)
452+
# standardize module name
453+
module_name = getattr(module, "__name__", None)
454+
if not is_library_module(module_name):
455+
cls.register_class()
456+
449457
@classmethod
450458
def is_possible_identifier(cls, obj):
451459
return isinstance(obj, str) or is_artifact_dict(obj)
@@ -458,18 +466,15 @@ def get_artifact_type(cls):
458466
if not is_library_module(module_name):
459467
non_library_module_warning = f"module named {module_name} is not importable. Class {cls} is thus registered into Artifact.class_register, indexed by {cls.__name__}, accessible there as long as this class_register lives."
460468
warnings.warn(non_library_module_warning, ImportWarning, stacklevel=2)
461-
cls.register_class(cls)
469+
cls.register_class()
462470
return {"module": "class_register", "name": cls.__name__}
463471
if hasattr(cls, "__qualname__") and "." in cls.__qualname__:
464472
return {"module": module_name, "name": cls.__qualname__}
465473
return {"module": module_name, "name": cls.__name__}
466474

467475
@classmethod
468-
def register_class(cls, artifact_class):
469-
Artifact._class_register[artifact_class.__name__] = artifact_class
470-
471-
def __init_subclass__(cls, **kwargs):
472-
super().__init_subclass__(**kwargs)
476+
def register_class(cls):
477+
Artifact._class_register[cls.__name__] = cls
473478

474479
@classmethod
475480
def is_artifact_file(cls, path):
@@ -603,7 +608,7 @@ def maybe_fix_type_to_ensure_instantiation_ability(self):
603608
not is_library_module(self.__type__["module"])
604609
or "<locals>" in self.__type__["name"]
605610
):
606-
self.__class__.register_class(self.__class__)
611+
self.__class__.register_class()
607612
self.__type__ = {
608613
"module": "class_register",
609614
"name": self.__class__.__name__,

‎src/unitxt/catalog/cards/judge_bench/inferential_strategies/sound_reasoning.json‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
"preprocess_steps": [
2020
{
2121
"__type__": {
22-
"module": "unitxt.operators",
23-
"name": "Rename"
22+
"module": "unitxt.processors",
23+
"name": "GroupDictWithRegex"
2424
},
2525
"field": "instance",
2626
"pattern": ".*?### PROBLEM STATEMENT\\s+(?P<problem_statement>.*?)\\s+Statements:\\s+(?P<statements>.*?)\\s+Let\\'s think step by step\\.\\s*### MODEL RESPONSE\\s+(?P<model_reasoning>.*)",
@@ -37,7 +37,6 @@
3737
"instance/model_reasoning": "model reasoning",
3838
"annotations/Sound Reasoning/majority_human": "label"
3939
}
40-
4140
},
4241
{
4342
"__type__": {

‎src/unitxt/catalog/cards/judge_bench/roscoe/overall/cosmos/coherence.json‎

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,6 @@
1717
"data_field": "instances"
1818
},
1919
"preprocess_steps": [
20-
{
21-
"__type__": {
22-
"module": "unitxt.operators",
23-
"name": "Rename"
24-
},
25-
"field": "annotations/Coherency/mean_human",
26-
"to_field": "mean_score"
27-
},
28-
{
29-
"__type__": {
30-
"module": "unitxt.operators",
31-
"name": "Cast"
32-
},
33-
"field": "mean_score",
34-
"to": "float"
35-
},
3620
{
3721
"__type__": {
3822
"module": "unitxt.processors",
@@ -56,12 +40,18 @@
5640
}
5741
},
5842
{
59-
"__type__": "cast",
43+
"__type__": {
44+
"module": "unitxt.operators",
45+
"name": "Cast"
46+
},
6047
"field": "mean_score",
6148
"to": "float"
6249
},
6350
{
64-
"__type__": "set",
51+
"__type__": {
52+
"module": "unitxt.operators",
53+
"name": "Set"
54+
},
6555
"fields": {
6656
"criteria": "metrics.llm_as_judge.direct.criteria.step_by_step_reasoning_coherency",
6757
"question": "Is the Hypothesis supported by the Premise?"

‎src/unitxt/catalog/cards/judge_bench/roscoe/overall/cosmos/contradiction.json‎

Lines changed: 16 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,27 @@
1717
"data_field": "instances"
1818
},
1919
"preprocess_steps": [
20+
{
21+
"__type__": {
22+
"module": "unitxt.processors",
23+
"name": "GroupDictWithRegex"
24+
},
25+
"field": "instance",
26+
"pattern": ".*?Situation \\(Premise\\):\\s+(?P<premise>.*?)\\s+Claim \\(Hypothesis\\):\\s+(?P<hypothesis>.*?)\\s+Is the Claim supported by the Situation\\?\\s+Correct Relationship \\(Yes or No\\):\\s(?P<correct_answer>.*?)\\s+GENERATED RESPONSE:\\s+(?P<model_reasoning>.*?)\\s+Judge the generated response:",
27+
"flags": 16
28+
},
2029
{
2130
"__type__": {
2231
"module": "unitxt.operators",
2332
"name": "Rename"
2433
},
25-
"field": "annotations/Contradiction/majority_human",
26-
"to_field": "label"
34+
"field_to_field": {
35+
"instance/premise": "premise",
36+
"instance/hypothesis": "hypothesis",
37+
"instance/model_reasoning": "generated response",
38+
"instance/correct_answer": "correct answer",
39+
"annotations/Contradiction/majority_human": "label"
40+
}
2741
},
2842
{
2943
"__type__": {
@@ -45,27 +59,6 @@
4559
"field": "label",
4660
"to_field": "label_value"
4761
},
48-
{
49-
"__type__": {
50-
"module": "unitxt.processors",
51-
"name": "GroupDictWithRegex"
52-
},
53-
"field": "instance",
54-
"pattern": ".*?Situation \\(Premise\\):\\s+(?P<premise>.*?)\\s+Claim \\(Hypothesis\\):\\s+(?P<hypothesis>.*?)\\s+Is the Claim supported by the Situation\\?\\s+Correct Relationship \\(Yes or No\\):\\s(?P<correct_answer>.*?)\\s+GENERATED RESPONSE:\\s+(?P<generated_response>.*?)\\s+Judge the generated response:",
55-
"flags": 16
56-
},
57-
{
58-
"__type__": {
59-
"module": "unitxt.operators",
60-
"name": "Rename"
61-
},
62-
"field_to_field": {
63-
"instance/premise": "premise",
64-
"instance/hypothesis": "hypothesis",
65-
"instance/generated_response": "generated response",
66-
"instance/correct_answer": "correct answer"
67-
}
68-
},
6962
{
7063
"__type__": {
7164
"module": "unitxt.operators",

‎src/unitxt/catalog/cards/judge_bench/roscoe/overall/cosmos/missing_steps.json‎

Lines changed: 16 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,27 @@
1717
"data_field": "instances"
1818
},
1919
"preprocess_steps": [
20+
{
21+
"__type__": {
22+
"module": "unitxt.processors",
23+
"name": "GroupDictWithRegex"
24+
},
25+
"field": "instance",
26+
"pattern": ".*?Situation \\(Premise\\):\\s+(?P<premise>.*?)\\s+Claim \\(Hypothesis\\):\\s+(?P<hypothesis>.*?)\\s+Is the Claim supported by the Situation\\?\\s+Correct Relationship \\(Yes or No\\):\\s(?P<correct_answer>.*?)\\s+GENERATED RESPONSE:\\s+(?P<model_reasoning>.*?)\\s+Judge the generated response:",
27+
"flags": 16
28+
},
2029
{
2130
"__type__": {
2231
"module": "unitxt.operators",
2332
"name": "Rename"
2433
},
25-
"field": "annotations/Missing Steps/majority_human",
26-
"to_field": "label"
34+
"field_to_field": {
35+
"instance/premise": "premise",
36+
"instance/hypothesis": "hypothesis",
37+
"instance/model_reasoning": "generated response",
38+
"instance/correct_answer": "correct answer",
39+
"annotations/Missing Steps/majority_human": "label"
40+
}
2741
},
2842
{
2943
"__type__": {
@@ -45,27 +59,6 @@
4559
"field": "label",
4660
"to_field": "label_value"
4761
},
48-
{
49-
"__type__": {
50-
"module": "unitxt.processors",
51-
"name": "GroupDictWithRegex"
52-
},
53-
"field": "instance",
54-
"pattern": ".*?Situation \\(Premise\\):\\s+(?P<premise>.*?)\\s+Claim \\(Hypothesis\\):\\s+(?P<hypothesis>.*?)\\s+Is the Claim supported by the Situation\\?\\s+Correct Relationship \\(Yes or No\\):\\s(?P<correct_answer>.*?)\\s+GENERATED RESPONSE:\\s+(?P<generated_response>.*?)\\s+Judge the generated response:",
55-
"flags": 16
56-
},
57-
{
58-
"__type__": {
59-
"module": "unitxt.operators",
60-
"name": "Rename"
61-
},
62-
"field_to_field": {
63-
"instance/premise": "premise",
64-
"instance/hypothesis": "hypothesis",
65-
"instance/generated_response": "generated response",
66-
"instance/correct_answer": "correct answer"
67-
}
68-
},
6962
{
7063
"__type__": {
7164
"module": "unitxt.operators",

‎src/unitxt/catalog/cards/judge_bench/roscoe/overall/cosmos/overall_quality.json‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,6 @@
1717
"data_field": "instances"
1818
},
1919
"preprocess_steps": [
20-
{
21-
"__type__": {
22-
"module": "unitxt.operators",
23-
"name": "Rename"
24-
},
25-
"field": "annotations/Overall Quality/mean_human",
26-
"to_field": "mean_score"
27-
},
28-
{
29-
"__type__": {
30-
"module": "unitxt.operators",
31-
"name": "Cast"
32-
},
33-
"field": "mean_score",
34-
"to": "float"
35-
},
3620
{
3721
"__type__": {
3822
"module": "unitxt.processors",
@@ -55,6 +39,14 @@
5539
"annotations/Overall Quality/mean_human": "mean_score"
5640
}
5741
},
42+
{
43+
"__type__": {
44+
"module": "unitxt.operators",
45+
"name": "Cast"
46+
},
47+
"field": "mean_score",
48+
"to": "float"
49+
},
5850
{
5951
"__type__": {
6052
"module": "unitxt.operators",

‎src/unitxt/catalog/cards/judge_bench/roscoe/overall/drop/coherence.json‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,13 @@
11
{
2-
"__type__": "task_card",
2+
"__type__": {
3+
"module": "unitxt.card",
4+
"name": "TaskCard"
5+
},
36
"loader": {
4-
"__type__": "load_json_file",
7+
"__type__": {
8+
"module": "unitxt.loaders",
9+
"name": "LoadJsonFile"
10+
},
511
"files": {
612
"test": "https://raw.githubusercontent.com/dmg-illc/JUDGE-BENCH/refs/heads/master/data/roscoe/roscoe-drop-overall.json"
713
},
@@ -12,13 +18,19 @@
1218
},
1319
"preprocess_steps": [
1420
{
15-
"__type__": "group_dict_with_regex",
21+
"__type__": {
22+
"module": "unitxt.processors",
23+
"name": "GroupDictWithRegex"
24+
},
1625
"field": "instance",
1726
"pattern": ".*?Situation \\(Premise\\):\\s+(?P<premise>.*?)\\s+Claim \\(Hypothesis\\):\\s+(?P<hypothesis>.*?)\\s+Is the Claim supported by the Situation\\?\\s+Correct Relationship \\(Yes or No\\):\\s(?P<correct_answer>.*?)\\s+GENERATED RESPONSE:\\s+(?P<model_reasoning>.*?)\\s+Judge the generated response:",
1827
"flags": 16
1928
},
2029
{
21-
"__type__": "rename",
30+
"__type__": {
31+
"module": "unitxt.operators",
32+
"name": "Rename"
33+
},
2234
"field_to_field": {
2335
"instance/premise": "premise",
2436
"instance/hypothesis": "hypothesis",
@@ -28,20 +40,29 @@
2840
}
2941
},
3042
{
31-
"__type__": "cast",
43+
"__type__": {
44+
"module": "unitxt.operators",
45+
"name": "Cast"
46+
},
3247
"field": "mean_score",
3348
"to": "float"
3449
},
3550
{
36-
"__type__": "set",
51+
"__type__": {
52+
"module": "unitxt.operators",
53+
"name": "Set"
54+
},
3755
"fields": {
3856
"criteria": "metrics.llm_as_judge.direct.criteria.step_by_step_reasoning_coherency",
3957
"question": "Is the Hypothesis supported by the Premise?"
4058
}
4159
}
4260
],
4361
"task": {
44-
"__type__": "task",
62+
"__type__": {
63+
"module": "unitxt.task",
64+
"name": "Task"
65+
},
4566
"input_fields": {
4667
"premise": "str",
4768
"hypothesis": "str",

0 commit comments

Comments
 (0)