add the splits to the only card that needs them, and a smallie for ma…

…ke docs-serve Signed-off-by: dafnapension <[email protected]>
IBM · Feb 13, 2025 · 5cd66bb · 5cd66bb
1 parent 8d06b62
commit 5cd66bb
Show file tree

Hide file tree

Showing 48 changed files with 308 additions and 77 deletions.
diff --git a/prepare/cards/xlsum.py b/prepare/cards/xlsum.py
@@ -16,7 +16,7 @@
 
 for lang in langs:
     card = TaskCard(
-        loader=LoadHF(path="GEM/xlsum", name=lang),
+        loader=LoadHF(path="GEM/xlsum", name=lang, splits=["test", "train", "validation"]),
         preprocess_steps=[
             Rename(field_to_field={"text": "document"}),
             Wrap(field="target", inside="list", to_field="summaries"),

diff --git a/src/unitxt/catalog/cards/xlsum/amharic.json b/src/unitxt/catalog/cards/xlsum/amharic.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "amharic"
+        "name": "amharic",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/arabic.json b/src/unitxt/catalog/cards/xlsum/arabic.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "arabic"
+        "name": "arabic",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/azerbaijani.json b/src/unitxt/catalog/cards/xlsum/azerbaijani.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "azerbaijani"
+        "name": "azerbaijani",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/bengali.json b/src/unitxt/catalog/cards/xlsum/bengali.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "bengali"
+        "name": "bengali",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/burmese.json b/src/unitxt/catalog/cards/xlsum/burmese.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "burmese"
+        "name": "burmese",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/chinese_simplified.json b/src/unitxt/catalog/cards/xlsum/chinese_simplified.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "chinese_simplified"
+        "name": "chinese_simplified",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/chinese_traditional.json b/src/unitxt/catalog/cards/xlsum/chinese_traditional.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "chinese_traditional"
+        "name": "chinese_traditional",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/english.json b/src/unitxt/catalog/cards/xlsum/english.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "english"
+        "name": "english",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/french.json b/src/unitxt/catalog/cards/xlsum/french.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "french"
+        "name": "french",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/gujarati.json b/src/unitxt/catalog/cards/xlsum/gujarati.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "gujarati"
+        "name": "gujarati",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/hausa.json b/src/unitxt/catalog/cards/xlsum/hausa.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "hausa"
+        "name": "hausa",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/hindi.json b/src/unitxt/catalog/cards/xlsum/hindi.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "hindi"
+        "name": "hindi",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/igbo.json b/src/unitxt/catalog/cards/xlsum/igbo.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "igbo"
+        "name": "igbo",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/indonesian.json b/src/unitxt/catalog/cards/xlsum/indonesian.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "indonesian"
+        "name": "indonesian",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/japanese.json b/src/unitxt/catalog/cards/xlsum/japanese.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "japanese"
+        "name": "japanese",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/kirundi.json b/src/unitxt/catalog/cards/xlsum/kirundi.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "kirundi"
+        "name": "kirundi",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/korean.json b/src/unitxt/catalog/cards/xlsum/korean.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "korean"
+        "name": "korean",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/kyrgyz.json b/src/unitxt/catalog/cards/xlsum/kyrgyz.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "kyrgyz"
+        "name": "kyrgyz",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/marathi.json b/src/unitxt/catalog/cards/xlsum/marathi.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "marathi"
+        "name": "marathi",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/nepali.json b/src/unitxt/catalog/cards/xlsum/nepali.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "nepali"
+        "name": "nepali",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/oromo.json b/src/unitxt/catalog/cards/xlsum/oromo.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "oromo"
+        "name": "oromo",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/pashto.json b/src/unitxt/catalog/cards/xlsum/pashto.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "pashto"
+        "name": "pashto",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/persian.json b/src/unitxt/catalog/cards/xlsum/persian.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "persian"
+        "name": "persian",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/pidgin.json b/src/unitxt/catalog/cards/xlsum/pidgin.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "pidgin"
+        "name": "pidgin",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/portuguese.json b/src/unitxt/catalog/cards/xlsum/portuguese.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "portuguese"
+        "name": "portuguese",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/punjabi.json b/src/unitxt/catalog/cards/xlsum/punjabi.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "punjabi"
+        "name": "punjabi",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/xlsum/russian.json b/src/unitxt/catalog/cards/xlsum/russian.json
@@ -3,7 +3,12 @@
     "loader": {
         "__type__": "load_hf",
         "path": "GEM/xlsum",
-        "name": "russian"
+        "name": "russian",
+        "splits": [
+            "test",
+            "train",
+            "validation"
+        ]
     },
     "preprocess_steps": [
         {