Merge pull request #263 from DashAISoftware/improvement/file-upload-and-docs

Improvement dataloaders and added plugins in the documentation

Author: cristian-tamblay

DashAI/back/app.py

@@ -129,11 +129,4 @@ def create_app(
     app.container = container
     logger.debug("Application successfully created.")
 
-    @app.on_event("startup")
-    async def maybe_start_job_loop():
-        if not hasattr(app.state, "job_loop") or app.state.job_loop.done():
-            app.state.job_loop = asyncio.create_task(
-                job_queue_loop(stop_when_queue_empties=False)
-            )
-
     return app

DashAI/back/dataloaders/classes/csv_dataloader.py

@@ -7,7 +7,9 @@
 from datasets import load_dataset
 
 from DashAI.back.core.schema_fields import (
+    bool_field,
     enum_field,
+    int_field,
     none_type,
     schema_field,
     string_field,
@@ -30,18 +32,102 @@ class CSVDataloaderSchema(BaseSchema):
         ),
     )  # type: ignore
     separator: schema_field(
-        enum_field([",", ";", "\u0020", "\t"]),
+        enum_field([",", ";", "blank space", "tab"]),
         ",",
         "A separator character delimits the data in a CSV file.",
     )  # type: ignore
 
+    header: schema_field(
+        string_field(),
+        "infer",
+        (
+            "Row number(s) containing column labels and marking the start of the data "
+            "(zero-indexed). Default behavior is to infer the column names. If column "
+            "names are passed explicitly, this should be set to '0'. "
+            "Header can also be a list of integers that specify row locations "
+            "for MultiIndex on the columns."
+        ),
+    )  # type: ignore
+
+    names: schema_field(
+        none_type(string_field()),
+        None,
+        (
+            "Comma-separated list of column names to use. If the file contains a "
+            "header row, "
+            "then you should explicitly pass header=0 to override the column names. "
+            "Example: 'col1,col2,col3'. Leave empty to use file headers."
+        ),
+    )  # type: ignore
+
+    encoding: schema_field(
+        enum_field(["utf-8", "latin1", "cp1252", "iso-8859-1"]),
+        "utf-8",
+        "Encoding to use for UTF when reading/writing. Most common encodings provided.",
+    )  # type: ignore
+
+    na_values: schema_field(
+        none_type(string_field()),
+        None,
+        (
+            "Comma-separated additional strings to recognize as NA/NaN. "
+            "Example: 'NULL,missing,n/a'"
+        ),
+    )  # type: ignore
+
+    keep_default_na: schema_field(
+        bool_field(),
+        True,
+        (
+            "Whether to include the default NaN values when parsing the data "
+            "(True recommended)."
+        ),
+    )  # type: ignore
+
+    true_values: schema_field(
+        none_type(string_field()),
+        None,
+        "Comma-separated values to consider as True. Example: 'yes,true,1,on'",
+    )  # type: ignore
+
+    false_values: schema_field(
+        none_type(string_field()),
+        None,
+        "Comma-separated values to consider as False. Example: 'no,false,0,off'",
+    )  # type: ignore
+
+    skip_blank_lines: schema_field(
+        bool_field(),
+        True,
+        "If True, skip over blank lines rather than interpreting as NaN values.",
+    )  # type: ignore
+
+    skiprows: schema_field(
+        none_type(int_field()),
+        None,
+        "Number of lines to skip at the beginning of the file. "
+        "Leave empty to skip none.",
+    )  # type: ignore
+
+    nrows: schema_field(
+        none_type(int_field()),
+        None,
+        "Number of rows to read from the file. Leave empty to read all rows.",
+    )  # type: ignore
+
 
 class CSVDataLoader(BaseDataLoader):
     """Data loader for tabular data in CSV files."""
 
     COMPATIBLE_COMPONENTS = ["TabularClassificationTask"]
     SCHEMA = CSVDataloaderSchema
 
+    DESCRIPTION: str = """
+    Data loader for tabular data in CSV files.
+    All uploaded CSV files must have the same column structure and use
+    consistent separators.
+    """
+
     def _check_params(
         self,
         params: Dict[str, Any],
@@ -51,12 +137,49 @@ def _check_params(
                 "Error trying to load the CSV dataset: "
                 "separator parameter was not provided."
             )
-        separator = params["separator"]
 
+        clean_params = {}
+
+        separator = params["separator"]
+        if separator == "blank space":
+            separator = " "
+        elif separator == "tab":
+            separator = "\t"
         if not isinstance(separator, str):
             raise TypeError(
                 f"Param separator should be a string, got {type(params['separator'])}"
             )
+        clean_params["delimiter"] = separator
+
+        if params.get("header") is not None:
+            clean_params["header"] = params["header"]
+
+        list_params = ["names", "na_values", "true_values", "false_values"]
+        for param in list_params:
+            if param in params and params[param]:
+                clean_params[param] = [val.strip() for val in params[param].split(",")]
+
+        bool_params = ["keep_default_na", "skip_blank_lines"]
+        for param in bool_params:
+            if param in params and params[param] is not None:
+                clean_params[param] = params[param]
+
+        int_params = ["skiprows", "nrows"]
+        for param in int_params:
+            if param in params and params[param] is not None:
+                if not isinstance(params[param], int):
+                    raise TypeError(
+                        f"Param {param} should be an integer, got {type(params[param])}"
+                    )
+                clean_params[param] = params[param]
+
+        if "encoding" in params and params["encoding"]:
+            valid_encodings = ["utf-8", "latin1", "cp1252", "iso-8859-1"]
+            if params["encoding"] not in valid_encodings:
+                raise ValueError(f"Invalid encoding: {params['encoding']}")
+            clean_params["encoding"] = params["encoding"]
+
+        return clean_params
 
     @beartype
     def load_data(
@@ -83,20 +206,21 @@ def load_data(
         DatasetDict
             A HuggingFace's Dataset with the loaded data.
         """
-        self._check_params(params)
-        separator = params["separator"]
+        print("parameters are", params)
+        clean_params = self._check_params(params)
+        print("cleaned parameters are", clean_params)
         prepared_path = self.prepare_files(filepath_or_buffer, temp_path)
         if prepared_path[1] == "file":
             dataset = load_dataset(
                 "csv",
                 data_files=prepared_path[0],
-                delimiter=separator,
+                **clean_params,
             )
         else:
             dataset = load_dataset(
                 "csv",
                 data_dir=prepared_path[0],
-                delimiter=separator,
+                **clean_params,
             )
             shutil.rmtree(prepared_path[0])
 

DashAI/back/dataloaders/classes/excel_dataloader.py

@@ -10,6 +10,7 @@
 from datasets.builder import DatasetGenerationError
 
 from DashAI.back.core.schema_fields import (
+    bool_field,
     int_field,
     none_type,
     schema_field,
@@ -47,7 +48,7 @@ class ExcelDataloaderSchema(BaseSchema):
     )  # type: ignore
     header: schema_field(
         none_type(int_field(ge=0)),
-        placeholder=0,
+        placeholder=None,
         description="""
         The row number where the column names are located, indexed from 0.
         If null, the file will be considered to have no column names.
@@ -63,13 +64,112 @@ class ExcelDataloaderSchema(BaseSchema):
         """,
     )  # type: ignore
 
+    skiprows: schema_field(
+        none_type(int_field(ge=0)),
+        None,
+        (
+            "Number of rows to skip at the start of the file. "
+            "Leave empty to not skip any rows."
+        ),
+    )  # type: ignore
+
+    nrows: schema_field(
+        none_type(int_field(ge=1)),
+        None,
+        "Number of rows to read. Leave empty to read all rows.",
+    )  # type: ignore
+
+    names: schema_field(
+        none_type(string_field()),
+        None,
+        (
+            "Comma-separated list of column names to use. Example: 'col1,col2,col3'. "
+            "Leave empty to use header row."
+        ),
+    )  # type: ignore
+
+    na_values: schema_field(
+        none_type(string_field()),
+        None,
+        (
+            "Comma-separated additional strings to recognize as NA/NaN. "
+            "Example: 'NA,N/A,null'."
+        ),
+    )  # type: ignore
+
+    keep_default_na: schema_field(
+        bool_field(),
+        True,
+        "Whether to include the default NaN values when parsing the data.",
+    )  # type: ignore
+
+    true_values: schema_field(
+        none_type(string_field()),
+        None,
+        "Comma-separated values to consider as True. Example: 'yes,true,1'.",
+    )  # type: ignore
+
+    false_values: schema_field(
+        none_type(string_field()),
+        None,
+        "Comma-separated values to consider as False. Example: 'no,false,0'.",
+    )  # type: ignore
+
 
 class ExcelDataLoader(BaseDataLoader):
     """Data loader for tabular data in Excel files."""
 
     COMPATIBLE_COMPONENTS = ["TabularClassificationTask"]
     SCHEMA = ExcelDataloaderSchema
 
+    DESCRIPTION: str = """
+    Data loader for tabular data in Excel files.
+    Supports xls, xlsx, xlsm, xlsb, odf, ods and odt file extensions.
+    """
+
+    def _prepare_pandas_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
+        """Prepare parameters for pandas.read_excel."""
+        pandas_params = {}
+
+        if "sheet" in params and params["sheet"] is not None:
+            pandas_params["sheet_name"] = params["sheet"]
+
+        pandas_params["header"] = params.get("header", 0)
+
+        if "usecols" in params and params["usecols"] is not None:
+            pandas_params["usecols"] = params["usecols"]
+
+        if "skiprows" in params and params["skiprows"] is not None:
+            pandas_params["skiprows"] = params["skiprows"]
+
+        if "nrows" in params and params["nrows"] is not None:
+            pandas_params["nrows"] = params["nrows"]
+
+        if "names" in params and params["names"] is not None:
+            pandas_params["names"] = [
+                name.strip() for name in params["names"].split(",")
+            ]
+
+        if "na_values" in params and params["na_values"] is not None:
+            pandas_params["na_values"] = [
+                val.strip() for val in params["na_values"].split(",")
+            ]
+
+        if "keep_default_na" in params and params["keep_default_na"] is not None:
+            pandas_params["keep_default_na"] = params["keep_default_na"]
+
+        if "true_values" in params and params["true_values"] is not None:
+            pandas_params["true_values"] = [
+                val.strip() for val in params["true_values"].split(",")
+            ]
+
+        if "false_values" in params and params["false_values"] is not None:
+            pandas_params["false_values"] = [
+                val.strip() for val in params["false_values"].split(",")
+            ]
+
+        return pandas_params
+
     @beartype
     def load_data(
         self,
@@ -95,13 +195,15 @@ def load_data(
             A HuggingFace's Dataset with the loaded data.
         """
         prepared_path = self.prepare_files(filepath_or_buffer, temp_path)
+        print("path prepared", prepared_path)
+
+        pandas_params = self._prepare_pandas_params(params)
+
         if prepared_path[1] == "file":
             try:
                 dataset = pd.read_excel(
                     io=prepared_path[0],
-                    sheet_name=params["sheet"],
-                    header=params["header"],
-                    usecols=params["usecols"],
+                    **pandas_params,
                 )
             except ValueError as e:
                 raise DatasetGenerationError from e
@@ -116,9 +218,7 @@ def load_data(
                 train_df_list = [
                     pd.read_excel(
                         io=file_path,
-                        sheet_name=params["sheet"],
-                        header=params["header"],
-                        usecols=params["usecols"],
+                        **pandas_params,
                     )
                     for file_path in sorted(train_files)
                 ]
@@ -127,9 +227,7 @@ def load_data(
                 test_df_list = [
                     pd.read_excel(
                         io=file_path,
-                        sheet_name=params["sheet"],
-                        header=params["header"],
-                        usecols=params["usecols"],
+                        **pandas_params,
                     )
                     for file_path in sorted(test_files)
                 ]
@@ -138,9 +236,7 @@ def load_data(
                 val_df_list = [
                     pd.read_excel(
                         io=file_path,
-                        sheet_name=params["sheet"],
-                        header=params["header"],
-                        usecols=params["usecols"],
+                        **pandas_params,
                     )
                     for file_path in sorted(val_files)
                 ]