Merge pull request #412 from legendu-net/dev

Merge dev into main

Author: dclong

aiutil/__init__.py

@@ -1,5 +1,6 @@
 """A utils Python package for data scientists.
 """
+
 from . import poetry
 
-__version__ = "0.82.0"
+__version__ = "0.83.0"

aiutil/collections.py

@@ -1,5 +1,6 @@
 """Collections related utills.
 """
+
 from typing import Callable, Any
 
 

aiutil/cv.py

@@ -1,5 +1,6 @@
 """Computer vision related utils.
 """
+
 from typing import Iterable
 from pathlib import Path
 from tqdm import tqdm, trange

aiutil/dataframe.py

@@ -1,9 +1,8 @@
 """Pandas DataFrame related utils.
 """
+
 from pathlib import Path
-from loguru import logger
 import pandas as pd
-from ydata_profiling import ProfileReport
 
 
 def table_2w(
@@ -45,39 +44,3 @@ def read_csv(path: str | Path, **kwargs) -> pd.DataFrame:
     if path.is_file():
         return pd.read_csv(path, **kwargs)
     return pd.concat(pd.read_csv(csv, **kwargs) for csv in path.glob("*.csv"))
-
-
-def dump_profile(df: pd.DataFrame | str | Path, title: str, output_dir: str | Path):
-    """Run ydata-profiling on a DataFrame and dump the report into files.
-
-    :param df: A pandas DataFrame.
-    :param title: The title of the generated report.
-    :param output_dir: The output directory for reports.
-    :raises ValueError: If an input file other than Parquet/Pickle/CSV is provided.
-    """
-    if isinstance(df, str):
-        df = Path(df)
-    if isinstance(df, Path):
-        logger.info("Reading the DataFrame from {}...", df)
-        ext = df.suffix.lower()
-        if ext == ".parquet":
-            df = pd.read_parquet(df)
-        elif ext == ".pickle":
-            df = pd.read_pickle(df)
-        elif ext == ".csv":
-            df = pd.read_csv(df)
-        else:
-            raise ValueError("Only Parquet, Pickle and CSV files are support!")
-    logger.info("Shape of the DataFrame: {}", df.shape)
-    logger.info("Profiling the DataFrame...")
-    report = ProfileReport(df, title=title, minimal=True, explorative=True)
-    if isinstance(output_dir, str):
-        output_dir = Path(output_dir)
-    output_dir.mkdir(exist_ok=True)
-    # dump report
-    logger.info("Dumping the report to HTML...")
-    report.to_file(output_dir / "report.html")
-    logger.info("Dumping the report to JSON...")
-    report.to_file(output_dir / "report.json")
-    logger.info("Dumping the report to Pickle...")
-    report.dump(output_dir / "report.pickle")