Fix insects dataset (#1731)

* Refactor insects dataset and remove variants:

incremental_abrupt_imbalanced
incremental_imbalanced
incremental_reoccurring_imbalanced

* Reformat file

* Remove unnecessary printouts

* Refactor archive_path

* add variants

* Reformat

* Add release note

Author: kulbachcedric

docs/releases/unreleased.md

@@ -1,10 +1,13 @@
 # Unreleased
 
+## datasets
+- Fixed download in Insects dataset. The datasets incremental_abrupt_imbalanced, incremental_imbalanced, incremental_reoccurring_imbalanced and out-of-control are not supported anymore.
+- Refactored `benchmarks` and added plotly dependency for interactive plots
+
 ## stats
 
 - Added `update_many` method to `stats.PearsonCorr`.
 - Changed the calculation of the Kuiper statistic in `base.KolmogorovSmirnov` to correspond to the reference implementation. The Kuiper statistic uses the difference between the maximum value and the minimum value.
-- Refactored `benchmarks` and added plotly dependency for interactive plots
 
 ## tree
 

river/datasets/base.py

@@ -267,7 +267,10 @@ def download(self, force=False, verbose=True):
         # Determine where to download the archive
         directory = self.path.parent
         directory.mkdir(parents=True, exist_ok=True)
-        archive_path = directory.joinpath(os.path.basename(self.url))
+        if self.unpack:
+            archive_path = directory.joinpath(os.path.basename(self.url))
+        else:
+            archive_path = directory.joinpath(os.path.basename(self.filename))
 
         with request.urlopen(self.url) as r:
             # Notify the user

river/datasets/insects.py

@@ -8,27 +8,24 @@
 class Insects(base.RemoteDataset):
     """Insects dataset.
 
-    This dataset has different variants, which are:
+    This dataset has different variants for concept drift evaluation:
 
     - abrupt_balanced
     - abrupt_imbalanced
     - gradual_balanced
     - gradual_imbalanced
-    - incremental-abrupt_balanced
-    - incremental-abrupt_imbalanced
-    - incremental-reoccurring_balanced
-    - incremental-reoccurring_imbalanced
+    - incremental_abrupt_balanced
+    - incremental_reoccurring_balanced
     - incremental_balanced
-    - incremental_imbalanced
-    - out-of-control
+
 
     The number of samples and the difficulty change from one variant to another. The number of
     classes is always the same (6), except for the last variant (24).
 
     Parameters
     ----------
     variant
-        Indicates which variant of the dataset to load.
+        Indicates which variant of the dataset to load. Defaults to "abrupt_balanced".
 
     References
     ----------
@@ -37,43 +34,97 @@ class Insects(base.RemoteDataset):
 
     """
 
-    variant_sizes = {
-        "abrupt_balanced": (52_848, 16_419_025),
-        "abrupt_imbalanced": (355_275, 110_043_637),
-        "gradual_balanced": (24_150, 7_503_750),
-        "gradual_imbalanced": (143_323, 44_371_501),
-        "incremental-abrupt_balanced": (79_986, 24_849_436),
-        "incremental-abrupt_imbalanced": (452_044, 140_004_225),
-        "incremental-reoccurring_balanced": (79_986, 24_849_092),
-        "incremental-reoccurring_imbalanced": (452_044, 140_004_230),
-        "incremental_balanced": (57_018, 17_713_574),
-        "incremental_imbalanced": (452_044, 140_004_218),
-        "out-of-control": (905_145, 277_777_854),
+    variant_configs = {
+        "abrupt_balanced": {
+            "n_samples": 52_848,
+            "size": 14_151_769,
+            "url": "https://drive.google.com/uc?export=download&id=1WQoIuuVgiuXfzv4kvao6XuLQG37V923O&confirm=t",
+            "filename": "abrupt_balanced.csv",
+        },
+        "abrupt_imbalanced": {
+            "n_samples": 355_275,
+            "size": 94_893_622,
+            "url": "https://drive.google.com/uc?export=download&id=1Z9W2_mwawobXeTihZDTDnFOCtfdgmgDa&confirm=t",
+            "filename": "abrupt_imbalanced.csv",
+        },
+        "gradual_balanced": {
+            "n_samples": 24_150,
+            "size": 6_474_831,
+            "url": "https://drive.google.com/uc?export=download&id=1fepYkDxwMbuoRUaG_fsymSzkuapS4vJp&confirm=t",
+            "filename": "gradual_balanced.csv",
+        },
+        "gradual_imbalanced": {
+            "n_samples": 143_323,
+            "size": 38_339_554,
+            "url": "https://drive.google.com/uc?export=download&id=1_WJ7lxK2sx1i6kq9Nqv4KrjT0ZXB0QbC&confirm=t",
+            "filename": "gradual_imbalanced.csv",
+        },
+        "incremental_abrupt_balanced": {
+            "n_samples": 79_986,
+            "size": 21_421_452,
+            "url": "https://drive.google.com/uc?export=download&id=1-J5WIBN8_F_tomdcrOaiLCxk9nzxtFsf&confirm=t",
+            "filename": "incremental_abrupt_balanced.csv",
+        },
+        # "incremental_abrupt_imbalanced": {
+        #    "n_samples": 452_044,
+        #    "size": 140_004_225,
+        #    "url": "https://drive.google.com/uc?export=download&id=1M6QfsernUlM0qvqXdbo9bPYAsHCcfuhb&confirm=t",
+        #    "filename": "incremental_abrupt_imbalanced.csv",
+        # },
+        "incremental_reoccurring_balanced": {
+            "n_samples": 79_986,
+            "size": 21_433_047,
+            "url": "https://drive.google.com/uc?export=download&id=1mSKTSsxzYMjdV005AJqrcMGajuu7dUfW&confirm=t",
+            "filename": "incremental_reoccurring_balanced.csv",
+        },
+        # "incremental_reoccurring_imbalanced": {
+        #    "n_samples": 452_044,
+        #    "size": 140_004_230,
+        #    "url": "https://drive.google.com/uc?export=download&id=1aSqdxvZvug-SwQw5NLY_9nTPhhEjB2ig&confirm=t",
+        #    "filename": "incremental_reoccurring_imbalanced.csv",
+        # },
+        "incremental_balanced": {
+            "n_samples": 57_018,
+            "size": 15_258_997,
+            "url": "https://drive.google.com/uc?export=download&id=1tKQ2KL4m-ACHCVKUDLFPrM4cyhioiOpu&confirm=t",
+            "filename": "incremental_balanced.csv",
+        },
+        # "incremental_imbalanced": {
+        #    "n_samples": 452_044,
+        #    "size": 140_004_218,
+        #    "url": "https://drive.google.com/uc?export=download&id=1K3vp0EjA4FPDeSgffiBe4CMosbB4CKbL&confirm=t",
+        #    "filename": "incremental_imbalanced.csv",
+        # }
     }
-
-    variants = list(variant_sizes.keys())
+    variants = variant_configs.keys()
 
     def __init__(self, variant="abrupt_balanced"):
-        try:
-            n_samples, size = self.variant_sizes[variant]
-        except KeyError:
-            variants = "\n".join(f"- {v}" for v in self.variant_sizes)
+        if variant not in self.variant_configs:
+            variants = "\n".join(f"- {v}" for v in self.variant_configs)
             raise ValueError(f"Unknown variant, possible choices are:\n{variants}")
+
+        config = self.variant_configs[variant]
+        n_samples = config["n_samples"]
+        size = config["size"]
+        url = config["url"]
+        filename = config["filename"]
         n_classes = 24 if variant == "out-of-control" else 6
 
         super().__init__(
             n_classes=n_classes,
             n_samples=n_samples,
             n_features=33,
             task=base.MULTI_CLF,
-            url=f"http://sites.labic.icmc.usp.br/vsouza/repository/creme/INSECTS-{variant}_norm.arff",
+            url=url,
             size=size,
             unpack=False,
+            filename=filename,
         )
         self.variant = variant
 
     def _iter(self):
-        return stream.iter_arff(self.path, target="class")
+        cols = [f"f{i}" for i in range(1, 34)] + ["class"]
+        return stream.iter_csv(self.path, target="class", fieldnames=cols)
 
     @property
     def _repr_content(self):