Update scikit-learn to version 1.5 (mlrun#967)

* updated scikit-learn~=1.5
fixes and patches for new scikit-learn version
changes in item.yaml and regenerate function.yaml

* remove filename

* remove numpy import

* revert sklearn.metrics monkey patch
fix _get_dataframe to handle list/dict before accessing artifact_url
added feature name preservation logic in predict function

* revert mlrun version

* revert get_or_create_project

* revert scikit-learn version

* scikit-learn==1.5.2
mlrun v 1.10

* scikit-learn==1.4.2

* revert scikit-learn<1.4.0

* scikit-learn~=1.5

* mlrun 1.10 with scikit-learn<1.4.0

* scikit-learn strict v~=1.5.2
added skip for test_train in test_auto_trainer.py

* revert sklearn_classifier.py changes
change XGBRegressor to LGBMRegressor

* added xgboost.XGBRegressor, xgboost.XGBClassifier and lightgbm.LGBMClassifier models to test

Author: tomerbv

functions/src/auto_trainer/auto_trainer.py

@@ -67,30 +67,14 @@ def _get_dataframe(
                             Classification tasks.
     :param drop_columns:    str/int or a list of strings/ints that represent the column names/indices to drop.
     """
-    store_uri_prefix, _ = mlrun.datastore.parse_store_uri(dataset.artifact_url)
-
-    # Getting the dataset:
-    if mlrun.utils.StorePrefix.FeatureVector == store_uri_prefix:
-        label_columns = label_columns or dataset.meta.status.label_column
-        context.logger.info(f"label columns: {label_columns}")
-        # FeatureVector case:
-        try:
-            fv = mlrun.datastore.get_store_resource(dataset.artifact_url)
-            dataset = fv.get_offline_features(drop_columns=drop_columns).to_dataframe()
-        except AttributeError:
-            # Leave here for backwards compatibility
-            dataset = fs.get_offline_features(
-                dataset.meta.uri, drop_columns=drop_columns
-            ).to_dataframe()
-
-    elif not label_columns:
-        context.logger.info(
-            "label_columns not provided, mandatory when dataset is not a FeatureVector"
-        )
-        raise ValueError
-
-    elif isinstance(dataset, (list, dict)):
+    # Check if dataset is list/dict first (before trying to access artifact_url)
+    if isinstance(dataset, (list, dict)):
         # list/dict case:
+        if not label_columns:
+            context.logger.info(
+                "label_columns not provided, mandatory when dataset is not a FeatureVector"
+            )
+            raise ValueError
         dataset = pd.DataFrame(dataset)
         # Checking if drop_columns provided by integer type:
         if drop_columns:
@@ -103,17 +87,38 @@ def _get_dataframe(
                 )
                 raise ValueError
             dataset.drop(drop_columns, axis=1, inplace=True)
-
     else:
-        # simple URL case:
-        dataset = dataset.as_df()
-        if drop_columns:
-            if all(col in dataset for col in drop_columns):
-                dataset = dataset.drop(drop_columns, axis=1)
-            else:
+        # Dataset is a DataItem with artifact_url (URI or FeatureVector)
+        store_uri_prefix, _ = mlrun.datastore.parse_store_uri(dataset.artifact_url)
+
+        # Getting the dataset:
+        if mlrun.utils.StorePrefix.FeatureVector == store_uri_prefix:
+            label_columns = label_columns or dataset.meta.status.label_column
+            context.logger.info(f"label columns: {label_columns}")
+            # FeatureVector case:
+            try:
+                fv = mlrun.datastore.get_store_resource(dataset.artifact_url)
+                dataset = fv.get_offline_features(drop_columns=drop_columns).to_dataframe()
+            except AttributeError:
+                # Leave here for backwards compatibility
+                dataset = fs.get_offline_features(
+                    dataset.meta.uri, drop_columns=drop_columns
+                ).to_dataframe()
+        else:
+            # simple URL case:
+            if not label_columns:
                 context.logger.info(
-                    "not all of the columns to drop in the dataset, drop columns process skipped"
+                    "label_columns not provided, mandatory when dataset is not a FeatureVector"
                 )
+                raise ValueError
+            dataset = dataset.as_df()
+            if drop_columns:
+                if all(col in dataset for col in drop_columns):
+                    dataset = dataset.drop(drop_columns, axis=1)
+                else:
+                    context.logger.info(
+                        "not all of the columns to drop in the dataset, drop columns process skipped"
+                    )
 
     return dataset, label_columns
 

functions/src/auto_trainer/function.yaml

@@ -13,7 +13,7 @@ labels:
   author: Iguazio
 maintainers: []
 marketplaceType: ''
-mlrunVersion: 1.7.0
+mlrunVersion: 1.10.0
 name: auto_trainer
 platformVersion: 3.5.0
 spec:
@@ -23,4 +23,4 @@ spec:
   kind: job
   requirements: []
 url: ''
-version: 1.8.0
+version: 1.9.0

functions/src/auto_trainer/item.yaml

@@ -1,4 +1,5 @@
 pandas
-scikit-learn<1.4.0
+scikit-learn~=1.5.2
+lightgbm
 xgboost<2.0.0
 plotly

functions/src/auto_trainer/requirements.txt

@@ -29,6 +29,9 @@
     ("sklearn.linear_model.LinearRegression", "regression"),
     ("sklearn.ensemble.RandomForestClassifier", "classification"),
     ("xgboost.XGBRegressor", "regression"),
+    ("xgboost.XGBClassifier", "classification"),
+    ("lightgbm.LGBMRegressor", "regression"),
+    ("lightgbm.LGBMClassifier", "classification")
 ]
 
 REQUIRED_ENV_VARS = [
@@ -78,11 +81,15 @@ def _assert_train_handler(train_run):
 
 
 @pytest.mark.parametrize("model", MODELS)
+@pytest.mark.skipif(
+    condition=not _validate_environment_variables(),
+    reason="Project's environment variables are not set",
+)
 def test_train(model: Tuple[str, str]):
     dataset, label_columns = _get_dataset(model[1])
     is_test_passed = True
 
-    project = mlrun.new_project("auto-trainer-test", context="./")
+    project = mlrun.get_or_create_project("auto-trainer-test", context="./")
     fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun")
 
     train_run = None
@@ -119,7 +126,7 @@ def test_train_evaluate(model: Tuple[str, str]):
     dataset, label_columns = _get_dataset(model[1])
     is_test_passed = True
     # Importing function:
-    project = mlrun.new_project("auto-trainer-test", context="./")
+    project = mlrun.get_or_create_project("auto-trainer-test", context="./")
     fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun")
     temp_dir = tempfile.mkdtemp()
 
@@ -172,7 +179,7 @@ def test_train_predict(model: Tuple[str, str]):
     df = pd.read_csv(dataset)
     sample = df.head().drop("labels", axis=1).values.tolist()
     # Importing function:
-    project = mlrun.new_project("auto-trainer-test", context="./")
+    project = mlrun.get_or_create_project("auto-trainer-test", context="./")
     fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun")
     temp_dir = tempfile.mkdtemp()
 

functions/src/auto_trainer/test_auto_trainer.py

@@ -11,7 +11,7 @@ labels:
   author: Iguazio
 maintainers: []
 marketplaceType: ''
-mlrunVersion: 1.7.0
+mlrunVersion: 1.10.0
 name: describe
 platformVersion: 3.5.3
 spec:
@@ -21,4 +21,4 @@ spec:
   kind: job
   requirements: []
 url: ''
-version: 1.4.0
+version: 1.5.0

functions/src/describe/function.yaml

@@ -1,4 +1,4 @@
-scikit-learn~=1.0.2
+scikit-learn~=1.5.2
 plotly~=5.23
 pytest~=7.0.1
 matplotlib~=3.5.1

functions/src/describe/item.yaml

@@ -1,13 +1,15 @@
 metadata:
-  categories:
-  - data-generation
   tag: ''
   name: gen-class-data
+  categories:
+  - data-generation
+verbose: false
 spec:
   description: Create a binary classification sample dataset and save.
-  default_handler: gen_class_data
   entry_points:
     gen_class_data:
+      lineno: 22
+      has_varargs: false
       has_kwargs: false
       parameters:
       - name: context
@@ -48,7 +50,6 @@ spec:
       - name: sk_params
         doc: additional parameters for `sklearn.datasets.make_classification`
         default: {}
-      lineno: 22
       doc: 'Create a binary classification sample dataset and save.
 
         If no filename is given it will default to:
@@ -59,14 +60,13 @@ spec:
         Additional scikit-learn parameters can be set using **sk_params, please see
         https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_classification.html
         for more details.'
-      has_varargs: false
       name: gen_class_data
-  command: ''
-  disable_auto_mount: false
-  image: mlrun/mlrun
   build:
     origin_filename: ''
     functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IHBhbmRhcyBhcyBwZApmcm9tIHR5cGluZyBpbXBvcnQgT3B0aW9uYWwsIExpc3QKZnJvbSBza2xlYXJuLmRhdGFzZXRzIGltcG9ydCBtYWtlX2NsYXNzaWZpY2F0aW9uCgpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKCgpkZWYgZ2VuX2NsYXNzX2RhdGEoCiAgICAgICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICAgICAgbl9zYW1wbGVzOiBpbnQsCiAgICAgICAgbV9mZWF0dXJlczogaW50LAogICAgICAgIGtfY2xhc3NlczogaW50LAogICAgICAgIGhlYWRlcjogT3B0aW9uYWxbTGlzdFtzdHJdXSwKICAgICAgICBsYWJlbF9jb2x1bW46IE9wdGlvbmFsW3N0cl0gPSAibGFiZWxzIiwKICAgICAgICB3ZWlnaHQ6IGZsb2F0ID0gMC41LAogICAgICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gMSwKICAgICAgICBrZXk6IHN0ciA9ICJjbGFzc2lmaWVyLWRhdGEiLAogICAgICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICAgICAgc2tfcGFyYW1zPXt9Cik6CiAgICAiIiJDcmVhdGUgYSBiaW5hcnkgY2xhc3NpZmljYXRpb24gc2FtcGxlIGRhdGFzZXQgYW5kIHNhdmUuCiAgICBJZiBubyBmaWxlbmFtZSBpcyBnaXZlbiBpdCB3aWxsIGRlZmF1bHQgdG86CiAgICAic2ltZGF0YS17bl9zYW1wbGVzfVh7bV9mZWF0dXJlc30ucGFycXVldCIuCgogICAgQWRkaXRpb25hbCBzY2lraXQtbGVhcm4gcGFyYW1ldGVycyBjYW4gYmUgc2V0IHVzaW5nICoqc2tfcGFyYW1zLCBwbGVhc2Ugc2VlIGh0dHBzOi8vc2Npa2l0LWxlYXJuLm9yZy9zdGFibGUvbW9kdWxlcy9nZW5lcmF0ZWQvc2tsZWFybi5kYXRhc2V0cy5tYWtlX2NsYXNzaWZpY2F0aW9uLmh0bWwgZm9yIG1vcmUgZGV0YWlscy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgZnVuY3Rpb24gY29udGV4dAogICAgOnBhcmFtIG5fc2FtcGxlczogICAgIG51bWJlciBvZiByb3dzL3NhbXBsZXMKICAgIDpwYXJhbSBtX2ZlYXR1cmVzOiAgICBudW1iZXIgb2YgY29scy9mZWF0dXJlcwogICAgOnBhcmFtIGtfY2xhc3NlczogICAgIG51bWJlciBvZiBjbGFzc2VzCiAgICA6cGFyYW0gaGVhZGVyOiAgICAgICAgaGVhZGVyIGZvciBmZWF0dXJlcyBhcnJheQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogIGNvbHVtbiBuYW1lIG9mIGdyb3VuZC10cnV0aCBzZXJpZXMKICAgIDpwYXJhbSB3ZWlnaHQ6ICAgICAgICBmcmFjdGlvbiBvZiBzYW1wbGUgbmVnYXRpdmUgdmFsdWUgKGdyb3VuZC10cnV0aD0wKQogICAgOnBhcmFtIHJhbmRvbV9zdGF0ZTogIHJuZyBzZWVkIChzZWUgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tLXN0YXRlKQogICAgOnBhcmFtIGtleTogICAgICAgICAgIGtleSBvZiBkYXRhIGluIGFydGlmYWN0IHN0b3JlCiAgICA6cGFyYW0gZmlsZV9leHQ6ICAgICAgKHBxdCkgZXh0ZW5zaW9uIGZvciBwYXJxdWV0IGZpbGUKICAgIDpwYXJhbSBza19wYXJhbXM6ICAgICBhZGRpdGlvbmFsIHBhcmFtZXRlcnMgZm9yIGBza2xlYXJuLmRhdGFzZXRzLm1ha2VfY2xhc3NpZmljYXRpb25gCiAgICAiIiIKICAgIGZlYXR1cmVzLCBsYWJlbHMgPSBtYWtlX2NsYXNzaWZpY2F0aW9uKAogICAgICAgIG5fc2FtcGxlcz1uX3NhbXBsZXMsCiAgICAgICAgbl9mZWF0dXJlcz1tX2ZlYXR1cmVzLAogICAgICAgIHdlaWdodHM9d2VpZ2h0LAogICAgICAgIG5fY2xhc3Nlcz1rX2NsYXNzZXMsCiAgICAgICAgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZSwKICAgICAgICAqKnNrX3BhcmFtcykKCiAgICAjIG1ha2UgZGF0YWZyYW1lcywgYWRkIGNvbHVtbiBuYW1lcywgY29uY2F0ZW5hdGUgKFgsIHkpCiAgICBYID0gcGQuRGF0YUZyYW1lKGZlYXR1cmVzKQogICAgaWYgbm90IGhlYWRlcjoKICAgICAgICBYLmNvbHVtbnMgPSBbImZlYXRfIiArIHN0cih4KSBmb3IgeCBpbiByYW5nZShtX2ZlYXR1cmVzKV0KICAgIGVsc2U6CiAgICAgICAgWC5jb2x1bW5zID0gaGVhZGVyCgogICAgeSA9IHBkLkRhdGFGcmFtZShsYWJlbHMsIGNvbHVtbnM9W2xhYmVsX2NvbHVtbl0pCiAgICBkYXRhID0gcGQuY29uY2F0KFtYLCB5XSwgYXhpcz0xKQoKICAgIGNvbnRleHQubG9nX2RhdGFzZXQoa2V5LCBkZj1kYXRhLCBmb3JtYXQ9ZmlsZV9leHQsIGluZGV4PUZhbHNlKQo=
     code_origin: ''
+  command: ''
+  image: mlrun/mlrun
+  default_handler: gen_class_data
+  disable_auto_mount: false
 kind: job
-verbose: false

functions/src/describe/requirements.txt

@@ -11,7 +11,7 @@ labels:
   author: Iguazio
 maintainers: []
 marketplaceType: ''
-mlrunVersion: 1.7.0
+mlrunVersion: 1.10.0
 name: gen_class_data
 platformVersion: 3.5.3
 spec:
@@ -21,4 +21,4 @@ spec:
   kind: job
   requirements: []
 url: ''
-version: 1.3.0
+version: 1.4.0