improve serialization and memory allocation

Author: deadsoul44

.gitignore

@@ -66,3 +66,6 @@ package-r/inst/c/
 
 # Environment variables
 .env
+
+# Temp folder
+tmp/

Cargo.toml

@@ -6,7 +6,7 @@ members = [
 
 [package]
 name = "perpetual"
-version = "2.0.0"
+version = "2.1.0"
 edition = "2024"
 authors = ["Mutlu Simsek <msimsek@perpetual-ml.com>", "Serkan Korkmaz <serkor1@duck.com>", "Pieter Pel <pelpieter@gmail.com>"]
 homepage = "https://perpetual-ml.com"

package-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "py-perpetual"
-version = "2.0.0"
+version = "2.1.0"
 edition = "2024"
 authors = ["Mutlu Simsek <msimsek@perpetual-ml.com>", "Serkan Korkmaz <serkor1@duck.com>", "Pieter Pel <pelpieter@gmail.com>"]
 homepage = "https://perpetual-ml.com"
@@ -17,7 +17,7 @@ name = "perpetual"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-perpetual_rs = {package="perpetual", version = "2.0.0", path = "../" }
+perpetual_rs = {package="perpetual", version = "2.1.0", path = "../" }
 pyo3 = { version = "0.28.2", features = ["extension-module"] }
 numpy = "0.28.0"
 ndarray = "0.17.2"

package-python/docs/source/conf.py

@@ -8,7 +8,7 @@
 author = "Mutlu Simsek, Serkan Korkmaz, Pieter Pel"
 
 # The full version, including alpha/beta/rc tags
-release = "2.0.0"
+release = "2.1.0"
 
 # -- General configuration ---------------------------------------------------
 

package-python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "perpetual"
-version = "2.0.0"
+version = "2.1.0"
 description = "A self-generalizing gradient boosting machine that doesn't need hyperparameter optimization"
 readme = "README_PYTHON.md"
 license = "Apache-2.0"

package-python/tests/core/test_tabarena.py

@@ -1,4 +1,5 @@
 import os
+import pickle
 import time
 import urllib.request
 
@@ -17,22 +18,29 @@
     "tabarena_wide_y.csv": "https://github.com/user-attachments/files/25682566/y.csv",
     "tabarena_var_x.csv": "https://github.com/user-attachments/files/25739177/X.csv",
     "tabarena_var_y.csv": "https://github.com/user-attachments/files/25739178/y.csv",
+    "tabarena_load_x.csv": "https://github.com/user-attachments/files/25764663/X.csv",
+    "tabarena_load_y.csv": "https://github.com/user-attachments/files/25764665/y.csv",
 }
 
 
 def _get_csv(filename):
     """Try to read a CSV from resources dir, then download, or skip the test."""
+    print(f"Loading {filename}...")
     local_path = os.path.join(RESOURCES_DIR, filename)
     if os.path.isfile(local_path):
-        return pd.read_csv(local_path)
+        res = pd.read_csv(local_path)
+        print(f"Loaded {filename} from resources.")
+        return res
 
     # Try downloading to the resources folder
     url = DOWNLOAD_URLS.get(filename)
     if url is None:
         pytest.skip(f"No download URL configured for {filename}")
     try:
+        print(f"Downloading {filename} from {url}...")
         os.makedirs(RESOURCES_DIR, exist_ok=True)
         urllib.request.urlretrieve(url, local_path)
+        print(f"Downloaded {filename}.")
     except Exception as exc:
         pytest.skip(f"Could not download {filename}: {exc}")
 
@@ -171,8 +179,8 @@ def test_tabarena_wide():
         objective="LogLoss",
         budget=2.0,
         categorical_features=categorical_features,
-        memory_limit=1,  # 30
-        iteration_limit=3,  # 10000
+        memory_limit=1,
+        iteration_limit=3,
         timeout=60 * 15,
     )
 
@@ -225,3 +233,104 @@ def test_tabarena_var():
         categorical_features=categorical_features, iteration_limit=3, memory_limit=1
     )
     model.fit(X, y)
+
+
+def test_tabarena_save_load():
+    print("test_tabarena_save_load started.")
+    X = _get_csv("tabarena_load_x.csv")
+    y = _get_csv("tabarena_load_y.csv")
+
+    categorical_features = [
+        "position_-30",
+        "position_-29",
+        "position_-28",
+        "position_-27",
+        "position_-26",
+        "position_-25",
+        "position_-24",
+        "position_-23",
+        "position_-22",
+        "position_-21",
+        "position_-20",
+        "position_-19",
+        "position_-18",
+        "position_-17",
+        "position_-16",
+        "position_-15",
+        "position_-14",
+        "position_-13",
+        "position_-12",
+        "position_-11",
+        "position_-10",
+        "position_-9",
+        "position_-8",
+        "position_-7",
+        "position_-6",
+        "position_-5",
+        "position_-4",
+        "position_-3",
+        "position_-2",
+        "position_-1",
+        "position_1",
+        "position_2",
+        "position_3",
+        "position_4",
+        "position_5",
+        "position_6",
+        "position_7",
+        "position_8",
+        "position_9",
+        "position_10",
+        "position_11",
+        "position_12",
+        "position_13",
+        "position_14",
+        "position_15",
+        "position_16",
+        "position_17",
+        "position_18",
+        "position_19",
+        "position_20",
+        "position_21",
+        "position_22",
+        "position_23",
+        "position_24",
+        "position_25",
+        "position_26",
+        "position_27",
+        "position_28",
+        "position_29",
+        "position_30",
+    ]
+
+    model = PerpetualBooster(
+        categorical_features=categorical_features,
+        memory_limit=3,
+        num_threads=8,
+        objective="LogLoss",
+        iteration_limit=10,
+        budget=2.0,
+    )
+
+    print(f"Starting fit... memory_limit={model.memory_limit}")
+    model.fit(X, y)
+    print("Fit completed.")
+    print(f"Number of trees: {model.number_of_trees}")
+
+    model_path = os.path.join(RESOURCES_DIR, "model.pkl")
+
+    print(f"Saving model to {model_path}...")
+    with open(model_path, "wb") as f:
+        pickle.dump(model, f)
+    print("Model saved.")
+
+    print("Loading model...")
+    with open(model_path, "rb") as f:
+        loaded_model = pickle.load(f)
+    print("Model loaded.")
+
+    del loaded_model
+    try:
+        os.remove(model_path)
+    except OSError:
+        pass

package-python/uv.lock

@@ -1,7 +1,7 @@
 Package: perpetual
 Type: Package
 Title: PerpetualBooster
-Version: 2.0.0
+Version: 2.1.0
 Authors@R: c(
     person("Mutlu", "Simsek", email = "msimsek@perpetual-ml.com", role = c("aut", "cre")),
     person("Serkan", "Korkmaz", email = "serkor1@duck.com", role = "aut"),

package-r/DESCRIPTION

@@ -3,7 +3,7 @@
 
 [package]
 name = "perpetual_r"
-version = "2.0.0"
+version = "2.1.0"
 edition = "2021"
 
 [lib]

package-r/src/rust/Cargo.toml

@@ -3,7 +3,7 @@
 import subprocess
 from pathlib import Path
 
-# python scripts/increment_version.py 2.0.0 --dry-run
+# python scripts/increment_version.py 2.1.0 --dry-run
 
 
 def update_file(file_path, pattern, replacement, dry_run=False):