Add DVC remote for DagsHub

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,3 @@
+['remote "origin"']
+    url = s3://dvc
+    endpointurl = https://dagshub.com/diegoklopf/examen-dvc.s3

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

data/raw_data/raw.csv.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 617b56af2992376d5c3e91ad80823898
+  size: 333757
+  hash: md5
+  path: raw.csv

dvc.lock

@@ -0,0 +1,126 @@
+schema: '2.0'
+stages:
+  split:
+    cmd: python src/data/split_data.py
+    deps:
+    - path: data/raw_data/raw.csv
+      hash: md5
+      md5: 617b56af2992376d5c3e91ad80823898
+      size: 333757
+    - path: src/data/split_data.py
+      hash: md5
+      md5: 4af06fafa15eb3ccfb3a1fa84a15f422
+      size: 706
+    outs:
+    - path: data/processed_data/X_test.csv
+      hash: md5
+      md5: b2af62a7e108b077aec89fa2622996db
+      size: 52624
+    - path: data/processed_data/X_train.csv
+      hash: md5
+      md5: 535251fa6c1a1d098cc7a36e0bd62f38
+      size: 209468
+    - path: data/processed_data/y_test.csv
+      hash: md5
+      md5: a051d29629c8bbb4cfb7e85c7f0315da
+      size: 6456
+    - path: data/processed_data/y_train.csv
+      hash: md5
+      md5: a9cd9d91da89f732843d72548f64ded3
+      size: 25951
+  scale:
+    cmd: python src/data/scale_data.py
+    deps:
+    - path: data/processed_data/X_test.csv
+      hash: md5
+      md5: b2af62a7e108b077aec89fa2622996db
+      size: 52624
+    - path: data/processed_data/X_train.csv
+      hash: md5
+      md5: 535251fa6c1a1d098cc7a36e0bd62f38
+      size: 209468
+    - path: src/data/scale_data.py
+      hash: md5
+      md5: b5496f2e92388384f2072ca3823a8b44
+      size: 677
+    outs:
+    - path: data/processed_data/X_test_scaled.csv
+      hash: md5
+      md5: 5e079189af6bd091d651a89fafb1c290
+      size: 57026
+    - path: data/processed_data/X_train_scaled.csv
+      hash: md5
+      md5: c1b80eb7f083888b625dd0369cf381f8
+      size: 227497
+  gridsearch:
+    cmd: python src/models/grid_search.py
+    deps:
+    - path: data/processed_data/X_train_scaled.csv
+      hash: md5
+      md5: c1b80eb7f083888b625dd0369cf381f8
+      size: 227497
+    - path: data/processed_data/y_train.csv
+      hash: md5
+      md5: a9cd9d91da89f732843d72548f64ded3
+      size: 25951
+    - path: src/models/grid_search.py
+      hash: md5
+      md5: 9500f288b8db02992dc58c71d411c9ba
+      size: 704
+    outs:
+    - path: models/best_params.pkl
+      hash: md5
+      md5: fd276a7d693d561052556d6ef3a8229f
+      size: 32
+  train:
+    cmd: python src/models/train_model.py
+    deps:
+    - path: data/processed_data/X_train_scaled.csv
+      hash: md5
+      md5: c1b80eb7f083888b625dd0369cf381f8
+      size: 227497
+    - path: data/processed_data/y_train.csv
+      hash: md5
+      md5: a9cd9d91da89f732843d72548f64ded3
+      size: 25951
+    - path: models/best_params.pkl
+      hash: md5
+      md5: fd276a7d693d561052556d6ef3a8229f
+      size: 32
+    - path: src/models/train_model.py
+      hash: md5
+      md5: 352bf1a7ec10cd505306eb51a3bbeed9
+      size: 559
+    outs:
+    - path: models/model.pkl
+      hash: md5
+      md5: 1b295d939facdf15dcc3f2856067af74
+      size: 694
+  evaluate:
+    cmd: python src/models/evaluate_model.py
+    deps:
+    - path: data/processed_data/X_test_scaled.csv
+      hash: md5
+      md5: 5e079189af6bd091d651a89fafb1c290
+      size: 57026
+    - path: data/processed_data/y_test.csv
+      hash: md5
+      md5: a051d29629c8bbb4cfb7e85c7f0315da
+      size: 6456
+    - path: models/model.pkl
+      hash: md5
+      md5: 1b295d939facdf15dcc3f2856067af74
+      size: 694
+    - path: src/models/evaluate_model.py
+      hash: md5
+      md5: 74cf5d1395fa2b21fc524d945829ba05
+      size: 790
+    outs:
+    - path: data/processed_data/predictions.csv
+      hash: md5
+      md5: 2e39ff526d95a082b2a9cf5efecefbb5
+      size: 6775
+    - path: metrics/scores.json
+      hash: md5
+      md5: 92d6cdb7ac22d23794090d94baa41d19
+      size: 63

dvc.yaml

@@ -0,0 +1,49 @@
+stages:
+  split:
+    cmd: python src/data/split_data.py
+    deps:
+    - data/raw_data/raw.csv
+    - src/data/split_data.py
+    outs:
+    - data/processed_data/X_test.csv
+    - data/processed_data/X_train.csv
+    - data/processed_data/y_test.csv
+    - data/processed_data/y_train.csv
+  scale:
+    cmd: python src/data/scale_data.py
+    deps:
+    - data/processed_data/X_test.csv
+    - data/processed_data/X_train.csv
+    - src/data/scale_data.py
+    outs:
+    - data/processed_data/X_test_scaled.csv
+    - data/processed_data/X_train_scaled.csv
+  gridsearch:
+    cmd: python src/models/grid_search.py
+    deps:
+    - data/processed_data/X_train_scaled.csv
+    - data/processed_data/y_train.csv
+    - src/models/grid_search.py
+    outs:
+    - models/best_params.pkl
+  train:
+    cmd: python src/models/train_model.py
+    deps:
+    - data/processed_data/X_train_scaled.csv
+    - data/processed_data/y_train.csv
+    - models/best_params.pkl
+    - src/models/train_model.py
+    outs:
+    - models/model.pkl
+  evaluate:
+    cmd: python src/models/evaluate_model.py
+    deps:
+    - data/processed_data/X_test_scaled.csv
+    - data/processed_data/y_test.csv
+    - models/model.pkl
+    - src/models/evaluate_model.py
+    outs:
+    - data/processed_data/predictions.csv
+    metrics:
+    - metrics/scores.json:
+        cache: false

metrics/scores.json

@@ -0,0 +1,4 @@
+{
+    "mse": 0.852834650746502,
+    "r2": 0.14791212554040034
+}

requirements.txt

@@ -0,0 +1,6 @@
+dvc
+
+# Librairies scientifiques
+numpy==1.26.0
+pandas==2.0.3
+scikit-learn==1.3.2

src/data/scale_data.py

@@ -0,0 +1,22 @@
+import pandas as pd
+from sklearn.preprocessing import StandardScaler
+
+INPUT_PATH = "data/processed_data"
+OUTPUT_PATH = "data/processed_data"
+
+def main():
+    X_train = pd.read_csv(f"{INPUT_PATH}/X_train.csv")
+    X_test = pd.read_csv(f"{INPUT_PATH}/X_test.csv")
+
+    scaler = StandardScaler()
+    X_train_scaled = scaler.fit_transform(X_train)
+    X_test_scaled = scaler.transform(X_test)
+
+    pd.DataFrame(X_train_scaled, columns=X_train.columns)\
+        .to_csv(f"{OUTPUT_PATH}/X_train_scaled.csv", index=False)
+
+    pd.DataFrame(X_test_scaled, columns=X_test.columns)\
+        .to_csv(f"{OUTPUT_PATH}/X_test_scaled.csv", index=False)
+
+if __name__ == "__main__":
+    main()

src/data/split_data.py

@@ -0,0 +1,23 @@
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+DATA_PATH = "data/raw_data/raw.csv"
+OUTPUT_PATH = "data/processed_data"
+
+def main():
+    df = pd.read_csv(DATA_PATH)
+
+    X = df.iloc[:,1 :-1]   # toutes les colonnes sauf la date et la cible
+    y = df.iloc[:, -1]    # silica_concentrate
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.2, random_state=42
+    )
+
+    X_train.to_csv(f"{OUTPUT_PATH}/X_train.csv", index=False)
+    X_test.to_csv(f"{OUTPUT_PATH}/X_test.csv", index=False)
+    y_train.to_csv(f"{OUTPUT_PATH}/y_train.csv", index=False)
+    y_test.to_csv(f"{OUTPUT_PATH}/y_test.csv", index=False)
+
+if __name__ == "__main__":
+    main()

src/models/evaluate_model.py

@@ -0,0 +1,28 @@
+import pandas as pd
+import pickle
+import json
+from sklearn.metrics import mean_squared_error, r2_score
+
+DATA_PATH = "data/processed_data"
+MODEL_PATH = "models"
+METRICS_PATH = "metrics"
+
+def main():
+    X_test = pd.read_csv(f"{DATA_PATH}/X_test_scaled.csv")
+    y_test = pd.read_csv(f"{DATA_PATH}/y_test.csv").values.ravel()
+
+    with open(f"{MODEL_PATH}/model.pkl", "rb") as f:
+        model = pickle.load(f)
+
+    predictions = model.predict(X_test)
+
+    mse = mean_squared_error(y_test, predictions)
+    r2 = r2_score(y_test, predictions)
+
+    pd.DataFrame({"prediction": predictions}).to_csv(f"{DATA_PATH}/predictions.csv", index=False)
+
+    with open(f"{METRICS_PATH}/scores.json", "w") as f:
+        json.dump({"mse": mse, "r2": r2}, f, indent=4)
+
+if __name__ == "__main__":
+    main()

src/models/grid_search.py

@@ -0,0 +1,32 @@
+import pandas as pd
+import pickle
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import GridSearchCV
+
+DATA_PATH = "data/processed_data"
+MODEL_PATH = "models"
+
+def main():
+    X_train = pd.read_csv(f"{DATA_PATH}/X_train_scaled.csv")
+    y_train = pd.read_csv(f"{DATA_PATH}/y_train.csv").values.ravel()
+
+    model = Ridge()
+
+    param_grid = {
+        "alpha": [0.01, 0.1, 1.0, 10.0]
+    }
+
+    grid = GridSearchCV(
+        model,
+        param_grid,
+        cv=5,
+        scoring="neg_mean_squared_error"
+    )
+
+    grid.fit(X_train, y_train)
+
+    with open(f"{MODEL_PATH}/best_params.pkl", "wb") as f:
+        pickle.dump(grid.best_params_, f)
+
+if __name__ == "__main__":
+    main()

src/models/train_model.py

@@ -0,0 +1,22 @@
+import pandas as pd
+import pickle
+from sklearn.linear_model import Ridge
+
+DATA_PATH = "data/processed_data"
+MODEL_PATH = "models"
+
+def main():
+    X_train = pd.read_csv(f"{DATA_PATH}/X_train_scaled.csv")
+    y_train = pd.read_csv(f"{DATA_PATH}/y_train.csv").values.ravel()
+
+    with open(f"{MODEL_PATH}/best_params.pkl", "rb") as f:
+        best_params = pickle.load(f)
+
+    model = Ridge(**best_params)
+    model.fit(X_train, y_train)
+
+    with open(f"{MODEL_PATH}/model.pkl", "wb") as f:
+        pickle.dump(model, f)
+
+if __name__ == "__main__":
+    main()