Updated .gitignore and added raw.csv data import scripts

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,7 @@
+[core]
+    remote = origin
+['remote "remote_storage"']
+    url = ../../dvc_remote
+['remote "origin"']
+    url = s3://dvc
+    endpointurl = https://dagshub.com/94O-O-O-O-O49/examen-dvc.s3

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

.gitignore

@@ -1,2 +1,109 @@
 /venv
 /clean_data.csv
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*.pyc
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env_dags/
+env/
+myenv/
+my_env/
+venv/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# DotEnv configuration
+.env
+
+# Database
+*.db
+*.rdb
+
+# Pycharm
+.idea
+
+# VS Code
+.vscode/
+
+# Spyder
+.spyproject/
+
+# Jupyter NB Checkpoints
+.ipynb_checkpoints/
+
+# exclude data from source control by default
+#/data/
+
+# Mac OS-specific storage files
+.DS_Store
+
+# vim
+*.swp
+*.swo
+
+# Mypy cache
+.mypy_cache/
+*.joblib
+
+# Ignore DVC cache
+.dvc/cache/
+
+# Ignore processed data outputs (they are versioned with DVC)
+dvc/data/processed/
+
+# Ignore model and metric artifacts (tracked via DVC, not Git)
+dvc/models/*.pkl
+dvc/metrics/*.json
+/predictions.csv

data/processed_data/.gitignore → data/processed/.gitignore

@@ -1,4 +1,6 @@
-/X_train.csv
 /X_test.csv
-/y_train.csv
+/X_train.csv
 /y_test.csv
+/y_train.csv
+/X_test_scaled.csv
+/X_train_scaled.csv

data/raw/.gitignore

@@ -0,0 +1 @@
+/raw.csv

dvc.lock

@@ -0,0 +1,146 @@
+schema: '2.0'
+stages:
+  import_data:
+    cmd: python src/data/import_raw_data.py
+    deps:
+    - path: src/data/import_raw_data.py
+      hash: md5
+      md5: 8b2291dc249a573768f8b67c8680dd42
+      size: 1703
+    outs:
+    - path: data/raw/raw.csv
+      hash: md5
+      md5: 617b56af2992376d5c3e91ad80823898
+      size: 333757
+  data_split:
+    cmd: python src/data/data_split.py
+    deps:
+    - path: data/raw/raw.csv
+      hash: md5
+      md5: 617b56af2992376d5c3e91ad80823898
+      size: 333757
+    - path: params.yaml
+      hash: md5
+      md5: 048ae28e2b265ba76f755c6bf1dc583a
+      size: 145
+    - path: src/data/data_split.py
+      hash: md5
+      md5: 06a1bc4e2dc87fca55bc7b8460e77508
+      size: 1403
+    outs:
+    - path: data/processed/X_test.csv
+      hash: md5
+      md5: b2af62a7e108b077aec89fa2622996db
+      size: 52624
+    - path: data/processed/X_train.csv
+      hash: md5
+      md5: 535251fa6c1a1d098cc7a36e0bd62f38
+      size: 209468
+    - path: data/processed/y_test.csv
+      hash: md5
+      md5: a051d29629c8bbb4cfb7e85c7f0315da
+      size: 6456
+    - path: data/processed/y_train.csv
+      hash: md5
+      md5: a9cd9d91da89f732843d72548f64ded3
+      size: 25951
+  data_normalization:
+    cmd: python src/data/normalize.py
+    deps:
+    - path: data/processed/X_test.csv
+      hash: md5
+      md5: b2af62a7e108b077aec89fa2622996db
+      size: 52624
+    - path: data/processed/X_train.csv
+      hash: md5
+      md5: 535251fa6c1a1d098cc7a36e0bd62f38
+      size: 209468
+    - path: src/data/normalize.py
+      hash: md5
+      md5: b6a6629f1164b59cb355307c0f7a0305
+      size: 1017
+    outs:
+    - path: data/processed/X_test_scaled.csv
+      hash: md5
+      md5: 5e079189af6bd091d651a89fafb1c290
+      size: 57026
+    - path: data/processed/X_train_scaled.csv
+      hash: md5
+      md5: c1b80eb7f083888b625dd0369cf381f8
+      size: 227497
+  gridsearch:
+    cmd: python src/models/grid_search.py
+    deps:
+    - path: data/processed/X_train_scaled.csv
+      hash: md5
+      md5: c1b80eb7f083888b625dd0369cf381f8
+      size: 227497
+    - path: data/processed/y_train.csv
+      hash: md5
+      md5: a9cd9d91da89f732843d72548f64ded3
+      size: 25951
+    - path: params.yaml
+      hash: md5
+      md5: 048ae28e2b265ba76f755c6bf1dc583a
+      size: 145
+    - path: src/models/grid_search.py
+      hash: md5
+      md5: 4c1b5b50de7cb97c96bdbaedafc71b71
+      size: 1508
+    outs:
+    - path: models/best_params.pkl
+      hash: md5
+      md5: a3f51b8e2ca564642519fd164a752d2a
+      size: 73
+  model_training:
+    cmd: python src/models/training.py
+    deps:
+    - path: data/processed/X_train_scaled.csv
+      hash: md5
+      md5: c1b80eb7f083888b625dd0369cf381f8
+      size: 227497
+    - path: data/processed/y_train.csv
+      hash: md5
+      md5: a9cd9d91da89f732843d72548f64ded3
+      size: 25951
+    - path: models/best_params.pkl
+      hash: md5
+      md5: a3f51b8e2ca564642519fd164a752d2a
+      size: 73
+    - path: src/models/training.py
+      hash: md5
+      md5: b8a70eac8a2888830cfa99ea48d8bb35
+      size: 935
+    outs:
+    - path: models/gbr_model.pkl
+      hash: md5
+      md5: 0afee3e7757bf99cd021cb829114b1d5
+      size: 1212806
+  model_evaluation:
+    cmd: python src/models/evaluate.py
+    deps:
+    - path: data/processed/X_test_scaled.csv
+      hash: md5
+      md5: 5e079189af6bd091d651a89fafb1c290
+      size: 57026
+    - path: data/processed/y_test.csv
+      hash: md5
+      md5: a051d29629c8bbb4cfb7e85c7f0315da
+      size: 6456
+    - path: models/gbr_model.pkl
+      hash: md5
+      md5: 0afee3e7757bf99cd021cb829114b1d5
+      size: 1212806
+    - path: src/models/evaluate.py
+      hash: md5
+      md5: fb3b3443bad319aa090cedf099c05c78
+      size: 1410
+    outs:
+    - path: metrics/predictions.csv
+      hash: md5
+      md5: 97947b2ba444c4fcb1cc7514bebda871
+      size: 6774
+    - path: metrics/scores.json
+      hash: md5
+      md5: 59a272929a8755a35d2c5b03d6aa1e04
+      size: 64

dvc.yaml

@@ -0,0 +1,56 @@
+stages:
+  import_data:
+    cmd: python src/data/import_raw_data.py
+    deps:
+    - src/data/import_raw_data.py
+    outs:
+    - data/raw/raw.csv
+  data_split:
+    cmd: python src/data/data_split.py
+    deps:
+    - data/raw/raw.csv
+    - params.yaml
+    - src/data/data_split.py
+    outs:
+    - data/processed/X_test.csv
+    - data/processed/X_train.csv
+    - data/processed/y_test.csv
+    - data/processed/y_train.csv
+  data_normalization:
+    cmd: python src/data/normalize.py
+    deps:
+    - data/processed/X_test.csv
+    - data/processed/X_train.csv
+    - src/data/normalize.py
+    outs:
+    - data/processed/X_test_scaled.csv
+    - data/processed/X_train_scaled.csv
+  gridsearch:
+    cmd: python src/models/grid_search.py
+    deps:
+    - data/processed/X_train_scaled.csv
+    - data/processed/y_train.csv
+    - params.yaml
+    - src/models/grid_search.py
+    outs:
+    - models/best_params.pkl
+  model_training:
+    cmd: python src/models/training.py
+    deps:
+    - data/processed/X_train_scaled.csv
+    - data/processed/y_train.csv
+    - models/best_params.pkl
+    - src/models/training.py
+    outs:
+    - models/gbr_model.pkl
+  model_evaluation:
+    cmd: python src/models/evaluate.py
+    deps:
+    - data/processed/X_test_scaled.csv
+    - data/processed/y_test.csv
+    - models/gbr_model.pkl
+    - src/models/evaluate.py
+    outs:
+    - metrics/predictions.csv
+    metrics:
+    - metrics/scores.json

metrics/.gitignore

@@ -0,0 +1,2 @@
+/scores.json
+/predictions.csv

models/.gitignore

@@ -0,0 +1,2 @@
+/best_params.pkl
+/gbr_model.pkl

params.yaml

@@ -0,0 +1,8 @@
+split:
+  test_size: 0.2
+  random_state: 42
+
+grid_search:
+  n_estimators: [100, 200, 300]
+  learning_rate: [0.01, 0.1, 0.2]
+  max_depth: [3, 5, 7]

requirements.txt

@@ -0,0 +1,18 @@
+# local package
+-e .
+
+# external requirements
+click
+Sphinx
+coverage
+#awscli>=1.29.0
+flake8
+pandas
+#logging
+numpy 
+pathlib 
+scikit-learn
+imbalanced-learn
+joblib
+#sys
+#json

setup.py

@@ -0,0 +1,10 @@
+from setuptools import find_packages, setup
+
+setup(
+    name='src',
+    packages=find_packages(),
+    version='0.1.0',
+    description='This project is a starting Pack for MLOps project. It s not perfect so feel free to make some modifications on it.',
+    author='DataScientest',
+    license='MIT',
+)