write_yaml_file function added in main_utils.utils

Author: mkr9395

data_schema/schema.yaml

@@ -0,0 +1,65 @@
+columns:
+  - having_IP_Address: int64
+  - URL_Length: int64
+  - Shortining_Service: int64
+  - having_At_Symbol: int64
+  - double_slash_redirecting: int64
+  - Prefix_Suffix: int64
+  - having_Sub_Domain: int64
+  - SSLfinal_State: int64
+  - Domain_registeration_length: int64
+  - Favicon: int64
+  - port: int64
+  - HTTPS_token: int64
+  - Request_URL: int64
+  - URL_of_Anchor: int64
+  - Links_in_tags: int64
+  - SFH: int64
+  - Submitting_to_email: int64
+  - Abnormal_URL: int64
+  - Redirect: int64
+  - on_mouseover: int64
+  - RightClick: int64
+  - popUpWidnow: int64
+  - Iframe: int64
+  - age_of_domain: int64
+  - DNSRecord: int64
+  - web_traffic: int64
+  - Page_Rank: int64
+  - Google_Index: int64
+  - Links_pointing_to_page: int64
+  - Statistical_report: int64
+  - Result: int64
+
+numerical_columns:
+ - having_IP_Address
+ - URL_Length
+ - Shortining_Service
+ - having_At_Symbol
+ - double_slash_redirecting
+ - Prefix_Suffix
+ - having_Sub_Domain
+ - SSLfinal_State
+ - Favicon
+ - port
+ - HTTPS_token
+ - Request_URL
+ - URL_of_Anchor
+ - Links_in_tags
+ - SFH
+ - Submitting_to_email
+ - Abnormal_URL
+ - Redirect
+ - on_mouseover
+ - RightClick
+ - popUpWidnow
+ - Iframe
+ - age_of_domain
+ - DNSRecord
+ - web_traffic
+ - Page_Rank
+ - Google_Index
+ - Links_pointing_to_page
+ - Statistical_report
+ - Domain_registeration_length
+ - Result

networksecurity/components/data_validation.py

@@ -1,8 +1,11 @@
 from networksecurity.entity.artifact_entity import DataIngestionArtifact
 from networksecurity.entity.artifact_entity import DataValidationArtifact
 from networksecurity.entity.config_entity import DataValidationConfig
+from networksecurity.constant.training_pipeline import SCHEMA_FILE_PATH
+
 from networksecurity.exception.exception import NetworkSecurityException
 from networksecurity.logging.logger import logging
+from networksecurity.utils.main_utils.utils import read_yaml_file
 
 from scipy.stats import ks_2samp
 import os, sys
@@ -11,3 +14,91 @@
 import numpy as np
 
 
+class DataValidation:
+    
+    def __init__(self, data_ingestion_artifact: DataIngestionArtifact, data_validation_config: DataValidationConfig):
+        
+        try:
+            self.data_ingestion_artifact = data_ingestion_artifact
+            self.data_validation_config = data_validation_config
+            self._schema_config = read_yaml_file(SCHEMA_FILE_PATH)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+        @staticmethod
+        def read_data(file_path)-> pd.DataFrame:
+            try:
+                return pd.read_csv(file_path)
+            except Exception as e:
+                raise NetworkSecurityException(e, sys)
+            
+        def validate_number_of_columns(self, dataframe: pd.DataFrame)-> bool:
+            try:
+                number_of_columns = len(self._schema_config)
+                logging.info(f'required number of columns : {number_of_columns}')
+                logging.info(f'dataframe has columns : {len(dataframe.columns)}')
+                
+                if len(dataframe.columns) == number_of_columns:
+                    return True
+                else:
+                    return False
+            except Exception as e:
+                raise NetworkSecurityException(e, sys)
+        
+    def detect_dataset_drift(self, base_df, current_df, threshold=0.05)-> bool:
+        try:
+            status = True
+            report = {}
+            
+            for column in base_df.columns:
+                d1 = base_df[column]
+                d2 = current_df[column]
+                is_same_dist = ks_2samp(d1, d2)
+                
+                if threshold <= is_same_dist.pvalue:
+                    is_found = False
+                else:
+                    is_found = True
+                    status = False
+                report.update({column: {
+                    "p_value": float(is_same_dist.p_value),
+                    "drift_status": is_found
+                }})
+            
+            drift_report_file_path = self.data_validation_config.data_drift_report_file_path
+            
+            # create directory
+            dir_path = os.path.dirname(drift_report_file_path)
+            os.makedirs(dir_path,exist_ok=True)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys) 
+     
+        
+    def initiate_data_validation(self) -> DataValidationArtifact:
+        
+        try:
+            train_file_path = self.data_ingestion_artifact.trained_file_path
+            test_file_path = self.data_ingestion_artifact.test_file_path
+            
+            # read data from train and test
+            train_dataframe = DataValidation.read_data(train_file_path)
+            test_dataframe = DataValidation.read_data(test_file_path)
+            
+            # validate number of columns for train set
+            status = self.validate_number_of_columns(dataframe = train_dataframe)          
+            if status == False:
+                error_message = "Train dataframe doesnot contain all columns.\n"
+            
+            # validate number of columns for test set
+            status = self.validate_number_of_columns(dataframe = test_dataframe)
+            if status == False:
+                error_message = "Test dataframe doesnot contain all columns.\n"
+            
+            ## check the data drift
+            
+                
+                
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+

networksecurity/entity/config_entity.py

@@ -105,6 +105,6 @@ def __init__(self, training_pipeline_config: TrainingPipelineConfig):
         self.invalid_test_file_path: str = os.path.join(self.invalid_data_dir, training_pipeline.TEST_FILE_NAME)
 
         #  Path to the data drift report file, used to track changes in data distribution
-        self.data_drift_report_file = os.path.join(self.data_validation_dir,
+        self.data_drift_report_file_path = os.path.join(self.data_validation_dir,
                                                    training_pipeline.DATA_VALIDATION_DRIFT_REPORT_DIR,  
                                                    training_pipeline.DATA_VALIDATION_DRIFT_REPORT_FILE_NAME)

networksecurity/utils/main_utils/__init__.py

@@ -0,0 +1,30 @@
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+
+import yaml
+import sys, os
+import pandas as pd
+import numpy as np
+
+import dill # for pickling of file
+import pickle
+
+def read_yaml_file(file_path: str) -> dict:
+    try:
+        with open(file_path, 'rb') as yaml_file:
+            return yaml.safe_load(yaml_file)
+    except Exception as e:
+        raise NetworkSecurityException(e, sys) from e
+    
+def write_yaml_file(file_path : str, content: object, replace: bool = False) -> None:
+    try:
+        if replace:
+            if os.path.exists(file_path):
+                os.remove(file_path)
+            os.makedirs(os.path.dirname(file_path), exist_ok = True)
+            
+            with open(file_path, 'w') as file:
+                yaml.dump(content, file)
+    except Exception as e:
+        raise NetworkSecurityException(e, sys)
+    

networksecurity/utils/main_utils/utils.py

@@ -9,5 +9,6 @@ pathlib
 pymongo
 pymongo[srv]==3.6
 certifi
-
+dill
+pyaml
 # -e .