AutoML/test_issue_detection.py at main · salarkhannn/AutoML · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
"""
Comprehensive test script for issue detection functionality.
"""
import pandas as pd
import numpy as np
from src.preprocessing import detect_issues

# Create a comprehensive test dataset with all types of issues
np.random.seed(42)
n = 100

df = pd.DataFrame({
    # Missing values (25% missing)
    'age': [np.nan if i < 25 else np.random.randint(18, 65) for i in range(n)],
    'income': [np.nan if i < 10 else np.random.randint(20000, 100000) for i in range(n)],

    # Outliers
    'score': list(np.random.normal(50, 10, 95)) + [200, 250, -50, -100, 300],

    # High cardinality categorical
    'user_id': ['user_' + str(i) for i in range(n)],

    # Constant feature
    'constant_col': ['same_value'] * n,

    # Near constant
    'near_constant': ['A'] * 96 + ['B'] * 4,

    # Normal categorical
    'category': np.random.choice(['cat1', 'cat2', 'cat3'], n),

    # Imbalanced target
    'target': ['positive'] * 10 + ['negative'] * 90
})

print('Testing comprehensive issue detection...\n')
issues = detect_issues(df, target_col='target')

print(f'Missing Values Issues: {len(issues["missing_values"])}')
for i in issues['missing_values']:
    print(f'  - {i["column"]}: {i["missing_percent"]}% missing ({i["severity"]})')

print(f'\nOutlier Issues (IQR): {len(issues["outliers_iqr"])}')
for i in issues['outliers_iqr']:
    print(f'  - {i["column"]}: {i["outlier_count"]} outliers ({i["severity"]})')

print(f'\nClass Imbalance: {"Yes" if issues["class_imbalance"] else "No"}')
if issues['class_imbalance']:
    print(f'  - Ratio: {issues["class_imbalance"]["imbalance_ratio"]:.2%}')
    print(f'  - Severity: {issues["class_imbalance"]["severity"]}')

print(f'\nHigh Cardinality Issues: {len(issues["high_cardinality"])}')
for i in issues['high_cardinality']:
    print(f'  - {i["column"]}: {i["n_unique"]} unique values ({i["severity"]})')

print(f'\nConstant Feature Issues: {len(issues["constant_features"])}')
for i in issues['constant_features']:
    status = 'constant' if i['is_constant'] else f'near-constant ({i["dominant_percent"]}%)'
    print(f'  - {i["column"]}: {status} ({i["severity"]})')

print('\n All issue detection tests passed!')