-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathseries_analyzer_cat.py
More file actions
156 lines (135 loc) · 7.7 KB
/
Copy pathseries_analyzer_cat.py
File metadata and controls
156 lines (135 loc) · 7.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
from sweetviz.config import config
from sweetviz.sv_types import NumWithPercent, FeatureType, FeatureToProcess
from sweetviz.graph_cat import GraphCat
import sweetviz.sv_html as sv_html
import sweetviz.utils as utils
from sweetviz.sv_types import OTHERS_GROUPED
def do_detail_categorical(to_process: FeatureToProcess, updated_dict: dict):
updated_dict["detail"] = dict()
detail = updated_dict["detail"]
# Compute COUNT stats (i.e. below graph)
# ----------------------------------------------------------------------------------------------
detail["full_count"] = []
# To get percentages
num_values = updated_dict["base_stats"]["num_values"].number
if to_process.compare_counts is not None:
num_values_compare = updated_dict["compare"]["base_stats"]["num_values"].number
category_counts = utils.get_clamped_value_counts(to_process.source_counts["value_counts_without_nan"], \
config["Graphs"].getint("detail_graph_max_categories"))
# Iterate through ALL VALUES and get stats
total_num_compare = 0
max_abs_value = 0
for item in category_counts.iteritems():
row = dict()
row["name"] = item[0]
row["count"] = NumWithPercent(item[1], num_values)
# Defaults to no comparison or target
row["count_compare"] = None
row["target_stats"] = None
row["target_stats_compare"] = None
row["is_total"] = None
if to_process.source_target is not None:
# HAS TARGET
# TODO: OPTIMIZE: CACHE FROM GRAPH?
if row["name"] == OTHERS_GROUPED:
this_value_target_only = to_process.source_target[
~to_process.source.isin(category_counts.keys())]
else:
this_value_target_only = to_process.source_target[to_process.source == row["name"]]
if to_process.predetermined_type_target == FeatureType.TYPE_BOOL:
# If value is only present in compared
if len(this_value_target_only) > 0:
count_this_value_target_only = float(this_value_target_only.count())
count_true = this_value_target_only.sum()
row["target_stats"] = NumWithPercent(count_true, count_this_value_target_only)
else:
# None will be correctly interpreted by our display, not nan
row["target_stats"] = None
elif to_process.predetermined_type_target == FeatureType.TYPE_NUM:
# If value is only present in compared
if len(this_value_target_only) > 0:
row["target_stats"] = NumWithPercent(this_value_target_only.mean(), 1.0)
max_abs_value = max(max_abs_value, row["target_stats"].number)
else:
# None will be correctly interpreted by our display, not nan
row["target_stats"] = None
if to_process.compare_counts is not None:
# HAS COMPARE...
if row["name"] in to_process.compare_counts["value_counts_without_nan"].index:
# ...and value exists in COMPARE
matching = to_process.compare_counts["value_counts_without_nan"][row["name"]]
row["count_compare"] = NumWithPercent(matching, num_values_compare)
if to_process.compare_target is not None:
# TODO: OPTIMIZE: CACHE FROM GRAPH?
if row["name"] == OTHERS_GROUPED:
this_value_target_only = to_process.compare_target[
~to_process.compare.isin(category_counts.keys())]
else:
this_value_target_only = to_process.compare_target[to_process.compare == row["name"]]
# HAS COMPARE-TARGET
if to_process.predetermined_type_target == FeatureType.TYPE_BOOL:
if len(this_value_target_only) > 0:
count_this_value_target_only = float(this_value_target_only.count())
count_true = this_value_target_only.sum()
row["target_stats_compare"] = NumWithPercent(count_true,
count_this_value_target_only)
else:
# None will be correctly interpreted by our display, not nan
row["target_stats_compare"] = None
elif to_process.predetermined_type_target == FeatureType.TYPE_NUM:
if len(this_value_target_only) > 0:
row["target_stats_compare"] = NumWithPercent(this_value_target_only.mean(), 1.0)
max_abs_value = max(max_abs_value, row["target_stats_compare"].number)
else:
# None will be correctly interpreted by our display, not nan
row["target_stats_compare"] = None
detail["full_count"].append(row)
detail["max_range"] = max_abs_value
# "ALL" row
# -----------------------------------------------
row = dict()
row["name"] = "ALL"
row["count"] = NumWithPercent(num_values, num_values)
# Defaults to no comparison or target
row["count_compare"] = None
row["target_stats"] = None
row["target_stats_compare"] = None
row["is_total"] = True
if to_process.source_target is not None:
# HAS TARGET
if to_process.predetermined_type_target == FeatureType.TYPE_BOOL:
# TODO: OPTIMIZE: CACHE FROM GRAPH?
count_this_value_target_only = float(to_process.source_target.count())
count_true = to_process.source_target.sum()
row["target_stats"] = NumWithPercent(count_true, count_this_value_target_only)
elif to_process.predetermined_type_target == FeatureType.TYPE_NUM:
# TODO: OPTIMIZE: CACHE FROM GRAPH?
row["target_stats"] = NumWithPercent(to_process.source_target.mean(), 1.0)
if to_process.compare_counts is not None:
row["count_compare"] = NumWithPercent(num_values_compare, num_values_compare)
if to_process.compare_target is not None:
# HAS COMPARE-TARGET
if to_process.predetermined_type_target == FeatureType.TYPE_BOOL:
# TODO: OPTIMIZE: CACHE FROM GRAPH?
count_this_value_target_only = float(to_process.compare_target.count())
count_true = to_process.compare_target.sum()
row["target_stats_compare"] = NumWithPercent(count_true, count_this_value_target_only)
elif to_process.predetermined_type_target == FeatureType.TYPE_NUM:
# TODO: OPTIMIZE: CACHE FROM GRAPH?
row["target_stats_compare"] = NumWithPercent(to_process.compare_target.mean(), 1.0)
detail["full_count"].append(row)
return
def analyze(to_process: FeatureToProcess, feature_dict: dict):
compare_dict = feature_dict.get("compare")
feature_dict["stats"] = dict()
if compare_dict:
compare_dict["stats"] = dict()
do_detail_categorical(to_process, feature_dict)
feature_dict["minigraph"] = GraphCat("mini", to_process)
feature_dict["detail_graphs"] = list()
feature_dict["detail_graphs"].append(GraphCat("detail", to_process))
if to_process.is_target():
feature_dict["html_summary"] = sv_html.generate_html_summary_target_cat(feature_dict, compare_dict)
else:
feature_dict["html_summary"] = sv_html.generate_html_summary_cat(feature_dict, compare_dict)
return