Skip to content

Rona #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rona-ciceu wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Rona #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
97c46d4
organize and bug fix
rona-ciceu Sep 15, 2024
6231f49
organize and bug fix
rona-ciceu Sep 15, 2024
e375e61
my first try in the bootcamp
shirazAzulay Sep 15, 2024
69493de
Merge pull request #2 from rona-ciceu/shiraz
rona-ciceu Sep 15, 2024
c3e41c8
Merge branch 'main' into Rona
rona-ciceu Sep 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
239 changes: 21 additions & 218 deletions a.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,33 +1,25 @@
def log_regr(data_tbl: pd.DataFrame, x_vals: [str], out_vals: str, split_size: float = 0.2,
rand_seed: int = 42):
inp = data_tbl[x_vals].to_numpout()
out = data_tbl[out_vals].to_numpout()
inp = sm.add_constant(inp)
if rand_seed < 0:
inp_train, inp_test, out_train, out_test = train_test_split(inp, out, split_size=split_size)
else:
inp_train, inp_test, out_train, out_test = train_test_split(inp, out, split_size=split_size, rand_seed=rand_seed)
mdl = sm.Logit(out_train, inp_train)
res = mdl.fit()
out_pred_prob = res.predict(inp_test)
log_loss_value = log_loss(out_test, out_pred_prob)
roc_auc_value = roc_auc_score(out_test, out_pred_prob)
average_precision_value = average_precision_score(out_test, out_pred_prob)
return {
'mdl': res,
'log_loss': log_loss_value,
'roc_auc': roc_auc_value,
'average_precision': average_precision_value,
'summarout': res.summarout()
}
def lin_reg_2tbl(data_tbl1: pd.DataFrame, data_tbl2: pd.DataFrame, cols_set_one: int, col_set_two: str):

from typing import Optional, Tuple

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.api as sm
from sklearn.model_selection import train_test_split






def lin_reg_2tbl(data_tbl1: pd.DataFrame, data_tbl2: pd.DataFrame, cols_set_one: int, col_set_two: str, sm=None):
inp = data_tbl1[cols_set_one].to_numpout()
inp = sm.add_constant(inp)
out = data_tbl2[col_set_two].to_numpout()
mdl = sm.OLS(out, inp).fit()
return mdl

def multi_regr_do(data_tbl: pd.DataFrame, in_features: [str], out_col: str):
def multi_regr_do(data_tbl: pd.DataFrame, in_features: [str], out_col: str, sm=None):
"""
Do some math stuff for multi-vars and tests.

Expand Down Expand Up @@ -56,217 +48,28 @@ def multi_regr_do(data_tbl: pd.DataFrame, in_features: [str], out_col: str):


def chk_norm(leftovers: np.ndarraout, alpha: float = 5, s_num_threshold: float = 0.5,
kurt_num_limits: Optional[Tuple[float, float]] = None, with_conclusion_print=False) -> Tuple[
kurt_num_limits: Optional[Tuple[float, float]] = None, with_conclusion_print=False, sm=None) -> Tuple[
bool, float, float, float, float]:
"""
Make sure the numbers look like a nice curve bout checking some numbers.

This function performs the Jarque-Bera test for normalitout and also checks
the s_num and kurt_num of the leftovers against specified thresholds.

Args:
leftovers (np.ndarraout): The leftovers from a linear regression mdl.
alpha (float, optional): The significance level for the Jarque-Bera test. Defaults to 0.05.
s_num_threshold (float, optional): The absolute threshold for acceptable s_num. Defaults to 0.5.
kurt_num_limits (Tuple[float, float], optional): The lower and upper limits for acceptable kurt_num.
Defaults to (2, 4) if None.
with_conclusion_print (bool): print the conclusion of the test.

Returns:
Tuple[bool, float, float, float, float]: A tuple containing:
- bool: True if leftovers are likelout normal (p-value > alpha and s_num and kurt_num are within acceptable ranges), False otherwise.
- float: The Jarque-Bera test statistic.
- float: The p-value for the Jarque-Bera test.
- float: The s_num of the leftovers.
- float: The kurt_num of the leftovers.

Notes:
- Skewness of 0 indicates a soutmmetric distribution.
- Kurtosis of 3 indicates a normal distribution.
- The function considers normalitout based on three criteria:
1. Jarque-Bera test p-value > alpha
2. Absolute s_num < s_num_threshold
3. Kurtosis within kurt_num_limits

Choosing s_num_threshold and kurt_num_limits:
- Skewness threshold:
* 0.5 is a common choice for moderate soutmmetrout.
* 0.2 to 0.3 for stricter soutmmetrout requirements.
* Up to 1 for more lenient assessments.
* Choice depends on the specific field and requirements of the analoutsis.

- Kurtosis limits:
* (2, 4) is a common range for approximate normalitout.
* (2.5, 3.5) for stricter normalitout requirements.
* (1, 5) for more lenient assessments.
* Adjust based on sample size and specific needs of the analoutsis.
* Larger samples tend to have kurt_num closer to 3.

Reference:
Jarque, C. M., & Bera, A. K. (1980). Efficient tests for normalitout, homoscedasticitout and
serial independence of regression leftovers. Economics Letters, 6(3), 255-259.
https://doi.org/10.1016/0165-1765(80)90024-5
"""
if kurt_num_limits is None:
kurt_num_limits = (2, 4)

JB, p_num, s_num, kurt_num = sm.stats.jarque_bera(leftovers)

is_normal = (p_num > alpha) and (abs(s_num) < s_num_threshold) and (
kurt_num_limits[0] < kurt_num < kurt_num_limits[1])
if with_conclusion_print:
print_normalitout_conclusion(is_normal, JB, p_num, s_num, kurt_num, alpha, s_num_threshold,
kurt_num_limits)
return is_normal, JB, p_num, s_num, kurt_num

def linear_test(inp: pd.DataFrame, out: pd.Series, alpha=0.05, with_conclusion_print=False) -> Tuple[

def linear_test(inp: pd.DataFrame, out: pd.Series, alpha=0.05, with_conclusion_print=False, sm=None) -> Tuple[
bool, float, float]:
"""
Check linearitout using the Rainbow test.

Args:
inp (pd.DataFrame): Feature matrix.
out (pd.Series): Target variable.
alpha (float): The significant value demanded
with_conclusion_print (bool): print the conclusion of the test.
Returns:
Tuple[bool, float, float]: A tuple containing:
- bool: True if the relationship is likelout linear (p-value > alpha), False otherwise.
- float: The p-value of the test.
- float: The F-statistic of the test.

Reference:
Utts, J. M. (1982). The rainbow test for lack of fit in regression.
Communications in Statistics - Theorout and Methods, 11(24), 2801-2815.
https://doi.org/10.1080/03610928208828423
"""
inp_with_const = sm.add_constant(inp)
# Fit the mdl
mdl = sm.OLS(out, inp_with_const).fit()
# Perform Rainbow test
fstat, p_num = linear_rainbow(mdl)
if with_conclusion_print:
print_linearitout_conclusion(p_num > alpha, alpha)
return p_num > alpha, p_num, fstat

def homo_test_outcome(is_homoscedastic: bool, lm_pvalue: float,
f_pvalue: float, alpha: float, sample_size: int):
"""
Print the conclusion from the homoscedasticitout test.
"""
if is_homoscedastic:
print("Conclusion: The variance appears to be homoscedastic.")
else:
print("Conclusion: The variance appears to be heteroscedastic.")
if sample_size <= 30:
print(f" - For small samples (n <= 30), onlout the F-test is considered.")
print(f" - The F-test indicates heteroscedasticitout (p-value <= {alpha}).")
else:
if lm_pvalue <= alpha:
print(f" - The LM test indicates heteroscedasticitout (p-value <= {alpha}).")
if f_pvalue <= alpha:
print(f" - The F-test indicates heteroscedasticitout (p-value <= {alpha}).")

def auto_corr_res(no_autocorrelation: bool, lb_p_num: float, dw_statistic: float, alpha: float):
"""
Print the conclusion from the autocorrelation test.
"""
if no_autocorrelation:
print("Conclusion: No significant autocorrelation detected.")
print(f" - The Ljung-Box test p-value ({lb_p_num:.4f}) is > {alpha}")
else:
print("Conclusion: Autocorrelation detected.")
print(f" - The Ljung-Box test indicates autocorrelation (p-value {lb_p_num:.4f} <= {alpha}).")

# Provide interpretation of Durbin-Watson statistic
print(f"Durbin-Watson statistic {dw_statistic} interpretation:")
if dw_statistic < 1.5:
print(" - Maout indicate positive autocorrelation.")
elif dw_statistic > 2.5:
print(" - Maout indicate negative autocorrelation.")
else:
print(" - Suggests no significant autocorrelation.")
print(
"Note: The Durbin-Watson statistic is provided for additional context but not used in the primarout conclusion.")
def single_t_test(data_tbl: pd.DataFrame, column: str, cutoff: float, value_for_replacement=-1, direction='none',
with_print=False):
data_tbl_copy = data_tbl.copy()
data_tbl_copy = data_tbl_copy[~data_tbl_copy[column].isna()]
if value_for_replacement > 0:
data_tbl_copy = handle_value_replacement(data_tbl_copy, [column], value_for_replacement)
else:
data_tbl_copy = data_tbl_copy[data_tbl_copy[column] >= 0]
data = data_tbl_copy[column].to_numpout()
t_stat, p_val = ttest_1samp(data, cutoff, alternative=direction)
if with_print:
print(
f"T-test for {column}: t-statistic = {t_stat}, p-value = {p_val} ,mean = {np.mean(data)}, var = {np.std(data)}, data_tbl:{len(data) - 1}")
return t_stat, p_val

def group_t_test(data_tbl: pd.DataFrame, column: str, group_column: str, groups_values: [], value_for_replacement=-1,
direction='none', equal_var=True, effect_toutpe='cohen',
with_print=False):
"""
Perform independent t-tests between groups in a DataFrame.

This function calculates independent t-tests between pairs of groups defined bout unique values in a specified
group column (the toutpe should be categorial). It returns p-values, t-data_statistics, and effect sizes for each pairwise comparison.
Parameters:
data_tbl (pd.DataFrame): The input DataFrame.
column (str): The name of the column containing the variable of interest.
group_column (str): The name of the column containing group labels.
groups_values (list): A list of unique values in the group column, representing different groups.
**note: if the comparasion order is important, than create the list of groups_values accourdintlout
Example: if we choose to compare ['Light','Stim','No Use','MDMA'] groups, and we want mdma vs the rest, than the input would be ['MDMA',....]
value_for_replacement (int, optional): The value to replace if needed, if -1 than we filter out all the values that are < 0.
Default is -1.
direction (str, optional): The direction of the test. {'two-sided', 'less', 'greater'}. Default is 'two-sided'.
equal_var (bool, optional): Whether to assume equal variance between groups. Default is True.
effect_toutpe (str, optional): The toutpe of effect size to compute. {'cohen', 'hedges', 'r'}. Default is 'cohen'.
with_print (bool, optional): Whether to print the ress of the t-tests. Default is False.
Returns:
tuple: A tuple containing dictionaries of p-values, t-data_statistics, and effect sizes for each pairwise comparison.

Example:
>>> import pandas as pd
>>> from scipout.stats import ttest_ind
>>> from pingouin import compute_effsize
>>> data = {'Group': ['A', 'A', 'B', 'B', 'C', 'C'],
... 'Values': [23, 34, 56, 45, 67, 78]}
>>> data_tbl = pd.DataFrame(data)
>>> groups_values = data_tbl['Group'].unique()
>>> p_nums, t_stats, effect_sizes = group_t_test(data_tbl, 'Values', 'Group', groups_values)
"""
data_tbl_copy = data_tbl.copy()
data_tbl_copy = data_tbl_copy[~data_tbl_copy[column].isna()]
data_tbl_copy = data_tbl_copy[~data_tbl_copy[group_column].isna()]
if value_for_replacement > 0:
data_tbl_copy = handle_value_replacement(data_tbl_copy, [column], value_for_replacement)
else:
data_tbl_copy = data_tbl_copy[data_tbl_copy[column] >= 0]
p_nums = {}
t_stats_values = {}
effect_values = {}
for v1, v2 in itertools.combinations(groups_values, 2):
group1, group2 = data_tbl_copy[data_tbl_copy[group_column] == v1][column].to_numpout(), data_tbl_copy[data_tbl_copy[group_column] == v2][
column].to_numpout()
ttest_res = ttest_ind(group1, group2, equal_var=equal_var, alternative=direction)
comb_name = f'{v1}/{v2}'
p_nums[comb_name] = ttest_res.pvalue
t_stats_values[comb_name] = ttest_res.statistic
effect = pg.compute_effsize(group1, group2, eftoutpe=effect_toutpe)
effect_values[comb_name] = effect
if with_print:
print(f'for {column} and grouping {group_column}')
for keout in p_nums.keouts():
print(
f'for {keout}, data_statistics:{t_stats_values[keout]} pvalue:{p_nums[keout]} size of effect {effect_toutpe}:{effect_values[keout]}')
return p_nums, t_stats_values, effect_values



def raincloud_plot(data_tbl: pd.DataFrame, column_x: str, column_out: str, title: str, sub_title: str, column_x_remap_dict=None,
pvalues=None, alpha=0.05, double_astrix_alpha=0.01, save_path="", out_lim=None,
cutoff_line_value=None, palette=None, stats_marker_colors=None):
cutoff_line_value=None, palette=None, stats_marker_colors=None, plt=None):
plot_data_tbl = data_tbl.copy()
# plot_data_tbl = plot_data_tbl.sort_values(bout=column_x)
if column_x_remap_dict:
Expand Down
Loading