diff --git a/README.md b/README.md index a815482..52a037b 100644 --- a/README.md +++ b/README.md @@ -30,4 +30,27 @@ It gets the data matrix from a supported CSV format from either a VFS or downloa ``` from cover_class.static.retrieval import generate_hdf5_from_config generate_hdf5_from_config('/path/to/my/config.yml') -``` \ No newline at end of file +``` + +### Outlier Detection +A separate feature of the cover-class repository is the ability to utilize outlier detectors which will save out a png highlighting any outliers and provide the indices in the dataset of them. + +The current options available are: +- z-score +- kmeans +- mahalanobis +- lof (Local Outlier Factor) + +Example: +``` +>>> import numpy as np +>>> my_data = np.load('my_data.npy') +>>> +>>> from cover_class.outlier_detection import show_outliers +>>> kwargs = {'outlier_percentile': 80} +>>> show_outliers(my_data, 'mahalanobis', png_name='my-data-outliers.png', **kwargs) +array([ 2, 4, 5, 37, 50, 59, 60]) +``` + +And an example of an output png: +![outlier detection figure](figs/outlier-detection-readme-fig.png) \ No newline at end of file diff --git a/figs/outlier-detection-readme-fig.png b/figs/outlier-detection-readme-fig.png new file mode 100644 index 0000000..2ced964 Binary files /dev/null and b/figs/outlier-detection-readme-fig.png differ diff --git a/src/cover_class/outlier_detection.py b/src/cover_class/outlier_detection.py new file mode 100644 index 0000000..0ea81d1 --- /dev/null +++ b/src/cover_class/outlier_detection.py @@ -0,0 +1,52 @@ +from numpy.typing import NDArray +import numpy as np +from scipy.stats import zscore #type: ignore +from sklearn.neighbors import NearestNeighbors, LocalOutlierFactor #type: ignore +from scipy.spatial.distance import mahalanobis #type: ignore +import matplotlib.pyplot as plt + +def zcore_outliers(data:NDArray, **kwargs) -> NDArray: + z = np.abs(zscore(data)) + return (z > 3).any(axis=1) + +def kmeans_outliers(data:NDArray, outlier_percentile:int=95, **kwargs) -> NDArray: + nbrs = NearestNeighbors(n_neighbors=5).fit(data) + dists, _ = nbrs.kneighbors(data) + return dists.mean(axis=1) > np.percentile(dists.mean(axis=1), outlier_percentile) + +def mahalanobis_distance(data:NDArray, outlier_percentile:int=95, **kwargs) -> NDArray: + cov = np.cov(data.T) + inv_cov = np.linalg.pinv(cov) + center = data.mean(0) + m = np.array([mahalanobis(x, center, inv_cov) for x in data]) + return m > np.percentile(m, outlier_percentile) + +def local_outlier_factor(data:NDArray, metric='cosine', **kwargs) -> NDArray: + return LocalOutlierFactor(metric=metric, **kwargs).fit_predict(data) == -1 + +def show_outliers(data:NDArray, method:str='z-score', png_name:str='', **kwargs) -> NDArray: + # returns the indices of outliers if there are any + outliers = np.ndarray([], dtype=bool) + match method: + case'z-score': + outliers = zcore_outliers(data, **kwargs) + case 'kmeans': + outliers = kmeans_outliers(data, **kwargs) + case 'mahalanobis': + outliers = mahalanobis_distance(data, **kwargs) + case 'lof': + outliers = local_outlier_factor(data, **kwargs) + case _: + raise ValueError('Unsupported outlier method: '+method) + if outliers.sum() == 0: + return np.array([]) + + plt.figure(figsize=(8, 5)) + plt.plot(data[~outliers].T, color='black', alpha=0.1) + plt.plot(data[outliers].T, color='red', alpha=0.9) + plt.title('Outliers using '+method) + if png_name != '': + plt.savefig(png_name) + plt.show() + + return np.where(outliers)[0] diff --git a/src/cover_class/train.py b/src/cover_class/train.py index a9d6786..de35647 100644 --- a/src/cover_class/train.py +++ b/src/cover_class/train.py @@ -51,7 +51,7 @@ def setup_training_from_config( odl = dataloader_from_config( config, FloatTensor(train_spectra), - train_labels, + LongTensor(train_labels.to(dtype=torch.long)), batch_size, shuffle, )