Looking into gower clustering to match the demographics
This commit is contained in:
James Twose
@@ -0,0 +1,333 @@
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import warnings
|
||||
from sklearn.preprocessing import MinMaxScaler, StandardScaler
|
||||
from sklearn.manifold import TSNE
|
||||
import matplotlib.pyplot as plt
|
||||
import seaborn as sns
|
||||
from typing import Union
|
||||
|
||||
|
||||
def variance_threshold(df:pd.DataFrame,
|
||||
threshold: float):
|
||||
scaled_df = pd.DataFrame(MinMaxScaler()
|
||||
.fit_transform(
|
||||
df.select_dtypes("number")
|
||||
),
|
||||
columns=df.columns)
|
||||
summary_df = (scaled_df
|
||||
.var()
|
||||
.to_frame(name="variance")
|
||||
.assign(feature_type=df.dtypes)
|
||||
.assign(discard=lambda x: x["variance"] < threshold)
|
||||
)
|
||||
|
||||
return summary_df
|
||||
|
||||
|
||||
def nunique_threshold(df:pd.DataFrame,
|
||||
threshold: int):
|
||||
scaled_df = pd.DataFrame(MinMaxScaler()
|
||||
.fit_transform(
|
||||
df.select_dtypes("number")
|
||||
),
|
||||
columns=df.columns)
|
||||
summary_df = (scaled_df
|
||||
.apply(lambda x: x.nunique())
|
||||
.to_frame(name="nunique")
|
||||
.assign(percent_unique=lambda x: x["nunique"] / df.shape[0] * 100)
|
||||
.assign(feature_type=df.dtypes)
|
||||
.assign(discard=lambda x: x["nunique"] < threshold)
|
||||
)
|
||||
|
||||
return summary_df
|
||||
|
||||
swell_eda_features_cols = ['MEAN',
|
||||
'MAX',
|
||||
'MIN',
|
||||
'RANGE',
|
||||
'KURT',
|
||||
'SKEW',
|
||||
'MEAN_1ST_GRAD',
|
||||
'STD_1ST_GRAD',
|
||||
'MEAN_2ND_GRAD',
|
||||
'STD_2ND_GRAD',
|
||||
'ALSC',
|
||||
'INSC',
|
||||
'APSC',
|
||||
'RMSC',
|
||||
'MIN_PEAKS',
|
||||
'MAX_PEAKS',
|
||||
'STD_PEAKS',
|
||||
'MEAN_PEAKS',
|
||||
'MIN_ONSET',
|
||||
'MAX_ONSET',
|
||||
'STD_ONSET',
|
||||
'MEAN_ONSET']
|
||||
|
||||
swell_eda_target_cols = [
|
||||
'condition',
|
||||
'Valence',
|
||||
'Arousal',
|
||||
'Dominance',
|
||||
'Stress',
|
||||
'MentalEffort',
|
||||
'MentalDemand',
|
||||
'PhysicalDemand',
|
||||
'TemporalDemand',
|
||||
'Effort',
|
||||
'Performance',
|
||||
'Frustration',
|
||||
'NasaTLX',
|
||||
]
|
||||
|
||||
|
||||
def plot_confusion_matrix(
|
||||
cf,
|
||||
group_names=None,
|
||||
categories="auto",
|
||||
count=True,
|
||||
percent=True,
|
||||
cbar=True,
|
||||
xyticks=True,
|
||||
xyplotlabels=True,
|
||||
sum_stats=True,
|
||||
figsize: tuple = (7, 5),
|
||||
cmap="Blues",
|
||||
title=None,
|
||||
):
|
||||
"""
|
||||
This function will make a pretty plot of an sklearn Confusion Matrix cm using a Seaborn heatmap visualization.
|
||||
Arguments
|
||||
---------
|
||||
cf: confusion matrix to be passed in
|
||||
group_names: List of strings that represent the labels row by row to be shown in each square.
|
||||
categories: List of strings containing the categories to be displayed on the x,y axis. Default is 'auto'
|
||||
count: If True, show the raw number in the confusion matrix. Default is True.
|
||||
normalize: If True, show the proportions for each category. Default is True.
|
||||
cbar: If True, show the color bar. The cbar values are based off the values in the confusion matrix.
|
||||
Default is True.
|
||||
xyticks: If True, show x and y ticks. Default is True.
|
||||
xyplotlabels: If True, show 'True Label' and 'Predicted Label' on the figure. Default is True.
|
||||
sum_stats: If True, display summary statistics below the figure. Default is True.
|
||||
figsize: Tuple representing the figure size. Default will be the matplotlib rcParams value.
|
||||
cmap: Colormap of the values displayed from matplotlib.pyplot.cm. Default is 'Blues'
|
||||
See http://matplotlib.org/examples/color/colormaps_reference.html
|
||||
title: Title for the heatmap. Default is None.
|
||||
"""
|
||||
|
||||
fig, ax = plt.subplots(figsize=figsize)
|
||||
|
||||
# CODE TO GENERATE TEXT INSIDE EACH SQUARE
|
||||
blanks = ["" for i in range(cf.size)]
|
||||
|
||||
if group_names and len(group_names) == cf.size:
|
||||
group_labels = ["{}\n".format(value) for value in group_names]
|
||||
else:
|
||||
group_labels = blanks
|
||||
|
||||
if count:
|
||||
group_counts = ["{0:0.0f}\n".format(value) for value in cf.flatten()]
|
||||
else:
|
||||
group_counts = blanks
|
||||
|
||||
if percent:
|
||||
group_percentages = [
|
||||
"{0:.2%}".format(value) for value in cf.flatten() / np.sum(cf)
|
||||
]
|
||||
else:
|
||||
group_percentages = blanks
|
||||
|
||||
box_labels = [
|
||||
f"{v1}{v2}{v3}".strip()
|
||||
for v1, v2, v3 in zip(group_labels, group_counts, group_percentages)
|
||||
]
|
||||
box_labels = np.asarray(box_labels).reshape(cf.shape[0], cf.shape[1])
|
||||
|
||||
# CODE TO GENERATE SUMMARY STATISTICS & TEXT FOR SUMMARY STATS
|
||||
if sum_stats:
|
||||
# Accuracy is sum of diagonal divided by total observations
|
||||
accuracy = np.trace(cf) / float(np.sum(cf))
|
||||
|
||||
# if it is a binary confusion matrix, show some more stats
|
||||
if len(cf) == 2:
|
||||
# Metrics for Binary Confusion Matrices
|
||||
precision = cf[1, 1] / sum(cf[:, 1])
|
||||
recall = cf[1, 1] / sum(cf[1, :])
|
||||
f1_score = 2 * precision * recall / (precision + recall)
|
||||
stats_text = "\n\nAccuracy={:0.3f}\nPrecision={:0.3f}\nRecall={:0.3f}\nF1 Score={:0.3f}".format(
|
||||
accuracy, precision, recall, f1_score
|
||||
)
|
||||
else:
|
||||
stats_text = "\n\nAccuracy={:0.3f}".format(accuracy)
|
||||
else:
|
||||
stats_text = ""
|
||||
|
||||
if xyticks == False:
|
||||
# Do not show categories if xyticks is False
|
||||
categories = False
|
||||
|
||||
# MAKE THE HEATMAP VISUALIZATION
|
||||
_ = sns.heatmap(
|
||||
cf,
|
||||
annot=box_labels,
|
||||
fmt="",
|
||||
cmap=cmap,
|
||||
cbar=cbar,
|
||||
xticklabels=categories,
|
||||
yticklabels=categories,
|
||||
)
|
||||
|
||||
if xyplotlabels:
|
||||
_ = plt.ylabel("True label")
|
||||
_ = plt.xlabel("Predicted label" + stats_text)
|
||||
else:
|
||||
_ = plt.xlabel(stats_text)
|
||||
|
||||
if title:
|
||||
_ = plt.title(title)
|
||||
|
||||
return fig, ax
|
||||
|
||||
|
||||
def tSNE(
|
||||
data: pd.DataFrame,
|
||||
n_components: int = 2,
|
||||
normalize: bool = True,
|
||||
hue: Union[str, None] = None,
|
||||
tag: Union[str, None] = None,
|
||||
label_fontsize: int = 14,
|
||||
figsize: tuple = (11.7, 8.27),
|
||||
**kwargs,
|
||||
):
|
||||
r"""Perform t-Distributed Stochastic Neighbor Embedding (t-SNE) Analysis
|
||||
More info : https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
|
||||
Parameters
|
||||
----------
|
||||
data: pandas.DataFrame
|
||||
Dataframe which contains some numerical feature
|
||||
n_components : int, optional (default: 2)
|
||||
Dimension of the embedded space (2D or 3D).
|
||||
normalize : bool, optional (default: True)
|
||||
Normalize data prior tSNE.
|
||||
hue: string, optional
|
||||
Grouping variable that will produce points with different colors.
|
||||
Can be either categorical or numeric, although color mapping will behave
|
||||
differently in latter case.
|
||||
tag: string, optional
|
||||
Tag each point with the value relative to the corresponding column (only 2D currently)
|
||||
label_fontsize: int, optional (default: 14)
|
||||
Font size for the `tag`
|
||||
figsize: tuple (default: (11.7, 8.27))
|
||||
Width and height of the figure in inches
|
||||
kwargs: key, value pairings
|
||||
Additional keyword arguments relative to tSNE() function. Additional info:
|
||||
https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
|
||||
Returns
|
||||
----------
|
||||
fig: matplotlib.pyplot.Figure
|
||||
Graph with clusters in embedded space
|
||||
Examples
|
||||
----------
|
||||
>>> data = sns.load_dataset("mpg")
|
||||
>>> from neuropy.cluster_analysis import tSNE
|
||||
>>> fig = tSNE(data, n_components=2, hue='origin', tag='name', generate_plot = False)
|
||||
"""
|
||||
|
||||
# check hue input
|
||||
if hue is None:
|
||||
warnings.warn(
|
||||
"A hue has not been set, hence it shall not be shown in the plot."
|
||||
)
|
||||
if not isinstance(hue, str):
|
||||
raise ValueError("hue input needs to be a string")
|
||||
|
||||
if hue not in data.columns:
|
||||
raise ValueError(f"hue='{hue}' is not contained in dataframe")
|
||||
|
||||
# check tag input
|
||||
if tag is not None:
|
||||
if not isinstance(tag, str):
|
||||
raise ValueError("tag needs to be str type")
|
||||
if tag not in data.columns:
|
||||
raise ValueError(f"tag='{tag}' is not contained in dataframe")
|
||||
else:
|
||||
pass
|
||||
|
||||
# t-SNE takes into account only numerical feature
|
||||
data_num = data.select_dtypes(include="number")
|
||||
if hue in data_num.columns:
|
||||
data_num = data_num.drop(hue, axis=1)
|
||||
data_obj = data.select_dtypes(exclude="number")
|
||||
if hue and hue not in data_obj.columns:
|
||||
# Add hue column if it is numerical
|
||||
data_obj = pd.concat([data_obj, data[[hue]]], axis=1)
|
||||
|
||||
# remove any row with NaNs and normalize data with z-score
|
||||
data_num = data_num.dropna(axis="index", how="any")
|
||||
|
||||
if normalize:
|
||||
# get z-score to treat different dimensions with equal importance
|
||||
data_num = StandardScaler().fit_transform(data_num)
|
||||
|
||||
# Apply t-SNE to normalized_movements: normalized_data
|
||||
tsne_features = TSNE(n_components=n_components, **kwargs).fit_transform(data_num)
|
||||
|
||||
# show t-SNE cluster
|
||||
if n_components == 2:
|
||||
# combine tsne feature with categorical and/or object variables
|
||||
df_tsne = pd.DataFrame(data=tsne_features, columns=["t-SNE (x)", "t-SNE (y)"])
|
||||
df_tsne = pd.concat([df_tsne, data_obj], axis=1)
|
||||
# plot 2D
|
||||
fig, ax = plt.subplots(figsize=figsize)
|
||||
_ = plt.title("t-Distributed Stochastic Neighbor Embedding (t-SNE)")
|
||||
_ = sns.scatterplot(
|
||||
x="t-SNE (x)",
|
||||
y="t-SNE (y)",
|
||||
hue=hue,
|
||||
legend="full",
|
||||
data=df_tsne,
|
||||
alpha=0.8,
|
||||
)
|
||||
|
||||
elif n_components == 3:
|
||||
# combine tsne feature with categorical and/or object variables
|
||||
df_tsne = pd.DataFrame(
|
||||
data=tsne_features, columns=["t-SNE (x)", "t-SNE (y)", "t-SNE (z)"]
|
||||
)
|
||||
df_tsne = pd.concat([df_tsne, data_obj], axis=1)
|
||||
# plot 3D
|
||||
fig = plt.figure(figsize=figsize)
|
||||
_ = plt.title("t-Distributed Stochastic Neighbor Embedding (t-SNE)")
|
||||
ax = fig.add_subplot(111, projection="3d")
|
||||
if hue:
|
||||
i = ax.scatter(
|
||||
df_tsne["t-SNE (x)"],
|
||||
df_tsne["t-SNE (y)"],
|
||||
df_tsne["t-SNE (z)"],
|
||||
c=df_tsne[hue],
|
||||
cmap="tab10",
|
||||
s=60,
|
||||
alpha=0.8,
|
||||
)
|
||||
fig.colorbar(i)
|
||||
else:
|
||||
ax.scatter(
|
||||
df_tsne["t-SNE (x)"],
|
||||
df_tsne["t-SNE (y)"],
|
||||
df_tsne["t-SNE (z)"],
|
||||
c="#75bbfd",
|
||||
s=60,
|
||||
alpha=0.8,
|
||||
)
|
||||
_ = ax.view_init(30, 185)
|
||||
|
||||
else:
|
||||
raise ValueError("n_components can be either 2 or 3")
|
||||
|
||||
# tag each point
|
||||
if tag is not None and n_components == 2:
|
||||
for x, y, tag in zip(df_tsne["t-SNE (x)"], df_tsne["t-SNE (y)"], df_tsne[tag]):
|
||||
plt.annotate(tag, (x, y), fontsize=label_fontsize, alpha=0.75)
|
||||
|
||||
return fig, ax
|
||||
Reference in New Issue
Block a user