Looking into gower clustering to match the demographics

.gitignore

@@ -133,3 +133,4 @@ dmypy.json
 .DS_Store
 data
 kp_determinants
+kp_determinants2

citizen_shield/extras.py

@@ -0,0 +1,333 @@
+import pandas as pd
+import numpy as np
+import warnings
+from sklearn.preprocessing import MinMaxScaler, StandardScaler
+from sklearn.manifold import TSNE
+import matplotlib.pyplot as plt
+import seaborn as sns
+from typing import Union
+
+
+def variance_threshold(df:pd.DataFrame, 
+                       threshold: float):
+    scaled_df = pd.DataFrame(MinMaxScaler()
+             .fit_transform(
+                 df.select_dtypes("number")
+             ), 
+             columns=df.columns)
+    summary_df = (scaled_df
+     .var()
+     .to_frame(name="variance")
+     .assign(feature_type=df.dtypes)
+     .assign(discard=lambda x: x["variance"] < threshold)
+    )
+    
+    return summary_df
+
+
+def nunique_threshold(df:pd.DataFrame, 
+                       threshold: int):
+    scaled_df = pd.DataFrame(MinMaxScaler()
+             .fit_transform(
+                 df.select_dtypes("number")
+             ), 
+             columns=df.columns)
+    summary_df = (scaled_df
+     .apply(lambda x: x.nunique())
+     .to_frame(name="nunique")
+                  .assign(percent_unique=lambda x: x["nunique"] / df.shape[0] * 100)
+     .assign(feature_type=df.dtypes)
+     .assign(discard=lambda x: x["nunique"] < threshold)
+    )
+    
+    return summary_df
+
+swell_eda_features_cols = ['MEAN',
+ 'MAX',
+ 'MIN',
+ 'RANGE',
+ 'KURT',
+ 'SKEW',
+ 'MEAN_1ST_GRAD',
+ 'STD_1ST_GRAD',
+ 'MEAN_2ND_GRAD',
+ 'STD_2ND_GRAD',
+ 'ALSC',
+ 'INSC',
+ 'APSC',
+ 'RMSC',
+ 'MIN_PEAKS',
+ 'MAX_PEAKS',
+ 'STD_PEAKS',
+ 'MEAN_PEAKS',
+ 'MIN_ONSET',
+ 'MAX_ONSET',
+ 'STD_ONSET',
+ 'MEAN_ONSET']
+
+swell_eda_target_cols = [
+ 'condition',
+ 'Valence',
+ 'Arousal',
+ 'Dominance',
+ 'Stress',
+ 'MentalEffort',
+ 'MentalDemand',
+ 'PhysicalDemand',
+ 'TemporalDemand',
+ 'Effort',
+ 'Performance',
+ 'Frustration',
+ 'NasaTLX',
+ ]
+
+
+def plot_confusion_matrix(
+    cf,
+    group_names=None,
+    categories="auto",
+    count=True,
+    percent=True,
+    cbar=True,
+    xyticks=True,
+    xyplotlabels=True,
+    sum_stats=True,
+    figsize: tuple = (7, 5),
+    cmap="Blues",
+    title=None,
+):
+    """
+    This function will make a pretty plot of an sklearn Confusion Matrix cm using a Seaborn heatmap visualization.
+    Arguments
+    ---------
+    cf:            confusion matrix to be passed in
+    group_names:   List of strings that represent the labels row by row to be shown in each square.
+    categories:    List of strings containing the categories to be displayed on the x,y axis. Default is 'auto'
+    count:         If True, show the raw number in the confusion matrix. Default is True.
+    normalize:     If True, show the proportions for each category. Default is True.
+    cbar:          If True, show the color bar. The cbar values are based off the values in the confusion matrix.
+                   Default is True.
+    xyticks:       If True, show x and y ticks. Default is True.
+    xyplotlabels:  If True, show 'True Label' and 'Predicted Label' on the figure. Default is True.
+    sum_stats:     If True, display summary statistics below the figure. Default is True.
+    figsize:       Tuple representing the figure size. Default will be the matplotlib rcParams value.
+    cmap:          Colormap of the values displayed from matplotlib.pyplot.cm. Default is 'Blues'
+                   See http://matplotlib.org/examples/color/colormaps_reference.html
+    title:         Title for the heatmap. Default is None.
+    """
+
+    fig, ax = plt.subplots(figsize=figsize)
+
+    # CODE TO GENERATE TEXT INSIDE EACH SQUARE
+    blanks = ["" for i in range(cf.size)]
+
+    if group_names and len(group_names) == cf.size:
+        group_labels = ["{}\n".format(value) for value in group_names]
+    else:
+        group_labels = blanks
+
+    if count:
+        group_counts = ["{0:0.0f}\n".format(value) for value in cf.flatten()]
+    else:
+        group_counts = blanks
+
+    if percent:
+        group_percentages = [
+            "{0:.2%}".format(value) for value in cf.flatten() / np.sum(cf)
+        ]
+    else:
+        group_percentages = blanks
+
+    box_labels = [
+        f"{v1}{v2}{v3}".strip()
+        for v1, v2, v3 in zip(group_labels, group_counts, group_percentages)
+    ]
+    box_labels = np.asarray(box_labels).reshape(cf.shape[0], cf.shape[1])
+
+    # CODE TO GENERATE SUMMARY STATISTICS & TEXT FOR SUMMARY STATS
+    if sum_stats:
+        # Accuracy is sum of diagonal divided by total observations
+        accuracy = np.trace(cf) / float(np.sum(cf))
+
+        # if it is a binary confusion matrix, show some more stats
+        if len(cf) == 2:
+            # Metrics for Binary Confusion Matrices
+            precision = cf[1, 1] / sum(cf[:, 1])
+            recall = cf[1, 1] / sum(cf[1, :])
+            f1_score = 2 * precision * recall / (precision + recall)
+            stats_text = "\n\nAccuracy={:0.3f}\nPrecision={:0.3f}\nRecall={:0.3f}\nF1 Score={:0.3f}".format(
+                accuracy, precision, recall, f1_score
+            )
+        else:
+            stats_text = "\n\nAccuracy={:0.3f}".format(accuracy)
+    else:
+        stats_text = ""
+
+    if xyticks == False:
+        # Do not show categories if xyticks is False
+        categories = False
+
+    # MAKE THE HEATMAP VISUALIZATION
+    _ = sns.heatmap(
+        cf,
+        annot=box_labels,
+        fmt="",
+        cmap=cmap,
+        cbar=cbar,
+        xticklabels=categories,
+        yticklabels=categories,
+    )
+
+    if xyplotlabels:
+        _ = plt.ylabel("True label")
+        _ = plt.xlabel("Predicted label" + stats_text)
+    else:
+        _ = plt.xlabel(stats_text)
+
+    if title:
+        _ = plt.title(title)
+
+    return fig, ax
+
+
+def tSNE(
+    data: pd.DataFrame,
+    n_components: int = 2,
+    normalize: bool = True,
+    hue: Union[str, None] = None,
+    tag: Union[str, None] = None,
+    label_fontsize: int = 14,
+    figsize: tuple = (11.7, 8.27),
+    **kwargs,
+):
+    r"""Perform t-Distributed Stochastic Neighbor Embedding (t-SNE) Analysis
+    More info : https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
+    Parameters
+    ----------
+    data: pandas.DataFrame
+        Dataframe which contains some numerical feature
+    n_components : int, optional (default: 2)
+        Dimension of the embedded space (2D or 3D).
+    normalize : bool, optional (default: True)
+        Normalize data prior tSNE.
+    hue: string, optional
+        Grouping variable that will produce points with different colors.
+        Can be either categorical or numeric, although color mapping will behave
+        differently in latter case.
+    tag: string, optional
+        Tag each point with the value relative to the corresponding column (only 2D currently)
+    label_fontsize: int, optional (default: 14)
+        Font size for the `tag`
+    figsize: tuple (default: (11.7, 8.27))
+        Width and height of the figure in inches
+    kwargs: key, value pairings
+        Additional keyword arguments relative to tSNE() function. Additional info:
+        https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
+    Returns
+    ----------
+    fig: matplotlib.pyplot.Figure
+        Graph with clusters in embedded space
+    Examples
+    ----------
+    >>> data = sns.load_dataset("mpg")
+    >>> from neuropy.cluster_analysis import tSNE
+    >>> fig = tSNE(data, n_components=2, hue='origin', tag='name', generate_plot = False)
+    """
+
+    # check hue input
+    if hue is None:
+        warnings.warn(
+            "A hue has not been set, hence it shall not be shown in the plot."
+        )
+    if not isinstance(hue, str):
+        raise ValueError("hue input needs to be a string")
+
+    if hue not in data.columns:
+        raise ValueError(f"hue='{hue}' is not contained in dataframe")
+
+    # check tag input
+    if tag is not None:
+        if not isinstance(tag, str):
+            raise ValueError("tag needs to be str type")
+        if tag not in data.columns:
+            raise ValueError(f"tag='{tag}' is not contained in dataframe")
+    else:
+        pass
+
+    # t-SNE takes into account only numerical feature
+    data_num = data.select_dtypes(include="number")
+    if hue in data_num.columns:
+        data_num = data_num.drop(hue, axis=1)
+    data_obj = data.select_dtypes(exclude="number")
+    if hue and hue not in data_obj.columns:
+        # Add hue column if it is numerical
+        data_obj = pd.concat([data_obj, data[[hue]]], axis=1)
+
+    # remove any row with NaNs and normalize data with z-score
+    data_num = data_num.dropna(axis="index", how="any")
+
+    if normalize:
+        # get z-score to treat different dimensions with equal importance
+        data_num = StandardScaler().fit_transform(data_num)
+
+    # Apply t-SNE to normalized_movements: normalized_data
+    tsne_features = TSNE(n_components=n_components, **kwargs).fit_transform(data_num)
+
+    # show t-SNE cluster
+    if n_components == 2:
+        # combine tsne feature with categorical and/or object variables
+        df_tsne = pd.DataFrame(data=tsne_features, columns=["t-SNE (x)", "t-SNE (y)"])
+        df_tsne = pd.concat([df_tsne, data_obj], axis=1)
+        # plot 2D
+        fig, ax = plt.subplots(figsize=figsize)
+        _ = plt.title("t-Distributed Stochastic Neighbor Embedding (t-SNE)")
+        _ = sns.scatterplot(
+            x="t-SNE (x)",
+            y="t-SNE (y)",
+            hue=hue,
+            legend="full",
+            data=df_tsne,
+            alpha=0.8,
+        )
+
+    elif n_components == 3:
+        # combine tsne feature with categorical and/or object variables
+        df_tsne = pd.DataFrame(
+            data=tsne_features, columns=["t-SNE (x)", "t-SNE (y)", "t-SNE (z)"]
+        )
+        df_tsne = pd.concat([df_tsne, data_obj], axis=1)
+        # plot 3D
+        fig = plt.figure(figsize=figsize)
+        _ = plt.title("t-Distributed Stochastic Neighbor Embedding (t-SNE)")
+        ax = fig.add_subplot(111, projection="3d")
+        if hue:
+            i = ax.scatter(
+                df_tsne["t-SNE (x)"],
+                df_tsne["t-SNE (y)"],
+                df_tsne["t-SNE (z)"],
+                c=df_tsne[hue],
+                cmap="tab10",
+                s=60,
+                alpha=0.8,
+            )
+            fig.colorbar(i)
+        else:
+            ax.scatter(
+                df_tsne["t-SNE (x)"],
+                df_tsne["t-SNE (y)"],
+                df_tsne["t-SNE (z)"],
+                c="#75bbfd",
+                s=60,
+                alpha=0.8,
+            )
+        _ = ax.view_init(30, 185)
+
+    else:
+        raise ValueError("n_components can be either 2 or 3")
+
+    # tag each point
+    if tag is not None and n_components == 2:
+        for x, y, tag in zip(df_tsne["t-SNE (x)"], df_tsne["t-SNE (y)"], df_tsne[tag]):
+            plt.annotate(tag, (x, y), fontsize=label_fontsize, alpha=0.75)
+
+    return fig, ax