James Twose
1.6 MiB
1.6 MiB
In [1]:
import pandas as pd import numpy as np from glob import glob import matplotlib.pyplot as plt import seaborn as sns from jmspack.utils import JmsColors
In [2]:
from tslearn.clustering import TimeSeriesKMeans, KernelKMeans
/opt/miniconda3/envs/ds_env/lib/python3.10/site-packages/tslearn/bases/bases.py:15: UserWarning: h5py not installed, hdf5 features will not be supported. Install h5py to use hdf5 features: http://docs.h5py.org/ warn(h5py_msg)
In [3]:
if "jms_style_sheet" in plt.style.available: _ = plt.style.use("jms_style_sheet")
In [4]:
df = pd.read_csv(glob("data/*")[0], index_col=0).set_index("id").dropna(thresh=1, axis=0)
In [5]:
df.info(max_cols=250)
<class 'pandas.core.frame.DataFrame'> Float64Index: 26584 entries, 1.0 to 26584.0 Data columns (total 207 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 fsd_start 26584 non-null object 1 fsd_end 26584 non-null object 2 fsd_round 26584 non-null float64 3 demographic_gender 26584 non-null float64 4 demographic_age 26584 non-null float64 5 demographic_region 26459 non-null float64 6 attitude_trust_others_generally 13913 non-null float64 7 attitude_officials_preparedness 20254 non-null float64 8 attitude_worry_covid_and_its_effects 26475 non-null float64 9 worry_health_own_illness 23249 non-null float64 10 worry_health_closeone_illness 23249 non-null float64 11 worry_health_contagion_asymptomatic 23249 non-null float64 12 worry_health_healthcare_system 23249 non-null float64 13 worry_health_parents_protective_behaviours 23249 non-null float64 14 worry_health_noncovid_treatment_availability_self 23249 non-null float64 15 worry_health_noncovid_treatment_availability_closeones 23249 non-null float64 16 worry_health_mental_wellbeing_own 23249 non-null float64 17 worry_health_alcohol_own 23249 non-null float64 18 worry_health_alcohol_closeones 23249 non-null float64 19 worry_health_domestic_violence 23249 non-null float64 20 worry_health_mental_wellbeing_children 23249 non-null float64 21 worry_health_other 23249 non-null float64 22 worry_health_none 23249 non-null float64 23 worry_health_unable_to_say 23249 non-null float64 24 worry_livelihood_sustenance_own 23249 non-null float64 25 worry_livelihood_sustenance_closeone 23249 non-null float64 26 worry_livelihood_unemployment 23249 non-null float64 27 worry_livelihood_layoff 23249 non-null float64 28 worry_livelihood_recession 23249 non-null float64 29 worry_livelihood_uncertainty_length 23249 non-null float64 30 worry_livelihood_increased_restrictions 23249 non-null float64 31 worry_livelihood_children_school 23249 non-null float64 32 worry_livelihood_work_and_childcare 23249 non-null float64 33 worry_livelihood_other 23249 non-null float64 34 worry_livelihood_none 23249 non-null float64 35 worry_livelihood_unable_to_say 23249 non-null float64 36 attitude_confidence_future 26402 non-null float64 37 worry_livelihood_sustainance_nextmonth 26392 non-null float64 38 emotion_stressed 26476 non-null float64 39 attitude_finnish_mood 24863 non-null float64 40 knowledge_satisfaction_effects_of_covid 22870 non-null float64 41 trust_institution_government 14849 non-null float64 42 trust_institution_municipality 14322 non-null float64 43 trust_institution_parliament 14720 non-null float64 44 trust_institution_tribunals 14288 non-null float64 45 trust_institution_parties 14234 non-null float64 46 trust_institution_police 14908 non-null float64 47 trust_institution_healthcare_system 14966 non-null float64 48 trust_institution_education_system 14612 non-null float64 49 trust_institution_public_servants 14068 non-null float64 50 trust_institution_media 14831 non-null float64 51 trust_institution_banks 11432 non-null float64 52 trust_institution_large_corporations 10878 non-null float64 53 attitude_communications_equality 24904 non-null float64 54 attitude_communications_trustworthiness 25878 non-null float64 55 attitude_communications_accuracy 25285 non-null float64 56 attitude_communications_clarity 26074 non-null float64 57 attitude_communications_speed 25905 non-null float64 58 attitude_communications_openness 25719 non-null float64 59 behaviour_compliance_self 20384 non-null float64 60 behaviour_compliance_others 19987 non-null float64 61 vaccination_intention 14099 non-null float64 62 vaccination_effectiveness 14258 non-null float64 63 demographic_education 26531 non-null float64 64 demographic_living_with 26387 non-null float64 65 demographic_underage_children 20286 non-null float64 66 demographic_income 26567 non-null float64 67 behaviour_distancing 26456 non-null float64 68 behaviour_handwashing 20428 non-null float64 69 behaviour_masks 26499 non-null float64 70 behaviour_facetouching 20326 non-null float64 71 behaviour_hand_desinfectant 20433 non-null float64 72 behaviour_avoid_meeting 26432 non-null float64 73 fsd_weight 26584 non-null float64 74 fsd_vnk 26584 non-null float64 75 attitude_justice 12428 non-null float64 76 trust_info_politicians 11287 non-null float64 77 trust_info_healthcare_experts 11377 non-null float64 78 trust_info_healthcare_employees 11167 non-null float64 79 trust_info_security_officials 10536 non-null float64 80 trust_info_other_officials 9940 non-null float64 81 trust_info_researchers 10982 non-null float64 82 trust_info_ngo 9943 non-null float64 83 trust_info_journalists 10968 non-null float64 84 trust_info_social_media_influencers 10013 non-null float64 85 trust_info_advocacy_groups 9534 non-null float64 86 attitude_democracy 3972 non-null float64 87 vaccination_children 1531 non-null float64 88 election_voted 1244 non-null float64 89 election_officials 932 non-null float64 90 election_healthy 932 non-null float64 91 election_corroded_trust_officials 1244 non-null float64 92 election_discussion_atmosphere 1244 non-null float64 93 election_voting_decision_covid 1210 non-null float64 94 vaccination_status 8786 non-null float64 95 spending_worry_covid 2390 non-null float64 96 spending_case_numbers 2381 non-null float64 97 spending_official_recommendations 2384 non-null float64 98 attitude_protection_necessity 11902 non-null float64 99 effectiveness_masks 5203 non-null float64 100 effectiveness_hometests 5203 non-null float64 101 effectiveness_ventilation 5203 non-null float64 102 effectiveness_quarantine 5203 non-null float64 103 effectiveness_avoid_meeting 5203 non-null float64 104 effectiveness_announce_test_result 5203 non-null float64 105 effectiveness_avoid_public_events 5203 non-null float64 106 effectiveness_covid_passport 5203 non-null float64 107 ease_masks 2459 non-null float64 108 ease_hometests 2459 non-null float64 109 ease_ventilation 2459 non-null float64 110 ease_quarantine 2459 non-null float64 111 ease_avoid_meeting 2459 non-null float64 112 ease_announce_test_result 2459 non-null float64 113 ease_avoid_public_events 2459 non-null float64 114 ease_covid_passport 2459 non-null float64 115 persistence_masks 5203 non-null float64 116 persistence_hometests 5203 non-null float64 117 persistence_ventilation 5203 non-null float64 118 persistence_quarantine 5203 non-null float64 119 persistence_avoid_meeting 5203 non-null float64 120 persistence_announce_test_result 5203 non-null float64 121 persistence_avoid_public_events 5203 non-null float64 122 persistence_covid_passport 5203 non-null float64 123 got_info_newspapers 1200 non-null float64 124 got_info_periodicals 1200 non-null float64 125 got_info_other_online_news_media 1200 non-null float64 126 got_info_tv 1200 non-null float64 127 got_info_radio 1200 non-null float64 128 got_info_international_press 1200 non-null float64 129 got_info_official_website 1200 non-null float64 130 got_info_official_social_media 1200 non-null float64 131 got_info_ngo 1200 non-null float64 132 got_info_other_social_media 1200 non-null float64 133 got_info_online_forums 1200 non-null float64 134 got_info_elsewhere_online 1200 non-null float64 135 got_info_work_or_education 1200 non-null float64 136 got_info_friends 1200 non-null float64 137 got_info_other 1200 non-null float64 138 got_info_nowhere 1200 non-null float64 139 got_info_can_not_say 1200 non-null float64 140 want_info_newspapers 1200 non-null float64 141 want_info_periodicals 1200 non-null float64 142 want_info_other_online_news_media 1200 non-null float64 143 want_info_tv 1200 non-null float64 144 want_info_radio 1200 non-null float64 145 want_info_international_press 1200 non-null float64 146 want_info_official_website 1200 non-null float64 147 want_info_official_social_media 1200 non-null float64 148 want_info_ngo 1200 non-null float64 149 want_info_other_social_media 1200 non-null float64 150 want_info_online_forums 1200 non-null float64 151 want_info_elsewhere_online 1200 non-null float64 152 want_info_work_or_education 1200 non-null float64 153 want_info_friends 1200 non-null float64 154 want_info_other 1200 non-null float64 155 want_info_nowhere 1200 non-null float64 156 want_info_can_not_say 1200 non-null float64 157 attitude_opening_up 1185 non-null float64 158 ukraine_worry_general 3315 non-null float64 159 ukraine_worry_finsafety 3318 non-null float64 160 ukraine_worry_finecon 3315 non-null float64 161 ukraine_worry_ownsafety 3306 non-null float64 162 ukraine_worry_ownmental 3311 non-null float64 163 ukraine_worry_ownecon 3290 non-null float64 164 ukraine_worry_warexpansion 3319 non-null float64 165 ukraine_officials_preparedness 3050 non-null float64 166 ukraine_accept_sanctions 3236 non-null float64 167 ukraine_accept_many_refugees 3266 non-null float64 168 crisis_connection_can_share 1131 non-null float64 169 crisis_connection_online_discussions 1131 non-null float64 170 crisis_connection_knows_mental_health_help 1131 non-null float64 171 crisis_action_donations 1131 non-null float64 172 crisis_action_refugee_accomodation 1131 non-null float64 173 crisis_action_gather_supplies 1131 non-null float64 174 crisis_action_other_volunteerwork 1131 non-null float64 175 crisis_infoneed_mental_health 1131 non-null float64 176 crisis_infoneed_preparation 1131 non-null float64 177 crisis_infoneed_howtohelp 1131 non-null float64 178 crisis_preparedness 3272 non-null float64 179 crisis_unprepared_time 1078 non-null float64 180 crisis_unprepared_expensive 1078 non-null float64 181 crisis_unprepared_not_thought 1078 non-null float64 182 crisis_unprepared_knowledge 1078 non-null float64 183 crisis_unprepared_futile 1078 non-null float64 184 crisis_unprepared_no_agency 1078 non-null float64 185 crisis_unprepared_trust_systems 1078 non-null float64 186 crisis_unprepared_no_opportunity 1078 non-null float64 187 crisis_unprepared_other 1078 non-null float64 188 crisis_unprepared_cannotsay 1078 non-null float64 189 crisis_volunteer_proactively 1131 non-null float64 190 crisis_volunteer_requested_by_official 1131 non-null float64 191 crisis_volunteer_requested_by_ngo 1131 non-null float64 192 crisis_volunteer_not_lacks_role 124 non-null float64 193 crisis_volunteer_not_prioritise_family 124 non-null float64 194 crisis_volunteer_not_prioritise_self 124 non-null float64 195 crisis_volunteer_not_lacks_skills 124 non-null float64 196 crisis_volunteer_not_senescense 124 non-null float64 197 crisis_volunteer_not_officials_duty 124 non-null float64 198 crisis_volunteer_not_lacks_interest 124 non-null float64 199 crisis_volunteer_not_other 124 non-null float64 200 crisis_volunteer_not_cannotsay 124 non-null float64 201 fsd_no 0 non-null float64 202 fsd_vr 0 non-null float64 203 fsd_id 0 non-null float64 204 demographic_age_factor 26584 non-null object 205 demographic_underage_children_factor 26316 non-null object 206 demographic_gender_factor 26489 non-null object dtypes: float64(202), object(5) memory usage: 42.2+ MB
In [6]:
display(df.sample(n=5)); df.shape
<style scoped="">
.dataframe tbody tr th:only-of-type {
vertical-align: middle;
}
.dataframe tbody tr th {
vertical-align: top;
}
.dataframe thead th {
text-align: right;
}
</style>
| fsd_start | fsd_end | fsd_round | demographic_gender | demographic_age | demographic_region | attitude_trust_others_generally | attitude_officials_preparedness | attitude_worry_covid_and_its_effects | worry_health_own_illness | ... | crisis_volunteer_not_officials_duty | crisis_volunteer_not_lacks_interest | crisis_volunteer_not_other | crisis_volunteer_not_cannotsay | fsd_no | fsd_vr | fsd_id | demographic_age_factor | demographic_underage_children_factor | demographic_gender_factor | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| id | |||||||||||||||||||||
| 8262.0 | 2021-05-19 | 2021-05-24 | 7.0 | 1.0 | 12.0 | 18.0 | 8.0 | 8.0 | 6.0 | 1.0 | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | 70+ | no_children_at_home | male |
| 25009.0 | 2022-04-06 | 2022-04-11 | 20.0 | 1.0 | 11.0 | 14.0 | NaN | NaN | 5.0 | NaN | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | 65-70 | no_children_at_home | male |
| 25775.0 | 2022-05-11 | 2022-05-16 | 21.0 | 1.0 | 8.0 | 7.0 | 9.0 | NaN | 9.0 | NaN | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | 50-54 | no_children_at_home | male |
| 520.0 | 2021-01-13 | 2021-01-18 | 1.0 | 2.0 | 6.0 | 9.0 | 8.0 | 8.0 | 9.0 | 0.0 | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | 40-44 | children_at_home | female |
| 8002.0 | 2021-04-28 | 2021-05-03 | 6.0 | 2.0 | 3.0 | 17.0 | NaN | 5.0 | 9.0 | 0.0 | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | 25-29 | no_children_at_home | female |
5 rows × 207 columns
Out[6]:
(26584, 207)
In [7]:
df.groupby("fsd_round").count()[["fsd_start"]].T
Out[7]:
<style scoped="">
.dataframe tbody tr th:only-of-type {
vertical-align: middle;
}
.dataframe tbody tr th {
vertical-align: top;
}
.dataframe thead th {
text-align: right;
}
</style>
| fsd_round | 1.0 | 2.0 | 3.0 | 4.0 | 5.0 | 6.0 | 7.0 | 8.0 | 9.0 | 10.0 | ... | 12.0 | 13.0 | 14.0 | 15.0 | 16.0 | 17.0 | 18.0 | 19.0 | 20.0 | 21.0 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| fsd_start | 1187 | 1343 | 1510 | 1465 | 1287 | 1292 | 1261 | 1285 | 1246 | 1288 | ... | 1192 | 1216 | 1154 | 1305 | 1200 | 1435 | 1309 | 1131 | 1056 | 1148 |
1 rows × 21 columns
In [8]:
_ = plt.figure(figsize=(8, 1)) _ = sns.heatmap(data=pd.crosstab(df["demographic_gender"], df["demographic_age"]), annot=True, fmt=".0f")
In [9]:
_ = plt.figure(figsize=(12, 6)) _ = sns.heatmap(data=pd.crosstab(df["fsd_round"], df["demographic_age"]), annot=True, fmt=".0f")
In [10]:
fs_demo_cols = sorted(df.filter(regex="fsd|demographic").columns.tolist()) np.array(fs_demo_cols)
Out[10]:
array(['demographic_age', 'demographic_age_factor',
'demographic_education', 'demographic_gender',
'demographic_gender_factor', 'demographic_income',
'demographic_living_with', 'demographic_region',
'demographic_underage_children',
'demographic_underage_children_factor', 'fsd_end', 'fsd_id',
'fsd_no', 'fsd_round', 'fsd_start', 'fsd_vnk', 'fsd_vr',
'fsd_weight'], dtype='<U36')
In [11]:
# features_list = df.filter(regex="^automaticity|attitude|^norms|^risk|^effective").columns.tolist() column_threshold_amount = int(df.shape[0]*0.8) features_list = df.drop(fs_demo_cols, axis=1).dropna(thresh=column_threshold_amount, axis=1).columns.tolist() len(features_list)
Out[11]:
42
In [12]:
X = df.loc[df["fsd_round"].isin([1, 2, 3]), features_list].dropna() X.shape
Out[12]:
(3423, 42)
In [13]:
# target="demographic_age_factor" target=fs_demo_cols[2] y = df.loc[X.index, target] y.value_counts()
Out[13]:
2.0 1328 3.0 888 4.0 843 1.0 311 5.0 46 Name: demographic_education, dtype: int64
In [17]:
amount_of_clusters = y.nunique() km = TimeSeriesKMeans(n_clusters=amount_of_clusters, metric="euclidean", max_iter=50, random_state=0) km.fit(X=X.values.reshape(X.shape[0], X.shape[1], 1)) pred_df = X.assign(**{"cluster": km.predict(X=X.values.reshape(X.shape[0], X.shape[1], 1)), # "cluster": pd.Series(km.predict(X)).astype("category"), target: y, # target: y.astype("category"), }) pred_df["cluster"].value_counts()
Out[17]:
1 1091 3 970 2 611 4 433 0 318 Name: cluster, dtype: int64
In [19]:
pred_df["cluster"].value_counts().sum()
Out[19]:
3423
In [20]:
def jitter(values,j): return values + np.random.normal(j,0.1,values.shape)
In [21]:
if amount_of_clusters == 3: chosen_palette=[JmsColors.BLUEGREEN, JmsColors.PURPLE, JmsColors.YELLOW] else: chosen_palette="viridis"
In [22]:
# _ = sns.pairplot(data=pred_df.drop(target, axis=1), hue="cluster") plot_df = pd.concat([jitter(pred_df.drop("cluster", axis=1), 3), pred_df[["cluster"]]], axis=1) _ = sns.scatterplot(data=plot_df, x="attitude_worry_covid_and_its_effects", y="behaviour_avoid_meeting", hue="cluster", palette=chosen_palette, alpha=0.5)
In [23]:
plot_df = pd.concat([jitter(pred_df.drop(target, axis=1), 3), pred_df[[target]]], axis=1) _ = sns.scatterplot(data=plot_df, x="attitude_worry_covid_and_its_effects", y="behaviour_avoid_meeting", hue=target, palette=chosen_palette, alpha=0.5)
In [24]:
_ = plt.figure(figsize=(20, 5)) _ = sns.lineplot(data=pred_df.reset_index(), x="id", y="attitude_worry_covid_and_its_effects", hue="cluster", palette=chosen_palette)
In [25]:
_ = plt.figure(figsize=(20, 5)) _ = sns.lineplot(data=pred_df.reset_index(), x="id", y="attitude_worry_covid_and_its_effects", hue=target, palette=chosen_palette)
In [26]:
gak_km = KernelKMeans(n_clusters=amount_of_clusters, kernel="gak", kernel_params={"sigma": "auto"}, n_init=20, verbose=True, random_state=42) y_pred = gak_km.fit_predict(X=X.values.reshape(X.shape[1], X.shape[0], 1)) sz = X.shape[1] plt.figure() for yi in range(3): plt.subplot(3, 1, 1 + yi) for xx in X[y_pred == yi]: plt.plot(xx.ravel(), "k-", alpha=.2) plt.xlim(0, sz) plt.ylim(-4, 4) plt.title("Cluster %d" % (yi + 1)) plt.tight_layout() plt.show()
[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 903 out of 903 | elapsed: 18.5min finished
Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster Init 1 Resumed because of empty cluster
--------------------------------------------------------------------------- ValueError Traceback (most recent call last) /Users/jamestwose/Coding/matti_jms_collabs/citizen_shield/citizen_shield_multi_wave_EDA.ipynb Cell 22' in <cell line: 12>() <a href='vscode-notebook-cell:/Users/jamestwose/Coding/matti_jms_collabs/citizen_shield/citizen_shield_multi_wave_EDA.ipynb#ch0000029?line=11'>12</a> for yi in range(3): <a href='vscode-notebook-cell:/Users/jamestwose/Coding/matti_jms_collabs/citizen_shield/citizen_shield_multi_wave_EDA.ipynb#ch0000029?line=12'>13</a> plt.subplot(3, 1, 1 + yi) ---> <a href='vscode-notebook-cell:/Users/jamestwose/Coding/matti_jms_collabs/citizen_shield/citizen_shield_multi_wave_EDA.ipynb#ch0000029?line=13'>14</a> for xx in X[y_pred == yi]: <a href='vscode-notebook-cell:/Users/jamestwose/Coding/matti_jms_collabs/citizen_shield/citizen_shield_multi_wave_EDA.ipynb#ch0000029?line=14'>15</a> plt.plot(xx.ravel(), "k-", alpha=.2) <a href='vscode-notebook-cell:/Users/jamestwose/Coding/matti_jms_collabs/citizen_shield/citizen_shield_multi_wave_EDA.ipynb#ch0000029?line=15'>16</a> plt.xlim(0, sz) File /opt/miniconda3/envs/ds_env/lib/python3.10/site-packages/pandas/core/frame.py:3496, in DataFrame.__getitem__(self, key) 3494 # Do we have a (boolean) 1d indexer? 3495 if com.is_bool_indexer(key): -> 3496 return self._getitem_bool_array(key) 3498 # We are left with two options: a single key, and a collection of keys, 3499 # We interpret tuples as collections only for non-MultiIndex 3500 is_single_key = isinstance(key, tuple) or not is_list_like(key) File /opt/miniconda3/envs/ds_env/lib/python3.10/site-packages/pandas/core/frame.py:3543, in DataFrame._getitem_bool_array(self, key) 3537 warnings.warn( 3538 "Boolean Series key will be reindexed to match DataFrame index.", 3539 UserWarning, 3540 stacklevel=find_stack_level(), 3541 ) 3542 elif len(key) != len(self.index): -> 3543 raise ValueError( 3544 f"Item wrong length {len(key)} instead of {len(self.index)}." 3545 ) 3547 # check_bool_indexer will throw exception if Series key cannot 3548 # be reindexed to match DataFrame rows 3549 key = check_bool_indexer(self.index, key) ValueError: Item wrong length 42 instead of 3423.
In [27]:
y_pred
Out[27]:
array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
In [ ]: