itsamejms
189 KiB
189 KiB
Visualize Most Prepared Users¶
This workflow loads user embeddings, generates a prompt embedding, computes similarity, and visualizes the most prepared users.
In [ ]:
import pandas as pd import numpy as np from sklearn.metrics.pairwise import cosine_similarity from sklearn.decomposition import PCA from sklearn.manifold import TSNE import plotly.express as px import matplotlib.pyplot as plt import seaborn as sns import umap import umap.plot from utils import get_ollama_embedding
/Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html from .autonotebook import tqdm as notebook_tqdm /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/numba/np/ufunc/dufunc.py:344: NumbaWarning: Compilation requested for previously compiled argument types ((uint32,)). This has no effect and perhaps indicates a bug in the calling code (compiling a ufunc more than once for the same signature warnings.warn(msg, errors.NumbaWarning) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/numba/np/ufunc/dufunc.py:344: NumbaWarning: Compilation requested for previously compiled argument types ((uint32,)). This has no effect and perhaps indicates a bug in the calling code (compiling a ufunc more than once for the same signature warnings.warn(msg, errors.NumbaWarning) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/numba/np/ufunc/dufunc.py:344: NumbaWarning: Compilation requested for previously compiled argument types ((uint32,)). This has no effect and perhaps indicates a bug in the calling code (compiling a ufunc more than once for the same signature warnings.warn(msg, errors.NumbaWarning)
Read in the file with user vectors¶
Update the path/format as needed.
In [ ]:
df = ( pd.read_csv("./data/eurobarometer_preparedness_model_data_v3.csv") .reset_index() .assign( **{ "user_id": lambda x: x["index"].astype(str) + "_" + x["country_code_iso_3166"] + "_" + x["age_recoded_6_categories"].astype(str) + "_" + x["gender"].astype(str) } ) # select only relevant countries for visualization .loc[ lambda x: x["country_code_iso_3166"].isin( ["FI", "DE-E", "DE-W", "FR", "ES", "PT"] ) ] ) df.shape # relabel None of the above/ Non binary/ do not recognize yourself in above categories/Prefer not to say to other df["gender"] = df["gender"].replace({ "None of the above/ Non binary/ do not recognize yourself in above categories/Prefer not to say": "Other", })
<ipython-input-2-7166d4ff9397>:3: DtypeWarning: Columns (58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84) have mixed types. Specify dtype option on import or set low_memory=False.
pd.read_csv("./data/eurobarometer_preparedness_model_data_v3.csv")
Convert string embeddings to lists if needed¶
In [ ]:
def parse_embedding(x): if isinstance(x, str): return [float(i) for i in x.strip("[]").split(",")] return x df["embedding"] = df["embedding"].apply(parse_embedding)
In [ ]:
# Select best user: # disaster_measures_in_hh_* == 1, and how_many_days_meet_* == (4 | 5) # Following relabeling in data_preparation_raw.py: # disaster_measures_in_hh_* != "Not mentioned" # how_many_days_meet_* == "More than 7 days" disaster_measures_in_hh_columns = [ col for col in df.columns if col.startswith("disaster_measures_in_hh_") ] how_many_days_meet_columns = [ col for col in df.columns if col.startswith("how_many_days_meet_") ] best_users = df[ (df[disaster_measures_in_hh_columns] != "Not mentioned").all(axis=1) & (df[how_many_days_meet_columns] == "More than 7 days").all(axis=1) ] print(f"Number of best users: {len(best_users)}") best_users.head(5)[["user_id", "user_text"]].to_dict(orient="records")
Number of best users: 2
Out[ ]:
[{'user_id': '7977_ES_35-44_Woman',
'user_text': 'risks_cntry_most_exposed_to_firstly: Terrorist attacks | risks_pers_most_exposed_to_firstly: Extreme weather events (violent storms, droughts, heatwaves, cold waves, etc.) | risks_pers_most_exposed_to_number_of_mentioned_risks: 2 mentions | pot_info_sources_to_learn_about_disaster_risks_firstly: National media | statements_disaster_risks_readseenheard_info_in_last_12m: Tend to agree | statements_disaster_risks_feel_well_informed: Tend to agree | statements_disaster_risks_trust_information_by_pub_auth_on_risks_where_you_live: Totally agree | statements_disaster_risks_easy_to_find_information_by_pub_auth_on_risks_where_you_live: Tend to agree | statements_disaster_risks_know_where_to_find_info_when_travelling_to_oth_eu_cntry: Totally agree | disaster_measures_in_hh_emergency_supply_drinksfood: Keep an emergency supply stock/pack of drinks, food | disaster_measures_in_hh_emergency_supply_water_cookinghygiene: Keep an emergency supply of water for cooking and hygiene | disaster_measures_in_hh_flashlightcandles: Have flashlight or candles accessible | disaster_measures_in_hh_batterypowered_radio: Have a battery-powered radio accessible | disaster_measures_in_hh_emergency_pharmacy: Keep a home pharmacy for emergencies | disaster_measures_in_hh_copies_imp_documentsstored_safely: Have made sure you have copies of your most important documents or have stored them safely | disaster_measures_in_hh_emergency_grabbag: Have prepared a grab-bag, in case you need to evacuate rapidly in an emergency | disaster_measures_in_hh_signed_up_for_alerts: Have signed up for alerts and warnings from emergency services or authorities | disaster_measures_in_hh_participated_in_trainingexercise: Have participated in a training or exercise, to learn how to react in an emergency | disaster_measures_in_hh_informed_about_official_response_plan: Got informed on the response plan your city, region or country has for a disaster or emergency (e.g. (...) | disaster_measures_in_hh_agreed_with_friendsfamily_to_contact: Agreed with family, friends on how to contact each other in case of an emergency | disaster_measures_in_hh_discussed_common_prot_measures_in_neighbourhood: Discussed common protective measures in your neighbourhood | disaster_measures_in_hh_invested_in_prot_measures_in_home: Have invested in protective measures in your home (e.g. flood-proofed the electricity installation, cleared (...) | how_many_days_meet_water_needs_if_water_services_disrupted: More than 7 days | how_many_days_power_essent_appliances_if_elec_interrupted: More than 7 days | how_many_days_cook_mealsheat_if_gas_disrupted: More than 7 days | how_many_days_provide_food_if_transportation_disrupted: More than 7 days | how_many_days_continued_treatment_if_medication_supply_disrupted: More than 7 days | personal_disaster_preparedness_better_able_to_cope_by_prep: Totally agree | personal_disaster_preparedness_feel_well_prepared: Tend to agree | personal_disaster_preparedness_no_timefin_resources_to_prep: Tend to disagree | personal_disaster_preparedness_easy_to_find_info_on_how_to_prep: Tend to agree | personal_disaster_preparedness_need_more_info_to_prep: Totally agree | personal_disaster_preparedness_know_how_emerg_services_will_alert: Tend to agree | personal_disaster_preparedness_know_what_to_do_in_event_of_disaster: Tend to agree | personal_disaster_preparedness_employerschool_encourages_trainingprep: Totally disagree | personal_disaster_preparedness_emerg_services_encourage_trainingprep: Tend to agree | relying_on_in_first_days_of_disaster_familyfriends: A great deal | relying_on_in_first_days_of_disaster_people_in_neighbourhood: A great deal | relying_on_in_first_days_of_disaster_assocsnonprofit_orgs: A great deal | relying_on_in_first_days_of_disaster_emerg_services: A great deal | relying_on_in_first_days_of_disaster_local_authgvmt_services: A great deal | relying_on_in_first_days_of_disaster_workemployerschooledu_institution: Not a lot | relying_on_in_first_days_of_disaster_private_sector_entities: Not a lot | trust_in_emerg_services_to_handle_disastersemerg_situations_properly: Tend to trust | engaging_in_voluntary_work_for_emerg_responder_orgs: No, you have never engaged in voluntary work and do not plan to do so | occupation_of_respondent: Skilled manual worker | age_recoded_6_categories: 35-44 | size_of_community: Towns/suburbs | direction_things_are_going_life_personally: Things are going in the right direction | political_discussion_local_matters: Never | political_discussion_national_matters: Never | internet_use_total: Everyday/almost everyday (at least once 1 in d62_1 to d62_4) | gender: Woman | age_education: 23.0 | standard_of_living_last_5yrs_in_light_of_crises: Your standard of living has not changed | personal_living_conditions_in_one_years_time: Worse | standard_of_living_next_5yrs: Your standard of living will not change | region_austria: nan | region_belgium: nan | region_bulgaria: nan | region_croatia: nan | region_cyprus: nan | region_czechia: nan | region_denmark: nan | region_germany: nan | region_estonia: nan | region_finland: nan | region_france: nan | region_greece: nan | region_hungary: nan | region_ireland: nan | region_italy: nan | region_latvia: nan | region_lithuania: nan | region_luxembourg: nan | region_malta: nan | region_netherlands: nan | region_poland: nan | region_portugal: nan | region_romania: nan | region_slovenia: nan | region_slovakia: nan | region_spain: ES61 - Andalucia | region_sweden: nan | education_level_preprimary_education_incl_no_education: Not mentioned | education_level_primary_education: Not mentioned | education_level_lower_secondary_education: Not mentioned | education_level_upper_secondary_education: Not mentioned | education_level_postsecondary_non_tertiary_incl_prevocationalvocational: Not mentioned | education_level_shortcycle_tertiary: Not mentioned | education_level_bachelor_or_equivalent: Bachelor or equivalent | education_level_master_or_equivalent: Not mentioned | education_level_doctoral_or_equivalent: Not mentioned'},
{'user_id': '9221_FI_55-64_Man',
'user_text': 'risks_cntry_most_exposed_to_firstly: Cybersecurity threats (e.g. cyberattacks, cybercrimes, etc.) | risks_pers_most_exposed_to_firstly: Extreme weather events (violent storms, droughts, heatwaves, cold waves, etc.) | risks_pers_most_exposed_to_number_of_mentioned_risks: 3 mentions | pot_info_sources_to_learn_about_disaster_risks_firstly: Civil society organisations, non-profit organisations | statements_disaster_risks_readseenheard_info_in_last_12m: Tend to agree | statements_disaster_risks_feel_well_informed: Totally agree | statements_disaster_risks_trust_information_by_pub_auth_on_risks_where_you_live: Totally agree | statements_disaster_risks_easy_to_find_information_by_pub_auth_on_risks_where_you_live: Totally agree | statements_disaster_risks_know_where_to_find_info_when_travelling_to_oth_eu_cntry: Totally agree | disaster_measures_in_hh_emergency_supply_drinksfood: Keep an emergency supply stock/pack of drinks, food | disaster_measures_in_hh_emergency_supply_water_cookinghygiene: Keep an emergency supply of water for cooking and hygiene | disaster_measures_in_hh_flashlightcandles: Have flashlight or candles accessible | disaster_measures_in_hh_batterypowered_radio: Have a battery-powered radio accessible | disaster_measures_in_hh_emergency_pharmacy: Keep a home pharmacy for emergencies | disaster_measures_in_hh_copies_imp_documentsstored_safely: Have made sure you have copies of your most important documents or have stored them safely | disaster_measures_in_hh_emergency_grabbag: Have prepared a grab-bag, in case you need to evacuate rapidly in an emergency | disaster_measures_in_hh_signed_up_for_alerts: Have signed up for alerts and warnings from emergency services or authorities | disaster_measures_in_hh_participated_in_trainingexercise: Have participated in a training or exercise, to learn how to react in an emergency | disaster_measures_in_hh_informed_about_official_response_plan: Got informed on the response plan your city, region or country has for a disaster or emergency (e.g. (...) | disaster_measures_in_hh_agreed_with_friendsfamily_to_contact: Agreed with family, friends on how to contact each other in case of an emergency | disaster_measures_in_hh_discussed_common_prot_measures_in_neighbourhood: Discussed common protective measures in your neighbourhood | disaster_measures_in_hh_invested_in_prot_measures_in_home: Have invested in protective measures in your home (e.g. flood-proofed the electricity installation, cleared (...) | how_many_days_meet_water_needs_if_water_services_disrupted: More than 7 days | how_many_days_power_essent_appliances_if_elec_interrupted: 1 day or less | how_many_days_cook_mealsheat_if_gas_disrupted: More than 7 days | how_many_days_provide_food_if_transportation_disrupted: More than 7 days | how_many_days_continued_treatment_if_medication_supply_disrupted: More than 7 days | personal_disaster_preparedness_better_able_to_cope_by_prep: Totally agree | personal_disaster_preparedness_feel_well_prepared: Totally agree | personal_disaster_preparedness_no_timefin_resources_to_prep: Totally disagree | personal_disaster_preparedness_easy_to_find_info_on_how_to_prep: Totally agree | personal_disaster_preparedness_need_more_info_to_prep: Tend to disagree | personal_disaster_preparedness_know_how_emerg_services_will_alert: Totally agree | personal_disaster_preparedness_know_what_to_do_in_event_of_disaster: Totally agree | personal_disaster_preparedness_employerschool_encourages_trainingprep: Totally agree | personal_disaster_preparedness_emerg_services_encourage_trainingprep: Totally agree | relying_on_in_first_days_of_disaster_familyfriends: A great deal | relying_on_in_first_days_of_disaster_people_in_neighbourhood: Somewhat | relying_on_in_first_days_of_disaster_assocsnonprofit_orgs: Somewhat | relying_on_in_first_days_of_disaster_emerg_services: A great deal | relying_on_in_first_days_of_disaster_local_authgvmt_services: A great deal | relying_on_in_first_days_of_disaster_workemployerschooledu_institution: A great deal | relying_on_in_first_days_of_disaster_private_sector_entities: A great deal | trust_in_emerg_services_to_handle_disastersemerg_situations_properly: Totally trust | engaging_in_voluntary_work_for_emerg_responder_orgs: Yes, you are already currently engaged in voluntary work | occupation_of_respondent: Unemployed, temporarily not working | age_recoded_6_categories: 55-64 | size_of_community: Rural | direction_things_are_going_life_personally: Things are going in the right direction | political_discussion_local_matters: Occasionally | political_discussion_national_matters: Occasionally | internet_use_total: Everyday/almost everyday (at least once 1 in d62_1 to d62_4) | gender: Man | age_education: 16.0 | standard_of_living_last_5yrs_in_light_of_crises: Your standard of living has not changed | personal_living_conditions_in_one_years_time: The same | standard_of_living_next_5yrs: Your standard of living will not change | region_austria: nan | region_belgium: nan | region_bulgaria: nan | region_croatia: nan | region_cyprus: nan | region_czechia: nan | region_denmark: nan | region_germany: nan | region_estonia: nan | region_finland: FI1D - Pohjois- ja Itae-Suomi | region_france: nan | region_greece: nan | region_hungary: nan | region_ireland: nan | region_italy: nan | region_latvia: nan | region_lithuania: nan | region_luxembourg: nan | region_malta: nan | region_netherlands: nan | region_poland: nan | region_portugal: nan | region_romania: nan | region_slovenia: nan | region_slovakia: nan | region_spain: nan | region_sweden: nan | education_level_preprimary_education_incl_no_education: Not mentioned | education_level_primary_education: Not mentioned | education_level_lower_secondary_education: Not mentioned | education_level_upper_secondary_education: Not mentioned | education_level_postsecondary_non_tertiary_incl_prevocationalvocational: Post-secondary non tertiary (including pre-vocational or vocational education) | education_level_shortcycle_tertiary: Not mentioned | education_level_bachelor_or_equivalent: Not mentioned | education_level_master_or_equivalent: Not mentioned | education_level_doctoral_or_equivalent: Not mentioned'}]
Write your prompt and generate its embedding¶
In [ ]:
# prompt = "The user is highly prepared for disasters, with emergency supplies and a clear plan." prompt = best_users.head(1)["user_text"].values[ 0 ] # Example: use the first user's text as the prompt prompt_embedding = get_ollama_embedding(prompt)
Compute similarity between each user and the prompt¶
In [ ]:
user_embeddings = np.vstack(df["embedding"].values) prompt_vec = np.array(prompt_embedding).reshape(1, -1) similarities = cosine_similarity(user_embeddings, prompt_vec).flatten() df["similarity"] = similarities
/Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:203: RuntimeWarning: divide by zero encountered in matmul ret = a @ b /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:203: RuntimeWarning: overflow encountered in matmul ret = a @ b /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:203: RuntimeWarning: invalid value encountered in matmul ret = a @ b
Visualize the users by similarity¶
In [ ]:
_ = plt.figure(figsize=(5, 7)) _ = sns.violinplot( x="similarity", y="country_code_iso_3166", hue="country_code_iso_3166", data=df ) _ = sns.stripplot( x="similarity", y="country_code_iso_3166", data=df, hue="country_code_iso_3166", alpha=0.8, jitter=True, linewidth=0.5, edgecolor="white", ) _ = plt.title("Distribution of User Similarities to Preparedness Prompt") _ = plt.xlabel("Cosine Similarity")
/Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: divide by zero encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: overflow encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: invalid value encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: divide by zero encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: overflow encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: invalid value encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: divide by zero encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: overflow encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: invalid value encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: divide by zero encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: overflow encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: invalid value encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: divide by zero encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: overflow encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: invalid value encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: divide by zero encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: overflow encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: invalid value encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: divide by zero encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: overflow encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: invalid value encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: divide by zero encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: overflow encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: invalid value encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: divide by zero encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: overflow encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: invalid value encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: divide by zero encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: overflow encountered in vecdot return np.vecdot(x1, x2, axis=axis) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/scipy/_lib/_util.py:1279: RuntimeWarning: invalid value encountered in vecdot return np.vecdot(x1, x2, axis=axis)
In [ ]:
# np.savetxt("./data/embeddings.tsv", np.vstack(df["embedding"].values), delimiter="\t") # # %% # df[["user_id"]].to_csv( # "./data/metadata.tsv", sep="\t", index=False # )
In [ ]:
# Reduce embeddings to 3D with PCA pca = PCA(n_components=3) embeddings_3d = pca.fit_transform(np.vstack(df["embedding"].values)) # Add PCA components to dataframe df["pca1"] = embeddings_3d[:, 0] df["pca2"] = embeddings_3d[:, 1] df["pca3"] = embeddings_3d[:, 2]
/Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:350: RuntimeWarning: divide by zero encountered in matmul Q, _ = normalizer(A @ Q) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:350: RuntimeWarning: overflow encountered in matmul Q, _ = normalizer(A @ Q) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:350: RuntimeWarning: invalid value encountered in matmul Q, _ = normalizer(A @ Q) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:351: RuntimeWarning: divide by zero encountered in matmul Q, _ = normalizer(A.T @ Q) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:351: RuntimeWarning: overflow encountered in matmul Q, _ = normalizer(A.T @ Q) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:351: RuntimeWarning: invalid value encountered in matmul Q, _ = normalizer(A.T @ Q) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:355: RuntimeWarning: divide by zero encountered in matmul Q, _ = qr_normalizer(A @ Q) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:355: RuntimeWarning: overflow encountered in matmul Q, _ = qr_normalizer(A @ Q) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:355: RuntimeWarning: invalid value encountered in matmul Q, _ = qr_normalizer(A @ Q) /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:577: RuntimeWarning: divide by zero encountered in matmul B = Q.T @ M /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:577: RuntimeWarning: overflow encountered in matmul B = Q.T @ M /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:577: RuntimeWarning: invalid value encountered in matmul B = Q.T @ M /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:590: RuntimeWarning: divide by zero encountered in matmul U = Q @ Uhat /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:590: RuntimeWarning: overflow encountered in matmul U = Q @ Uhat /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:590: RuntimeWarning: invalid value encountered in matmul U = Q @ Uhat
In [ ]:
# Interactive 3D scatter plot fig = px.scatter_3d( df, x="pca1", y="pca2", z="pca3", color="age_recoded_6_categories", hover_data=["user_id", "gender", "age_recoded_6_categories"], title="User Embeddings (PCA 3D)" ) fig.write_html("./output/user_embeddings_pca_3d.html")
In [ ]:
# Reduce embeddings to 3D with t-SNE tsne = TSNE(n_components=3, random_state=42, perplexity=30) embeddings_3d = tsne.fit_transform(np.vstack(df["embedding"].values)) # Add t-SNE components to dataframe df["tsne1"] = embeddings_3d[:, 0] df["tsne2"] = embeddings_3d[:, 1] df["tsne3"] = embeddings_3d[:, 2] # Interactive 3D scatter plot fig = px.scatter_3d( df, x="tsne1", y="tsne2", z="tsne3", color="age_recoded_6_categories", hover_data=["user_id", "gender", "country_code_iso_3166"], title="User Embeddings (t-SNE 3D)" ) fig.write_html("./output/user_embeddings_tsne_3d.html")
/Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:350: RuntimeWarning: divide by zero encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:350: RuntimeWarning: overflow encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:350: RuntimeWarning: invalid value encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:351: RuntimeWarning: divide by zero encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:351: RuntimeWarning: overflow encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:351: RuntimeWarning: invalid value encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:355: RuntimeWarning: divide by zero encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:355: RuntimeWarning: overflow encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:355: RuntimeWarning: invalid value encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:577: RuntimeWarning: divide by zero encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:577: RuntimeWarning: overflow encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:577: RuntimeWarning: invalid value encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:590: RuntimeWarning: divide by zero encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:590: RuntimeWarning: overflow encountered in matmul /Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/sklearn/utils/extmath.py:590: RuntimeWarning: invalid value encountered in matmul
In [ ]:
# example comparison based on t-sne projection index_1 = 9561 index_2 = 5096 df.loc[[index_1, index_2]][["user_id", "user_text", "similarity"]].to_dict(orient="records")
Out[ ]:
[{'user_id': '9561_FR_55-64_Man',
'user_text': "risks_cntry_most_exposed_to_firstly: Cybersecurity threats (e.g. cyberattacks, cybercrimes, etc.) | risks_pers_most_exposed_to_firstly: Nuclear accidents | risks_pers_most_exposed_to_number_of_mentioned_risks: 2 mentions | pot_info_sources_to_learn_about_disaster_risks_firstly: National media | statements_disaster_risks_readseenheard_info_in_last_12m: Tend to agree | statements_disaster_risks_feel_well_informed: Tend to agree | statements_disaster_risks_trust_information_by_pub_auth_on_risks_where_you_live: Tend to disagree | statements_disaster_risks_easy_to_find_information_by_pub_auth_on_risks_where_you_live: Tend to agree | statements_disaster_risks_know_where_to_find_info_when_travelling_to_oth_eu_cntry: Don't know (SPONTANEOUS) | disaster_measures_in_hh_emergency_supply_drinksfood: Not mentioned | disaster_measures_in_hh_emergency_supply_water_cookinghygiene: Not mentioned | disaster_measures_in_hh_flashlightcandles: Not mentioned | disaster_measures_in_hh_batterypowered_radio: Not mentioned | disaster_measures_in_hh_emergency_pharmacy: Not mentioned | disaster_measures_in_hh_copies_imp_documentsstored_safely: Not mentioned | disaster_measures_in_hh_emergency_grabbag: Not mentioned | disaster_measures_in_hh_signed_up_for_alerts: Not mentioned | disaster_measures_in_hh_participated_in_trainingexercise: Not mentioned | disaster_measures_in_hh_informed_about_official_response_plan: Not mentioned | disaster_measures_in_hh_agreed_with_friendsfamily_to_contact: Not mentioned | disaster_measures_in_hh_discussed_common_prot_measures_in_neighbourhood: Not mentioned | disaster_measures_in_hh_invested_in_prot_measures_in_home: Not mentioned | how_many_days_meet_water_needs_if_water_services_disrupted: 4 - 7 days | how_many_days_power_essent_appliances_if_elec_interrupted: 2 - 3 days | how_many_days_cook_mealsheat_if_gas_disrupted: 1 day or less | how_many_days_provide_food_if_transportation_disrupted: 2 - 3 days | how_many_days_continued_treatment_if_medication_supply_disrupted: 2 - 3 days | personal_disaster_preparedness_better_able_to_cope_by_prep: Tend to agree | personal_disaster_preparedness_feel_well_prepared: Don't know (SPONTANEOUS) | personal_disaster_preparedness_no_timefin_resources_to_prep: Tend to agree | personal_disaster_preparedness_easy_to_find_info_on_how_to_prep: Don't know (SPONTANEOUS) | personal_disaster_preparedness_need_more_info_to_prep: Totally agree | personal_disaster_preparedness_know_how_emerg_services_will_alert: Tend to disagree | personal_disaster_preparedness_know_what_to_do_in_event_of_disaster: Don't know (SPONTANEOUS) | personal_disaster_preparedness_employerschool_encourages_trainingprep: It depends on the type of disaster (SPONTANEOUS) | personal_disaster_preparedness_emerg_services_encourage_trainingprep: Don't know (SPONTANEOUS) | relying_on_in_first_days_of_disaster_familyfriends: Somewhat | relying_on_in_first_days_of_disaster_people_in_neighbourhood: Somewhat | relying_on_in_first_days_of_disaster_assocsnonprofit_orgs: Somewhat | relying_on_in_first_days_of_disaster_emerg_services: Somewhat | relying_on_in_first_days_of_disaster_local_authgvmt_services: Somewhat | relying_on_in_first_days_of_disaster_workemployerschooledu_institution: Somewhat | relying_on_in_first_days_of_disaster_private_sector_entities: Somewhat | trust_in_emerg_services_to_handle_disastersemerg_situations_properly: Tend to trust | engaging_in_voluntary_work_for_emerg_responder_orgs: No, you have never engaged in voluntary work and do not plan to do so | occupation_of_respondent: Unemployed, temporarily not working | age_recoded_6_categories: 55-64 | size_of_community: Towns/suburbs | direction_things_are_going_life_personally: Things are going in the wrong direction | political_discussion_local_matters: Occasionally | political_discussion_national_matters: Occasionally | internet_use_total: Everyday/almost everyday (at least once 1 in d62_1 to d62_4) | gender: Man | age_education: 16.0 | standard_of_living_last_5yrs_in_light_of_crises: Your standard of living has decreased | personal_living_conditions_in_one_years_time: Worse | standard_of_living_next_5yrs: Your standard of living will decrease | region_austria: nan | region_belgium: nan | region_bulgaria: nan | region_croatia: nan | region_cyprus: nan | region_czechia: nan | region_denmark: nan | region_germany: nan | region_estonia: nan | region_finland: nan | region_france: FRK2 Rhone-Alpes | region_greece: nan | region_hungary: nan | region_ireland: nan | region_italy: nan | region_latvia: nan | region_lithuania: nan | region_luxembourg: nan | region_malta: nan | region_netherlands: nan | region_poland: nan | region_portugal: nan | region_romania: nan | region_slovenia: nan | region_slovakia: nan | region_spain: nan | region_sweden: nan | education_level_preprimary_education_incl_no_education: Not mentioned | education_level_primary_education: Not mentioned | education_level_lower_secondary_education: Lower secondary education | education_level_upper_secondary_education: Not mentioned | education_level_postsecondary_non_tertiary_incl_prevocationalvocational: Not mentioned | education_level_shortcycle_tertiary: Not mentioned | education_level_bachelor_or_equivalent: Not mentioned | education_level_master_or_equivalent: Not mentioned | education_level_doctoral_or_equivalent: Not mentioned",
'similarity': 0.8321311374074787},
{'user_id': '5096_DE-W_65-74_Woman',
'user_text': 'risks_cntry_most_exposed_to_firstly: Terrorist attacks | risks_pers_most_exposed_to_firstly: Extreme weather events (violent storms, droughts, heatwaves, cold waves, etc.) | risks_pers_most_exposed_to_number_of_mentioned_risks: +5 mentions | pot_info_sources_to_learn_about_disaster_risks_firstly: Emergency management services (e.g. police, firefighters, civil protection) | statements_disaster_risks_readseenheard_info_in_last_12m: Tend to agree | statements_disaster_risks_feel_well_informed: Tend to agree | statements_disaster_risks_trust_information_by_pub_auth_on_risks_where_you_live: Totally agree | statements_disaster_risks_easy_to_find_information_by_pub_auth_on_risks_where_you_live: Tend to disagree | statements_disaster_risks_know_where_to_find_info_when_travelling_to_oth_eu_cntry: Totally disagree | disaster_measures_in_hh_emergency_supply_drinksfood: Keep an emergency supply stock/pack of drinks, food | disaster_measures_in_hh_emergency_supply_water_cookinghygiene: Keep an emergency supply of water for cooking and hygiene | disaster_measures_in_hh_flashlightcandles: Have flashlight or candles accessible | disaster_measures_in_hh_batterypowered_radio: Not mentioned | disaster_measures_in_hh_emergency_pharmacy: Not mentioned | disaster_measures_in_hh_copies_imp_documentsstored_safely: Have made sure you have copies of your most important documents or have stored them safely | disaster_measures_in_hh_emergency_grabbag: Not mentioned | disaster_measures_in_hh_signed_up_for_alerts: Not mentioned | disaster_measures_in_hh_participated_in_trainingexercise: Not mentioned | disaster_measures_in_hh_informed_about_official_response_plan: Not mentioned | disaster_measures_in_hh_agreed_with_friendsfamily_to_contact: Agreed with family, friends on how to contact each other in case of an emergency | disaster_measures_in_hh_discussed_common_prot_measures_in_neighbourhood: Not mentioned | disaster_measures_in_hh_invested_in_prot_measures_in_home: Have invested in protective measures in your home (e.g. flood-proofed the electricity installation, cleared (...) | how_many_days_meet_water_needs_if_water_services_disrupted: 4 - 7 days | how_many_days_power_essent_appliances_if_elec_interrupted: 1 day or less | how_many_days_cook_mealsheat_if_gas_disrupted: 2 - 3 days | how_many_days_provide_food_if_transportation_disrupted: 4 - 7 days | how_many_days_continued_treatment_if_medication_supply_disrupted: 4 - 7 days | personal_disaster_preparedness_better_able_to_cope_by_prep: Tend to agree | personal_disaster_preparedness_feel_well_prepared: Tend to agree | personal_disaster_preparedness_no_timefin_resources_to_prep: Totally disagree | personal_disaster_preparedness_easy_to_find_info_on_how_to_prep: It depends on the type of disaster (SPONTANEOUS) | personal_disaster_preparedness_need_more_info_to_prep: Tend to agree | personal_disaster_preparedness_know_how_emerg_services_will_alert: Totally agree | personal_disaster_preparedness_know_what_to_do_in_event_of_disaster: Totally agree | personal_disaster_preparedness_employerschool_encourages_trainingprep: Totally disagree | personal_disaster_preparedness_emerg_services_encourage_trainingprep: Tend to agree | relying_on_in_first_days_of_disaster_familyfriends: A great deal | relying_on_in_first_days_of_disaster_people_in_neighbourhood: Somewhat | relying_on_in_first_days_of_disaster_assocsnonprofit_orgs: Somewhat | relying_on_in_first_days_of_disaster_emerg_services: Somewhat | relying_on_in_first_days_of_disaster_local_authgvmt_services: Somewhat | relying_on_in_first_days_of_disaster_workemployerschooledu_institution: Not at all | relying_on_in_first_days_of_disaster_private_sector_entities: Not a lot | trust_in_emerg_services_to_handle_disastersemerg_situations_properly: Totally trust | engaging_in_voluntary_work_for_emerg_responder_orgs: No, you have never engaged in voluntary work and do not plan to do so | occupation_of_respondent: Retired, unable to work | age_recoded_6_categories: 65-74 | size_of_community: Rural | direction_things_are_going_life_personally: Things are going in the right direction | political_discussion_local_matters: Frequently | political_discussion_national_matters: Frequently | internet_use_total: Often/sometimes (at least once 2-5 in d62_1 to d62_4) | gender: Woman | age_education: 17.0 | standard_of_living_last_5yrs_in_light_of_crises: Your standard of living has not changed | personal_living_conditions_in_one_years_time: Better | standard_of_living_next_5yrs: Your standard of living will not change | region_austria: nan | region_belgium: nan | region_bulgaria: nan | region_croatia: nan | region_cyprus: nan | region_czechia: nan | region_denmark: nan | region_germany: DE9 - Niedersachsen | region_estonia: nan | region_finland: nan | region_france: nan | region_greece: nan | region_hungary: nan | region_ireland: nan | region_italy: nan | region_latvia: nan | region_lithuania: nan | region_luxembourg: nan | region_malta: nan | region_netherlands: nan | region_poland: nan | region_portugal: nan | region_romania: nan | region_slovenia: nan | region_slovakia: nan | region_spain: nan | region_sweden: nan | education_level_preprimary_education_incl_no_education: Not mentioned | education_level_primary_education: Not mentioned | education_level_lower_secondary_education: Lower secondary education | education_level_upper_secondary_education: Not mentioned | education_level_postsecondary_non_tertiary_incl_prevocationalvocational: Not mentioned | education_level_shortcycle_tertiary: Not mentioned | education_level_bachelor_or_equivalent: Not mentioned | education_level_master_or_equivalent: Not mentioned | education_level_doctoral_or_equivalent: Not mentioned',
'similarity': 0.8967985948973097}]
In [ ]:
row1 = df.loc[index_1].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"]) row2 = df.loc[index_2].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"]) # column_list = df.columns.difference(["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"]) column_list = disaster_measures_in_hh_columns + how_many_days_meet_columns diffs = {} for col in column_list: val1 = row1[col] val2 = row2[col] if pd.isnull(val1) and pd.isnull(val2): continue if val1 != val2: diffs[col] = (val1, val2) # Print or display the differing columns and their values for col, (v1, v2) in diffs.items(): print(f"{col}: {v1} | {v2}")
disaster_measures_in_hh_emergency_supply_drinksfood: Not mentioned | Keep an emergency supply stock/pack of drinks, food disaster_measures_in_hh_emergency_supply_water_cookinghygiene: Not mentioned | Keep an emergency supply of water for cooking and hygiene disaster_measures_in_hh_flashlightcandles: Not mentioned | Have flashlight or candles accessible disaster_measures_in_hh_copies_imp_documentsstored_safely: Not mentioned | Have made sure you have copies of your most important documents or have stored them safely disaster_measures_in_hh_agreed_with_friendsfamily_to_contact: Not mentioned | Agreed with family, friends on how to contact each other in case of an emergency disaster_measures_in_hh_invested_in_prot_measures_in_home: Not mentioned | Have invested in protective measures in your home (e.g. flood-proofed the electricity installation, cleared (...)
In [ ]:
# Reduce embeddings to 3D with UMAP umap_3d = umap.UMAP(n_components=3, random_state=42, metric="cosine").fit_transform(np.vstack(df["embedding"].values)) # Add UMAP components to dataframe df["umap1"] = umap_3d[:, 0] df["umap2"] = umap_3d[:, 1] df["umap3"] = umap_3d[:, 2] # Interactive 3D scatter plot fig = px.scatter_3d( df, x="umap1", y="umap2", z="umap3", color="country_code_iso_3166", hover_data=["user_id", "gender", "country_code_iso_3166"], title="User Embeddings (UMAP 3D)" ) fig.write_html("./output/user_embeddings_umap_3d.html")
/Users/jamestwose/Coding/matti_jms_collabs/.venv/lib/python3.12/site-packages/umap/umap_.py:1952: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism.
In [ ]:
# example comparison based on UMAP projection index_1 = 21396 index_2 = 10444 df.loc[[index_1, index_2]][["user_id", "user_text", "similarity"]].to_dict(orient="records")
Out[ ]:
[{'user_id': '21396_PT_45-54_Man',
'user_text': 'risks_cntry_most_exposed_to_firstly: Extreme weather events (violent storms, droughts, heatwaves, cold waves, etc.) | risks_pers_most_exposed_to_firstly: Extreme weather events (violent storms, droughts, heatwaves, cold waves, etc.) | risks_pers_most_exposed_to_number_of_mentioned_risks: 3 mentions | pot_info_sources_to_learn_about_disaster_risks_firstly: Emergency management services (e.g. police, firefighters, civil protection) | statements_disaster_risks_readseenheard_info_in_last_12m: Tend to disagree | statements_disaster_risks_feel_well_informed: Tend to disagree | statements_disaster_risks_trust_information_by_pub_auth_on_risks_where_you_live: Tend to agree | statements_disaster_risks_easy_to_find_information_by_pub_auth_on_risks_where_you_live: Tend to disagree | statements_disaster_risks_know_where_to_find_info_when_travelling_to_oth_eu_cntry: Tend to disagree | disaster_measures_in_hh_emergency_supply_drinksfood: Not mentioned | disaster_measures_in_hh_emergency_supply_water_cookinghygiene: Not mentioned | disaster_measures_in_hh_flashlightcandles: Have flashlight or candles accessible | disaster_measures_in_hh_batterypowered_radio: Not mentioned | disaster_measures_in_hh_emergency_pharmacy: Keep a home pharmacy for emergencies | disaster_measures_in_hh_copies_imp_documentsstored_safely: Not mentioned | disaster_measures_in_hh_emergency_grabbag: Not mentioned | disaster_measures_in_hh_signed_up_for_alerts: Not mentioned | disaster_measures_in_hh_participated_in_trainingexercise: Not mentioned | disaster_measures_in_hh_informed_about_official_response_plan: Not mentioned | disaster_measures_in_hh_agreed_with_friendsfamily_to_contact: Not mentioned | disaster_measures_in_hh_discussed_common_prot_measures_in_neighbourhood: Not mentioned | disaster_measures_in_hh_invested_in_prot_measures_in_home: Not mentioned | how_many_days_meet_water_needs_if_water_services_disrupted: 1 day or less | how_many_days_power_essent_appliances_if_elec_interrupted: 2 - 3 days | how_many_days_cook_mealsheat_if_gas_disrupted: 1 day or less | how_many_days_provide_food_if_transportation_disrupted: 4 - 7 days | how_many_days_continued_treatment_if_medication_supply_disrupted: More than 7 days | personal_disaster_preparedness_better_able_to_cope_by_prep: Tend to agree | personal_disaster_preparedness_feel_well_prepared: Tend to disagree | personal_disaster_preparedness_no_timefin_resources_to_prep: Tend to agree | personal_disaster_preparedness_easy_to_find_info_on_how_to_prep: Tend to disagree | personal_disaster_preparedness_need_more_info_to_prep: Tend to agree | personal_disaster_preparedness_know_how_emerg_services_will_alert: Tend to disagree | personal_disaster_preparedness_know_what_to_do_in_event_of_disaster: Tend to disagree | personal_disaster_preparedness_employerschool_encourages_trainingprep: Tend to disagree | personal_disaster_preparedness_emerg_services_encourage_trainingprep: Tend to disagree | relying_on_in_first_days_of_disaster_familyfriends: A great deal | relying_on_in_first_days_of_disaster_people_in_neighbourhood: Not a lot | relying_on_in_first_days_of_disaster_assocsnonprofit_orgs: Somewhat | relying_on_in_first_days_of_disaster_emerg_services: A great deal | relying_on_in_first_days_of_disaster_local_authgvmt_services: A great deal | relying_on_in_first_days_of_disaster_workemployerschooledu_institution: Not at all | relying_on_in_first_days_of_disaster_private_sector_entities: Not a lot | trust_in_emerg_services_to_handle_disastersemerg_situations_properly: Tend to trust | engaging_in_voluntary_work_for_emerg_responder_orgs: No, you have never engaged in voluntary work and do not plan to do so | occupation_of_respondent: Employed position, travelling | age_recoded_6_categories: 45-54 | size_of_community: Rural | direction_things_are_going_life_personally: Neither the one nor the other (SPONTANEOUS) | political_discussion_local_matters: Never | political_discussion_national_matters: Never | internet_use_total: Everyday/almost everyday (at least once 1 in d62_1 to d62_4) | gender: Man | age_education: 15.0 | standard_of_living_last_5yrs_in_light_of_crises: Your standard of living has decreased | personal_living_conditions_in_one_years_time: Worse | standard_of_living_next_5yrs: Your standard of living will decrease | region_austria: nan | region_belgium: nan | region_bulgaria: nan | region_croatia: nan | region_cyprus: nan | region_czechia: nan | region_denmark: nan | region_germany: nan | region_estonia: nan | region_finland: nan | region_france: nan | region_greece: nan | region_hungary: nan | region_ireland: nan | region_italy: nan | region_latvia: nan | region_lithuania: nan | region_luxembourg: nan | region_malta: nan | region_netherlands: nan | region_poland: nan | region_portugal: PT16 - Centro (PT) | region_romania: nan | region_slovenia: nan | region_slovakia: nan | region_spain: nan | region_sweden: nan | education_level_preprimary_education_incl_no_education: Not mentioned | education_level_primary_education: Not mentioned | education_level_lower_secondary_education: Lower secondary education | education_level_upper_secondary_education: Not mentioned | education_level_postsecondary_non_tertiary_incl_prevocationalvocational: Not mentioned | education_level_shortcycle_tertiary: Not mentioned | education_level_bachelor_or_equivalent: Not mentioned | education_level_master_or_equivalent: Not mentioned | education_level_doctoral_or_equivalent: Not mentioned',
'similarity': 0.8832531753098408},
{'user_id': '10444_FR_65-74_Woman',
'user_text': "risks_cntry_most_exposed_to_firstly: Extreme weather events (violent storms, droughts, heatwaves, cold waves, etc.) | risks_pers_most_exposed_to_firstly: Extreme weather events (violent storms, droughts, heatwaves, cold waves, etc.) | risks_pers_most_exposed_to_number_of_mentioned_risks: 4 mentions | pot_info_sources_to_learn_about_disaster_risks_firstly: Local or national authorities or agencies | statements_disaster_risks_readseenheard_info_in_last_12m: Tend to agree | statements_disaster_risks_feel_well_informed: Totally disagree | statements_disaster_risks_trust_information_by_pub_auth_on_risks_where_you_live: Tend to agree | statements_disaster_risks_easy_to_find_information_by_pub_auth_on_risks_where_you_live: Totally disagree | statements_disaster_risks_know_where_to_find_info_when_travelling_to_oth_eu_cntry: Totally disagree | disaster_measures_in_hh_emergency_supply_drinksfood: Not mentioned | disaster_measures_in_hh_emergency_supply_water_cookinghygiene: Not mentioned | disaster_measures_in_hh_flashlightcandles: Not mentioned | disaster_measures_in_hh_batterypowered_radio: Not mentioned | disaster_measures_in_hh_emergency_pharmacy: Not mentioned | disaster_measures_in_hh_copies_imp_documentsstored_safely: Not mentioned | disaster_measures_in_hh_emergency_grabbag: Not mentioned | disaster_measures_in_hh_signed_up_for_alerts: Not mentioned | disaster_measures_in_hh_participated_in_trainingexercise: Not mentioned | disaster_measures_in_hh_informed_about_official_response_plan: Not mentioned | disaster_measures_in_hh_agreed_with_friendsfamily_to_contact: Not mentioned | disaster_measures_in_hh_discussed_common_prot_measures_in_neighbourhood: Not mentioned | disaster_measures_in_hh_invested_in_prot_measures_in_home: Not mentioned | how_many_days_meet_water_needs_if_water_services_disrupted: 2 - 3 days | how_many_days_power_essent_appliances_if_elec_interrupted: 1 day or less | how_many_days_cook_mealsheat_if_gas_disrupted: 1 day or less | how_many_days_provide_food_if_transportation_disrupted: 2 - 3 days | how_many_days_continued_treatment_if_medication_supply_disrupted: Don't know (SPONTANEOUS) | personal_disaster_preparedness_better_able_to_cope_by_prep: Tend to agree | personal_disaster_preparedness_feel_well_prepared: Totally disagree | personal_disaster_preparedness_no_timefin_resources_to_prep: Don't know (SPONTANEOUS) | personal_disaster_preparedness_easy_to_find_info_on_how_to_prep: Totally disagree | personal_disaster_preparedness_need_more_info_to_prep: Tend to agree | personal_disaster_preparedness_know_how_emerg_services_will_alert: Totally disagree | personal_disaster_preparedness_know_what_to_do_in_event_of_disaster: Totally disagree | personal_disaster_preparedness_employerschool_encourages_trainingprep: Don't know (SPONTANEOUS) | personal_disaster_preparedness_emerg_services_encourage_trainingprep: Don't know (SPONTANEOUS) | relying_on_in_first_days_of_disaster_familyfriends: A great deal | relying_on_in_first_days_of_disaster_people_in_neighbourhood: A great deal | relying_on_in_first_days_of_disaster_assocsnonprofit_orgs: A great deal | relying_on_in_first_days_of_disaster_emerg_services: A great deal | relying_on_in_first_days_of_disaster_local_authgvmt_services: A great deal | relying_on_in_first_days_of_disaster_workemployerschooledu_institution: Don't know (SPONTANEOUS) | relying_on_in_first_days_of_disaster_private_sector_entities: Don't know (SPONTANEOUS) | trust_in_emerg_services_to_handle_disastersemerg_situations_properly: Tend to trust | engaging_in_voluntary_work_for_emerg_responder_orgs: No, but you have engaged in voluntary work in the past | occupation_of_respondent: Retired, unable to work | age_recoded_6_categories: 65-74 | size_of_community: Cities/urban | direction_things_are_going_life_personally: DK (SPONT.) | political_discussion_local_matters: Occasionally | political_discussion_national_matters: Occasionally | internet_use_total: Often/sometimes (at least once 2-5 in d62_1 to d62_4) | gender: Woman | age_education: 17.0 | standard_of_living_last_5yrs_in_light_of_crises: Your standard of living has decreased | personal_living_conditions_in_one_years_time: Worse | standard_of_living_next_5yrs: Your standard of living will decrease | region_austria: nan | region_belgium: nan | region_bulgaria: nan | region_croatia: nan | region_cyprus: nan | region_czechia: nan | region_denmark: nan | region_germany: nan | region_estonia: nan | region_finland: nan | region_france: FR10 Ile-de-France | region_greece: nan | region_hungary: nan | region_ireland: nan | region_italy: nan | region_latvia: nan | region_lithuania: nan | region_luxembourg: nan | region_malta: nan | region_netherlands: nan | region_poland: nan | region_portugal: nan | region_romania: nan | region_slovenia: nan | region_slovakia: nan | region_spain: nan | region_sweden: nan | education_level_preprimary_education_incl_no_education: Not mentioned | education_level_primary_education: Not mentioned | education_level_lower_secondary_education: Lower secondary education | education_level_upper_secondary_education: Not mentioned | education_level_postsecondary_non_tertiary_incl_prevocationalvocational: Not mentioned | education_level_shortcycle_tertiary: Not mentioned | education_level_bachelor_or_equivalent: Not mentioned | education_level_master_or_equivalent: Not mentioned | education_level_doctoral_or_equivalent: Not mentioned",
'similarity': 0.8832531753098408}]
In [ ]:
row1 = df.loc[index_1].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"]) row2 = df.loc[index_2].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"]) # column_list = df.columns.difference(["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"]) column_list = disaster_measures_in_hh_columns + how_many_days_meet_columns diffs = {} for col in column_list: val1 = row1[col] val2 = row2[col] if pd.isnull(val1) and pd.isnull(val2): continue if val1 != val2: diffs[col] = (val1, val2) # Print or display the differing columns and their values for col, (v1, v2) in diffs.items(): print(f"{col}: {v1} | {v2}")
disaster_measures_in_hh_flashlightcandles: Have flashlight or candles accessible | Not mentioned disaster_measures_in_hh_emergency_pharmacy: Keep a home pharmacy for emergencies | Not mentioned how_many_days_meet_water_needs_if_water_services_disrupted: 1 day or less | 2 - 3 days