diff --git a/preparedness/main.ipynb b/preparedness/analysis.ipynb similarity index 100% rename from preparedness/main.ipynb rename to preparedness/analysis.ipynb diff --git a/preparedness/analysis.py b/preparedness/analysis.py new file mode 100644 index 0000000..a6e46d8 --- /dev/null +++ b/preparedness/analysis.py @@ -0,0 +1,239 @@ +# %% [markdown] +# # Visualize Most Prepared Users +# This workflow loads user embeddings, generates a prompt embedding, computes similarity, and visualizes the most prepared users. + +# %% +import pandas as pd +import numpy as np +from sklearn.metrics.pairwise import cosine_similarity +from sklearn.decomposition import PCA +from sklearn.manifold import TSNE +import plotly.express as px +import matplotlib.pyplot as plt +import seaborn as sns +import umap +import umap.plot +from utils import get_ollama_embedding + +# %% [markdown] +# ## Read in the file with user vectors +# Update the path/format as needed. +# %% +df = ( + pd.read_csv("./data/eurobarometer_preparedness_model_data_v3.csv") + .reset_index() + .assign( + **{ + "user_id": lambda x: x["index"].astype(str) + + "_" + + x["country_code_iso_3166"] + + "_" + + x["age_recoded_6_categories"].astype(str) + + "_" + + x["gender"].astype(str) + } + ) + # select only relevant countries for visualization + .loc[ + lambda x: x["country_code_iso_3166"].isin( + ["FI", "DE-E", "DE-W", "FR", "ES", "PT"] + ) + ] +) +df.shape + +# relabel None of the above/ Non binary/ do not recognize yourself in above categories/Prefer not to say to other +df["gender"] = df["gender"].replace({ + "None of the above/ Non binary/ do not recognize yourself in above categories/Prefer not to say": "Other", +}) + +# %% [markdown] +# ## Convert string embeddings to lists if needed +# %% +def parse_embedding(x): + if isinstance(x, str): + return [float(i) for i in x.strip("[]").split(",")] + return x + + +df["embedding"] = df["embedding"].apply(parse_embedding) + +# %% +# Select best user: +# disaster_measures_in_hh_* == 1, and how_many_days_meet_* == (4 | 5) +# Following relabeling in data_preparation_raw.py: +# disaster_measures_in_hh_* != "Not mentioned" +# how_many_days_meet_* == "More than 7 days" +disaster_measures_in_hh_columns = [ + col for col in df.columns if col.startswith("disaster_measures_in_hh_") +] +how_many_days_meet_columns = [ + col for col in df.columns if col.startswith("how_many_days_meet_") +] +best_users = df[ + (df[disaster_measures_in_hh_columns] != "Not mentioned").all(axis=1) + & (df[how_many_days_meet_columns] == "More than 7 days").all(axis=1) +] +print(f"Number of best users: {len(best_users)}") + +best_users.head(5)[["user_id", "user_text"]].to_dict(orient="records") + +# %% +best_users.columns.tolist() + +# %% [markdown] +# ## Write your prompt and generate its embedding +# %% +# prompt = "The user is highly prepared for disasters, with emergency supplies and a clear plan." +prompt = best_users.head(1)["user_text"].values[ + 0 +] # Example: use the first user's text as the prompt +prompt_embedding = get_ollama_embedding(prompt) + +# %% [markdown] +# ## Compute similarity between each user and the prompt +# %% +user_embeddings = np.vstack(df["embedding"].values) +prompt_vec = np.array(prompt_embedding).reshape(1, -1) +similarities = cosine_similarity(user_embeddings, prompt_vec).flatten() +df["similarity"] = similarities + +# %% [markdown] +# ## Visualize the users by similarity +# %% +_ = plt.figure(figsize=(5, 7)) +_ = sns.violinplot( + x="similarity", y="country_code_iso_3166", hue="country_code_iso_3166", data=df +) +_ = sns.stripplot( + x="similarity", + y="country_code_iso_3166", + data=df, + hue="country_code_iso_3166", + alpha=0.8, + jitter=True, + linewidth=0.5, + edgecolor="white", +) +_ = plt.title("Distribution of User Similarities to Preparedness Prompt") +_ = plt.xlabel("Cosine Similarity") + +# %% +# np.savetxt("./data/embeddings.tsv", np.vstack(df["embedding"].values), delimiter="\t") + +# # %% +# df[["user_id"]].to_csv( +# "./data/metadata.tsv", sep="\t", index=False +# ) + +# %% +# Reduce embeddings to 3D with PCA +pca = PCA(n_components=3) +embeddings_3d = pca.fit_transform(np.vstack(df["embedding"].values)) + +# Add PCA components to dataframe +df["pca1"] = embeddings_3d[:, 0] +df["pca2"] = embeddings_3d[:, 1] +df["pca3"] = embeddings_3d[:, 2] + +# %% +# Interactive 3D scatter plot +fig = px.scatter_3d( + df, + x="pca1", + y="pca2", + z="pca3", + color="age_recoded_6_categories", + hover_data=["user_id", "gender", "age_recoded_6_categories"], + title="User Embeddings (PCA 3D)" +) +fig.write_html("./output/user_embeddings_pca_3d.html") +# %% +# Reduce embeddings to 3D with t-SNE +tsne = TSNE(n_components=3, random_state=42, perplexity=30) +embeddings_3d = tsne.fit_transform(np.vstack(df["embedding"].values)) + +# Add t-SNE components to dataframe +df["tsne1"] = embeddings_3d[:, 0] +df["tsne2"] = embeddings_3d[:, 1] +df["tsne3"] = embeddings_3d[:, 2] + +# Interactive 3D scatter plot +fig = px.scatter_3d( + df, + x="tsne1", + y="tsne2", + z="tsne3", + color="age_recoded_6_categories", + hover_data=["user_id", "gender", "country_code_iso_3166"], + title="User Embeddings (t-SNE 3D)" +) +fig.write_html("./output/user_embeddings_tsne_3d.html") + +# %% +# example comparison based on t-sne projection +index_1 = 9561 +index_2 = 5096 +df.loc[[index_1, index_2]][["user_id", "user_text", "similarity"]].to_dict(orient="records") +# %% +row1 = df.loc[index_1].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"]) +row2 = df.loc[index_2].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"]) +# column_list = df.columns.difference(["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"]) +column_list = disaster_measures_in_hh_columns + how_many_days_meet_columns + +diffs = {} +for col in column_list: + val1 = row1[col] + val2 = row2[col] + if pd.isnull(val1) and pd.isnull(val2): + continue + if val1 != val2: + diffs[col] = (val1, val2) + +# Print or display the differing columns and their values +for col, (v1, v2) in diffs.items(): + print(f"{col}: {v1} | {v2}") +# %% +# Reduce embeddings to 3D with UMAP +umap_3d = umap.UMAP(n_components=3, random_state=42, metric="cosine").fit_transform(np.vstack(df["embedding"].values)) + +# Add UMAP components to dataframe +df["umap1"] = umap_3d[:, 0] +df["umap2"] = umap_3d[:, 1] +df["umap3"] = umap_3d[:, 2] + +# Interactive 3D scatter plot +fig = px.scatter_3d( + df, + x="umap1", + y="umap2", + z="umap3", + color="country_code_iso_3166", + hover_data=["user_id", "gender", "country_code_iso_3166"], + title="User Embeddings (UMAP 3D)" +) +fig.write_html("./output/user_embeddings_umap_3d.html") + +# %% +# example comparison based on UMAP projection +index_1 = 21396 +index_2 = 10444 +df.loc[[index_1, index_2]][["user_id", "user_text", "similarity"]].to_dict(orient="records") +# %% +row1 = df.loc[index_1].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"]) +row2 = df.loc[index_2].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"]) +# column_list = df.columns.difference(["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"]) +column_list = disaster_measures_in_hh_columns + how_many_days_meet_columns +diffs = {} +for col in column_list: + val1 = row1[col] + val2 = row2[col] + if pd.isnull(val1) and pd.isnull(val2): + continue + if val1 != val2: + diffs[col] = (val1, val2) + +# Print or display the differing columns and their values +for col, (v1, v2) in diffs.items(): + print(f"{col}: {v1} | {v2}") +# %% \ No newline at end of file diff --git a/preparedness/api.py b/preparedness/api.py new file mode 100644 index 0000000..0be0748 --- /dev/null +++ b/preparedness/api.py @@ -0,0 +1,305 @@ +import re +from fastapi import FastAPI, Response +import json +from fastapi.middleware.cors import CORSMiddleware +import os +from sklearn.metrics.pairwise import cosine_similarity + +from schemas import PredictionRequest +from utils import get_ollama_embedding, parse_combined_text_to_dict + + +app = FastAPI( + title="preparedness-api", + version="0.0.1", + description="General API for the preparedness campaign.", + contact={ + "email": "contact@jamestwose.com", + }, + # lifespan=lifespan, +) + +# Whitelist: only these variables will be exposed to the client form renderer. +# Update this list when you want to add/remove fields shown in the UI. +WHITELIST = { + "risks_cntry_most_exposed_to_firstly", + "risks_pers_most_exposed_to_firstly", + "risks_pers_most_exposed_to_number_of_mentioned_risks", + "pot_info_sources_to_learn_about_disaster_risks_firstly", + "statements_disaster_risks_readseenheard_info_in_last_12m", + "statements_disaster_risks_feel_well_informed", + "statements_disaster_risks_trust_information_by_pub_auth_on_risks_where_you_live", + "statements_disaster_risks_easy_to_find_information_by_pub_auth_on_risks_where_you_live", + "statements_disaster_risks_know_where_to_find_info_when_travelling_to_oth_eu_cntry", + "disaster_measures_in_hh_emergency_supply_drinksfood", + "disaster_measures_in_hh_emergency_supply_water_cookinghygiene", + "disaster_measures_in_hh_flashlightcandles", + "disaster_measures_in_hh_batterypowered_radio", + "disaster_measures_in_hh_emergency_pharmacy", + "disaster_measures_in_hh_copies_imp_documentsstored_safely", + "disaster_measures_in_hh_emergency_grabbag", + "disaster_measures_in_hh_signed_up_for_alerts", + "disaster_measures_in_hh_participated_in_trainingexercise", + "disaster_measures_in_hh_informed_about_official_response_plan", + "disaster_measures_in_hh_agreed_with_friendsfamily_to_contact", + "disaster_measures_in_hh_discussed_common_prot_measures_in_neighbourhood", + "disaster_measures_in_hh_invested_in_prot_measures_in_home", + "how_many_days_meet_water_needs_if_water_services_disrupted", + "how_many_days_power_essent_appliances_if_elec_interrupted", + "how_many_days_cook_mealsheat_if_gas_disrupted", + "how_many_days_provide_food_if_transportation_disrupted", + "how_many_days_continued_treatment_if_medication_supply_disrupted", + "personal_disaster_preparedness_better_able_to_cope_by_prep", + "personal_disaster_preparedness_feel_well_prepared", + "personal_disaster_preparedness_no_timefin_resources_to_prep", + "personal_disaster_preparedness_easy_to_find_info_on_how_to_prep", + "personal_disaster_preparedness_need_more_info_to_prep", + "personal_disaster_preparedness_know_how_emerg_services_will_alert", + "personal_disaster_preparedness_know_what_to_do_in_event_of_disaster", + "personal_disaster_preparedness_employerschool_encourages_trainingprep", + "personal_disaster_preparedness_emerg_services_encourage_trainingprep", + "relying_on_in_first_days_of_disaster_familyfriends", + "relying_on_in_first_days_of_disaster_people_in_neighbourhood", + "relying_on_in_first_days_of_disaster_assocsnonprofit_orgs", + "relying_on_in_first_days_of_disaster_emerg_services", + "relying_on_in_first_days_of_disaster_local_authgvmt_services", + "relying_on_in_first_days_of_disaster_workemployerschooledu_institution", + "relying_on_in_first_days_of_disaster_private_sector_entities", + "trust_in_emerg_services_to_handle_disastersemerg_situations_properly", + "engaging_in_voluntary_work_for_emerg_responder_orgs", + "occupation_of_respondent", + "age_recoded_6_categories", + "size_of_community", + "direction_things_are_going_life_personally", + "political_discussion_local_matters", + "political_discussion_national_matters", + "internet_use_total", + "gender", + "age_education", + "standard_of_living_last_5yrs_in_light_of_crises", + "personal_living_conditions_in_one_years_time", + "standard_of_living_next_5yrs", + # region and education levels are included but will often be null in examples + "region_spain", + "region_austria", + "region_belgium", + "education_level_bachelor_or_equivalent", +} + + +def format_column_name(col: str) -> str: + """Format a column name to match the style used in the WHITELIST.""" + col = re.sub(r"[.\s]+", "_", col) + col = re.sub(r"[^0-9a-zA-Z_]", "", col) + col = col.lower() + col = re.sub(r"__+", "_", col) # Replace double (or more) underscores with single + col = col.strip("_") # Remove leading/trailing underscores + return col + + +origins = ["*"] + +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/") +async def root(): + template_path = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "preparedness/templates", + "user_survey.html", + ) + try: + with open(template_path, "r", encoding="utf-8") as f: + html_content = f.read() + return Response(content=html_content, media_type="text/html") + except Exception as e: + return Response( + content=f"Error loading template: {e}", + media_type="text/plain", + status_code=500, + ) + + +@app.post("/predict") +async def predict(request: PredictionRequest): + example_best_user = "risks_cntry_most_exposed_to_firstly: Terrorist attacks | risks_pers_most_exposed_to_firstly: Extreme weather events (violent storms, droughts, heatwaves, cold waves, etc.) | risks_pers_most_exposed_to_number_of_mentioned_risks: 2 mentions | pot_info_sources_to_learn_about_disaster_risks_firstly: National media | statements_disaster_risks_readseenheard_info_in_last_12m: Tend to agree | statements_disaster_risks_feel_well_informed: Tend to agree | statements_disaster_risks_trust_information_by_pub_auth_on_risks_where_you_live: Totally agree | statements_disaster_risks_easy_to_find_information_by_pub_auth_on_risks_where_you_live: Tend to agree | statements_disaster_risks_know_where_to_find_info_when_travelling_to_oth_eu_cntry: Totally agree | disaster_measures_in_hh_emergency_supply_drinksfood: Keep an emergency supply stock/pack of drinks, food | disaster_measures_in_hh_emergency_supply_water_cookinghygiene: Keep an emergency supply of water for cooking and hygiene | disaster_measures_in_hh_flashlightcandles: Have flashlight or candles accessible | disaster_measures_in_hh_batterypowered_radio: Have a battery-powered radio accessible | disaster_measures_in_hh_emergency_pharmacy: Keep a home pharmacy for emergencies | disaster_measures_in_hh_copies_imp_documentsstored_safely: Have made sure you have copies of your most important documents or have stored them safely | disaster_measures_in_hh_emergency_grabbag: Have prepared a grab-bag, in case you need to evacuate rapidly in an emergency | disaster_measures_in_hh_signed_up_for_alerts: Have signed up for alerts and warnings from emergency services or authorities | disaster_measures_in_hh_participated_in_trainingexercise: Have participated in a training or exercise, to learn how to react in an emergency | disaster_measures_in_hh_informed_about_official_response_plan: Got informed on the response plan your city, region or country has for a disaster or emergency (e.g. (...) | disaster_measures_in_hh_agreed_with_friendsfamily_to_contact: Agreed with family, friends on how to contact each other in case of an emergency | disaster_measures_in_hh_discussed_common_prot_measures_in_neighbourhood: Discussed common protective measures in your neighbourhood | disaster_measures_in_hh_invested_in_prot_measures_in_home: Have invested in protective measures in your home (e.g. flood-proofed the electricity installation, cleared (...) | how_many_days_meet_water_needs_if_water_services_disrupted: More than 7 days | how_many_days_power_essent_appliances_if_elec_interrupted: More than 7 days | how_many_days_cook_mealsheat_if_gas_disrupted: More than 7 days | how_many_days_provide_food_if_transportation_disrupted: More than 7 days | how_many_days_continued_treatment_if_medication_supply_disrupted: More than 7 days | personal_disaster_preparedness_better_able_to_cope_by_prep: Totally agree | personal_disaster_preparedness_feel_well_prepared: Tend to agree | personal_disaster_preparedness_no_timefin_resources_to_prep: Tend to disagree | personal_disaster_preparedness_easy_to_find_info_on_how_to_prep: Tend to agree | personal_disaster_preparedness_need_more_info_to_prep: Totally agree | personal_disaster_preparedness_know_how_emerg_services_will_alert: Tend to agree | personal_disaster_preparedness_know_what_to_do_in_event_of_disaster: Tend to agree | personal_disaster_preparedness_employerschool_encourages_trainingprep: Totally disagree | personal_disaster_preparedness_emerg_services_encourage_trainingprep: Tend to agree | relying_on_in_first_days_of_disaster_familyfriends: A great deal | relying_on_in_first_days_of_disaster_people_in_neighbourhood: A great deal | relying_on_in_first_days_of_disaster_assocsnonprofit_orgs: A great deal | relying_on_in_first_days_of_disaster_emerg_services: A great deal | relying_on_in_first_days_of_disaster_local_authgvmt_services: A great deal | relying_on_in_first_days_of_disaster_workemployerschooledu_institution: Not a lot | relying_on_in_first_days_of_disaster_private_sector_entities: Not a lot | trust_in_emerg_services_to_handle_disastersemerg_situations_properly: Tend to trust | engaging_in_voluntary_work_for_emerg_responder_orgs: No, you have never engaged in voluntary work and do not plan to do so | occupation_of_respondent: Skilled manual worker | age_recoded_6_categories: 35-44 | size_of_community: Towns/suburbs | direction_things_are_going_life_personally: Things are going in the right direction | political_discussion_local_matters: Never | political_discussion_national_matters: Never | internet_use_total: Everyday/almost everyday (at least once 1 in d62_1 to d62_4) | gender: Woman | age_education: 23.0 | standard_of_living_last_5yrs_in_light_of_crises: Your standard of living has not changed | personal_living_conditions_in_one_years_time: Worse | standard_of_living_next_5yrs: Your standard of living will not change | region_austria: nan | region_belgium: nan | region_bulgaria: nan | region_croatia: nan | region_cyprus: nan | region_czechia: nan | region_denmark: nan | region_germany: nan | region_estonia: nan | region_finland: nan | region_france: nan | region_greece: nan | region_hungary: nan | region_ireland: nan | region_italy: nan | region_latvia: nan | region_lithuania: nan | region_luxembourg: nan | region_malta: nan | region_netherlands: nan | region_poland: nan | region_portugal: nan | region_romania: nan | region_slovenia: nan | region_slovakia: nan | region_spain: ES61 - Andalucia | region_sweden: nan | education_level_preprimary_education_incl_no_education: Not mentioned | education_level_primary_education: Not mentioned | education_level_lower_secondary_education: Not mentioned | education_level_upper_secondary_education: Not mentioned | education_level_postsecondary_non_tertiary_incl_prevocationalvocational: Not mentioned | education_level_shortcycle_tertiary: Not mentioned | education_level_bachelor_or_equivalent: Bachelor or equivalent | education_level_master_or_equivalent: Not mentioned | education_level_doctoral_or_equivalent: Not mentioned" + # parse combined example and request text into dicts for field-wise comparison + example_dict = parse_combined_text_to_dict(example_best_user) + + # Load metadata and build an inverted mapping from internal_label -> canonical variable id + meta_path = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "preparedness", + "data", + "za8841_meta.json", + ) + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) + var_labels = meta.get("column_names_to_labels", {}) + col_names = meta.get("column_names", []) + # Build mapping from internal_label -> canonical id using the var_labels and column list + internal_to_id = {} + for col in col_names: + human_label = var_labels.get(col) + if isinstance(human_label, str): + internal = format_column_name(human_label) + internal_to_id[internal] = col + except Exception: + internal_to_id = {} + + + request_parsed_internal = parse_combined_text_to_dict(request.text) + + # # Map internal_label keys back to canonical variable ids when possible + # request_dict = {} + # for ik, val in request_parsed_internal.items(): + # # prefer exact match by internal label + # varid = internal_to_id.get(ik) + # if not varid: + # # try case-insensitive normalized match + # for internal_label, cid in internal_to_id.items(): + # if internal_label.strip().lower() == ik.strip().lower(): + # varid = cid + # break + # # fall back to the original internal key if we couldn't map + # request_dict[varid or ik] = val + + # compute simple field diffs: keys present in either dict; value equal, different or missing + keys = sorted(set(list(example_dict.keys()) + list(request_parsed_internal.keys()))) + field_diffs = {} + for k in keys: + if "region" in k: + continue + a = example_dict.get(k) + b = request_parsed_internal.get(k) + if a == b: + # field_diffs[k] = {"status": "same", "example": a, "request": b} + continue + else: + field_diffs[k] = {"status": "different", "example": a, "request": b} + + example_best_user_embedding = get_ollama_embedding(example_best_user) + prompt_embedding = get_ollama_embedding(request.text) + similarity = float( + cosine_similarity([example_best_user_embedding], [prompt_embedding])[0][0] + ) + + return { + "similarity": similarity, + "example_parsed": example_dict, + # "request_parsed": request_dict, + "request_parsed_internal_keys": request_parsed_internal, + "field_diffs": field_diffs, + } + + +@app.get("/value_labels") +async def value_labels(): + """Return the variable-level value labels extracted from the za8841 metadata JSON. + + This endpoint returns the `variable_value_labels` section as a JSON object so the client + can use canonical, human-friendly labels for variables where available. + """ + meta_path = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "preparedness", + "data", + "za8841_meta.json", + ) + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) + variable_value_labels = meta.get("variable_value_labels", {}) + # filter to whitelist + filtered = {k: v for k, v in variable_value_labels.items() if k in WHITELIST} + return filtered + except Exception as e: + return Response( + content=json.dumps({"error": str(e)}), + media_type="application/json", + status_code=500, + ) + + +@app.get("/variable_map") +async def variable_map(): + """Return a mapping from human-readable variable label -> variable id. + + The source JSON contains a mapping of variable id -> human label (e.g. "qc5_3": "STATEMENTS ..."). + This endpoint inverts that mapping so the client can look up the variable id by visible label. + """ + meta_path = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "preparedness", + "data", + "za8841_meta.json", + ) + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) + var_labels = meta.get("column_names_to_labels", {}) + # filter to whitelist + filtered = {k: v for k, v in var_labels.items() if k in WHITELIST} + # invert: map human label -> variable id + inv = {v: k for k, v in filtered.items() if isinstance(v, str)} + return inv + except Exception as e: + return Response( + content=json.dumps({"error": str(e)}), + media_type="application/json", + status_code=500, + ) + + +@app.get("/variables") +async def variables(): + """Return an ordered list of variable metadata suitable for client-side form rendering. + + The response format is a list of objects: + [{"id": "qc5_3", "label": "STATEMENTS ...", "values": {"1.0": "Totally agree", ...}}, ...] + + This lets the client render each variable using canonical labels and codes. + """ + meta_path = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "preparedness", + "data", + "za8841_meta.json", + ) + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) + + col_names = meta.get("column_names", []) + # human-friendly labels live under `column_names_to_labels` in the metadata + var_labels = meta.get("column_names_to_labels", {}) + value_labels = meta.get("variable_value_labels", {}) + + vars_out = [] + for col in col_names: + if format_column_name(var_labels.get(col)) not in WHITELIST: + print(format_column_name(var_labels.get(col))) + continue + item = { + "id": col, + "internal_label": format_column_name(var_labels.get(col)), + "label": var_labels.get(col) or col, + "values": value_labels.get(col) or {}, + } + vars_out.append(item) + + return vars_out + except Exception as e: + return Response( + content=json.dumps({"error": str(e)}), + media_type="application/json", + status_code=500, + ) diff --git a/preparedness/main.py b/preparedness/main.py index 0d634f9..8cd326c 100644 --- a/preparedness/main.py +++ b/preparedness/main.py @@ -1,236 +1,20 @@ -# %% [markdown] -# # Visualize Most Prepared Users -# This workflow loads user embeddings, generates a prompt embedding, computes similarity, and visualizes the most prepared users. +import logging +import uvicorn +import os +from api import app -# %% -import pandas as pd -import numpy as np -from sklearn.metrics.pairwise import cosine_similarity -from sklearn.decomposition import PCA -from sklearn.manifold import TSNE -import plotly.express as px -import matplotlib.pyplot as plt -import seaborn as sns -import umap -import umap.plot -from utils import get_ollama_embedding - -# %% [markdown] -# ## Read in the file with user vectors -# Update the path/format as needed. -# %% -df = ( - pd.read_csv("./data/eurobarometer_preparedness_model_data_v3.csv") - .reset_index() - .assign( - **{ - "user_id": lambda x: x["index"].astype(str) - + "_" - + x["country_code_iso_3166"] - + "_" - + x["age_recoded_6_categories"].astype(str) - + "_" - + x["gender"].astype(str) - } - ) - # select only relevant countries for visualization - .loc[ - lambda x: x["country_code_iso_3166"].isin( - ["FI", "DE-E", "DE-W", "FR", "ES", "PT"] - ) - ] -) -df.shape - -# relabel None of the above/ Non binary/ do not recognize yourself in above categories/Prefer not to say to other -df["gender"] = df["gender"].replace({ - "None of the above/ Non binary/ do not recognize yourself in above categories/Prefer not to say": "Other", -}) - -# %% [markdown] -# ## Convert string embeddings to lists if needed -# %% -def parse_embedding(x): - if isinstance(x, str): - return [float(i) for i in x.strip("[]").split(",")] - return x +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) -df["embedding"] = df["embedding"].apply(parse_embedding) +def run_api() -> None: + """Run the FastAPI application.""" + if "james" in os.environ.get("USER", ""): + logger.info("Running in James's environment") + uvicorn.run("api:app", host="0.0.0.0", port=8080, reload=True) + else: + uvicorn.run(app, host="0.0.0.0", port=8080) -# %% -# Select best user: -# disaster_measures_in_hh_* == 1, and how_many_days_meet_* == (4 | 5) -# Following relabeling in data_preparation_raw.py: -# disaster_measures_in_hh_* != "Not mentioned" -# how_many_days_meet_* == "More than 7 days" -disaster_measures_in_hh_columns = [ - col for col in df.columns if col.startswith("disaster_measures_in_hh_") -] -how_many_days_meet_columns = [ - col for col in df.columns if col.startswith("how_many_days_meet_") -] -best_users = df[ - (df[disaster_measures_in_hh_columns] != "Not mentioned").all(axis=1) - & (df[how_many_days_meet_columns] == "More than 7 days").all(axis=1) -] -print(f"Number of best users: {len(best_users)}") -best_users.head(5)[["user_id", "user_text"]].to_dict(orient="records") - -# %% [markdown] -# ## Write your prompt and generate its embedding -# %% -# prompt = "The user is highly prepared for disasters, with emergency supplies and a clear plan." -prompt = best_users.head(1)["user_text"].values[ - 0 -] # Example: use the first user's text as the prompt -prompt_embedding = get_ollama_embedding(prompt) - -# %% [markdown] -# ## Compute similarity between each user and the prompt -# %% -user_embeddings = np.vstack(df["embedding"].values) -prompt_vec = np.array(prompt_embedding).reshape(1, -1) -similarities = cosine_similarity(user_embeddings, prompt_vec).flatten() -df["similarity"] = similarities - -# %% [markdown] -# ## Visualize the users by similarity -# %% -_ = plt.figure(figsize=(5, 7)) -_ = sns.violinplot( - x="similarity", y="country_code_iso_3166", hue="country_code_iso_3166", data=df -) -_ = sns.stripplot( - x="similarity", - y="country_code_iso_3166", - data=df, - hue="country_code_iso_3166", - alpha=0.8, - jitter=True, - linewidth=0.5, - edgecolor="white", -) -_ = plt.title("Distribution of User Similarities to Preparedness Prompt") -_ = plt.xlabel("Cosine Similarity") - -# %% -# np.savetxt("./data/embeddings.tsv", np.vstack(df["embedding"].values), delimiter="\t") - -# # %% -# df[["user_id"]].to_csv( -# "./data/metadata.tsv", sep="\t", index=False -# ) - -# %% -# Reduce embeddings to 3D with PCA -pca = PCA(n_components=3) -embeddings_3d = pca.fit_transform(np.vstack(df["embedding"].values)) - -# Add PCA components to dataframe -df["pca1"] = embeddings_3d[:, 0] -df["pca2"] = embeddings_3d[:, 1] -df["pca3"] = embeddings_3d[:, 2] - -# %% -# Interactive 3D scatter plot -fig = px.scatter_3d( - df, - x="pca1", - y="pca2", - z="pca3", - color="age_recoded_6_categories", - hover_data=["user_id", "gender", "age_recoded_6_categories"], - title="User Embeddings (PCA 3D)" -) -fig.write_html("./output/user_embeddings_pca_3d.html") -# %% -# Reduce embeddings to 3D with t-SNE -tsne = TSNE(n_components=3, random_state=42, perplexity=30) -embeddings_3d = tsne.fit_transform(np.vstack(df["embedding"].values)) - -# Add t-SNE components to dataframe -df["tsne1"] = embeddings_3d[:, 0] -df["tsne2"] = embeddings_3d[:, 1] -df["tsne3"] = embeddings_3d[:, 2] - -# Interactive 3D scatter plot -fig = px.scatter_3d( - df, - x="tsne1", - y="tsne2", - z="tsne3", - color="age_recoded_6_categories", - hover_data=["user_id", "gender", "country_code_iso_3166"], - title="User Embeddings (t-SNE 3D)" -) -fig.write_html("./output/user_embeddings_tsne_3d.html") - -# %% -# example comparison based on t-sne projection -index_1 = 9561 -index_2 = 5096 -df.loc[[index_1, index_2]][["user_id", "user_text", "similarity"]].to_dict(orient="records") -# %% -row1 = df.loc[index_1].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"]) -row2 = df.loc[index_2].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"]) -# column_list = df.columns.difference(["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"]) -column_list = disaster_measures_in_hh_columns + how_many_days_meet_columns - -diffs = {} -for col in column_list: - val1 = row1[col] - val2 = row2[col] - if pd.isnull(val1) and pd.isnull(val2): - continue - if val1 != val2: - diffs[col] = (val1, val2) - -# Print or display the differing columns and their values -for col, (v1, v2) in diffs.items(): - print(f"{col}: {v1} | {v2}") -# %% -# Reduce embeddings to 3D with UMAP -umap_3d = umap.UMAP(n_components=3, random_state=42, metric="cosine").fit_transform(np.vstack(df["embedding"].values)) - -# Add UMAP components to dataframe -df["umap1"] = umap_3d[:, 0] -df["umap2"] = umap_3d[:, 1] -df["umap3"] = umap_3d[:, 2] - -# Interactive 3D scatter plot -fig = px.scatter_3d( - df, - x="umap1", - y="umap2", - z="umap3", - color="country_code_iso_3166", - hover_data=["user_id", "gender", "country_code_iso_3166"], - title="User Embeddings (UMAP 3D)" -) -fig.write_html("./output/user_embeddings_umap_3d.html") - -# %% -# example comparison based on UMAP projection -index_1 = 21396 -index_2 = 10444 -df.loc[[index_1, index_2]][["user_id", "user_text", "similarity"]].to_dict(orient="records") -# %% -row1 = df.loc[index_1].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"]) -row2 = df.loc[index_2].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"]) -# column_list = df.columns.difference(["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"]) -column_list = disaster_measures_in_hh_columns + how_many_days_meet_columns -diffs = {} -for col in column_list: - val1 = row1[col] - val2 = row2[col] - if pd.isnull(val1) and pd.isnull(val2): - continue - if val1 != val2: - diffs[col] = (val1, val2) - -# Print or display the differing columns and their values -for col, (v1, v2) in diffs.items(): - print(f"{col}: {v1} | {v2}") -# %% \ No newline at end of file +if __name__ == "__main__": + run_api() \ No newline at end of file diff --git a/preparedness/requirements.txt b/preparedness/requirements.txt index f15c112..090b00d 100644 --- a/preparedness/requirements.txt +++ b/preparedness/requirements.txt @@ -7,4 +7,6 @@ seaborn pyreadstat plotly nbformat>=4.2.0 -umap-learn[plot] \ No newline at end of file +umap-learn[plot] +fastapi +uvicorn \ No newline at end of file diff --git a/preparedness/schemas.py b/preparedness/schemas.py new file mode 100644 index 0000000..dae29ec --- /dev/null +++ b/preparedness/schemas.py @@ -0,0 +1,4 @@ +from pydantic import BaseModel + +class PredictionRequest(BaseModel): + text: str \ No newline at end of file diff --git a/preparedness/templates/user_survey.html b/preparedness/templates/user_survey.html new file mode 100644 index 0000000..a6e0cbb --- /dev/null +++ b/preparedness/templates/user_survey.html @@ -0,0 +1,757 @@ + + + + + + + Preparedness Survey — Quick Form + + + + +
+

Preparedness survey — quick form

+

Fill the fields you have, then use Predict to send the form as JSON to the server's + /predict endpoint and see the model response below. +

+
+ +
+ + + +
+ + +
+ +

JSON preview

+ 
{ }
+
+ + +

If you need CSV export or different field typing, tell me and I'll add it.

+
+ + + \ No newline at end of file diff --git a/preparedness/utils.py b/preparedness/utils.py index 71a17f3..c8b243a 100644 --- a/preparedness/utils.py +++ b/preparedness/utils.py @@ -21,4 +21,34 @@ def get_ollama_embedding(text, model="nomic-embed-text"): } response = requests.post(url, json=payload) response.raise_for_status() - return response.json()["embedding"] \ No newline at end of file + return response.json()["embedding"] + +def parse_combined_text_to_dict(s: str) -> dict: + """Parse a string of the form 'key1: value1 | key2: value2' into a dict. + + Rules: + - Split on ' | ' to get key:value segments. + - For each segment, split on the first ':' to separate key and value. + - Strip whitespace. If a value is 'nan' (case-insensitive) or empty, use None. + - Return a dict mapping keys to values or None. + """ + if not s: + return {} + result = {} + parts = [p.strip() for p in s.split("|")] + for part in parts: + if not part: + continue + # split on the first colon + if ':' in part: + k, v = part.split(':', 1) + k = k.strip() + v = v.strip() + if v.lower() == 'nan' or v == '': + result[k] = None + else: + result[k] = v + else: + # fallback: store whole segment under a numeric key + result.setdefault('_extra', []).append(part) + return result \ No newline at end of file