Compare commits
18 Commits
ef9bfed85c
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
 itsamejms
|
95c3bd7338 | ||
|
itsamejms
|
3a12bccaff | ||
 jameshtwose
|
682788637c | ||
|
itsamejms
|
7f67f3d09f | ||
|
itsamejms
|
b2e412b3c5 | ||
|
itsamejms
|
a44833d054 | ||
|
itsamejms
|
a0b38fa37d | ||
|
itsamejms
|
fc0d58cd1b | ||
|
itsamejms
|
8a2e7fdb6e | ||
|
James Twose
|
a86232a039 | ||
|
James Twose
|
fd25bd5fe5 | ||
|
James Twose
|
0be9b3b6b5 | ||
|
jameshtwose
|
1945de9567 | ||
|
James Twose
|
733ad68cef | ||
|
James Twose
|
9af36c67d0 | ||
|
James Twose
|
d3d4571467 | ||
|
James
|
ed87bbdd38 | ||
|
James
|
f30b446110 |
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"projects": {
|
||||
"default": "itsamejms"
|
||||
}
|
||||
}
|
||||
+5
-1
@@ -135,5 +135,9 @@ data
|
||||
kp_determinants
|
||||
kp_determinants2
|
||||
.Rproj.user
|
||||
|
||||
*.Rproj
|
||||
.Rhistory
|
||||
.Rhistory
|
||||
scratch*
|
||||
# Firebase
|
||||
.firebase/
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
# matti_jms_collabs
|
||||
All the avenues of fun analyses that Matti Heino and James Twose have embarked on - predominantly using a Complexity Sciences approach.
|
||||
|
||||
Main results can be found here: https://jameshtwose.github.io/matti_jms_collabs/
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,381 @@
|
||||
---
|
||||
title: "Citizen Shield Networks"
|
||||
output: html_notebook
|
||||
---
|
||||
|
||||
# Outline
|
||||
Looking into different network approaches in the multi wave part of the citizen shield project
|
||||
|
||||
### Preliminaries
|
||||
#### Loading Libraries
|
||||
|
||||
Loading libraries used in this script
|
||||
|
||||
```{r}
|
||||
# Check to see if necessary packages are installed, and install if not
|
||||
packages <- c("psych", "pompom", "mlVAR", "bootnet", "psychonetrics", "GGMncv", "EstimateGroupNetwork", "relaimpo")
|
||||
if (length(setdiff(packages, rownames(installed.packages()))) > 0) {
|
||||
install.packages(setdiff(packages, rownames(installed.packages())))
|
||||
}
|
||||
# Load packages
|
||||
library(tidyr)
|
||||
library(lubridate)
|
||||
library(stringr)
|
||||
library(psych) #for data description
|
||||
library(plyr) #for data manipulation
|
||||
library(dplyr)
|
||||
library(ggplot2) #for data visualization
|
||||
# library(pompom) #for uSEM
|
||||
# library(mlVAR) #for mlVAR models
|
||||
library(bootnet)
|
||||
library(psychonetrics)
|
||||
library(EstimateGroupNetwork)
|
||||
```
|
||||
|
||||
### Loading Data
|
||||
|
||||
```{r}
|
||||
fs_demo_columns <- c('id', 'demographic_age', 'demographic_age_factor',
|
||||
'demographic_education', 'demographic_gender',
|
||||
'demographic_gender_factor', 'demographic_income',
|
||||
'demographic_living_with', 'demographic_region',
|
||||
# 'demographic_underage_children',
|
||||
'demographic_underage_children_factor', 'fsd_end',
|
||||
# 'fsd_id',
|
||||
# 'fsd_no',
|
||||
'fsd_round',
|
||||
'fsd_start', 'fsd_vnk',
|
||||
# 'fsd_vr',
|
||||
'fsd_weight')
|
||||
|
||||
```
|
||||
|
||||
```{r}
|
||||
df <- read.csv("../citizen_shield/data/kp_df_eng_ordinal.csv", header=TRUE, row.names = "X")
|
||||
remove_na <- function(DF, n=0) {
|
||||
DF[, colSums(is.na(DF)) <= n]
|
||||
}
|
||||
|
||||
df <- remove_na(df, n=4000)
|
||||
|
||||
```
|
||||
|
||||
```{r}
|
||||
# describeBy(df, group="user_id")
|
||||
# tmp <- df %>%
|
||||
# group_by(user_id) %>%
|
||||
# summarise_at(vars(feature_list), funs(sd(., na.rm=TRUE)))
|
||||
```
|
||||
|
||||
|
||||
|
||||
#### create a feature list based on the column names of the data frame
|
||||
|
||||
```{r}
|
||||
feature_list <- colnames(df %>% select(-fs_demo_columns))
|
||||
```
|
||||
|
||||
#### Show the head and description of the overall data frame
|
||||
|
||||
```{r}
|
||||
head(df)
|
||||
```
|
||||
|
||||
```{r}
|
||||
# describeBy(df, group = "user_id")
|
||||
# describe(df)
|
||||
```
|
||||
|
||||
#### Show the head and descrption of an example user
|
||||
|
||||
```{r}
|
||||
example_round <- 1
|
||||
data_indiv <- df[df$fsd_round == example_round, ]
|
||||
# head(data_indiv)
|
||||
# describe(data_indiv)
|
||||
```
|
||||
|
||||
|
||||
```{r}
|
||||
plot_df <- data_indiv %>%
|
||||
select(c(id, fsd_round), all_of(feature_list[1:20])) %>%
|
||||
gather(key = "variable", value = "value", -c(id, fsd_round))
|
||||
```
|
||||
|
||||
#### plotting intraindividual change
|
||||
|
||||
```{r}
|
||||
#plotting intraindividual change
|
||||
ggplot(data = plot_df,
|
||||
aes(x = id, y=value, group= fsd_round)) +
|
||||
#first variable
|
||||
geom_line(aes(color = variable)) +
|
||||
#plot layouts
|
||||
scale_x_continuous(name="Arbitrary Time") +
|
||||
scale_y_continuous(name="Raw Values") +
|
||||
theme_classic() +
|
||||
theme(axis.title=element_text(size=14),
|
||||
axis.text=element_text(size=14),
|
||||
plot.title=element_text(size=14, hjust=.5)) +
|
||||
ggtitle(example_round)
|
||||
```
|
||||
#### Normalize the data frame (consider min max maybe)
|
||||
|
||||
```{r}
|
||||
# standardize specific data columns (not the id or time variables in first 3 columns)
|
||||
data_indiv[feature_list] <- lapply(data_indiv[feature_list],
|
||||
function(x) c(scale(x, center=TRUE, scale=TRUE)))
|
||||
# describe(data_indiv)
|
||||
```
|
||||
|
||||
#### plotting normalized intraindividual change
|
||||
```{r}
|
||||
plot_df <- data_indiv %>%
|
||||
select(c(id, fsd_round), all_of(feature_list[1:20])) %>%
|
||||
gather(key = "variable", value = "value", -c(id, fsd_round))
|
||||
```
|
||||
|
||||
|
||||
```{r}
|
||||
#plotting intraindividual change
|
||||
ggplot(data = plot_df,
|
||||
aes(x = id, y=value, group= fsd_round)) +
|
||||
#first variable
|
||||
geom_line(aes(color = variable)) +
|
||||
#plot layouts
|
||||
scale_x_continuous(name="Arbitrary Time") +
|
||||
scale_y_continuous(name="Raw Values") +
|
||||
theme_classic() +
|
||||
theme(axis.title=element_text(size=14),
|
||||
axis.text=element_text(size=14),
|
||||
plot.title=element_text(size=14, hjust=.5)) +
|
||||
ggtitle(example_round)
|
||||
```
|
||||
|
||||
|
||||
Now we see that all the variables are in standardized form.
|
||||
|
||||
### Check the data
|
||||
|
||||
It is useful to check that there are data in all columns. If any one of the variables is all missing (or has no variance), the model cannot be fit. Missing data on a few observations within a column is ok.
|
||||
|
||||
```{r}
|
||||
# check column missing
|
||||
na_col <- 0
|
||||
for (col in 1:ncol(data_indiv)) {
|
||||
if (sum(is.na(data_indiv[,col])) == nrow(data_indiv)){
|
||||
na_col <- na_col + 1
|
||||
}
|
||||
}
|
||||
na_col
|
||||
```
|
||||
|
||||
All columns are reported.
|
||||
|
||||
### Describing the data.
|
||||
```{r}
|
||||
describe(df)
|
||||
```
|
||||
|
||||
### Not standardizing the data
|
||||
|
||||
For the following networks, the data are kept in their original form.
|
||||
|
||||
```{r}
|
||||
## Impute the feature_list missingness
|
||||
# imp.cart <- mice::mice(df[, feature_list], method="cart", printFlag = FALSE)
|
||||
# df[, feature_list] <- mice::complete(imp.cart)
|
||||
complete_df <- df[complete.cases(df[, c("id", "fsd_round", feature_list)]), c("id", "fsd_round", feature_list)]
|
||||
```
|
||||
|
||||
### ggmModSelect and EBICglasso networks
|
||||
There is no accounting for the hierarchical nature of the data here. Unregularized Gaussian Graphical Model ("ggmModSelect"; GGM) using the glasso algorithm and stepwise model selection. Gaussian Markov random field estimation using graphical LASSO and extended Bayesian information criterion ("EBICglasso") to select optimal regularization parameter.
|
||||
|
||||
```{r}
|
||||
net_modSelect <- estimateNetwork(complete_df[feature_list],
|
||||
default = "ggmModSelect",
|
||||
stepwise = FALSE,
|
||||
corMethod = "cor")
|
||||
```
|
||||
|
||||
```{r}
|
||||
net_thresh <- estimateNetwork(complete_df[feature_list],
|
||||
tuning = 0, # EBICglasso sets tuning to 0.5 by default
|
||||
default = "EBICglasso",
|
||||
threshold = TRUE,
|
||||
corMethod = "cor")
|
||||
```
|
||||
|
||||
```{r}
|
||||
Layout <- qgraph::averageLayout(net_modSelect, net_thresh)
|
||||
layout(t(1:2))
|
||||
plot(net_modSelect, layout = Layout, title = "ggmModSelect", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
plot(net_thresh, layout = Layout, title = "Thresholded EBICglasso", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
```
|
||||
|
||||
Here the principal direction is forced - this rescales variables according to the sign of the first eigen-vector. This will lead to most correlations to be positive (positive manifold), leading to negative edges to be substantively interpretable. (not sure this is preferable in this instance as the variables are not all from the same questionnaire).
|
||||
|
||||
```{r}
|
||||
net_modSelect_rescale <- estimateNetwork(complete_df[feature_list],
|
||||
default = "ggmModSelect",
|
||||
stepwise = FALSE,
|
||||
principalDirection = TRUE)
|
||||
net_thresh_rescale <- estimateNetwork(complete_df[feature_list],
|
||||
tuning = 0,
|
||||
default = "EBICglasso",
|
||||
threshold = TRUE,
|
||||
principalDirection = TRUE)
|
||||
layout(t(1:2))
|
||||
plot(net_modSelect_rescale, layout = Layout,
|
||||
title = "ggmModSelect", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
plot(net_thresh_rescale, layout = Layout,
|
||||
title = "Thresholded EBICglasso", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
```
|
||||
```{r}
|
||||
qgraph::centralityPlot(
|
||||
list(
|
||||
ggmModSelect = net_modSelect_rescale,
|
||||
EBICGlasso_thresh = net_thresh_rescale
|
||||
), include = "ExpectedInfluence"
|
||||
)
|
||||
```
|
||||
|
||||
```{r}
|
||||
boots <- bootnet(net_thresh_rescale, statistics = "ExpectedInfluence",
|
||||
nBoots = 100, nCores = 2, type = "case")
|
||||
plot(boots, statistics = "ExpectedInfluence") +
|
||||
theme(legend.position = "none")
|
||||
```
|
||||
### relative importance network
|
||||
There is no accounting for the hierarchical nature of the data here
|
||||
|
||||
```{r}
|
||||
# net_relimp <- estimateNetwork(complete_df[feature_list],
|
||||
# default = "relimp",
|
||||
# normalize = FALSE)
|
||||
# net_relimp2 <- estimateNetwork(complete_df[feature_list],
|
||||
# default = "relimp",
|
||||
# normalize = FALSE,
|
||||
# structureDefault = "ggmModSelect",
|
||||
# stepwise = FALSE # Sent to structureDefault function
|
||||
# )
|
||||
```
|
||||
|
||||
|
||||
```{r}
|
||||
# Layout <- qgraph::averageLayout(net_relimp, net_relimp2)
|
||||
# layout(t(1:2))
|
||||
# plot(net_relimp, layout = Layout, title = "Saturated", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038")
|
||||
# plot(net_relimp2, layout = Layout, title = "Non-saturated", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038")
|
||||
```
|
||||
### graphicalVAR
|
||||
This approach is not really valid as it uses all time points as if they are from the same person which is not the case here
|
||||
|
||||
```{r}
|
||||
# Estimate model:
|
||||
gvar <- estimateNetwork(
|
||||
complete_df, default = "graphicalVAR", vars = feature_list,
|
||||
tuning = 0, dayvar = "fsd_round", nLambda = 10
|
||||
)
|
||||
```
|
||||
|
||||
|
||||
```{r}
|
||||
Layout <- qgraph::averageLayout(gvar$graph$temporal,
|
||||
gvar$graph$contemporaneous)
|
||||
layout(t(1:2))
|
||||
plot(gvar, graph = "temporal", layout = Layout,
|
||||
title = "Temporal", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
plot(gvar, graph = "contemporaneous", layout = Layout,
|
||||
title = "Contemporaneous", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
```
|
||||
|
||||
```{r}
|
||||
# gvar_boot <- bootnet(gvar, nBoots = 2, nCores = 2)
|
||||
```
|
||||
|
||||
```{r}
|
||||
# plot(gvar_boot, graph = "contemporaneous", plot = "interval")
|
||||
```
|
||||
|
||||
### Mixed Graphical model
|
||||
|
||||
```{r}
|
||||
net_mgm <- estimateNetwork(complete_df[feature_list],
|
||||
default = "mgm",
|
||||
type="g",
|
||||
level=1
|
||||
# type=c("g", "g", "g", "g", "c", "c", "c", "g"),
|
||||
# level=c(1, 1, 1, 1, 12, 7, 15, 1)
|
||||
)
|
||||
```
|
||||
|
||||
```{r}
|
||||
plot(net_mgm, layout = Layout,
|
||||
title = "mgm", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
```
|
||||
|
||||
```{r}
|
||||
par(mar=c(5,1,10,1)+7)
|
||||
for (checkpoint in unique(complete_df[, "fsd_round"])) {
|
||||
net_mgm <- estimateNetwork(complete_df[complete_df["fsd_round"]==checkpoint, feature_list],
|
||||
default = "mgm",
|
||||
type="g",
|
||||
level=1)
|
||||
plot(net_mgm, layout = Layout,
|
||||
# title = paste("mgm network", "checkpoint =", checkpoint),
|
||||
edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
|
||||
title(main=paste("Mixed Graphical Model network", "checkpoint =", checkpoint), line=16.25)
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
### Partial Correlation network
|
||||
|
||||
```{r}
|
||||
net_par_cor <- estimateNetwork(complete_df[feature_list],
|
||||
default = "pcor")
|
||||
```
|
||||
|
||||
```{r}
|
||||
plot(net_par_cor, layout = Layout,
|
||||
title = "Partial Correlation", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
```
|
||||
|
||||
```{r}
|
||||
par(mar=c(5,1,10,1)+7)
|
||||
for (checkpoint in unique(complete_df[, "fsd_round"])) {
|
||||
net_par_cor <- estimateNetwork(complete_df[complete_df["fsd_round"]==checkpoint, feature_list],
|
||||
default = "pcor")
|
||||
plot(net_par_cor, layout = Layout,
|
||||
edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
|
||||
title(main=paste("Partial Correlation network", "checkpoint =", checkpoint), line=16.25)
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
### Correlation network
|
||||
|
||||
```{r}
|
||||
net_cor <- estimateNetwork(complete_df[feature_list],
|
||||
default = "cor")
|
||||
```
|
||||
|
||||
```{r}
|
||||
plot(net_cor, layout = Layout,
|
||||
title = "Correlation", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
```
|
||||
|
||||
### GGMncv network
|
||||
|
||||
```{r}
|
||||
net_GGMncv <- estimateNetwork(complete_df[feature_list],
|
||||
default = "GGMncv")
|
||||
```
|
||||
|
||||
```{r}
|
||||
plot(net_GGMncv, layout = Layout,
|
||||
title = "GGMncv", edge.labels=TRUE, posCol="#306fbe", negCol="#e58038", label.scale.equal=TRUE, label.cex=10)
|
||||
```
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -7,12 +7,14 @@
|
||||
library(tidyverse)
|
||||
library(dplyr)
|
||||
|
||||
country_in_question <- "Finland"
|
||||
country_in_question <- "Netherlands"
|
||||
|
||||
## World Survey
|
||||
|
||||
# matti date range 2021-06-09 to 2022-01-18
|
||||
|
||||
# Ask for an indicator that's not found:
|
||||
path <- paste0("https://covidmap.umd.edu/api/resources?indicator=all&type=smoothed&country=", country_in_question, "&daterange=20201115-20201130")
|
||||
path <- paste0("https://covidmap.umd.edu/api/resources?indicator=all&type=smoothed&country=", country_in_question, "&daterange=20210609-20220118")
|
||||
request <- httr::GET(url = path)
|
||||
response <- httr::content(request, as = "text", encoding = "UTF-8")
|
||||
|
||||
@@ -47,7 +49,7 @@ for (i in 1:length(all_indicators)){
|
||||
all_indicators[i],
|
||||
"&type=daily&country=",
|
||||
country_in_question,
|
||||
"&daterange=20200101-20221130")
|
||||
"&daterange=20210609-20220118")
|
||||
|
||||
# request data from api
|
||||
request <- httr::GET(url = path)
|
||||
@@ -102,7 +104,7 @@ latest_date <- data_for_pca$date %>% tail(1)
|
||||
readr::write_csv(x = data_for_pca,
|
||||
file = paste0("shield-complexity/data/",
|
||||
country_in_question,
|
||||
"_worldsurvey_nonmissing_since_2021-06-08_to_",
|
||||
"_worldsurvey_nonmissing_since_2021-06-09_to_",
|
||||
latest_date, ".csv"))
|
||||
|
||||
#### Coefficient of variation
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the Sphinx docs and deploy to Firebase Hosting (site: matti-jms).
|
||||
set -euo pipefail
|
||||
sphinx-build -b html docs/source/ docs/build/html
|
||||
firebase deploy --only hosting:matti-jms
|
||||
@@ -0,0 +1,20 @@
|
||||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line, and also
|
||||
# from the environment for the first two.
|
||||
SPHINXOPTS ?=
|
||||
SPHINXBUILD ?= sphinx-build
|
||||
SOURCEDIR = source
|
||||
BUILDDIR = build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
@@ -0,0 +1,35 @@
|
||||
@ECHO OFF
|
||||
|
||||
pushd %~dp0
|
||||
|
||||
REM Command file for Sphinx documentation
|
||||
|
||||
if "%SPHINXBUILD%" == "" (
|
||||
set SPHINXBUILD=sphinx-build
|
||||
)
|
||||
set SOURCEDIR=source
|
||||
set BUILDDIR=build
|
||||
|
||||
%SPHINXBUILD% >NUL 2>NUL
|
||||
if errorlevel 9009 (
|
||||
echo.
|
||||
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
|
||||
echo.installed, then set the SPHINXBUILD environment variable to point
|
||||
echo.to the full path of the 'sphinx-build' executable. Alternatively you
|
||||
echo.may add the Sphinx directory to PATH.
|
||||
echo.
|
||||
echo.If you don't have Sphinx installed, grab it from
|
||||
echo.https://www.sphinx-doc.org/
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
if "%1" == "" goto help
|
||||
|
||||
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
goto end
|
||||
|
||||
:help
|
||||
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
|
||||
:end
|
||||
popd
|
||||
@@ -0,0 +1,6 @@
|
||||
sphinx>=4.4.0
|
||||
sphinx-bootstrap-theme>=0.8.1
|
||||
nbsphinx>=0.8.8
|
||||
nbsphinx-link>=1.3.0
|
||||
sphinx-gallery>=0.10.1
|
||||
numpydoc>=1.2
|
||||
@@ -0,0 +1,79 @@
|
||||
body {
|
||||
font-family: "Raleway", sans-serif;
|
||||
--darkgrey: grey;
|
||||
--white: #ffffff;
|
||||
--greenblue: #005dc7;
|
||||
color: var(--greenblue);
|
||||
}
|
||||
|
||||
h1, h2, h3, h4, h5, h6 {
|
||||
color: var(--greenblue);
|
||||
}
|
||||
|
||||
li {
|
||||
color: var(--greenblue);
|
||||
}
|
||||
|
||||
li.toctree-l1 a.reference.internal code {
|
||||
color: var(--greenblue);
|
||||
}
|
||||
|
||||
li .reference.internal {
|
||||
color: var(--greenblue);
|
||||
}
|
||||
|
||||
.nav.navbar-nav li a {
|
||||
color: var(--darkgrey);
|
||||
}
|
||||
|
||||
.nav.navbar-nav li a:hover {
|
||||
color: var(--darkgrey);
|
||||
}
|
||||
|
||||
#dLabelGlobalToc {
|
||||
color: var(--darkgrey);
|
||||
}
|
||||
|
||||
#dLabelLocalToc {
|
||||
color: var(--darkgrey);
|
||||
}
|
||||
|
||||
.genindex-jumpbox a {
|
||||
color: var(--greenblue);
|
||||
}
|
||||
|
||||
table td li a {
|
||||
color: var(--greenblue);
|
||||
}
|
||||
|
||||
.footer a {
|
||||
color: var(--greenblue);
|
||||
}
|
||||
|
||||
.dropdown-menu>li>a:hover, .dropdown-menu>li>a:focus {
|
||||
color: var(--greenblue);
|
||||
text-decoration: none;
|
||||
background-color: var(--greenyellow);
|
||||
}
|
||||
|
||||
code {
|
||||
color: var(--greenblue);
|
||||
}
|
||||
|
||||
a {
|
||||
color: var(--darkgrey);
|
||||
}
|
||||
|
||||
a.headerlink:hover {
|
||||
color: var(--darkgrey);
|
||||
}
|
||||
|
||||
.navbar-default .navbar-brand {
|
||||
color: var(--darkgrey);
|
||||
/* background-color: transparent; */
|
||||
}
|
||||
|
||||
.navbar-default .navbar-brand:hover, .navbar-default .navbar-brand:focus {
|
||||
color: var(--darkgrey);
|
||||
background-color: transparent;
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"path": "../../preparedness/analysis.ipynb"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"path": "../../citizen_shield/citizen_shield_demographics_classification.ipynb"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"path": "../../citizen_shield/citizen_shield_demographics_clustering.ipynb"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"path": "../../citizen_shield/citizen_shield_permutation_approach.ipynb"
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
# Configuration file for the Sphinx documentation builder.
|
||||
#
|
||||
# This file only contains a selection of the most common options. For a full
|
||||
# list see the documentation:
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
||||
|
||||
# -- Path setup --------------------------------------------------------------
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#
|
||||
import os
|
||||
import sys
|
||||
|
||||
import sphinx_bootstrap_theme
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../../"))
|
||||
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
|
||||
project = 'Matti Jms Collabs'
|
||||
copyright = '2022, James Twose, Matti Heino'
|
||||
author = 'James Twose, Matti Heino'
|
||||
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = ["sphinx.ext.duration",
|
||||
"sphinx.ext.doctest",
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.autosummary",
|
||||
"sphinx.ext.mathjax",
|
||||
"sphinx.ext.viewcode",
|
||||
"numpydoc",
|
||||
"sphinx.ext.inheritance_diagram",
|
||||
"nbsphinx",
|
||||
"nbsphinx_link",
|
||||
"sphinx_gallery.load_style",
|
||||
]
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This pattern also affects html_static_path and html_extra_path.
|
||||
exclude_patterns = ["_build", "**.ipynb_checkpoints"]
|
||||
|
||||
|
||||
# -- Options for HTML output -------------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#
|
||||
html_theme = "bootstrap"
|
||||
html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()
|
||||
|
||||
html_theme_options = {
|
||||
"source_link_position": "footer",
|
||||
"bootswatch_theme": "simplex",
|
||||
"navbar_title": "Matti and James Collaboration Code",
|
||||
"navbar_sidebarrel": False,
|
||||
"bootstrap_version": "3",
|
||||
"nosidebar": True,
|
||||
"body_max_width": "100%",
|
||||
"navbar_links": [
|
||||
# ("Gallery", "examples/index"),
|
||||
# ("Tutorial", "tutorial"),
|
||||
# ("Home", "index"),
|
||||
# ("API", "api"),
|
||||
],
|
||||
}
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
|
||||
# Add the 'copybutton' javascript, to hide/show the prompt in code
|
||||
# examples, originally taken from scikit-learn's doc/conf.py
|
||||
def setup(app):
|
||||
app.add_js_file("copybutton.js")
|
||||
app.add_css_file("css/custom.css")
|
||||
@@ -0,0 +1,70 @@
|
||||
.. Matti Jms Collabs documentation master file, created by
|
||||
sphinx-quickstart on Tue Jun 28 20:18:11 2022.
|
||||
You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
|
||||
Welcome to Matti Jms Collabs's documentation!
|
||||
=============================================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents:
|
||||
|
||||
|
||||
Citizen Shield
|
||||
----------------------------
|
||||
.. nbgallery::
|
||||
:caption: Citizen Shield Notebooks
|
||||
:name: notebook-gallery
|
||||
:glob:
|
||||
|
||||
citizen_shield_demographics_classification.ipynb
|
||||
citizen_shield_demographics_clustering.ipynb
|
||||
citizen_shield_permutation_approach.ipynb
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<div id="content">
|
||||
<li><a href="_static/html/citizen_shield_networks.nb.html">Citizen Shield Network Approach in R</a></li>
|
||||
</div>
|
||||
|
||||
|
|
||||
|
||||
References
|
||||
^^^^^^^^^^^
|
||||
.. raw:: html
|
||||
|
||||
<div id="content">
|
||||
<li>Borsboom, D., Deserno, M. K., Rhemtulla, M., Epskamp, S., Fried, E. I., McNally, R. J., ... & Waldorp, L. J. (2021).
|
||||
Network analysis of multivariate data in psychological science.
|
||||
<i>Nature Reviews Methods Primers, 1</i> (1), 1-18.
|
||||
<a href="https://www.nature.com/articles/s43586-021-00055-w.pdf"> Article Link</a></li>
|
||||
</div>
|
||||
|
||||
|
|
||||
|
||||
Preparedness
|
||||
----------------------------
|
||||
.. nbgallery::
|
||||
:caption: Preparedness Notebooks
|
||||
:name: preparedness-notebook-gallery
|
||||
:glob:
|
||||
|
||||
analysis.ipynb
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<div id="content">
|
||||
<li><a href="_static/html/user_embeddings_pca_3d.html">PCA Preparedness</a></li>
|
||||
<li><a href="_static/html/user_embeddings_tsne_3d.html">t-SNE Preparedness</a></li>
|
||||
<li><a href="_static/html/user_embeddings_umap_3d.html">UMAP Preparedness</a></li>
|
||||
</div>
|
||||
|
||||
|
|
||||
|
||||
Indices and tables
|
||||
==================
|
||||
|
||||
* :ref:`genindex`
|
||||
* :ref:`modindex`
|
||||
* :ref:`search`
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"hosting": {
|
||||
"site": "matti-jms",
|
||||
"public": "docs/build/html",
|
||||
"ignore": ["firebase.json", "**/.*", "**/node_modules/**"],
|
||||
"cleanUrls": true,
|
||||
"trailingSlash": false
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,239 @@
|
||||
# %% [markdown]
|
||||
# # Visualize Most Prepared Users
|
||||
# This workflow loads user embeddings, generates a prompt embedding, computes similarity, and visualizes the most prepared users.
|
||||
|
||||
# %%
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.manifold import TSNE
|
||||
import plotly.express as px
|
||||
import matplotlib.pyplot as plt
|
||||
import seaborn as sns
|
||||
import umap
|
||||
import umap.plot
|
||||
from utils import get_ollama_embedding
|
||||
|
||||
# %% [markdown]
|
||||
# ## Read in the file with user vectors
|
||||
# Update the path/format as needed.
|
||||
# %%
|
||||
df = (
|
||||
pd.read_csv("./data/eurobarometer_preparedness_model_data_v3.csv")
|
||||
.reset_index()
|
||||
.assign(
|
||||
**{
|
||||
"user_id": lambda x: x["index"].astype(str)
|
||||
+ "_"
|
||||
+ x["country_code_iso_3166"]
|
||||
+ "_"
|
||||
+ x["age_recoded_6_categories"].astype(str)
|
||||
+ "_"
|
||||
+ x["gender"].astype(str)
|
||||
}
|
||||
)
|
||||
# select only relevant countries for visualization
|
||||
.loc[
|
||||
lambda x: x["country_code_iso_3166"].isin(
|
||||
["FI", "DE-E", "DE-W", "FR", "ES", "PT"]
|
||||
)
|
||||
]
|
||||
)
|
||||
df.shape
|
||||
|
||||
# relabel None of the above/ Non binary/ do not recognize yourself in above categories/Prefer not to say to other
|
||||
df["gender"] = df["gender"].replace({
|
||||
"None of the above/ Non binary/ do not recognize yourself in above categories/Prefer not to say": "Other",
|
||||
})
|
||||
|
||||
# %% [markdown]
|
||||
# ## Convert string embeddings to lists if needed
|
||||
# %%
|
||||
def parse_embedding(x):
|
||||
if isinstance(x, str):
|
||||
return [float(i) for i in x.strip("[]").split(",")]
|
||||
return x
|
||||
|
||||
|
||||
df["embedding"] = df["embedding"].apply(parse_embedding)
|
||||
|
||||
# %%
|
||||
# Select best user:
|
||||
# disaster_measures_in_hh_* == 1, and how_many_days_meet_* == (4 | 5)
|
||||
# Following relabeling in data_preparation_raw.py:
|
||||
# disaster_measures_in_hh_* != "Not mentioned"
|
||||
# how_many_days_meet_* == "More than 7 days"
|
||||
disaster_measures_in_hh_columns = [
|
||||
col for col in df.columns if col.startswith("disaster_measures_in_hh_")
|
||||
]
|
||||
how_many_days_meet_columns = [
|
||||
col for col in df.columns if col.startswith("how_many_days_meet_")
|
||||
]
|
||||
best_users = df[
|
||||
(df[disaster_measures_in_hh_columns] != "Not mentioned").all(axis=1)
|
||||
& (df[how_many_days_meet_columns] == "More than 7 days").all(axis=1)
|
||||
]
|
||||
print(f"Number of best users: {len(best_users)}")
|
||||
|
||||
best_users.head(5)[["user_id", "user_text"]].to_dict(orient="records")
|
||||
|
||||
# %%
|
||||
best_users.columns.tolist()
|
||||
|
||||
# %% [markdown]
|
||||
# ## Write your prompt and generate its embedding
|
||||
# %%
|
||||
# prompt = "The user is highly prepared for disasters, with emergency supplies and a clear plan."
|
||||
prompt = best_users.head(1)["user_text"].values[
|
||||
0
|
||||
] # Example: use the first user's text as the prompt
|
||||
prompt_embedding = get_ollama_embedding(prompt)
|
||||
|
||||
# %% [markdown]
|
||||
# ## Compute similarity between each user and the prompt
|
||||
# %%
|
||||
user_embeddings = np.vstack(df["embedding"].values)
|
||||
prompt_vec = np.array(prompt_embedding).reshape(1, -1)
|
||||
similarities = cosine_similarity(user_embeddings, prompt_vec).flatten()
|
||||
df["similarity"] = similarities
|
||||
|
||||
# %% [markdown]
|
||||
# ## Visualize the users by similarity
|
||||
# %%
|
||||
_ = plt.figure(figsize=(5, 7))
|
||||
_ = sns.violinplot(
|
||||
x="similarity", y="country_code_iso_3166", hue="country_code_iso_3166", data=df
|
||||
)
|
||||
_ = sns.stripplot(
|
||||
x="similarity",
|
||||
y="country_code_iso_3166",
|
||||
data=df,
|
||||
hue="country_code_iso_3166",
|
||||
alpha=0.8,
|
||||
jitter=True,
|
||||
linewidth=0.5,
|
||||
edgecolor="white",
|
||||
)
|
||||
_ = plt.title("Distribution of User Similarities to Preparedness Prompt")
|
||||
_ = plt.xlabel("Cosine Similarity")
|
||||
|
||||
# %%
|
||||
# np.savetxt("./data/embeddings.tsv", np.vstack(df["embedding"].values), delimiter="\t")
|
||||
|
||||
# # %%
|
||||
# df[["user_id"]].to_csv(
|
||||
# "./data/metadata.tsv", sep="\t", index=False
|
||||
# )
|
||||
|
||||
# %%
|
||||
# Reduce embeddings to 3D with PCA
|
||||
pca = PCA(n_components=3)
|
||||
embeddings_3d = pca.fit_transform(np.vstack(df["embedding"].values))
|
||||
|
||||
# Add PCA components to dataframe
|
||||
df["pca1"] = embeddings_3d[:, 0]
|
||||
df["pca2"] = embeddings_3d[:, 1]
|
||||
df["pca3"] = embeddings_3d[:, 2]
|
||||
|
||||
# %%
|
||||
# Interactive 3D scatter plot
|
||||
fig = px.scatter_3d(
|
||||
df,
|
||||
x="pca1",
|
||||
y="pca2",
|
||||
z="pca3",
|
||||
color="age_recoded_6_categories",
|
||||
hover_data=["user_id", "gender", "age_recoded_6_categories"],
|
||||
title="User Embeddings (PCA 3D)"
|
||||
)
|
||||
fig.write_html("./output/user_embeddings_pca_3d.html")
|
||||
# %%
|
||||
# Reduce embeddings to 3D with t-SNE
|
||||
tsne = TSNE(n_components=3, random_state=42, perplexity=30)
|
||||
embeddings_3d = tsne.fit_transform(np.vstack(df["embedding"].values))
|
||||
|
||||
# Add t-SNE components to dataframe
|
||||
df["tsne1"] = embeddings_3d[:, 0]
|
||||
df["tsne2"] = embeddings_3d[:, 1]
|
||||
df["tsne3"] = embeddings_3d[:, 2]
|
||||
|
||||
# Interactive 3D scatter plot
|
||||
fig = px.scatter_3d(
|
||||
df,
|
||||
x="tsne1",
|
||||
y="tsne2",
|
||||
z="tsne3",
|
||||
color="age_recoded_6_categories",
|
||||
hover_data=["user_id", "gender", "country_code_iso_3166"],
|
||||
title="User Embeddings (t-SNE 3D)"
|
||||
)
|
||||
fig.write_html("./output/user_embeddings_tsne_3d.html")
|
||||
|
||||
# %%
|
||||
# example comparison based on t-sne projection
|
||||
index_1 = 9561
|
||||
index_2 = 5096
|
||||
df.loc[[index_1, index_2]][["user_id", "user_text", "similarity"]].to_dict(orient="records")
|
||||
# %%
|
||||
row1 = df.loc[index_1].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"])
|
||||
row2 = df.loc[index_2].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"])
|
||||
# column_list = df.columns.difference(["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3"])
|
||||
column_list = disaster_measures_in_hh_columns + how_many_days_meet_columns
|
||||
|
||||
diffs = {}
|
||||
for col in column_list:
|
||||
val1 = row1[col]
|
||||
val2 = row2[col]
|
||||
if pd.isnull(val1) and pd.isnull(val2):
|
||||
continue
|
||||
if val1 != val2:
|
||||
diffs[col] = (val1, val2)
|
||||
|
||||
# Print or display the differing columns and their values
|
||||
for col, (v1, v2) in diffs.items():
|
||||
print(f"{col}: {v1} | {v2}")
|
||||
# %%
|
||||
# Reduce embeddings to 3D with UMAP
|
||||
umap_3d = umap.UMAP(n_components=3, random_state=42, metric="cosine").fit_transform(np.vstack(df["embedding"].values))
|
||||
|
||||
# Add UMAP components to dataframe
|
||||
df["umap1"] = umap_3d[:, 0]
|
||||
df["umap2"] = umap_3d[:, 1]
|
||||
df["umap3"] = umap_3d[:, 2]
|
||||
|
||||
# Interactive 3D scatter plot
|
||||
fig = px.scatter_3d(
|
||||
df,
|
||||
x="umap1",
|
||||
y="umap2",
|
||||
z="umap3",
|
||||
color="country_code_iso_3166",
|
||||
hover_data=["user_id", "gender", "country_code_iso_3166"],
|
||||
title="User Embeddings (UMAP 3D)"
|
||||
)
|
||||
fig.write_html("./output/user_embeddings_umap_3d.html")
|
||||
|
||||
# %%
|
||||
# example comparison based on UMAP projection
|
||||
index_1 = 21396
|
||||
index_2 = 10444
|
||||
df.loc[[index_1, index_2]][["user_id", "user_text", "similarity"]].to_dict(orient="records")
|
||||
# %%
|
||||
row1 = df.loc[index_1].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"])
|
||||
row2 = df.loc[index_2].drop(labels=["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"])
|
||||
# column_list = df.columns.difference(["user_id", "embedding", "user_text", "similarity", "pca1", "pca2", "pca3", "tsne1", "tsne2", "tsne3", "umap1", "umap2", "umap3"])
|
||||
column_list = disaster_measures_in_hh_columns + how_many_days_meet_columns
|
||||
diffs = {}
|
||||
for col in column_list:
|
||||
val1 = row1[col]
|
||||
val2 = row2[col]
|
||||
if pd.isnull(val1) and pd.isnull(val2):
|
||||
continue
|
||||
if val1 != val2:
|
||||
diffs[col] = (val1, val2)
|
||||
|
||||
# Print or display the differing columns and their values
|
||||
for col, (v1, v2) in diffs.items():
|
||||
print(f"{col}: {v1} | {v2}")
|
||||
# %%
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,142 @@
|
||||
# %%
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from utils import get_ollama_embedding
|
||||
|
||||
# %%[markdown]
|
||||
#### Data Preparation Steps
|
||||
|
||||
# - Select relevant columns from the cleaned dataset, including a range and specific variables.
|
||||
# - Exclude columns containing substrings like "2nd", "spont", "other", and some specific variables.
|
||||
# - Add columns starting with "region_" and "education_level_".
|
||||
# - Convert character columns to categorical type.
|
||||
# - Generate a `subregion` variable by coalescing region columns, then drop the originals.
|
||||
# - Generate an `education_level` variable by coalescing education columns, replacing "Not mentioned" with NA, then drop the originals.
|
||||
# - Define core preparedness items and their human-readable labels.
|
||||
# - Map country codes to country names.
|
||||
|
||||
# %%
|
||||
# Load your data
|
||||
data_cleaned = pd.read_csv('./data/eurobarometer_data_cleaned_csv.csv')
|
||||
|
||||
print(data_cleaned.shape)
|
||||
|
||||
# Select columns by name and range
|
||||
cols_to_select = [
|
||||
'country_code_iso_3166',
|
||||
'risks_cntry_most_exposed_to_firstly',
|
||||
'risks_pers_most_exposed_to_firstly',
|
||||
'risks_pers_most_exposed_to_number_of_mentioned_risks',
|
||||
'pot_info_sources_to_learn_about_disaster_risks_firstly',
|
||||
*data_cleaned.columns[404:448], # Python is 0-indexed
|
||||
'occupation_of_respondent',
|
||||
'age_recoded_6_categories',
|
||||
'size_of_community',
|
||||
'social_class_self_assessment_5_cat',
|
||||
'direction_things_are_going_life_personally',
|
||||
'political_discussion_local_matters',
|
||||
'political_discussion_national_matters',
|
||||
'left_right_placement_recoded_5_cat',
|
||||
'internet_use_total',
|
||||
'gender',
|
||||
'age_education',
|
||||
'standard_of_living_last_5yrs_in_light_of_crises',
|
||||
'personal_living_conditions_in_one_years_time',
|
||||
'standard_of_living_next_5yrs'
|
||||
]
|
||||
|
||||
# Remove columns containing certain substrings
|
||||
exclude_patterns = ['2nd', 'spont', 'other']
|
||||
cols_to_exclude = [col for col in data_cleaned.columns if any(p in col for p in exclude_patterns)]
|
||||
cols_to_exclude += [
|
||||
'disaster_measures_in_hh_number_of_measures',
|
||||
'disaster_pers_experienced_past_10yrs_none',
|
||||
'pot_info_sources_to_learn_about_disaster_risks_interested_in_at_least_one_source'
|
||||
]
|
||||
|
||||
# Add region_ and education_level_ columns
|
||||
cols_to_select += [col for col in data_cleaned.columns if col.startswith('region_')]
|
||||
cols_to_select += [col for col in data_cleaned.columns if col.startswith('education_level_')]
|
||||
|
||||
final_cols = [col for col in cols_to_select if col not in cols_to_exclude]
|
||||
df_model = data_cleaned[final_cols].copy()
|
||||
|
||||
# %% [markdown]
|
||||
# #### Combine all columns into a single string per user
|
||||
# This step creates a text representation of each user, which can be sent to an embedding model.
|
||||
|
||||
def row_to_string(row):
|
||||
return ' | '.join(f'{col}: {row[col]}' for col in row.index)
|
||||
|
||||
# Combine all columns into a single string per user (except country_code_iso_3166)
|
||||
df_model['user_text'] = df_model.drop(columns=['country_code_iso_3166']).apply(row_to_string, axis=1)
|
||||
|
||||
df_model.head()
|
||||
|
||||
# %%
|
||||
# Convert character columns to category BEFORE adding embedding column
|
||||
for col in df_model.select_dtypes(include='object').columns:
|
||||
if col != 'user_text':
|
||||
df_model[col] = df_model[col].astype('category')
|
||||
|
||||
# Generate embeddings for each user using Ollama
|
||||
df_model['embedding'] = df_model['user_text'].apply(get_ollama_embedding)
|
||||
|
||||
# Add new columns to final_cols
|
||||
final_cols.extend(["user_text", "embedding"])
|
||||
|
||||
# Check the lengths of all embeddings
|
||||
embedding_lengths = df_model['embedding'].apply(lambda x: len(x) if isinstance(x, list) else None)
|
||||
print('Embedding lengths:', embedding_lengths.tolist())
|
||||
df_model = df_model[final_cols].copy()
|
||||
|
||||
# Generate subregion variable
|
||||
region_cols = [col for col in df_model.columns if col.startswith('region_')]
|
||||
df_model['subregion'] = df_model[region_cols].bfill(axis=1).iloc[:, 0]
|
||||
df_model.drop(columns=region_cols, inplace=True)
|
||||
|
||||
# Generate education_level variable
|
||||
edu_cols = [col for col in df_model.columns if col.startswith('education_level_')]
|
||||
for col in edu_cols:
|
||||
df_model[col] = df_model[col].replace('Not mentioned', np.nan)
|
||||
df_model['education_level'] = df_model[edu_cols].bfill(axis=1).iloc[:, 0]
|
||||
df_model['education_level'] = df_model['education_level'].astype('category')
|
||||
df_model.drop(columns=edu_cols, inplace=True)
|
||||
|
||||
# Core preparedness items
|
||||
CORE_ITEMS_MAPPED = [
|
||||
"disaster_measures_in_hh_emergency_supply_drinks_food",
|
||||
"disaster_measures_in_hh_emergency_supply_water_cooking_hygiene",
|
||||
"disaster_measures_in_hh_agreed_with_friends_family_to_contact",
|
||||
"disaster_measures_in_hh_discussed_common_prot_measures_in_neighbourhood",
|
||||
"disaster_measures_in_hh_battery_powered_radio"
|
||||
]
|
||||
|
||||
CORE_ITEM_LABELS = {
|
||||
"disaster_measures_in_hh_emergency_supply_drinks_food": "Emergency supply of drinks, food",
|
||||
"disaster_measures_in_hh_emergency_supply_water_cooking_hygiene": "Emergency supply of cooking and hygiene water",
|
||||
"disaster_measures_in_hh_agreed_with_friends_family_to_contact": "Agreed with family, friends on how to contact in an emergency",
|
||||
"disaster_measures_in_hh_discussed_common_prot_measures_in_neighbourhood": "Discussed precautions in neighbourhood",
|
||||
"disaster_measures_in_hh_battery_powered_radio": "Battery-powered radio accessible"
|
||||
}
|
||||
|
||||
COUNTRY_NAME_MAP = {
|
||||
"FI": "Finland",
|
||||
"DE-E": "East Germany",
|
||||
"DE-W": "West Germany",
|
||||
"FR": "France",
|
||||
"ES": "Spain",
|
||||
"PT": "Portugal",
|
||||
# "EE": "Estonia",
|
||||
# "DK": "Denmark",
|
||||
# "SE": "Sweden",
|
||||
# "NL": "Netherlands"
|
||||
}
|
||||
|
||||
# %%
|
||||
df_model.head()
|
||||
|
||||
# %%
|
||||
df_model.head().to_dict(orient='records')
|
||||
# %%
|
||||
df_model.to_csv('./data/eurobarometer_preparedness_model_data_v2.csv', index=False)
|
||||
@@ -0,0 +1,137 @@
|
||||
# %%
|
||||
import pyreadstat
|
||||
import json
|
||||
import re
|
||||
import datetime
|
||||
|
||||
from utils import get_ollama_summary, get_ollama_embedding
|
||||
|
||||
# %%
|
||||
df_sav, meta = pyreadstat.read_sav('./data/ZA8841_v1-0-0.sav')
|
||||
print(df_sav.shape)
|
||||
|
||||
# %%
|
||||
df_sav.head(1).to_dict(orient='records')
|
||||
|
||||
# %%
|
||||
def safe_json(obj):
|
||||
if isinstance(obj, (datetime.datetime, datetime.date)):
|
||||
return obj.isoformat()
|
||||
if isinstance(obj, set):
|
||||
return list(obj)
|
||||
if hasattr(obj, '__dict__'):
|
||||
return str(obj)
|
||||
return obj
|
||||
|
||||
meta_dict = vars(meta)
|
||||
meta_json = json.dumps(meta_dict, default=safe_json, indent=2)
|
||||
|
||||
# %%
|
||||
with open('./data/za8841_meta.json', 'w') as f:
|
||||
f.write(meta_json)
|
||||
|
||||
# %%
|
||||
# Load your mapping dictionary (from JSON file or directly)
|
||||
with open("./data/za8841_meta.json") as f:
|
||||
meta = json.load(f)
|
||||
value_labels = meta["variable_value_labels"]
|
||||
|
||||
# %%
|
||||
# Assume df_sav is your loaded SPSS dataframe
|
||||
# For each column in the mapping, map values if the column exists in df_sav
|
||||
for col, mapping in value_labels.items():
|
||||
if col in df_sav.columns:
|
||||
# Convert keys to float if needed (SPSS values often are float)
|
||||
mapping_float = {float(k): v for k, v in mapping.items()}
|
||||
df_sav[col] = df_sav[col].map(mapping_float).fillna(df_sav[col])
|
||||
|
||||
# Now all mapped columns have human-readable values
|
||||
print(df_sav.head())
|
||||
|
||||
# %%
|
||||
labels_map = meta["column_names_to_labels"]
|
||||
def make_pandas_friendly(col):
|
||||
col = labels_map.get(col, col)
|
||||
col = re.sub(r'[.\s]+', '_', col)
|
||||
col = re.sub(r'[^0-9a-zA-Z_]', '', col)
|
||||
col = col.lower()
|
||||
col = re.sub(r'__+', '_', col) # Replace double (or more) underscores with single
|
||||
col = col.strip('_') # Remove leading/trailing underscores
|
||||
return col
|
||||
|
||||
df_sav.columns = [make_pandas_friendly(col) for col in df_sav.columns]
|
||||
|
||||
# %%
|
||||
df_sav.head(1).to_dict(orient='records')
|
||||
|
||||
# %%
|
||||
cols_to_select = [
|
||||
'country_code_iso_3166',
|
||||
'risks_cntry_most_exposed_to_firstly',
|
||||
'risks_pers_most_exposed_to_firstly',
|
||||
'risks_pers_most_exposed_to_number_of_mentioned_risks',
|
||||
'pot_info_sources_to_learn_about_disaster_risks_firstly',
|
||||
*df_sav.columns[404:448], # Python is 0-indexed
|
||||
'occupation_of_respondent',
|
||||
'age_recoded_6_categories',
|
||||
'size_of_community',
|
||||
# 'social_class_self_assessment_5_cat', # not in the data due to mapping
|
||||
'direction_things_are_going_life_personally',
|
||||
'political_discussion_local_matters',
|
||||
'political_discussion_national_matters',
|
||||
# 'left_right_placement_recoded_5_cat', # not in the data due to mapping
|
||||
'internet_use_total',
|
||||
'gender',
|
||||
'age_education',
|
||||
'standard_of_living_last_5yrs_in_light_of_crises',
|
||||
'personal_living_conditions_in_one_years_time',
|
||||
'standard_of_living_next_5yrs'
|
||||
]
|
||||
|
||||
# Remove columns containing certain substrings
|
||||
exclude_patterns = ['2nd', 'spont', 'other']
|
||||
cols_to_exclude = [col for col in df_sav.columns if any(p in col for p in exclude_patterns)]
|
||||
cols_to_exclude += [
|
||||
'disaster_measures_in_hh_number_of_measures',
|
||||
'disaster_pers_experienced_past_10yrs_none',
|
||||
'pot_info_sources_to_learn_about_disaster_risks_interested_in_at_least_one_source'
|
||||
]
|
||||
|
||||
# Add region_ and education_level_ columns
|
||||
cols_to_select += [col for col in df_sav.columns if col.startswith('region_')]
|
||||
cols_to_select += [col for col in df_sav.columns if col.startswith('education_level_')]
|
||||
|
||||
final_cols = [col for col in cols_to_select if col not in cols_to_exclude]
|
||||
|
||||
print(f"Final number of columns: {len(final_cols)}")
|
||||
final_cols
|
||||
|
||||
# %%
|
||||
df_model = df_sav[final_cols].copy()
|
||||
|
||||
# %%
|
||||
# %% [markdown]
|
||||
# #### Combine all columns into a single string per user
|
||||
# This step creates a text representation of each user, which can be sent to an embedding model.
|
||||
|
||||
def row_to_string(row):
|
||||
return ' | '.join(f'{col}: {row[col]}' for col in row.index)
|
||||
|
||||
# Combine all columns into a single string per user (except country_code_iso_3166)
|
||||
df_model['user_text'] = df_model.drop(columns=['country_code_iso_3166']).apply(row_to_string, axis=1)
|
||||
|
||||
df_model.head()
|
||||
|
||||
# %%
|
||||
# Convert character columns to category BEFORE adding embedding column
|
||||
for col in df_model.select_dtypes(include='object').columns:
|
||||
if col != 'user_text':
|
||||
df_model[col] = df_model[col].astype('category')
|
||||
|
||||
# %%
|
||||
# df_model["summary"] = df_model['user_text'].apply(get_ollama_summary)
|
||||
# df_model.head(1)['user_text'].apply(get_ollama_summary).to_list()
|
||||
df_model["embedding"] = df_model['user_text'].apply(get_ollama_embedding)
|
||||
|
||||
# %%
|
||||
df_model.to_csv('./data/eurobarometer_preparedness_model_data_v3.csv', index=False)
|
||||
@@ -0,0 +1,20 @@
|
||||
import logging
|
||||
import uvicorn
|
||||
import os
|
||||
from api import app
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_api() -> None:
|
||||
"""Run the FastAPI application."""
|
||||
if "james" in os.environ.get("USER", ""):
|
||||
logger.info("Running in James's environment")
|
||||
uvicorn.run("api:app", host="0.0.0.0", port=8080, reload=True)
|
||||
else:
|
||||
uvicorn.run(app, host="0.0.0.0", port=8080)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_api()
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,12 @@
|
||||
pandas
|
||||
ipykernel
|
||||
requests
|
||||
scikit-learn
|
||||
matplotlib
|
||||
seaborn
|
||||
pyreadstat
|
||||
plotly
|
||||
nbformat>=4.2.0
|
||||
umap-learn[plot]
|
||||
fastapi
|
||||
uvicorn
|
||||
@@ -0,0 +1,4 @@
|
||||
from pydantic import BaseModel
|
||||
|
||||
class PredictionRequest(BaseModel):
|
||||
text: str
|
||||
@@ -0,0 +1,757 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Preparedness Survey — Quick Form</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: system-ui, Segoe UI, Roboto, Helvetica, Arial, sans-serif;
|
||||
margin: 28px;
|
||||
background: #f7fafc
|
||||
}
|
||||
|
||||
.container {
|
||||
max-width: 980px;
|
||||
margin: 0 auto;
|
||||
background: #fff;
|
||||
padding: 28px;
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 6px 20px rgba(2, 6, 23, .08)
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin-top: 0;
|
||||
margin-bottom: 18px;
|
||||
font-size: 28px;
|
||||
}
|
||||
|
||||
.row {
|
||||
display: flex;
|
||||
gap: 16px;
|
||||
margin-bottom: 14px
|
||||
}
|
||||
|
||||
label {
|
||||
display: block;
|
||||
font-weight: 600;
|
||||
margin-bottom: 4px
|
||||
}
|
||||
|
||||
input[type=text],
|
||||
select,
|
||||
textarea,
|
||||
input[type=number] {
|
||||
width: 100%;
|
||||
padding: 12px;
|
||||
border: 1px solid #d1d5db;
|
||||
border-radius: 8px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.grid {
|
||||
display: block;
|
||||
margin-top: 18px;
|
||||
margin-bottom: 18px;
|
||||
}
|
||||
|
||||
/* form rows: labels above inputs for cleaner reading and to avoid overlap */
|
||||
.form-row {
|
||||
display: block;
|
||||
padding: 12px 0;
|
||||
border-bottom: 1px solid #eef2f6;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
/* label shown above the input area */
|
||||
.form-row>label,
|
||||
.label-top {
|
||||
display: block;
|
||||
width: 100%;
|
||||
margin: 0 0 8px 0;
|
||||
font-weight: 600;
|
||||
font-size: 14px;
|
||||
color: #111827;
|
||||
}
|
||||
|
||||
.form-row .input-wrap {
|
||||
width: 100%;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
/* keep Likert rows visually compact with the question label above */
|
||||
.form-row.likert-row {
|
||||
display: block
|
||||
}
|
||||
|
||||
.form-row.likert-row .input-wrap.likert {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
flex-wrap: wrap
|
||||
}
|
||||
|
||||
@media (max-width:700px) {
|
||||
.form-row {
|
||||
flex-direction: column;
|
||||
align-items: stretch
|
||||
}
|
||||
|
||||
.form-row label {
|
||||
width: 100%
|
||||
}
|
||||
|
||||
.form-row .input-wrap {
|
||||
width: 100%
|
||||
}
|
||||
}
|
||||
|
||||
.likert {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
align-items: center;
|
||||
flex-wrap: wrap
|
||||
}
|
||||
|
||||
/* each option is an inline radio + text so the options form a horizontal row */
|
||||
.likert label {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-weight: 400;
|
||||
white-space: nowrap;
|
||||
padding: 4px 6px;
|
||||
border-radius: 6px
|
||||
}
|
||||
|
||||
.likert input[type=radio] {
|
||||
margin: 0 6px 0 0
|
||||
}
|
||||
|
||||
/* Normalize likert groups so all questions display consistently */
|
||||
.input-wrap.likert {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.input-wrap.likert .likert,
|
||||
.input-wrap.likert {
|
||||
width: 100%;
|
||||
align-items: center
|
||||
}
|
||||
|
||||
.input-wrap.likert .likert {
|
||||
justify-content: flex-start;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.input-wrap.likert label {
|
||||
padding: 6px 10px;
|
||||
border-radius: 8px;
|
||||
background: transparent
|
||||
}
|
||||
|
||||
.input-wrap.likert label:hover {
|
||||
background: rgba(2, 6, 23, 0.03)
|
||||
}
|
||||
|
||||
.input-wrap.likert input[type=radio] {
|
||||
transform: scale(1.05);
|
||||
}
|
||||
|
||||
/* toggle switch for binary yes/no selects */
|
||||
.toggle-switch {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
cursor: pointer
|
||||
}
|
||||
|
||||
.toggle-switch input {
|
||||
display: none
|
||||
}
|
||||
|
||||
.toggle-switch .slider {
|
||||
width: 48px;
|
||||
height: 26px;
|
||||
background: #e5e7eb;
|
||||
border-radius: 999px;
|
||||
position: relative;
|
||||
transition: background .18s ease
|
||||
}
|
||||
|
||||
.toggle-switch .slider::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
left: 4px;
|
||||
top: 4px;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
background: #fff;
|
||||
border-radius: 50%;
|
||||
transition: transform .18s ease
|
||||
}
|
||||
|
||||
.toggle-switch input:checked+.slider {
|
||||
background: #10b981
|
||||
}
|
||||
|
||||
.toggle-switch input:checked+.slider::after {
|
||||
transform: translateX(22px)
|
||||
}
|
||||
|
||||
.actions {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
margin-top: 20px
|
||||
}
|
||||
|
||||
button {
|
||||
padding: 10px 14px;
|
||||
border-radius: 8px;
|
||||
border: 0;
|
||||
background: #0069ff;
|
||||
color: #fff;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
pre {
|
||||
background: #0b1220;
|
||||
color: #e6eef8;
|
||||
padding: 16px;
|
||||
border-radius: 8px;
|
||||
overflow: auto
|
||||
}
|
||||
|
||||
h3 {
|
||||
margin-top: 28px;
|
||||
margin-bottom: 12px;
|
||||
font-size: 18px;
|
||||
color: #0f172a
|
||||
}
|
||||
|
||||
p {
|
||||
margin-bottom: 18px;
|
||||
color: #374151
|
||||
}
|
||||
|
||||
small {
|
||||
color: #6b7280
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>Preparedness survey — quick form</h1>
|
||||
<p>Fill the fields you have, then use <strong>Predict</strong> to send the form as JSON to the server's
|
||||
<code>/predict</code> endpoint and see the model response below.
|
||||
</p>
|
||||
<form id="survey">
|
||||
<!-- dynamic form will be rendered here from metadata -->
|
||||
<div id="dynamic-grid" class="grid"></div>
|
||||
|
||||
<!-- All static questions removed. The form is fully metadata-driven and rendered into #dynamic-grid -->
|
||||
|
||||
<div class="actions">
|
||||
<button type="button" id="predict">Predict</button>
|
||||
<button type="button" id="selftest">Self-test</button>
|
||||
</div>
|
||||
|
||||
<h3>JSON preview</h3>
|
||||
<pre id="output">{ }</pre>
|
||||
</form>
|
||||
<script>
|
||||
// Note: selects were converted to static markup; label/likert styling is handled in the CSS above.
|
||||
const form = document.getElementById('survey');
|
||||
const output = document.getElementById('output');
|
||||
|
||||
function collectForm() {
|
||||
const data = {};
|
||||
// include all inputs/selects
|
||||
const elements = Array.from(form.elements).filter(e => e.name);
|
||||
// handle radios by grouping
|
||||
const handled = new Set();
|
||||
elements.forEach(el => {
|
||||
const name = el.name;
|
||||
if (handled.has(name)) return;
|
||||
if (el.type === 'radio') {
|
||||
const radios = form.querySelectorAll(`input[name="${name}"]`);
|
||||
const checked = Array.from(radios).find(r => r.checked);
|
||||
data[name] = checked ? checked.value : null;
|
||||
handled.add(name);
|
||||
return;
|
||||
}
|
||||
if (el.type === 'checkbox') {
|
||||
// if this checkbox was created as a toggle with data-yes/data-no, return the string
|
||||
const yes = el.dataset && el.dataset.yes;
|
||||
const no = el.dataset && el.dataset.no;
|
||||
if (yes !== undefined && no !== undefined) {
|
||||
data[name] = el.checked ? yes : no;
|
||||
} else {
|
||||
data[name] = el.checked ? 1 : 0;
|
||||
}
|
||||
handled.add(name);
|
||||
return;
|
||||
}
|
||||
if (el.type === 'number' || el.type === 'range') {
|
||||
const v = el.value;
|
||||
data[name] = v === '' ? null : Number(v);
|
||||
handled.add(name);
|
||||
return;
|
||||
}
|
||||
// select, text, textarea, etc.
|
||||
data[name] = el.value === '' ? null : el.value;
|
||||
handled.add(name);
|
||||
});
|
||||
return data;
|
||||
}
|
||||
|
||||
// Map compact option values to human-friendly labels used in the canonical example
|
||||
const VALUE_LABELS = {
|
||||
// general likert / frequency
|
||||
'very_much': 'A great deal',
|
||||
'somewhat': 'Somewhat',
|
||||
'not_much': 'Not much',
|
||||
'not_at_all': 'Not at all',
|
||||
|
||||
'strongly_agree': 'Strongly agree',
|
||||
'agree': 'Agree',
|
||||
'neutral': 'Neutral',
|
||||
'disagree': 'Disagree',
|
||||
'strongly_disagree': 'Strongly disagree',
|
||||
|
||||
'very_trust': 'Very trust',
|
||||
'somewhat_trust': 'Somewhat trust',
|
||||
'somewhat_distrust': 'Somewhat distrust',
|
||||
'very_distrust': 'Very distrust',
|
||||
|
||||
'daily': 'Daily',
|
||||
'weekly': 'Weekly',
|
||||
'monthly': 'Monthly',
|
||||
'rarely': 'Rarely',
|
||||
'never': 'Never',
|
||||
|
||||
'female': 'Female',
|
||||
'male': 'Male',
|
||||
'other': 'Other',
|
||||
'prefer_not': 'Prefer not to say',
|
||||
|
||||
'getting_better': 'Getting better',
|
||||
'staying_same': 'Staying the same',
|
||||
'getting_worse': 'Getting worse',
|
||||
|
||||
'often': 'Often',
|
||||
'sometimes': 'Sometimes',
|
||||
|
||||
'improved': 'Improved',
|
||||
'same': 'Same',
|
||||
'worse': 'Worse',
|
||||
'better': 'Better',
|
||||
|
||||
'rural': 'Rural',
|
||||
'small_town': 'Small town',
|
||||
'suburb': 'Suburb',
|
||||
'city': 'City',
|
||||
|
||||
// toggles / yes-no
|
||||
'yes': 'Yes',
|
||||
'no': 'No'
|
||||
};
|
||||
|
||||
// SERVER_VALUE_LABELS will hold the variable-level mappings returned from the server
|
||||
// structure: { varName: { codeStr: label, ... }, ... }
|
||||
let SERVER_VALUE_LABELS = {};
|
||||
// FIELD_LABELS maps our stable field name (input.name) -> human-friendly label
|
||||
// FIELD_INTERNALS maps stable field name -> server-provided internal_label (snake_case id)
|
||||
// populated when we render the variables list so we can use internal_label when sending
|
||||
// the combined text (and still show human-friendly labels in the UI).
|
||||
let FIELD_LABELS = {};
|
||||
let FIELD_INTERNALS = {};
|
||||
// FIELD_VARNAMES maps stable field name -> canonical variable id (as returned in variables[].id)
|
||||
// we need this so we can look up SERVER_VALUE_LABELS by the canonical id when resolving labels
|
||||
let FIELD_VARNAMES = {};
|
||||
// VAR_VALUES stores the values map returned in /variables for each canonical var id
|
||||
// structure: { varId: { code: label, ... }, ... }
|
||||
let VAR_VALUES = {};
|
||||
|
||||
// Load server-provided variable metadata and render the entire form from it.
|
||||
(async function loadVariablesAndRender() {
|
||||
try {
|
||||
// Load variable metadata (ordered list)
|
||||
const varsResp = await fetch('/variables');
|
||||
if (!varsResp.ok) throw new Error('Could not load /variables');
|
||||
const variables = await varsResp.json();
|
||||
|
||||
// load server-side value labels for quick lookup too
|
||||
const vlResp = await fetch('/value_labels');
|
||||
if (vlResp.ok) {
|
||||
SERVER_VALUE_LABELS = await vlResp.json();
|
||||
}
|
||||
|
||||
const grid = document.getElementById('dynamic-grid');
|
||||
grid.innerHTML = '';
|
||||
console.log('[survey] Loaded', variables.length, 'variables from /variables');
|
||||
|
||||
// heuristic: certain variable name prefixes indicate a range/numeric field
|
||||
function looksLikeRange(varId) {
|
||||
return varId.startsWith('how_many_days') || varId.startsWith('age_') || varId.startsWith('age') || varId.startsWith('age_recoded');
|
||||
}
|
||||
|
||||
variables.forEach(v => {
|
||||
const row = document.createElement('div');
|
||||
row.className = 'form-row';
|
||||
|
||||
const lbl = document.createElement('label');
|
||||
lbl.textContent = v.label || v.id;
|
||||
row.appendChild(lbl);
|
||||
|
||||
const wrap = document.createElement('div');
|
||||
wrap.className = 'input-wrap';
|
||||
// stable field name for submission: use snake_case of id
|
||||
const fieldName = String(v.id).toLowerCase().replace(/[^a-z0-9]+/g, '_');
|
||||
wrap.setAttribute('data-field', fieldName);
|
||||
wrap.setAttribute('data-var', v.id);
|
||||
// remember the human label and internal_label for display and submission
|
||||
FIELD_LABELS[fieldName] = v.label || v.id;
|
||||
FIELD_INTERNALS[fieldName] = v.internal_label || fieldName;
|
||||
FIELD_VARNAMES[fieldName] = v.id;
|
||||
VAR_VALUES[v.id] = v.values || {};
|
||||
console.log('[survey] FIELD_LABELS set:', fieldName, '=>', FIELD_LABELS[fieldName]);
|
||||
console.log('[survey] FIELD_INTERNALS set:', fieldName, '=>', FIELD_INTERNALS[fieldName]);
|
||||
|
||||
// if we have value labels for this var, render radios or toggles
|
||||
const vals = v.values || {};
|
||||
const codes = Object.keys(vals || {});
|
||||
if (codes.length === 0) {
|
||||
// no canonical values: choose text or range based on heuristics
|
||||
if (looksLikeRange(v.id)) {
|
||||
const inp = document.createElement('input');
|
||||
inp.type = 'range';
|
||||
inp.name = fieldName;
|
||||
inp.id = fieldName;
|
||||
inp.min = 0;
|
||||
inp.max = 30;
|
||||
inp.step = 1;
|
||||
wrap.appendChild(inp);
|
||||
const span = document.createElement('span');
|
||||
span.className = 'range-value';
|
||||
span.setAttribute('data-for', fieldName);
|
||||
span.textContent = '0';
|
||||
wrap.appendChild(span);
|
||||
} else {
|
||||
const inp = document.createElement('input');
|
||||
inp.type = 'text';
|
||||
inp.name = fieldName;
|
||||
inp.id = fieldName;
|
||||
wrap.appendChild(inp);
|
||||
}
|
||||
} else {
|
||||
// render canonical codes -> labels; detect binary numeric yes/no
|
||||
const numericCodes = codes.map(c => Number(String(c))).filter(n => !Number.isNaN(n));
|
||||
const isBinary = (codes.length === 2) && (numericCodes.includes(0) || numericCodes.includes(1) || numericCodes.includes(2));
|
||||
if (isBinary) {
|
||||
const yesCode = codes.find(c => Number(c) === 1) || codes.find(c => Number(c) === 2) || codes[0];
|
||||
const noCode = codes.find(c => c !== yesCode) || null;
|
||||
const wrapper = document.createElement('div');
|
||||
const label = document.createElement('label');
|
||||
label.className = 'toggle-switch';
|
||||
const input = document.createElement('input');
|
||||
input.type = 'checkbox';
|
||||
input.name = fieldName;
|
||||
input.setAttribute('data-yes', yesCode.replace(/\.0$/, ''));
|
||||
input.setAttribute('data-no', noCode ? noCode.replace(/\.0$/, '') : '0');
|
||||
const slider = document.createElement('span');
|
||||
slider.className = 'slider';
|
||||
label.appendChild(input);
|
||||
label.appendChild(slider);
|
||||
wrapper.appendChild(label);
|
||||
const small = document.createElement('small');
|
||||
small.style.marginLeft = '10px';
|
||||
small.textContent = vals[yesCode] || 'Yes';
|
||||
wrapper.appendChild(small);
|
||||
wrap.appendChild(wrapper);
|
||||
} else {
|
||||
const likert = document.createElement('div');
|
||||
likert.className = 'likert';
|
||||
codes.slice().sort((a,b) => {
|
||||
const na = Number(a), nb = Number(b);
|
||||
if (!Number.isNaN(na) && !Number.isNaN(nb)) return na - nb;
|
||||
return String(a).localeCompare(String(b));
|
||||
}).forEach(code => {
|
||||
const label = document.createElement('label');
|
||||
const input = document.createElement('input');
|
||||
input.type = 'radio';
|
||||
input.name = fieldName;
|
||||
input.value = String(code).replace(/\.0$/, '');
|
||||
label.appendChild(input);
|
||||
label.appendChild(document.createTextNode(' ' + (vals[code] || code)));
|
||||
likert.appendChild(label);
|
||||
});
|
||||
wrap.appendChild(likert);
|
||||
}
|
||||
}
|
||||
|
||||
row.appendChild(wrap);
|
||||
grid.appendChild(row);
|
||||
});
|
||||
|
||||
// init sliders to show values
|
||||
initSliders();
|
||||
|
||||
} catch (e) {
|
||||
console.warn('Failed to load variables:', e && e.message ? e.message : e);
|
||||
}
|
||||
})();
|
||||
|
||||
// Render inputs for elements with data-var attribute using SERVER_VALUE_LABELS.
|
||||
// For each container with data-var="<varname>", we create controls based on the mapping:
|
||||
// - If the mapping looks binary (codes like 0/1) we render a toggle checkbox (data-yes/data-no)
|
||||
// - Otherwise we render radio inputs for each code->label pair (values are code without .0)
|
||||
function renderVariableControls() {
|
||||
console.log('[survey] renderVariableControls start, SERVER_VALUE_LABELS keys:', Object.keys(SERVER_VALUE_LABELS).length);
|
||||
const containers = Array.from(document.querySelectorAll('[data-var]'));
|
||||
containers.forEach(container => {
|
||||
const varName = container.getAttribute('data-var');
|
||||
if (!varName) return;
|
||||
const map = SERVER_VALUE_LABELS[varName];
|
||||
if (!map || typeof map !== 'object') return;
|
||||
// clear existing content
|
||||
container.innerHTML = '';
|
||||
const codes = Object.keys(map);
|
||||
// detect binary mapping (common pattern: 0.0/1.0 or 1.0/2.0 for yes/no)
|
||||
const numericCodes = codes.map(c => Number(String(c)) ).filter(n => !Number.isNaN(n));
|
||||
const isBinary = (codes.length === 2) && (numericCodes.includes(0) || numericCodes.includes(1) || numericCodes.includes(2));
|
||||
const fieldName = container.getAttribute('data-field') || varName;
|
||||
|
||||
if (isBinary) {
|
||||
// determine the 'yes' code (prefer 1, then 2)
|
||||
const yesCode = codes.find(c => Number(c) === 1) || codes.find(c => Number(c) === 2) || codes[0];
|
||||
const noCode = codes.find(c => c !== yesCode) || null;
|
||||
const yesLabel = map[yesCode] || 'Yes';
|
||||
const noLabel = noCode ? map[noCode] : 'No';
|
||||
// checkbox: when checked -> yes (1), unchecked -> no (0)
|
||||
const wrapper = document.createElement('div');
|
||||
const label = document.createElement('label');
|
||||
label.className = 'toggle-switch';
|
||||
const input = document.createElement('input');
|
||||
input.type = 'checkbox';
|
||||
input.name = fieldName;
|
||||
// encode yes/no as data attributes so collectForm interprets correctly
|
||||
input.setAttribute('data-yes', yesCode.replace(/\.0$/, ''));
|
||||
input.setAttribute('data-no', noCode ? noCode.replace(/\.0$/, '') : '0');
|
||||
const slider = document.createElement('span');
|
||||
slider.className = 'slider';
|
||||
label.appendChild(input);
|
||||
label.appendChild(slider);
|
||||
wrapper.appendChild(label);
|
||||
const small = document.createElement('small');
|
||||
small.style.marginLeft = '10px';
|
||||
small.textContent = yesLabel;
|
||||
wrapper.appendChild(small);
|
||||
container.appendChild(wrapper);
|
||||
} else {
|
||||
// render radios; sort codes numerically when possible for stable ordering
|
||||
const sorted = codes.slice().sort((a, b) => {
|
||||
const na = Number(a), nb = Number(b);
|
||||
if (!Number.isNaN(na) && !Number.isNaN(nb)) return na - nb;
|
||||
return String(a).localeCompare(String(b));
|
||||
});
|
||||
sorted.forEach(code => {
|
||||
const labelText = map[code];
|
||||
const valueToken = String(code).replace(/\.0$/, '');
|
||||
const label = document.createElement('label');
|
||||
const input = document.createElement('input');
|
||||
input.type = 'radio';
|
||||
input.name = fieldName;
|
||||
input.value = valueToken;
|
||||
label.appendChild(input);
|
||||
label.appendChild(document.createTextNode(' ' + labelText));
|
||||
container.appendChild(label);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Helper: return the human-friendly label for a given field and token.
|
||||
// Lookup order:
|
||||
// 1) If SERVER_VALUE_LABELS has an entry for fieldName and a matching code -> return label.
|
||||
// We attempt exact match on token, token+'.0', and token with trailing '.0' removed.
|
||||
// 2) If VALUE_LABELS has an entry for token -> return that.
|
||||
// 3) Else return null to indicate no mapping.
|
||||
function labelFor(fieldName, token) {
|
||||
if (!token || token === 'nan') return null;
|
||||
// server-side per-variable lookup
|
||||
// determine canonical var id for this field so we can resolve per-variable value labels
|
||||
const varKey = FIELD_VARNAMES[fieldName] || fieldName;
|
||||
// Prefer VAR_VALUES (values shipped with /variables) as it contains the recoded labels
|
||||
if (VAR_VALUES && VAR_VALUES[varKey]) {
|
||||
const map = VAR_VALUES[varKey];
|
||||
// 1) If the token directly matches a code key, return the server label
|
||||
if (map.hasOwnProperty(token)) return map[token];
|
||||
// try token + '.0' and stripped
|
||||
if (map.hasOwnProperty(token + '.0')) return map[token + '.0'];
|
||||
const strippedToken = token.replace(/\.0$/, '');
|
||||
if (map.hasOwnProperty(strippedToken)) return map[strippedToken];
|
||||
// 2) If the token maps locally to a human label, see if the server map contains that label
|
||||
// (this lets us map semantic tokens like 'very_much' -> 'A great deal' if the server
|
||||
// contains that exact human label for the field)
|
||||
if (VALUE_LABELS.hasOwnProperty(token)) {
|
||||
const human = VALUE_LABELS[token];
|
||||
// try exact value match (case-sensitive); also try trimmed/case-insensitive
|
||||
for (const k of Object.keys(map)) {
|
||||
const serverLabel = map[k];
|
||||
if (!serverLabel) continue;
|
||||
if (serverLabel === human) return serverLabel;
|
||||
if (serverLabel.trim().toLowerCase() === human.trim().toLowerCase()) return serverLabel;
|
||||
}
|
||||
}
|
||||
// 3) additional numeric equivalence handled above
|
||||
// 4) numeric equivalence match
|
||||
try {
|
||||
const numToken = Number(token);
|
||||
if (!Number.isNaN(numToken)) {
|
||||
for (const k of Object.keys(map)) {
|
||||
if (!isNaN(Number(k)) && Number(k) === numToken) return map[k];
|
||||
}
|
||||
}
|
||||
} catch (e) { /* ignore */ }
|
||||
}
|
||||
// fallback to server-provided per-variable mapping if present
|
||||
if (SERVER_VALUE_LABELS && SERVER_VALUE_LABELS[varKey]) {
|
||||
const map = SERVER_VALUE_LABELS[varKey];
|
||||
if (map.hasOwnProperty(token)) return map[token];
|
||||
if (map.hasOwnProperty(token + '.0')) return map[token + '.0'];
|
||||
const stripped2 = token.replace(/\.0$/, '');
|
||||
if (map.hasOwnProperty(stripped2)) return map[stripped2];
|
||||
try {
|
||||
const numToken = Number(token);
|
||||
if (!Number.isNaN(numToken)) {
|
||||
for (const k of Object.keys(map)) {
|
||||
if (!isNaN(Number(k)) && Number(k) === numToken) return map[k];
|
||||
}
|
||||
}
|
||||
} catch (e) { /* ignore */ }
|
||||
}
|
||||
// fallback to global VALUE_LABELS exact match
|
||||
if (VALUE_LABELS.hasOwnProperty(token)) {
|
||||
console.log('[survey] labelFor fallback global VALUE_LABELS for', token, '->', VALUE_LABELS[token]);
|
||||
return VALUE_LABELS[token];
|
||||
}
|
||||
console.log('[survey] labelFor no mapping for', fieldName, token);
|
||||
return null;
|
||||
}
|
||||
|
||||
document.getElementById('predict').addEventListener('click', async () => {
|
||||
const d = collectForm();
|
||||
// Build a combined string of key:value for all form fields in a stable order
|
||||
// Missing values become 'nan' to mirror downstream expectations
|
||||
const parts = [];
|
||||
Object.keys(d).forEach(k => {
|
||||
let v = d[k];
|
||||
if (v === null || v === undefined || v === '') {
|
||||
v = 'nan';
|
||||
} else if (typeof v === 'object') {
|
||||
v = JSON.stringify(v);
|
||||
} else {
|
||||
v = String(v);
|
||||
}
|
||||
|
||||
// Attempt to map token to friendly label using per-field labels first,
|
||||
// then global fallback.
|
||||
if (typeof v === 'string') {
|
||||
const fieldLabelVal = labelFor(k, v);
|
||||
if (fieldLabelVal) {
|
||||
v = fieldLabelVal;
|
||||
}
|
||||
}
|
||||
|
||||
// Use server-provided internal_label (if available) as the key in the combined text
|
||||
// Fall back to the human-friendly FIELD_LABELS or the field name
|
||||
const internalKey = FIELD_INTERNALS[k] || FIELD_LABELS[k] || k;
|
||||
parts.push(`${internalKey}: ${v}`);
|
||||
});
|
||||
const combined = parts.join(' | ');
|
||||
|
||||
output.textContent = 'Sending...';
|
||||
console.log('[survey] Sending combined payload to /predict');
|
||||
try {
|
||||
// Build an internal-keyed mapping to send alongside the combined text
|
||||
const request_parsed_internal = {};
|
||||
Object.keys(d).forEach(k => {
|
||||
const internalKey = FIELD_INTERNALS[k] || FIELD_LABELS[k] || k;
|
||||
request_parsed_internal[internalKey] = d[k] === null || d[k] === undefined || d[k] === '' ? 'nan' : d[k];
|
||||
});
|
||||
|
||||
const res = await fetch('/predict', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text: combined })
|
||||
});
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
output.textContent = `Error ${res.status}: ${text}`;
|
||||
return;
|
||||
}
|
||||
const json = await res.json();
|
||||
output.textContent = JSON.stringify(json, null, 2);
|
||||
} catch (err) {
|
||||
output.textContent = 'Network error: ' + (err && err.message ? err.message : String(err));
|
||||
}
|
||||
});
|
||||
|
||||
// Self-test: programmatically choose first option in each likert group, enable toggles and fill a couple fields,
|
||||
// then trigger the Predict flow so testers can quickly exercise the endpoint.
|
||||
function fillSelfTest() {
|
||||
// pick first radio of each group
|
||||
console.log('[survey] fillSelfTest starting');
|
||||
const radios = Array.from(document.querySelectorAll('input[type=radio]'));
|
||||
const grouped = {};
|
||||
radios.forEach(r => { if (!grouped[r.name]) grouped[r.name] = []; grouped[r.name].push(r); });
|
||||
Object.values(grouped).forEach(g => { if (g.length) g[0].checked = true; });
|
||||
|
||||
// enable all toggle checkboxes (rendered as .toggle-switch input)
|
||||
document.querySelectorAll('.toggle-switch input[type=checkbox]').forEach(ch => ch.checked = true);
|
||||
|
||||
// fill first text inputs
|
||||
const firstText = document.querySelector('input[type=text]'); if (firstText) firstText.value = 'Sample';
|
||||
|
||||
// set ranges to mid values
|
||||
document.querySelectorAll('input[type=range]').forEach(r => {
|
||||
const min = Number(r.min || 0);
|
||||
const max = Number(r.max || 10);
|
||||
r.value = Math.floor((min + max) / 2);
|
||||
r.dispatchEvent(new Event('input'));
|
||||
});
|
||||
|
||||
console.log('[survey] fillSelfTest done populating fields');
|
||||
// trigger predict
|
||||
const btn = document.getElementById('predict');
|
||||
if (btn) btn.click();
|
||||
}
|
||||
|
||||
document.getElementById('selftest').addEventListener('click', fillSelfTest);
|
||||
|
||||
// Keep range-value spans in sync with range inputs
|
||||
function initSliders() {
|
||||
const ranges = Array.from(document.querySelectorAll('input[type=range]'));
|
||||
ranges.forEach(r => {
|
||||
const span = document.querySelector(`.range-value[data-for="${r.id}"]`);
|
||||
const update = () => {
|
||||
if (span) span.textContent = r.value;
|
||||
// set ARIA value
|
||||
r.setAttribute('aria-valuenow', r.value);
|
||||
};
|
||||
// initialize
|
||||
update();
|
||||
r.addEventListener('input', update);
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', initSliders);
|
||||
</script>
|
||||
|
||||
<p><small>If you need CSV export or different field typing, tell me and I'll add it.</small></p>
|
||||
</div>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -0,0 +1,54 @@
|
||||
import requests
|
||||
|
||||
def get_ollama_summary(text, model="granite3.1-moe:1b"):
|
||||
url = "http://localhost:11434/v1/chat/completions"
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant that summarizes text."},
|
||||
{"role": "user", "content": text}
|
||||
]
|
||||
}
|
||||
response = requests.post(url, json=payload)
|
||||
response.raise_for_status()
|
||||
return response.json()["choices"][0]["message"]["content"]
|
||||
|
||||
def get_ollama_embedding(text, model="nomic-embed-text"):
|
||||
url = "http://localhost:11434/api/embeddings"
|
||||
payload = {
|
||||
"model": model,
|
||||
"prompt": text
|
||||
}
|
||||
response = requests.post(url, json=payload)
|
||||
response.raise_for_status()
|
||||
return response.json()["embedding"]
|
||||
|
||||
def parse_combined_text_to_dict(s: str) -> dict:
|
||||
"""Parse a string of the form 'key1: value1 | key2: value2' into a dict.
|
||||
|
||||
Rules:
|
||||
- Split on ' | ' to get key:value segments.
|
||||
- For each segment, split on the first ':' to separate key and value.
|
||||
- Strip whitespace. If a value is 'nan' (case-insensitive) or empty, use None.
|
||||
- Return a dict mapping keys to values or None.
|
||||
"""
|
||||
if not s:
|
||||
return {}
|
||||
result = {}
|
||||
parts = [p.strip() for p in s.split("|")]
|
||||
for part in parts:
|
||||
if not part:
|
||||
continue
|
||||
# split on the first colon
|
||||
if ':' in part:
|
||||
k, v = part.split(':', 1)
|
||||
k = k.strip()
|
||||
v = v.strip()
|
||||
if v.lower() == 'nan' or v == '':
|
||||
result[k] = None
|
||||
else:
|
||||
result[k] = v
|
||||
else:
|
||||
# fallback: store whole segment under a numeric key
|
||||
result.setdefault('_extra', []).append(part)
|
||||
return result
|
||||
Reference in New Issue
Block a user