jameshtwose/matti_jms_collabs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
matti_jms_collabs/covid19/uni_maryland_fb_covid19_windowed_NLTSA.ipynb
T

4.4 MiB
Raw Permalink Blame History

Windowed NLTSA in COVID19 data

In [1]:
import requests
import json
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from jmspack.utils import apply_scaling, JmsColors
from sklearn import decomposition
from sklearn.linear_model import LinearRegression
from scipy.stats import norm
import time
In [2]:
start = time.time()
In [3]:
if "jms_style_sheet" in plt.style.available:
    plt.style.use("jms_style_sheet")
In [4]:
# request data from api
# https://covidmap.umd.edu/api/resources?indicator={indicator}&type={type}&country={country}&date={date}
# type: daily/ smoothed
country = "Finland"
type = "daily"
response = requests.get(f"https://covidmap.umd.edu/api/resources?indicator=mask&type={type}&country={country}&daterange=20201115-20220111").text

#convert json data to dic data for use!
jsonData = json.loads(response)

# convert to pandas dataframe
df = pd.DataFrame.from_dict(jsonData['data'])
In [5]:
df.head()
Out[5]:
<style scoped=""> .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } </style>
percent_mc mc_se percent_mc_unw mc_se_unw sample_size country iso_code gid_0 survey_date
0 0.680969 0.031385 0.698361 0.026281 305.0 Finland FIN FIN 20201115
1 0.657667 0.034046 0.701493 0.025002 335.0 Finland FIN FIN 20201116
2 0.665093 0.028629 0.691667 0.024339 360.0 Finland FIN FIN 20201117
3 0.595547 0.028435 0.645514 0.022377 457.0 Finland FIN FIN 20201118
4 0.648205 0.025742 0.668687 0.021156 495.0 Finland FIN FIN 20201119
In [6]:
endpoints = [
    # "mask",
"contact",
"work_outside_home_1d",	
"shop_1d",
"restaurant_1d",
"spent_time_1d",
"large_event_1d",
"public_transit_1d",
"wash_hands_24h_1to2",
"access_wash",
"wash_hands_24h_3to6",
"wash_hands_24h_7orMore",
"activity_large_event",
"activity_public_transit",
"activity_restaurant_bar",
"activity_shop",
"activity_spent_time",
"activity_work_outside_home",
"mask_work_outside_home_1d",
"mask_shop_1d",
"mask_restaurant_1d",
"mask_spent_time_1d",
# "mask_large_event_1d",
"mask_public_transit_1d"]
In [7]:
for endpoint in endpoints:
    api_string = f"https://covidmap.umd.edu/api/resources?indicator={endpoint}&type={type}&country={country}&daterange=20201115-20220111"
    response = requests.get(api_string).text

    #convert json data to dic data for use!
    jsonData = json.loads(response)

    if "error" in jsonData:
        print(f"Endpoint: {endpoint} failed")
    else:
        # convert to pandas dataframe
        tmp_df = pd.DataFrame.from_dict(jsonData['data'])

        if tmp_df.shape[0] > 0:
            # df = pd.merge(df, tmp_df, on=["sample_size", "country", "iso_code", "gid_0", "survey_date"])
            df = pd.concat([df, tmp_df.drop(["sample_size", "country", "iso_code", "gid_0", "survey_date"], axis=1)], axis=1)

Endpoint: work_outside_home_1d failed
Endpoint: shop_1d failed
Endpoint: restaurant_1d failed
Endpoint: spent_time_1d failed
Endpoint: large_event_1d failed
Endpoint: public_transit_1d failed
Endpoint: wash_hands_24h_1to2 failed
In [8]:
df["survey_date"] = pd.to_datetime(df["survey_date"])
In [9]:
display(df.head()); df.shape
<style scoped=""> .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } </style>
percent_mc mc_se percent_mc_unw mc_se_unw sample_size country iso_code gid_0 survey_date percent_dc ... pct_mask_shop_1d_unw mask_shop_1d_se_unw pct_mask_restaurant_1d mask_restaurant_1d_se pct_mask_restaurant_1d_unw mask_restaurant_1d_se_unw pct_mask_spent_time_1d mask_spent_time_1d_se pct_mask_spent_time_1d_unw mask_spent_time_1d_se_unw
0 0.680969 0.031385 0.698361 0.026281 305.0 Finland FIN FIN 2020-11-15 0.545597 ... 0.908517 0.016192 0.446433 0.064626 0.435185 0.047707 0.331995 0.041313 0.415789 0.035756
1 0.657667 0.034046 0.701493 0.025002 335.0 Finland FIN FIN 2020-11-16 0.568379 ... 0.886731 0.018029 0.265386 0.051648 0.310000 0.046249 0.325892 0.044393 0.383784 0.035754
2 0.665093 0.028629 0.691667 0.024339 360.0 Finland FIN FIN 2020-11-17 0.556073 ... 0.926984 0.014658 0.306528 0.050647 0.324324 0.044432 0.306346 0.047284 0.338624 0.034423
3 0.595547 0.028435 0.645514 0.022377 457.0 Finland FIN FIN 2020-11-18 0.520839 ... 0.913495 0.016536 0.334798 0.058626 0.362745 0.047606 0.200042 0.029019 0.253731 0.030693
4 0.648205 0.025742 0.668687 0.021156 495.0 Finland FIN FIN 2020-11-19 0.512703 ... 0.930556 0.014979 0.334441 0.048391 0.363636 0.043731 0.362410 0.044485 0.347594 0.034824

5 rows × 65 columns

Out[9]:
(422, 65)
In [10]:
drop_columns = ["sample_size", "country", "iso_code", "gid_0", "survey_date"]
_ = plt.figure(figsize=(30, 7))
_ = sns.heatmap(df
.drop(drop_columns[:-1], axis=1)
.set_index("survey_date")
.T
)
No description has been provided for this image
In [11]:
_ = plt.figure(figsize=(30, 7))
_ = sns.heatmap(df
.drop(drop_columns[:-1], axis=1)
.set_index("survey_date")
.pipe(apply_scaling)
.T
)
No description has been provided for this image
In [12]:
df_melt = (df
.drop(drop_columns[:-1], axis=1)
.set_index("survey_date")
.pipe(apply_scaling)
.reset_index()
.melt(id_vars="survey_date"))
In [13]:
_ = plt.figure(figsize=(30, 7))
_ = sns.lineplot(data=df_melt, x="survey_date", y="value", hue="variable", legend=False)
No description has been provided for this image
In [14]:
g = sns.FacetGrid(df_melt, col="variable", col_wrap=3, aspect=2)
_ = g.map(sns.lineplot, "survey_date", "value")
No description has been provided for this image
In [15]:
def summary_window_FUN(x: pd.DataFrame, window_size: int = 7, user_func=decomposition.PCA, kwargs: dict = {}):
    window_range = np.arange(0, len(x)-window_size, window_size)

    cp_df = pd.DataFrame()
    for window_begin in window_range:
        current_cp_df = pd.DataFrame(user_func(n_components=None, **kwargs)
                                     .fit_transform(x.iloc[window_begin: window_begin + window_size])).iloc[:, 0]
        cp_df = pd.concat([cp_df, current_cp_df])

    return cp_df.rename(columns={0: f"windowed_{user_func.__name__}"}).reset_index(drop=True)
In [16]:
decomps_list = [decomposition.DictionaryLearning,
                decomposition.FactorAnalysis,
                decomposition.FastICA,
                # decomposition.IncrementalPCA,
                decomposition.KernelPCA,
                decomposition.NMF,
                decomposition.PCA
                ]

window_choice=28
if window_choice == 28:
    date_end = window_choice-18
else:
    date_end = window_choice-4

plot_df=pd.concat([summary_window_FUN(df.drop(drop_columns, axis=1).dropna(axis=0).pipe(apply_scaling), window_size=window_choice, user_func=window_function,
                                        kwargs={"random_state": 42}) for window_function in decomps_list],
                    axis=1).set_index(df.dropna(axis=0)["survey_date"][:-date_end]).reset_index().melt(id_vars="survey_date")

/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_dict_learning.py:207: RuntimeWarning: Orthogonal matching pursuit ended prematurely due to linear dependence in the dictionary. The requested precision might not have been met.
  new_code = orthogonal_mp_gram(
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_dict_learning.py:207: RuntimeWarning: Orthogonal matching pursuit ended prematurely due to linear dependence in the dictionary. The requested precision might not have been met.
  new_code = orthogonal_mp_gram(
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_dict_learning.py:207: RuntimeWarning: Orthogonal matching pursuit ended prematurely due to linear dependence in the dictionary. The requested precision might not have been met.
  new_code = orthogonal_mp_gram(
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_dict_learning.py:207: RuntimeWarning: Orthogonal matching pursuit ended prematurely due to linear dependence in the dictionary. The requested precision might not have been met.
  new_code = orthogonal_mp_gram(
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:289: FutureWarning: The 'init' value, when 'init=None' and n_components is less than n_samples and n_features, will be changed from 'nndsvd' to 'nndsvda' in 1.1 (renaming of 0.26).
  warnings.warn(
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:289: FutureWarning: The 'init' value, when 'init=None' and n_components is less than n_samples and n_features, will be changed from 'nndsvd' to 'nndsvda' in 1.1 (renaming of 0.26).
  warnings.warn(
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:1637: ConvergenceWarning: Maximum number of iterations 200 reached. Increase it to improve convergence.
  warnings.warn(
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:289: FutureWarning: The 'init' value, when 'init=None' and n_components is less than n_samples and n_features, will be changed from 'nndsvd' to 'nndsvda' in 1.1 (renaming of 0.26).
  warnings.warn(
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:1637: ConvergenceWarning: Maximum number of iterations 200 reached. Increase it to improve convergence.
  warnings.warn(
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:289: FutureWarning: The 'init' value, when 'init=None' and n_components is less than n_samples and n_features, will be changed from 'nndsvd' to 'nndsvda' in 1.1 (renaming of 0.26).
  warnings.warn(
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:1637: ConvergenceWarning: Maximum number of iterations 200 reached. Increase it to improve convergence.
  warnings.warn(
In [17]:
_ = plt.figure(figsize=(20, 7))
_ = sns.lineplot(data=plot_df, x="survey_date", y="value", hue="variable", legend=True)
No description has been provided for this image
In [18]:
g = sns.FacetGrid(plot_df, col="variable", col_wrap=2, aspect=2, sharey=False)
_ = g.map(sns.lineplot, "survey_date", "value")
No description has been provided for this image
In [19]:
# def summary_window_slope(x: pd.DataFrame, window_size: int = 7, user_func=decomposition.PCA, kwargs: dict = {}):
#     window_range = np.arange(0, len(x)-window_size, window_size)

#     cp_df = pd.DataFrame()
#     for window_begin in window_range:
#         current_cp_df = pd.DataFrame(user_func(n_components=None, **kwargs)
#                                      .fit_transform(x.iloc[window_begin: window_begin + window_size])).iloc[:, 0]
#         cp_df = pd.concat([cp_df, current_cp_df])

#     return cp_df.rename(columns={0: f"windowed_{user_func.__name__}"}).reset_index(drop=True)
In [20]:
tmp = apply_scaling(plot_df
.loc[plot_df["variable"] == "windowed_PCA", ["survey_date", "value"]]
.set_index("survey_date"), 
method="Standard")
In [21]:
significance_level=0.1
sig_plot_df = tmp.mask(
        tmp > norm.ppf(1 - significance_level), 1
).mask(tmp <= norm.ppf(1 - significance_level), 0)
In [22]:
norm.ppf(1 - significance_level)
norm.ppf(1 - significance_level)
Out[22]:
1.2815515655446004
In [23]:
# _ = sns.heatmap(sig_plot_df.T)
In [24]:
_ = plt.figure(figsize=(15, 5))
_ = plt.plot(tmp)
for sig_date in sig_plot_df[sig_plot_df==True].dropna().index:
    _ = plt.axvline(sig_date, ls="--", c=JmsColors.YELLOW)
No description has been provided for this image
In [25]:
window_size=7
window_range = np.arange(0, len(tmp)-window_size, window_size)
coefs_list = []
for window_begin in window_range:
    win_df = tmp.iloc[window_begin: window_begin + window_size].iloc[:, 0]
    mod = LinearRegression()
    _ = mod.fit(X=win_df.index.to_numpy().reshape(-1, 1), y=win_df)
    coefs_list.append(mod.coef_[0])
In [26]:
_ = plt.plot(pd.Series(coefs_list))
No description has been provided for this image
In [27]:
print(f"time elapsed: {time.time()-start} seconds")

time elapsed: 170.37806701660156 seconds
Reference in New Issue View Git Blame Copy Permalink