James
4.4 MiB
4.4 MiB
Windowed NLTSA in COVID19 data¶
In [1]:
import requests import json import pandas as pd import numpy as np import seaborn as sns import matplotlib.pyplot as plt from jmspack.utils import apply_scaling, JmsColors from sklearn import decomposition from sklearn.linear_model import LinearRegression from scipy.stats import norm import time
In [2]:
start = time.time()
In [3]:
if "jms_style_sheet" in plt.style.available: plt.style.use("jms_style_sheet")
In [4]:
# request data from api # https://covidmap.umd.edu/api/resources?indicator={indicator}&type={type}&country={country}&date={date} # type: daily/ smoothed country = "Finland" type = "daily" response = requests.get(f"https://covidmap.umd.edu/api/resources?indicator=mask&type={type}&country={country}&daterange=20201115-20220111").text #convert json data to dic data for use! jsonData = json.loads(response) # convert to pandas dataframe df = pd.DataFrame.from_dict(jsonData['data'])
In [5]:
df.head()
Out[5]:
<style scoped="">
.dataframe tbody tr th:only-of-type {
vertical-align: middle;
}
.dataframe tbody tr th {
vertical-align: top;
}
.dataframe thead th {
text-align: right;
}
</style>
| percent_mc | mc_se | percent_mc_unw | mc_se_unw | sample_size | country | iso_code | gid_0 | survey_date | |
|---|---|---|---|---|---|---|---|---|---|
| 0 | 0.680969 | 0.031385 | 0.698361 | 0.026281 | 305.0 | Finland | FIN | FIN | 20201115 |
| 1 | 0.657667 | 0.034046 | 0.701493 | 0.025002 | 335.0 | Finland | FIN | FIN | 20201116 |
| 2 | 0.665093 | 0.028629 | 0.691667 | 0.024339 | 360.0 | Finland | FIN | FIN | 20201117 |
| 3 | 0.595547 | 0.028435 | 0.645514 | 0.022377 | 457.0 | Finland | FIN | FIN | 20201118 |
| 4 | 0.648205 | 0.025742 | 0.668687 | 0.021156 | 495.0 | Finland | FIN | FIN | 20201119 |
In [6]:
endpoints = [ # "mask", "contact", "work_outside_home_1d", "shop_1d", "restaurant_1d", "spent_time_1d", "large_event_1d", "public_transit_1d", "wash_hands_24h_1to2", "access_wash", "wash_hands_24h_3to6", "wash_hands_24h_7orMore", "activity_large_event", "activity_public_transit", "activity_restaurant_bar", "activity_shop", "activity_spent_time", "activity_work_outside_home", "mask_work_outside_home_1d", "mask_shop_1d", "mask_restaurant_1d", "mask_spent_time_1d", # "mask_large_event_1d", "mask_public_transit_1d"]
In [7]:
for endpoint in endpoints: api_string = f"https://covidmap.umd.edu/api/resources?indicator={endpoint}&type={type}&country={country}&daterange=20201115-20220111" response = requests.get(api_string).text #convert json data to dic data for use! jsonData = json.loads(response) if "error" in jsonData: print(f"Endpoint: {endpoint} failed") else: # convert to pandas dataframe tmp_df = pd.DataFrame.from_dict(jsonData['data']) if tmp_df.shape[0] > 0: # df = pd.merge(df, tmp_df, on=["sample_size", "country", "iso_code", "gid_0", "survey_date"]) df = pd.concat([df, tmp_df.drop(["sample_size", "country", "iso_code", "gid_0", "survey_date"], axis=1)], axis=1)
Endpoint: work_outside_home_1d failed Endpoint: shop_1d failed Endpoint: restaurant_1d failed Endpoint: spent_time_1d failed Endpoint: large_event_1d failed Endpoint: public_transit_1d failed Endpoint: wash_hands_24h_1to2 failed
In [8]:
df["survey_date"] = pd.to_datetime(df["survey_date"])
In [9]:
display(df.head()); df.shape
<style scoped="">
.dataframe tbody tr th:only-of-type {
vertical-align: middle;
}
.dataframe tbody tr th {
vertical-align: top;
}
.dataframe thead th {
text-align: right;
}
</style>
| percent_mc | mc_se | percent_mc_unw | mc_se_unw | sample_size | country | iso_code | gid_0 | survey_date | percent_dc | ... | pct_mask_shop_1d_unw | mask_shop_1d_se_unw | pct_mask_restaurant_1d | mask_restaurant_1d_se | pct_mask_restaurant_1d_unw | mask_restaurant_1d_se_unw | pct_mask_spent_time_1d | mask_spent_time_1d_se | pct_mask_spent_time_1d_unw | mask_spent_time_1d_se_unw | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 0.680969 | 0.031385 | 0.698361 | 0.026281 | 305.0 | Finland | FIN | FIN | 2020-11-15 | 0.545597 | ... | 0.908517 | 0.016192 | 0.446433 | 0.064626 | 0.435185 | 0.047707 | 0.331995 | 0.041313 | 0.415789 | 0.035756 |
| 1 | 0.657667 | 0.034046 | 0.701493 | 0.025002 | 335.0 | Finland | FIN | FIN | 2020-11-16 | 0.568379 | ... | 0.886731 | 0.018029 | 0.265386 | 0.051648 | 0.310000 | 0.046249 | 0.325892 | 0.044393 | 0.383784 | 0.035754 |
| 2 | 0.665093 | 0.028629 | 0.691667 | 0.024339 | 360.0 | Finland | FIN | FIN | 2020-11-17 | 0.556073 | ... | 0.926984 | 0.014658 | 0.306528 | 0.050647 | 0.324324 | 0.044432 | 0.306346 | 0.047284 | 0.338624 | 0.034423 |
| 3 | 0.595547 | 0.028435 | 0.645514 | 0.022377 | 457.0 | Finland | FIN | FIN | 2020-11-18 | 0.520839 | ... | 0.913495 | 0.016536 | 0.334798 | 0.058626 | 0.362745 | 0.047606 | 0.200042 | 0.029019 | 0.253731 | 0.030693 |
| 4 | 0.648205 | 0.025742 | 0.668687 | 0.021156 | 495.0 | Finland | FIN | FIN | 2020-11-19 | 0.512703 | ... | 0.930556 | 0.014979 | 0.334441 | 0.048391 | 0.363636 | 0.043731 | 0.362410 | 0.044485 | 0.347594 | 0.034824 |
5 rows × 65 columns
Out[9]:
(422, 65)
In [10]:
drop_columns = ["sample_size", "country", "iso_code", "gid_0", "survey_date"] _ = plt.figure(figsize=(30, 7)) _ = sns.heatmap(df .drop(drop_columns[:-1], axis=1) .set_index("survey_date") .T )
In [11]:
_ = plt.figure(figsize=(30, 7)) _ = sns.heatmap(df .drop(drop_columns[:-1], axis=1) .set_index("survey_date") .pipe(apply_scaling) .T )
In [12]:
df_melt = (df .drop(drop_columns[:-1], axis=1) .set_index("survey_date") .pipe(apply_scaling) .reset_index() .melt(id_vars="survey_date"))
In [13]:
_ = plt.figure(figsize=(30, 7)) _ = sns.lineplot(data=df_melt, x="survey_date", y="value", hue="variable", legend=False)
In [14]:
g = sns.FacetGrid(df_melt, col="variable", col_wrap=3, aspect=2) _ = g.map(sns.lineplot, "survey_date", "value")
In [15]:
def summary_window_FUN(x: pd.DataFrame, window_size: int = 7, user_func=decomposition.PCA, kwargs: dict = {}): window_range = np.arange(0, len(x)-window_size, window_size) cp_df = pd.DataFrame() for window_begin in window_range: current_cp_df = pd.DataFrame(user_func(n_components=None, **kwargs) .fit_transform(x.iloc[window_begin: window_begin + window_size])).iloc[:, 0] cp_df = pd.concat([cp_df, current_cp_df]) return cp_df.rename(columns={0: f"windowed_{user_func.__name__}"}).reset_index(drop=True)
In [16]:
decomps_list = [decomposition.DictionaryLearning, decomposition.FactorAnalysis, decomposition.FastICA, # decomposition.IncrementalPCA, decomposition.KernelPCA, decomposition.NMF, decomposition.PCA ] window_choice=28 if window_choice == 28: date_end = window_choice-18 else: date_end = window_choice-4 plot_df=pd.concat([summary_window_FUN(df.drop(drop_columns, axis=1).dropna(axis=0).pipe(apply_scaling), window_size=window_choice, user_func=window_function, kwargs={"random_state": 42}) for window_function in decomps_list], axis=1).set_index(df.dropna(axis=0)["survey_date"][:-date_end]).reset_index().melt(id_vars="survey_date")
/Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_dict_learning.py:207: RuntimeWarning: Orthogonal matching pursuit ended prematurely due to linear dependence in the dictionary. The requested precision might not have been met. new_code = orthogonal_mp_gram( /Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_dict_learning.py:207: RuntimeWarning: Orthogonal matching pursuit ended prematurely due to linear dependence in the dictionary. The requested precision might not have been met. new_code = orthogonal_mp_gram( /Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_dict_learning.py:207: RuntimeWarning: Orthogonal matching pursuit ended prematurely due to linear dependence in the dictionary. The requested precision might not have been met. new_code = orthogonal_mp_gram( /Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_dict_learning.py:207: RuntimeWarning: Orthogonal matching pursuit ended prematurely due to linear dependence in the dictionary. The requested precision might not have been met. new_code = orthogonal_mp_gram( /Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:289: FutureWarning: The 'init' value, when 'init=None' and n_components is less than n_samples and n_features, will be changed from 'nndsvd' to 'nndsvda' in 1.1 (renaming of 0.26). warnings.warn( /Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:289: FutureWarning: The 'init' value, when 'init=None' and n_components is less than n_samples and n_features, will be changed from 'nndsvd' to 'nndsvda' in 1.1 (renaming of 0.26). warnings.warn( /Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:1637: ConvergenceWarning: Maximum number of iterations 200 reached. Increase it to improve convergence. warnings.warn( /Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:289: FutureWarning: The 'init' value, when 'init=None' and n_components is less than n_samples and n_features, will be changed from 'nndsvd' to 'nndsvda' in 1.1 (renaming of 0.26). warnings.warn( /Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:1637: ConvergenceWarning: Maximum number of iterations 200 reached. Increase it to improve convergence. warnings.warn( /Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:289: FutureWarning: The 'init' value, when 'init=None' and n_components is less than n_samples and n_features, will be changed from 'nndsvd' to 'nndsvda' in 1.1 (renaming of 0.26). warnings.warn( /Users/james/miniconda3/envs/ds_env/lib/python3.8/site-packages/sklearn/decomposition/_nmf.py:1637: ConvergenceWarning: Maximum number of iterations 200 reached. Increase it to improve convergence. warnings.warn(
In [17]:
_ = plt.figure(figsize=(20, 7)) _ = sns.lineplot(data=plot_df, x="survey_date", y="value", hue="variable", legend=True)
In [18]:
g = sns.FacetGrid(plot_df, col="variable", col_wrap=2, aspect=2, sharey=False) _ = g.map(sns.lineplot, "survey_date", "value")
In [19]:
# def summary_window_slope(x: pd.DataFrame, window_size: int = 7, user_func=decomposition.PCA, kwargs: dict = {}): # window_range = np.arange(0, len(x)-window_size, window_size) # cp_df = pd.DataFrame() # for window_begin in window_range: # current_cp_df = pd.DataFrame(user_func(n_components=None, **kwargs) # .fit_transform(x.iloc[window_begin: window_begin + window_size])).iloc[:, 0] # cp_df = pd.concat([cp_df, current_cp_df]) # return cp_df.rename(columns={0: f"windowed_{user_func.__name__}"}).reset_index(drop=True)
In [20]:
tmp = apply_scaling(plot_df .loc[plot_df["variable"] == "windowed_PCA", ["survey_date", "value"]] .set_index("survey_date"), method="Standard")
In [21]:
significance_level=0.1 sig_plot_df = tmp.mask( tmp > norm.ppf(1 - significance_level), 1 ).mask(tmp <= norm.ppf(1 - significance_level), 0)
In [22]:
norm.ppf(1 - significance_level) norm.ppf(1 - significance_level)
Out[22]:
1.2815515655446004
In [23]:
# _ = sns.heatmap(sig_plot_df.T)
In [24]:
_ = plt.figure(figsize=(15, 5)) _ = plt.plot(tmp) for sig_date in sig_plot_df[sig_plot_df==True].dropna().index: _ = plt.axvline(sig_date, ls="--", c=JmsColors.YELLOW)
In [25]:
window_size=7 window_range = np.arange(0, len(tmp)-window_size, window_size) coefs_list = [] for window_begin in window_range: win_df = tmp.iloc[window_begin: window_begin + window_size].iloc[:, 0] mod = LinearRegression() _ = mod.fit(X=win_df.index.to_numpy().reshape(-1, 1), y=win_df) coefs_list.append(mod.coef_[0])
In [26]:
_ = plt.plot(pd.Series(coefs_list))
In [27]:
print(f"time elapsed: {time.time()-start} seconds")
time elapsed: 170.37806701660156 seconds