getting started on the preparedness data

preparedness/data_preparation_raw.py

@@ -0,0 +1,65 @@
+# %%
+import pyreadstat
+import json
+import re
+import datetime
+
+# %%
+df_sav, meta = pyreadstat.read_sav('./data/ZA8841_v1-0-0.sav')
+print(df_sav.shape)
+
+# %%
+df_sav.head(1).to_dict(orient='records')
+
+# %%
+def safe_json(obj):
+    if isinstance(obj, (datetime.datetime, datetime.date)):
+        return obj.isoformat()
+    if isinstance(obj, set):
+        return list(obj)
+    if hasattr(obj, '__dict__'):
+        return str(obj)
+    return obj
+
+meta_dict = vars(meta)
+meta_json = json.dumps(meta_dict, default=safe_json, indent=2)
+
+# %%
+with open('./data/za8841_meta.json', 'w') as f:
+    f.write(meta_json)
+
+# %%
+# Load your mapping dictionary (from JSON file or directly)
+with open("./data/za8841_meta.json") as f:
+    meta = json.load(f)
+value_labels = meta["variable_value_labels"]
+
+# %%
+# Assume df_sav is your loaded SPSS dataframe
+# For each column in the mapping, map values if the column exists in df_sav
+for col, mapping in value_labels.items():
+    if col in df_sav.columns:
+        # Convert keys to float if needed (SPSS values often are float)
+        mapping_float = {float(k): v for k, v in mapping.items()}
+        df_sav[col] = df_sav[col].map(mapping_float).fillna(df_sav[col])
+
+# Now all mapped columns have human-readable values
+print(df_sav.head())
+
+# %%
+labels_map = meta["column_names_to_labels"]
+def make_pandas_friendly(col):
+    col = labels_map.get(col, col)
+    col = re.sub(r'[.\s]+', '_', col)
+    col = re.sub(r'[^0-9a-zA-Z_]', '', col)
+    col = col.lower()
+    col = re.sub(r'__+', '_', col)      # Replace double (or more) underscores with single
+    col = col.strip('_')                # Remove leading/trailing underscores
+    return col
+
+df_sav.columns = [make_pandas_friendly(col) for col in df_sav.columns]
+
+# %%
+df_sav.head(1).to_dict(orient='records')
+
+# %%