retrieving data as Sta and Sen arrays

data.py

@@ -0,0 +1,108 @@
+import pandas as pd
+import numpy as np
+
+file = "./data/pnad_trimestral_trimestre_012026.parquet"
+
+
+regions_ufs = {
+    "Sul": [41, 42, 43],
+    "Sudeste": [31, 32, 33, 35],
+    "CentroOeste": [50, 51, 52, 53],
+    "Norte": [11, 12, 13, 14, 15, 16, 17],
+    "Nordeste": [21, 22, 23, 24, 25, 26, 27, 28, 29],
+}
+regions_names = list(regions_ufs.keys())
+regions: dict[str, dict[str, float]] = {}
+
+
+carteira_assinada = "V4029"
+# carteira_assinada_opcoes = [1, # sim
+#                           2] # não
+
+renda = "VD4019"
+renda_norm = "VD4019_norm"
+
+
+def clip_and_normalize(df: pd.DataFrame, c):
+    df = df.dropna(subset=[renda]).copy()
+    df[renda_norm] = df[renda].clip(0, c) / c
+    return df
+
+
+def get_regions(df: pd.DataFrame, C: float):
+    for region_name in regions_names:
+        region_df = df[df["UF"].isin(regions_ufs[region_name])]
+        n_formal = (region_df[carteira_assinada] == 1).sum()
+        n_informal = (region_df[carteira_assinada] == 2).sum()
+
+        regions[region_name] = {
+            "informal_count": n_informal,
+            "informal_mean": region_df.loc[
+                region_df[carteira_assinada] == 2, renda_norm
+            ].mean(),
+            "informal_std": region_df.loc[
+                region_df[carteira_assinada] == 2, renda_norm
+            ].std(ddof=0),
+            "formal_count": n_formal,
+            "formal_mean": region_df.loc[
+                region_df[carteira_assinada] == 1, renda_norm
+            ].mean(),
+            "formal_std": region_df.loc[
+                region_df[carteira_assinada] == 1, renda_norm
+            ].std(ddof=0),
+            "sens_count": 1.0,
+            "sens_formal_mean": C / (n_formal - 1),
+            "sens_formal_std": C / np.sqrt(n_formal - 1),
+            "sens_informal_mean": C / (n_informal - 1),
+            "sens_informal_std": C / np.sqrt(n_informal - 1),
+        }
+    return regions
+
+
+def get_Sta(regions):
+    groups = [
+        (region, formality)
+        for region in regions_names
+        for formality in ["formal", "informal"]
+    ]
+    Sta = []
+    stat_kinds = ["count", "mean", "std"]
+
+    for region, formality in groups:
+        for stat_kind in stat_kinds:
+            Sta.append(regions[region][f"{formality}_{stat_kind}"])
+
+    return Sta
+
+
+def get_Sen(regions):
+    groups = [
+        (region, formality)
+        for region in regions_names
+        for formality in ["formal", "informal"]
+    ]
+    Sen = []
+    stat_kinds = ["count", "mean", "std"]
+
+    for region, formality in groups:
+        for stat_kind in stat_kinds:
+            if stat_kind == "count":
+                Sen.append(1.0)
+                continue
+            Sen.append(regions[region][f"sens_{formality}_{stat_kind}"])
+
+    return Sen
+
+
+def main():
+    df = pd.read_parquet(file)
+    df = clip_and_normalize(df, 30_000)
+    regions = get_regions(df, 30_000)
+    Sta = get_Sta(regions)
+    print(len(Sta))
+    Sen = get_Sen(regions)
+    print(len(Sen))
+
+
+if __name__ == "__main__":
+    main()

dp.py

@@ -0,0 +1,21 @@
+import numpy as np
+
+
+def epsilon_dp(x, epsilon, sensitivity):
+    return x + np.random.laplace(loc=0, scale=(sensitivity / epsilon))
+
+
+if __name__ == "__main__":
+    from data import get_regions
+
+    regions = get_regions()
+
+    result = regions["Sul"]["informal_count"]
+
+    epsilon = 1
+    # count query
+    sensitivity = 1
+
+    query_count = 10
+    for i in range(query_count):
+        print(epsilon_dp(result, epsilon / query_count, sensitivity))

get_statistics.py

@@ -1,62 +0,0 @@
-import pandas as pd
-
-file = "./data/pnad_trimestral_trimestre_012026.parquet"
-
-
-regions_ufs = {
-    "Sul": [41, 42, 43],
-    "Sudeste": [31, 32, 33, 35],
-    "CentroOeste": [50, 51, 52, 53],
-    "Norte": [11, 12, 13, 14, 15, 16, 17],
-    "Nordeste": [21, 22, 23, 24, 25, 26, 27, 28, 29],
-}
-regions_names = list(regions_ufs.keys())
-regions: dict[str, dict[str, int]] = {}
-
-
-carteira_assinada = "V4029"
-# carteira_assinada_opcoes = [1, # sim
-#                           2] # não
-
-renda = "VD4019"
-
-df = pd.read_parquet(file)
-
-
-def get_regions():
-
-    for region_name in regions_names:
-        region_df = df[df["UF"].isin(regions_ufs[region_name])]
-        regions[region_name] = {
-            "informal_count": (region_df[carteira_assinada] == 2).shape[0],
-            "informal_mean": region_df.loc[
-                region_df[carteira_assinada] == 2, renda
-            ].mean(),
-            "informal_std": region_df.loc[
-                region_df[carteira_assinada] == 2, renda
-            ].std(),
-            "formal_count": (region_df[carteira_assinada] == 1).shape[0],
-            "formal_mean": region_df.loc[
-                region_df[carteira_assinada] == 1, renda
-            ].mean(),
-            "formal_std": region_df.loc[region_df[carteira_assinada] == 1, renda].std(),
-        }
-    return regions
-
-
-def main():
-    get_regions()
-    print(f"TOTAL: {df.shape[0]}")
-    print(f"QUANTIDADE DE INFORMAIS: {(df[carteira_assinada] == 2).sum()}")
-    print(f"QUANTIDADE DE FORMAIS: {(df[carteira_assinada] == 1).sum()}")
-    print("\n")
-    for region_name in regions_names:
-        print(f"REGIÃO: {region_name}")
-        for key, value in regions[region_name].items():
-            print(f"{key}: {value}")
-        print("------------------------------")
-        print("")
-
-
-if __name__ == "__main__":
-    main()

t_tests.py

@@ -1,8 +1,6 @@
-from get_statistics import get_regions, regions_names
+from data import regions_names
 import math
 
-regions = get_regions()
-
 
 def t(n1, mean1, std1, n2, mean2, std2):
     return (mean1 - mean2) / math.sqrt(
@@ -10,7 +8,7 @@ def t(n1, mean1, std1, n2, mean2, std2):
     )
 
 
-def run_all_t_tests():
+def run_all_t_tests(regions):
     tests = {}
     for name1 in regions_names:
         for name2 in regions_names:
@@ -54,4 +52,7 @@ def run_all_t_tests():
 
 
 if __name__ == "__main__":
-    print(run_all_t_tests())
+    from data import get_regions
+
+    regions = get_regions()
+    print(run_all_t_tests(regions))