From 0b2484025e81adf5cb46febf9d3069dc14d0cd6b Mon Sep 17 00:00:00 2001
From: Gabriel Franco <gabefgonc@gmail.com>
Date: Mon, 17 Aug 2026 12:25:09 -0300
Subject: [PATCH] retrieving data as Sta and Sen arrays

---
 __pycache__/get_statistics.cpython-314.pyc | Bin 3436 -> 3435 bytes
 data.py                                    | 108 +++++++++++++++++++++
 dp.py                                      |  21 ++++
 get_statistics.py                          |  62 ------------
 t_tests.py                                 |  11 ++-
 5 files changed, 135 insertions(+), 67 deletions(-)
 create mode 100644 data.py
 create mode 100644 dp.py
 delete mode 100644 get_statistics.py

diff --git a/__pycache__/get_statistics.cpython-314.pyc b/__pycache__/get_statistics.cpython-314.pyc
index 1390113ace902a8decc8a95ea4c0fe984e00498b..47cc16220cfed83fde0524aba13f17f8cc1170e9 100644
GIT binary patch
delta 332
zcmaDO^;(Kon~#@^0SIdPO0pC;^4c+S+~Rgl%_}L&-yFhd!@{Vrc_Qn2<_HC#l3T0=
zMVUYuVIaFm97KqLM45|AbBlyPTp19THM1l&w^)<8NER%aomzQ|wJb5GG_{x+D4?LA
zP$UnMV98C)%q!BGJcIKRqvGUjuFZ@xljXUcMZl6g@$rdydHE%YC8_c8MaGlMxGjN-
zHgHd`*8}NgD@e>sNi4p_RFHCuB`q^2wMZ0fWKL#r$t@P3jl~G}fZbRm36ke2N=;0O
zFGwsZEKP;_LIEgVqy>`Xh>y=p%mrHq<Z3b&fov*f1QHGq2Xolu=BKcg=A_ycg-t%g
Tqr)P2NmzOEAD$c*MW7-8Q)^hj

delta 333
zcmaDY^+t+Un~#@^0SN3qr(`K@<h5hutm1J_%_}L&*WDb<Xv4xNzj*@ddgcgupqg8(
z1x1-aDIp-cNDM@Xf<&2%OLL0^L0oANmo>8_HMdxkxkv^qnVnjBi?u8<r!=*g87QEj
zpim?Sl3>YA%*-p&oIIWL5~IT8EUwLr(v#)5okhTsJn`{~d3pIIi6yD=@kK_HOSvt9
ziq>;auh#|XWh+R`OGzxg#Z-`TizO{HC$&ffY-CPmamg(fppC@{_ki74Bmt7=DN0RD
zi7!YjDlAQf`U2#wB2ADaM|^x<VlLP^AXk&I2xL<+Bam=_IGDpGH$R1~G$+-rD0K2^
U9vv2;OCl<hfAi$9C;$}!042Lvga7~l

diff --git a/data.py b/data.py
new file mode 100644
index 0000000..1b601d4
--- /dev/null
+++ b/data.py
@@ -0,0 +1,108 @@
+import pandas as pd
+import numpy as np
+
+file = "./data/pnad_trimestral_trimestre_012026.parquet"
+
+
+regions_ufs = {
+    "Sul": [41, 42, 43],
+    "Sudeste": [31, 32, 33, 35],
+    "CentroOeste": [50, 51, 52, 53],
+    "Norte": [11, 12, 13, 14, 15, 16, 17],
+    "Nordeste": [21, 22, 23, 24, 25, 26, 27, 28, 29],
+}
+regions_names = list(regions_ufs.keys())
+regions: dict[str, dict[str, float]] = {}
+
+
+carteira_assinada = "V4029"
+# carteira_assinada_opcoes = [1, # sim
+#                           2] # não
+
+renda = "VD4019"
+renda_norm = "VD4019_norm"
+
+
+def clip_and_normalize(df: pd.DataFrame, c):
+    df = df.dropna(subset=[renda]).copy()
+    df[renda_norm] = df[renda].clip(0, c) / c
+    return df
+
+
+def get_regions(df: pd.DataFrame, C: float):
+    for region_name in regions_names:
+        region_df = df[df["UF"].isin(regions_ufs[region_name])]
+        n_formal = (region_df[carteira_assinada] == 1).sum()
+        n_informal = (region_df[carteira_assinada] == 2).sum()
+
+        regions[region_name] = {
+            "informal_count": n_informal,
+            "informal_mean": region_df.loc[
+                region_df[carteira_assinada] == 2, renda_norm
+            ].mean(),
+            "informal_std": region_df.loc[
+                region_df[carteira_assinada] == 2, renda_norm
+            ].std(ddof=0),
+            "formal_count": n_formal,
+            "formal_mean": region_df.loc[
+                region_df[carteira_assinada] == 1, renda_norm
+            ].mean(),
+            "formal_std": region_df.loc[
+                region_df[carteira_assinada] == 1, renda_norm
+            ].std(ddof=0),
+            "sens_count": 1.0,
+            "sens_formal_mean": C / (n_formal - 1),
+            "sens_formal_std": C / np.sqrt(n_formal - 1),
+            "sens_informal_mean": C / (n_informal - 1),
+            "sens_informal_std": C / np.sqrt(n_informal - 1),
+        }
+    return regions
+
+
+def get_Sta(regions):
+    groups = [
+        (region, formality)
+        for region in regions_names
+        for formality in ["formal", "informal"]
+    ]
+    Sta = []
+    stat_kinds = ["count", "mean", "std"]
+
+    for region, formality in groups:
+        for stat_kind in stat_kinds:
+            Sta.append(regions[region][f"{formality}_{stat_kind}"])
+
+    return Sta
+
+
+def get_Sen(regions):
+    groups = [
+        (region, formality)
+        for region in regions_names
+        for formality in ["formal", "informal"]
+    ]
+    Sen = []
+    stat_kinds = ["count", "mean", "std"]
+
+    for region, formality in groups:
+        for stat_kind in stat_kinds:
+            if stat_kind == "count":
+                Sen.append(1.0)
+                continue
+            Sen.append(regions[region][f"sens_{formality}_{stat_kind}"])
+
+    return Sen
+
+
+def main():
+    df = pd.read_parquet(file)
+    df = clip_and_normalize(df, 30_000)
+    regions = get_regions(df, 30_000)
+    Sta = get_Sta(regions)
+    print(len(Sta))
+    Sen = get_Sen(regions)
+    print(len(Sen))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/dp.py b/dp.py
new file mode 100644
index 0000000..fa4e1d1
--- /dev/null
+++ b/dp.py
@@ -0,0 +1,21 @@
+import numpy as np
+
+
+def epsilon_dp(x, epsilon, sensitivity):
+    return x + np.random.laplace(loc=0, scale=(sensitivity / epsilon))
+
+
+if __name__ == "__main__":
+    from data import get_regions
+
+    regions = get_regions()
+
+    result = regions["Sul"]["informal_count"]
+
+    epsilon = 1
+    # count query
+    sensitivity = 1
+
+    query_count = 10
+    for i in range(query_count):
+        print(epsilon_dp(result, epsilon / query_count, sensitivity))
diff --git a/get_statistics.py b/get_statistics.py
deleted file mode 100644
index 3122377..0000000
--- a/get_statistics.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import pandas as pd
-
-file = "./data/pnad_trimestral_trimestre_012026.parquet"
-
-
-regions_ufs = {
-    "Sul": [41, 42, 43],
-    "Sudeste": [31, 32, 33, 35],
-    "CentroOeste": [50, 51, 52, 53],
-    "Norte": [11, 12, 13, 14, 15, 16, 17],
-    "Nordeste": [21, 22, 23, 24, 25, 26, 27, 28, 29],
-}
-regions_names = list(regions_ufs.keys())
-regions: dict[str, dict[str, int]] = {}
-
-
-carteira_assinada = "V4029"
-# carteira_assinada_opcoes = [1, # sim
-#                           2] # não
-
-renda = "VD4019"
-
-df = pd.read_parquet(file)
-
-
-def get_regions():
-
-    for region_name in regions_names:
-        region_df = df[df["UF"].isin(regions_ufs[region_name])]
-        regions[region_name] = {
-            "informal_count": (region_df[carteira_assinada] == 2).shape[0],
-            "informal_mean": region_df.loc[
-                region_df[carteira_assinada] == 2, renda
-            ].mean(),
-            "informal_std": region_df.loc[
-                region_df[carteira_assinada] == 2, renda
-            ].std(),
-            "formal_count": (region_df[carteira_assinada] == 1).shape[0],
-            "formal_mean": region_df.loc[
-                region_df[carteira_assinada] == 1, renda
-            ].mean(),
-            "formal_std": region_df.loc[region_df[carteira_assinada] == 1, renda].std(),
-        }
-    return regions
-
-
-def main():
-    get_regions()
-    print(f"TOTAL: {df.shape[0]}")
-    print(f"QUANTIDADE DE INFORMAIS: {(df[carteira_assinada] == 2).sum()}")
-    print(f"QUANTIDADE DE FORMAIS: {(df[carteira_assinada] == 1).sum()}")
-    print("\n")
-    for region_name in regions_names:
-        print(f"REGIÃO: {region_name}")
-        for key, value in regions[region_name].items():
-            print(f"{key}: {value}")
-        print("------------------------------")
-        print("")
-
-
-if __name__ == "__main__":
-    main()
diff --git a/t_tests.py b/t_tests.py
index 645fc10..03c2c51 100644
--- a/t_tests.py
+++ b/t_tests.py
@@ -1,8 +1,6 @@
-from get_statistics import get_regions, regions_names
+from data import regions_names
 import math
 
-regions = get_regions()
-
 
 def t(n1, mean1, std1, n2, mean2, std2):
     return (mean1 - mean2) / math.sqrt(
@@ -10,7 +8,7 @@ def t(n1, mean1, std1, n2, mean2, std2):
     )
 
 
-def run_all_t_tests():
+def run_all_t_tests(regions):
     tests = {}
     for name1 in regions_names:
         for name2 in regions_names:
@@ -54,4 +52,7 @@ def run_all_t_tests():
 
 
 if __name__ == "__main__":
-    print(run_all_t_tests())
+    from data import get_regions
+
+    regions = get_regions()
+    print(run_all_t_tests(regions))