gabeefranco/differential_privacy_budget_allocation_metric
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

retrieving data as Sta and Sen arrays

Browse source
This commit is contained in:
Gabriel Franco 2026-08-17 12:25:09 -03:00
parent 746440d325
commit 0b2484025e
5 changed files with 135 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
__pycache__/get_statistics.cpython-314.pyc
View file

Binary file not shown.

108
data.py Normal file
View file

@ -0,0 +1,108 @@
import pandas as pd
import numpy as np
file = "./data/pnad_trimestral_trimestre_012026.parquet"
regions_ufs = {
"Sul": [41, 42, 43],
"Sudeste": [31, 32, 33, 35],
"CentroOeste": [50, 51, 52, 53],
"Norte": [11, 12, 13, 14, 15, 16, 17],
"Nordeste": [21, 22, 23, 24, 25, 26, 27, 28, 29],
}
regions_names = list(regions_ufs.keys())
regions: dict[str, dict[str, float]] = {}
carteira_assinada = "V4029"
# carteira_assinada_opcoes = [1, # sim
# 2] # não
renda = "VD4019"
renda_norm = "VD4019_norm"
def clip_and_normalize(df: pd.DataFrame, c):
df = df.dropna(subset=[renda]).copy()
df[renda_norm] = df[renda].clip(0, c) / c
return df
def get_regions(df: pd.DataFrame, C: float):
for region_name in regions_names:
region_df = df[df["UF"].isin(regions_ufs[region_name])]
n_formal = (region_df[carteira_assinada] == 1).sum()
n_informal = (region_df[carteira_assinada] == 2).sum()
regions[region_name] = {
"informal_count": n_informal,
"informal_mean": region_df.loc[
region_df[carteira_assinada] == 2, renda_norm
].mean(),
"informal_std": region_df.loc[
region_df[carteira_assinada] == 2, renda_norm
].std(ddof=0),
"formal_count": n_formal,
"formal_mean": region_df.loc[
region_df[carteira_assinada] == 1, renda_norm
].mean(),
"formal_std": region_df.loc[
region_df[carteira_assinada] == 1, renda_norm
].std(ddof=0),
"sens_count": 1.0,
"sens_formal_mean": C / (n_formal - 1),
"sens_formal_std": C / np.sqrt(n_formal - 1),
"sens_informal_mean": C / (n_informal - 1),
"sens_informal_std": C / np.sqrt(n_informal - 1),
}
return regions
def get_Sta(regions):
groups = [
(region, formality)
for region in regions_names
for formality in ["formal", "informal"]
]
Sta = []
stat_kinds = ["count", "mean", "std"]
for region, formality in groups:
for stat_kind in stat_kinds:
Sta.append(regions[region][f"{formality}_{stat_kind}"])
return Sta
def get_Sen(regions):
groups = [
(region, formality)
for region in regions_names
for formality in ["formal", "informal"]
]
Sen = []
stat_kinds = ["count", "mean", "std"]
for region, formality in groups:
for stat_kind in stat_kinds:
if stat_kind == "count":
Sen.append(1.0)
continue
Sen.append(regions[region][f"sens_{formality}_{stat_kind}"])
return Sen
def main():
df = pd.read_parquet(file)
df = clip_and_normalize(df, 30_000)
regions = get_regions(df, 30_000)
Sta = get_Sta(regions)
print(len(Sta))
Sen = get_Sen(regions)
print(len(Sen))
if __name__ == "__main__":
main()

21
dp.py Normal file
View file

@ -0,0 +1,21 @@
import numpy as np
def epsilon_dp(x, epsilon, sensitivity):
return x + np.random.laplace(loc=0, scale=(sensitivity / epsilon))
if __name__ == "__main__":
from data import get_regions
regions = get_regions()
result = regions["Sul"]["informal_count"]
epsilon = 1
# count query
sensitivity = 1
query_count = 10
for i in range(query_count):
print(epsilon_dp(result, epsilon / query_count, sensitivity))

62
get_statistics.py
View file

@ -1,62 +0,0 @@
import pandas as pd
file = "./data/pnad_trimestral_trimestre_012026.parquet"
regions_ufs = {
"Sul": [41, 42, 43],
"Sudeste": [31, 32, 33, 35],
"CentroOeste": [50, 51, 52, 53],
"Norte": [11, 12, 13, 14, 15, 16, 17],
"Nordeste": [21, 22, 23, 24, 25, 26, 27, 28, 29],
}
regions_names = list(regions_ufs.keys())
regions: dict[str, dict[str, int]] = {}
carteira_assinada = "V4029"
# carteira_assinada_opcoes = [1, # sim
# 2] # não
renda = "VD4019"
df = pd.read_parquet(file)
def get_regions():
for region_name in regions_names:
region_df = df[df["UF"].isin(regions_ufs[region_name])]
regions[region_name] = {
"informal_count": (region_df[carteira_assinada] == 2).shape[0],
"informal_mean": region_df.loc[
region_df[carteira_assinada] == 2, renda
].mean(),
"informal_std": region_df.loc[
region_df[carteira_assinada] == 2, renda
].std(),
"formal_count": (region_df[carteira_assinada] == 1).shape[0],
"formal_mean": region_df.loc[
region_df[carteira_assinada] == 1, renda
].mean(),
"formal_std": region_df.loc[region_df[carteira_assinada] == 1, renda].std(),
}
return regions
def main():
get_regions()
print(f"TOTAL: {df.shape[0]}")
print(f"QUANTIDADE DE INFORMAIS: {(df[carteira_assinada] == 2).sum()}")
print(f"QUANTIDADE DE FORMAIS: {(df[carteira_assinada] == 1).sum()}")
print("\n")
for region_name in regions_names:
print(f"REGIÃO: {region_name}")
for key, value in regions[region_name].items():
print(f"{key}: {value}")
print("------------------------------")
print("")
if __name__ == "__main__":
main()

11
t_tests.py
View file

@ -1,8 +1,6 @@
from get_statistics import get_regions, regions_names
from data import regions_names
import math
regions = get_regions()
def t(n1, mean1, std1, n2, mean2, std2):
return (mean1 - mean2) / math.sqrt(
@ -10,7 +8,7 @@ def t(n1, mean1, std1, n2, mean2, std2):
)
def run_all_t_tests():
def run_all_t_tests(regions):
tests = {}
for name1 in regions_names:
for name2 in regions_names:
@ -54,4 +52,7 @@ def run_all_t_tests():
if __name__ == "__main__":
print(run_all_t_tests())
from data import get_regions
regions = get_regions()
print(run_all_t_tests(regions))