diff --git a/__pycache__/get_statistics.cpython-314.pyc b/__pycache__/get_statistics.cpython-314.pyc index 1390113..47cc162 100644 Binary files a/__pycache__/get_statistics.cpython-314.pyc and b/__pycache__/get_statistics.cpython-314.pyc differ diff --git a/data.py b/data.py new file mode 100644 index 0000000..1b601d4 --- /dev/null +++ b/data.py @@ -0,0 +1,108 @@ +import pandas as pd +import numpy as np + +file = "./data/pnad_trimestral_trimestre_012026.parquet" + + +regions_ufs = { + "Sul": [41, 42, 43], + "Sudeste": [31, 32, 33, 35], + "CentroOeste": [50, 51, 52, 53], + "Norte": [11, 12, 13, 14, 15, 16, 17], + "Nordeste": [21, 22, 23, 24, 25, 26, 27, 28, 29], +} +regions_names = list(regions_ufs.keys()) +regions: dict[str, dict[str, float]] = {} + + +carteira_assinada = "V4029" +# carteira_assinada_opcoes = [1, # sim +# 2] # não + +renda = "VD4019" +renda_norm = "VD4019_norm" + + +def clip_and_normalize(df: pd.DataFrame, c): + df = df.dropna(subset=[renda]).copy() + df[renda_norm] = df[renda].clip(0, c) / c + return df + + +def get_regions(df: pd.DataFrame, C: float): + for region_name in regions_names: + region_df = df[df["UF"].isin(regions_ufs[region_name])] + n_formal = (region_df[carteira_assinada] == 1).sum() + n_informal = (region_df[carteira_assinada] == 2).sum() + + regions[region_name] = { + "informal_count": n_informal, + "informal_mean": region_df.loc[ + region_df[carteira_assinada] == 2, renda_norm + ].mean(), + "informal_std": region_df.loc[ + region_df[carteira_assinada] == 2, renda_norm + ].std(ddof=0), + "formal_count": n_formal, + "formal_mean": region_df.loc[ + region_df[carteira_assinada] == 1, renda_norm + ].mean(), + "formal_std": region_df.loc[ + region_df[carteira_assinada] == 1, renda_norm + ].std(ddof=0), + "sens_count": 1.0, + "sens_formal_mean": C / (n_formal - 1), + "sens_formal_std": C / np.sqrt(n_formal - 1), + "sens_informal_mean": C / (n_informal - 1), + "sens_informal_std": C / np.sqrt(n_informal - 1), + } + return regions + + +def get_Sta(regions): + groups = [ + (region, formality) + for region in regions_names + for formality in ["formal", "informal"] + ] + Sta = [] + stat_kinds = ["count", "mean", "std"] + + for region, formality in groups: + for stat_kind in stat_kinds: + Sta.append(regions[region][f"{formality}_{stat_kind}"]) + + return Sta + + +def get_Sen(regions): + groups = [ + (region, formality) + for region in regions_names + for formality in ["formal", "informal"] + ] + Sen = [] + stat_kinds = ["count", "mean", "std"] + + for region, formality in groups: + for stat_kind in stat_kinds: + if stat_kind == "count": + Sen.append(1.0) + continue + Sen.append(regions[region][f"sens_{formality}_{stat_kind}"]) + + return Sen + + +def main(): + df = pd.read_parquet(file) + df = clip_and_normalize(df, 30_000) + regions = get_regions(df, 30_000) + Sta = get_Sta(regions) + print(len(Sta)) + Sen = get_Sen(regions) + print(len(Sen)) + + +if __name__ == "__main__": + main() diff --git a/dp.py b/dp.py new file mode 100644 index 0000000..fa4e1d1 --- /dev/null +++ b/dp.py @@ -0,0 +1,21 @@ +import numpy as np + + +def epsilon_dp(x, epsilon, sensitivity): + return x + np.random.laplace(loc=0, scale=(sensitivity / epsilon)) + + +if __name__ == "__main__": + from data import get_regions + + regions = get_regions() + + result = regions["Sul"]["informal_count"] + + epsilon = 1 + # count query + sensitivity = 1 + + query_count = 10 + for i in range(query_count): + print(epsilon_dp(result, epsilon / query_count, sensitivity)) diff --git a/get_statistics.py b/get_statistics.py deleted file mode 100644 index 3122377..0000000 --- a/get_statistics.py +++ /dev/null @@ -1,62 +0,0 @@ -import pandas as pd - -file = "./data/pnad_trimestral_trimestre_012026.parquet" - - -regions_ufs = { - "Sul": [41, 42, 43], - "Sudeste": [31, 32, 33, 35], - "CentroOeste": [50, 51, 52, 53], - "Norte": [11, 12, 13, 14, 15, 16, 17], - "Nordeste": [21, 22, 23, 24, 25, 26, 27, 28, 29], -} -regions_names = list(regions_ufs.keys()) -regions: dict[str, dict[str, int]] = {} - - -carteira_assinada = "V4029" -# carteira_assinada_opcoes = [1, # sim -# 2] # não - -renda = "VD4019" - -df = pd.read_parquet(file) - - -def get_regions(): - - for region_name in regions_names: - region_df = df[df["UF"].isin(regions_ufs[region_name])] - regions[region_name] = { - "informal_count": (region_df[carteira_assinada] == 2).shape[0], - "informal_mean": region_df.loc[ - region_df[carteira_assinada] == 2, renda - ].mean(), - "informal_std": region_df.loc[ - region_df[carteira_assinada] == 2, renda - ].std(), - "formal_count": (region_df[carteira_assinada] == 1).shape[0], - "formal_mean": region_df.loc[ - region_df[carteira_assinada] == 1, renda - ].mean(), - "formal_std": region_df.loc[region_df[carteira_assinada] == 1, renda].std(), - } - return regions - - -def main(): - get_regions() - print(f"TOTAL: {df.shape[0]}") - print(f"QUANTIDADE DE INFORMAIS: {(df[carteira_assinada] == 2).sum()}") - print(f"QUANTIDADE DE FORMAIS: {(df[carteira_assinada] == 1).sum()}") - print("\n") - for region_name in regions_names: - print(f"REGIÃO: {region_name}") - for key, value in regions[region_name].items(): - print(f"{key}: {value}") - print("------------------------------") - print("") - - -if __name__ == "__main__": - main() diff --git a/t_tests.py b/t_tests.py index 645fc10..03c2c51 100644 --- a/t_tests.py +++ b/t_tests.py @@ -1,8 +1,6 @@ -from get_statistics import get_regions, regions_names +from data import regions_names import math -regions = get_regions() - def t(n1, mean1, std1, n2, mean2, std2): return (mean1 - mean2) / math.sqrt( @@ -10,7 +8,7 @@ def t(n1, mean1, std1, n2, mean2, std2): ) -def run_all_t_tests(): +def run_all_t_tests(regions): tests = {} for name1 in regions_names: for name2 in regions_names: @@ -54,4 +52,7 @@ def run_all_t_tests(): if __name__ == "__main__": - print(run_all_t_tests()) + from data import get_regions + + regions = get_regions() + print(run_all_t_tests(regions))