""" Health Information Privacy Lab Steve L. Nyemba & Brad. Malin This is an extension to the pandas data-frame that will perform a risk assessment on a variety of attributes This implementation puts the responsibility on the user of the framework to join datasets and load the final results into a pandas data-frame. The code will randomly select fields and compute the risk (marketer and prosecutor) and perform a given number of runs. Usage: from pandas_risk import * mydataframe = pd.DataFrame('/myfile.csv') risk = mydataframe.deid.risk(id=,num_runs=) @TODO: - Provide a selected number of fields and risk will be computed for those fields. - include journalist risk """ import pandas as pd import numpy as np @pd.api.extensions.register_dataframe_accessor("deid") class deid : """ This class is a deidentification class that will compute risk (marketer, prosecutor) given a pandas dataframe """ def __init__(self,df): self._df = df def risk(self,**args): """ @param id name of patient field @params num_runs number of runs (default will be 100) """ id = args['id'] num_runs = args['num_runs'] if 'num_runs' in args else 100 r = pd.DataFrame() columns = list(set(self._df.columns) - set([id])) k = len(columns) for i in range(0,num_runs) : # # let's chose a random number of columns and compute marketer and prosecutor risk # Once the fields are selected we run a groupby clause # n = np.random.randint(2,k) #-- number of random fields we are picking ii = np.random.choice(k,n,replace=False) cols = np.array(columns)[ii].tolist() x_ = self._df.groupby(cols).count()[id].values r = r.append( pd.DataFrame( [ { "selected":n, "marketer": x_.size / np.float64(np.sum(x_)), "prosecutor":1 / np.float64(np.min(x_)) } ] ) ) g_size = x_.size n_ids = np.float64(np.sum(x_)) return r # import pandas as pd # import numpy as np # from io import StringIO # csv = """ # id,sex,age,profession,drug_test # 1,M,37,doctor,- # 2,F,28,doctor,+ # 3,M,37,doctor,- # 4,M,28,doctor,+ # 5,M,28,doctor,- # 6,M,37,doctor,- # """ # f = StringIO() # f.write(unicode(csv)) # f.seek(0) # df = pd.read_csv(f) # print df.deid.risk(id='id',num_runs=2)