""" Health Information Privacy Lab Steve L. Nyemba & Brad. Malin This is an extension to the pandas data-frame that will perform a risk assessment on a variety of attributes This implementation puts the responsibility on the user of the framework to join datasets and load the final results into a pandas data-frame. The code will randomly select fields and compute the risk (marketer and prosecutor) and perform a given number of runs. Usage: from pandas_risk import * mydataframe = pd.DataFrame('/myfile.csv') risk = mydataframe.deid.risk(id=,num_runs=) @TODO: - Provide a selected number of fields and risk will be computed for those fields. - include journalist risk """ import pandas as pd import numpy as np @pd.api.extensions.register_dataframe_accessor("deid") class deid : """ This class is a deidentification class that will compute risk (marketer, prosecutor) given a pandas dataframe """ def __init__(self,df): self._df = df def risk(self,**args): """ @param id name of patient field @params num_runs number of runs (default will be 100) """ id = args['id'] num_runs = args['num_runs'] if 'num_runs' in args else 100 r = pd.DataFrame() columns = list(set(self._df.columns) - set([id])) k = len(columns) for i in range(0,num_runs) : # # let's chose a random number of columns and compute marketer and prosecutor risk # Once the fields are selected we run a groupby clause # n = np.random.randint(2,k) #-- number of random fields we are picking ii = np.random.choice(k,n,replace=False) cols = np.array(columns)[ii].tolist() x_ = self._df.groupby(cols).count()[id].values r = r.append( pd.DataFrame( [ { "selected":n, "marketer": x_.size / np.float64(np.sum(x_)), "prosecutor":1 / np.float64(np.min(x_)) } ] ) ) g_size = x_.size n_ids = np.float64(np.sum(x_)) return r import pandas as pd import numpy as np from io import StringIO csv = """ id,sex,age,profession,drug_test 1,M,37,doctor,- 2,F,28,doctor,+ 3,M,37,doctor,- 4,M,28,doctor,+ 5,M,28,doctor,- 6,M,37,doctor,- """ f = StringIO() f.write(unicode(csv)) f.seek(0) df = pd.read_csv(f) print df.deid.risk(id='id',num_runs=1)