diff --git a/notebooks/registered-tier-history.ipynb b/notebooks/registered-tier-history.ipynb new file mode 100644 index 0000000..3278968 --- /dev/null +++ b/notebooks/registered-tier-history.ipynb @@ -0,0 +1,385 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"\n", + "The experiments here describe medical/family history as they associate with risk measures\n", + "Additionally we will have fractional risk assessments\n", + "\"\"\"\n", + "import pandas as pd\n", + "import numpy as np\n", + "from pandas_risk import *\n", + "dfm = pd.read_gbq(\"SELECT * FROM deid_risk.registered_medical_history_dec_001\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n", + "dff = pd.read_gbq(\"SELECT * FROM deid_risk.registered_family_history_dec_001\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n", + "df = pd.read_gbq(\"SELECT person_id, birth_date,city,state,home_owner,race,ethnicity,gender,birth_place,marital_status,orientation,education,employment_status,income,travel_abroad_6_months,active_duty_status FROM deid_risk.registered_dec_01\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "med_cols = np.random.choice(list(set(dfm.columns.tolist()) - set(['person_id'])),3).tolist()\n", + "fam_cols = np.random.choice(list(set(dff.columns.tolist()) - set(['person_id'])),3).tolist()\n", + "medical = pd.merge(df,dfm[med_cols+['person_id']],on='person_id')\n", + "family = pd.merge(df,dff[fam_cols + ['person_id']],on='person_id')\n", + "_tmp = pd.merge(dfm[med_cols +['person_id']],dff[fam_cols+['person_id']])\n", + "data = pd.merge(df,_tmp,on='person_id')" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
field_countflaggroup_countmarketerprosecutorunique_row_ratio
021full history1153080.9926911.00.987663
118medical1153060.9926741.00.987629
218family1153040.9926561.00.987594
315no-history1153000.9926221.00.987526
43medical-only270.0002320.50.000000
53family-only1460.0012571.00.000551
\n", + "
" + ], + "text/plain": [ + " field_count flag group_count marketer prosecutor \\\n", + "0 21 full history 115308 0.992691 1.0 \n", + "1 18 medical 115306 0.992674 1.0 \n", + "2 18 family 115304 0.992656 1.0 \n", + "3 15 no-history 115300 0.992622 1.0 \n", + "4 3 medical-only 27 0.000232 0.5 \n", + "5 3 family-only 146 0.001257 1.0 \n", + "\n", + " unique_row_ratio \n", + "0 0.987663 \n", + "1 0.987629 \n", + "2 0.987594 \n", + "3 0.987526 \n", + "4 0.000000 \n", + "5 0.000551 " + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.concat([data.deid.evaluate(flag='full history',cols= list(set(data.columns.tolist()) - set(['person_id'])) )\n", + " ,medical.deid.evaluate(flag='medical',cols=list( set(medical.columns.tolist() ) - set(['person_id']) ) )\n", + " ,family.deid.evaluate(flag='family',cols=list( set(family.columns.tolist() ) - set(['person_id']) ) )\n", + " ,df.deid.evaluate(flag='no-history',cols=list( set(df.columns.tolist() ) - set(['person_id']) ) )\n", + " , dfm.deid.evaluate(flag='medical-only',cols=med_cols )\n", + " , dff.deid.evaluate(flag='family-only',cols=fam_cols )\n", + " ],ignore_index=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import division\n", + "def evaluate(df) :\n", + " cols = list(set(df.columns.tolist()) - set(['person_id']))\n", + " \n", + " portions = np.round(np.random.random_sample(4),3).tolist() + np.arange(5,105,5).tolist()\n", + " \n", + " N = df.shape[0] - 1\n", + " portions = np.divide(np.multiply(portions,N),100).astype(np.int64)\n", + " portions = np.unique([n for n in portions if n > 1])\n", + " \n", + " r = pd.DataFrame()\n", + " for num_rows in portions :\n", + " \n", + " indices = np.random.choice(N,num_rows,replace=False)\n", + "# print (indices.size / N)\n", + " flag = \" \".join([str( np.round(100*indices.size/ N,2)),'%'])\n", + " r = r.append(df.loc[indices].deid.evaluate(cols=cols,flag=flag,min_group_size=2))\n", + " return r" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
field_countflaggroup_countmarketerprosecutorunique_row_ratio
011UNFLAGGED1148860.9890581.00.980535
\n", + "
" + ], + "text/plain": [ + " field_count flag group_count marketer prosecutor unique_row_ratio\n", + "0 11 UNFLAGGED 114886 0.989058 1.0 0.980535" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cols = list(set (df.columns.tolist()) - set(['person_id']))\n", + "df[['race','state','gender_identity','ethnicity','marital_status','education','orientation','sex_at_birth','birth_date','travel_abroad_6_months','active_duty_status']].deid.evaluate()" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['person_id',\n", + " 'HearingVision_FarSightedness',\n", + " 'HearingVision_Glaucoma',\n", + " 'Digestive_Pancreatitis']" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#\n", + "# This is the merge with medical history\n", + "\n", + "cols = ['person_id'] + np.random.choice(dfm.columns[1:],3,replace=False).tolist()\n", + "p = pd.merge(df,dfm[cols],on='person_id')\n", + "cols\n", + "# # cols = list(set(p.columns.tolist()) - set(['person_id']))\n", + "# evaluate(p) #p.deid.explore(cols=cols,num_runs=100)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "cols = list( set(dfm.columns.tolist()) - set(['person_id']))\n", + "cols = np.random.choice(cols,3,replace=False).tolist()\n", + "p = pd.merge(dfm[['person_id']+cols],df)\n", + "fcols = list(set(p.columns.tolist()) - set(['person_id']))\n", + "# dfm[cols].deid.evaluate(cols=list( set(cols) - set(['person_id'])))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "variables": { + " \" ; \".join(cols)": "InfectiousDiseases_HepatitisC ; Cancer_StomachCancer ; Circulatory_Hypertension", + " p.shape[0] ": "116157", + " p[fcols].deid.evaluate() ": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
field_countflaggroup_countmarketerprosecutorunique_row_ratio
037UNFLAGGED1153970.9934571.00.98886
\n
" + } + }, + "source": [ + "### Medical History\n", + "\n", + " We randomly select three a tributes {{ \" ; \".join(cols)}} . \n", + " The dataset associated risk evaluation contains {{ p.shape[0] }} records\n", + "{{ p[fcols].deid.evaluate() }}\n", + "\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['person_id',\n", + " 'InfectiousDiseases_Tuberculosis',\n", + " 'SkeletalMuscular_Fibromyalgia',\n", + " 'Cancer_ProstateCancer']" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cols" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "execution_count": 67, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# dfm[cols[1:]].head()\n", + "np.sum(dfm.fillna(' ').groupby(cols[1:],as_index=False).size().values <= 1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.15rc1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/registered-tier.ipynb b/notebooks/registered-tier.ipynb new file mode 100644 index 0000000..c8acbbf --- /dev/null +++ b/notebooks/registered-tier.ipynb @@ -0,0 +1,859 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"\n", + "This notebook is designed to run experiments around demographics on registered tier\n", + "The \n", + "\"\"\"\n", + "import pandas as pd\n", + "import numpy as np\n", + "from pandas_risk import *\n", + "\n", + "ATTRIBUTES = ['race','ethnicity','birth_date','state','city','zip','marital_status','education','language','home_owner','income','employment_status','living_situation','active_duty_status','gender_identity','birth_place','death_date','death_cause','orientation']\n", + "dfs = pd.read_csv('scenario-settings.csv')\n", + "dfc = pd.read_gbq(\"SELECT * FROM deid_risk.registered_dec_01\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "cols_o = dfs.loc[(dfs.fo & dfs.fi) ==1].feature.tolist()\n", + "cols_i = dfs.loc[(dfs.fo + dfs.fi )>=1 ].feature.tolist()\n", + "cols_a = dfs.feature.tolist()\n", + "\n", + "cols_v = ['birth_date','gender_identity','race','state','city','birth_place'] #-- voter registration\n", + "#remove the dates fields because dates are shifted\n", + "cols_o = [i for i in cols_o if i not in ['birth_date','death_date']]\n", + "cols_i = [i for i in cols_i if i not in ['birth_date','death_date']]\n", + "cols_a = [i for i in cols_a if i not in ['birth_date','death_date']]\n", + "cols_v = [i for i in cols_v if i not in ['birth_date', 'death_date']]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# print(dfs)\n", + "# print(cols_o)\n", + "# print(cols_i)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
field_countflaggroup_countmarketerprosecutorunique_row_ratio
08high-conj65320.0562341.00.021368
111high-disj474470.4084731.00.278554
216all607180.5227241.00.408189
35voter-reg13160.0113291.00.002944
\n", + "
" + ], + "text/plain": [ + " field_count flag group_count marketer prosecutor unique_row_ratio\n", + "0 8 high-conj 6532 0.056234 1.0 0.021368\n", + "1 11 high-disj 47447 0.408473 1.0 0.278554\n", + "2 16 all 60718 0.522724 1.0 0.408189\n", + "3 5 voter-reg 1316 0.011329 1.0 0.002944" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r = pd.concat([dfc[cols_o].deid.evaluate(),dfc[cols_i].deid.evaluate(),dfc[cols_a].deid.evaluate(),dfc[cols_v].deid.evaluate() ])\n", + "r.index = np.arange(r.shape[0]).astype(np.int64)\n", + "r['flag']=['high-conj','high-disj','all','voter-reg']\n", + "\n", + "r\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEuCAYAAAB1QVLBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFxVJREFUeJzt3X+U3XV95/HnK+FHtohSIcsiAZO6UYGFEjdEUY/yU0E0bP3RwuourFa2q1A89lSBFqzUY6ly3HqUrlKbo9a1ILCWGKIgVVyRU8gAWWzMcow0QmArETGy7okQfe8f906cjHcydyZ3cplPno9z5uR+P9/PzH3nnpnXfOf9/d7vJ1WFJKktc4ZdgCRp8Ax3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoP2GtYTH3TQQbVw4cJhPb0kzUp33333D6tq/mTzhhbuCxcuZGRkZFhPL0mzUpLv9zPPtowkNchwl6QGGe6S1KCh9dwlteupp55i06ZNbN26ddilzFrz5s1jwYIF7L333tP6fMNd0sBt2rSJ/fffn4ULF5Jk2OXMOlXFY489xqZNm1i0aNG0voZtGUkDt3XrVg488ECDfZqScOCBB+7SXz6Gu6QZYbDvml19/Qx3SWqQPXdpN1h40U3DLqEvG684Y0a+7qD//zNV50Ruu+02rrzySlatWtXX/LVr1/LII4/wmte8ZoYrm5hH7pK0E9u2bZvy56xdu5bVq1fP+PPsjOEuqUkbN27khS98Ieeeey7Pf/7zefOb38ytt97Ky172MhYvXsxdd93FXXfdxfHHH8+SJUt46Utfyv333w/Apz/9aZYvX85JJ53EySefvMPXXbNmDUuWLOF73/seP/3pT3nrW9/KsmXLWLJkCTfeeCNPPvkkl112Gddeey3HHnss1157bc95kz3PrrItI6lZGzZs4LrrrmPFihUcd9xxfP7zn+f2229n5cqVfPCDH+Szn/0s3/zmN9lrr7249dZbueSSS7jhhhsAuOeee7jvvvt49rOfzW233QbAHXfcwQUXXMCNN97I4YcfziWXXMJJJ53EihUr+PGPf8yyZcs45ZRTuPzyyxkZGeHjH/84wITzxj/PIBnukpq1aNEijj76aACOOuooTj75ZJJw9NFHs3HjRrZs2cI555zDd7/7XZLw1FNPbf/cU089dYfAXb9+Peeddx633HILz3nOcwC45ZZbWLlyJVdeeSXQuQT0wQcf/JU6djZv/PMMiuEuqVn77rvv9sdz5szZvj1nzhy2bdvGpZdeyoknnsgXv/hFNm7cyAknnLB9/n777bfD1zrkkEPYunUr99577/ZwrypuuOEGXvCCF+ww984779xhe2fzxj/PoPTVc09yWpL7k2xIclGP/ecm2ZxkbffjdwdfqiQN1pYtWzj00EOBTv97Zw444ABuuukmLr744u1tmle/+tV87GMfo6oAuPfeewHYf//9eeKJJ7Z/7kTzZtKkR+5J5gJXAacCm4A1SVZW1XfGTb22qs6fgRolzXK7+9LFfr3nPe/hnHPO4QMf+ABnnDF5jQcffDCrVq3i9NNPZ8WKFVx66aW8613v4phjjuEXv/gFixYtYtWqVZx44olcccUVHHvssVx88cUTzptJGf1NMuGE5HjgT6rq1d3tiwGq6s/GzDkXWDqVcF+6dGm5WIf2FHvade7r16/niCOOGMjX2pP1eh2T3F1VSyf73H7aMocCD43Z3tQdG+8NSe5Lcn2Sw3p9oSTnJRlJMrJ58+Y+nlqSNB2Dus79S8DCqjoG+CrwmV6TqurqqlpaVUvnz590CUBJ0jT1E+4PA2OPxBd0x7arqseq6mfdzU8B/3Yw5UmarSZr+WrndvX16yfc1wCLkyxKsg9wFrBy7IQkh4zZXA6s36WqJM1q8+bN47HHHjPgp2n0fu7z5s2b9teY9GqZqtqW5HzgZmAusKKq1iW5HBipqpXA7ydZDmwDfgScO+2KJM16CxYsYNOmTXhubfpGV2Karr7exFRVq4HV48YuG/P4YuDiaVchqSl77733tFcQ0mB44zBJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhq0Vz+TkpwGfBSYC3yqqq6YYN4bgOuB46pqZGBVaigWXnTTsEvoy8Yrzhh2CdLTzqRH7knmAlcBpwNHAmcnObLHvP2BC4E7B12kJGlq+mnLLAM2VNUDVfUkcA1wZo95fwr8ObB1gPVJkqahn3A/FHhozPam7th2SV4EHFZVO/07Psl5SUaSjGzevHnKxUqS+rPLJ1STzAE+AvzBZHOr6uqqWlpVS+fPn7+rTy1JmkA/4f4wcNiY7QXdsVH7A/8GuC3JRuAlwMokSwdVpCRpavoJ9zXA4iSLkuwDnAWsHN1ZVVuq6qCqWlhVC4F/AJZ7tYwkDc+k4V5V24DzgZuB9cAXqmpdksuTLJ/pAiVJU9fXde5VtRpYPW7ssgnmnrDrZUmSdoXvUJWkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkN6ivck5yW5P4kG5Jc1GP/7yX5dpK1SW5PcuTgS5Uk9WvScE8yF7gKOB04Eji7R3h/vqqOrqpjgQ8BHxl4pZKkvvVz5L4M2FBVD1TVk8A1wJljJ1TVT8Zs7gfU4EqUJE3VXn3MORR4aMz2JuDF4ycleSfwbmAf4KSBVCdJmpaBnVCtqquq6nnAe4E/7jUnyXlJRpKMbN68eVBPLUkap59wfxg4bMz2gu7YRK4B/l2vHVV1dVUtraql8+fP779KSdKU9BPua4DFSRYl2Qc4C1g5dkKSxWM2zwC+O7gSJUlTNWnPvaq2JTkfuBmYC6yoqnVJLgdGqmolcH6SU4CngMeBc2ayaEnSzvVzQpWqWg2sHjd22ZjHFw64LknSLvAdqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNaivcE9yWpL7k2xIclGP/e9O8p0k9yX5+yTPHXypkqR+TRruSeYCVwGnA0cCZyc5cty0e4GlVXUMcD3woUEXKknqXz9H7suADVX1QFU9CVwDnDl2QlV9var+X3fzH4AFgy1TkjQV/YT7ocBDY7Y3dccm8jbgy712JDkvyUiSkc2bN/dfpSRpSgZ6QjXJW4ClwId77a+qq6tqaVUtnT9//iCfWpI0xl59zHkYOGzM9oLu2A6SnAL8EfDKqvrZYMqTJE1HP0fua4DFSRYl2Qc4C1g5dkKSJcAngeVV9ejgy5QkTcWk4V5V24DzgZuB9cAXqmpdksuTLO9O+zDwDOC6JGuTrJzgy0mSdoN+2jJU1Wpg9bixy8Y8PmXAdUmSdoHvUJWkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUF9hXuS05Lcn2RDkot67H9FknuSbEvyxsGXKUmaiknDPclc4CrgdOBI4OwkR46b9iBwLvD5QRcoSZq6vfqYswzYUFUPACS5BjgT+M7ohKra2N33ixmoUZI0Rf20ZQ4FHhqzvak7NmVJzksykmRk8+bN0/kSkqQ+7NYTqlV1dVUtraql8+fP351PLUl7lH7C/WHgsDHbC7pjkqSnqX7CfQ2wOMmiJPsAZwErZ7YsSdKumDTcq2obcD5wM7Ae+EJVrUtyeZLlAEmOS7IJeBPwySTrZrJoSdLO9XO1DFW1Glg9buyyMY/X0GnXSJKeBnyHqiQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KC+7i0zWyy86KZhl9CXjVecMewSJDXOI3dJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqUFMrMUnaM7jq2uT6OnJPclqS+5NsSHJRj/37Jrm2u//OJAsHXagkqX+ThnuSucBVwOnAkcDZSY4cN+1twONV9a+B/wr8+aALlST1r58j92XAhqp6oKqeBK4Bzhw350zgM93H1wMnJ8ngypQkTUU/PfdDgYfGbG8CXjzRnKralmQLcCDww7GTkpwHnNfd/L9J7p9O0bvZQYz7f+yq7Nl/1/h6Do6v5WDNltfzuf1M2q0nVKvqauDq3fmcuyrJSFUtHXYdrfD1HBxfy8Fq7fXspy3zMHDYmO0F3bGec5LsBTwLeGwQBUqSpq6fcF8DLE6yKMk+wFnAynFzVgLndB+/EfhaVdXgypQkTcWkbZluD/184GZgLrCiqtYluRwYqaqVwF8Df5NkA/AjOr8AWjGr2kizgK/n4PhaDlZTr2c8wJak9nj7AUlqkOEuSQ0y3CWpQYa7JDXIu0JqxiR5ZlX9JMmze+wu4CdV9fPdXZcEkOT1PYa3AN+uqkd3dz2D5tUyYyR5S1V9Lsm7e+wuOpd5rqyqx3dzabNSklVV9dok/0Tn9Rt/v6FnAH9VVZfs/upmlyRP0HkNf2UXUFX1zN1c0qyX5CbgeODr3aETgLuBRcDlVfU3QyptIDxy39F+3X/3n2D/IuC/AC/ZPeXMblX12u6/i3rt795x9B8Bw30SVTXR96Smby/giKr6AUCSg4HP0rl31v8EDPdWVNUnu/++f6I53TdvaQqSvAxYW1U/TfIW4EXAX1TVg8ARw61udpigtbVdVf1od9XSkMNGg73r0e7Yj5I8NayiBsW2TA9J5gNvBxYy5hdgVb11WDXNZknuA34TOAb4NPAp4Ler6pXDrGs2maC1NbpdVfUbQylsFkvyl8DhwHXdoTfSubvtHwKrqurEYdU2CIZ7D0nuAL5Jp/+2/YRfVd0wtKJmsST3VNWLklwGPFxVfz06NuzaZqPuUfxiYN7oWFV9Y3gVzU7dNSdeD7y8O/Qt4IZW7otlW6a3X6uq9w67iIY8keRi4C3AK5LMAfYeck2zUpLfBS6kc3fWtXTO/9wBnDzMumajqqokI8CWqro1ya/ROcn/xJBLGwivc+9tVZLXDLuIhvwO8DPgbVX1z3SC6cPDLWnWuhA4Dvh+t22whM7le5qiJG+ns3LcJ7tDhwJ/N7yKBsu2TA/dy872A54ERk+seLmZhi7Jmqo6Lsla4MVV9bMk66rqqGHXNtt0X8NlwJ1VtaQ79u2qOnq4lQ2GbZkevOxsMJLcXlUv73GNttdmT9+mJAfQOcL8apLHge8PuabZ6mdV9eTocs/dhYaaOdr1yH0CSZYDr+hu3lZVq4ZZjzReklfSWfXsK93F6zUFST4E/Bj4j8AFwDuA71TVHw21sAEx3HtIcgWdvuZ/7w6dTWdhkouHV9Xs47XZejrrnth/G/AqOn9N3gx8qpWrZQz3HrrXZR9bVb/obs8F7q2qY4Zb2ewy7trsw4HHu48PAB6c6J2r0kzr/kx/tqrePOxaZopXy0zsgDGPnzW0KmaxqlrUfXPNrcDrquqgqjoQeC1wy3Cr056se8O653bXhW6SJ1R7+zPg3iRfp3Ok+QrgouGWNKu9pKrePrpRVV/u9julYXoA+FaSlcBPRwer6iPDK2lwDPcequpvk9xGp+8O8N7u9dmankeS/DHwue72m4FHhliPBPC97sccJr5Z4Kxlz72HJL8FfK2qtnS3DwBOqKpm3uCwO3VPrL6PX1599A06t1T1hKqeFpL8q9YO4Az3HpKsrapjx43dO/pGB01fkkOq6v8Muw5prBbvdeQJ1d56vS62sAbjpmEXIPUwfiGZWc9w720kyUeSPK/78RE6d4jUrmvuh0hN+KthFzBohntvF9C5r8y1wDXAVuCdQ62oHc39EGn2STI3yf8e3a6qvxxmPTPBnvsk7BEPRvdNIwez4+InDw6vIu3pktwIXNDq96F95MndRGdZOE1TkgvoXC3zAzqLn4TOO1d9x6+G6deBdUnuYsfr3JcPr6TBMdwnZ494110IvKCqHht2IdIYlw67gJlkuE/OHvGuewgXlNDTTFV9I8lzgcVjVmKaO+y6BsWe+wTsEe+6JO/uPjwKeAGdFtfPRve38jZvzU7dlZjOA55dVc9Lshj4RFU1sWShR+492CMemNG3dD/Y/din+yE9HbyT7kpMAFX13ST/crglDY7h3ps94gGoqvcPuwZpJ5peiclw780e8QAl+RK/+kOzBRgBPllVW3d/VRLfSHIJ8C+SnEpnJaYvDbmmgbHnPoY94pmR5KPAfOBvu0O/A/yETuA/s6r+w7Bq056r10pMVdXMBRQeue/IHvHMeGlVHTdm+0tJ1lTVcUnWDa0q7ekuqKqPMuaKuCQXdsdmPY/cNeOSrAdePXq1UZLD6RwlHeHdNjUsve4E2dL3o0fuPdgjHrg/AG5P8j06f/4uAt6RZD/gM0OtTHucJGcD/x5Y1F2FadQzgWbWGPDIvQd7xIOXZF/ghd3N+/0FqWHpvnFpEZ3lNMcun/kEcF9VbRtKYQNmuPcw2g/uNZZkXVUdNazaZpMkJ1XV15K8vtf+qvofu7smaawkB/PL5TTvqqpHh1nPINmW6e0ZSQ4f1yN+Rnffk8Mra9Z5JfA14HXd7dEjidE3hRnuGpokbwKuBG6j8z35sSR/WFXXD7WwAfHIvYckrwE+QWfx3O09YjrfBG+vqr8YXnWzT5J5wBuAhfzygKKq6vKhFaU9XpL/BZw6erSeZD5wa1X95nArGwyP3HuoqtXd+0z06hEb7FP3d8CPgXvoLHwCDb0TULPWnHFtmMdoaAEjw32MnfSIn5fEHvH0Laiq04ZdhDTOl5PczI4XTqweYj0DZbjvyB7xzLgjydFV9e1hFyKN8SjwOeDY7vbVVfXFIdYzUPbce7BHPBhJvk3nl+JewGLgATq3cwid19O7bGpokrwP+G0617ZfC1xXVT8YblWDY7j3kOQr/LJH/PPucHlvmanpXk88oar6/u6qRZpIkmPotGTeAGyqqlOGXNJA2JbpzR7xABjemiUeBf6ZzgnVZu7n3syZ4QG7I8nRwy5C0sxJ8o4ktwF/DxxI5zLnZlqFHrmPMa5H/J+S2COW2nUY8K6qWjvsQmaCPfcx7BFLaoXhLkkNsucuSQ0y3CWpQYa79jhJfj/J+iQPJ/n4sOuRZoJXy2hP9A7glO7H0iHXIs0Ij9y1R0nyCeA3gC8Dvz5m/HVJ7kxyb5Jbu4s4kGR+kq8mWZfkU0m+n+SgIZUv9c1w1x6lqn4PeAQ4EXh8zK7bgZd0F0e+BnhPd/x9wNe6q29dDxy+G8uVps22jNSxALg2ySHAPsA/dcdfDvwWQFV9JcnjE3y+9LTikbvU8THg41V1NPCfgXlDrkfaJYa71PEs4OHu43PGjH+Lzm1hSfIqxvTppaczw13q+BPguiR3Az8cM/5+4FVJ/hF4E527Bz6x+8uTpsbbD0g7kWRf4OdVtS3J8cB/q6pjJ/s8adg8oSrt3OHAF5LMAZ4E3j7keqS+eOQuSQ2y5y5JDTLcJalBhrskNchwl6QGGe6S1KD/DzwQEyuRP8dtAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "fig_o = r.plot(kind='bar',x='flag',y=['marketer']).get_figure()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "writer = pd.ExcelWriter('out-116kpatients-phase-1.xlsx',engine='xlsxwriter')\n", + "r.to_excel(writer,'phase-1')\n", + "writer.save()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
featurefifo
0race11
1ethnicity11
2birth_date11
3city11
4state11
5marital_status11
6education10
7language00
8home_owner11
9income01
10employment_status10
11living_situation00
12active_duty_status00
13gender_identity11
14birth_place00
15death_date11
16death_cause11
17orientation00
\n", + "
" + ], + "text/plain": [ + " feature fi fo\n", + "0 race 1 1\n", + "1 ethnicity 1 1\n", + "2 birth_date 1 1\n", + "3 city 1 1\n", + "4 state 1 1\n", + "5 marital_status 1 1\n", + "6 education 1 0\n", + "7 language 0 0\n", + "8 home_owner 1 1\n", + "9 income 0 1\n", + "10 employment_status 1 0\n", + "11 living_situation 0 0\n", + "12 active_duty_status 0 0\n", + "13 gender_identity 1 1\n", + "14 birth_place 0 0\n", + "15 death_date 1 1\n", + "16 death_cause 1 1\n", + "17 orientation 0 0" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dfs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "names = pd.read_csv('family-history.csv').name.tolist()\n", + "path ='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json'\n", + "sql = \"\"\"\n", + "SELECT * FROM deid_risk.registered_medical_history_dec_001\n", + "\"\"\"\n", + "dfm = pd.read_gbq(\"SELECT * FROM deid_risk.registered_medical_history_dec_001\",private_key=path,dialect='standard')" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.9343780009344719, 1.269831148073964)" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cols = list( set(dfm.columns.tolist()) - set(['person_id']))\n", + "r = pd.DataFrame(dfm[cols].count(),columns=['counts'])\n", + "r['attributes'] = r.index\n", + "r['rate'] = 100*(r.counts / dfm.shape[0])\n", + "r.rate.mean(),np.sqrt(r.rate.var())" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": {}, + "outputs": [], + "source": [ + "writer = pd.ExcelWriter('/home/steve/tmp/simple.xlsx', engine='xlsxwriter')\n", + "r.to_excel(writer,sheet_name='p1')\n", + "workbook = writer.book\n", + "worksheet = workbook.add_worksheet()\n", + "b = pd.DataFrame({\"id\":np.random.choice(10,30)})" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['__class__',\n", + " '__delattr__',\n", + " '__dict__',\n", + " '__doc__',\n", + " '__format__',\n", + " '__getattribute__',\n", + " '__hash__',\n", + " '__init__',\n", + " '__module__',\n", + " '__new__',\n", + " '__reduce__',\n", + " '__reduce_ex__',\n", + " '__repr__',\n", + " '__setattr__',\n", + " '__sizeof__',\n", + " '__str__',\n", + " '__subclasshook__',\n", + " '__weakref__',\n", + " '_assemble_xml_file',\n", + " '_button_params',\n", + " '_calculate_spans',\n", + " '_calculate_x_split_width',\n", + " '_check_dimensions',\n", + " '_comment_params',\n", + " '_convert_date_time',\n", + " '_convert_name_area',\n", + " '_csv_join',\n", + " '_encode_password',\n", + " '_escape_attributes',\n", + " '_escape_data',\n", + " '_escape_url',\n", + " '_extract_filter_tokens',\n", + " '_get_palette_color',\n", + " '_get_range_data',\n", + " '_initialize',\n", + " '_isinf',\n", + " '_isnan',\n", + " '_opt_close',\n", + " '_opt_reopen',\n", + " '_parse_filter_expression',\n", + " '_parse_filter_tokens',\n", + " '_position_object_emus',\n", + " '_position_object_pixels',\n", + " '_prepare_chart',\n", + " '_prepare_header_image',\n", + " '_prepare_header_vml_objects',\n", + " '_prepare_image',\n", + " '_prepare_shape',\n", + " '_prepare_tables',\n", + " '_prepare_vml_objects',\n", + " '_set_filehandle',\n", + " '_set_icon_props',\n", + " '_set_spark_color',\n", + " '_set_xml_writer',\n", + " '_size_col',\n", + " '_size_row',\n", + " '_sort_pagebreaks',\n", + " '_table_function_to_formula',\n", + " '_write',\n", + " '_write_array_formula',\n", + " '_write_auto_filter',\n", + " '_write_autofilters',\n", + " '_write_blank',\n", + " '_write_boolean',\n", + " '_write_brk',\n", + " '_write_cell',\n", + " '_write_cell_array_formula',\n", + " '_write_cell_value',\n", + " '_write_cf_rule',\n", + " '_write_cfvo',\n", + " '_write_col_breaks',\n", + " '_write_col_info',\n", + " '_write_color',\n", + " '_write_color_axis',\n", + " '_write_color_first',\n", + " '_write_color_high',\n", + " '_write_color_last',\n", + " '_write_color_low',\n", + " '_write_color_markers',\n", + " '_write_color_negative',\n", + " '_write_color_scale',\n", + " '_write_color_series',\n", + " '_write_cols',\n", + " '_write_conditional_formats',\n", + " '_write_conditional_formatting',\n", + " '_write_conditional_formatting_2010',\n", + " '_write_custom_filter',\n", + " '_write_custom_filters',\n", + " '_write_data_bar',\n", + " '_write_data_bar_ext',\n", + " '_write_data_validation',\n", + " '_write_data_validations',\n", + " '_write_datetime',\n", + " '_write_dimension',\n", + " '_write_drawing',\n", + " '_write_drawings',\n", + " '_write_empty_row',\n", + " '_write_ext',\n", + " '_write_ext_list',\n", + " '_write_ext_list_data_bars',\n", + " '_write_ext_list_sparklines',\n", + " '_write_filter',\n", + " '_write_filter_column',\n", + " '_write_filters',\n", + " '_write_font',\n", + " '_write_formula',\n", + " '_write_formula_1',\n", + " '_write_formula_2',\n", + " '_write_formula_element',\n", + " '_write_freeze_panes',\n", + " '_write_header_footer',\n", + " '_write_hyperlink_external',\n", + " '_write_hyperlink_internal',\n", + " '_write_hyperlinks',\n", + " '_write_icon_set',\n", + " '_write_legacy_drawing',\n", + " '_write_legacy_drawing_hf',\n", + " '_write_merge_cell',\n", + " '_write_merge_cells',\n", + " '_write_number',\n", + " '_write_odd_footer',\n", + " '_write_odd_header',\n", + " '_write_optimized_sheet_data',\n", + " '_write_outline_pr',\n", + " '_write_page_margins',\n", + " '_write_page_set_up_pr',\n", + " '_write_page_setup',\n", + " '_write_panes',\n", + " '_write_phonetic_pr',\n", + " '_write_print_options',\n", + " '_write_rich_string',\n", + " '_write_row',\n", + " '_write_row_breaks',\n", + " '_write_rows',\n", + " '_write_rstring_color',\n", + " '_write_selection',\n", + " '_write_selections',\n", + " '_write_sheet_data',\n", + " '_write_sheet_format_pr',\n", + " '_write_sheet_pr',\n", + " '_write_sheet_protection',\n", + " '_write_sheet_view',\n", + " '_write_sheet_views',\n", + " '_write_single_row',\n", + " '_write_spark_color',\n", + " '_write_sparkline_group',\n", + " '_write_sparkline_groups',\n", + " '_write_sparklines',\n", + " '_write_split_panes',\n", + " '_write_string',\n", + " '_write_tab_color',\n", + " '_write_table_part',\n", + " '_write_table_parts',\n", + " '_write_token_as_string',\n", + " '_write_underline',\n", + " '_write_url',\n", + " '_write_vert_align',\n", + " '_write_worksheet',\n", + " '_write_x14_axis_color',\n", + " '_write_x14_border_color',\n", + " '_write_x14_cf_rule',\n", + " '_write_x14_cfvo',\n", + " '_write_x14_data_bar',\n", + " '_write_x14_negative_border_color',\n", + " '_write_x14_negative_fill_color',\n", + " '_xml_close',\n", + " '_xml_data_element',\n", + " '_xml_declaration',\n", + " '_xml_empty_tag',\n", + " '_xml_empty_tag_unencoded',\n", + " '_xml_end_tag',\n", + " '_xml_formula_element',\n", + " '_xml_inline_string',\n", + " '_xml_number_element',\n", + " '_xml_rich_inline_string',\n", + " '_xml_rich_si_element',\n", + " '_xml_si_element',\n", + " '_xml_start_tag',\n", + " '_xml_start_tag_unencoded',\n", + " '_xml_string_element',\n", + " 'activate',\n", + " 'active',\n", + " 'active_pane',\n", + " 'add_sparkline',\n", + " 'add_table',\n", + " 'autofilter',\n", + " 'autofilter_area',\n", + " 'autofilter_ref',\n", + " 'black_white',\n", + " 'buttons_list',\n", + " 'center_horizontally',\n", + " 'center_vertically',\n", + " 'charts',\n", + " 'col_formats',\n", + " 'col_size_changed',\n", + " 'col_sizes',\n", + " 'colinfo',\n", + " 'comments',\n", + " 'comments_author',\n", + " 'comments_list',\n", + " 'comments_visible',\n", + " 'cond_formats',\n", + " 'conditional_format',\n", + " 'constant_memory',\n", + " 'data_bars_2010',\n", + " 'data_validation',\n", + " 'date_1904',\n", + " 'default_col_pixels',\n", + " 'default_date_format',\n", + " 'default_row_height',\n", + " 'default_row_pixels',\n", + " 'default_row_zeroed',\n", + " 'default_url_format',\n", + " 'dim_colmax',\n", + " 'dim_colmin',\n", + " 'dim_rowmax',\n", + " 'dim_rowmin',\n", + " 'draft_quality',\n", + " 'drawing',\n", + " 'drawing_links',\n", + " 'dxf_priority',\n", + " 'escapes',\n", + " 'excel2003_style',\n", + " 'excel_version',\n", + " 'ext_sheets',\n", + " 'external_comment_links',\n", + " 'external_drawing_links',\n", + " 'external_hyper_links',\n", + " 'external_table_links',\n", + " 'external_vml_links',\n", + " 'fh',\n", + " 'fileclosed',\n", + " 'filter_cols',\n", + " 'filter_column',\n", + " 'filter_column_list',\n", + " 'filter_on',\n", + " 'filter_range',\n", + " 'filter_type',\n", + " 'fit_height',\n", + " 'fit_page',\n", + " 'fit_to_pages',\n", + " 'fit_width',\n", + " 'footer',\n", + " 'footer_images',\n", + " 'freeze_panes',\n", + " 'get_name',\n", + " 'has_comments',\n", + " 'has_header_vml',\n", + " 'has_vml',\n", + " 'hbreaks',\n", + " 'hcenter',\n", + " 'header',\n", + " 'header_footer_aligns',\n", + " 'header_footer_changed',\n", + " 'header_footer_scales',\n", + " 'header_images',\n", + " 'header_images_list',\n", + " 'hidden',\n", + " 'hide',\n", + " 'hide_gridlines',\n", + " 'hide_row_col_headers',\n", + " 'hide_zero',\n", + " 'hlink_count',\n", + " 'hlink_refs',\n", + " 'horizontal_dpi',\n", + " 'hyperlinks',\n", + " 'images',\n", + " 'index',\n", + " 'insert_button',\n", + " 'insert_chart',\n", + " 'insert_image',\n", + " 'insert_textbox',\n", + " 'internal_fh',\n", + " 'is_chartsheet',\n", + " 'is_right_to_left',\n", + " 'last_shape_id',\n", + " 'leading_zeros',\n", + " 'margin_bottom',\n", + " 'margin_footer',\n", + " 'margin_header',\n", + " 'margin_left',\n", + " 'margin_right',\n", + " 'margin_top',\n", + " 'merge',\n", + " 'merge_range',\n", + " 'name',\n", + " 'names',\n", + " 'nan_inf_to_errors',\n", + " 'orientation',\n", + " 'original_row_height',\n", + " 'outline_below',\n", + " 'outline_changed',\n", + " 'outline_col_level',\n", + " 'outline_on',\n", + " 'outline_right',\n", + " 'outline_row_level',\n", + " 'outline_settings',\n", + " 'outline_style',\n", + " 'page_order',\n", + " 'page_setup_changed',\n", + " 'page_start',\n", + " 'page_view',\n", + " 'palette',\n", + " 'panes',\n", + " 'paper_size',\n", + " 'previous_row',\n", + " 'print_across',\n", + " 'print_area',\n", + " 'print_area_range',\n", + " 'print_comments',\n", + " 'print_gridlines',\n", + " 'print_headers',\n", + " 'print_options_changed',\n", + " 'print_row_col_headers',\n", + " 'print_scale',\n", + " 'protect',\n", + " 'protect_options',\n", + " 'rel_count',\n", + " 'remove_timezone',\n", + " 'repeat_col_range',\n", + " 'repeat_columns',\n", + " 'repeat_row_range',\n", + " 'repeat_rows',\n", + " 'right_to_left',\n", + " 'row_col_headers',\n", + " 'row_data_fh',\n", + " 'row_data_fh_closed',\n", + " 'row_data_filename',\n", + " 'row_size_changed',\n", + " 'row_sizes',\n", + " 'row_spans',\n", + " 'rstring',\n", + " 'screen_gridlines',\n", + " 'select',\n", + " 'selected',\n", + " 'selections',\n", + " 'set_cols',\n", + " 'set_column',\n", + " 'set_comments_author',\n", + " 'set_default_row',\n", + " 'set_first_sheet',\n", + " 'set_footer',\n", + " 'set_h_pagebreaks',\n", + " 'set_header',\n", + " 'set_landscape',\n", + " 'set_margins',\n", + " 'set_page_view',\n", + " 'set_paper',\n", + " 'set_portrait',\n", + " 'set_print_scale',\n", + " 'set_row',\n", + " 'set_rows',\n", + " 'set_selection',\n", + " 'set_start_page',\n", + " 'set_tab_color',\n", + " 'set_v_pagebreaks',\n", + " 'set_vba_name',\n", + " 'set_zoom',\n", + " 'shape_hash',\n", + " 'shapes',\n", + " 'show_comments',\n", + " 'show_zeros',\n", + " 'sparklines',\n", + " 'split_panes',\n", + " 'str_table',\n", + " 'strings_to_formulas',\n", + " 'strings_to_numbers',\n", + " 'strings_to_urls',\n", + " 'tab_color',\n", + " 'table',\n", + " 'tables',\n", + " 'tmpdir',\n", + " 'use_data_bars_2010',\n", + " 'validations',\n", + " 'vba_codename',\n", + " 'vbreaks',\n", + " 'vcenter',\n", + " 'vertical_dpi',\n", + " 'vml_data_id',\n", + " 'vml_drawing_links',\n", + " 'vml_header_id',\n", + " 'vml_shape_id',\n", + " 'worksheet_meta',\n", + " 'write',\n", + " 'write_array_formula',\n", + " 'write_blank',\n", + " 'write_boolean',\n", + " 'write_column',\n", + " 'write_comment',\n", + " 'write_datetime',\n", + " 'write_formula',\n", + " 'write_match',\n", + " 'write_number',\n", + " 'write_rich_string',\n", + " 'write_row',\n", + " 'write_string',\n", + " 'write_url',\n", + " 'xls_colmax',\n", + " 'xls_rowmax',\n", + " 'xls_strmax',\n", + " 'zoom',\n", + " 'zoom_scale_normal']" + ] + }, + "execution_count": 80, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dir(worksheet)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.15rc1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/pandas_risk.py b/src/pandas_risk.py index e3af3e0..665582d 100644 --- a/src/pandas_risk.py +++ b/src/pandas_risk.py @@ -87,12 +87,13 @@ class deid : x_i = pd.DataFrame(self._df) elif args and 'sample' in args : x_i = args['sample'] - if (args and 'cols' not in args) or not args : + if not args or 'cols' not in args: cols = x_i.columns.tolist() # cols = self._df.columns.tolist() elif args and 'cols' in args : cols = args['cols'] flag = args['flag'] if 'flag' in args else 'UNFLAGGED' + MIN_GROUP_SIZE = args['min_group_size'] if 'min_group_size' in args else 1 # if args and 'sample' in args : # x_i = pd.DataFrame(self._df) @@ -100,15 +101,16 @@ class deid : # cols = args['cols'] if 'cols' in args else self._df.columns.tolist() # x_i = x_i.groupby(cols,as_index=False).size().values x_i_values = x_i.groupby(cols,as_index=False).size().values - SAMPLE_GROUP_COUNT = x_i_values.size + SAMPLE_GROUP_COUNT = x_i_values.size SAMPLE_FIELD_COUNT = len(cols) SAMPLE_POPULATION = x_i_values.sum() - + UNIQUE_REC_RATIO = np.divide(np.sum(x_i_values <= MIN_GROUP_SIZE) , np.float64( SAMPLE_POPULATION)) SAMPLE_MARKETER = SAMPLE_GROUP_COUNT / np.float64(SAMPLE_POPULATION) SAMPLE_PROSECUTOR = 1/ np.min(x_i_values).astype(np.float64) if 'pop' in args : Yi = args['pop'] y_i= pd.DataFrame({"group_size":Yi.groupby(cols,as_index=False).size()}).reset_index() + UNIQUE_REC_RATIO = np.sum(y_i.group_size < MIN_GROUP_SIZE) , np.float64(Yi.shape[0]) # y_i['group'] = pd.DataFrame({"group_size":args['pop'].groupby(cols,as_index=False).size().values}).reset_index() # x_i = pd.DataFrame({"group_size":x_i.groupby(cols,as_index=False).size().values}).reset_index() x_i = pd.DataFrame({"group_size":x_i.groupby(cols,as_index=False).size()}).reset_index() @@ -120,7 +122,8 @@ class deid : r['sample marketer'] = np.repeat(SAMPLE_MARKETER,r.shape[0]) r = r.groupby(['sample %','tier','sample marketer'],as_index=False).sum()[['sample %','marketer','sample marketer','tier']] else: - r = pd.DataFrame({"marketer":[SAMPLE_MARKETER],"prosecutor":[SAMPLE_PROSECUTOR],"field_count":[SAMPLE_FIELD_COUNT],"group_count":[SAMPLE_GROUP_COUNT]}) + r = pd.DataFrame({"marketer":[SAMPLE_MARKETER],"flag":[flag],"prosecutor":[SAMPLE_PROSECUTOR],"field_count":[SAMPLE_FIELD_COUNT],"group_count":[SAMPLE_GROUP_COUNT]}) + r['unique_row_ratio'] = np.repeat(UNIQUE_REC_RATIO,r.shape[0]) return r def _risk(self,**args): diff --git a/src/sql/basic-observation.sql b/src/sql/basic-observation.sql new file mode 100644 index 0000000..96b7067 --- /dev/null +++ b/src/sql/basic-observation.sql @@ -0,0 +1,83 @@ +SELECT person.person_id,sex_at_birth,birth_date, race,zip,city,state, gender +FROM + (SELECT DISTINCT person_id from deid_tmp.observation order by person_id) as person +FULL JOIN ( + SELECT + person_id,MAX(value_as_string) as race + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'Race_WhatRace') and value_as_string IS NOT NULL + + GROUP BY person_id + order by person_id +) as lang +ON lang.person_id = person.person_id + +FULL JOIN ( + SELECT + person_id,MAX(value_as_string) as zip + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'PIIZIP') and value_as_string IS NOT NULL + GROUP BY person_id + order by person_id +) as work_add +ON work_add.person_id = person.person_id + + + +FULL JOIN ( + SELECT + person_id,max(value_as_string) as city + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'PIICity') and value_as_string IS NOT NULL + GROUP BY person_id + order by person_id +) as u_city +ON u_city.person_id = person.person_id + +FULL JOIN ( + SELECT + person_id,max(value_as_string) as state + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'PIIState') and value_as_string IS NOT NULL + GROUP BY person_id + order by person_id + +) as p_addr_o +ON p_addr_o.person_id = person.person_id + +FULL JOIN ( + SELECT + person_id,max(value_as_string) as gender + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'Gender_GenderIdentity') and value_as_string IS NOT NULL + GROUP BY person_id + order by person_id + +) as p_gender +ON p_gender.person_id = person.person_id + +FULL JOIN ( + SELECT + person_id,max(value_as_string) as birth_date + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'PIIBirthInformation_BirthDate') and value_as_string IS NOT NULL + GROUP BY person_id + order by person_id + +) as p_birth +ON p_birth.person_id = person.person_id + +FULL JOIN ( + SELECT + person_id,max(value_as_string) as sex_at_birth + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'BiologicalSexAtBirth_SexAtBirth') and value_as_string IS NOT NULL + GROUP BY person_id + order by person_id + +) as p_sex +ON p_sex.person_id = person.person_id + + +ORDER BY person.person_id + diff --git a/src/sql/observation-registered.sql b/src/sql/observation-registered.sql new file mode 100644 index 0000000..abfb249 --- /dev/null +++ b/src/sql/observation-registered.sql @@ -0,0 +1,376 @@ +SELECT * +FROM ( + SELECT person.person_id,first_name,last_name,birth_date,city,family_history_aware,current_hyper_tension,sex_at_birth, race,state, gender,ethnicity,birth_place,orientation,education,employment_status, + marital_status,language,home_owner,sd_bloodbank, nhpi, living_situation,income,death_cause, death_date, active_duty_status, + gender_identity, insurance_type, work_address_state,consent_18_years_age,person_one_state,person_two_state,sc_site, + health_abroad_6_months,travel_abroad_6_months + FROM + (SELECT DISTINCT person_id from deid_tmp.observation order by person_id) as person + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as travel_abroad_6_months + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'OutsideTravel6Month_OutsideTravel6MonthWhere') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as te_ + ON te_.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as health_abroad_6_months + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'OverallHealth_OutsideTravel6Month') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as he_ + ON he_.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as active_duty_status + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'ActiveDuty_AvtiveDutyServeStatus') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as mil_ + ON mil_.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as sc_site + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'SouthCarolinaSitePairing_EauClaireAppointment') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as sc_ + ON sc_.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as person_one_state + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'PersonOneAddress_PersonOneAddressState') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as p1_ + ON p1_.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as person_two_state + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'SecondContactsAddress_SecondContactState') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as p2_ + ON p2_.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,MAX(value_as_string) as work_address_state + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'EmploymentWorkAddress_State') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as ws_ + ON ws_.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as consent_18_years_age + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'ExtraConsent_18YearsofAge') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as c18_ + ON c18_.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as gender_identity + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'Gender_GenderIdentity') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as gi_ + ON gi_.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as income + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'Income_AnnualIncome') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as income_ + ON income_.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as living_situation + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'LivingSituation_CurrentLiving') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as living_ + ON living_.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as nhpi + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'NHPI_NHPISpecific') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as nhpi_ + ON nhpi_.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,MAX(value_as_string) as sd_bloodbank + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'SanDiegoBloodBank') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as sd + ON sd.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as education + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'EducationLevel_HighestGrade') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as edu + ON edu.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as home_owner + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'HomeOwn_CurrentHomeOwn') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as h_owner + ON h_owner.person_id = person.person_id + + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as employment_status + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'Employment_EmploymentStatus') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as empl + ON empl.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as marital_status + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'MaritalStatus_CurrentMaritalStatus') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as marital + ON marital.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as language + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'Language_SpokenWrittenLanguage') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as lang_ + ON lang_.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as race + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'Race_WhatRace') and value_source_value IS NOT NULL + + GROUP BY person_id + order by person_id + ) as lang + ON lang.person_id = person.person_id + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as ethnicity + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'Race_WhatRaceEthnicity') and value_source_value IS NOT NULL + + GROUP BY person_id + order by person_id + ) as ethnic + ON ethnic.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as birth_place + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'TheBasics_Birthplace') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as birthp + ON birthp.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,MAX(value_source_value) as orientation + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'TheBasics_SexualOrientation') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + ) as sexo + ON sexo.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,max(value_source_value) as state + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'PIIState') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + + ) as p_addr_o + ON p_addr_o.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,max(value_source_value) as gender + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'Gender_GenderIdentity') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + + ) as p_gender + ON p_gender.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,max(value_source_value) as sex_at_birth + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'_SexAtBirth') --and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + + ) as p_sex + ON p_sex.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,max(value_source_value) as insurance_type + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'HealthInsurance_HealthInsuranceType') and value_source_value IS NOT NULL + GROUP BY person_id + order by person_id + + ) as ins_ + ON ins_.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,max(value_as_string) as last_name + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'PIIName_Last') and value_as_string IS NOT NULL + GROUP BY person_id + order by person_id + + ) as ln_ + ON ln_.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,max(value_as_string) as first_name + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'PIIName_First') + GROUP BY person_id + order by person_id + + ) as fn_ + ON fn_.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,max(value_as_string) as current_hyper_tension + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'Circulatory_HypertensionCurrently') + GROUP BY person_id + order by person_id + + ) as cht_ + ON cht_.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,max( cast(value_as_string as DATE)) as birth_date + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'PIIBirthInformation_BirthDate') + GROUP BY person_id + order by person_id + + ) as bd_ + ON bd_.person_id = person.person_id + + + FULL JOIN ( + SELECT + person_id,max(value_as_string) as city + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'StreetAddress_PIICity') + GROUP BY person_id + order by person_id + + ) as city_ + ON city_.person_id = person.person_id + + FULL JOIN ( + SELECT + person_id,max(value_as_string) as family_history_aware + FROM deid_tmp.observation + WHERE REGEXP_CONTAINS(observation_source_value,'FamilyHistory_FamilyMedicalHistoryAware') + GROUP BY person_id + order by person_id + + ) as bro_ + ON bro_.person_id = person.person_id + FULL JOIN ( + SELECT person_id, max(death_date) AS death_date + FROM deid_tmp.death + GROUP BY person_id + order BY person_id + + ) as death_ + ON death_.person_id = person.person_id + + FULL JOIN ( + SELECT person_id, max(cause_source_value) as death_cause + FROM deid_tmp.death + GROUP BY person_id + order BY person_id + + + ) as death_c ON death_c.person_id = person.person_id + ORDER BY person.person_id +) as frame + +-- WHERE first_name is not NULL