misc updates ...
This commit is contained in:
Steve Nyemba
parent
c3066408c9
commit
43cbd12a1f
|
|
@ -0,0 +1,385 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\"\"\"\n",
|
||||
"The experiments here describe medical/family history as they associate with risk measures\n",
|
||||
"Additionally we will have fractional risk assessments\n",
|
||||
"\"\"\"\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"from pandas_risk import *\n",
|
||||
"dfm = pd.read_gbq(\"SELECT * FROM deid_risk.registered_medical_history_dec_001\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n",
|
||||
"dff = pd.read_gbq(\"SELECT * FROM deid_risk.registered_family_history_dec_001\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n",
|
||||
"df = pd.read_gbq(\"SELECT person_id, birth_date,city,state,home_owner,race,ethnicity,gender,birth_place,marital_status,orientation,education,employment_status,income,travel_abroad_6_months,active_duty_status FROM deid_risk.registered_dec_01\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 32,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"med_cols = np.random.choice(list(set(dfm.columns.tolist()) - set(['person_id'])),3).tolist()\n",
|
||||
"fam_cols = np.random.choice(list(set(dff.columns.tolist()) - set(['person_id'])),3).tolist()\n",
|
||||
"medical = pd.merge(df,dfm[med_cols+['person_id']],on='person_id')\n",
|
||||
"family = pd.merge(df,dff[fam_cols + ['person_id']],on='person_id')\n",
|
||||
"_tmp = pd.merge(dfm[med_cols +['person_id']],dff[fam_cols+['person_id']])\n",
|
||||
"data = pd.merge(df,_tmp,on='person_id')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>field_count</th>\n",
|
||||
" <th>flag</th>\n",
|
||||
" <th>group_count</th>\n",
|
||||
" <th>marketer</th>\n",
|
||||
" <th>prosecutor</th>\n",
|
||||
" <th>unique_row_ratio</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>21</td>\n",
|
||||
" <td>full history</td>\n",
|
||||
" <td>115308</td>\n",
|
||||
" <td>0.992691</td>\n",
|
||||
" <td>1.0</td>\n",
|
||||
" <td>0.987663</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>18</td>\n",
|
||||
" <td>medical</td>\n",
|
||||
" <td>115306</td>\n",
|
||||
" <td>0.992674</td>\n",
|
||||
" <td>1.0</td>\n",
|
||||
" <td>0.987629</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>18</td>\n",
|
||||
" <td>family</td>\n",
|
||||
" <td>115304</td>\n",
|
||||
" <td>0.992656</td>\n",
|
||||
" <td>1.0</td>\n",
|
||||
" <td>0.987594</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>15</td>\n",
|
||||
" <td>no-history</td>\n",
|
||||
" <td>115300</td>\n",
|
||||
" <td>0.992622</td>\n",
|
||||
" <td>1.0</td>\n",
|
||||
" <td>0.987526</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>3</td>\n",
|
||||
" <td>medical-only</td>\n",
|
||||
" <td>27</td>\n",
|
||||
" <td>0.000232</td>\n",
|
||||
" <td>0.5</td>\n",
|
||||
" <td>0.000000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>3</td>\n",
|
||||
" <td>family-only</td>\n",
|
||||
" <td>146</td>\n",
|
||||
" <td>0.001257</td>\n",
|
||||
" <td>1.0</td>\n",
|
||||
" <td>0.000551</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" field_count flag group_count marketer prosecutor \\\n",
|
||||
"0 21 full history 115308 0.992691 1.0 \n",
|
||||
"1 18 medical 115306 0.992674 1.0 \n",
|
||||
"2 18 family 115304 0.992656 1.0 \n",
|
||||
"3 15 no-history 115300 0.992622 1.0 \n",
|
||||
"4 3 medical-only 27 0.000232 0.5 \n",
|
||||
"5 3 family-only 146 0.001257 1.0 \n",
|
||||
"\n",
|
||||
" unique_row_ratio \n",
|
||||
"0 0.987663 \n",
|
||||
"1 0.987629 \n",
|
||||
"2 0.987594 \n",
|
||||
"3 0.987526 \n",
|
||||
"4 0.000000 \n",
|
||||
"5 0.000551 "
|
||||
]
|
||||
},
|
||||
"execution_count": 33,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"pd.concat([data.deid.evaluate(flag='full history',cols= list(set(data.columns.tolist()) - set(['person_id'])) )\n",
|
||||
" ,medical.deid.evaluate(flag='medical',cols=list( set(medical.columns.tolist() ) - set(['person_id']) ) )\n",
|
||||
" ,family.deid.evaluate(flag='family',cols=list( set(family.columns.tolist() ) - set(['person_id']) ) )\n",
|
||||
" ,df.deid.evaluate(flag='no-history',cols=list( set(df.columns.tolist() ) - set(['person_id']) ) )\n",
|
||||
" , dfm.deid.evaluate(flag='medical-only',cols=med_cols )\n",
|
||||
" , dff.deid.evaluate(flag='family-only',cols=fam_cols )\n",
|
||||
" ],ignore_index=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from __future__ import division\n",
|
||||
"def evaluate(df) :\n",
|
||||
" cols = list(set(df.columns.tolist()) - set(['person_id']))\n",
|
||||
" \n",
|
||||
" portions = np.round(np.random.random_sample(4),3).tolist() + np.arange(5,105,5).tolist()\n",
|
||||
" \n",
|
||||
" N = df.shape[0] - 1\n",
|
||||
" portions = np.divide(np.multiply(portions,N),100).astype(np.int64)\n",
|
||||
" portions = np.unique([n for n in portions if n > 1])\n",
|
||||
" \n",
|
||||
" r = pd.DataFrame()\n",
|
||||
" for num_rows in portions :\n",
|
||||
" \n",
|
||||
" indices = np.random.choice(N,num_rows,replace=False)\n",
|
||||
"# print (indices.size / N)\n",
|
||||
" flag = \" \".join([str( np.round(100*indices.size/ N,2)),'%'])\n",
|
||||
" r = r.append(df.loc[indices].deid.evaluate(cols=cols,flag=flag,min_group_size=2))\n",
|
||||
" return r"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>field_count</th>\n",
|
||||
" <th>flag</th>\n",
|
||||
" <th>group_count</th>\n",
|
||||
" <th>marketer</th>\n",
|
||||
" <th>prosecutor</th>\n",
|
||||
" <th>unique_row_ratio</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>11</td>\n",
|
||||
" <td>UNFLAGGED</td>\n",
|
||||
" <td>114886</td>\n",
|
||||
" <td>0.989058</td>\n",
|
||||
" <td>1.0</td>\n",
|
||||
" <td>0.980535</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" field_count flag group_count marketer prosecutor unique_row_ratio\n",
|
||||
"0 11 UNFLAGGED 114886 0.989058 1.0 0.980535"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"cols = list(set (df.columns.tolist()) - set(['person_id']))\n",
|
||||
"df[['race','state','gender_identity','ethnicity','marital_status','education','orientation','sex_at_birth','birth_date','travel_abroad_6_months','active_duty_status']].deid.evaluate()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 68,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['person_id',\n",
|
||||
" 'HearingVision_FarSightedness',\n",
|
||||
" 'HearingVision_Glaucoma',\n",
|
||||
" 'Digestive_Pancreatitis']"
|
||||
]
|
||||
},
|
||||
"execution_count": 68,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"#\n",
|
||||
"# This is the merge with medical history\n",
|
||||
"\n",
|
||||
"cols = ['person_id'] + np.random.choice(dfm.columns[1:],3,replace=False).tolist()\n",
|
||||
"p = pd.merge(df,dfm[cols],on='person_id')\n",
|
||||
"cols\n",
|
||||
"# # cols = list(set(p.columns.tolist()) - set(['person_id']))\n",
|
||||
"# evaluate(p) #p.deid.explore(cols=cols,num_runs=100)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"cols = list( set(dfm.columns.tolist()) - set(['person_id']))\n",
|
||||
"cols = np.random.choice(cols,3,replace=False).tolist()\n",
|
||||
"p = pd.merge(dfm[['person_id']+cols],df)\n",
|
||||
"fcols = list(set(p.columns.tolist()) - set(['person_id']))\n",
|
||||
"# dfm[cols].deid.evaluate(cols=list( set(cols) - set(['person_id'])))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"variables": {
|
||||
" \" ; \".join(cols)": "InfectiousDiseases_HepatitisC ; Cancer_StomachCancer ; Circulatory_Hypertension",
|
||||
" p.shape[0] ": "116157",
|
||||
" p[fcols].deid.evaluate() ": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>field_count</th>\n <th>flag</th>\n <th>group_count</th>\n <th>marketer</th>\n <th>prosecutor</th>\n <th>unique_row_ratio</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>37</td>\n <td>UNFLAGGED</td>\n <td>115397</td>\n <td>0.993457</td>\n <td>1.0</td>\n <td>0.98886</td>\n </tr>\n </tbody>\n</table>\n</div>"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### Medical History\n",
|
||||
"\n",
|
||||
" We randomly select three a tributes {{ \" ; \".join(cols)}} . \n",
|
||||
" The dataset associated risk evaluation contains {{ p.shape[0] }} records\n",
|
||||
"{{ p[fcols].deid.evaluate() }}\n",
|
||||
"\n",
|
||||
" \n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 52,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['person_id',\n",
|
||||
" 'InfectiousDiseases_Tuberculosis',\n",
|
||||
" 'SkeletalMuscular_Fibromyalgia',\n",
|
||||
" 'Cancer_ProstateCancer']"
|
||||
]
|
||||
},
|
||||
"execution_count": 52,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"cols"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 67,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"3"
|
||||
]
|
||||
},
|
||||
"execution_count": 67,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# dfm[cols[1:]].head()\n",
|
||||
"np.sum(dfm.fillna(' ').groupby(cols[1:],as_index=False).size().values <= 1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 2",
|
||||
"language": "python",
|
||||
"name": "python2"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 2
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython2",
|
||||
"version": "2.7.15rc1"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -87,12 +87,13 @@ class deid :
|
|||
x_i = pd.DataFrame(self._df)
|
||||
elif args and 'sample' in args :
|
||||
x_i = args['sample']
|
||||
if (args and 'cols' not in args) or not args :
|
||||
if not args or 'cols' not in args:
|
||||
cols = x_i.columns.tolist()
|
||||
# cols = self._df.columns.tolist()
|
||||
elif args and 'cols' in args :
|
||||
cols = args['cols']
|
||||
flag = args['flag'] if 'flag' in args else 'UNFLAGGED'
|
||||
MIN_GROUP_SIZE = args['min_group_size'] if 'min_group_size' in args else 1
|
||||
# if args and 'sample' in args :
|
||||
|
||||
# x_i = pd.DataFrame(self._df)
|
||||
|
|
@ -103,12 +104,13 @@ class deid :
|
|||
SAMPLE_GROUP_COUNT = x_i_values.size
|
||||
SAMPLE_FIELD_COUNT = len(cols)
|
||||
SAMPLE_POPULATION = x_i_values.sum()
|
||||
|
||||
UNIQUE_REC_RATIO = np.divide(np.sum(x_i_values <= MIN_GROUP_SIZE) , np.float64( SAMPLE_POPULATION))
|
||||
SAMPLE_MARKETER = SAMPLE_GROUP_COUNT / np.float64(SAMPLE_POPULATION)
|
||||
SAMPLE_PROSECUTOR = 1/ np.min(x_i_values).astype(np.float64)
|
||||
if 'pop' in args :
|
||||
Yi = args['pop']
|
||||
y_i= pd.DataFrame({"group_size":Yi.groupby(cols,as_index=False).size()}).reset_index()
|
||||
UNIQUE_REC_RATIO = np.sum(y_i.group_size < MIN_GROUP_SIZE) , np.float64(Yi.shape[0])
|
||||
# y_i['group'] = pd.DataFrame({"group_size":args['pop'].groupby(cols,as_index=False).size().values}).reset_index()
|
||||
# x_i = pd.DataFrame({"group_size":x_i.groupby(cols,as_index=False).size().values}).reset_index()
|
||||
x_i = pd.DataFrame({"group_size":x_i.groupby(cols,as_index=False).size()}).reset_index()
|
||||
|
|
@ -120,7 +122,8 @@ class deid :
|
|||
r['sample marketer'] = np.repeat(SAMPLE_MARKETER,r.shape[0])
|
||||
r = r.groupby(['sample %','tier','sample marketer'],as_index=False).sum()[['sample %','marketer','sample marketer','tier']]
|
||||
else:
|
||||
r = pd.DataFrame({"marketer":[SAMPLE_MARKETER],"prosecutor":[SAMPLE_PROSECUTOR],"field_count":[SAMPLE_FIELD_COUNT],"group_count":[SAMPLE_GROUP_COUNT]})
|
||||
r = pd.DataFrame({"marketer":[SAMPLE_MARKETER],"flag":[flag],"prosecutor":[SAMPLE_PROSECUTOR],"field_count":[SAMPLE_FIELD_COUNT],"group_count":[SAMPLE_GROUP_COUNT]})
|
||||
r['unique_row_ratio'] = np.repeat(UNIQUE_REC_RATIO,r.shape[0])
|
||||
return r
|
||||
|
||||
def _risk(self,**args):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
SELECT person.person_id,sex_at_birth,birth_date, race,zip,city,state, gender
|
||||
FROM
|
||||
(SELECT DISTINCT person_id from deid_tmp.observation order by person_id) as person
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_as_string) as race
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'Race_WhatRace') and value_as_string IS NOT NULL
|
||||
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as lang
|
||||
ON lang.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_as_string) as zip
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'PIIZIP') and value_as_string IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as work_add
|
||||
ON work_add.person_id = person.person_id
|
||||
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_as_string) as city
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'PIICity') and value_as_string IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as u_city
|
||||
ON u_city.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_as_string) as state
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'PIIState') and value_as_string IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as p_addr_o
|
||||
ON p_addr_o.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_as_string) as gender
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'Gender_GenderIdentity') and value_as_string IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as p_gender
|
||||
ON p_gender.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_as_string) as birth_date
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'PIIBirthInformation_BirthDate') and value_as_string IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as p_birth
|
||||
ON p_birth.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_as_string) as sex_at_birth
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'BiologicalSexAtBirth_SexAtBirth') and value_as_string IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as p_sex
|
||||
ON p_sex.person_id = person.person_id
|
||||
|
||||
|
||||
ORDER BY person.person_id
|
||||
|
||||
|
|
@ -0,0 +1,376 @@
|
|||
SELECT *
|
||||
FROM (
|
||||
SELECT person.person_id,first_name,last_name,birth_date,city,family_history_aware,current_hyper_tension,sex_at_birth, race,state, gender,ethnicity,birth_place,orientation,education,employment_status,
|
||||
marital_status,language,home_owner,sd_bloodbank, nhpi, living_situation,income,death_cause, death_date, active_duty_status,
|
||||
gender_identity, insurance_type, work_address_state,consent_18_years_age,person_one_state,person_two_state,sc_site,
|
||||
health_abroad_6_months,travel_abroad_6_months
|
||||
FROM
|
||||
(SELECT DISTINCT person_id from deid_tmp.observation order by person_id) as person
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as travel_abroad_6_months
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'OutsideTravel6Month_OutsideTravel6MonthWhere') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as te_
|
||||
ON te_.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as health_abroad_6_months
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'OverallHealth_OutsideTravel6Month') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as he_
|
||||
ON he_.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as active_duty_status
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'ActiveDuty_AvtiveDutyServeStatus') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as mil_
|
||||
ON mil_.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as sc_site
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'SouthCarolinaSitePairing_EauClaireAppointment') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as sc_
|
||||
ON sc_.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as person_one_state
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'PersonOneAddress_PersonOneAddressState') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as p1_
|
||||
ON p1_.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as person_two_state
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'SecondContactsAddress_SecondContactState') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as p2_
|
||||
ON p2_.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_as_string) as work_address_state
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'EmploymentWorkAddress_State') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as ws_
|
||||
ON ws_.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as consent_18_years_age
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'ExtraConsent_18YearsofAge') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as c18_
|
||||
ON c18_.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as gender_identity
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'Gender_GenderIdentity') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as gi_
|
||||
ON gi_.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as income
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'Income_AnnualIncome') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as income_
|
||||
ON income_.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as living_situation
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'LivingSituation_CurrentLiving') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as living_
|
||||
ON living_.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as nhpi
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'NHPI_NHPISpecific') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as nhpi_
|
||||
ON nhpi_.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_as_string) as sd_bloodbank
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'SanDiegoBloodBank') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as sd
|
||||
ON sd.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as education
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'EducationLevel_HighestGrade') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as edu
|
||||
ON edu.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as home_owner
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'HomeOwn_CurrentHomeOwn') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as h_owner
|
||||
ON h_owner.person_id = person.person_id
|
||||
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as employment_status
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'Employment_EmploymentStatus') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as empl
|
||||
ON empl.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as marital_status
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'MaritalStatus_CurrentMaritalStatus') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as marital
|
||||
ON marital.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as language
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'Language_SpokenWrittenLanguage') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as lang_
|
||||
ON lang_.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as race
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'Race_WhatRace') and value_source_value IS NOT NULL
|
||||
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as lang
|
||||
ON lang.person_id = person.person_id
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as ethnicity
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'Race_WhatRaceEthnicity') and value_source_value IS NOT NULL
|
||||
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as ethnic
|
||||
ON ethnic.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as birth_place
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'TheBasics_Birthplace') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as birthp
|
||||
ON birthp.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,MAX(value_source_value) as orientation
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'TheBasics_SexualOrientation') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
) as sexo
|
||||
ON sexo.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_source_value) as state
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'PIIState') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as p_addr_o
|
||||
ON p_addr_o.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_source_value) as gender
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'Gender_GenderIdentity') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as p_gender
|
||||
ON p_gender.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_source_value) as sex_at_birth
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'_SexAtBirth') --and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as p_sex
|
||||
ON p_sex.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_source_value) as insurance_type
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'HealthInsurance_HealthInsuranceType') and value_source_value IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as ins_
|
||||
ON ins_.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_as_string) as last_name
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'PIIName_Last') and value_as_string IS NOT NULL
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as ln_
|
||||
ON ln_.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_as_string) as first_name
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'PIIName_First')
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as fn_
|
||||
ON fn_.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_as_string) as current_hyper_tension
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'Circulatory_HypertensionCurrently')
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as cht_
|
||||
ON cht_.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max( cast(value_as_string as DATE)) as birth_date
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'PIIBirthInformation_BirthDate')
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as bd_
|
||||
ON bd_.person_id = person.person_id
|
||||
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_as_string) as city
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'StreetAddress_PIICity')
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as city_
|
||||
ON city_.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT
|
||||
person_id,max(value_as_string) as family_history_aware
|
||||
FROM deid_tmp.observation
|
||||
WHERE REGEXP_CONTAINS(observation_source_value,'FamilyHistory_FamilyMedicalHistoryAware')
|
||||
GROUP BY person_id
|
||||
order by person_id
|
||||
|
||||
) as bro_
|
||||
ON bro_.person_id = person.person_id
|
||||
FULL JOIN (
|
||||
SELECT person_id, max(death_date) AS death_date
|
||||
FROM deid_tmp.death
|
||||
GROUP BY person_id
|
||||
order BY person_id
|
||||
|
||||
) as death_
|
||||
ON death_.person_id = person.person_id
|
||||
|
||||
FULL JOIN (
|
||||
SELECT person_id, max(cause_source_value) as death_cause
|
||||
FROM deid_tmp.death
|
||||
GROUP BY person_id
|
||||
order BY person_id
|
||||
|
||||
|
||||
) as death_c ON death_c.person_id = person.person_id
|
||||
ORDER BY person.person_id
|
||||
) as frame
|
||||
|
||||
-- WHERE first_name is not NULL
|
||||
Loading…
Reference in New Issue