Merge pull request #37 from tomwhite/impala
Add scripts for creating a schema and importing data into Impala.
This commit is contained in:
Patrick Ryan
GitHub
commit
d99820c5e2
|
|
@ -0,0 +1,15 @@
|
|||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_CARE_SITE.csv' OVERWRITE INTO TABLE care_site;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_CONDITION_OCCURRENCE.csv' OVERWRITE INTO TABLE condition_occurrence;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_DEATH.csv' OVERWRITE INTO TABLE death;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_DRUG_EXPOSURE.csv' OVERWRITE INTO TABLE drug_exposure;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_DEVICE_EXPOSURE.csv' OVERWRITE INTO TABLE device_exposure;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_LOCATION.csv' OVERWRITE INTO TABLE `location`;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_MEASUREMENT.csv' OVERWRITE INTO TABLE measurement;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_OBSERVATION.csv' OVERWRITE INTO TABLE observation;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_OBSERVATION_PERIOD.csv' OVERWRITE INTO TABLE observation_period;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_PERSON.csv' OVERWRITE INTO TABLE person;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_PROCEDURE_OCCURRENCE.csv' OVERWRITE INTO TABLE procedure_occurrence;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_PROVIDER.csv' OVERWRITE INTO TABLE provider;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_VISIT_OCCURRENCE.csv' OVERWRITE INTO TABLE visit_occurrence;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_DRUG_ERA.csv' OVERWRITE INTO TABLE drug_era;
|
||||
LOAD DATA INPATH '${VAR:OMOP_SYNPUF_PATH}/CDM_CONDITION_ERA.csv' OVERWRITE INTO TABLE condition_era;
|
||||
|
|
@ -0,0 +1,763 @@
|
|||
/*********************************************************************************
|
||||
# Copyright 2014-6 Observational Health Data Sciences and Informatics
|
||||
#
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
********************************************************************************/
|
||||
|
||||
/************************
|
||||
|
||||
####### # # ####### ###### ##### ###### # # #######
|
||||
# # ## ## # # # # # # # # ## ## # # #
|
||||
# # # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # ###### # # # # # # # # ######
|
||||
# # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # #
|
||||
####### # # ####### # ##### ###### # # ## #####
|
||||
|
||||
|
||||
script to create OMOP common data model, version 5.0 for Hadoop (Hive/Impala) database
|
||||
|
||||
Based on the PostgreSQL version, with the following changes:
|
||||
* NULL/NOT NULL is not used.
|
||||
* Dates are stored as VARCHAR(8). See http://stackoverflow.com/questions/33024309/convert-yyyymmdd-string-to-date-in-impala for how to treat date columns as dates.
|
||||
* PostgreSQL NUMERIC is stored as DOUBLE for amounts, and DECIMAL(19,4) for prices.
|
||||
* PostgreSQL TEXT is stored as STRING.
|
||||
|
||||
*************************/
|
||||
|
||||
|
||||
/************************
|
||||
|
||||
Standardized vocabulary
|
||||
|
||||
************************/
|
||||
|
||||
|
||||
CREATE TABLE concept (
|
||||
concept_id INTEGER,
|
||||
concept_name VARCHAR(255),
|
||||
domain_id VARCHAR(20),
|
||||
vocabulary_id VARCHAR(20),
|
||||
concept_class_id VARCHAR(20),
|
||||
standard_concept VARCHAR(1),
|
||||
concept_code VARCHAR(50),
|
||||
valid_start_date VARCHAR(8), -- DATE
|
||||
valid_end_date VARCHAR(8), -- DATE
|
||||
invalid_reason VARCHAR(1)
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
CREATE TABLE vocabulary (
|
||||
vocabulary_id VARCHAR(20),
|
||||
vocabulary_name VARCHAR(255),
|
||||
vocabulary_reference VARCHAR(255),
|
||||
vocabulary_version VARCHAR(255),
|
||||
vocabulary_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
CREATE TABLE domain (
|
||||
domain_id VARCHAR(20),
|
||||
domain_name VARCHAR(255),
|
||||
domain_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE concept_class (
|
||||
concept_class_id VARCHAR(20),
|
||||
concept_class_name VARCHAR(255),
|
||||
concept_class_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
CREATE TABLE concept_relationship (
|
||||
concept_id_1 INTEGER,
|
||||
concept_id_2 INTEGER,
|
||||
relationship_id VARCHAR(20),
|
||||
valid_start_date VARCHAR(8), -- DATE
|
||||
valid_end_date VARCHAR(8), -- DATE
|
||||
invalid_reason VARCHAR(1)
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE relationship (
|
||||
relationship_id VARCHAR(20),
|
||||
relationship_name VARCHAR(255),
|
||||
is_hierarchical VARCHAR(1),
|
||||
defines_ancestry VARCHAR(1),
|
||||
reverse_relationship_id VARCHAR(20),
|
||||
relationship_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
CREATE TABLE concept_synonym (
|
||||
concept_id INTEGER,
|
||||
concept_synonym_name VARCHAR(1000),
|
||||
language_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
CREATE TABLE concept_ancestor (
|
||||
ancestor_concept_id INTEGER,
|
||||
descendant_concept_id INTEGER,
|
||||
min_levels_of_separation INTEGER,
|
||||
max_levels_of_separation INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE source_to_concept_map (
|
||||
source_code VARCHAR(50),
|
||||
source_concept_id INTEGER,
|
||||
source_vocabulary_id VARCHAR(20),
|
||||
source_code_description VARCHAR(255),
|
||||
target_concept_id INTEGER,
|
||||
target_vocabulary_id VARCHAR(20),
|
||||
valid_start_date VARCHAR(8), -- DATE
|
||||
valid_end_date VARCHAR(8), -- DATE
|
||||
invalid_reason VARCHAR(1)
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
CREATE TABLE drug_strength (
|
||||
drug_concept_id INTEGER,
|
||||
ingredient_concept_id INTEGER,
|
||||
amount_value DOUBLE, -- NUMERIC
|
||||
amount_unit_concept_id INTEGER,
|
||||
numerator_value DOUBLE, -- NUMERIC
|
||||
numerator_unit_concept_id INTEGER,
|
||||
denominator_value DOUBLE, -- NUMERIC
|
||||
denominator_unit_concept_id INTEGER,
|
||||
box_size INTEGER,
|
||||
valid_start_date VARCHAR(8), -- DATE
|
||||
valid_end_date VARCHAR(8), -- DATE
|
||||
invalid_reason VARCHAR(1)
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE cohort_definition (
|
||||
cohort_definition_id INTEGER,
|
||||
cohort_definition_name VARCHAR(255),
|
||||
cohort_definition_description STRING, -- TEXT
|
||||
definition_type_concept_id INTEGER,
|
||||
cohort_definition_syntax STRING, -- TEXT
|
||||
subject_concept_id INTEGER,
|
||||
cohort_initiation_date VARCHAR(8) -- DATE
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
CREATE TABLE attribute_definition (
|
||||
attribute_definition_id INTEGER,
|
||||
attribute_name VARCHAR(255),
|
||||
attribute_description STRING, -- TEXT
|
||||
attribute_type_concept_id INTEGER,
|
||||
attribute_syntax STRING -- TEXT
|
||||
)
|
||||
;
|
||||
|
||||
/**************************
|
||||
|
||||
Standardized meta-data
|
||||
|
||||
***************************/
|
||||
|
||||
|
||||
CREATE TABLE cdm_source
|
||||
(
|
||||
cdm_source_name VARCHAR(255),
|
||||
cdm_source_abbreviation VARCHAR(25),
|
||||
cdm_holder VARCHAR(255),
|
||||
source_description STRING, -- TEXT
|
||||
source_documentation_reference VARCHAR(255),
|
||||
cdm_etl_reference VARCHAR(255),
|
||||
source_release_date VARCHAR(8), -- DATE
|
||||
cdm_release_date VARCHAR(8), -- DATE
|
||||
cdm_version VARCHAR(10),
|
||||
vocabulary_version VARCHAR(20)
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/************************
|
||||
|
||||
Standardized clinical data
|
||||
|
||||
************************/
|
||||
|
||||
|
||||
CREATE TABLE person (
|
||||
person_id INTEGER,
|
||||
gender_concept_id INTEGER,
|
||||
year_of_birth INTEGER,
|
||||
month_of_birth INTEGER,
|
||||
day_of_birth INTEGER,
|
||||
time_of_birth VARCHAR(10),
|
||||
race_concept_id INTEGER,
|
||||
ethnicity_concept_id INTEGER,
|
||||
location_id INTEGER,
|
||||
provider_id INTEGER,
|
||||
care_site_id INTEGER,
|
||||
person_source_value VARCHAR(50),
|
||||
gender_source_value VARCHAR(50),
|
||||
gender_source_concept_id INTEGER,
|
||||
race_source_value VARCHAR(50),
|
||||
race_source_concept_id INTEGER,
|
||||
ethnicity_source_value VARCHAR(50),
|
||||
ethnicity_source_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
CREATE TABLE observation_period (
|
||||
observation_period_id INTEGER,
|
||||
person_id INTEGER,
|
||||
observation_period_start_date VARCHAR(8), -- DATE
|
||||
observation_period_end_date VARCHAR(8), -- DATE
|
||||
period_type_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE specimen (
|
||||
specimen_id INTEGER,
|
||||
person_id INTEGER,
|
||||
specimen_concept_id INTEGER,
|
||||
specimen_type_concept_id INTEGER,
|
||||
specimen_date VARCHAR(8), -- DATE
|
||||
specimen_time VARCHAR(10),
|
||||
quantity DOUBLE, -- NUMERIC
|
||||
unit_concept_id INTEGER,
|
||||
anatomic_site_concept_id INTEGER,
|
||||
disease_status_concept_id INTEGER,
|
||||
specimen_source_id VARCHAR(50),
|
||||
specimen_source_value VARCHAR(50),
|
||||
unit_source_value VARCHAR(50),
|
||||
anatomic_site_source_value VARCHAR(50),
|
||||
disease_status_source_value VARCHAR(50)
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE death (
|
||||
person_id INTEGER,
|
||||
death_date VARCHAR(8), -- DATE
|
||||
death_type_concept_id INTEGER,
|
||||
cause_concept_id INTEGER,
|
||||
cause_source_value VARCHAR(50),
|
||||
cause_source_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE visit_occurrence (
|
||||
visit_occurrence_id INTEGER,
|
||||
person_id INTEGER,
|
||||
visit_concept_id INTEGER,
|
||||
visit_start_date VARCHAR(8), -- DATE
|
||||
visit_start_time VARCHAR(10),
|
||||
visit_end_date VARCHAR(8), -- DATE
|
||||
visit_end_time VARCHAR(10),
|
||||
visit_type_concept_id INTEGER,
|
||||
provider_id INTEGER,
|
||||
care_site_id INTEGER,
|
||||
visit_source_value VARCHAR(50),
|
||||
visit_source_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE procedure_occurrence (
|
||||
procedure_occurrence_id INTEGER,
|
||||
person_id INTEGER,
|
||||
procedure_concept_id INTEGER,
|
||||
procedure_date VARCHAR(8), -- DATE
|
||||
procedure_type_concept_id INTEGER,
|
||||
modifier_concept_id INTEGER,
|
||||
quantity INTEGER,
|
||||
provider_id INTEGER,
|
||||
visit_occurrence_id INTEGER,
|
||||
procedure_source_value VARCHAR(50),
|
||||
procedure_source_concept_id INTEGER,
|
||||
qualifier_source_value VARCHAR(50)
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE drug_exposure (
|
||||
drug_exposure_id INTEGER,
|
||||
person_id INTEGER,
|
||||
drug_concept_id INTEGER,
|
||||
drug_exposure_start_date VARCHAR(8), -- DATE
|
||||
drug_exposure_end_date VARCHAR(8), -- DATE
|
||||
drug_type_concept_id INTEGER,
|
||||
stop_reason VARCHAR(20),
|
||||
refills INTEGER,
|
||||
quantity DOUBLE, -- NUMERIC
|
||||
days_supply INTEGER,
|
||||
sig STRING, -- TEXT
|
||||
route_concept_id INTEGER,
|
||||
effective_drug_dose DOUBLE, -- NUMERIC
|
||||
dose_unit_concept_id INTEGER,
|
||||
lot_number VARCHAR(50),
|
||||
provider_id INTEGER,
|
||||
visit_occurrence_id INTEGER,
|
||||
drug_source_value VARCHAR(50),
|
||||
drug_source_concept_id INTEGER,
|
||||
route_source_value VARCHAR(50),
|
||||
dose_unit_source_value VARCHAR(50)
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
CREATE TABLE device_exposure (
|
||||
device_exposure_id INTEGER,
|
||||
person_id INTEGER,
|
||||
device_concept_id INTEGER,
|
||||
device_exposure_start_date VARCHAR(8), -- DATE
|
||||
device_exposure_end_date VARCHAR(8), -- DATE
|
||||
device_type_concept_id INTEGER,
|
||||
unique_device_id VARCHAR(50),
|
||||
quantity INTEGER,
|
||||
provider_id INTEGER,
|
||||
visit_occurrence_id INTEGER,
|
||||
device_source_value VARCHAR(100),
|
||||
device_source_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
CREATE TABLE condition_occurrence (
|
||||
condition_occurrence_id INTEGER,
|
||||
person_id INTEGER,
|
||||
condition_concept_id INTEGER,
|
||||
condition_start_date VARCHAR(8), -- DATE
|
||||
condition_end_date VARCHAR(8), -- DATE
|
||||
condition_type_concept_id INTEGER,
|
||||
stop_reason VARCHAR(20),
|
||||
provider_id INTEGER,
|
||||
visit_occurrence_id INTEGER,
|
||||
condition_source_value VARCHAR(50),
|
||||
condition_source_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE measurement (
|
||||
measurement_id INTEGER,
|
||||
person_id INTEGER,
|
||||
measurement_concept_id INTEGER,
|
||||
measurement_date VARCHAR(8), -- DATE
|
||||
measurement_time VARCHAR(10),
|
||||
measurement_type_concept_id INTEGER,
|
||||
operator_concept_id INTEGER,
|
||||
value_as_number DOUBLE, -- NUMERIC
|
||||
value_as_concept_id INTEGER,
|
||||
unit_concept_id INTEGER,
|
||||
range_low DOUBLE, -- NUMERIC
|
||||
range_high DOUBLE, -- NUMERIC
|
||||
provider_id INTEGER,
|
||||
visit_occurrence_id INTEGER,
|
||||
measurement_source_value VARCHAR(50),
|
||||
measurement_source_concept_id INTEGER,
|
||||
unit_source_value VARCHAR(50),
|
||||
value_source_value VARCHAR(50)
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE note (
|
||||
note_id INTEGER,
|
||||
person_id INTEGER,
|
||||
note_date VARCHAR(8), -- DATE
|
||||
note_time VARCHAR(10),
|
||||
note_type_concept_id INTEGER,
|
||||
note_text STRING, -- TEXT
|
||||
provider_id INTEGER,
|
||||
visit_occurrence_id INTEGER,
|
||||
note_source_value VARCHAR(50)
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE observation (
|
||||
observation_id INTEGER,
|
||||
person_id INTEGER,
|
||||
observation_concept_id INTEGER,
|
||||
observation_date VARCHAR(8), -- DATE
|
||||
observation_time VARCHAR(10),
|
||||
observation_type_concept_id INTEGER,
|
||||
value_as_number DOUBLE, -- NUMERIC
|
||||
value_as_string VARCHAR(60),
|
||||
value_as_concept_id INTEGER,
|
||||
qualifier_concept_id INTEGER,
|
||||
unit_concept_id INTEGER,
|
||||
provider_id INTEGER,
|
||||
visit_occurrence_id INTEGER,
|
||||
observation_source_value VARCHAR(50),
|
||||
observation_source_concept_id INTEGER,
|
||||
unit_source_value VARCHAR(50),
|
||||
qualifier_source_value VARCHAR(50)
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE fact_relationship (
|
||||
domain_concept_id_1 INTEGER,
|
||||
fact_id_1 INTEGER,
|
||||
domain_concept_id_2 INTEGER,
|
||||
fact_id_2 INTEGER,
|
||||
relationship_concept_id INTEGER
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
/************************
|
||||
|
||||
Standardized health system data
|
||||
|
||||
************************/
|
||||
|
||||
|
||||
|
||||
CREATE TABLE `location` (
|
||||
location_id INTEGER,
|
||||
address_1 VARCHAR(50),
|
||||
address_2 VARCHAR(50),
|
||||
city VARCHAR(50),
|
||||
state VARCHAR(2),
|
||||
zip VARCHAR(9),
|
||||
county VARCHAR(20),
|
||||
location_source_value VARCHAR(50)
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE care_site (
|
||||
care_site_id INTEGER,
|
||||
care_site_name VARCHAR(255),
|
||||
place_of_service_concept_id INTEGER,
|
||||
location_id INTEGER,
|
||||
care_site_source_value VARCHAR(50),
|
||||
place_of_service_source_value VARCHAR(50)
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE provider (
|
||||
provider_id INTEGER,
|
||||
provider_name VARCHAR(255),
|
||||
NPI VARCHAR(20),
|
||||
DEA VARCHAR(20),
|
||||
specialty_concept_id INTEGER,
|
||||
care_site_id INTEGER,
|
||||
year_of_birth INTEGER,
|
||||
gender_concept_id INTEGER,
|
||||
provider_source_value VARCHAR(50),
|
||||
specialty_source_value VARCHAR(50),
|
||||
specialty_source_concept_id INTEGER,
|
||||
gender_source_value VARCHAR(50),
|
||||
gender_source_concept_id INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
/************************
|
||||
|
||||
Standardized health economics
|
||||
|
||||
************************/
|
||||
|
||||
|
||||
CREATE TABLE payer_plan_period (
|
||||
payer_plan_period_id INTEGER,
|
||||
person_id INTEGER,
|
||||
payer_plan_period_start_date VARCHAR(8), -- DATE
|
||||
payer_plan_period_end_date VARCHAR(8), -- DATE
|
||||
payer_source_value VARCHAR (50),
|
||||
plan_source_value VARCHAR (50),
|
||||
family_source_value VARCHAR (50)
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
/* The individual cost tables are being phased out and will disappear soon
|
||||
|
||||
CREATE TABLE visit_cost (
|
||||
visit_cost_id INTEGER,
|
||||
visit_occurrence_id INTEGER,
|
||||
currency_concept_id INTEGER,
|
||||
paid_copay DECIMAL(19,4), -- NUMERIC
|
||||
paid_coinsurance DECIMAL(19,4), -- NUMERIC
|
||||
paid_toward_deductible DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_payer DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_coordination_benefits DECIMAL(19,4), -- NUMERIC
|
||||
total_out_of_pocket DECIMAL(19,4), -- NUMERIC
|
||||
total_paid DECIMAL(19,4), -- NUMERIC
|
||||
payer_plan_period_id INTEGER
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE procedure_cost (
|
||||
procedure_cost_id INTEGER,
|
||||
procedure_occurrence_id INTEGER,
|
||||
currency_concept_id INTEGER,
|
||||
paid_copay DECIMAL(19,4), -- NUMERIC
|
||||
paid_coinsurance DECIMAL(19,4), -- NUMERIC
|
||||
paid_toward_deductible DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_payer DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_coordination_benefits DECIMAL(19,4), -- NUMERIC
|
||||
total_out_of_pocket DECIMAL(19,4), -- NUMERIC
|
||||
total_paid DECIMAL(19,4), -- NUMERIC
|
||||
revenue_code_concept_id INTEGER,
|
||||
payer_plan_period_id INTEGER,
|
||||
revenue_code_source_value VARCHAR(50)
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE drug_cost (
|
||||
drug_cost_id INTEGER,
|
||||
drug_exposure_id INTEGER,
|
||||
currency_concept_id INTEGER,
|
||||
paid_copay DECIMAL(19,4), -- NUMERIC
|
||||
paid_coinsurance DECIMAL(19,4), -- NUMERIC
|
||||
paid_toward_deductible DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_payer DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_coordination_benefits DECIMAL(19,4), -- NUMERIC
|
||||
total_out_of_pocket DECIMAL(19,4), -- NUMERIC
|
||||
total_paid DECIMAL(19,4), -- NUMERIC
|
||||
ingredient_cost DECIMAL(19,4), -- NUMERIC
|
||||
dispensing_fee DECIMAL(19,4), -- NUMERIC
|
||||
average_wholesale_price DECIMAL(19,4), -- NUMERIC
|
||||
payer_plan_period_id INTEGER
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
CREATE TABLE device_cost (
|
||||
device_cost_id INTEGER,
|
||||
device_exposure_id INTEGER,
|
||||
currency_concept_id INTEGER,
|
||||
paid_copay DECIMAL(19,4), -- NUMERIC
|
||||
paid_coinsurance DECIMAL(19,4), -- NUMERIC
|
||||
paid_toward_deductible DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_payer DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_coordination_benefits DECIMAL(19,4), -- NUMERIC
|
||||
total_out_of_pocket DECIMAL(19,4), -- NUMERIC
|
||||
total_paid DECIMAL(19,4), -- NUMERIC
|
||||
payer_plan_period_id INTEGER
|
||||
)
|
||||
;
|
||||
*/
|
||||
|
||||
|
||||
CREATE TABLE cost (
|
||||
cost_id INTEGER,
|
||||
cost_event_id INTEGER,
|
||||
cost_domain_id VARCHAR(20),
|
||||
cost_type_concept_id INTEGER,
|
||||
currency_concept_id INTEGER,
|
||||
total_charge DECIMAL(19,4), -- NUMERIC
|
||||
total_cost DECIMAL(19,4), -- NUMERIC
|
||||
total_paid DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_payer DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_patient DECIMAL(19,4), -- NUMERIC
|
||||
paid_patient_copay DECIMAL(19,4), -- NUMERIC
|
||||
paid_patient_coinsurance DECIMAL(19,4), -- NUMERIC
|
||||
paid_patient_deductible DECIMAL(19,4), -- NUMERIC
|
||||
paid_by_primary DECIMAL(19,4), -- NUMERIC
|
||||
paid_ingredient_cost DECIMAL(19,4), -- NUMERIC
|
||||
paid_dispensing_fee DECIMAL(19,4), -- NUMERIC
|
||||
payer_plan_period_id INTEGER,
|
||||
amount_allowed DECIMAL(19,4), -- NUMERIC
|
||||
revenue_code_concept_id INTEGER,
|
||||
reveue_code_source_value VARCHAR(50)
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/************************
|
||||
|
||||
Standardized derived elements
|
||||
|
||||
************************/
|
||||
|
||||
CREATE TABLE cohort (
|
||||
cohort_definition_id INTEGER,
|
||||
subject_id INTEGER,
|
||||
cohort_start_date VARCHAR(8), -- DATE
|
||||
cohort_end_date VARCHAR(8) -- DATE
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
CREATE TABLE cohort_attribute (
|
||||
cohort_definition_id INTEGER,
|
||||
cohort_start_date VARCHAR(8), -- DATE
|
||||
cohort_end_date VARCHAR(8), -- DATE
|
||||
subject_id INTEGER,
|
||||
attribute_definition_id INTEGER,
|
||||
value_as_number DOUBLE, -- NUMERIC
|
||||
value_as_concept_id INTEGER
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
CREATE TABLE drug_era (
|
||||
drug_era_id INTEGER,
|
||||
person_id INTEGER,
|
||||
drug_concept_id INTEGER,
|
||||
drug_era_start_date VARCHAR(8), -- DATE
|
||||
drug_era_end_date VARCHAR(8), -- DATE
|
||||
drug_exposure_count INTEGER,
|
||||
gap_days INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
CREATE TABLE dose_era (
|
||||
dose_era_id INTEGER,
|
||||
person_id INTEGER,
|
||||
drug_concept_id INTEGER,
|
||||
unit_concept_id INTEGER,
|
||||
dose_value DOUBLE, -- NUMERIC
|
||||
dose_era_start_date VARCHAR(8), -- DATE
|
||||
dose_era_end_date VARCHAR(8) -- DATE
|
||||
)
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
CREATE TABLE condition_era (
|
||||
condition_era_id INTEGER,
|
||||
person_id INTEGER,
|
||||
condition_concept_id INTEGER,
|
||||
condition_era_start_date VARCHAR(8), -- DATE
|
||||
condition_era_end_date VARCHAR(8), -- DATE
|
||||
condition_occurrence_count INTEGER
|
||||
)
|
||||
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
|
||||
TBLPROPERTIES ("skip.header.line.count"="1")
|
||||
;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
Common-Data-Model / Impala
|
||||
=================
|
||||
|
||||
This folder contains the SQL scripts for Impala.
|
||||
|
||||
In order to create your instantiation of the Common Data Model, we recommend following these steps:
|
||||
|
||||
1. Create an empty schema.
|
||||
|
||||
```bash
|
||||
impala-shell -q 'CREATE DATABASE omop_cdm'
|
||||
```
|
||||
|
||||
2. Execute the script `OMOP_CDM_ddl_Impala.sql` to create the tables and fields.
|
||||
|
||||
```bash
|
||||
impala-shell -d omop_cdm -f OMOP_CDM_ddl_Impala.sql
|
||||
```
|
||||
|
||||
3. Load your data into the schema.
|
||||
|
||||
a. Load the vocabulary tables.
|
||||
|
||||
First, download the data from
|
||||
[http://www.ohdsi.org/web/athena/](http://www.ohdsi.org/web/athena/)
|
||||
and unzip into a _cdmv5vocab_ directory, then run
|
||||
|
||||
```bash
|
||||
hadoop fs -put cdmv5vocab cdmv5vocab
|
||||
hadoop fs -chmod +w cdmv5vocab
|
||||
impala-shell -d omop_cdm -f VocabImport/OMOP_CDM_vocabulary_load_Impala.sql --var=OMOP_VOCAB_PATH=/user/$USER/cdmv5vocab
|
||||
```
|
||||
|
||||
b. Load the patient data.
|
||||
|
||||
For example, download the 1000 person sample of simulated CMS SynPUF patient data from
|
||||
[http://www.ltscomputingllc.com/downloads/](http://www.ltscomputingllc.com/downloads/)
|
||||
and unzip into a _synpuf_ directory, then run
|
||||
|
||||
```bash
|
||||
hadoop fs -put synpuf synpuf
|
||||
hadoop fs -chmod +w synpuf
|
||||
impala-shell -d omop_cdm -f DataImport/OMOP_CDM_synpuf_load_Impala.sql --var=OMOP_SYNPUF_PATH=/user/$USER/synpuf
|
||||
```
|
||||
|
||||
4. Run simple queries to sanity check.
|
||||
|
||||
```bash
|
||||
impala-shell -d omop_cdm -q 'SELECT COUNT(1) FROM concept'
|
||||
impala-shell -d omop_cdm -q 'SELECT COUNT(1) FROM person'
|
||||
```
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
LOAD DATA INPATH '${VAR:OMOP_VOCAB_PATH}/CONCEPT.csv' OVERWRITE INTO TABLE concept;
|
||||
LOAD DATA INPATH '${VAR:OMOP_VOCAB_PATH}/VOCABULARY.csv' OVERWRITE INTO TABLE vocabulary;
|
||||
LOAD DATA INPATH '${VAR:OMOP_VOCAB_PATH}/DOMAIN.csv' OVERWRITE INTO TABLE domain;
|
||||
LOAD DATA INPATH '${VAR:OMOP_VOCAB_PATH}/CONCEPT_CLASS.csv' OVERWRITE INTO TABLE concept_class;
|
||||
LOAD DATA INPATH '${VAR:OMOP_VOCAB_PATH}/CONCEPT_RELATIONSHIP.csv' OVERWRITE INTO TABLE concept_relationship;
|
||||
LOAD DATA INPATH '${VAR:OMOP_VOCAB_PATH}/RELATIONSHIP.csv' OVERWRITE INTO TABLE relationship;
|
||||
LOAD DATA INPATH '${VAR:OMOP_VOCAB_PATH}/CONCEPT_SYNONYM.csv' OVERWRITE INTO TABLE concept_synonym;
|
||||
LOAD DATA INPATH '${VAR:OMOP_VOCAB_PATH}/CONCEPT_ANCESTOR.csv' OVERWRITE INTO TABLE concept_ancestor;
|
||||
LOAD DATA INPATH '${VAR:OMOP_VOCAB_PATH}/DRUG_STRENGTH.csv' OVERWRITE INTO TABLE drug_strength;
|
||||
Loading…
Reference in New Issue