From 386793b10106f8f91669aae724162902bdafab75 Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Mon, 6 Jul 2015 08:44:31 -0400 Subject: [PATCH 1/5] Addressing bugs from PostgreSQL/Oracle testing --- .../Conversion-QA.sql | 6 +- .../OMOP CDMv4 to CDMv5 - templateSQL.sql | 321 +++++++------ .../Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql | 450 +++++++++--------- .../OMOP CDMv4 to CDMv5 - PostgreSQL.sql | 442 ++++++++--------- .../OMOP CDMv4 to CDMv5 - SQL Server.sql | 443 ++++++++--------- 5 files changed, 862 insertions(+), 800 deletions(-) diff --git a/Version4 To Version5 Conversion/Conversion-QA.sql b/Version4 To Version5 Conversion/Conversion-QA.sql index 5ded8fb..2012c96 100644 --- a/Version4 To Version5 Conversion/Conversion-QA.sql +++ b/Version4 To Version5 Conversion/Conversion-QA.sql @@ -55,7 +55,8 @@ INSTRUCTIONS 2. Run the resulting script on your target RDBDMS. *********************************************************************************/ -USE [TARGET_CDMV5] +--USE [TARGET_CDMV5] +USE [CDMV5_Conversion_Target] GO IF OBJECT_ID('tempdb..#v5_stats', 'U') IS NOT NULL @@ -83,7 +84,8 @@ GROUP BY ORDER BY object_name(i.object_id) -USE [SOURCE_CDMV4] +--USE [SOURCE_CDMV4] +USE [CDM_TRUVEN_CCAE_6k] GO IF OBJECT_ID('tempdb..#v4_stats', 'U') IS NOT NULL diff --git a/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql b/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql index 3af6472..73679d1 100644 --- a/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql +++ b/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql @@ -77,10 +77,30 @@ TemplateSQL File Instructions *********************************************************************************/ /* SCRIPT PARAMETERS */ -{DEFAULT @SOURCE_CDMV4 = '[SOURCE_CDMV4]'} -- The CDMv4 database name -{DEFAULT @SOURCE_CDMV4_SCHEMA = '[SOURCE_CDMV4].[SCHEMA]'} -- The CDMv4 database plus schema -{DEFAULT @TARGET_CDMV5 = '[TARGET_CDMV5]'} -- The target CDMv5 database name -{DEFAULT @TARGET_CDMV5_SCHEMA = '[TARGET_CDMV5].[SCHEMA]'} -- the target CDMv5 database plus schema +--{DEFAULT @SOURCE_CDMV4 = '[SOURCE_CDMV4]'} -- The CDMv4 database name +--{DEFAULT @SOURCE_CDMV4_SCHEMA = '[SOURCE_CDMV4].[SCHEMA]'} -- The CDMv4 database plus schema +--{DEFAULT @TARGET_CDMV5 = '[TARGET_CDMV5]'} -- The target CDMv5 database name +--{DEFAULT @TARGET_CDMV5_SCHEMA = '[TARGET_CDMV5].[SCHEMA]'} -- the target CDMv5 database plus schema +/* LOCAL SQL Server */ +--{DEFAULT @SOURCE_CDMV4 = '[CDMV4]'} -- The CDMv4 database name +--{DEFAULT @SOURCE_CDMV4_SCHEMA = '[CDMV4].[dbo]'} -- The CDMv4 database plus schema +--{DEFAULT @TARGET_CDMV5 = '[CDMV5]'} -- The target CDMv5 database name +--{DEFAULT @TARGET_CDMV5_SCHEMA = '[CDMV5].[dbo]'} -- the target CDMv5 database plus schema +/* QA Settings */ +--{DEFAULT @SOURCE_CDMV4 = 'CDM_TRUVEN_CCAE_6k'} -- The CDMv4 database name +--{DEFAULT @SOURCE_CDMV4_SCHEMA = 'CDM_TRUVEN_CCAE_6k.dbo'} -- The CDMv4 database plus schema +--{DEFAULT @TARGET_CDMV5 = 'CDMV5_Conversion_Target'} -- The target CDMv5 database name +--{DEFAULT @TARGET_CDMV5_SCHEMA = 'CDMV5_Conversion_Target.dbo'} -- the target CDMv5 database plus schema +/* PostgreSQL Settings */ +--{DEFAULT @SOURCE_CDMV4 = 'sandbox'} -- The CDMv4 database name +--{DEFAULT @SOURCE_CDMV4_SCHEMA = 'sandbox.cdmv4'} -- The CDMv4 database plus schema +--{DEFAULT @TARGET_CDMV5 = 'sandbox'} -- The target CDMv5 database name +--{DEFAULT @TARGET_CDMV5_SCHEMA = 'sandbox.cdmv5'} -- the target CDMv5 database plus schema +/* Oracle Settings */ +{DEFAULT @SOURCE_CDMV4 = 'sandbox'} -- The CDMv4 database name +{DEFAULT @SOURCE_CDMV4_SCHEMA = 'sandbox.cdmv4'} -- The CDMv4 database plus schema +{DEFAULT @TARGET_CDMV5 = 'sandbox'} -- The target CDMv5 database name +{DEFAULT @TARGET_CDMV5_SCHEMA = 'sandbox.cdmv5'} -- the target CDMv5 database plus schema USE @TARGET_CDMV5; @@ -117,7 +137,7 @@ select concept_id as source_concept_id, domain_id, NULL as source_concept_mapping_occurrence into #concept_map -from dbo.concept +from @TARGET_CDMV5_SCHEMA.concept where standard_concept = 'S' and invalid_reason is null @@ -131,16 +151,16 @@ select distinct c1.concept_id as source_concept_id, from ( select concept_id -from dbo.concept +from @TARGET_CDMV5_SCHEMA.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -dbo.concept_relationship cr1 +@TARGET_CDMV5_SCHEMA.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +@TARGET_CDMV5_SCHEMA.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -158,7 +178,7 @@ select distinct c1.concept_id as source_concept_id, from ( select concept_id -from dbo.concept +from @TARGET_CDMV5_SCHEMA.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) @@ -167,16 +187,16 @@ and concept_id not in ( from ( select concept_id - from dbo.concept + from @TARGET_CDMV5_SCHEMA.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join - dbo.concept_relationship cr1 + @TARGET_CDMV5_SCHEMA.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join - dbo.concept c2 + @TARGET_CDMV5_SCHEMA.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -186,10 +206,10 @@ and concept_id not in ( ) c1 inner join -dbo.concept_relationship cr1 +@TARGET_CDMV5_SCHEMA.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +@TARGET_CDMV5_SCHEMA.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -214,7 +234,7 @@ select distinct c1.concept_id as source_concept_id, from ( select concept_id -from dbo.concept +from @TARGET_CDMV5_SCHEMA.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) @@ -223,16 +243,16 @@ and concept_id not in ( from ( select concept_id - from dbo.concept + from @TARGET_CDMV5_SCHEMA.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join - dbo.concept_relationship cr1 + @TARGET_CDMV5_SCHEMA.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join - dbo.concept c2 + @TARGET_CDMV5_SCHEMA.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -251,10 +271,10 @@ and concept_id not in ( ) c1 inner join -dbo.concept_relationship cr1 +@TARGET_CDMV5_SCHEMA.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +@TARGET_CDMV5_SCHEMA.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -267,31 +287,31 @@ and cr1.invalid_reason is null; -- the script to ensure that we generate new primary keys -- for the target tables when applicable UPDATE #concept_map - SET #concept_map.source_concept_mapping_occurrence = A.[Rowcount] + SET #concept_map.source_concept_mapping_occurrence = A.CountOfRows FROM #concept_map, ( - select source_concept_id, domain_id, count(*) as "rowcount" + select source_concept_id, domain_id, count(*) as "CountOfRows" from #concept_map group by source_concept_id, domain_id ) AS A -WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id +WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; - IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL +IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL DROP TABLE #concept_map_distinct; SELECT DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" INTO #concept_map_distinct FROM #concept_map - GROUP BY source_concept_id, domain_id + GROUP BY source_concept_id, domain_id; - IF OBJECT_ID('@TARGET_CDMV5_SCHEMA.ETL_WARNINGS') IS NOT NULL - DROP TABLE @TARGET_CDMV5_SCHEMA.ETL_WARNINGS +IF OBJECT_ID('ETL_WARNINGS', 'U') IS NOT NULL + DROP TABLE ETL_WARNINGS; -CREATE TABLE @TARGET_CDMV5_SCHEMA.ETL_WARNINGS +CREATE TABLE ETL_WARNINGS ( -WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(8000) ); /**** @@ -300,9 +320,9 @@ CDM_SOURCE ****/ -INSERT INTO dbo.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +INSERT INTO @TARGET_CDMV5_SCHEMA.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) select '@TARGET_CDMV5', 'V5', v.vocabulary_version, getDate() -from dbo.vocabulary v +from @TARGET_CDMV5_SCHEMA.vocabulary v where vocabulary_id = 'Vocabulary'; /**** @@ -311,7 +331,7 @@ LOCATION ****/ -insert into dbo.location +insert into @TARGET_CDMV5_SCHEMA.location select location_id, address_1, address_2, city, state, zip, county, location_source_value from @SOURCE_CDMV4_SCHEMA.LOCATION; @@ -321,7 +341,7 @@ CARE_SITE ****/ -insert into dbo.care_site +insert into @TARGET_CDMV5_SCHEMA.care_site select care_site_id, cast(null as varchar(255)) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value from @SOURCE_CDMV4_SCHEMA.CARE_SITE; @@ -331,7 +351,7 @@ Provider ****/ -insert dbo.provider +insert into @TARGET_CDMV5_SCHEMA.provider select provider_id, cast(null as varchar(255)) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, cast(null as varchar(50)) as gender_source_value, cast(null as integer) as gender_source_concept_id @@ -345,7 +365,7 @@ from @SOURCE_CDMV4_SCHEMA.provider ****/ - INSERT dbo.person + INSERT into @TARGET_CDMV5_SCHEMA.person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -371,7 +391,7 @@ from @SOURCE_CDMV4_SCHEMA.provider LEFT JOIN #concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( @@ -379,7 +399,7 @@ from @SOURCE_CDMV4_SCHEMA.provider FROM @SOURCE_CDMV4_SCHEMA.PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -390,7 +410,7 @@ from @SOURCE_CDMV4_SCHEMA.provider ) warn ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( @@ -399,7 +419,7 @@ from @SOURCE_CDMV4_SCHEMA.provider WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -410,7 +430,7 @@ from @SOURCE_CDMV4_SCHEMA.provider ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( @@ -419,7 +439,7 @@ from @SOURCE_CDMV4_SCHEMA.provider WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -435,7 +455,7 @@ from @SOURCE_CDMV4_SCHEMA.provider ****/ - INSERT dbo.observation_period + INSERT INTO @TARGET_CDMV5_SCHEMA.observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id FROM @SOURCE_CDMV4_SCHEMA.OBSERVATION_PERIOD; @@ -445,7 +465,7 @@ from @SOURCE_CDMV4_SCHEMA.provider ****/ - INSERT dbo.death + INSERT INTO @TARGET_CDMV5_SCHEMA.death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, @@ -458,7 +478,7 @@ from @SOURCE_CDMV4_SCHEMA.provider AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( @@ -466,7 +486,7 @@ from @SOURCE_CDMV4_SCHEMA.provider FROM @SOURCE_CDMV4_SCHEMA.DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -483,7 +503,7 @@ from @SOURCE_CDMV4_SCHEMA.provider ****/ - INSERT dbo.visit_occurrence + INSERT INTO @TARGET_CDMV5_SCHEMA.visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, CAST(null as varchar(10)) visit_start_time, @@ -497,7 +517,7 @@ from @SOURCE_CDMV4_SCHEMA.provider ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( @@ -505,7 +525,7 @@ from @SOURCE_CDMV4_SCHEMA.provider FROM @SOURCE_CDMV4_SCHEMA.VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -710,37 +730,37 @@ UNION ALL ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO [dbo].[procedure_occurrence] - ([procedure_occurrence_id] - ,[person_id] - ,[procedure_concept_id] - ,[procedure_date] - ,[procedure_type_concept_id] - ,[modifier_concept_id] - ,[quantity] - ,[provider_id] - ,[visit_occurrence_id] - ,[procedure_source_value] - ,[procedure_source_concept_id] - ,[qualifier_source_value]) +INSERT INTO procedure_occurrence + (procedure_occurrence_id + ,person_id + ,procedure_concept_id + ,procedure_date + ,procedure_type_concept_id + ,modifier_concept_id + ,quantity + ,provider_id + ,visit_occurrence_id + ,procedure_source_value + ,procedure_source_concept_id + ,qualifier_source_value) SELECT - [procedure_occurrence_id] - ,[person_id] - ,[procedure_concept_id] - ,[procedure_date] - ,[procedure_type_concept_id] - ,[modifier_concept_id] - ,[quantity] - ,[provider_id] - ,[visit_occurrence_id] - ,[procedure_source_value] - ,[procedure_source_concept_id] - ,[qualifier_source_value] -FROM #procedure_occurrence_map + procedure_occurrence_id + ,person_id + ,procedure_concept_id + ,procedure_date + ,procedure_type_concept_id + ,modifier_concept_id + ,quantity + ,provider_id + ,visit_occurrence_id + ,procedure_source_value + ,procedure_source_concept_id + ,qualifier_source_value +FROM #procedure_occurrence_map; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( @@ -748,7 +768,7 @@ FROM #procedure_occurrence_map FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -758,7 +778,7 @@ FROM #procedure_occurrence_map ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( @@ -766,7 +786,7 @@ FROM #procedure_occurrence_map FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -970,55 +990,55 @@ FROM AND LOWER(cm1.domain_id) IN ('drug') ) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE) MAXROW ; -INSERT INTO [dbo].[drug_exposure] - ([drug_exposure_id] - ,[person_id] - ,[drug_concept_id] - ,[drug_exposure_start_date] - ,[drug_exposure_end_date] - ,[drug_type_concept_id] - ,[stop_reason] - ,[refills] - ,[quantity] - ,[days_supply] - ,[sig] - ,[route_concept_id] - ,[effective_drug_dose] - ,[dose_unit_concept_id] - ,[lot_number] - ,[provider_id] - ,[visit_occurrence_id] - ,[drug_source_value] - ,[drug_source_concept_id] - ,[route_source_value] - ,[dose_unit_source_value]) +INSERT INTO drug_exposure + (drug_exposure_id + ,person_id + ,drug_concept_id + ,drug_exposure_start_date + ,drug_exposure_end_date + ,drug_type_concept_id + ,stop_reason + ,refills + ,quantity + ,days_supply + ,sig + ,route_concept_id + ,effective_drug_dose + ,dose_unit_concept_id + ,lot_number + ,provider_id + ,visit_occurrence_id + ,drug_source_value + ,drug_source_concept_id + ,route_source_value + ,dose_unit_source_value) SELECT - [drug_exposure_id] - ,[person_id] - ,[drug_concept_id] - ,[drug_exposure_start_date] - ,[drug_exposure_end_date] - ,[drug_type_concept_id] - ,[stop_reason] - ,[refills] - ,[quantity] - ,[days_supply] - ,[sig] - ,[route_concept_id] - ,[effective_drug_dose] - ,[dose_unit_concept_id] - ,[lot_number] - ,[provider_id] - ,[visit_occurrence_id] - ,[drug_source_value] - ,[drug_source_concept_id] - ,[route_source_value] - ,[dose_unit_source_value] -FROM #drug_exposure_map + drug_exposure_id + ,person_id + ,drug_concept_id + ,drug_exposure_start_date + ,drug_exposure_end_date + ,drug_type_concept_id + ,stop_reason + ,refills + ,quantity + ,days_supply + ,sig + ,route_concept_id + ,effective_drug_dose + ,dose_unit_concept_id + ,lot_number + ,provider_id + ,visit_occurrence_id + ,drug_source_value + ,drug_source_concept_id + ,route_source_value + ,dose_unit_source_value +FROM #drug_exposure_map; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( @@ -1026,7 +1046,7 @@ FROM #drug_exposure_map FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1036,7 +1056,7 @@ FROM #drug_exposure_map ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( @@ -1044,7 +1064,7 @@ FROM #drug_exposure_map FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1062,7 +1082,7 @@ FROM #drug_exposure_map ****/ --find valid conditions from condition_occurrence table - INSERT dbo.condition_occurrence + INSERT INTO @TARGET_CDMV5_SCHEMA.condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1136,7 +1156,8 @@ FROM ( condition_end_date, COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, - condition_source_value, CAST(null as integer) condition_source_concept_id + condition_source_value, CAST(null as integer) condition_source_concept_id, + NULL as OCCURRENCE_ID FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id @@ -1192,7 +1213,7 @@ FROM ( --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( @@ -1200,7 +1221,7 @@ FROM ( FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1210,7 +1231,7 @@ FROM ( ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( @@ -1218,7 +1239,7 @@ FROM ( FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM @TARGET_CDMV5_SCHEMA.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1237,7 +1258,7 @@ FROM ( ****/ -INSERT dbo.device_exposure +INSERT INTO @TARGET_CDMV5_SCHEMA.device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1308,7 +1329,7 @@ FROM ****/ --find valid measurements from observation table -INSERT dbo.measurement +INSERT INTO @TARGET_CDMV5_SCHEMA.measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1426,7 +1447,7 @@ from --find valid observation from observation table - INSERT dbo.observation + INSERT INTO @TARGET_CDMV5_SCHEMA.observation SELECT observation_id, person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, value_as_number, value_as_string, value_as_concept_id, CAST(null as integer) qualifier_concept_id, unit_concept_id, associated_provider_id as provider_id, @@ -1498,7 +1519,7 @@ FROM ****/ -INSERT dbo.payer_plan_period +INSERT INTO @TARGET_CDMV5_SCHEMA.payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value FROM @SOURCE_CDMV4_SCHEMA.PAYER_PLAN_PERIOD; @@ -1511,7 +1532,7 @@ FROM @SOURCE_CDMV4_SCHEMA.PAYER_PLAN_PERIOD; ****/ -INSERT dbo.drug_cost +INSERT INTO @TARGET_CDMV5_SCHEMA.drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id @@ -1519,7 +1540,7 @@ FROM @SOURCE_CDMV4_SCHEMA.DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO dbo.drug_cost +INSERT INTO @TARGET_CDMV5_SCHEMA.drug_cost select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, @@ -1544,7 +1565,7 @@ FROM ( ****/ -INSERT INTO dbo.procedure_cost +INSERT INTO @TARGET_CDMV5_SCHEMA.procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value @@ -1552,7 +1573,7 @@ FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO dbo.procedure_cost +INSERT INTO @TARGET_CDMV5_SCHEMA.procedure_cost SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, @@ -1583,8 +1604,8 @@ with cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONC COALESCE(DRUG_EXPOSURE_END_DATE, DATEADD(day,DAYS_SUPPLY,DRUG_EXPOSURE_START_DATE), DATEADD(day,1,DRUG_EXPOSURE_START_DATE)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID FROM @TARGET_CDMV5_SCHEMA.DRUG_EXPOSURE d - join dbo.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join dbo.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + join @TARGET_CDMV5_SCHEMA.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join @TARGET_CDMV5_SCHEMA.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1634,7 +1655,7 @@ GROUP BY d.PERSON_ID, d.DRUG_TYPE_CONCEPT_ID, d.DRUG_EXPOSURE_START_DATE ) -insert into dbo.drug_era +insert into @TARGET_CDMV5_SCHEMA.drug_era select row_number() over (order by person_id) as drug_era_id, person_id, drug_concept_id, min(DRUG_EXPOSURE_START_DATE) as drug_era_start_date, drug_era_end_date, COUNT(*) as DRUG_EXPOSURE_COUNT, 30 as gap_days from cteDrugExposureEnds GROUP BY person_id, drug_concept_id, drug_type_concept_id, DRUG_ERA_END_DATE @@ -1708,13 +1729,13 @@ from cteConditionEnds GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO [dbo].[condition_era] - ([condition_era_id] - ,[person_id] - ,[condition_concept_id] - ,[condition_era_start_date] - ,[condition_era_end_date] - ,[condition_occurrence_count]) +INSERT INTO condition_era + (condition_era_id + ,person_id + ,condition_concept_id + ,condition_era_start_date + ,condition_era_end_date + ,condition_occurrence_count) SELECT condition_era_id, PERSON_ID, diff --git a/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql b/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql index 7482bf7..78664d5 100644 --- a/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql +++ b/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) + (sandbox, sandbox.cdmv4) 2. Set parameter name of schema that contains CDMv5 instance - ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) + (sandbox, sandbox.cdmv5) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -77,12 +77,22 @@ TemplateSQL File Instructions *********************************************************************************/ /* SCRIPT PARAMETERS */ +-- -- The CDMv4 database name +-- -- The CDMv4 database plus schema +-- -- The target CDMv5 database name +-- -- the target CDMv5 database plus schema +/* LOCAL SQL Server */ +-- -- The CDMv4 database name +-- -- The CDMv4 database plus schema +-- -- The target CDMv5 database name +-- -- the target CDMv5 database plus schema +/* PostgreSQL Settings */ -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema -ALTER SESSION SET current_schema = [TARGET_CDMV5]; +ALTER SESSION SET current_schema = sandbox; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -129,7 +139,7 @@ SELECT NULL as source_concept_mapping_occurrence FROM - dbo.concept + sandbox.cdmv5.concept WHERE standard_concept = 'S' and invalid_reason is null @@ -143,16 +153,16 @@ SELECT distinct c1.concept_id as source_concept_id, FROM ( select concept_id -from dbo.concept +from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -dbo.concept_relationship cr1 +sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' and c2.invalid_reason is null @@ -170,7 +180,7 @@ SELECT distinct c1.concept_id as source_concept_id, FROM ( select concept_id -from dbo.concept +from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) @@ -179,16 +189,16 @@ and concept_id not in ( from ( select concept_id - from dbo.concept + from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join - dbo.concept_relationship cr1 + sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join - dbo.concept c2 + sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -198,10 +208,10 @@ and concept_id not in ( ) c1 inner join -dbo.concept_relationship cr1 +sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' and c2.invalid_reason is null @@ -226,7 +236,7 @@ SELECT distinct c1.concept_id as source_concept_id, FROM ( select concept_id -from dbo.concept +from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) @@ -235,16 +245,16 @@ and concept_id not in ( from ( select concept_id - from dbo.concept + from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join - dbo.concept_relationship cr1 + sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join - dbo.concept c2 + sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -263,10 +273,10 @@ and concept_id not in ( ) c1 inner join -dbo.concept_relationship cr1 +sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' and c2.invalid_reason is null @@ -279,17 +289,17 @@ and cr1.invalid_reason is null; -- the script to ensure that we generate new primary keys -- for the target tables when applicable UPDATE fviywowuconcept_map - SET fviywowuconcept_map.source_concept_mapping_occurrence = A.[Rowcount] + SET fviywowuconcept_map.source_concept_mapping_occurrence = A.CountOfRows FROM fviywowuconcept_map, ( - select source_concept_id, domain_id, count(*) as "rowcount" + select source_concept_id, domain_id, count(*) as "CountOfRows" from fviywowuconcept_map group by source_concept_id, domain_id ) AS A -WHERE fviywowuconcept_map.source_concept_id = A.source_concept_id AND fviywowuconcept_map.domain_id = A.domain_id +WHERE fviywowuconcept_map.source_concept_id = A.source_concept_id AND fviywowuconcept_map.domain_id = A.domain_id; - BEGIN +BEGIN EXECUTE IMMEDIATE 'TRUNCATE TABLE fviywowuconcept_map_distinct'; EXECUTE IMMEDIATE 'DROP TABLE fviywowuconcept_map_distinct'; EXCEPTION @@ -307,15 +317,22 @@ SELECT FROM fviywowuconcept_map - GROUP BY source_concept_id, domain_id + GROUP BY source_concept_id, domain_id; - IF OBJECT_ID('[TARGET_CDMV5].[SCHEMA].ETL_WARNINGS') IS NOT NULL - DROP TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS +BEGIN + EXECUTE IMMEDIATE 'TRUNCATE TABLE ETL_WARNINGS'; + EXECUTE IMMEDIATE 'DROP TABLE ETL_WARNINGS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; -CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS +CREATE TABLE ETL_WARNINGS ( -WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(8000) ); /**** @@ -324,9 +341,9 @@ CDM_SOURCE ****/ -INSERT INTO dbo.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -SELECT '[TARGET_CDMV5]', 'V5', v.vocabulary_version, SYSDATE - FROM dbo.vocabulary v +INSERT INTO sandbox.cdmv5.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +SELECT 'sandbox', 'V5', v.vocabulary_version, SYSDATE + FROM sandbox.cdmv5.vocabulary v WHERE vocabulary_id = 'Vocabulary'; /**** @@ -335,9 +352,9 @@ LOCATION ****/ -insert into dbo.location +insert into sandbox.cdmv5.location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from [SOURCE_CDMV4].[SCHEMA].LOCATION; +from sandbox.cdmv4.LOCATION; /**** @@ -345,9 +362,9 @@ CARE_SITE ****/ -insert into dbo.care_site +insert into sandbox.cdmv5.care_site select care_site_id, TO_CHAR(null ) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from [SOURCE_CDMV4].[SCHEMA].CARE_SITE; +from sandbox.cdmv4.CARE_SITE; /**** @@ -355,11 +372,11 @@ Provider ****/ -insert dbo.provider +insert into sandbox.cdmv5.provider select provider_id, TO_CHAR(null ) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, TO_CHAR(null ) as gender_source_value, cast(null as integer) as gender_source_concept_id -from [SOURCE_CDMV4].[SCHEMA].provider +from sandbox.cdmv4.provider ; @@ -369,7 +386,7 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.person + INSERT into sandbox.cdmv5.person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -389,21 +406,21 @@ from [SOURCE_CDMV4].[SCHEMA].provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].PERSON p + FROM sandbox.cdmv4.PERSON p left JOIN fviywowuconcept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN fviywowuconcept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN fviywowuconcept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON + FROM sandbox.cdmv4.PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -414,16 +431,16 @@ from [SOURCE_CDMV4].[SCHEMA].provider ) warn ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON + FROM sandbox.cdmv4.PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -434,16 +451,16 @@ from [SOURCE_CDMV4].[SCHEMA].provider ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON + FROM sandbox.cdmv4.PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -459,9 +476,9 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.observation_period + INSERT INTO sandbox.cdmv5.observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; + FROM sandbox.cdmv4.OBSERVATION_PERIOD; /**** @@ -469,28 +486,28 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.death + INSERT INTO sandbox.cdmv5.death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].DEATH + FROM sandbox.cdmv4.DEATH LEFT JOIN fviywowuconcept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DEATH + FROM sandbox.cdmv4.DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -507,7 +524,7 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.visit_occurrence + INSERT INTO sandbox.cdmv5.visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, TO_CHAR(null ) visit_start_time, @@ -516,20 +533,20 @@ from [SOURCE_CDMV4].[SCHEMA].provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE + FROM sandbox.cdmv4.VISIT_OCCURRENCE LEFT JOIN fviywowuconcept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE + FROM sandbox.cdmv4.VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -584,7 +601,7 @@ SELECT CAST(null as NUMBER(19)) as origional_drug_id FROM - [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -611,7 +628,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE FROM DUAL FROM @@ -635,7 +652,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE LEFT JOIN fviywowuconcept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN fviywowuconcept_map cm2 @@ -678,7 +695,7 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id, NULL as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -704,7 +721,7 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -725,7 +742,7 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN fviywowuconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -741,54 +758,54 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN fviywowuconcept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO [dbo].[procedure_occurrence] - ([procedure_occurrence_id] - ,[person_id] - ,[procedure_concept_id] - ,[procedure_date] - ,[procedure_type_concept_id] - ,[modifier_concept_id] - ,[quantity] - ,[provider_id] - ,[visit_occurrence_id] - ,[procedure_source_value] - ,[procedure_source_concept_id] - ,[qualifier_source_value]) +INSERT INTO procedure_occurrence + (procedure_occurrence_id + ,person_id + ,procedure_concept_id + ,procedure_date + ,procedure_type_concept_id + ,modifier_concept_id + ,quantity + ,provider_id + ,visit_occurrence_id + ,procedure_source_value + ,procedure_source_concept_id + ,qualifier_source_value) SELECT - [procedure_occurrence_id] - ,[person_id] - ,[procedure_concept_id] - ,[procedure_date] - ,[procedure_type_concept_id] - ,[modifier_concept_id] - ,[quantity] - ,[provider_id] - ,[visit_occurrence_id] - ,[procedure_source_value] - ,[procedure_source_concept_id] - ,[qualifier_source_value] -FROM fviywowuprocedure_occurrence_map + procedure_occurrence_id + ,person_id + ,procedure_concept_id + ,procedure_date + ,procedure_type_concept_id + ,modifier_concept_id + ,quantity + ,provider_id + ,visit_occurrence_id + ,procedure_source_value + ,procedure_source_concept_id + ,qualifier_source_value +FROM fviywowuprocedure_occurrence_map; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -798,15 +815,15 @@ FROM fviywowuprocedure_occurrence_map ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -862,7 +879,7 @@ SELECT CAST(null as NUMBER(19)) origional_procedure_id FROM - [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN fviywowuconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -898,7 +915,7 @@ UNION ALL TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -926,7 +943,7 @@ UNION ALL TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE LEFT JOIN fviywowuconcept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN fviywowuconcept_map cm2 @@ -968,7 +985,7 @@ FROM TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, NULL as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN fviywowuconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -987,7 +1004,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1002,7 +1019,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1016,69 +1033,69 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, observation_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN fviywowuconcept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM sandbox.cdmv4.DRUG_EXPOSURE) MAXROW ; -INSERT INTO [dbo].[drug_exposure] - ([drug_exposure_id] - ,[person_id] - ,[drug_concept_id] - ,[drug_exposure_start_date] - ,[drug_exposure_end_date] - ,[drug_type_concept_id] - ,[stop_reason] - ,[refills] - ,[quantity] - ,[days_supply] - ,[sig] - ,[route_concept_id] - ,[effective_drug_dose] - ,[dose_unit_concept_id] - ,[lot_number] - ,[provider_id] - ,[visit_occurrence_id] - ,[drug_source_value] - ,[drug_source_concept_id] - ,[route_source_value] - ,[dose_unit_source_value]) +INSERT INTO drug_exposure + (drug_exposure_id + ,person_id + ,drug_concept_id + ,drug_exposure_start_date + ,drug_exposure_end_date + ,drug_type_concept_id + ,stop_reason + ,refills + ,quantity + ,days_supply + ,sig + ,route_concept_id + ,effective_drug_dose + ,dose_unit_concept_id + ,lot_number + ,provider_id + ,visit_occurrence_id + ,drug_source_value + ,drug_source_concept_id + ,route_source_value + ,dose_unit_source_value) SELECT - [drug_exposure_id] - ,[person_id] - ,[drug_concept_id] - ,[drug_exposure_start_date] - ,[drug_exposure_end_date] - ,[drug_type_concept_id] - ,[stop_reason] - ,[refills] - ,[quantity] - ,[days_supply] - ,[sig] - ,[route_concept_id] - ,[effective_drug_dose] - ,[dose_unit_concept_id] - ,[lot_number] - ,[provider_id] - ,[visit_occurrence_id] - ,[drug_source_value] - ,[drug_source_concept_id] - ,[route_source_value] - ,[dose_unit_source_value] -FROM fviywowudrug_exposure_map + drug_exposure_id + ,person_id + ,drug_concept_id + ,drug_exposure_start_date + ,drug_exposure_end_date + ,drug_type_concept_id + ,stop_reason + ,refills + ,quantity + ,days_supply + ,sig + ,route_concept_id + ,effective_drug_dose + ,dose_unit_concept_id + ,lot_number + ,provider_id + ,visit_occurrence_id + ,drug_source_value + ,drug_source_concept_id + ,route_source_value + ,dose_unit_source_value +FROM fviywowudrug_exposure_map; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1088,15 +1105,15 @@ FROM fviywowudrug_exposure_map ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1114,7 +1131,7 @@ FROM fviywowudrug_exposure_map ****/ --find valid conditions from condition_occurrence table - INSERT dbo.condition_occurrence + INSERT INTO sandbox.cdmv5.condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1123,7 +1140,7 @@ FROM fviywowudrug_exposure_map COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1147,7 +1164,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1164,7 +1181,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE LEFT JOIN fviywowuconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN fviywowuconcept_map cm2 @@ -1188,8 +1205,9 @@ FROM ( condition_end_date, COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, - condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + condition_source_value, CAST(null as integer) condition_source_concept_id, + NULL as OCCURRENCE_ID + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1208,7 +1226,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1221,7 +1239,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN fviywowuconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1234,25 +1252,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN fviywowuconcept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM sandbox.cdmv4.CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1262,15 +1280,15 @@ FROM ( ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1289,7 +1307,7 @@ FROM ( ****/ -INSERT dbo.device_exposure +INSERT INTO sandbox.cdmv5.device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1303,7 +1321,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1317,7 +1335,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN fviywowuconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1331,7 +1349,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1345,7 +1363,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN fviywowuconcept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1360,7 +1378,7 @@ FROM ****/ --find valid measurements from observation table -INSERT dbo.measurement +INSERT INTO sandbox.cdmv5.measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1381,7 +1399,7 @@ from unit_source_value AS unit_source_value, TO_CHAR(null ) as value_source_value, observation_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN fviywowuconcept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1409,7 +1427,7 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, procedure_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1434,7 +1452,7 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, condition_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1459,7 +1477,7 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, drug_exposure_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure + FROM sandbox.cdmv4.drug_exposure INNER JOIN fviywowuconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1478,13 +1496,13 @@ from --find valid observation from observation table - INSERT dbo.observation + INSERT INTO sandbox.cdmv5.observation SELECT observation_id, person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, value_as_number, value_as_string, value_as_concept_id, CAST(null as integer) qualifier_concept_id, unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value, CAST(null as integer) observation_source_concept_id, unit_source_value, TO_CHAR(null ) qualifier_source_value - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM fviywowuconcept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) @@ -1505,7 +1523,7 @@ FROM visit_occurrence_id, procedure_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, null as unit_source_value, TO_CHAR(null ) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1520,7 +1538,7 @@ FROM visit_occurrence_id, condition_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, null as unit_source_value, TO_CHAR(null ) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN fviywowuconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1535,11 +1553,11 @@ FROM visit_occurrence_id, drug_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, null as unit_source_value, TO_CHAR(null ) qualifier_source_value, drug_exposure_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN fviywowuconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM sandbox.cdmv4.OBSERVATION) MAXROW ; @@ -1550,10 +1568,10 @@ FROM ****/ -INSERT dbo.payer_plan_period +INSERT INTO sandbox.cdmv5.payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; +FROM sandbox.cdmv4.PAYER_PLAN_PERIOD; /**** @@ -1563,15 +1581,15 @@ FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; ****/ -INSERT dbo.drug_cost +INSERT INTO sandbox.cdmv5.drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc +FROM sandbox.cdmv4.DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO dbo.drug_cost +INSERT INTO sandbox.cdmv5.drug_cost select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, @@ -1580,11 +1598,11 @@ FROM ( SELECT drug_exposure_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, null as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po - join [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE po + join sandbox.cdmv4.PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id JOIN fviywowudrug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW ; /**** @@ -1596,15 +1614,15 @@ FROM ( ****/ -INSERT INTO dbo.procedure_cost +INSERT INTO sandbox.cdmv5.procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; +FROM sandbox.cdmv4.PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO dbo.procedure_cost +INSERT INTO sandbox.cdmv5.procedure_cost SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, @@ -1613,11 +1631,11 @@ SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_n SELECT po.procedure_occurrence_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, null as average_wholesale_price, payer_plan_period_id, null as revenue_code_concept_id, null as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de - join [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + FROM sandbox.cdmv4.DRUG_EXPOSURE de + join sandbox.cdmv4.DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id JOIN fviywowuprocedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW ; /**** @@ -1628,16 +1646,16 @@ Note: Eras derived from DRUG_EXPOSURE table, using 30d gap ****/ -- drop table dbo.drug_era -INSERT INTO dbo.drug_era +INSERT INTO sandbox.cdmv5.drug_era WITH cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, DRUG_EXPOSURE_END_DATE, INGREDIENT_CONCEPT_ID) AS ( -- Normalize DRUG_EXPOSURE_END_DATE to either the existing drug exposure end date, or add days supply, or add 1 day to the start date select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, (DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), (DRUG_EXPOSURE_START_DATE + 1)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d - join dbo.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join dbo.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM sandbox.cdmv5.DRUG_EXPOSURE d + join sandbox.cdmv5.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join sandbox.cdmv5.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1717,7 +1735,7 @@ WITH cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, (CONDITION_START_DATE + 1)) as CONDITION_END_DATE - FROM [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co + FROM sandbox.cdmv5.CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1772,13 +1790,13 @@ FROM GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO [dbo].[condition_era] - ([condition_era_id] - ,[person_id] - ,[condition_concept_id] - ,[condition_era_start_date] - ,[condition_era_end_date] - ,[condition_occurrence_count]) +INSERT INTO condition_era + (condition_era_id + ,person_id + ,condition_concept_id + ,condition_era_start_date + ,condition_era_end_date + ,condition_occurrence_count) SELECT condition_era_id, PERSON_ID, diff --git a/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql b/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql index 67c4f42..012dc82 100644 --- a/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql +++ b/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) + (sandbox, sandbox.cdmv4) 2. Set parameter name of schema that contains CDMv5 instance - ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) + (sandbox, sandbox.cdmv5) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -77,12 +77,22 @@ TemplateSQL File Instructions *********************************************************************************/ /* SCRIPT PARAMETERS */ +-- -- The CDMv4 database name +-- -- The CDMv4 database plus schema +-- -- The target CDMv5 database name +-- -- the target CDMv5 database plus schema +/* LOCAL SQL Server */ +-- -- The CDMv4 database name +-- -- The CDMv4 database plus schema +-- -- The target CDMv5 database name +-- -- the target CDMv5 database plus schema +/* PostgreSQL Settings */ -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema -SET search_path TO [TARGET_CDMV5]; +SET search_path TO sandbox; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -121,7 +131,7 @@ SELECT NULL as source_concept_mapping_occurrence FROM - dbo.concept + sandbox.cdmv5.concept where standard_concept = 'S' and invalid_reason is null @@ -135,16 +145,16 @@ select distinct c1.concept_id as source_concept_id, from ( select concept_id -from dbo.concept +from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -dbo.concept_relationship cr1 +sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -162,7 +172,7 @@ select distinct c1.concept_id as source_concept_id, from ( select concept_id -from dbo.concept +from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) @@ -171,16 +181,16 @@ and concept_id not in ( from ( select concept_id - from dbo.concept + from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join - dbo.concept_relationship cr1 + sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join - dbo.concept c2 + sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -190,10 +200,10 @@ and concept_id not in ( ) c1 inner join -dbo.concept_relationship cr1 +sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -218,7 +228,7 @@ select distinct c1.concept_id as source_concept_id, from ( select concept_id -from dbo.concept +from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) @@ -227,16 +237,16 @@ and concept_id not in ( from ( select concept_id - from dbo.concept + from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join - dbo.concept_relationship cr1 + sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join - dbo.concept c2 + sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -255,10 +265,10 @@ and concept_id not in ( ) c1 inner join -dbo.concept_relationship cr1 +sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -271,17 +281,17 @@ and cr1.invalid_reason is null; -- the script to ensure that we generate new primary keys -- for the target tables when applicable UPDATE concept_map - SET concept_map.source_concept_mapping_occurrence = A.[Rowcount] + SET concept_map.source_concept_mapping_occurrence = A.CountOfRows FROM concept_map, ( - select source_concept_id, domain_id, count(*) as "rowcount" + select source_concept_id, domain_id, count(*) as "CountOfRows" from concept_map group by source_concept_id, domain_id ) AS A -WHERE concept_map.source_concept_id = A.source_concept_id AND concept_map.domain_id = A.domain_id +WHERE concept_map.source_concept_id = A.source_concept_id AND concept_map.domain_id = A.domain_id; - DROP TABLE IF EXISTS concept_map_distinct; +DROP TABLE IF EXISTS concept_map_distinct; CREATE TEMP TABLE concept_map_distinct @@ -291,15 +301,14 @@ SELECT FROM concept_map - GROUP BY source_concept_id, domain_id + GROUP BY source_concept_id, domain_id; - IF OBJECT_ID('[TARGET_CDMV5].[SCHEMA].ETL_WARNINGS') IS NOT NULL - DROP TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS +DROP TABLE IF EXISTS ETL_WARNINGS; -CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS +CREATE TABLE ETL_WARNINGS ( -WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(8000) ); /**** @@ -308,9 +317,9 @@ CDM_SOURCE ****/ -INSERT INTO dbo.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -select '[TARGET_CDMV5]', 'V5', v.vocabulary_version, CURRENT_DATE -from dbo.vocabulary v +INSERT INTO sandbox.cdmv5.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +select 'sandbox', 'V5', v.vocabulary_version, CURRENT_DATE +from sandbox.cdmv5.vocabulary v where vocabulary_id = 'Vocabulary'; /**** @@ -319,9 +328,9 @@ LOCATION ****/ -insert into dbo.location +insert into sandbox.cdmv5.location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from [SOURCE_CDMV4].[SCHEMA].LOCATION; +from sandbox.cdmv4.LOCATION; /**** @@ -329,9 +338,9 @@ CARE_SITE ****/ -insert into dbo.care_site +insert into sandbox.cdmv5.care_site select care_site_id, cast(null as varchar(255)) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from [SOURCE_CDMV4].[SCHEMA].CARE_SITE; +from sandbox.cdmv4.CARE_SITE; /**** @@ -339,11 +348,11 @@ Provider ****/ -insert dbo.provider +insert into sandbox.cdmv5.provider select provider_id, cast(null as varchar(255)) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, cast(null as varchar(50)) as gender_source_value, cast(null as integer) as gender_source_concept_id -from [SOURCE_CDMV4].[SCHEMA].provider +from sandbox.cdmv4.provider ; @@ -353,7 +362,7 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.person + INSERT into sandbox.cdmv5.person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -373,21 +382,21 @@ from [SOURCE_CDMV4].[SCHEMA].provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].PERSON p + FROM sandbox.cdmv4.PERSON p left JOIN concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON + FROM sandbox.cdmv4.PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -398,16 +407,16 @@ from [SOURCE_CDMV4].[SCHEMA].provider ) warn ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON + FROM sandbox.cdmv4.PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -418,16 +427,16 @@ from [SOURCE_CDMV4].[SCHEMA].provider ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON + FROM sandbox.cdmv4.PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -443,9 +452,9 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.observation_period + INSERT INTO sandbox.cdmv5.observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; + FROM sandbox.cdmv4.OBSERVATION_PERIOD; /**** @@ -453,28 +462,28 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.death + INSERT INTO sandbox.cdmv5.death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].DEATH + FROM sandbox.cdmv4.DEATH LEFT JOIN concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DEATH + FROM sandbox.cdmv4.DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -491,7 +500,7 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.visit_occurrence + INSERT INTO sandbox.cdmv5.visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, CAST(null as varchar(10)) visit_start_time, @@ -500,20 +509,20 @@ from [SOURCE_CDMV4].[SCHEMA].provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE + FROM sandbox.cdmv4.VISIT_OCCURRENCE LEFT JOIN concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE + FROM sandbox.cdmv4.VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -560,7 +569,7 @@ SELECT CAST(null as bigint) as origional_drug_id FROM - [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -587,7 +596,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE WHERE procedure_concept_id = 0 UNION ALL @@ -607,7 +616,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE LEFT JOIN concept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -650,7 +659,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, NULL as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -676,7 +685,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -697,7 +706,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -713,54 +722,54 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO [dbo].[procedure_occurrence] - ([procedure_occurrence_id] - ,[person_id] - ,[procedure_concept_id] - ,[procedure_date] - ,[procedure_type_concept_id] - ,[modifier_concept_id] - ,[quantity] - ,[provider_id] - ,[visit_occurrence_id] - ,[procedure_source_value] - ,[procedure_source_concept_id] - ,[qualifier_source_value]) +INSERT INTO procedure_occurrence + (procedure_occurrence_id + ,person_id + ,procedure_concept_id + ,procedure_date + ,procedure_type_concept_id + ,modifier_concept_id + ,quantity + ,provider_id + ,visit_occurrence_id + ,procedure_source_value + ,procedure_source_concept_id + ,qualifier_source_value) SELECT - [procedure_occurrence_id] - ,[person_id] - ,[procedure_concept_id] - ,[procedure_date] - ,[procedure_type_concept_id] - ,[modifier_concept_id] - ,[quantity] - ,[provider_id] - ,[visit_occurrence_id] - ,[procedure_source_value] - ,[procedure_source_concept_id] - ,[qualifier_source_value] -FROM procedure_occurrence_map + procedure_occurrence_id + ,person_id + ,procedure_concept_id + ,procedure_date + ,procedure_type_concept_id + ,modifier_concept_id + ,quantity + ,provider_id + ,visit_occurrence_id + ,procedure_source_value + ,procedure_source_concept_id + ,qualifier_source_value +FROM procedure_occurrence_map; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -770,15 +779,15 @@ FROM procedure_occurrence_map ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -826,7 +835,7 @@ SELECT CAST(null as bigint) origional_procedure_id FROM - [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -862,7 +871,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -890,7 +899,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE LEFT JOIN concept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -932,7 +941,7 @@ FROM CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, NULL as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -951,7 +960,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -966,7 +975,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -980,69 +989,69 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, observation_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM sandbox.cdmv4.DRUG_EXPOSURE) MAXROW ; -INSERT INTO [dbo].[drug_exposure] - ([drug_exposure_id] - ,[person_id] - ,[drug_concept_id] - ,[drug_exposure_start_date] - ,[drug_exposure_end_date] - ,[drug_type_concept_id] - ,[stop_reason] - ,[refills] - ,[quantity] - ,[days_supply] - ,[sig] - ,[route_concept_id] - ,[effective_drug_dose] - ,[dose_unit_concept_id] - ,[lot_number] - ,[provider_id] - ,[visit_occurrence_id] - ,[drug_source_value] - ,[drug_source_concept_id] - ,[route_source_value] - ,[dose_unit_source_value]) +INSERT INTO drug_exposure + (drug_exposure_id + ,person_id + ,drug_concept_id + ,drug_exposure_start_date + ,drug_exposure_end_date + ,drug_type_concept_id + ,stop_reason + ,refills + ,quantity + ,days_supply + ,sig + ,route_concept_id + ,effective_drug_dose + ,dose_unit_concept_id + ,lot_number + ,provider_id + ,visit_occurrence_id + ,drug_source_value + ,drug_source_concept_id + ,route_source_value + ,dose_unit_source_value) SELECT - [drug_exposure_id] - ,[person_id] - ,[drug_concept_id] - ,[drug_exposure_start_date] - ,[drug_exposure_end_date] - ,[drug_type_concept_id] - ,[stop_reason] - ,[refills] - ,[quantity] - ,[days_supply] - ,[sig] - ,[route_concept_id] - ,[effective_drug_dose] - ,[dose_unit_concept_id] - ,[lot_number] - ,[provider_id] - ,[visit_occurrence_id] - ,[drug_source_value] - ,[drug_source_concept_id] - ,[route_source_value] - ,[dose_unit_source_value] -FROM drug_exposure_map + drug_exposure_id + ,person_id + ,drug_concept_id + ,drug_exposure_start_date + ,drug_exposure_end_date + ,drug_type_concept_id + ,stop_reason + ,refills + ,quantity + ,days_supply + ,sig + ,route_concept_id + ,effective_drug_dose + ,dose_unit_concept_id + ,lot_number + ,provider_id + ,visit_occurrence_id + ,drug_source_value + ,drug_source_concept_id + ,route_source_value + ,dose_unit_source_value +FROM drug_exposure_map; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1052,15 +1061,15 @@ FROM drug_exposure_map ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1078,7 +1087,7 @@ FROM drug_exposure_map ****/ --find valid conditions from condition_occurrence table - INSERT dbo.condition_occurrence + INSERT INTO sandbox.cdmv5.condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1087,7 +1096,7 @@ FROM drug_exposure_map COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1111,7 +1120,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1128,7 +1137,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE LEFT JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -1152,8 +1161,9 @@ FROM ( condition_end_date, COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, - condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + condition_source_value, CAST(null as integer) condition_source_concept_id, + NULL as OCCURRENCE_ID + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1172,7 +1182,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1185,7 +1195,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1198,25 +1208,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM sandbox.cdmv4.CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1226,15 +1236,15 @@ FROM ( ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1253,7 +1263,7 @@ FROM ( ****/ -INSERT dbo.device_exposure +INSERT INTO sandbox.cdmv5.device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1267,7 +1277,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1281,7 +1291,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1295,7 +1305,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1309,7 +1319,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1324,7 +1334,7 @@ FROM ****/ --find valid measurements from observation table -INSERT dbo.measurement +INSERT INTO sandbox.cdmv5.measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1345,7 +1355,7 @@ from unit_source_value AS unit_source_value, cast(null as varchar(50)) as value_source_value, observation_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1373,7 +1383,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, procedure_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1398,7 +1408,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, condition_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1423,7 +1433,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, drug_exposure_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure + FROM sandbox.cdmv4.drug_exposure INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1442,13 +1452,13 @@ from --find valid observation from observation table - INSERT dbo.observation + INSERT INTO sandbox.cdmv5.observation SELECT observation_id, person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, value_as_number, value_as_string, value_as_concept_id, CAST(null as integer) qualifier_concept_id, unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value, CAST(null as integer) observation_source_concept_id, unit_source_value, cast(null as varchar(50)) qualifier_source_value - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) @@ -1469,7 +1479,7 @@ FROM visit_occurrence_id, procedure_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1484,7 +1494,7 @@ FROM visit_occurrence_id, condition_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1499,11 +1509,11 @@ FROM visit_occurrence_id, drug_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, drug_exposure_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM sandbox.cdmv4.OBSERVATION) MAXROW ; @@ -1514,10 +1524,10 @@ FROM ****/ -INSERT dbo.payer_plan_period +INSERT INTO sandbox.cdmv5.payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; +FROM sandbox.cdmv4.PAYER_PLAN_PERIOD; /**** @@ -1527,15 +1537,15 @@ FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; ****/ -INSERT dbo.drug_cost +INSERT INTO sandbox.cdmv5.drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc +FROM sandbox.cdmv4.DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO dbo.drug_cost +INSERT INTO sandbox.cdmv5.drug_cost select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, @@ -1544,11 +1554,11 @@ FROM ( SELECT drug_exposure_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, null as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po - join [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE po + join sandbox.cdmv4.PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id JOIN drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW ; /**** @@ -1560,15 +1570,15 @@ FROM ( ****/ -INSERT INTO dbo.procedure_cost +INSERT INTO sandbox.cdmv5.procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; +FROM sandbox.cdmv4.PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO dbo.procedure_cost +INSERT INTO sandbox.cdmv5.procedure_cost SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, @@ -1577,11 +1587,11 @@ SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_n SELECT po.procedure_occurrence_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, null as average_wholesale_price, payer_plan_period_id, null as revenue_code_concept_id, null as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de - join [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + FROM sandbox.cdmv4.DRUG_EXPOSURE de + join sandbox.cdmv4.DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id JOIN procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW ; /**** @@ -1598,9 +1608,9 @@ with cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONC select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, (DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), (DRUG_EXPOSURE_START_DATE + 1)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d - join dbo.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join dbo.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM sandbox.cdmv5.DRUG_EXPOSURE d + join sandbox.cdmv5.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join sandbox.cdmv5.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1650,7 +1660,7 @@ GROUP BY d.PERSON_ID, d.DRUG_TYPE_CONCEPT_ID, d.DRUG_EXPOSURE_START_DATE ) -insert into dbo.drug_era +insert into sandbox.cdmv5.drug_era select row_number() over (order by person_id) as drug_era_id, person_id, drug_concept_id, min(DRUG_EXPOSURE_START_DATE) as drug_era_start_date, drug_era_end_date, COUNT(*) as DRUG_EXPOSURE_COUNT, 30 as gap_days from cteDrugExposureEnds GROUP BY person_id, drug_concept_id, drug_type_concept_id, DRUG_ERA_END_DATE @@ -1673,7 +1683,7 @@ WITH cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, (CONDITION_START_DATE + 1)) as CONDITION_END_DATE - FROM [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co + FROM sandbox.cdmv5.CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1728,13 +1738,13 @@ FROM GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO [dbo].[condition_era] - ([condition_era_id] - ,[person_id] - ,[condition_concept_id] - ,[condition_era_start_date] - ,[condition_era_end_date] - ,[condition_occurrence_count]) +INSERT INTO condition_era + (condition_era_id + ,person_id + ,condition_concept_id + ,condition_era_start_date + ,condition_era_end_date + ,condition_occurrence_count) SELECT condition_era_id, PERSON_ID, diff --git a/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql b/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql index c4dac0b..be114be 100644 --- a/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql +++ b/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) + (sandbox, sandbox.cdmv4) 2. Set parameter name of schema that contains CDMv5 instance - ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) + (sandbox, sandbox.cdmv5) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -77,12 +77,22 @@ TemplateSQL File Instructions *********************************************************************************/ /* SCRIPT PARAMETERS */ +-- -- The CDMv4 database name +-- -- The CDMv4 database plus schema +-- -- The target CDMv5 database name +-- -- the target CDMv5 database plus schema +/* LOCAL SQL Server */ +-- -- The CDMv4 database name +-- -- The CDMv4 database plus schema +-- -- The target CDMv5 database name +-- -- the target CDMv5 database plus schema +/* PostgreSQL Settings */ -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema -USE [TARGET_CDMV5]; +USE sandbox; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -117,7 +127,7 @@ select concept_id as source_concept_id, domain_id, NULL as source_concept_mapping_occurrence into #concept_map -from dbo.concept +from sandbox.cdmv5.concept where standard_concept = 'S' and invalid_reason is null @@ -131,16 +141,16 @@ select distinct c1.concept_id as source_concept_id, from ( select concept_id -from dbo.concept +from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -dbo.concept_relationship cr1 +sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -158,7 +168,7 @@ select distinct c1.concept_id as source_concept_id, from ( select concept_id -from dbo.concept +from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) @@ -167,16 +177,16 @@ and concept_id not in ( from ( select concept_id - from dbo.concept + from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join - dbo.concept_relationship cr1 + sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join - dbo.concept c2 + sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -186,10 +196,10 @@ and concept_id not in ( ) c1 inner join -dbo.concept_relationship cr1 +sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -214,7 +224,7 @@ select distinct c1.concept_id as source_concept_id, from ( select concept_id -from dbo.concept +from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) @@ -223,16 +233,16 @@ and concept_id not in ( from ( select concept_id - from dbo.concept + from sandbox.cdmv5.concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join - dbo.concept_relationship cr1 + sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join - dbo.concept c2 + sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -251,10 +261,10 @@ and concept_id not in ( ) c1 inner join -dbo.concept_relationship cr1 +sandbox.cdmv5.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -dbo.concept c2 +sandbox.cdmv5.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -267,31 +277,31 @@ and cr1.invalid_reason is null; -- the script to ensure that we generate new primary keys -- for the target tables when applicable UPDATE #concept_map - SET #concept_map.source_concept_mapping_occurrence = A.[Rowcount] + SET #concept_map.source_concept_mapping_occurrence = A.CountOfRows FROM #concept_map, ( - select source_concept_id, domain_id, count(*) as "rowcount" + select source_concept_id, domain_id, count(*) as "CountOfRows" from #concept_map group by source_concept_id, domain_id ) AS A -WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id +WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; - IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL +IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL DROP TABLE #concept_map_distinct; SELECT DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" INTO #concept_map_distinct FROM #concept_map - GROUP BY source_concept_id, domain_id + GROUP BY source_concept_id, domain_id; - IF OBJECT_ID('[TARGET_CDMV5].[SCHEMA].ETL_WARNINGS') IS NOT NULL - DROP TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS +IF OBJECT_ID('ETL_WARNINGS', 'U') IS NOT NULL + DROP TABLE ETL_WARNINGS; -CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS +CREATE TABLE ETL_WARNINGS ( -WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(8000) ); /**** @@ -300,9 +310,9 @@ CDM_SOURCE ****/ -INSERT INTO dbo.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -select '[TARGET_CDMV5]', 'V5', v.vocabulary_version, getDate() -from dbo.vocabulary v +INSERT INTO sandbox.cdmv5.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +select 'sandbox', 'V5', v.vocabulary_version, getDate() +from sandbox.cdmv5.vocabulary v where vocabulary_id = 'Vocabulary'; /**** @@ -311,9 +321,9 @@ LOCATION ****/ -insert into dbo.location +insert into sandbox.cdmv5.location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from [SOURCE_CDMV4].[SCHEMA].LOCATION; +from sandbox.cdmv4.LOCATION; /**** @@ -321,9 +331,9 @@ CARE_SITE ****/ -insert into dbo.care_site +insert into sandbox.cdmv5.care_site select care_site_id, cast(null as varchar(255)) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from [SOURCE_CDMV4].[SCHEMA].CARE_SITE; +from sandbox.cdmv4.CARE_SITE; /**** @@ -331,11 +341,11 @@ Provider ****/ -insert dbo.provider +insert into sandbox.cdmv5.provider select provider_id, cast(null as varchar(255)) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, cast(null as varchar(50)) as gender_source_value, cast(null as integer) as gender_source_concept_id -from [SOURCE_CDMV4].[SCHEMA].provider +from sandbox.cdmv4.provider ; @@ -345,7 +355,7 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.person + INSERT into sandbox.cdmv5.person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -365,21 +375,21 @@ from [SOURCE_CDMV4].[SCHEMA].provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].PERSON p + FROM sandbox.cdmv4.PERSON p left JOIN #concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN #concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN #concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON + FROM sandbox.cdmv4.PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -390,16 +400,16 @@ from [SOURCE_CDMV4].[SCHEMA].provider ) warn ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON + FROM sandbox.cdmv4.PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -410,16 +420,16 @@ from [SOURCE_CDMV4].[SCHEMA].provider ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON + FROM sandbox.cdmv4.PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -435,9 +445,9 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.observation_period + INSERT INTO sandbox.cdmv5.observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; + FROM sandbox.cdmv4.OBSERVATION_PERIOD; /**** @@ -445,28 +455,28 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.death + INSERT INTO sandbox.cdmv5.death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].DEATH + FROM sandbox.cdmv4.DEATH LEFT JOIN #concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DEATH + FROM sandbox.cdmv4.DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -483,7 +493,7 @@ from [SOURCE_CDMV4].[SCHEMA].provider ****/ - INSERT dbo.visit_occurrence + INSERT INTO sandbox.cdmv5.visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, CAST(null as varchar(10)) visit_start_time, @@ -492,20 +502,20 @@ from [SOURCE_CDMV4].[SCHEMA].provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE + FROM sandbox.cdmv4.VISIT_OCCURRENCE LEFT JOIN #concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE + FROM sandbox.cdmv4.VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -548,7 +558,7 @@ from [SOURCE_CDMV4].[SCHEMA].provider CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id INTO #procedure_occurrence_map - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -575,7 +585,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE WHERE procedure_concept_id = 0 UNION ALL @@ -595,7 +605,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE LEFT JOIN #concept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -638,7 +648,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, NULL as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -664,7 +674,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -685,7 +695,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -701,54 +711,54 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO [dbo].[procedure_occurrence] - ([procedure_occurrence_id] - ,[person_id] - ,[procedure_concept_id] - ,[procedure_date] - ,[procedure_type_concept_id] - ,[modifier_concept_id] - ,[quantity] - ,[provider_id] - ,[visit_occurrence_id] - ,[procedure_source_value] - ,[procedure_source_concept_id] - ,[qualifier_source_value]) +INSERT INTO procedure_occurrence + (procedure_occurrence_id + ,person_id + ,procedure_concept_id + ,procedure_date + ,procedure_type_concept_id + ,modifier_concept_id + ,quantity + ,provider_id + ,visit_occurrence_id + ,procedure_source_value + ,procedure_source_concept_id + ,qualifier_source_value) SELECT - [procedure_occurrence_id] - ,[person_id] - ,[procedure_concept_id] - ,[procedure_date] - ,[procedure_type_concept_id] - ,[modifier_concept_id] - ,[quantity] - ,[provider_id] - ,[visit_occurrence_id] - ,[procedure_source_value] - ,[procedure_source_concept_id] - ,[qualifier_source_value] -FROM #procedure_occurrence_map + procedure_occurrence_id + ,person_id + ,procedure_concept_id + ,procedure_date + ,procedure_type_concept_id + ,modifier_concept_id + ,quantity + ,provider_id + ,visit_occurrence_id + ,procedure_source_value + ,procedure_source_concept_id + ,qualifier_source_value +FROM #procedure_occurrence_map; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -758,15 +768,15 @@ FROM #procedure_occurrence_map ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -810,7 +820,7 @@ IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id INTO #drug_exposure_map - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -846,7 +856,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -874,7 +884,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE LEFT JOIN #concept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -916,7 +926,7 @@ FROM CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, NULL as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -935,7 +945,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -950,7 +960,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -964,69 +974,69 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, observation_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM sandbox.cdmv4.DRUG_EXPOSURE) MAXROW ; -INSERT INTO [dbo].[drug_exposure] - ([drug_exposure_id] - ,[person_id] - ,[drug_concept_id] - ,[drug_exposure_start_date] - ,[drug_exposure_end_date] - ,[drug_type_concept_id] - ,[stop_reason] - ,[refills] - ,[quantity] - ,[days_supply] - ,[sig] - ,[route_concept_id] - ,[effective_drug_dose] - ,[dose_unit_concept_id] - ,[lot_number] - ,[provider_id] - ,[visit_occurrence_id] - ,[drug_source_value] - ,[drug_source_concept_id] - ,[route_source_value] - ,[dose_unit_source_value]) +INSERT INTO drug_exposure + (drug_exposure_id + ,person_id + ,drug_concept_id + ,drug_exposure_start_date + ,drug_exposure_end_date + ,drug_type_concept_id + ,stop_reason + ,refills + ,quantity + ,days_supply + ,sig + ,route_concept_id + ,effective_drug_dose + ,dose_unit_concept_id + ,lot_number + ,provider_id + ,visit_occurrence_id + ,drug_source_value + ,drug_source_concept_id + ,route_source_value + ,dose_unit_source_value) SELECT - [drug_exposure_id] - ,[person_id] - ,[drug_concept_id] - ,[drug_exposure_start_date] - ,[drug_exposure_end_date] - ,[drug_type_concept_id] - ,[stop_reason] - ,[refills] - ,[quantity] - ,[days_supply] - ,[sig] - ,[route_concept_id] - ,[effective_drug_dose] - ,[dose_unit_concept_id] - ,[lot_number] - ,[provider_id] - ,[visit_occurrence_id] - ,[drug_source_value] - ,[drug_source_concept_id] - ,[route_source_value] - ,[dose_unit_source_value] -FROM #drug_exposure_map + drug_exposure_id + ,person_id + ,drug_concept_id + ,drug_exposure_start_date + ,drug_exposure_end_date + ,drug_type_concept_id + ,stop_reason + ,refills + ,quantity + ,days_supply + ,sig + ,route_concept_id + ,effective_drug_dose + ,dose_unit_concept_id + ,lot_number + ,provider_id + ,visit_occurrence_id + ,drug_source_value + ,drug_source_concept_id + ,route_source_value + ,dose_unit_source_value +FROM #drug_exposure_map; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1036,15 +1046,15 @@ FROM #drug_exposure_map ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1062,7 +1072,7 @@ FROM #drug_exposure_map ****/ --find valid conditions from condition_occurrence table - INSERT dbo.condition_occurrence + INSERT INTO sandbox.cdmv5.condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1071,7 +1081,7 @@ FROM #drug_exposure_map COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1095,7 +1105,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1112,7 +1122,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE LEFT JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -1136,8 +1146,9 @@ FROM ( condition_end_date, COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, - condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + condition_source_value, CAST(null as integer) condition_source_concept_id, + NULL as OCCURRENCE_ID + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1156,7 +1167,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1169,7 +1180,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1182,25 +1193,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM sandbox.cdmv4.CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1210,15 +1221,15 @@ FROM ( ; - INSERT ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM CONCEPT + FROM sandbox.cdmv5.CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1237,7 +1248,7 @@ FROM ( ****/ -INSERT dbo.device_exposure +INSERT INTO sandbox.cdmv5.device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1251,7 +1262,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1265,7 +1276,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1279,7 +1290,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1293,7 +1304,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1308,7 +1319,7 @@ FROM ****/ --find valid measurements from observation table -INSERT dbo.measurement +INSERT INTO sandbox.cdmv5.measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1329,7 +1340,7 @@ from unit_source_value AS unit_source_value, cast(null as varchar(50)) as value_source_value, observation_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1357,7 +1368,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, procedure_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1382,7 +1393,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, condition_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1407,7 +1418,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, drug_exposure_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure + FROM sandbox.cdmv4.drug_exposure INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1426,13 +1437,13 @@ from --find valid observation from observation table - INSERT dbo.observation + INSERT INTO sandbox.cdmv5.observation SELECT observation_id, person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, value_as_number, value_as_string, value_as_concept_id, CAST(null as integer) qualifier_concept_id, unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value, CAST(null as integer) observation_source_concept_id, unit_source_value, cast(null as varchar(50)) qualifier_source_value - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION + FROM sandbox.cdmv4.OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM #concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) @@ -1453,7 +1464,7 @@ FROM visit_occurrence_id, procedure_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1468,7 +1479,7 @@ FROM visit_occurrence_id, condition_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE + FROM sandbox.cdmv4.CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1483,11 +1494,11 @@ FROM visit_occurrence_id, drug_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, drug_exposure_id as occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE + FROM sandbox.cdmv4.DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM sandbox.cdmv4.OBSERVATION) MAXROW ; @@ -1498,10 +1509,10 @@ FROM ****/ -INSERT dbo.payer_plan_period +INSERT INTO sandbox.cdmv5.payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; +FROM sandbox.cdmv4.PAYER_PLAN_PERIOD; /**** @@ -1511,15 +1522,15 @@ FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; ****/ -INSERT dbo.drug_cost +INSERT INTO sandbox.cdmv5.drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc +FROM sandbox.cdmv4.DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO dbo.drug_cost +INSERT INTO sandbox.cdmv5.drug_cost select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, @@ -1528,11 +1539,11 @@ FROM ( SELECT drug_exposure_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, null as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po - join [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE po + join sandbox.cdmv4.PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id JOIN #drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW ; /**** @@ -1544,15 +1555,15 @@ FROM ( ****/ -INSERT INTO dbo.procedure_cost +INSERT INTO sandbox.cdmv5.procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; +FROM sandbox.cdmv4.PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO dbo.procedure_cost +INSERT INTO sandbox.cdmv5.procedure_cost SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, @@ -1561,11 +1572,11 @@ SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_n SELECT po.procedure_occurrence_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, null as average_wholesale_price, payer_plan_period_id, null as revenue_code_concept_id, null as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de - join [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + FROM sandbox.cdmv4.DRUG_EXPOSURE de + join sandbox.cdmv4.DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id JOIN #procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW ; /**** @@ -1582,9 +1593,9 @@ with cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONC select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, DATEADD(day,DAYS_SUPPLY,DRUG_EXPOSURE_START_DATE), DATEADD(day,1,DRUG_EXPOSURE_START_DATE)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d - join dbo.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join dbo.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM sandbox.cdmv5.DRUG_EXPOSURE d + join sandbox.cdmv5.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join sandbox.cdmv5.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1634,7 +1645,7 @@ GROUP BY d.PERSON_ID, d.DRUG_TYPE_CONCEPT_ID, d.DRUG_EXPOSURE_START_DATE ) -insert into dbo.drug_era +insert into sandbox.cdmv5.drug_era select row_number() over (order by person_id) as drug_era_id, person_id, drug_concept_id, min(DRUG_EXPOSURE_START_DATE) as drug_era_start_date, drug_era_end_date, COUNT(*) as DRUG_EXPOSURE_COUNT, 30 as gap_days from cteDrugExposureEnds GROUP BY person_id, drug_concept_id, drug_type_concept_id, DRUG_ERA_END_DATE @@ -1655,7 +1666,7 @@ with cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, DATEADD(day,1,CONDITION_START_DATE)) as CONDITION_END_DATE - FROM [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co + FROM sandbox.cdmv5.CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1708,13 +1719,13 @@ from cteConditionEnds GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO [dbo].[condition_era] - ([condition_era_id] - ,[person_id] - ,[condition_concept_id] - ,[condition_era_start_date] - ,[condition_era_end_date] - ,[condition_occurrence_count]) +INSERT INTO condition_era + (condition_era_id + ,person_id + ,condition_concept_id + ,condition_era_start_date + ,condition_era_end_date + ,condition_occurrence_count) SELECT condition_era_id, PERSON_ID, From 0fd4f7ebe11a73e95b1b384f37f5a0561e6ad9cb Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Tue, 7 Jul 2015 14:32:08 -0400 Subject: [PATCH 2/5] Saving progress before I rewrite the source_concept_map_occurrence table to remove the need for an UPDATE statement in this script. --- .../OMOP CDMv4 to CDMv5 - templateSQL.sql | 504 +++++++---- .../Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql | 849 ++++++++++-------- .../OMOP CDMv4 to CDMv5 - PostgreSQL.sql | 719 +++++++++------ .../OMOP CDMv4 to CDMv5 - SQL Server.sql | 706 +++++++++------ 4 files changed, 1679 insertions(+), 1099 deletions(-) diff --git a/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql b/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql index 73679d1..80d83e5 100644 --- a/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql +++ b/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql @@ -76,32 +76,36 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS */ +/* SCRIPT PARAMETERS --{DEFAULT @SOURCE_CDMV4 = '[SOURCE_CDMV4]'} -- The CDMv4 database name --{DEFAULT @SOURCE_CDMV4_SCHEMA = '[SOURCE_CDMV4].[SCHEMA]'} -- The CDMv4 database plus schema --{DEFAULT @TARGET_CDMV5 = '[TARGET_CDMV5]'} -- The target CDMv5 database name --{DEFAULT @TARGET_CDMV5_SCHEMA = '[TARGET_CDMV5].[SCHEMA]'} -- the target CDMv5 database plus schema -/* LOCAL SQL Server */ ---{DEFAULT @SOURCE_CDMV4 = '[CDMV4]'} -- The CDMv4 database name ---{DEFAULT @SOURCE_CDMV4_SCHEMA = '[CDMV4].[dbo]'} -- The CDMv4 database plus schema ---{DEFAULT @TARGET_CDMV5 = '[CDMV5]'} -- The target CDMv5 database name ---{DEFAULT @TARGET_CDMV5_SCHEMA = '[CDMV5].[dbo]'} -- the target CDMv5 database plus schema -/* QA Settings */ ---{DEFAULT @SOURCE_CDMV4 = 'CDM_TRUVEN_CCAE_6k'} -- The CDMv4 database name ---{DEFAULT @SOURCE_CDMV4_SCHEMA = 'CDM_TRUVEN_CCAE_6k.dbo'} -- The CDMv4 database plus schema ---{DEFAULT @TARGET_CDMV5 = 'CDMV5_Conversion_Target'} -- The target CDMv5 database name ---{DEFAULT @TARGET_CDMV5_SCHEMA = 'CDMV5_Conversion_Target.dbo'} -- the target CDMv5 database plus schema -/* PostgreSQL Settings */ ---{DEFAULT @SOURCE_CDMV4 = 'sandbox'} -- The CDMv4 database name ---{DEFAULT @SOURCE_CDMV4_SCHEMA = 'sandbox.cdmv4'} -- The CDMv4 database plus schema ---{DEFAULT @TARGET_CDMV5 = 'sandbox'} -- The target CDMv5 database name ---{DEFAULT @TARGET_CDMV5_SCHEMA = 'sandbox.cdmv5'} -- the target CDMv5 database plus schema -/* Oracle Settings */ +*/ +/* QA Settings +{DEFAULT @SOURCE_CDMV4 = 'CDM_TRUVEN_CCAE_6k'} -- The CDMv4 database name +{DEFAULT @SOURCE_CDMV4_SCHEMA = 'CDM_TRUVEN_CCAE_6k.dbo'} -- The CDMv4 database plus schema +{DEFAULT @TARGET_CDMV5 = 'CDMV5_Conversion_Target'} -- The target CDMv5 database name +{DEFAULT @TARGET_CDMV5_SCHEMA = 'CDMV5_Conversion_Target.dbo'} -- the target CDMv5 database plus schema + */ +/* PostgreSQL Settings {DEFAULT @SOURCE_CDMV4 = 'sandbox'} -- The CDMv4 database name {DEFAULT @SOURCE_CDMV4_SCHEMA = 'sandbox.cdmv4'} -- The CDMv4 database plus schema {DEFAULT @TARGET_CDMV5 = 'sandbox'} -- The target CDMv5 database name {DEFAULT @TARGET_CDMV5_SCHEMA = 'sandbox.cdmv5'} -- the target CDMv5 database plus schema - +*/ +/* Oracle Settings +{DEFAULT @SOURCE_CDMV4 = 'CDMV4'} -- The CDMv4 database name +{DEFAULT @SOURCE_CDMV4_SCHEMA = 'CDMV4'} -- The CDMv4 database plus schema +{DEFAULT @TARGET_CDMV5 = 'CDMV5'} -- The target CDMv5 database name +{DEFAULT @TARGET_CDMV5_SCHEMA = 'CDMV5'} -- the target CDMv5 database plus schema +*/ +/* LOCAL SQL Server */ +{DEFAULT @SOURCE_CDMV4 = '[CDMV4]'} -- The CDMv4 database name +{DEFAULT @SOURCE_CDMV4_SCHEMA = '[CDMV4].[dbo]'} -- The CDMv4 database plus schema +{DEFAULT @TARGET_CDMV5 = '[CDMV5]'} -- The target CDMv5 database name +{DEFAULT @TARGET_CDMV5_SCHEMA = '[CDMV5].[dbo]'} -- the target CDMv5 database plus schema + USE @TARGET_CDMV5; /* @@ -131,12 +135,20 @@ USE @TARGET_CDMV5; IF OBJECT_ID('tempdb..#concept_map', 'U') IS NOT NULL DROP TABLE #concept_map; +CREATE TABLE #concept_map +( + source_concept_id int, + target_concept_id int, + domain_id varchar(20), + source_concept_map_occurrence int +); + --standard concepts +INSERT INTO #concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) select concept_id as source_concept_id, concept_id as target_concept_id, domain_id, - NULL as source_concept_mapping_occurrence -into #concept_map + 0 as source_concept_map_occurrence from @TARGET_CDMV5_SCHEMA.concept where standard_concept = 'S' and invalid_reason is null @@ -147,7 +159,7 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,NULL + ,0 from ( select concept_id @@ -174,89 +186,46 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,NULL + ,0 from ( -select concept_id -from @TARGET_CDMV5_SCHEMA.concept -where ((standard_concept <> 'S' or standard_concept is null) -or invalid_reason is not null -) -and concept_id not in ( - select distinct c1.concept_id - from + SELECT A.concept_id + FROM @TARGET_CDMV5_SCHEMA.concept A + LEFT JOIN ( - select concept_id - from @TARGET_CDMV5_SCHEMA.concept - where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) - ) c1 - inner join - @TARGET_CDMV5_SCHEMA.concept_relationship cr1 - on c1.concept_id = cr1.concept_id_1 - inner join - @TARGET_CDMV5_SCHEMA.concept c2 - on cr1.concept_id_2 = c2.concept_id - where c2.standard_concept = 'S' - and c2.invalid_reason is null - and cr1.relationship_id in ('Maps to') - and cr1.invalid_reason is null -) - + SELECT DISTINCT c1.concept_id + FROM ( + SELECT concept_id + FROM @TARGET_CDMV5_SCHEMA.concept + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + ) c1 + INNER JOIN @TARGET_CDMV5_SCHEMA.concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN @TARGET_CDMV5_SCHEMA.concept c2 ON cr1.concept_id_2 = c2.concept_id + WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ('Maps to') + AND cr1.invalid_reason IS NULL + ) B ON A.concept_id = B.concept_id + WHERE ( + ( + A.standard_concept <> 'S' + OR A.standard_concept IS NULL + ) + OR A.invalid_reason IS NOT NULL + ) + AND B.concept_id IS NULL ) c1 -inner join -@TARGET_CDMV5_SCHEMA.concept_relationship cr1 -on c1.concept_id = cr1.concept_id_1 -inner join -@TARGET_CDMV5_SCHEMA.concept c2 -on cr1.concept_id_2 = c2.concept_id +inner join @TARGET_CDMV5_SCHEMA.concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 +inner join @TARGET_CDMV5_SCHEMA.concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' -and c2.invalid_reason is null -and cr1.relationship_id in ('RxNorm replaced by', -'SNOMED replaced by', -'UCUM replaced by', -'Concept replaced by', -'ICD9P replaced by', -'LOINC replaced by', -'Concept same_as to', -'Concept was_a to', -'Concept alt_to to') -and cr1.invalid_reason is null - -union - ---concepts without 'map to' standard with 'is a' relation to standard -select distinct c1.concept_id as source_concept_id, - c2.concept_id as target_concept_id, - c2.domain_id - ,NULL -from -( -select concept_id -from @TARGET_CDMV5_SCHEMA.concept -where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) -and concept_id not in ( - select distinct c1.concept_id - from - ( - select concept_id - from @TARGET_CDMV5_SCHEMA.concept - where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) - ) c1 - inner join - @TARGET_CDMV5_SCHEMA.concept_relationship cr1 - on c1.concept_id = cr1.concept_id_1 - inner join - @TARGET_CDMV5_SCHEMA.concept c2 - on cr1.concept_id_2 = c2.concept_id - where c2.standard_concept = 'S' - and c2.invalid_reason is null - and cr1.relationship_id in ('Maps to', + and c2.invalid_reason is null + and cr1.relationship_id in ( 'RxNorm replaced by', 'SNOMED replaced by', 'UCUM replaced by', @@ -265,29 +234,76 @@ and concept_id not in ( 'LOINC replaced by', 'Concept same_as to', 'Concept was_a to', - 'Concept alt_to to') - and cr1.invalid_reason is null + 'Concept alt_to to' ) +and cr1.invalid_reason is null -) c1 -inner join -@TARGET_CDMV5_SCHEMA.concept_relationship cr1 -on c1.concept_id = cr1.concept_id_1 -inner join -@TARGET_CDMV5_SCHEMA.concept c2 -on cr1.concept_id_2 = c2.concept_id -where c2.standard_concept = 'S' -and c2.invalid_reason is null -and cr1.relationship_id in ('Is a') -and cr1.invalid_reason is null; +union --- Update the source_concept_mapping_occurrence column +--concepts without 'map to' standard with 'is a' relation to standard +SELECT DISTINCT c1.concept_id AS source_concept_id + ,c2.concept_id AS target_concept_id + ,c2.domain_id + ,0 +FROM ( + SELECT A.concept_id + FROM @TARGET_CDMV5_SCHEMA.concept A + LEFT JOIN + ( + SELECT DISTINCT c1.concept_id + FROM ( + SELECT concept_id + FROM @TARGET_CDMV5_SCHEMA.concept + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + ) c1 + INNER JOIN @TARGET_CDMV5_SCHEMA.concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN @TARGET_CDMV5_SCHEMA.concept c2 ON cr1.concept_id_2 = c2.concept_id + WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ( + 'Maps to' + ,'RxNorm replaced by' + ,'SNOMED replaced by' + ,'UCUM replaced by' + ,'Concept replaced by' + ,'ICD9P replaced by' + ,'LOINC replaced by' + ,'Concept same_as to' + ,'Concept was_a to' + ,'Concept alt_to to' + ) + AND cr1.invalid_reason IS NULL + ) B ON A.concept_id = B.concept_id + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + AND B.concept_id IS NULL + ) c1 +INNER JOIN @TARGET_CDMV5_SCHEMA.concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 +INNER JOIN @TARGET_CDMV5_SCHEMA.concept c2 ON cr1.concept_id_2 = c2.concept_id +WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ('Is a') + AND cr1.invalid_reason IS NULL; + +-- Update the source_concept_map_occurrence column -- to contain a count to indicate the number of target_concept_ids -- map to that source_concept_id. This will be used elsewhere in -- the script to ensure that we generate new primary keys -- for the target tables when applicable +/* UPDATE #concept_map - SET #concept_map.source_concept_mapping_occurrence = A.CountOfRows + SET #concept_map.source_concept_map_occurrence = A.CountOfRows FROM #concept_map, ( @@ -296,7 +312,26 @@ and cr1.invalid_reason is null; group by source_concept_id, domain_id ) AS A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; - + +UPDATE #concept_map +SET source_concept_map_occurrence = "CountOfRows" +FROM + ( + select source_concept_id, domain_id, count(*) as "CountOfRows" + from #concept_map + group by source_concept_id, domain_id + ) A +WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; +*/ + +UPDATE #concept_map +SET source_concept_map_occurrence = ( + select count(*) as "CountOfRows" + from #concept_map A + WHERE A.source_concept_id = #concept_map.source_concept_id AND A.domain_id = #concept_map.domain_id + group by source_concept_id, domain_id + ) + IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL DROP TABLE #concept_map_distinct; @@ -306,10 +341,10 @@ IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL GROUP BY source_concept_id, domain_id; -IF OBJECT_ID('ETL_WARNINGS', 'U') IS NOT NULL - DROP TABLE ETL_WARNINGS; +IF OBJECT_ID('@TARGET_CDMV5_SCHEMA.ETL_WARNINGS', 'U') IS NOT NULL + DROP TABLE @TARGET_CDMV5_SCHEMA.ETL_WARNINGS; -CREATE TABLE ETL_WARNINGS +CREATE TABLE @TARGET_CDMV5_SCHEMA.ETL_WARNINGS ( WARNING_MESSAGE varchar(8000) ); @@ -391,7 +426,7 @@ from @SOURCE_CDMV4_SCHEMA.provider LEFT JOIN #concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( @@ -410,7 +445,7 @@ from @SOURCE_CDMV4_SCHEMA.provider ) warn ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( @@ -430,7 +465,7 @@ from @SOURCE_CDMV4_SCHEMA.provider ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( @@ -478,7 +513,7 @@ from @SOURCE_CDMV4_SCHEMA.provider AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( @@ -517,7 +552,7 @@ from @SOURCE_CDMV4_SCHEMA.provider ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( @@ -572,11 +607,11 @@ from @SOURCE_CDMV4_SCHEMA.provider INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_mapping_occurrence = 1 + AND cm1.source_concept_map_occurrence = 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_mapping_occurrence = 1 + AND cm2.source_concept_map_occurrence = 1 UNION ALL @@ -662,7 +697,7 @@ UNION ALL INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_mapping_occurrence > 1 + AND cm1.source_concept_map_occurrence > 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -730,7 +765,7 @@ UNION ALL ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO procedure_occurrence +INSERT INTO @TARGET_CDMV5_SCHEMA.procedure_occurrence (procedure_occurrence_id ,person_id ,procedure_concept_id @@ -760,7 +795,7 @@ FROM #procedure_occurrence_map; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( @@ -778,7 +813,7 @@ FROM #procedure_occurrence_map; ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( @@ -834,11 +869,11 @@ IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_mapping_occurrence = 1 + AND cm1.source_concept_map_occurrence = 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_mapping_occurrence = 1 + AND cm2.source_concept_map_occurrence = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -940,7 +975,7 @@ FROM INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_mapping_occurrence > 1 + AND cm1.source_concept_map_occurrence > 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -990,7 +1025,7 @@ FROM AND LOWER(cm1.domain_id) IN ('drug') ) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE) MAXROW ; -INSERT INTO drug_exposure +INSERT INTO @TARGET_CDMV5_SCHEMA.drug_exposure (drug_exposure_id ,person_id ,drug_concept_id @@ -1038,7 +1073,7 @@ FROM #drug_exposure_map; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( @@ -1056,7 +1091,7 @@ FROM #drug_exposure_map; ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( @@ -1095,7 +1130,7 @@ FROM #drug_exposure_map; INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_mapping_occurrence = 1 + AND cm1.source_concept_map_occurrence = 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1162,7 +1197,7 @@ FROM ( INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_mapping_occurrence > 1 + AND cm1.source_concept_map_occurrence > 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1213,7 +1248,7 @@ FROM ( --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( @@ -1231,7 +1266,7 @@ FROM ( ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( @@ -1337,9 +1372,13 @@ from ( --find mesaurements that were previously classified as observations select person_id, - cm1.target_concept_id AS measurement_concept_id, OBSERVATION_DATE AS measurement_date, - OBSERVATION_TIME AS measurement_time, 0 AS measurement_type_concept_id, CAST(null as integer) operator_concept_id, - value_as_number, value_as_concept_id, + cm1.target_concept_id AS measurement_concept_id, + OBSERVATION_DATE AS measurement_date, + CAST(OBSERVATION_TIME as varchar(50)) AS measurement_time, + 0 AS measurement_type_concept_id, + CAST(null as integer) operator_concept_id, + value_as_number, + value_as_concept_id, COALESCE(cm2.target_concept_id,0) AS unit_concept_id, range_low, range_high, @@ -1448,31 +1487,66 @@ from --find valid observation from observation table INSERT INTO @TARGET_CDMV5_SCHEMA.observation - SELECT observation_id, person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, - value_as_number, value_as_string, value_as_concept_id, CAST(null as integer) qualifier_concept_id, - unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, observation_source_value, CAST(null as integer) observation_source_concept_id, - unit_source_value, cast(null as varchar(50)) qualifier_source_value + SELECT + observation_id, + person_id, + observation_concept_id, + observation_date, + CAST(observation_time as varchar(50)) as observation_time, + observation_type_concept_id, + value_as_number, + value_as_string, + value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + observation_source_value, + CAST(null as integer) observation_source_concept_id, + unit_source_value, + cast(null as varchar(50)) qualifier_source_value FROM @SOURCE_CDMV4_SCHEMA.OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM #concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) --find observations that were previously classified as procedure UNION ALL -select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS observation_id, - person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, - value_as_number, value_as_string, value_as_concept_id, qualifier_concept_id, - unit_concept_id, provider_id, visit_occurrence_id, observation_source_value, observation_source_concept_id, - unit_source_value, qualifier_source_value +select + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS observation_id, + person_id, + observation_concept_id, + observation_date, + observation_time, + observation_type_concept_id, + value_as_number, + value_as_string, + value_as_concept_id, + qualifier_concept_id, + unit_concept_id, + provider_id, + visit_occurrence_id, + observation_source_value, + observation_source_concept_id, + unit_source_value, + qualifier_source_value FROM ( select person_id, cm1.target_concept_id as observation_concept_id, - procedure_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, procedure_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, + procedure_date as observation_date, + CAST(null as varchar(50)) as observation_time, + 0 as observation_type_concept_id, + CAST(null as float) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + null as unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + procedure_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + cast(null as varchar(50)) qualifier_source_value, procedure_occurrence_id as occurrence_id FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 @@ -1483,11 +1557,20 @@ FROM UNION ALL SELECT person_id, cm1.target_concept_id as observation_concept_id, - condition_start_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, condition_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, + condition_start_date as observation_date, + CAST(null as varchar(50)) as observation_time, + 0 as observation_type_concept_id, + CAST(null as float) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + null as unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + condition_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + cast(null as varchar(50)) qualifier_source_value, condition_occurrence_id as occurrence_id FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 @@ -1498,11 +1581,20 @@ FROM UNION ALL SELECT person_id, cm1.target_concept_id as observation_concept_id, - drug_exposure_start_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, null as provider_id, - visit_occurrence_id, drug_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, + drug_exposure_start_date as observation_date, + CAST(null as varchar(10)) as observation_time, + 0 as observation_type_concept_id, + CAST(null as float) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + CAST(null as integer) as unit_concept_id, + CAST(null as integer) as provider_id, + visit_occurrence_id, + drug_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + cast(null as varchar(50)) qualifier_source_value, drug_exposure_id as occurrence_id FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE INNER JOIN #concept_map cm1 @@ -1541,14 +1633,37 @@ FROM @SOURCE_CDMV4_SCHEMA.DRUG_COST dc -- insert procedure costs for procedures that were inserted into the drug_exposure table INSERT INTO @TARGET_CDMV5_SCHEMA.drug_cost -select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, - drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, - average_wholesale_price, payer_plan_period_id +select + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, + drug_exposure_id, + cast(null as integer) currency_concept_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + ingredient_cost, + dispensing_fee, + average_wholesale_price, + payer_plan_period_id FROM ( - SELECT drug_exposure_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, - null as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID + SELECT + drug_exposure_id, + po.person_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + CAST(null as float) as ingredient_cost, + CAST(null as float) as dispensing_fee, + CAST(null as float) as average_wholesale_price, + payer_plan_period_id, + procedure_cost_id as OCCURRENCE_ID FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE po join @SOURCE_CDMV4_SCHEMA.PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id @@ -1574,14 +1689,39 @@ FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table INSERT INTO @TARGET_CDMV5_SCHEMA.procedure_cost -SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, - procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, - paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, - payer_plan_period_id,revenue_code_concept_id, revenue_code_source_value +SELECT + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, + procedure_occurrence_id, + cast(null as integer) + currency_concept_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + payer_plan_period_id, + revenue_code_concept_id, + revenue_code_source_value FROM ( - SELECT po.procedure_occurrence_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, - null as average_wholesale_price, payer_plan_period_id, null as revenue_code_concept_id, null as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID + SELECT + po.procedure_occurrence_id, + po.person_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + CAST(null as float) as ingredient_cost, + CAST(null as float) as dispensing_fee, + CAST(null as float) as average_wholesale_price, + payer_plan_period_id, + CAST(null as integer) as revenue_code_concept_id, + CAST(null as integer) as revenue_code_source_value, + drug_cost_id as OCCURRENCE_ID FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE de join @SOURCE_CDMV4_SCHEMA.DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id @@ -1729,7 +1869,7 @@ from cteConditionEnds GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO condition_era +INSERT INTO @TARGET_CDMV5_SCHEMA.condition_era (condition_era_id ,person_id ,condition_concept_id diff --git a/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql b/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql index 78664d5..8cae4c9 100644 --- a/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql +++ b/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - (sandbox, sandbox.cdmv4) + ([CDMV4], [CDMV4].[dbo]) 2. Set parameter name of schema that contains CDMv5 instance - (sandbox, sandbox.cdmv5) + ([CDMV5], [CDMV5].[dbo]) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -76,23 +76,37 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS */ +/* SCRIPT PARAMETERS -- -- The CDMv4 database name -- -- The CDMv4 database plus schema -- -- The target CDMv5 database name -- -- the target CDMv5 database plus schema -/* LOCAL SQL Server */ --- -- The CDMv4 database name --- -- The CDMv4 database plus schema --- -- The target CDMv5 database name --- -- the target CDMv5 database plus schema -/* PostgreSQL Settings */ +*/ +/* QA Settings + -- The CDMv4 database name + -- The CDMv4 database plus schema + -- The target CDMv5 database name + -- the target CDMv5 database plus schema + */ +/* PostgreSQL Settings -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema - -ALTER SESSION SET current_schema = sandbox; +*/ +/* Oracle Settings + -- The CDMv4 database name + -- The CDMv4 database plus schema + -- The target CDMv5 database name + -- the target CDMv5 database plus schema +*/ +/* LOCAL SQL Server */ + -- The CDMv4 database name + -- The CDMv4 database plus schema + -- The target CDMv5 database name + -- the target CDMv5 database plus schema + +ALTER SESSION SET current_schema = [CDMV5]; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -119,8 +133,8 @@ ALTER SESSION SET current_schema = sandbox; * Also, as of the date which this script was authored, no source_concept_ids map to multiple domains */ BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE fviywowuconcept_map'; - EXECUTE IMMEDIATE 'DROP TABLE fviywowuconcept_map'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0concept_map'; + EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0concept_map'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -128,18 +142,21 @@ EXCEPTION END IF; END; +CREATE GLOBAL TEMPORARY TABLE w3m16jd0concept_map + ( + source_concept_id int, + target_concept_id int, + domain_id varchar(20), + source_concept_map_occurrence int +) ON COMMIT PRESERVE ROWS; + --standard concepts -CREATE GLOBAL TEMPORARY TABLE fviywowuconcept_map - ON COMMIT PRESERVE ROWS -AS -SELECT - concept_id as source_concept_id, +INSERT INTO w3m16jd0concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) +SELECT concept_id as source_concept_id, concept_id as target_concept_id, domain_id, - NULL as source_concept_mapping_occurrence - -FROM - sandbox.cdmv5.concept + 0 as source_concept_map_occurrence + FROM [CDMV5].[dbo].concept WHERE standard_concept = 'S' and invalid_reason is null @@ -149,20 +166,20 @@ union SELECT distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,NULL + ,0 FROM ( select concept_id -from sandbox.cdmv5.concept +from [CDMV5].[dbo].concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -sandbox.cdmv5.concept_relationship cr1 +[CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -sandbox.cdmv5.concept c2 +[CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' and c2.invalid_reason is null @@ -176,89 +193,46 @@ union SELECT distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,NULL + ,0 FROM ( -select concept_id -from sandbox.cdmv5.concept -where ((standard_concept <> 'S' or standard_concept is null) -or invalid_reason is not null -) -and concept_id not in ( - select distinct c1.concept_id - from + SELECT A.concept_id + FROM [CDMV5].[dbo].concept A + LEFT JOIN ( - select concept_id - from sandbox.cdmv5.concept - where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) - ) c1 - inner join - sandbox.cdmv5.concept_relationship cr1 - on c1.concept_id = cr1.concept_id_1 - inner join - sandbox.cdmv5.concept c2 - on cr1.concept_id_2 = c2.concept_id - where c2.standard_concept = 'S' - and c2.invalid_reason is null - and cr1.relationship_id in ('Maps to') - and cr1.invalid_reason is null -) - + SELECT DISTINCT c1.concept_id + FROM ( + SELECT concept_id + FROM [CDMV5].[dbo].concept + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + ) c1 + INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ('Maps to') + AND cr1.invalid_reason IS NULL + ) B ON A.concept_id = B.concept_id + WHERE ( + ( + A.standard_concept <> 'S' + OR A.standard_concept IS NULL + ) + OR A.invalid_reason IS NOT NULL + ) + AND B.concept_id IS NULL ) c1 -inner join -sandbox.cdmv5.concept_relationship cr1 -on c1.concept_id = cr1.concept_id_1 -inner join -sandbox.cdmv5.concept c2 -on cr1.concept_id_2 = c2.concept_id +inner join [CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 +inner join [CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' -and c2.invalid_reason is null -and cr1.relationship_id in ('RxNorm replaced by', -'SNOMED replaced by', -'UCUM replaced by', -'Concept replaced by', -'ICD9P replaced by', -'LOINC replaced by', -'Concept same_as to', -'Concept was_a to', -'Concept alt_to to') -and cr1.invalid_reason is null - -union - ---concepts without 'map to' standard with 'is a' relation to standard -SELECT distinct c1.concept_id as source_concept_id, - c2.concept_id as target_concept_id, - c2.domain_id - ,NULL - FROM -( -select concept_id -from sandbox.cdmv5.concept -where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) -and concept_id not in ( - select distinct c1.concept_id - from - ( - select concept_id - from sandbox.cdmv5.concept - where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) - ) c1 - inner join - sandbox.cdmv5.concept_relationship cr1 - on c1.concept_id = cr1.concept_id_1 - inner join - sandbox.cdmv5.concept c2 - on cr1.concept_id_2 = c2.concept_id - where c2.standard_concept = 'S' - and c2.invalid_reason is null - and cr1.relationship_id in ('Maps to', + and c2.invalid_reason is null + and cr1.relationship_id in ( 'RxNorm replaced by', 'SNOMED replaced by', 'UCUM replaced by', @@ -267,41 +241,107 @@ and concept_id not in ( 'LOINC replaced by', 'Concept same_as to', 'Concept was_a to', - 'Concept alt_to to') - and cr1.invalid_reason is null + 'Concept alt_to to' ) +and cr1.invalid_reason is null -) c1 -inner join -sandbox.cdmv5.concept_relationship cr1 -on c1.concept_id = cr1.concept_id_1 -inner join -sandbox.cdmv5.concept c2 -on cr1.concept_id_2 = c2.concept_id +union + +--concepts without 'map to' standard with 'is a' relation to standard +SELECT DISTINCT c1.concept_id AS source_concept_id + ,c2.concept_id AS target_concept_id + ,c2.domain_id + ,0 + FROM ( + SELECT A.concept_id + FROM [CDMV5].[dbo].concept A + LEFT JOIN + ( + SELECT DISTINCT c1.concept_id + FROM ( + SELECT concept_id + FROM [CDMV5].[dbo].concept + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + ) c1 + INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ( + 'Maps to' + ,'RxNorm replaced by' + ,'SNOMED replaced by' + ,'UCUM replaced by' + ,'Concept replaced by' + ,'ICD9P replaced by' + ,'LOINC replaced by' + ,'Concept same_as to' + ,'Concept was_a to' + ,'Concept alt_to to' + ) + AND cr1.invalid_reason IS NULL + ) B ON A.concept_id = B.concept_id + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + AND B.concept_id IS NULL + ) c1 +INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 +INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' -and c2.invalid_reason is null -and cr1.relationship_id in ('Is a') -and cr1.invalid_reason is null; + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ('Is a') + AND cr1.invalid_reason IS NULL; --- Update the source_concept_mapping_occurrence column +-- Update the source_concept_map_occurrence column -- to contain a count to indicate the number of target_concept_ids -- map to that source_concept_id. This will be used elsewhere in -- the script to ensure that we generate new primary keys -- for the target tables when applicable - UPDATE fviywowuconcept_map - SET fviywowuconcept_map.source_concept_mapping_occurrence = A.CountOfRows +/* + UPDATE #concept_map + SET #concept_map.source_concept_map_occurrence = A.CountOfRows FROM - fviywowuconcept_map, + #concept_map, ( select source_concept_id, domain_id, count(*) as "CountOfRows" - from fviywowuconcept_map + from #concept_map group by source_concept_id, domain_id ) AS A -WHERE fviywowuconcept_map.source_concept_id = A.source_concept_id AND fviywowuconcept_map.domain_id = A.domain_id; - +WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; + +UPDATE #concept_map +SET source_concept_map_occurrence = "CountOfRows" +FROM + ( + select source_concept_id, domain_id, count(*) as "CountOfRows" + from #concept_map + group by source_concept_id, domain_id + ) A +WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; +*/ + +UPDATE w3m16jd0concept_map +SET source_concept_map_occurrence = ( + select count(*) as "CountOfRows" + from w3m16jd0concept_map A + WHERE A.source_concept_id = w3m16jd0concept_map.source_concept_id AND A.domain_id = w3m16jd0concept_map.domain_id + group by source_concept_id, domain_id + ) + BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE fviywowuconcept_map_distinct'; - EXECUTE IMMEDIATE 'DROP TABLE fviywowuconcept_map_distinct'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0concept_map_distinct'; + EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0concept_map_distinct'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -309,20 +349,20 @@ EXCEPTION END IF; END; - CREATE GLOBAL TEMPORARY TABLE fviywowuconcept_map_distinct + CREATE GLOBAL TEMPORARY TABLE w3m16jd0concept_map_distinct ON COMMIT PRESERVE ROWS AS SELECT DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" FROM - fviywowuconcept_map + w3m16jd0concept_map GROUP BY source_concept_id, domain_id; BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE ETL_WARNINGS'; - EXECUTE IMMEDIATE 'DROP TABLE ETL_WARNINGS'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE [CDMV5].[dbo].ETL_WARNINGS'; + EXECUTE IMMEDIATE 'DROP TABLE [CDMV5].[dbo].ETL_WARNINGS'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -330,7 +370,7 @@ EXCEPTION END IF; END; -CREATE TABLE ETL_WARNINGS +CREATE TABLE [CDMV5].[dbo].ETL_WARNINGS ( WARNING_MESSAGE varchar(8000) ); @@ -341,9 +381,9 @@ CDM_SOURCE ****/ -INSERT INTO sandbox.cdmv5.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -SELECT 'sandbox', 'V5', v.vocabulary_version, SYSDATE - FROM sandbox.cdmv5.vocabulary v +INSERT INTO [CDMV5].[dbo].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +SELECT '[CDMV5]', 'V5', v.vocabulary_version, SYSDATE + FROM [CDMV5].[dbo].vocabulary v WHERE vocabulary_id = 'Vocabulary'; /**** @@ -352,9 +392,9 @@ LOCATION ****/ -insert into sandbox.cdmv5.location +insert into [CDMV5].[dbo].location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from sandbox.cdmv4.LOCATION; +from [CDMV4].[dbo].LOCATION; /**** @@ -362,9 +402,9 @@ CARE_SITE ****/ -insert into sandbox.cdmv5.care_site +insert into [CDMV5].[dbo].care_site select care_site_id, TO_CHAR(null ) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from sandbox.cdmv4.CARE_SITE; +from [CDMV4].[dbo].CARE_SITE; /**** @@ -372,11 +412,11 @@ Provider ****/ -insert into sandbox.cdmv5.provider +insert into [CDMV5].[dbo].provider select provider_id, TO_CHAR(null ) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, TO_CHAR(null ) as gender_source_value, cast(null as integer) as gender_source_concept_id -from sandbox.cdmv4.provider +from [CDMV4].[dbo].provider ; @@ -386,7 +426,7 @@ from sandbox.cdmv4.provider ****/ - INSERT into sandbox.cdmv5.person + INSERT into [CDMV5].[dbo].person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -406,21 +446,21 @@ from sandbox.cdmv4.provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM sandbox.cdmv4.PERSON p - left JOIN fviywowuconcept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id - left JOIN fviywowuconcept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id - LEFT JOIN fviywowuconcept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; + FROM [CDMV4].[dbo].PERSON p + left JOIN w3m16jd0concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id + left JOIN w3m16jd0concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id + LEFT JOIN w3m16jd0concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PERSON + FROM [CDMV4].[dbo].PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -431,16 +471,16 @@ from sandbox.cdmv4.provider ) warn ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PERSON + FROM [CDMV4].[dbo].PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -451,16 +491,16 @@ from sandbox.cdmv4.provider ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PERSON + FROM [CDMV4].[dbo].PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -476,9 +516,9 @@ from sandbox.cdmv4.provider ****/ - INSERT INTO sandbox.cdmv5.observation_period + INSERT INTO [CDMV5].[dbo].observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM sandbox.cdmv4.OBSERVATION_PERIOD; + FROM [CDMV4].[dbo].OBSERVATION_PERIOD; /**** @@ -486,28 +526,28 @@ from sandbox.cdmv4.provider ****/ - INSERT INTO sandbox.cdmv5.death + INSERT INTO [CDMV5].[dbo].death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM sandbox.cdmv4.DEATH - LEFT JOIN fviywowuconcept_map_distinct cm1 + FROM [CDMV4].[dbo].DEATH + LEFT JOIN w3m16jd0concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.DEATH + FROM [CDMV4].[dbo].DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -524,7 +564,7 @@ from sandbox.cdmv4.provider ****/ - INSERT INTO sandbox.cdmv5.visit_occurrence + INSERT INTO [CDMV5].[dbo].visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, TO_CHAR(null ) visit_start_time, @@ -533,20 +573,20 @@ from sandbox.cdmv4.provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM sandbox.cdmv4.VISIT_OCCURRENCE - LEFT JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].VISIT_OCCURRENCE + LEFT JOIN w3m16jd0concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.VISIT_OCCURRENCE + FROM [CDMV4].[dbo].VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -571,8 +611,8 @@ from sandbox.cdmv4.provider -- *************************************************************************** BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE fviywowuprocedure_occurrence_map'; - EXECUTE IMMEDIATE 'DROP TABLE fviywowuprocedure_occurrence_map'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0procedure_occurrence_map'; + EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0procedure_occurrence_map'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -581,7 +621,7 @@ EXCEPTION END; --find valid procedures from procedure table - CREATE GLOBAL TEMPORARY TABLE fviywowuprocedure_occurrence_map + CREATE GLOBAL TEMPORARY TABLE w3m16jd0procedure_occurrence_map ON COMMIT PRESERVE ROWS AS SELECT @@ -601,15 +641,15 @@ SELECT CAST(null as NUMBER(19)) as origional_drug_id FROM - sandbox.cdmv4.PROCEDURE_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + [CDMV4].[dbo].PROCEDURE_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_mapping_occurrence = 1 - LEFT JOIN fviywowuconcept_map cm2 + AND cm1.source_concept_map_occurrence = 1 + LEFT JOIN w3m16jd0concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_mapping_occurrence = 1 + AND cm2.source_concept_map_occurrence = 1 UNION ALL @@ -628,7 +668,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE FROM DUAL FROM @@ -652,10 +692,10 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE - LEFT JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + LEFT JOIN w3m16jd0concept_map cm1 ON procedure_concept_id = cm1.source_concept_id - LEFT JOIN fviywowuconcept_map cm2 + LEFT JOIN w3m16jd0concept_map cm2 ON procedure_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') WHERE procedure_concept_id <> 0 @@ -695,12 +735,12 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id, NULL as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_mapping_occurrence > 1 - LEFT JOIN fviywowuconcept_map cm2 + AND cm1.source_concept_map_occurrence > 1 + LEFT JOIN w3m16jd0concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -721,8 +761,8 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM sandbox.cdmv4.CONDITION_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -742,8 +782,8 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].DRUG_EXPOSURE + INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -758,16 +798,16 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.OBSERVATION - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].OBSERVATION + INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO procedure_occurrence +INSERT INTO [CDMV5].[dbo].procedure_occurrence (procedure_occurrence_id ,person_id ,procedure_concept_id @@ -793,19 +833,19 @@ SELECT ,procedure_source_value ,procedure_source_concept_id ,qualifier_source_value -FROM fviywowuprocedure_occurrence_map; +FROM w3m16jd0procedure_occurrence_map; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -815,15 +855,15 @@ FROM fviywowuprocedure_occurrence_map; ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -842,8 +882,8 @@ FROM fviywowuprocedure_occurrence_map; --find valid drugs from drug_exposure table BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE fviywowudrug_exposure_map'; - EXECUTE IMMEDIATE 'DROP TABLE fviywowudrug_exposure_map'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0drug_exposure_map'; + EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0drug_exposure_map'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -851,7 +891,7 @@ EXCEPTION END IF; END; - CREATE GLOBAL TEMPORARY TABLE fviywowudrug_exposure_map + CREATE GLOBAL TEMPORARY TABLE w3m16jd0drug_exposure_map ON COMMIT PRESERVE ROWS AS SELECT @@ -879,15 +919,15 @@ SELECT CAST(null as NUMBER(19)) origional_procedure_id FROM - sandbox.cdmv4.DRUG_EXPOSURE - INNER JOIN fviywowuconcept_map cm1 + [CDMV4].[dbo].DRUG_EXPOSURE + INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_mapping_occurrence = 1 - LEFT JOIN fviywowuconcept_map cm2 + AND cm1.source_concept_map_occurrence = 1 + LEFT JOIN w3m16jd0concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_mapping_occurrence = 1 + AND cm2.source_concept_map_occurrence = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -915,7 +955,7 @@ UNION ALL TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -943,10 +983,10 @@ UNION ALL TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id - FROM sandbox.cdmv4.DRUG_EXPOSURE - LEFT JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].DRUG_EXPOSURE + LEFT JOIN w3m16jd0concept_map cm1 ON drug_concept_id = cm1.source_concept_id - LEFT JOIN fviywowuconcept_map cm2 + LEFT JOIN w3m16jd0concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') WHERE drug_concept_id <> 0 @@ -985,12 +1025,12 @@ FROM TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, NULL as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].DRUG_EXPOSURE + INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_mapping_occurrence > 1 - LEFT JOIN fviywowuconcept_map cm2 + AND cm1.source_concept_map_occurrence > 1 + LEFT JOIN w3m16jd0concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -1004,8 +1044,8 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1019,8 +1059,8 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1033,13 +1073,13 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, observation_id as occurrence_id - FROM sandbox.cdmv4.OBSERVATION - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].OBSERVATION + INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM sandbox.cdmv4.DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [CDMV4].[dbo].DRUG_EXPOSURE) MAXROW ; -INSERT INTO drug_exposure +INSERT INTO [CDMV5].[dbo].drug_exposure (drug_exposure_id ,person_id ,drug_concept_id @@ -1083,19 +1123,19 @@ SELECT ,drug_source_concept_id ,route_source_value ,dose_unit_source_value -FROM fviywowudrug_exposure_map; +FROM w3m16jd0drug_exposure_map; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1105,15 +1145,15 @@ FROM fviywowudrug_exposure_map; ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1131,7 +1171,7 @@ FROM fviywowudrug_exposure_map; ****/ --find valid conditions from condition_occurrence table - INSERT INTO sandbox.cdmv5.condition_occurrence + INSERT INTO [CDMV5].[dbo].condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1140,12 +1180,12 @@ FROM fviywowudrug_exposure_map; COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_mapping_occurrence = 1 - LEFT JOIN fviywowuconcept_map cm2 + AND cm1.source_concept_map_occurrence = 1 + LEFT JOIN w3m16jd0concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 @@ -1164,7 +1204,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1181,10 +1221,10 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE - LEFT JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + LEFT JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - LEFT JOIN fviywowuconcept_map cm2 + LEFT JOIN w3m16jd0concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') WHERE condition_concept_id <> 0 @@ -1207,12 +1247,12 @@ FROM ( stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id, NULL as OCCURRENCE_ID - FROM sandbox.cdmv4.CONDITION_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_mapping_occurrence > 1 - LEFT JOIN fviywowuconcept_map cm2 + AND cm1.source_concept_map_occurrence > 1 + LEFT JOIN w3m16jd0concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 @@ -1226,8 +1266,8 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1239,8 +1279,8 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].DRUG_EXPOSURE + INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1252,25 +1292,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM sandbox.cdmv4.OBSERVATION - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].OBSERVATION + INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM sandbox.cdmv4.CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [CDMV4].[dbo].CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1280,15 +1320,15 @@ FROM ( ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1307,7 +1347,7 @@ FROM ( ****/ -INSERT INTO sandbox.cdmv5.device_exposure +INSERT INTO [CDMV5].[dbo].device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1321,8 +1361,8 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1335,8 +1375,8 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].DRUG_EXPOSURE + INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1349,8 +1389,8 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.CONDITION_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1363,8 +1403,8 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.OBSERVATION - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].OBSERVATION + INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') ) OTHERS @@ -1378,7 +1418,7 @@ FROM ****/ --find valid measurements from observation table -INSERT INTO sandbox.cdmv5.measurement +INSERT INTO [CDMV5].[dbo].measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1386,9 +1426,13 @@ from ( --find mesaurements that were previously classified as observations select person_id, - cm1.target_concept_id AS measurement_concept_id, OBSERVATION_DATE AS measurement_date, - OBSERVATION_TIME AS measurement_time, 0 AS measurement_type_concept_id, CAST(null as integer) operator_concept_id, - value_as_number, value_as_concept_id, + cm1.target_concept_id AS measurement_concept_id, + OBSERVATION_DATE AS measurement_date, + TO_CHAR(OBSERVATION_TIME ) AS measurement_time, + 0 AS measurement_type_concept_id, + CAST(null as integer) operator_concept_id, + value_as_number, + value_as_concept_id, COALESCE(cm2.target_concept_id,0) AS unit_concept_id, range_low, range_high, @@ -1399,11 +1443,11 @@ from unit_source_value AS unit_source_value, TO_CHAR(null ) as value_source_value, observation_id as occurrence_id - FROM sandbox.cdmv4.OBSERVATION - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].OBSERVATION + INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') - LEFT JOIN fviywowuconcept_map cm2 + LEFT JOIN w3m16jd0concept_map cm2 ON observation.unit_concept_id = cm2.source_concept_id AND LOWER(cm1.domain_id) IN ('unit') @@ -1427,8 +1471,8 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, procedure_occurrence_id as occurrence_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1452,8 +1496,8 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, condition_occurrence_id as occurrence_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1477,8 +1521,8 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, drug_exposure_id as occurrence_id - FROM sandbox.cdmv4.drug_exposure - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].drug_exposure + INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1496,35 +1540,70 @@ from --find valid observation from observation table - INSERT INTO sandbox.cdmv5.observation - SELECT observation_id, person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, - value_as_number, value_as_string, value_as_concept_id, CAST(null as integer) qualifier_concept_id, - unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, observation_source_value, CAST(null as integer) observation_source_concept_id, - unit_source_value, TO_CHAR(null ) qualifier_source_value - FROM sandbox.cdmv4.OBSERVATION - WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM fviywowuconcept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) + INSERT INTO [CDMV5].[dbo].observation + SELECT + observation_id, + person_id, + observation_concept_id, + observation_date, + TO_CHAR(observation_time ) as observation_time, + observation_type_concept_id, + value_as_number, + value_as_string, + value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + observation_source_value, + CAST(null as integer) observation_source_concept_id, + unit_source_value, + TO_CHAR(null ) qualifier_source_value + FROM [CDMV4].[dbo].OBSERVATION + WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM w3m16jd0concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) --find observations that were previously classified as procedure UNION ALL -select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS observation_id, - person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, - value_as_number, value_as_string, value_as_concept_id, qualifier_concept_id, - unit_concept_id, provider_id, visit_occurrence_id, observation_source_value, observation_source_concept_id, - unit_source_value, qualifier_source_value +select + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS observation_id, + person_id, + observation_concept_id, + observation_date, + observation_time, + observation_type_concept_id, + value_as_number, + value_as_string, + value_as_concept_id, + qualifier_concept_id, + unit_concept_id, + provider_id, + visit_occurrence_id, + observation_source_value, + observation_source_concept_id, + unit_source_value, + qualifier_source_value FROM ( select person_id, cm1.target_concept_id as observation_concept_id, - procedure_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, procedure_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, TO_CHAR(null ) qualifier_source_value, + procedure_date as observation_date, + TO_CHAR(null ) as observation_time, + 0 as observation_type_concept_id, + CAST(null as float) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + null as unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + procedure_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + TO_CHAR(null ) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1532,14 +1611,23 @@ FROM UNION ALL SELECT person_id, cm1.target_concept_id as observation_concept_id, - condition_start_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, condition_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, TO_CHAR(null ) qualifier_source_value, + condition_start_date as observation_date, + TO_CHAR(null ) as observation_time, + 0 as observation_type_concept_id, + CAST(null as float) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + null as unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + condition_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + TO_CHAR(null ) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1547,17 +1635,26 @@ FROM UNION ALL SELECT person_id, cm1.target_concept_id as observation_concept_id, - drug_exposure_start_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, null as provider_id, - visit_occurrence_id, drug_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, TO_CHAR(null ) qualifier_source_value, + drug_exposure_start_date as observation_date, + TO_CHAR(null ) as observation_time, + 0 as observation_type_concept_id, + CAST(null as float) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + CAST(null as integer) as unit_concept_id, + CAST(null as integer) as provider_id, + visit_occurrence_id, + drug_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + TO_CHAR(null ) qualifier_source_value, drug_exposure_id as occurrence_id - FROM sandbox.cdmv4.DRUG_EXPOSURE - INNER JOIN fviywowuconcept_map cm1 + FROM [CDMV4].[dbo].DRUG_EXPOSURE + INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM sandbox.cdmv4.OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [CDMV4].[dbo].OBSERVATION) MAXROW ; @@ -1568,10 +1665,10 @@ FROM ****/ -INSERT INTO sandbox.cdmv5.payer_plan_period +INSERT INTO [CDMV5].[dbo].payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM sandbox.cdmv4.PAYER_PLAN_PERIOD; +FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; /**** @@ -1581,28 +1678,51 @@ FROM sandbox.cdmv4.PAYER_PLAN_PERIOD; ****/ -INSERT INTO sandbox.cdmv5.drug_cost +INSERT INTO [CDMV5].[dbo].drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM sandbox.cdmv4.DRUG_COST dc +FROM [CDMV4].[dbo].DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO sandbox.cdmv5.drug_cost -select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, - drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, - average_wholesale_price, payer_plan_period_id +INSERT INTO [CDMV5].[dbo].drug_cost +select + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, + drug_exposure_id, + cast(null as integer) currency_concept_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + ingredient_cost, + dispensing_fee, + average_wholesale_price, + payer_plan_period_id FROM ( - SELECT drug_exposure_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, - null as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE po - join sandbox.cdmv4.PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + SELECT + drug_exposure_id, + po.person_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + CAST(null as float) as ingredient_cost, + CAST(null as float) as dispensing_fee, + CAST(null as float) as average_wholesale_price, + payer_plan_period_id, + procedure_cost_id as OCCURRENCE_ID + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE po + join [CDMV4].[dbo].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - JOIN fviywowudrug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW + JOIN w3m16jd0drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW ; /**** @@ -1614,28 +1734,53 @@ FROM ( ****/ -INSERT INTO sandbox.cdmv5.procedure_cost +INSERT INTO [CDMV5].[dbo].procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM sandbox.cdmv4.PROCEDURE_COST; +FROM [CDMV4].[dbo].PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO sandbox.cdmv5.procedure_cost -SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, - procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, - paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, - payer_plan_period_id,revenue_code_concept_id, revenue_code_source_value +INSERT INTO [CDMV5].[dbo].procedure_cost +SELECT + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, + procedure_occurrence_id, + cast(null as integer) + currency_concept_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + payer_plan_period_id, + revenue_code_concept_id, + revenue_code_source_value FROM ( - SELECT po.procedure_occurrence_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, - null as average_wholesale_price, payer_plan_period_id, null as revenue_code_concept_id, null as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE de - join sandbox.cdmv4.DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + SELECT + po.procedure_occurrence_id, + po.person_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + CAST(null as float) as ingredient_cost, + CAST(null as float) as dispensing_fee, + CAST(null as float) as average_wholesale_price, + payer_plan_period_id, + CAST(null as integer) as revenue_code_concept_id, + CAST(null as integer) as revenue_code_source_value, + drug_cost_id as OCCURRENCE_ID + FROM [CDMV4].[dbo].DRUG_EXPOSURE de + join [CDMV4].[dbo].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - JOIN fviywowuprocedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW + JOIN w3m16jd0procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW ; /**** @@ -1646,16 +1791,16 @@ Note: Eras derived from DRUG_EXPOSURE table, using 30d gap ****/ -- drop table dbo.drug_era -INSERT INTO sandbox.cdmv5.drug_era +INSERT INTO [CDMV5].[dbo].drug_era WITH cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, DRUG_EXPOSURE_END_DATE, INGREDIENT_CONCEPT_ID) AS ( -- Normalize DRUG_EXPOSURE_END_DATE to either the existing drug exposure end date, or add days supply, or add 1 day to the start date select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, (DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), (DRUG_EXPOSURE_START_DATE + 1)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM sandbox.cdmv5.DRUG_EXPOSURE d - join sandbox.cdmv5.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join sandbox.cdmv5.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM [CDMV5].[dbo].DRUG_EXPOSURE d + join [CDMV5].[dbo].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join [CDMV5].[dbo].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1718,8 +1863,8 @@ Note: Eras derived from CONDITION_OCCURRENCE table, using 30d gap ****/ BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE fviywowucondition_era_phase_1'; - EXECUTE IMMEDIATE 'DROP TABLE fviywowucondition_era_phase_1'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0condition_era_phase_1'; + EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0condition_era_phase_1'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -1727,7 +1872,7 @@ EXCEPTION END IF; END; -CREATE GLOBAL TEMPORARY TABLE fviywowucondition_era_phase_1 +CREATE GLOBAL TEMPORARY TABLE w3m16jd0condition_era_phase_1 ON COMMIT PRESERVE ROWS AS WITH cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, CONDITION_END_DATE) AS @@ -1735,7 +1880,7 @@ WITH cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, (CONDITION_START_DATE + 1)) as CONDITION_END_DATE - FROM sandbox.cdmv5.CONDITION_OCCURRENCE co + FROM [CDMV5].[dbo].CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1790,7 +1935,7 @@ FROM GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO condition_era +INSERT INTO [CDMV5].[dbo].condition_era (condition_era_id ,person_id ,condition_concept_id @@ -1804,6 +1949,6 @@ SELECT CONDITION_ERA_START_DATE, CONDITION_ERA_END_DATE, CONDITIOn_OCCURRENCE_COUNT -FROM fviywowucondition_era_phase_1 +FROM w3m16jd0condition_era_phase_1 diff --git a/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql b/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql index 012dc82..dc02cf5 100644 --- a/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql +++ b/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - (sandbox, sandbox.cdmv4) + ([CDMV4], [CDMV4].[dbo]) 2. Set parameter name of schema that contains CDMv5 instance - (sandbox, sandbox.cdmv5) + ([CDMV5], [CDMV5].[dbo]) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -76,23 +76,37 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS */ +/* SCRIPT PARAMETERS -- -- The CDMv4 database name -- -- The CDMv4 database plus schema -- -- The target CDMv5 database name -- -- the target CDMv5 database plus schema -/* LOCAL SQL Server */ --- -- The CDMv4 database name --- -- The CDMv4 database plus schema --- -- The target CDMv5 database name --- -- the target CDMv5 database plus schema -/* PostgreSQL Settings */ +*/ +/* QA Settings + -- The CDMv4 database name + -- The CDMv4 database plus schema + -- The target CDMv5 database name + -- the target CDMv5 database plus schema + */ +/* PostgreSQL Settings -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema - -SET search_path TO sandbox; +*/ +/* Oracle Settings + -- The CDMv4 database name + -- The CDMv4 database plus schema + -- The target CDMv5 database name + -- the target CDMv5 database plus schema +*/ +/* LOCAL SQL Server */ + -- The CDMv4 database name + -- The CDMv4 database plus schema + -- The target CDMv5 database name + -- the target CDMv5 database plus schema + +SET search_path TO [CDMV5]; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -120,18 +134,21 @@ SET search_path TO sandbox; */ DROP TABLE IF EXISTS concept_map; ---standard concepts -CREATE TEMP TABLE concept_map +CREATE TEMP TABLE concept_map + ( + source_concept_id int, + target_concept_id int, + domain_id varchar(20), + source_concept_map_occurrence int +); -AS -SELECT - concept_id as source_concept_id, +--standard concepts +INSERT INTO concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) +select concept_id as source_concept_id, concept_id as target_concept_id, domain_id, - NULL as source_concept_mapping_occurrence - -FROM - sandbox.cdmv5.concept + 0 as source_concept_map_occurrence +from [CDMV5].[dbo].concept where standard_concept = 'S' and invalid_reason is null @@ -141,20 +158,20 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,NULL + ,0 from ( select concept_id -from sandbox.cdmv5.concept +from [CDMV5].[dbo].concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -sandbox.cdmv5.concept_relationship cr1 +[CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -sandbox.cdmv5.concept c2 +[CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -168,89 +185,46 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,NULL + ,0 from ( -select concept_id -from sandbox.cdmv5.concept -where ((standard_concept <> 'S' or standard_concept is null) -or invalid_reason is not null -) -and concept_id not in ( - select distinct c1.concept_id - from + SELECT A.concept_id + FROM [CDMV5].[dbo].concept A + LEFT JOIN ( - select concept_id - from sandbox.cdmv5.concept - where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) - ) c1 - inner join - sandbox.cdmv5.concept_relationship cr1 - on c1.concept_id = cr1.concept_id_1 - inner join - sandbox.cdmv5.concept c2 - on cr1.concept_id_2 = c2.concept_id - where c2.standard_concept = 'S' - and c2.invalid_reason is null - and cr1.relationship_id in ('Maps to') - and cr1.invalid_reason is null -) - + SELECT DISTINCT c1.concept_id + FROM ( + SELECT concept_id + FROM [CDMV5].[dbo].concept + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + ) c1 + INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ('Maps to') + AND cr1.invalid_reason IS NULL + ) B ON A.concept_id = B.concept_id + WHERE ( + ( + A.standard_concept <> 'S' + OR A.standard_concept IS NULL + ) + OR A.invalid_reason IS NOT NULL + ) + AND B.concept_id IS NULL ) c1 -inner join -sandbox.cdmv5.concept_relationship cr1 -on c1.concept_id = cr1.concept_id_1 -inner join -sandbox.cdmv5.concept c2 -on cr1.concept_id_2 = c2.concept_id +inner join [CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 +inner join [CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' -and c2.invalid_reason is null -and cr1.relationship_id in ('RxNorm replaced by', -'SNOMED replaced by', -'UCUM replaced by', -'Concept replaced by', -'ICD9P replaced by', -'LOINC replaced by', -'Concept same_as to', -'Concept was_a to', -'Concept alt_to to') -and cr1.invalid_reason is null - -union - ---concepts without 'map to' standard with 'is a' relation to standard -select distinct c1.concept_id as source_concept_id, - c2.concept_id as target_concept_id, - c2.domain_id - ,NULL -from -( -select concept_id -from sandbox.cdmv5.concept -where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) -and concept_id not in ( - select distinct c1.concept_id - from - ( - select concept_id - from sandbox.cdmv5.concept - where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) - ) c1 - inner join - sandbox.cdmv5.concept_relationship cr1 - on c1.concept_id = cr1.concept_id_1 - inner join - sandbox.cdmv5.concept c2 - on cr1.concept_id_2 = c2.concept_id - where c2.standard_concept = 'S' - and c2.invalid_reason is null - and cr1.relationship_id in ('Maps to', + and c2.invalid_reason is null + and cr1.relationship_id in ( 'RxNorm replaced by', 'SNOMED replaced by', 'UCUM replaced by', @@ -259,38 +233,104 @@ and concept_id not in ( 'LOINC replaced by', 'Concept same_as to', 'Concept was_a to', - 'Concept alt_to to') - and cr1.invalid_reason is null + 'Concept alt_to to' ) +and cr1.invalid_reason is null -) c1 -inner join -sandbox.cdmv5.concept_relationship cr1 -on c1.concept_id = cr1.concept_id_1 -inner join -sandbox.cdmv5.concept c2 -on cr1.concept_id_2 = c2.concept_id -where c2.standard_concept = 'S' -and c2.invalid_reason is null -and cr1.relationship_id in ('Is a') -and cr1.invalid_reason is null; +union --- Update the source_concept_mapping_occurrence column +--concepts without 'map to' standard with 'is a' relation to standard +SELECT DISTINCT c1.concept_id AS source_concept_id + ,c2.concept_id AS target_concept_id + ,c2.domain_id + ,0 +FROM ( + SELECT A.concept_id + FROM [CDMV5].[dbo].concept A + LEFT JOIN + ( + SELECT DISTINCT c1.concept_id + FROM ( + SELECT concept_id + FROM [CDMV5].[dbo].concept + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + ) c1 + INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ( + 'Maps to' + ,'RxNorm replaced by' + ,'SNOMED replaced by' + ,'UCUM replaced by' + ,'Concept replaced by' + ,'ICD9P replaced by' + ,'LOINC replaced by' + ,'Concept same_as to' + ,'Concept was_a to' + ,'Concept alt_to to' + ) + AND cr1.invalid_reason IS NULL + ) B ON A.concept_id = B.concept_id + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + AND B.concept_id IS NULL + ) c1 +INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 +INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id +WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ('Is a') + AND cr1.invalid_reason IS NULL; + +-- Update the source_concept_map_occurrence column -- to contain a count to indicate the number of target_concept_ids -- map to that source_concept_id. This will be used elsewhere in -- the script to ensure that we generate new primary keys -- for the target tables when applicable - UPDATE concept_map - SET concept_map.source_concept_mapping_occurrence = A.CountOfRows +/* + UPDATE #concept_map + SET #concept_map.source_concept_map_occurrence = A.CountOfRows FROM - concept_map, + #concept_map, ( select source_concept_id, domain_id, count(*) as "CountOfRows" - from concept_map + from #concept_map group by source_concept_id, domain_id ) AS A -WHERE concept_map.source_concept_id = A.source_concept_id AND concept_map.domain_id = A.domain_id; - +WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; + +UPDATE #concept_map +SET source_concept_map_occurrence = "CountOfRows" +FROM + ( + select source_concept_id, domain_id, count(*) as "CountOfRows" + from #concept_map + group by source_concept_id, domain_id + ) A +WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; +*/ + +UPDATE concept_map +SET source_concept_map_occurrence = ( + select count(*) as "CountOfRows" + from concept_map A + WHERE A.source_concept_id = concept_map.source_concept_id AND A.domain_id = concept_map.domain_id + group by source_concept_id, domain_id + ) + DROP TABLE IF EXISTS concept_map_distinct; CREATE TEMP TABLE concept_map_distinct @@ -304,9 +344,9 @@ FROM GROUP BY source_concept_id, domain_id; -DROP TABLE IF EXISTS ETL_WARNINGS; +DROP TABLE IF EXISTS [CDMV5].[dbo].ETL_WARNINGS; -CREATE TABLE ETL_WARNINGS +CREATE TABLE [CDMV5].[dbo].ETL_WARNINGS ( WARNING_MESSAGE varchar(8000) ); @@ -317,9 +357,9 @@ CDM_SOURCE ****/ -INSERT INTO sandbox.cdmv5.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -select 'sandbox', 'V5', v.vocabulary_version, CURRENT_DATE -from sandbox.cdmv5.vocabulary v +INSERT INTO [CDMV5].[dbo].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +select '[CDMV5]', 'V5', v.vocabulary_version, CURRENT_DATE +from [CDMV5].[dbo].vocabulary v where vocabulary_id = 'Vocabulary'; /**** @@ -328,9 +368,9 @@ LOCATION ****/ -insert into sandbox.cdmv5.location +insert into [CDMV5].[dbo].location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from sandbox.cdmv4.LOCATION; +from [CDMV4].[dbo].LOCATION; /**** @@ -338,9 +378,9 @@ CARE_SITE ****/ -insert into sandbox.cdmv5.care_site +insert into [CDMV5].[dbo].care_site select care_site_id, cast(null as varchar(255)) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from sandbox.cdmv4.CARE_SITE; +from [CDMV4].[dbo].CARE_SITE; /**** @@ -348,11 +388,11 @@ Provider ****/ -insert into sandbox.cdmv5.provider +insert into [CDMV5].[dbo].provider select provider_id, cast(null as varchar(255)) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, cast(null as varchar(50)) as gender_source_value, cast(null as integer) as gender_source_concept_id -from sandbox.cdmv4.provider +from [CDMV4].[dbo].provider ; @@ -362,7 +402,7 @@ from sandbox.cdmv4.provider ****/ - INSERT into sandbox.cdmv5.person + INSERT into [CDMV5].[dbo].person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -382,21 +422,21 @@ from sandbox.cdmv4.provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM sandbox.cdmv4.PERSON p + FROM [CDMV4].[dbo].PERSON p left JOIN concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PERSON + FROM [CDMV4].[dbo].PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -407,16 +447,16 @@ from sandbox.cdmv4.provider ) warn ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PERSON + FROM [CDMV4].[dbo].PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -427,16 +467,16 @@ from sandbox.cdmv4.provider ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PERSON + FROM [CDMV4].[dbo].PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -452,9 +492,9 @@ from sandbox.cdmv4.provider ****/ - INSERT INTO sandbox.cdmv5.observation_period + INSERT INTO [CDMV5].[dbo].observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM sandbox.cdmv4.OBSERVATION_PERIOD; + FROM [CDMV4].[dbo].OBSERVATION_PERIOD; /**** @@ -462,28 +502,28 @@ from sandbox.cdmv4.provider ****/ - INSERT INTO sandbox.cdmv5.death + INSERT INTO [CDMV5].[dbo].death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM sandbox.cdmv4.DEATH + FROM [CDMV4].[dbo].DEATH LEFT JOIN concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.DEATH + FROM [CDMV4].[dbo].DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -500,7 +540,7 @@ from sandbox.cdmv4.provider ****/ - INSERT INTO sandbox.cdmv5.visit_occurrence + INSERT INTO [CDMV5].[dbo].visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, CAST(null as varchar(10)) visit_start_time, @@ -509,20 +549,20 @@ from sandbox.cdmv4.provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM sandbox.cdmv4.VISIT_OCCURRENCE + FROM [CDMV4].[dbo].VISIT_OCCURRENCE LEFT JOIN concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.VISIT_OCCURRENCE + FROM [CDMV4].[dbo].VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -569,15 +609,15 @@ SELECT CAST(null as bigint) as origional_drug_id FROM - sandbox.cdmv4.PROCEDURE_OCCURRENCE + [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_mapping_occurrence = 1 + AND cm1.source_concept_map_occurrence = 1 LEFT JOIN concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_mapping_occurrence = 1 + AND cm2.source_concept_map_occurrence = 1 UNION ALL @@ -596,7 +636,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE WHERE procedure_concept_id = 0 UNION ALL @@ -616,7 +656,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE LEFT JOIN concept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -659,11 +699,11 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, NULL as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_mapping_occurrence > 1 + AND cm1.source_concept_map_occurrence > 1 LEFT JOIN concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -685,7 +725,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -706,7 +746,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -722,16 +762,16 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.OBSERVATION + FROM [CDMV4].[dbo].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO procedure_occurrence +INSERT INTO [CDMV5].[dbo].procedure_occurrence (procedure_occurrence_id ,person_id ,procedure_concept_id @@ -761,15 +801,15 @@ FROM procedure_occurrence_map; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -779,15 +819,15 @@ FROM procedure_occurrence_map; ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -835,15 +875,15 @@ SELECT CAST(null as bigint) origional_procedure_id FROM - sandbox.cdmv4.DRUG_EXPOSURE + [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_mapping_occurrence = 1 + AND cm1.source_concept_map_occurrence = 1 LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_mapping_occurrence = 1 + AND cm2.source_concept_map_occurrence = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -871,7 +911,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -899,7 +939,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE LEFT JOIN concept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -941,11 +981,11 @@ FROM CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, NULL as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_mapping_occurrence > 1 + AND cm1.source_concept_map_occurrence > 1 LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -960,7 +1000,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -975,7 +1015,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -989,13 +1029,13 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, observation_id as occurrence_id - FROM sandbox.cdmv4.OBSERVATION + FROM [CDMV4].[dbo].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM sandbox.cdmv4.DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [CDMV4].[dbo].DRUG_EXPOSURE) MAXROW ; -INSERT INTO drug_exposure +INSERT INTO [CDMV5].[dbo].drug_exposure (drug_exposure_id ,person_id ,drug_concept_id @@ -1043,15 +1083,15 @@ FROM drug_exposure_map; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1061,15 +1101,15 @@ FROM drug_exposure_map; ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1087,7 +1127,7 @@ FROM drug_exposure_map; ****/ --find valid conditions from condition_occurrence table - INSERT INTO sandbox.cdmv5.condition_occurrence + INSERT INTO [CDMV5].[dbo].condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1096,11 +1136,11 @@ FROM drug_exposure_map; COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_mapping_occurrence = 1 + AND cm1.source_concept_map_occurrence = 1 LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1120,7 +1160,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1137,7 +1177,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE LEFT JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -1163,11 +1203,11 @@ FROM ( stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id, NULL as OCCURRENCE_ID - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_mapping_occurrence > 1 + AND cm1.source_concept_map_occurrence > 1 LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1182,7 +1222,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1195,7 +1235,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1208,25 +1248,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM sandbox.cdmv4.OBSERVATION + FROM [CDMV4].[dbo].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM sandbox.cdmv4.CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [CDMV4].[dbo].CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1236,15 +1276,15 @@ FROM ( ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1263,7 +1303,7 @@ FROM ( ****/ -INSERT INTO sandbox.cdmv5.device_exposure +INSERT INTO [CDMV5].[dbo].device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1277,7 +1317,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1291,7 +1331,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1305,7 +1345,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1319,7 +1359,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.OBSERVATION + FROM [CDMV4].[dbo].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1334,7 +1374,7 @@ FROM ****/ --find valid measurements from observation table -INSERT INTO sandbox.cdmv5.measurement +INSERT INTO [CDMV5].[dbo].measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1342,9 +1382,13 @@ from ( --find mesaurements that were previously classified as observations select person_id, - cm1.target_concept_id AS measurement_concept_id, OBSERVATION_DATE AS measurement_date, - OBSERVATION_TIME AS measurement_time, 0 AS measurement_type_concept_id, CAST(null as integer) operator_concept_id, - value_as_number, value_as_concept_id, + cm1.target_concept_id AS measurement_concept_id, + OBSERVATION_DATE AS measurement_date, + CAST(OBSERVATION_TIME as varchar(50)) AS measurement_time, + 0 AS measurement_type_concept_id, + CAST(null as integer) operator_concept_id, + value_as_number, + value_as_concept_id, COALESCE(cm2.target_concept_id,0) AS unit_concept_id, range_low, range_high, @@ -1355,7 +1399,7 @@ from unit_source_value AS unit_source_value, cast(null as varchar(50)) as value_source_value, observation_id as occurrence_id - FROM sandbox.cdmv4.OBSERVATION + FROM [CDMV4].[dbo].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1383,7 +1427,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, procedure_occurrence_id as occurrence_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1408,7 +1452,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, condition_occurrence_id as occurrence_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1433,7 +1477,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, drug_exposure_id as occurrence_id - FROM sandbox.cdmv4.drug_exposure + FROM [CDMV4].[dbo].drug_exposure INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1452,34 +1496,69 @@ from --find valid observation from observation table - INSERT INTO sandbox.cdmv5.observation - SELECT observation_id, person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, - value_as_number, value_as_string, value_as_concept_id, CAST(null as integer) qualifier_concept_id, - unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, observation_source_value, CAST(null as integer) observation_source_concept_id, - unit_source_value, cast(null as varchar(50)) qualifier_source_value - FROM sandbox.cdmv4.OBSERVATION + INSERT INTO [CDMV5].[dbo].observation + SELECT + observation_id, + person_id, + observation_concept_id, + observation_date, + CAST(observation_time as varchar(50)) as observation_time, + observation_type_concept_id, + value_as_number, + value_as_string, + value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + observation_source_value, + CAST(null as integer) observation_source_concept_id, + unit_source_value, + cast(null as varchar(50)) qualifier_source_value + FROM [CDMV4].[dbo].OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) --find observations that were previously classified as procedure UNION ALL -select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS observation_id, - person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, - value_as_number, value_as_string, value_as_concept_id, qualifier_concept_id, - unit_concept_id, provider_id, visit_occurrence_id, observation_source_value, observation_source_concept_id, - unit_source_value, qualifier_source_value +select + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS observation_id, + person_id, + observation_concept_id, + observation_date, + observation_time, + observation_type_concept_id, + value_as_number, + value_as_string, + value_as_concept_id, + qualifier_concept_id, + unit_concept_id, + provider_id, + visit_occurrence_id, + observation_source_value, + observation_source_concept_id, + unit_source_value, + qualifier_source_value FROM ( select person_id, cm1.target_concept_id as observation_concept_id, - procedure_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, procedure_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, + procedure_date as observation_date, + CAST(null as varchar(50)) as observation_time, + 0 as observation_type_concept_id, + CAST(null as NUMERIC) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + null as unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + procedure_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + cast(null as varchar(50)) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1488,13 +1567,22 @@ FROM UNION ALL SELECT person_id, cm1.target_concept_id as observation_concept_id, - condition_start_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, condition_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, + condition_start_date as observation_date, + CAST(null as varchar(50)) as observation_time, + 0 as observation_type_concept_id, + CAST(null as NUMERIC) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + null as unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + condition_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + cast(null as varchar(50)) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1503,17 +1591,26 @@ FROM UNION ALL SELECT person_id, cm1.target_concept_id as observation_concept_id, - drug_exposure_start_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, null as provider_id, - visit_occurrence_id, drug_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, + drug_exposure_start_date as observation_date, + CAST(null as varchar(10)) as observation_time, + 0 as observation_type_concept_id, + CAST(null as NUMERIC) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + CAST(null as integer) as unit_concept_id, + CAST(null as integer) as provider_id, + visit_occurrence_id, + drug_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + cast(null as varchar(50)) qualifier_source_value, drug_exposure_id as occurrence_id - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM sandbox.cdmv4.OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [CDMV4].[dbo].OBSERVATION) MAXROW ; @@ -1524,10 +1621,10 @@ FROM ****/ -INSERT INTO sandbox.cdmv5.payer_plan_period +INSERT INTO [CDMV5].[dbo].payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM sandbox.cdmv4.PAYER_PLAN_PERIOD; +FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; /**** @@ -1537,28 +1634,51 @@ FROM sandbox.cdmv4.PAYER_PLAN_PERIOD; ****/ -INSERT INTO sandbox.cdmv5.drug_cost +INSERT INTO [CDMV5].[dbo].drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM sandbox.cdmv4.DRUG_COST dc +FROM [CDMV4].[dbo].DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO sandbox.cdmv5.drug_cost -select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, - drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, - average_wholesale_price, payer_plan_period_id +INSERT INTO [CDMV5].[dbo].drug_cost +select + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, + drug_exposure_id, + cast(null as integer) currency_concept_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + ingredient_cost, + dispensing_fee, + average_wholesale_price, + payer_plan_period_id FROM ( - SELECT drug_exposure_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, - null as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE po - join sandbox.cdmv4.PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + SELECT + drug_exposure_id, + po.person_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + CAST(null as NUMERIC) as ingredient_cost, + CAST(null as NUMERIC) as dispensing_fee, + CAST(null as NUMERIC) as average_wholesale_price, + payer_plan_period_id, + procedure_cost_id as OCCURRENCE_ID + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE po + join [CDMV4].[dbo].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id JOIN drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW ; /**** @@ -1570,28 +1690,53 @@ FROM ( ****/ -INSERT INTO sandbox.cdmv5.procedure_cost +INSERT INTO [CDMV5].[dbo].procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM sandbox.cdmv4.PROCEDURE_COST; +FROM [CDMV4].[dbo].PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO sandbox.cdmv5.procedure_cost -SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, - procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, - paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, - payer_plan_period_id,revenue_code_concept_id, revenue_code_source_value +INSERT INTO [CDMV5].[dbo].procedure_cost +SELECT + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, + procedure_occurrence_id, + cast(null as integer) + currency_concept_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + payer_plan_period_id, + revenue_code_concept_id, + revenue_code_source_value FROM ( - SELECT po.procedure_occurrence_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, - null as average_wholesale_price, payer_plan_period_id, null as revenue_code_concept_id, null as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE de - join sandbox.cdmv4.DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + SELECT + po.procedure_occurrence_id, + po.person_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + CAST(null as NUMERIC) as ingredient_cost, + CAST(null as NUMERIC) as dispensing_fee, + CAST(null as NUMERIC) as average_wholesale_price, + payer_plan_period_id, + CAST(null as integer) as revenue_code_concept_id, + CAST(null as integer) as revenue_code_source_value, + drug_cost_id as OCCURRENCE_ID + FROM [CDMV4].[dbo].DRUG_EXPOSURE de + join [CDMV4].[dbo].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id JOIN procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW ; /**** @@ -1608,9 +1753,9 @@ with cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONC select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, (DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), (DRUG_EXPOSURE_START_DATE + 1)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM sandbox.cdmv5.DRUG_EXPOSURE d - join sandbox.cdmv5.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join sandbox.cdmv5.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM [CDMV5].[dbo].DRUG_EXPOSURE d + join [CDMV5].[dbo].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join [CDMV5].[dbo].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1660,7 +1805,7 @@ GROUP BY d.PERSON_ID, d.DRUG_TYPE_CONCEPT_ID, d.DRUG_EXPOSURE_START_DATE ) -insert into sandbox.cdmv5.drug_era +insert into [CDMV5].[dbo].drug_era select row_number() over (order by person_id) as drug_era_id, person_id, drug_concept_id, min(DRUG_EXPOSURE_START_DATE) as drug_era_start_date, drug_era_end_date, COUNT(*) as DRUG_EXPOSURE_COUNT, 30 as gap_days from cteDrugExposureEnds GROUP BY person_id, drug_concept_id, drug_type_concept_id, DRUG_ERA_END_DATE @@ -1683,7 +1828,7 @@ WITH cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, (CONDITION_START_DATE + 1)) as CONDITION_END_DATE - FROM sandbox.cdmv5.CONDITION_OCCURRENCE co + FROM [CDMV5].[dbo].CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1738,7 +1883,7 @@ FROM GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO condition_era +INSERT INTO [CDMV5].[dbo].condition_era (condition_era_id ,person_id ,condition_concept_id diff --git a/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql b/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql index be114be..41f4232 100644 --- a/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql +++ b/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - (sandbox, sandbox.cdmv4) + ([CDMV4], [CDMV4].[dbo]) 2. Set parameter name of schema that contains CDMv5 instance - (sandbox, sandbox.cdmv5) + ([CDMV5], [CDMV5].[dbo]) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -76,23 +76,37 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS */ +/* SCRIPT PARAMETERS -- -- The CDMv4 database name -- -- The CDMv4 database plus schema -- -- The target CDMv5 database name -- -- the target CDMv5 database plus schema -/* LOCAL SQL Server */ --- -- The CDMv4 database name --- -- The CDMv4 database plus schema --- -- The target CDMv5 database name --- -- the target CDMv5 database plus schema -/* PostgreSQL Settings */ +*/ +/* QA Settings + -- The CDMv4 database name + -- The CDMv4 database plus schema + -- The target CDMv5 database name + -- the target CDMv5 database plus schema + */ +/* PostgreSQL Settings -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema - -USE sandbox; +*/ +/* Oracle Settings + -- The CDMv4 database name + -- The CDMv4 database plus schema + -- The target CDMv5 database name + -- the target CDMv5 database plus schema +*/ +/* LOCAL SQL Server */ + -- The CDMv4 database name + -- The CDMv4 database plus schema + -- The target CDMv5 database name + -- the target CDMv5 database plus schema + +USE [CDMV5]; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -121,13 +135,21 @@ USE sandbox; IF OBJECT_ID('tempdb..#concept_map', 'U') IS NOT NULL DROP TABLE #concept_map; +CREATE TABLE #concept_map +( + source_concept_id int, + target_concept_id int, + domain_id varchar(20), + source_concept_map_occurrence int +); + --standard concepts +INSERT INTO #concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) select concept_id as source_concept_id, concept_id as target_concept_id, domain_id, - NULL as source_concept_mapping_occurrence -into #concept_map -from sandbox.cdmv5.concept + 0 as source_concept_map_occurrence +from [CDMV5].[dbo].concept where standard_concept = 'S' and invalid_reason is null @@ -137,20 +159,20 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,NULL + ,0 from ( select concept_id -from sandbox.cdmv5.concept +from [CDMV5].[dbo].concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -sandbox.cdmv5.concept_relationship cr1 +[CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -sandbox.cdmv5.concept c2 +[CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -164,89 +186,46 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,NULL + ,0 from ( -select concept_id -from sandbox.cdmv5.concept -where ((standard_concept <> 'S' or standard_concept is null) -or invalid_reason is not null -) -and concept_id not in ( - select distinct c1.concept_id - from + SELECT A.concept_id + FROM [CDMV5].[dbo].concept A + LEFT JOIN ( - select concept_id - from sandbox.cdmv5.concept - where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) - ) c1 - inner join - sandbox.cdmv5.concept_relationship cr1 - on c1.concept_id = cr1.concept_id_1 - inner join - sandbox.cdmv5.concept c2 - on cr1.concept_id_2 = c2.concept_id - where c2.standard_concept = 'S' - and c2.invalid_reason is null - and cr1.relationship_id in ('Maps to') - and cr1.invalid_reason is null -) - + SELECT DISTINCT c1.concept_id + FROM ( + SELECT concept_id + FROM [CDMV5].[dbo].concept + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + ) c1 + INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ('Maps to') + AND cr1.invalid_reason IS NULL + ) B ON A.concept_id = B.concept_id + WHERE ( + ( + A.standard_concept <> 'S' + OR A.standard_concept IS NULL + ) + OR A.invalid_reason IS NOT NULL + ) + AND B.concept_id IS NULL ) c1 -inner join -sandbox.cdmv5.concept_relationship cr1 -on c1.concept_id = cr1.concept_id_1 -inner join -sandbox.cdmv5.concept c2 -on cr1.concept_id_2 = c2.concept_id +inner join [CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 +inner join [CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' -and c2.invalid_reason is null -and cr1.relationship_id in ('RxNorm replaced by', -'SNOMED replaced by', -'UCUM replaced by', -'Concept replaced by', -'ICD9P replaced by', -'LOINC replaced by', -'Concept same_as to', -'Concept was_a to', -'Concept alt_to to') -and cr1.invalid_reason is null - -union - ---concepts without 'map to' standard with 'is a' relation to standard -select distinct c1.concept_id as source_concept_id, - c2.concept_id as target_concept_id, - c2.domain_id - ,NULL -from -( -select concept_id -from sandbox.cdmv5.concept -where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) -and concept_id not in ( - select distinct c1.concept_id - from - ( - select concept_id - from sandbox.cdmv5.concept - where ((standard_concept <> 'S' or standard_concept is null) - or invalid_reason is not null - ) - ) c1 - inner join - sandbox.cdmv5.concept_relationship cr1 - on c1.concept_id = cr1.concept_id_1 - inner join - sandbox.cdmv5.concept c2 - on cr1.concept_id_2 = c2.concept_id - where c2.standard_concept = 'S' - and c2.invalid_reason is null - and cr1.relationship_id in ('Maps to', + and c2.invalid_reason is null + and cr1.relationship_id in ( 'RxNorm replaced by', 'SNOMED replaced by', 'UCUM replaced by', @@ -255,29 +234,76 @@ and concept_id not in ( 'LOINC replaced by', 'Concept same_as to', 'Concept was_a to', - 'Concept alt_to to') - and cr1.invalid_reason is null + 'Concept alt_to to' ) +and cr1.invalid_reason is null -) c1 -inner join -sandbox.cdmv5.concept_relationship cr1 -on c1.concept_id = cr1.concept_id_1 -inner join -sandbox.cdmv5.concept c2 -on cr1.concept_id_2 = c2.concept_id -where c2.standard_concept = 'S' -and c2.invalid_reason is null -and cr1.relationship_id in ('Is a') -and cr1.invalid_reason is null; +union --- Update the source_concept_mapping_occurrence column +--concepts without 'map to' standard with 'is a' relation to standard +SELECT DISTINCT c1.concept_id AS source_concept_id + ,c2.concept_id AS target_concept_id + ,c2.domain_id + ,0 +FROM ( + SELECT A.concept_id + FROM [CDMV5].[dbo].concept A + LEFT JOIN + ( + SELECT DISTINCT c1.concept_id + FROM ( + SELECT concept_id + FROM [CDMV5].[dbo].concept + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + ) c1 + INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ( + 'Maps to' + ,'RxNorm replaced by' + ,'SNOMED replaced by' + ,'UCUM replaced by' + ,'Concept replaced by' + ,'ICD9P replaced by' + ,'LOINC replaced by' + ,'Concept same_as to' + ,'Concept was_a to' + ,'Concept alt_to to' + ) + AND cr1.invalid_reason IS NULL + ) B ON A.concept_id = B.concept_id + WHERE ( + ( + standard_concept <> 'S' + OR standard_concept IS NULL + ) + OR invalid_reason IS NOT NULL + ) + AND B.concept_id IS NULL + ) c1 +INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 +INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id +WHERE c2.standard_concept = 'S' + AND c2.invalid_reason IS NULL + AND cr1.relationship_id IN ('Is a') + AND cr1.invalid_reason IS NULL; + +-- Update the source_concept_map_occurrence column -- to contain a count to indicate the number of target_concept_ids -- map to that source_concept_id. This will be used elsewhere in -- the script to ensure that we generate new primary keys -- for the target tables when applicable +/* UPDATE #concept_map - SET #concept_map.source_concept_mapping_occurrence = A.CountOfRows + SET #concept_map.source_concept_map_occurrence = A.CountOfRows FROM #concept_map, ( @@ -286,7 +312,26 @@ and cr1.invalid_reason is null; group by source_concept_id, domain_id ) AS A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; - + +UPDATE #concept_map +SET source_concept_map_occurrence = "CountOfRows" +FROM + ( + select source_concept_id, domain_id, count(*) as "CountOfRows" + from #concept_map + group by source_concept_id, domain_id + ) A +WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; +*/ + +UPDATE #concept_map +SET source_concept_map_occurrence = ( + select count(*) as "CountOfRows" + from #concept_map A + WHERE A.source_concept_id = #concept_map.source_concept_id AND A.domain_id = #concept_map.domain_id + group by source_concept_id, domain_id + ) + IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL DROP TABLE #concept_map_distinct; @@ -296,10 +341,10 @@ IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL GROUP BY source_concept_id, domain_id; -IF OBJECT_ID('ETL_WARNINGS', 'U') IS NOT NULL - DROP TABLE ETL_WARNINGS; +IF OBJECT_ID('[CDMV5].[dbo].ETL_WARNINGS', 'U') IS NOT NULL + DROP TABLE [CDMV5].[dbo].ETL_WARNINGS; -CREATE TABLE ETL_WARNINGS +CREATE TABLE [CDMV5].[dbo].ETL_WARNINGS ( WARNING_MESSAGE varchar(8000) ); @@ -310,9 +355,9 @@ CDM_SOURCE ****/ -INSERT INTO sandbox.cdmv5.cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -select 'sandbox', 'V5', v.vocabulary_version, getDate() -from sandbox.cdmv5.vocabulary v +INSERT INTO [CDMV5].[dbo].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +select '[CDMV5]', 'V5', v.vocabulary_version, getDate() +from [CDMV5].[dbo].vocabulary v where vocabulary_id = 'Vocabulary'; /**** @@ -321,9 +366,9 @@ LOCATION ****/ -insert into sandbox.cdmv5.location +insert into [CDMV5].[dbo].location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from sandbox.cdmv4.LOCATION; +from [CDMV4].[dbo].LOCATION; /**** @@ -331,9 +376,9 @@ CARE_SITE ****/ -insert into sandbox.cdmv5.care_site +insert into [CDMV5].[dbo].care_site select care_site_id, cast(null as varchar(255)) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from sandbox.cdmv4.CARE_SITE; +from [CDMV4].[dbo].CARE_SITE; /**** @@ -341,11 +386,11 @@ Provider ****/ -insert into sandbox.cdmv5.provider +insert into [CDMV5].[dbo].provider select provider_id, cast(null as varchar(255)) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, cast(null as varchar(50)) as gender_source_value, cast(null as integer) as gender_source_concept_id -from sandbox.cdmv4.provider +from [CDMV4].[dbo].provider ; @@ -355,7 +400,7 @@ from sandbox.cdmv4.provider ****/ - INSERT into sandbox.cdmv5.person + INSERT into [CDMV5].[dbo].person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -375,21 +420,21 @@ from sandbox.cdmv4.provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM sandbox.cdmv4.PERSON p + FROM [CDMV4].[dbo].PERSON p left JOIN #concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN #concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN #concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PERSON + FROM [CDMV4].[dbo].PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -400,16 +445,16 @@ from sandbox.cdmv4.provider ) warn ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PERSON + FROM [CDMV4].[dbo].PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -420,16 +465,16 @@ from sandbox.cdmv4.provider ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PERSON + FROM [CDMV4].[dbo].PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -445,9 +490,9 @@ from sandbox.cdmv4.provider ****/ - INSERT INTO sandbox.cdmv5.observation_period + INSERT INTO [CDMV5].[dbo].observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM sandbox.cdmv4.OBSERVATION_PERIOD; + FROM [CDMV4].[dbo].OBSERVATION_PERIOD; /**** @@ -455,28 +500,28 @@ from sandbox.cdmv4.provider ****/ - INSERT INTO sandbox.cdmv5.death + INSERT INTO [CDMV5].[dbo].death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM sandbox.cdmv4.DEATH + FROM [CDMV4].[dbo].DEATH LEFT JOIN #concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.DEATH + FROM [CDMV4].[dbo].DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -493,7 +538,7 @@ from sandbox.cdmv4.provider ****/ - INSERT INTO sandbox.cdmv5.visit_occurrence + INSERT INTO [CDMV5].[dbo].visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, CAST(null as varchar(10)) visit_start_time, @@ -502,20 +547,20 @@ from sandbox.cdmv4.provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM sandbox.cdmv4.VISIT_OCCURRENCE + FROM [CDMV4].[dbo].VISIT_OCCURRENCE LEFT JOIN #concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.VISIT_OCCURRENCE + FROM [CDMV4].[dbo].VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -558,15 +603,15 @@ from sandbox.cdmv4.provider CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id INTO #procedure_occurrence_map - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_mapping_occurrence = 1 + AND cm1.source_concept_map_occurrence = 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_mapping_occurrence = 1 + AND cm2.source_concept_map_occurrence = 1 UNION ALL @@ -585,7 +630,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE WHERE procedure_concept_id = 0 UNION ALL @@ -605,7 +650,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE LEFT JOIN #concept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -648,11 +693,11 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, NULL as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_mapping_occurrence > 1 + AND cm1.source_concept_map_occurrence > 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -674,7 +719,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -695,7 +740,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -711,16 +756,16 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.OBSERVATION + FROM [CDMV4].[dbo].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO procedure_occurrence +INSERT INTO [CDMV5].[dbo].procedure_occurrence (procedure_occurrence_id ,person_id ,procedure_concept_id @@ -750,15 +795,15 @@ FROM #procedure_occurrence_map; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -768,15 +813,15 @@ FROM #procedure_occurrence_map; ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -820,15 +865,15 @@ IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id INTO #drug_exposure_map - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_mapping_occurrence = 1 + AND cm1.source_concept_map_occurrence = 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_mapping_occurrence = 1 + AND cm2.source_concept_map_occurrence = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -856,7 +901,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -884,7 +929,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE LEFT JOIN #concept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -926,11 +971,11 @@ FROM CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, NULL as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_mapping_occurrence > 1 + AND cm1.source_concept_map_occurrence > 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -945,7 +990,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -960,7 +1005,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -974,13 +1019,13 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, observation_id as occurrence_id - FROM sandbox.cdmv4.OBSERVATION + FROM [CDMV4].[dbo].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM sandbox.cdmv4.DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [CDMV4].[dbo].DRUG_EXPOSURE) MAXROW ; -INSERT INTO drug_exposure +INSERT INTO [CDMV5].[dbo].drug_exposure (drug_exposure_id ,person_id ,drug_concept_id @@ -1028,15 +1073,15 @@ FROM #drug_exposure_map; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1046,15 +1091,15 @@ FROM #drug_exposure_map; ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1072,7 +1117,7 @@ FROM #drug_exposure_map; ****/ --find valid conditions from condition_occurrence table - INSERT INTO sandbox.cdmv5.condition_occurrence + INSERT INTO [CDMV5].[dbo].condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1081,11 +1126,11 @@ FROM #drug_exposure_map; COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_mapping_occurrence = 1 + AND cm1.source_concept_map_occurrence = 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1105,7 +1150,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1122,7 +1167,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE LEFT JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -1148,11 +1193,11 @@ FROM ( stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id, NULL as OCCURRENCE_ID - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_mapping_occurrence > 1 + AND cm1.source_concept_map_occurrence > 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1167,7 +1212,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1180,7 +1225,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1193,25 +1238,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM sandbox.cdmv4.OBSERVATION + FROM [CDMV4].[dbo].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM sandbox.cdmv4.CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [CDMV4].[dbo].CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1221,15 +1266,15 @@ FROM ( ; - INSERT INTO ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM sandbox.cdmv5.CONCEPT + FROM [CDMV5].[dbo].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1248,7 +1293,7 @@ FROM ( ****/ -INSERT INTO sandbox.cdmv5.device_exposure +INSERT INTO [CDMV5].[dbo].device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1262,7 +1307,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1276,7 +1321,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1290,7 +1335,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1304,7 +1349,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM sandbox.cdmv4.OBSERVATION + FROM [CDMV4].[dbo].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1319,7 +1364,7 @@ FROM ****/ --find valid measurements from observation table -INSERT INTO sandbox.cdmv5.measurement +INSERT INTO [CDMV5].[dbo].measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1327,9 +1372,13 @@ from ( --find mesaurements that were previously classified as observations select person_id, - cm1.target_concept_id AS measurement_concept_id, OBSERVATION_DATE AS measurement_date, - OBSERVATION_TIME AS measurement_time, 0 AS measurement_type_concept_id, CAST(null as integer) operator_concept_id, - value_as_number, value_as_concept_id, + cm1.target_concept_id AS measurement_concept_id, + OBSERVATION_DATE AS measurement_date, + CAST(OBSERVATION_TIME as varchar(50)) AS measurement_time, + 0 AS measurement_type_concept_id, + CAST(null as integer) operator_concept_id, + value_as_number, + value_as_concept_id, COALESCE(cm2.target_concept_id,0) AS unit_concept_id, range_low, range_high, @@ -1340,7 +1389,7 @@ from unit_source_value AS unit_source_value, cast(null as varchar(50)) as value_source_value, observation_id as occurrence_id - FROM sandbox.cdmv4.OBSERVATION + FROM [CDMV4].[dbo].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1368,7 +1417,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, procedure_occurrence_id as occurrence_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1393,7 +1442,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, condition_occurrence_id as occurrence_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1418,7 +1467,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, drug_exposure_id as occurrence_id - FROM sandbox.cdmv4.drug_exposure + FROM [CDMV4].[dbo].drug_exposure INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1437,34 +1486,69 @@ from --find valid observation from observation table - INSERT INTO sandbox.cdmv5.observation - SELECT observation_id, person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, - value_as_number, value_as_string, value_as_concept_id, CAST(null as integer) qualifier_concept_id, - unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, observation_source_value, CAST(null as integer) observation_source_concept_id, - unit_source_value, cast(null as varchar(50)) qualifier_source_value - FROM sandbox.cdmv4.OBSERVATION + INSERT INTO [CDMV5].[dbo].observation + SELECT + observation_id, + person_id, + observation_concept_id, + observation_date, + CAST(observation_time as varchar(50)) as observation_time, + observation_type_concept_id, + value_as_number, + value_as_string, + value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + observation_source_value, + CAST(null as integer) observation_source_concept_id, + unit_source_value, + cast(null as varchar(50)) qualifier_source_value + FROM [CDMV4].[dbo].OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM #concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) --find observations that were previously classified as procedure UNION ALL -select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS observation_id, - person_id, observation_concept_id, observation_date, observation_time, observation_type_concept_id, - value_as_number, value_as_string, value_as_concept_id, qualifier_concept_id, - unit_concept_id, provider_id, visit_occurrence_id, observation_source_value, observation_source_concept_id, - unit_source_value, qualifier_source_value +select + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS observation_id, + person_id, + observation_concept_id, + observation_date, + observation_time, + observation_type_concept_id, + value_as_number, + value_as_string, + value_as_concept_id, + qualifier_concept_id, + unit_concept_id, + provider_id, + visit_occurrence_id, + observation_source_value, + observation_source_concept_id, + unit_source_value, + qualifier_source_value FROM ( select person_id, cm1.target_concept_id as observation_concept_id, - procedure_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, procedure_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, + procedure_date as observation_date, + CAST(null as varchar(50)) as observation_time, + 0 as observation_type_concept_id, + CAST(null as float) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + null as unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + procedure_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + cast(null as varchar(50)) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1473,13 +1557,22 @@ FROM UNION ALL SELECT person_id, cm1.target_concept_id as observation_concept_id, - condition_start_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, associated_provider_id as provider_id, - visit_occurrence_id, condition_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, + condition_start_date as observation_date, + CAST(null as varchar(50)) as observation_time, + 0 as observation_type_concept_id, + CAST(null as float) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + null as unit_concept_id, + associated_provider_id as provider_id, + visit_occurrence_id, + condition_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + cast(null as varchar(50)) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM sandbox.cdmv4.CONDITION_OCCURRENCE + FROM [CDMV4].[dbo].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1488,17 +1581,26 @@ FROM UNION ALL SELECT person_id, cm1.target_concept_id as observation_concept_id, - drug_exposure_start_date as observation_date, null as observation_time, - 0 as observation_type_concept_id, null as value_as_number, null as value_as_string, null as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, null as provider_id, - visit_occurrence_id, drug_source_value as observation_source_value, CAST(null as integer) observation_source_concept_id, - null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, + drug_exposure_start_date as observation_date, + CAST(null as varchar(10)) as observation_time, + 0 as observation_type_concept_id, + CAST(null as float) as value_as_number, + null as value_as_string, + CAST(null as integer) as value_as_concept_id, + CAST(null as integer) qualifier_concept_id, + CAST(null as integer) as unit_concept_id, + CAST(null as integer) as provider_id, + visit_occurrence_id, + drug_source_value as observation_source_value, + CAST(null as integer) observation_source_concept_id, + null as unit_source_value, + cast(null as varchar(50)) qualifier_source_value, drug_exposure_id as occurrence_id - FROM sandbox.cdmv4.DRUG_EXPOSURE + FROM [CDMV4].[dbo].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM sandbox.cdmv4.OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [CDMV4].[dbo].OBSERVATION) MAXROW ; @@ -1509,10 +1611,10 @@ FROM ****/ -INSERT INTO sandbox.cdmv5.payer_plan_period +INSERT INTO [CDMV5].[dbo].payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM sandbox.cdmv4.PAYER_PLAN_PERIOD; +FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; /**** @@ -1522,28 +1624,51 @@ FROM sandbox.cdmv4.PAYER_PLAN_PERIOD; ****/ -INSERT INTO sandbox.cdmv5.drug_cost +INSERT INTO [CDMV5].[dbo].drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM sandbox.cdmv4.DRUG_COST dc +FROM [CDMV4].[dbo].DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO sandbox.cdmv5.drug_cost -select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, - drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, - average_wholesale_price, payer_plan_period_id +INSERT INTO [CDMV5].[dbo].drug_cost +select + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, + drug_exposure_id, + cast(null as integer) currency_concept_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + ingredient_cost, + dispensing_fee, + average_wholesale_price, + payer_plan_period_id FROM ( - SELECT drug_exposure_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, - null as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM sandbox.cdmv4.PROCEDURE_OCCURRENCE po - join sandbox.cdmv4.PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + SELECT + drug_exposure_id, + po.person_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + CAST(null as float) as ingredient_cost, + CAST(null as float) as dispensing_fee, + CAST(null as float) as average_wholesale_price, + payer_plan_period_id, + procedure_cost_id as OCCURRENCE_ID + FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE po + join [CDMV4].[dbo].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id JOIN #drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW ; /**** @@ -1555,28 +1680,53 @@ FROM ( ****/ -INSERT INTO sandbox.cdmv5.procedure_cost +INSERT INTO [CDMV5].[dbo].procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM sandbox.cdmv4.PROCEDURE_COST; +FROM [CDMV4].[dbo].PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO sandbox.cdmv5.procedure_cost -SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, - procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, - paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, - payer_plan_period_id,revenue_code_concept_id, revenue_code_source_value +INSERT INTO [CDMV5].[dbo].procedure_cost +SELECT + CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, + procedure_occurrence_id, + cast(null as integer) + currency_concept_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + payer_plan_period_id, + revenue_code_concept_id, + revenue_code_source_value FROM ( - SELECT po.procedure_occurrence_id, po.person_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, - paid_by_coordination_benefits, total_out_of_pocket, total_paid, null as ingredient_cost, null as dispensing_fee, - null as average_wholesale_price, payer_plan_period_id, null as revenue_code_concept_id, null as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM sandbox.cdmv4.DRUG_EXPOSURE de - join sandbox.cdmv4.DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + SELECT + po.procedure_occurrence_id, + po.person_id, + paid_copay, + paid_coinsurance, + paid_toward_deductible, + paid_by_payer, + paid_by_coordination_benefits, + total_out_of_pocket, + total_paid, + CAST(null as float) as ingredient_cost, + CAST(null as float) as dispensing_fee, + CAST(null as float) as average_wholesale_price, + payer_plan_period_id, + CAST(null as integer) as revenue_code_concept_id, + CAST(null as integer) as revenue_code_source_value, + drug_cost_id as OCCURRENCE_ID + FROM [CDMV4].[dbo].DRUG_EXPOSURE de + join [CDMV4].[dbo].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id JOIN #procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM sandbox.cdmv4.DRUG_COST) MAXROW + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW ; /**** @@ -1593,9 +1743,9 @@ with cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONC select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, DATEADD(day,DAYS_SUPPLY,DRUG_EXPOSURE_START_DATE), DATEADD(day,1,DRUG_EXPOSURE_START_DATE)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM sandbox.cdmv5.DRUG_EXPOSURE d - join sandbox.cdmv5.CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join sandbox.cdmv5.CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM [CDMV5].[dbo].DRUG_EXPOSURE d + join [CDMV5].[dbo].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join [CDMV5].[dbo].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1645,7 +1795,7 @@ GROUP BY d.PERSON_ID, d.DRUG_TYPE_CONCEPT_ID, d.DRUG_EXPOSURE_START_DATE ) -insert into sandbox.cdmv5.drug_era +insert into [CDMV5].[dbo].drug_era select row_number() over (order by person_id) as drug_era_id, person_id, drug_concept_id, min(DRUG_EXPOSURE_START_DATE) as drug_era_start_date, drug_era_end_date, COUNT(*) as DRUG_EXPOSURE_COUNT, 30 as gap_days from cteDrugExposureEnds GROUP BY person_id, drug_concept_id, drug_type_concept_id, DRUG_ERA_END_DATE @@ -1666,7 +1816,7 @@ with cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, DATEADD(day,1,CONDITION_START_DATE)) as CONDITION_END_DATE - FROM sandbox.cdmv5.CONDITION_OCCURRENCE co + FROM [CDMV5].[dbo].CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1719,7 +1869,7 @@ from cteConditionEnds GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO condition_era +INSERT INTO [CDMV5].[dbo].condition_era (condition_era_id ,person_id ,condition_concept_id From c3e7bb7eeae4ef92997dcd484335fdb44a73308a Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Thu, 9 Jul 2015 08:47:18 -0400 Subject: [PATCH 3/5] Final commit before push --- .../OMOP CDMv4 to CDMv5 - templateSQL.sql | 212 +++++--- .../Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql | 501 ++++++++++-------- .../OMOP CDMv4 to CDMv5 - PostgreSQL.sql | 495 +++++++++-------- .../OMOP CDMv4 to CDMv5 - SQL Server.sql | 474 ++++++++++------- 4 files changed, 962 insertions(+), 720 deletions(-) diff --git a/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql b/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql index 80d83e5..347f4df 100644 --- a/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql +++ b/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql @@ -19,7 +19,7 @@ PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. -last revised: 01 July 2015 +last revised: 09 July 2015 author: Patrick Ryan, Chris Knoll editor: Anthony Sena @@ -34,7 +34,7 @@ instructions will be slightly different. General Assumptions ------------------- -This script assumes that your V4 and V5 database either located on the same +This script assumes that your V4 and V5 database are located on the same RDBMS server. Getting Started @@ -76,36 +76,12 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS ---{DEFAULT @SOURCE_CDMV4 = '[SOURCE_CDMV4]'} -- The CDMv4 database name ---{DEFAULT @SOURCE_CDMV4_SCHEMA = '[SOURCE_CDMV4].[SCHEMA]'} -- The CDMv4 database plus schema ---{DEFAULT @TARGET_CDMV5 = '[TARGET_CDMV5]'} -- The target CDMv5 database name ---{DEFAULT @TARGET_CDMV5_SCHEMA = '[TARGET_CDMV5].[SCHEMA]'} -- the target CDMv5 database plus schema -*/ -/* QA Settings -{DEFAULT @SOURCE_CDMV4 = 'CDM_TRUVEN_CCAE_6k'} -- The CDMv4 database name -{DEFAULT @SOURCE_CDMV4_SCHEMA = 'CDM_TRUVEN_CCAE_6k.dbo'} -- The CDMv4 database plus schema -{DEFAULT @TARGET_CDMV5 = 'CDMV5_Conversion_Target'} -- The target CDMv5 database name -{DEFAULT @TARGET_CDMV5_SCHEMA = 'CDMV5_Conversion_Target.dbo'} -- the target CDMv5 database plus schema - */ -/* PostgreSQL Settings -{DEFAULT @SOURCE_CDMV4 = 'sandbox'} -- The CDMv4 database name -{DEFAULT @SOURCE_CDMV4_SCHEMA = 'sandbox.cdmv4'} -- The CDMv4 database plus schema -{DEFAULT @TARGET_CDMV5 = 'sandbox'} -- The target CDMv5 database name -{DEFAULT @TARGET_CDMV5_SCHEMA = 'sandbox.cdmv5'} -- the target CDMv5 database plus schema -*/ -/* Oracle Settings -{DEFAULT @SOURCE_CDMV4 = 'CDMV4'} -- The CDMv4 database name -{DEFAULT @SOURCE_CDMV4_SCHEMA = 'CDMV4'} -- The CDMv4 database plus schema -{DEFAULT @TARGET_CDMV5 = 'CDMV5'} -- The target CDMv5 database name -{DEFAULT @TARGET_CDMV5_SCHEMA = 'CDMV5'} -- the target CDMv5 database plus schema -*/ -/* LOCAL SQL Server */ -{DEFAULT @SOURCE_CDMV4 = '[CDMV4]'} -- The CDMv4 database name -{DEFAULT @SOURCE_CDMV4_SCHEMA = '[CDMV4].[dbo]'} -- The CDMv4 database plus schema -{DEFAULT @TARGET_CDMV5 = '[CDMV5]'} -- The target CDMv5 database name -{DEFAULT @TARGET_CDMV5_SCHEMA = '[CDMV5].[dbo]'} -- the target CDMv5 database plus schema - +/* SCRIPT PARAMETERS */ +{DEFAULT @SOURCE_CDMV4 = '[SOURCE_CDMV4]'} -- The CDMv4 database name +{DEFAULT @SOURCE_CDMV4_SCHEMA = '[SOURCE_CDMV4].[SCHEMA]'} -- The CDMv4 database plus schema +{DEFAULT @TARGET_CDMV5 = '[TARGET_CDMV5]'} -- The target CDMv5 database name +{DEFAULT @TARGET_CDMV5_SCHEMA = '[TARGET_CDMV5].[SCHEMA]'} -- the target CDMv5 database plus schema + USE @TARGET_CDMV5; /* @@ -139,16 +115,14 @@ CREATE TABLE #concept_map ( source_concept_id int, target_concept_id int, - domain_id varchar(20), - source_concept_map_occurrence int + domain_id varchar(20) ); --standard concepts -INSERT INTO #concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) +INSERT INTO #concept_map (source_concept_id, target_concept_id, domain_id) select concept_id as source_concept_id, concept_id as target_concept_id, - domain_id, - 0 as source_concept_map_occurrence + domain_id from @TARGET_CDMV5_SCHEMA.concept where standard_concept = 'S' and invalid_reason is null @@ -159,7 +133,6 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( select concept_id @@ -186,7 +159,6 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( SELECT A.concept_id @@ -244,7 +216,6 @@ union SELECT DISTINCT c1.concept_id AS source_concept_id ,c2.concept_id AS target_concept_id ,c2.domain_id - ,0 FROM ( SELECT A.concept_id FROM @TARGET_CDMV5_SCHEMA.concept A @@ -303,40 +274,47 @@ WHERE c2.standard_concept = 'S' -- for the target tables when applicable /* UPDATE #concept_map - SET #concept_map.source_concept_map_occurrence = A.CountOfRows + SET #concept_map.source_concept_map_occurrence = A.targetConceptCount FROM #concept_map, ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) AS A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; UPDATE #concept_map -SET source_concept_map_occurrence = "CountOfRows" +SET source_concept_map_occurrence = "targetConceptCount" FROM ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; -*/ UPDATE #concept_map SET source_concept_map_occurrence = ( - select count(*) as "CountOfRows" + select count(*) as "targetConceptCount" from #concept_map A WHERE A.source_concept_id = #concept_map.source_concept_id AND A.domain_id = #concept_map.domain_id group by source_concept_id, domain_id ) +*/ IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL DROP TABLE #concept_map_distinct; - SELECT DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" - INTO #concept_map_distinct +CREATE TABLE #concept_map_distinct +( + source_concept_id int, + domain_id varchar(20), + targetConceptCount int +); + + INSERT INTO #concept_map_distinct (source_concept_id, domain_id, targetConceptCount) + SELECT source_concept_id, domain_id, COUNT(*) FROM #concept_map GROUP BY source_concept_id, domain_id; @@ -346,7 +324,7 @@ IF OBJECT_ID('@TARGET_CDMV5_SCHEMA.ETL_WARNINGS', 'U') IS NOT NULL CREATE TABLE @TARGET_CDMV5_SCHEMA.ETL_WARNINGS ( - WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(4000) ); /**** @@ -579,15 +557,48 @@ from @SOURCE_CDMV4_SCHEMA.provider -- *************************************************************************** -- AGS: Modifying this section to insert this information into the temp - -- table #procedure_occurrence_map but this may need to be revisited for + -- table #po_map but this may need to be revisited for -- performance tuning on APS as a large temp table may cause processing -- time issues. -- *************************************************************************** - IF OBJECT_ID('tempdb..#procedure_occurrence_map', 'U') IS NOT NULL - DROP TABLE #procedure_occurrence_map; + IF OBJECT_ID('tempdb..#po_map', 'U') IS NOT NULL + DROP TABLE #po_map; + +CREATE TABLE #po_map +( + procedure_occurrence_id int, + person_id int, + procedure_concept_id int, + procedure_date date, + procedure_type_concept_id int, + modifier_concept_id int, + quantity int, + provider_id int, + visit_occurrence_id int, + procedure_source_value varchar(50), + procedure_source_concept_id int, + qualifier_source_value varchar(50), + origional_drug_id bigint +); --find valid procedures from procedure table +INSERT INTO #po_map +( + procedure_occurrence_id, + person_id, + procedure_concept_id, + procedure_date, + procedure_type_concept_id, + modifier_concept_id, + quantity, + provider_id, + visit_occurrence_id, + procedure_source_value, + procedure_source_concept_id, + qualifier_source_value, + origional_drug_id +) SELECT procedure_occurrence_id, person_id, @@ -602,16 +613,17 @@ from @SOURCE_CDMV4_SCHEMA.provider CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - INTO #procedure_occurrence_map FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence = 1 + AND LOWER(cm1.domain_id) IN ('procedure') + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_map_occurrence = 1 + LEFT JOIN #concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 UNION ALL @@ -631,7 +643,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - WHERE procedure_concept_id = 0 + WHERE procedure_concept_id = 0 UNION ALL @@ -697,7 +709,8 @@ UNION ALL INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -791,7 +804,7 @@ SELECT ,procedure_source_value ,procedure_source_concept_id ,qualifier_source_value -FROM #procedure_occurrence_map; +FROM #po_map; --warnings of invalid records @@ -839,9 +852,60 @@ FROM #procedure_occurrence_map; ****/ --find valid drugs from drug_exposure table -IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL - DROP TABLE #drug_exposure_map; +IF OBJECT_ID('tempdb..#drgexp_map', 'U') IS NOT NULL + DROP TABLE #drgexp_map; +CREATE TABLE #drgexp_map +( + drug_exposure_id int, + person_id int, + drug_concept_id int, + drug_exposure_start_date date, + drug_exposure_end_date date, + drug_type_concept_id int, + stop_reason varchar(20), + refills int, + quantity float, + days_supply int, + sig varchar(max), + route_concept_id int, + effective_drug_dose float, + dose_unit_concept_id int, + lot_number varchar(50), + provider_id int, + visit_occurrence_id int, + drug_source_value varchar(50), + drug_source_concept_id int, + route_source_value varchar(50), + dose_unit_source_value varchar(50), + origional_procedure_id int +); + +INSERT INTO #drgexp_map +( + drug_exposure_id, + person_id, + drug_concept_id, + drug_exposure_start_date, + drug_exposure_end_date, + drug_type_concept_id, + stop_reason, + refills, + quantity, + days_supply, + sig, + route_concept_id, + effective_drug_dose, + dose_unit_concept_id, + lot_number, + provider_id, + visit_occurrence_id, + drug_source_value, + drug_source_concept_id, + route_source_value, + dose_unit_source_value, + origional_procedure_id +) SELECT drug_exposure_id, person_id, COALESCE(cm1.target_concept_id,0) as drug_concept_id, @@ -864,16 +928,17 @@ IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - INTO #drug_exposure_map FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -975,7 +1040,8 @@ FROM INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -1069,7 +1135,7 @@ SELECT ,drug_source_concept_id ,route_source_value ,dose_unit_source_value -FROM #drug_exposure_map; +FROM #drgexp_map; --warnings of invalid records @@ -1130,7 +1196,8 @@ FROM #drug_exposure_map; INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1197,7 +1264,8 @@ FROM ( INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1540,7 +1608,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as observation_source_value, @@ -1564,7 +1632,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value as observation_source_value, @@ -1667,7 +1735,7 @@ FROM ( FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE po join @SOURCE_CDMV4_SCHEMA.PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - JOIN #drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id + JOIN #drgexp_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id ) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM @SOURCE_CDMV4_SCHEMA.DRUG_COST) MAXROW ; @@ -1725,7 +1793,7 @@ SELECT FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE de join @SOURCE_CDMV4_SCHEMA.DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - JOIN #procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id + JOIN #po_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM @SOURCE_CDMV4_SCHEMA.DRUG_COST) MAXROW ; diff --git a/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql b/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql index 8cae4c9..b485f41 100644 --- a/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql +++ b/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql @@ -19,7 +19,7 @@ PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. -last revised: 01 July 2015 +last revised: 09 July 2015 author: Patrick Ryan, Chris Knoll editor: Anthony Sena @@ -34,7 +34,7 @@ instructions will be slightly different. General Assumptions ------------------- -This script assumes that your V4 and V5 database either located on the same +This script assumes that your V4 and V5 database are located on the same RDBMS server. Getting Started @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - ([CDMV4], [CDMV4].[dbo]) + ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) 2. Set parameter name of schema that contains CDMv5 instance - ([CDMV5], [CDMV5].[dbo]) + ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -76,37 +76,13 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS --- -- The CDMv4 database name --- -- The CDMv4 database plus schema --- -- The target CDMv5 database name --- -- the target CDMv5 database plus schema -*/ -/* QA Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - */ -/* PostgreSQL Settings +/* SCRIPT PARAMETERS */ -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema -*/ -/* Oracle Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema -*/ -/* LOCAL SQL Server */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -ALTER SESSION SET current_schema = [CDMV5]; + +ALTER SESSION SET current_schema = [TARGET_CDMV5]; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -146,17 +122,15 @@ CREATE GLOBAL TEMPORARY TABLE w3m16jd0concept_map ( source_concept_id int, target_concept_id int, - domain_id varchar(20), - source_concept_map_occurrence int + domain_id varchar(20) ) ON COMMIT PRESERVE ROWS; --standard concepts -INSERT INTO w3m16jd0concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) +INSERT INTO w3m16jd0concept_map (source_concept_id, target_concept_id, domain_id) SELECT concept_id as source_concept_id, concept_id as target_concept_id, - domain_id, - 0 as source_concept_map_occurrence - FROM [CDMV5].[dbo].concept + domain_id + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE standard_concept = 'S' and invalid_reason is null @@ -166,20 +140,19 @@ union SELECT distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 FROM ( select concept_id -from [CDMV5].[dbo].concept +from [TARGET_CDMV5].[SCHEMA].concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -[CDMV5].[dbo].concept_relationship cr1 +[TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -[CDMV5].[dbo].concept c2 +[TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' and c2.invalid_reason is null @@ -193,17 +166,16 @@ union SELECT distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 FROM ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -212,8 +184,8 @@ SELECT distinct c1.concept_id as source_concept_id, OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Maps to') @@ -228,8 +200,8 @@ SELECT distinct c1.concept_id as source_concept_id, ) AND B.concept_id IS NULL ) c1 -inner join [CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 -inner join [CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id +inner join [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 +inner join [TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' and c2.invalid_reason is null and cr1.relationship_id in ( @@ -251,16 +223,15 @@ union SELECT DISTINCT c1.concept_id AS source_concept_id ,c2.concept_id AS target_concept_id ,c2.domain_id - ,0 FROM ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -269,8 +240,8 @@ SELECT DISTINCT c1.concept_id AS source_concept_id OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ( @@ -296,8 +267,8 @@ SELECT DISTINCT c1.concept_id AS source_concept_id ) AND B.concept_id IS NULL ) c1 -INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Is a') @@ -310,34 +281,34 @@ INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id -- for the target tables when applicable /* UPDATE #concept_map - SET #concept_map.source_concept_map_occurrence = A.CountOfRows + SET #concept_map.source_concept_map_occurrence = A.targetConceptCount FROM #concept_map, ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) AS A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; UPDATE #concept_map -SET source_concept_map_occurrence = "CountOfRows" +SET source_concept_map_occurrence = "targetConceptCount" FROM ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; -*/ -UPDATE w3m16jd0concept_map +UPDATE #concept_map SET source_concept_map_occurrence = ( - select count(*) as "CountOfRows" - from w3m16jd0concept_map A - WHERE A.source_concept_id = w3m16jd0concept_map.source_concept_id AND A.domain_id = w3m16jd0concept_map.domain_id + select count(*) as "targetConceptCount" + from #concept_map A + WHERE A.source_concept_id = #concept_map.source_concept_id AND A.domain_id = #concept_map.domain_id group by source_concept_id, domain_id ) +*/ BEGIN EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0concept_map_distinct'; @@ -349,20 +320,22 @@ EXCEPTION END IF; END; - CREATE GLOBAL TEMPORARY TABLE w3m16jd0concept_map_distinct - ON COMMIT PRESERVE ROWS -AS -SELECT - DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" - -FROM - w3m16jd0concept_map +CREATE GLOBAL TEMPORARY TABLE w3m16jd0concept_map_distinct + ( + source_concept_id int, + domain_id varchar(20), + targetConceptCount int +) ON COMMIT PRESERVE ROWS; + + INSERT INTO w3m16jd0concept_map_distinct (source_concept_id, domain_id, targetConceptCount) + SELECT source_concept_id, domain_id, COUNT(*) + FROM w3m16jd0concept_map GROUP BY source_concept_id, domain_id; BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE [CDMV5].[dbo].ETL_WARNINGS'; - EXECUTE IMMEDIATE 'DROP TABLE [CDMV5].[dbo].ETL_WARNINGS'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS'; + EXECUTE IMMEDIATE 'DROP TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -370,9 +343,9 @@ EXCEPTION END IF; END; -CREATE TABLE [CDMV5].[dbo].ETL_WARNINGS +CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS ( - WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(4000) ); /**** @@ -381,9 +354,9 @@ CDM_SOURCE ****/ -INSERT INTO [CDMV5].[dbo].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -SELECT '[CDMV5]', 'V5', v.vocabulary_version, SYSDATE - FROM [CDMV5].[dbo].vocabulary v +INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +SELECT '[TARGET_CDMV5]', 'V5', v.vocabulary_version, SYSDATE + FROM [TARGET_CDMV5].[SCHEMA].vocabulary v WHERE vocabulary_id = 'Vocabulary'; /**** @@ -392,9 +365,9 @@ LOCATION ****/ -insert into [CDMV5].[dbo].location +insert into [TARGET_CDMV5].[SCHEMA].location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from [CDMV4].[dbo].LOCATION; +from [SOURCE_CDMV4].[SCHEMA].LOCATION; /**** @@ -402,9 +375,9 @@ CARE_SITE ****/ -insert into [CDMV5].[dbo].care_site +insert into [TARGET_CDMV5].[SCHEMA].care_site select care_site_id, TO_CHAR(null ) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from [CDMV4].[dbo].CARE_SITE; +from [SOURCE_CDMV4].[SCHEMA].CARE_SITE; /**** @@ -412,11 +385,11 @@ Provider ****/ -insert into [CDMV5].[dbo].provider +insert into [TARGET_CDMV5].[SCHEMA].provider select provider_id, TO_CHAR(null ) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, TO_CHAR(null ) as gender_source_value, cast(null as integer) as gender_source_concept_id -from [CDMV4].[dbo].provider +from [SOURCE_CDMV4].[SCHEMA].provider ; @@ -426,7 +399,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT into [CDMV5].[dbo].person + INSERT into [TARGET_CDMV5].[SCHEMA].person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -446,21 +419,21 @@ from [CDMV4].[dbo].provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM [CDMV4].[dbo].PERSON p + FROM [SOURCE_CDMV4].[SCHEMA].PERSON p left JOIN w3m16jd0concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN w3m16jd0concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN w3m16jd0concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -471,16 +444,16 @@ from [CDMV4].[dbo].provider ) warn ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -491,16 +464,16 @@ from [CDMV4].[dbo].provider ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -516,9 +489,9 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].observation_period + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM [CDMV4].[dbo].OBSERVATION_PERIOD; + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; /**** @@ -526,28 +499,28 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].death + INSERT INTO [TARGET_CDMV5].[SCHEMA].death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH LEFT JOIN w3m16jd0concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -564,7 +537,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].visit_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, TO_CHAR(null ) visit_start_time, @@ -573,20 +546,20 @@ from [CDMV4].[dbo].provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE LEFT JOIN w3m16jd0concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -605,14 +578,14 @@ from [CDMV4].[dbo].provider -- *************************************************************************** -- AGS: Modifying this section to insert this information into the temp - -- table #procedure_occurrence_map but this may need to be revisited for + -- table #po_map but this may need to be revisited for -- performance tuning on APS as a large temp table may cause processing -- time issues. -- *************************************************************************** BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0procedure_occurrence_map'; - EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0procedure_occurrence_map'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0po_map'; + EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0po_map'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -620,12 +593,41 @@ EXCEPTION END IF; END; +CREATE GLOBAL TEMPORARY TABLE w3m16jd0po_map + ( + procedure_occurrence_id int, + person_id int, + procedure_concept_id int, + procedure_date date, + procedure_type_concept_id int, + modifier_concept_id int, + quantity int, + provider_id int, + visit_occurrence_id int, + procedure_source_value varchar(50), + procedure_source_concept_id int, + qualifier_source_value varchar(50), + origional_drug_id NUMBER(19) +) ON COMMIT PRESERVE ROWS; + --find valid procedures from procedure table - CREATE GLOBAL TEMPORARY TABLE w3m16jd0procedure_occurrence_map - ON COMMIT PRESERVE ROWS -AS -SELECT - +INSERT INTO w3m16jd0po_map +( + procedure_occurrence_id, + person_id, + procedure_concept_id, + procedure_date, + procedure_type_concept_id, + modifier_concept_id, + quantity, + provider_id, + visit_occurrence_id, + procedure_source_value, + procedure_source_concept_id, + qualifier_source_value, + origional_drug_id +) + SELECT procedure_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as procedure_concept_id, @@ -639,17 +641,17 @@ SELECT CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id - -FROM - [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence = 1 + AND LOWER(cm1.domain_id) IN ('procedure') + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN w3m16jd0concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_map_occurrence = 1 + LEFT JOIN w3m16jd0concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 UNION ALL @@ -668,12 +670,12 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE FROM DUAL FROM DUAL -WHERE procedure_concept_id = 0 +WHERE procedure_concept_id = 0 UNION ALL @@ -692,7 +694,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE LEFT JOIN w3m16jd0concept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN w3m16jd0concept_map cm2 @@ -735,11 +737,12 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN w3m16jd0concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -761,7 +764,7 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -782,7 +785,7 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -798,16 +801,16 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO [CDMV5].[dbo].procedure_occurrence +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence (procedure_occurrence_id ,person_id ,procedure_concept_id @@ -833,19 +836,19 @@ SELECT ,procedure_source_value ,procedure_source_concept_id ,qualifier_source_value -FROM w3m16jd0procedure_occurrence_map; +FROM w3m16jd0po_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -855,15 +858,15 @@ FROM w3m16jd0procedure_occurrence_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -882,8 +885,8 @@ FROM w3m16jd0procedure_occurrence_map; --find valid drugs from drug_exposure table BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0drug_exposure_map'; - EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0drug_exposure_map'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0drgexp_map'; + EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0drgexp_map'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -891,11 +894,58 @@ EXCEPTION END IF; END; - CREATE GLOBAL TEMPORARY TABLE w3m16jd0drug_exposure_map - ON COMMIT PRESERVE ROWS -AS -SELECT - drug_exposure_id, +CREATE GLOBAL TEMPORARY TABLE w3m16jd0drgexp_map + ( + drug_exposure_id int, + person_id int, + drug_concept_id int, + drug_exposure_start_date date, + drug_exposure_end_date date, + drug_type_concept_id int, + stop_reason varchar(20), + refills int, + quantity float, + days_supply int, + sig CLOB, + route_concept_id int, + effective_drug_dose float, + dose_unit_concept_id int, + lot_number varchar(50), + provider_id int, + visit_occurrence_id int, + drug_source_value varchar(50), + drug_source_concept_id int, + route_source_value varchar(50), + dose_unit_source_value varchar(50), + origional_procedure_id int +) ON COMMIT PRESERVE ROWS; + +INSERT INTO w3m16jd0drgexp_map +( + drug_exposure_id, + person_id, + drug_concept_id, + drug_exposure_start_date, + drug_exposure_end_date, + drug_type_concept_id, + stop_reason, + refills, + quantity, + days_supply, + sig, + route_concept_id, + effective_drug_dose, + dose_unit_concept_id, + lot_number, + provider_id, + visit_occurrence_id, + drug_source_value, + drug_source_concept_id, + route_source_value, + dose_unit_source_value, + origional_procedure_id +) + SELECT drug_exposure_id, person_id, COALESCE(cm1.target_concept_id,0) as drug_concept_id, drug_exposure_start_date, @@ -917,17 +967,17 @@ SELECT TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id - -FROM - [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN w3m16jd0concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_map_occurrence = 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -955,7 +1005,7 @@ UNION ALL TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -983,7 +1033,7 @@ UNION ALL TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE LEFT JOIN w3m16jd0concept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN w3m16jd0concept_map cm2 @@ -1025,11 +1075,12 @@ FROM TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN w3m16jd0concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -1044,7 +1095,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1059,7 +1110,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1073,13 +1124,13 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [CDMV4].[dbo].DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE) MAXROW ; -INSERT INTO [CDMV5].[dbo].drug_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure (drug_exposure_id ,person_id ,drug_concept_id @@ -1123,19 +1174,19 @@ SELECT ,drug_source_concept_id ,route_source_value ,dose_unit_source_value -FROM w3m16jd0drug_exposure_map; +FROM w3m16jd0drgexp_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1145,15 +1196,15 @@ FROM w3m16jd0drug_exposure_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1171,7 +1222,7 @@ FROM w3m16jd0drug_exposure_map; ****/ --find valid conditions from condition_occurrence table - INSERT INTO [CDMV5].[dbo].condition_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1180,11 +1231,12 @@ FROM w3m16jd0drug_exposure_map; COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN w3m16jd0concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1204,7 +1256,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1221,7 +1273,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE LEFT JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN w3m16jd0concept_map cm2 @@ -1247,11 +1299,12 @@ FROM ( stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN w3m16jd0concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1266,7 +1319,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1279,7 +1332,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1292,25 +1345,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [CDMV4].[dbo].CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1320,15 +1373,15 @@ FROM ( ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1347,7 +1400,7 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].device_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1361,7 +1414,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1375,7 +1428,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1389,7 +1442,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1403,7 +1456,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1418,7 +1471,7 @@ FROM ****/ --find valid measurements from observation table -INSERT INTO [CDMV5].[dbo].measurement +INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1443,7 +1496,7 @@ from unit_source_value AS unit_source_value, TO_CHAR(null ) as value_source_value, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1471,7 +1524,7 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1496,7 +1549,7 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1521,7 +1574,7 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].drug_exposure + FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1540,7 +1593,7 @@ from --find valid observation from observation table - INSERT INTO [CDMV5].[dbo].observation + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation SELECT observation_id, person_id, @@ -1559,7 +1612,7 @@ from CAST(null as integer) observation_source_concept_id, unit_source_value, TO_CHAR(null ) qualifier_source_value - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM w3m16jd0concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) @@ -1594,7 +1647,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as observation_source_value, @@ -1602,7 +1655,7 @@ FROM null as unit_source_value, TO_CHAR(null ) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1618,7 +1671,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value as observation_source_value, @@ -1626,7 +1679,7 @@ FROM null as unit_source_value, TO_CHAR(null ) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1650,11 +1703,11 @@ FROM null as unit_source_value, TO_CHAR(null ) qualifier_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [CDMV4].[dbo].OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION) MAXROW ; @@ -1665,10 +1718,10 @@ FROM ****/ -INSERT INTO [CDMV5].[dbo].payer_plan_period +INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; +FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; /**** @@ -1678,15 +1731,15 @@ FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; ****/ -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM [CDMV4].[dbo].DRUG_COST dc +FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, drug_exposure_id, @@ -1718,11 +1771,11 @@ FROM ( CAST(null as float) as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE po - join [CDMV4].[dbo].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po + join [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - JOIN w3m16jd0drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN w3m16jd0drgexp_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1734,15 +1787,15 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM [CDMV4].[dbo].PROCEDURE_COST; +FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, procedure_occurrence_id, @@ -1776,11 +1829,11 @@ SELECT CAST(null as integer) as revenue_code_concept_id, CAST(null as integer) as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE de - join [CDMV4].[dbo].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de + join [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - JOIN w3m16jd0procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN w3m16jd0po_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1791,16 +1844,16 @@ Note: Eras derived from DRUG_EXPOSURE table, using 30d gap ****/ -- drop table dbo.drug_era -INSERT INTO [CDMV5].[dbo].drug_era +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_era WITH cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, DRUG_EXPOSURE_END_DATE, INGREDIENT_CONCEPT_ID) AS ( -- Normalize DRUG_EXPOSURE_END_DATE to either the existing drug exposure end date, or add days supply, or add 1 day to the start date select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, (DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), (DRUG_EXPOSURE_START_DATE + 1)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM [CDMV5].[dbo].DRUG_EXPOSURE d - join [CDMV5].[dbo].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join [CDMV5].[dbo].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d + join [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join [TARGET_CDMV5].[SCHEMA].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1880,7 +1933,7 @@ WITH cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, (CONDITION_START_DATE + 1)) as CONDITION_END_DATE - FROM [CDMV5].[dbo].CONDITION_OCCURRENCE co + FROM [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1935,7 +1988,7 @@ FROM GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO [CDMV5].[dbo].condition_era +INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era (condition_era_id ,person_id ,condition_concept_id diff --git a/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql b/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql index dc02cf5..156123f 100644 --- a/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql +++ b/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql @@ -19,7 +19,7 @@ PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. -last revised: 01 July 2015 +last revised: 09 July 2015 author: Patrick Ryan, Chris Knoll editor: Anthony Sena @@ -34,7 +34,7 @@ instructions will be slightly different. General Assumptions ------------------- -This script assumes that your V4 and V5 database either located on the same +This script assumes that your V4 and V5 database are located on the same RDBMS server. Getting Started @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - ([CDMV4], [CDMV4].[dbo]) + ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) 2. Set parameter name of schema that contains CDMv5 instance - ([CDMV5], [CDMV5].[dbo]) + ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -76,37 +76,13 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS --- -- The CDMv4 database name --- -- The CDMv4 database plus schema --- -- The target CDMv5 database name --- -- the target CDMv5 database plus schema -*/ -/* QA Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - */ -/* PostgreSQL Settings +/* SCRIPT PARAMETERS */ -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema -*/ -/* Oracle Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema -*/ -/* LOCAL SQL Server */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -SET search_path TO [CDMV5]; + +SET search_path TO [TARGET_CDMV5]; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -138,17 +114,15 @@ CREATE TEMP TABLE concept_map ( source_concept_id int, target_concept_id int, - domain_id varchar(20), - source_concept_map_occurrence int + domain_id varchar(20) ); --standard concepts -INSERT INTO concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) +INSERT INTO concept_map (source_concept_id, target_concept_id, domain_id) select concept_id as source_concept_id, concept_id as target_concept_id, - domain_id, - 0 as source_concept_map_occurrence -from [CDMV5].[dbo].concept + domain_id +from [TARGET_CDMV5].[SCHEMA].concept where standard_concept = 'S' and invalid_reason is null @@ -158,20 +132,19 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( select concept_id -from [CDMV5].[dbo].concept +from [TARGET_CDMV5].[SCHEMA].concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -[CDMV5].[dbo].concept_relationship cr1 +[TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -[CDMV5].[dbo].concept c2 +[TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -185,17 +158,16 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -204,8 +176,8 @@ from OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Maps to') @@ -220,8 +192,8 @@ from ) AND B.concept_id IS NULL ) c1 -inner join [CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 -inner join [CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id +inner join [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 +inner join [TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null and cr1.relationship_id in ( @@ -243,16 +215,15 @@ union SELECT DISTINCT c1.concept_id AS source_concept_id ,c2.concept_id AS target_concept_id ,c2.domain_id - ,0 FROM ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -261,8 +232,8 @@ FROM ( OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ( @@ -288,8 +259,8 @@ FROM ( ) AND B.concept_id IS NULL ) c1 -INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Is a') @@ -302,53 +273,55 @@ WHERE c2.standard_concept = 'S' -- for the target tables when applicable /* UPDATE #concept_map - SET #concept_map.source_concept_map_occurrence = A.CountOfRows + SET #concept_map.source_concept_map_occurrence = A.targetConceptCount FROM #concept_map, ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) AS A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; UPDATE #concept_map -SET source_concept_map_occurrence = "CountOfRows" +SET source_concept_map_occurrence = "targetConceptCount" FROM ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; -*/ -UPDATE concept_map +UPDATE #concept_map SET source_concept_map_occurrence = ( - select count(*) as "CountOfRows" - from concept_map A - WHERE A.source_concept_id = concept_map.source_concept_id AND A.domain_id = concept_map.domain_id + select count(*) as "targetConceptCount" + from #concept_map A + WHERE A.source_concept_id = #concept_map.source_concept_id AND A.domain_id = #concept_map.domain_id group by source_concept_id, domain_id ) +*/ DROP TABLE IF EXISTS concept_map_distinct; - CREATE TEMP TABLE concept_map_distinct - -AS -SELECT - DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" - -FROM - concept_map +CREATE TEMP TABLE concept_map_distinct + ( + source_concept_id int, + domain_id varchar(20), + targetConceptCount int +); + + INSERT INTO concept_map_distinct (source_concept_id, domain_id, targetConceptCount) + SELECT source_concept_id, domain_id, COUNT(*) + FROM concept_map GROUP BY source_concept_id, domain_id; -DROP TABLE IF EXISTS [CDMV5].[dbo].ETL_WARNINGS; +DROP TABLE IF EXISTS [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS; -CREATE TABLE [CDMV5].[dbo].ETL_WARNINGS +CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS ( - WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(4000) ); /**** @@ -357,9 +330,9 @@ CDM_SOURCE ****/ -INSERT INTO [CDMV5].[dbo].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -select '[CDMV5]', 'V5', v.vocabulary_version, CURRENT_DATE -from [CDMV5].[dbo].vocabulary v +INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +select '[TARGET_CDMV5]', 'V5', v.vocabulary_version, CURRENT_DATE +from [TARGET_CDMV5].[SCHEMA].vocabulary v where vocabulary_id = 'Vocabulary'; /**** @@ -368,9 +341,9 @@ LOCATION ****/ -insert into [CDMV5].[dbo].location +insert into [TARGET_CDMV5].[SCHEMA].location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from [CDMV4].[dbo].LOCATION; +from [SOURCE_CDMV4].[SCHEMA].LOCATION; /**** @@ -378,9 +351,9 @@ CARE_SITE ****/ -insert into [CDMV5].[dbo].care_site +insert into [TARGET_CDMV5].[SCHEMA].care_site select care_site_id, cast(null as varchar(255)) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from [CDMV4].[dbo].CARE_SITE; +from [SOURCE_CDMV4].[SCHEMA].CARE_SITE; /**** @@ -388,11 +361,11 @@ Provider ****/ -insert into [CDMV5].[dbo].provider +insert into [TARGET_CDMV5].[SCHEMA].provider select provider_id, cast(null as varchar(255)) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, cast(null as varchar(50)) as gender_source_value, cast(null as integer) as gender_source_concept_id -from [CDMV4].[dbo].provider +from [SOURCE_CDMV4].[SCHEMA].provider ; @@ -402,7 +375,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT into [CDMV5].[dbo].person + INSERT into [TARGET_CDMV5].[SCHEMA].person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -422,21 +395,21 @@ from [CDMV4].[dbo].provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM [CDMV4].[dbo].PERSON p + FROM [SOURCE_CDMV4].[SCHEMA].PERSON p left JOIN concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -447,16 +420,16 @@ from [CDMV4].[dbo].provider ) warn ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -467,16 +440,16 @@ from [CDMV4].[dbo].provider ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -492,9 +465,9 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].observation_period + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM [CDMV4].[dbo].OBSERVATION_PERIOD; + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; /**** @@ -502,28 +475,28 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].death + INSERT INTO [TARGET_CDMV5].[SCHEMA].death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH LEFT JOIN concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -540,7 +513,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].visit_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, CAST(null as varchar(10)) visit_start_time, @@ -549,20 +522,20 @@ from [CDMV4].[dbo].provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE LEFT JOIN concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -581,19 +554,48 @@ from [CDMV4].[dbo].provider -- *************************************************************************** -- AGS: Modifying this section to insert this information into the temp - -- table #procedure_occurrence_map but this may need to be revisited for + -- table #po_map but this may need to be revisited for -- performance tuning on APS as a large temp table may cause processing -- time issues. -- *************************************************************************** - DROP TABLE IF EXISTS procedure_occurrence_map; + DROP TABLE IF EXISTS po_map; + +CREATE TEMP TABLE po_map + ( + procedure_occurrence_id int, + person_id int, + procedure_concept_id int, + procedure_date date, + procedure_type_concept_id int, + modifier_concept_id int, + quantity int, + provider_id int, + visit_occurrence_id int, + procedure_source_value varchar(50), + procedure_source_concept_id int, + qualifier_source_value varchar(50), + origional_drug_id bigint +); --find valid procedures from procedure table - CREATE TEMP TABLE procedure_occurrence_map - -AS -SELECT - +INSERT INTO po_map +( + procedure_occurrence_id, + person_id, + procedure_concept_id, + procedure_date, + procedure_type_concept_id, + modifier_concept_id, + quantity, + provider_id, + visit_occurrence_id, + procedure_source_value, + procedure_source_concept_id, + qualifier_source_value, + origional_drug_id +) + SELECT procedure_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as procedure_concept_id, @@ -607,17 +609,17 @@ SELECT CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - -FROM - [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence = 1 + AND LOWER(cm1.domain_id) IN ('procedure') + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_map_occurrence = 1 + LEFT JOIN concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 UNION ALL @@ -636,8 +638,8 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE - WHERE procedure_concept_id = 0 + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + WHERE procedure_concept_id = 0 UNION ALL @@ -656,7 +658,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE LEFT JOIN concept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -699,11 +701,12 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -725,7 +728,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -746,7 +749,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -762,16 +765,16 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO [CDMV5].[dbo].procedure_occurrence +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence (procedure_occurrence_id ,person_id ,procedure_concept_id @@ -797,19 +800,19 @@ SELECT ,procedure_source_value ,procedure_source_concept_id ,qualifier_source_value -FROM procedure_occurrence_map; +FROM po_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -819,15 +822,15 @@ FROM procedure_occurrence_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -845,13 +848,60 @@ FROM procedure_occurrence_map; ****/ --find valid drugs from drug_exposure table -DROP TABLE IF EXISTS drug_exposure_map; +DROP TABLE IF EXISTS drgexp_map; - CREATE TEMP TABLE drug_exposure_map - -AS -SELECT - drug_exposure_id, +CREATE TEMP TABLE drgexp_map + ( + drug_exposure_id int, + person_id int, + drug_concept_id int, + drug_exposure_start_date date, + drug_exposure_end_date date, + drug_type_concept_id int, + stop_reason varchar(20), + refills int, + quantity NUMERIC, + days_supply int, + sig TEXT, + route_concept_id int, + effective_drug_dose NUMERIC, + dose_unit_concept_id int, + lot_number varchar(50), + provider_id int, + visit_occurrence_id int, + drug_source_value varchar(50), + drug_source_concept_id int, + route_source_value varchar(50), + dose_unit_source_value varchar(50), + origional_procedure_id int +); + +INSERT INTO drgexp_map +( + drug_exposure_id, + person_id, + drug_concept_id, + drug_exposure_start_date, + drug_exposure_end_date, + drug_type_concept_id, + stop_reason, + refills, + quantity, + days_supply, + sig, + route_concept_id, + effective_drug_dose, + dose_unit_concept_id, + lot_number, + provider_id, + visit_occurrence_id, + drug_source_value, + drug_source_concept_id, + route_source_value, + dose_unit_source_value, + origional_procedure_id +) + SELECT drug_exposure_id, person_id, COALESCE(cm1.target_concept_id,0) as drug_concept_id, drug_exposure_start_date, @@ -873,17 +923,17 @@ SELECT CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - -FROM - [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_map_occurrence = 1 + INNER JOIN concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -911,7 +961,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -939,7 +989,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE LEFT JOIN concept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -981,11 +1031,12 @@ FROM CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -1000,7 +1051,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1015,7 +1066,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1029,13 +1080,13 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [CDMV4].[dbo].DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE) MAXROW ; -INSERT INTO [CDMV5].[dbo].drug_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure (drug_exposure_id ,person_id ,drug_concept_id @@ -1079,19 +1130,19 @@ SELECT ,drug_source_concept_id ,route_source_value ,dose_unit_source_value -FROM drug_exposure_map; +FROM drgexp_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1101,15 +1152,15 @@ FROM drug_exposure_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1127,7 +1178,7 @@ FROM drug_exposure_map; ****/ --find valid conditions from condition_occurrence table - INSERT INTO [CDMV5].[dbo].condition_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1136,11 +1187,12 @@ FROM drug_exposure_map; COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1160,7 +1212,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1177,7 +1229,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE LEFT JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -1203,11 +1255,12 @@ FROM ( stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1222,7 +1275,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1235,7 +1288,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1248,25 +1301,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [CDMV4].[dbo].CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1276,15 +1329,15 @@ FROM ( ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1303,7 +1356,7 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].device_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1317,7 +1370,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1331,7 +1384,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1345,7 +1398,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1359,7 +1412,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1374,7 +1427,7 @@ FROM ****/ --find valid measurements from observation table -INSERT INTO [CDMV5].[dbo].measurement +INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1399,7 +1452,7 @@ from unit_source_value AS unit_source_value, cast(null as varchar(50)) as value_source_value, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1427,7 +1480,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1452,7 +1505,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1477,7 +1530,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].drug_exposure + FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1496,7 +1549,7 @@ from --find valid observation from observation table - INSERT INTO [CDMV5].[dbo].observation + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation SELECT observation_id, person_id, @@ -1515,7 +1568,7 @@ from CAST(null as integer) observation_source_concept_id, unit_source_value, cast(null as varchar(50)) qualifier_source_value - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) @@ -1550,7 +1603,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as observation_source_value, @@ -1558,7 +1611,7 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1574,7 +1627,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value as observation_source_value, @@ -1582,7 +1635,7 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1606,11 +1659,11 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [CDMV4].[dbo].OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION) MAXROW ; @@ -1621,10 +1674,10 @@ FROM ****/ -INSERT INTO [CDMV5].[dbo].payer_plan_period +INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; +FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; /**** @@ -1634,15 +1687,15 @@ FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; ****/ -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM [CDMV4].[dbo].DRUG_COST dc +FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, drug_exposure_id, @@ -1674,11 +1727,11 @@ FROM ( CAST(null as NUMERIC) as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE po - join [CDMV4].[dbo].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po + join [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - JOIN drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN drgexp_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1690,15 +1743,15 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM [CDMV4].[dbo].PROCEDURE_COST; +FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, procedure_occurrence_id, @@ -1732,11 +1785,11 @@ SELECT CAST(null as integer) as revenue_code_concept_id, CAST(null as integer) as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE de - join [CDMV4].[dbo].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de + join [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - JOIN procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN po_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1753,9 +1806,9 @@ with cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONC select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, (DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), (DRUG_EXPOSURE_START_DATE + 1)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM [CDMV5].[dbo].DRUG_EXPOSURE d - join [CDMV5].[dbo].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join [CDMV5].[dbo].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d + join [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join [TARGET_CDMV5].[SCHEMA].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1805,7 +1858,7 @@ GROUP BY d.PERSON_ID, d.DRUG_TYPE_CONCEPT_ID, d.DRUG_EXPOSURE_START_DATE ) -insert into [CDMV5].[dbo].drug_era +insert into [TARGET_CDMV5].[SCHEMA].drug_era select row_number() over (order by person_id) as drug_era_id, person_id, drug_concept_id, min(DRUG_EXPOSURE_START_DATE) as drug_era_start_date, drug_era_end_date, COUNT(*) as DRUG_EXPOSURE_COUNT, 30 as gap_days from cteDrugExposureEnds GROUP BY person_id, drug_concept_id, drug_type_concept_id, DRUG_ERA_END_DATE @@ -1828,7 +1881,7 @@ WITH cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, (CONDITION_START_DATE + 1)) as CONDITION_END_DATE - FROM [CDMV5].[dbo].CONDITION_OCCURRENCE co + FROM [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1883,7 +1936,7 @@ FROM GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO [CDMV5].[dbo].condition_era +INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era (condition_era_id ,person_id ,condition_concept_id diff --git a/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql b/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql index 41f4232..ca655a0 100644 --- a/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql +++ b/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql @@ -19,7 +19,7 @@ PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. -last revised: 01 July 2015 +last revised: 09 July 2015 author: Patrick Ryan, Chris Knoll editor: Anthony Sena @@ -34,7 +34,7 @@ instructions will be slightly different. General Assumptions ------------------- -This script assumes that your V4 and V5 database either located on the same +This script assumes that your V4 and V5 database are located on the same RDBMS server. Getting Started @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - ([CDMV4], [CDMV4].[dbo]) + ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) 2. Set parameter name of schema that contains CDMv5 instance - ([CDMV5], [CDMV5].[dbo]) + ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -76,37 +76,13 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS --- -- The CDMv4 database name --- -- The CDMv4 database plus schema --- -- The target CDMv5 database name --- -- the target CDMv5 database plus schema -*/ -/* QA Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - */ -/* PostgreSQL Settings +/* SCRIPT PARAMETERS */ -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema -*/ -/* Oracle Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema -*/ -/* LOCAL SQL Server */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -USE [CDMV5]; + +USE [TARGET_CDMV5]; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -139,17 +115,15 @@ CREATE TABLE #concept_map ( source_concept_id int, target_concept_id int, - domain_id varchar(20), - source_concept_map_occurrence int + domain_id varchar(20) ); --standard concepts -INSERT INTO #concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) +INSERT INTO #concept_map (source_concept_id, target_concept_id, domain_id) select concept_id as source_concept_id, concept_id as target_concept_id, - domain_id, - 0 as source_concept_map_occurrence -from [CDMV5].[dbo].concept + domain_id +from [TARGET_CDMV5].[SCHEMA].concept where standard_concept = 'S' and invalid_reason is null @@ -159,20 +133,19 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( select concept_id -from [CDMV5].[dbo].concept +from [TARGET_CDMV5].[SCHEMA].concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -[CDMV5].[dbo].concept_relationship cr1 +[TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -[CDMV5].[dbo].concept c2 +[TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -186,17 +159,16 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -205,8 +177,8 @@ from OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Maps to') @@ -221,8 +193,8 @@ from ) AND B.concept_id IS NULL ) c1 -inner join [CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 -inner join [CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id +inner join [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 +inner join [TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null and cr1.relationship_id in ( @@ -244,16 +216,15 @@ union SELECT DISTINCT c1.concept_id AS source_concept_id ,c2.concept_id AS target_concept_id ,c2.domain_id - ,0 FROM ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -262,8 +233,8 @@ FROM ( OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ( @@ -289,8 +260,8 @@ FROM ( ) AND B.concept_id IS NULL ) c1 -INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Is a') @@ -303,50 +274,57 @@ WHERE c2.standard_concept = 'S' -- for the target tables when applicable /* UPDATE #concept_map - SET #concept_map.source_concept_map_occurrence = A.CountOfRows + SET #concept_map.source_concept_map_occurrence = A.targetConceptCount FROM #concept_map, ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) AS A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; UPDATE #concept_map -SET source_concept_map_occurrence = "CountOfRows" +SET source_concept_map_occurrence = "targetConceptCount" FROM ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; -*/ UPDATE #concept_map SET source_concept_map_occurrence = ( - select count(*) as "CountOfRows" + select count(*) as "targetConceptCount" from #concept_map A WHERE A.source_concept_id = #concept_map.source_concept_id AND A.domain_id = #concept_map.domain_id group by source_concept_id, domain_id ) +*/ IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL DROP TABLE #concept_map_distinct; - SELECT DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" - INTO #concept_map_distinct +CREATE TABLE #concept_map_distinct +( + source_concept_id int, + domain_id varchar(20), + targetConceptCount int +); + + INSERT INTO #concept_map_distinct (source_concept_id, domain_id, targetConceptCount) + SELECT source_concept_id, domain_id, COUNT(*) FROM #concept_map GROUP BY source_concept_id, domain_id; -IF OBJECT_ID('[CDMV5].[dbo].ETL_WARNINGS', 'U') IS NOT NULL - DROP TABLE [CDMV5].[dbo].ETL_WARNINGS; +IF OBJECT_ID('[TARGET_CDMV5].[SCHEMA].ETL_WARNINGS', 'U') IS NOT NULL + DROP TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS; -CREATE TABLE [CDMV5].[dbo].ETL_WARNINGS +CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS ( - WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(4000) ); /**** @@ -355,9 +333,9 @@ CDM_SOURCE ****/ -INSERT INTO [CDMV5].[dbo].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -select '[CDMV5]', 'V5', v.vocabulary_version, getDate() -from [CDMV5].[dbo].vocabulary v +INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +select '[TARGET_CDMV5]', 'V5', v.vocabulary_version, getDate() +from [TARGET_CDMV5].[SCHEMA].vocabulary v where vocabulary_id = 'Vocabulary'; /**** @@ -366,9 +344,9 @@ LOCATION ****/ -insert into [CDMV5].[dbo].location +insert into [TARGET_CDMV5].[SCHEMA].location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from [CDMV4].[dbo].LOCATION; +from [SOURCE_CDMV4].[SCHEMA].LOCATION; /**** @@ -376,9 +354,9 @@ CARE_SITE ****/ -insert into [CDMV5].[dbo].care_site +insert into [TARGET_CDMV5].[SCHEMA].care_site select care_site_id, cast(null as varchar(255)) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from [CDMV4].[dbo].CARE_SITE; +from [SOURCE_CDMV4].[SCHEMA].CARE_SITE; /**** @@ -386,11 +364,11 @@ Provider ****/ -insert into [CDMV5].[dbo].provider +insert into [TARGET_CDMV5].[SCHEMA].provider select provider_id, cast(null as varchar(255)) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, cast(null as varchar(50)) as gender_source_value, cast(null as integer) as gender_source_concept_id -from [CDMV4].[dbo].provider +from [SOURCE_CDMV4].[SCHEMA].provider ; @@ -400,7 +378,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT into [CDMV5].[dbo].person + INSERT into [TARGET_CDMV5].[SCHEMA].person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -420,21 +398,21 @@ from [CDMV4].[dbo].provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM [CDMV4].[dbo].PERSON p + FROM [SOURCE_CDMV4].[SCHEMA].PERSON p left JOIN #concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN #concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN #concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -445,16 +423,16 @@ from [CDMV4].[dbo].provider ) warn ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -465,16 +443,16 @@ from [CDMV4].[dbo].provider ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -490,9 +468,9 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].observation_period + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM [CDMV4].[dbo].OBSERVATION_PERIOD; + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; /**** @@ -500,28 +478,28 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].death + INSERT INTO [TARGET_CDMV5].[SCHEMA].death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH LEFT JOIN #concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -538,7 +516,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].visit_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, CAST(null as varchar(10)) visit_start_time, @@ -547,20 +525,20 @@ from [CDMV4].[dbo].provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE LEFT JOIN #concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -579,15 +557,48 @@ from [CDMV4].[dbo].provider -- *************************************************************************** -- AGS: Modifying this section to insert this information into the temp - -- table #procedure_occurrence_map but this may need to be revisited for + -- table #po_map but this may need to be revisited for -- performance tuning on APS as a large temp table may cause processing -- time issues. -- *************************************************************************** - IF OBJECT_ID('tempdb..#procedure_occurrence_map', 'U') IS NOT NULL - DROP TABLE #procedure_occurrence_map; + IF OBJECT_ID('tempdb..#po_map', 'U') IS NOT NULL + DROP TABLE #po_map; + +CREATE TABLE #po_map +( + procedure_occurrence_id int, + person_id int, + procedure_concept_id int, + procedure_date date, + procedure_type_concept_id int, + modifier_concept_id int, + quantity int, + provider_id int, + visit_occurrence_id int, + procedure_source_value varchar(50), + procedure_source_concept_id int, + qualifier_source_value varchar(50), + origional_drug_id bigint +); --find valid procedures from procedure table +INSERT INTO #po_map +( + procedure_occurrence_id, + person_id, + procedure_concept_id, + procedure_date, + procedure_type_concept_id, + modifier_concept_id, + quantity, + provider_id, + visit_occurrence_id, + procedure_source_value, + procedure_source_concept_id, + qualifier_source_value, + origional_drug_id +) SELECT procedure_occurrence_id, person_id, @@ -602,16 +613,17 @@ from [CDMV4].[dbo].provider CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - INTO #procedure_occurrence_map - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence = 1 + AND LOWER(cm1.domain_id) IN ('procedure') + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_map_occurrence = 1 + LEFT JOIN #concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 UNION ALL @@ -630,8 +642,8 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE - WHERE procedure_concept_id = 0 + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + WHERE procedure_concept_id = 0 UNION ALL @@ -650,7 +662,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE LEFT JOIN #concept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -693,11 +705,12 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -719,7 +732,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -740,7 +753,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -756,16 +769,16 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO [CDMV5].[dbo].procedure_occurrence +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence (procedure_occurrence_id ,person_id ,procedure_concept_id @@ -791,19 +804,19 @@ SELECT ,procedure_source_value ,procedure_source_concept_id ,qualifier_source_value -FROM #procedure_occurrence_map; +FROM #po_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -813,15 +826,15 @@ FROM #procedure_occurrence_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -839,9 +852,60 @@ FROM #procedure_occurrence_map; ****/ --find valid drugs from drug_exposure table -IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL - DROP TABLE #drug_exposure_map; +IF OBJECT_ID('tempdb..#drgexp_map', 'U') IS NOT NULL + DROP TABLE #drgexp_map; +CREATE TABLE #drgexp_map +( + drug_exposure_id int, + person_id int, + drug_concept_id int, + drug_exposure_start_date date, + drug_exposure_end_date date, + drug_type_concept_id int, + stop_reason varchar(20), + refills int, + quantity float, + days_supply int, + sig varchar(max), + route_concept_id int, + effective_drug_dose float, + dose_unit_concept_id int, + lot_number varchar(50), + provider_id int, + visit_occurrence_id int, + drug_source_value varchar(50), + drug_source_concept_id int, + route_source_value varchar(50), + dose_unit_source_value varchar(50), + origional_procedure_id int +); + +INSERT INTO #drgexp_map +( + drug_exposure_id, + person_id, + drug_concept_id, + drug_exposure_start_date, + drug_exposure_end_date, + drug_type_concept_id, + stop_reason, + refills, + quantity, + days_supply, + sig, + route_concept_id, + effective_drug_dose, + dose_unit_concept_id, + lot_number, + provider_id, + visit_occurrence_id, + drug_source_value, + drug_source_concept_id, + route_source_value, + dose_unit_source_value, + origional_procedure_id +) SELECT drug_exposure_id, person_id, COALESCE(cm1.target_concept_id,0) as drug_concept_id, @@ -864,16 +928,17 @@ IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - INTO #drug_exposure_map - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -901,7 +966,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -929,7 +994,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE LEFT JOIN #concept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -971,11 +1036,12 @@ FROM CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -990,7 +1056,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1005,7 +1071,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1019,13 +1085,13 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [CDMV4].[dbo].DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE) MAXROW ; -INSERT INTO [CDMV5].[dbo].drug_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure (drug_exposure_id ,person_id ,drug_concept_id @@ -1069,19 +1135,19 @@ SELECT ,drug_source_concept_id ,route_source_value ,dose_unit_source_value -FROM #drug_exposure_map; +FROM #drgexp_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1091,15 +1157,15 @@ FROM #drug_exposure_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1117,7 +1183,7 @@ FROM #drug_exposure_map; ****/ --find valid conditions from condition_occurrence table - INSERT INTO [CDMV5].[dbo].condition_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1126,11 +1192,12 @@ FROM #drug_exposure_map; COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1150,7 +1217,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1167,7 +1234,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE LEFT JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -1193,11 +1260,12 @@ FROM ( stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1212,7 +1280,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1225,7 +1293,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1238,25 +1306,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [CDMV4].[dbo].CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1266,15 +1334,15 @@ FROM ( ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1293,7 +1361,7 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].device_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1307,7 +1375,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1321,7 +1389,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1335,7 +1403,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1349,7 +1417,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1364,7 +1432,7 @@ FROM ****/ --find valid measurements from observation table -INSERT INTO [CDMV5].[dbo].measurement +INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1389,7 +1457,7 @@ from unit_source_value AS unit_source_value, cast(null as varchar(50)) as value_source_value, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1417,7 +1485,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1442,7 +1510,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1467,7 +1535,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].drug_exposure + FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1486,7 +1554,7 @@ from --find valid observation from observation table - INSERT INTO [CDMV5].[dbo].observation + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation SELECT observation_id, person_id, @@ -1505,7 +1573,7 @@ from CAST(null as integer) observation_source_concept_id, unit_source_value, cast(null as varchar(50)) qualifier_source_value - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM #concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) @@ -1540,7 +1608,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as observation_source_value, @@ -1548,7 +1616,7 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1564,7 +1632,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value as observation_source_value, @@ -1572,7 +1640,7 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1596,11 +1664,11 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [CDMV4].[dbo].OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION) MAXROW ; @@ -1611,10 +1679,10 @@ FROM ****/ -INSERT INTO [CDMV5].[dbo].payer_plan_period +INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; +FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; /**** @@ -1624,15 +1692,15 @@ FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; ****/ -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM [CDMV4].[dbo].DRUG_COST dc +FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, drug_exposure_id, @@ -1664,11 +1732,11 @@ FROM ( CAST(null as float) as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE po - join [CDMV4].[dbo].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po + join [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - JOIN #drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN #drgexp_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1680,15 +1748,15 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM [CDMV4].[dbo].PROCEDURE_COST; +FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, procedure_occurrence_id, @@ -1722,11 +1790,11 @@ SELECT CAST(null as integer) as revenue_code_concept_id, CAST(null as integer) as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE de - join [CDMV4].[dbo].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de + join [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - JOIN #procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN #po_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1743,9 +1811,9 @@ with cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONC select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, DATEADD(day,DAYS_SUPPLY,DRUG_EXPOSURE_START_DATE), DATEADD(day,1,DRUG_EXPOSURE_START_DATE)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM [CDMV5].[dbo].DRUG_EXPOSURE d - join [CDMV5].[dbo].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join [CDMV5].[dbo].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d + join [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join [TARGET_CDMV5].[SCHEMA].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1795,7 +1863,7 @@ GROUP BY d.PERSON_ID, d.DRUG_TYPE_CONCEPT_ID, d.DRUG_EXPOSURE_START_DATE ) -insert into [CDMV5].[dbo].drug_era +insert into [TARGET_CDMV5].[SCHEMA].drug_era select row_number() over (order by person_id) as drug_era_id, person_id, drug_concept_id, min(DRUG_EXPOSURE_START_DATE) as drug_era_start_date, drug_era_end_date, COUNT(*) as DRUG_EXPOSURE_COUNT, 30 as gap_days from cteDrugExposureEnds GROUP BY person_id, drug_concept_id, drug_type_concept_id, DRUG_ERA_END_DATE @@ -1816,7 +1884,7 @@ with cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, DATEADD(day,1,CONDITION_START_DATE)) as CONDITION_END_DATE - FROM [CDMV5].[dbo].CONDITION_OCCURRENCE co + FROM [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1869,7 +1937,7 @@ from cteConditionEnds GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO [CDMV5].[dbo].condition_era +INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era (condition_era_id ,person_id ,condition_concept_id From 3af8f2e76b535d6805b88b9416d4c5f8778f6303 Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Thu, 9 Jul 2015 09:53:40 -0400 Subject: [PATCH 4/5] README.md updates --- ...-QA.sql => Conversion-QA - Sql Server.sql} | 0 ... => Conversion-QA-Part-2 - Sql Server.sql} | 0 Version4 To Version5 Conversion/README.md | 35 +++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) rename Version4 To Version5 Conversion/{Conversion-QA.sql => Conversion-QA - Sql Server.sql} (100%) rename Version4 To Version5 Conversion/{Conversion-QA-Part-2.sql => Conversion-QA-Part-2 - Sql Server.sql} (100%) diff --git a/Version4 To Version5 Conversion/Conversion-QA.sql b/Version4 To Version5 Conversion/Conversion-QA - Sql Server.sql similarity index 100% rename from Version4 To Version5 Conversion/Conversion-QA.sql rename to Version4 To Version5 Conversion/Conversion-QA - Sql Server.sql diff --git a/Version4 To Version5 Conversion/Conversion-QA-Part-2.sql b/Version4 To Version5 Conversion/Conversion-QA-Part-2 - Sql Server.sql similarity index 100% rename from Version4 To Version5 Conversion/Conversion-QA-Part-2.sql rename to Version4 To Version5 Conversion/Conversion-QA-Part-2 - Sql Server.sql diff --git a/Version4 To Version5 Conversion/README.md b/Version4 To Version5 Conversion/README.md index 2bfedbd..2e1ea74 100644 --- a/Version4 To Version5 Conversion/README.md +++ b/Version4 To Version5 Conversion/README.md @@ -1,4 +1,33 @@ -Common-Data-Model - Conversion from CDM v4 to CDM v5 -==================================================== +Conversion from CDM v4 to CDM v5 +============================================================== -Scrips for converting data from the Common Data Model V4 to V5. +The scripts in this directory will aid you in moving your data from the Common Data Model (CDM) version 4 to version 5. + +Usage & Assumptions +============================================================== + +We have created a directory per RDBMS that contains the conversion script for that database platform. All of the script assume the following: + +1. Your source CDM V4 database is on the same sever as your target CDM v5 database. +2. You have read rights to the CDM V4 database and database owner privileges on the target V5 database as this script will create an "ETL_WARNINGS" table in the process. + +** **NOTE** ** If you are running the Oracle script via Sql Developer or similar, you may need to alter the script to include the appropriate "/" symbols to mark the end of the anonymous code blocks. + +Quality Assurance +=================== + +We have included 2 scripts in the root of this directory that were used while doing quality assurance on the conversion scripts: + +* Conversion-QA - Sql Server.sql +* Conversion-QA-Part-2 - Sql Server.sql + +As noted in the file names, these scripts were written specifically for Sql Server but should be a fairly easy port to your RDBMS target. The goals of these scripts were to measure the following: + +* **Conversion-QA - Sql Server.sql**: provides row counts from each table in the V4 and V5 databases. It also includes a column called "Migration Target" which notes if that table was a target of the migration. The full list will help you to see if there were any tables in V4 that were either missed or are not targeted as part of the migration. Of particular note: **Cohort** and **Source\_To\_Concept\_Map** are not targeted for this migration. + +* **Conversion-QA-Part-2 - Sql Server.sql**: provides 2 summary tables to help verify the output from the first script. The first summary table provides row counts for specific V4 tables and how the rows in the tables map to the V5 domains. This summary is useful to understand why the row counts for these tables will vary between the V4 and V5. The second summary table provides a row count sum by domain which should then match the V5 row counts for the corresponding V5 tables. The tables that are summarized in this script are: condition\_occurrence, drug\_exposure, observation, procedure\_occurrence. + +Contributions +============================================================== + +Each script found in the RDBMS directory was generated from the template SQL file: *OMOP CDMv4 to CDMv5 - templateSQL.sql* found in the root of this directory. If you would like to contribute to this script, we'd suggest you modify this script and use **[SqlRender](https://github.com/OHDSI/SqlRender "SqlRender")** to re-generate the specific RDBMS scripts. From 04fd25a73dcf43d881ebb98fb54c33f877be99d7 Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Thu, 9 Jul 2015 10:06:20 -0400 Subject: [PATCH 5/5] One more edit to the README.md --- Version4 To Version5 Conversion/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Version4 To Version5 Conversion/README.md b/Version4 To Version5 Conversion/README.md index 2e1ea74..bf78ecf 100644 --- a/Version4 To Version5 Conversion/README.md +++ b/Version4 To Version5 Conversion/README.md @@ -3,7 +3,7 @@ Conversion from CDM v4 to CDM v5 The scripts in this directory will aid you in moving your data from the Common Data Model (CDM) version 4 to version 5. -Usage & Assumptions +Assumptions ============================================================== We have created a directory per RDBMS that contains the conversion script for that database platform. All of the script assume the following: @@ -11,6 +11,18 @@ We have created a directory per RDBMS that contains the conversion script for th 1. Your source CDM V4 database is on the same sever as your target CDM v5 database. 2. You have read rights to the CDM V4 database and database owner privileges on the target V5 database as this script will create an "ETL_WARNINGS" table in the process. +Usage +===== + + 1. The conversion script will hold a number of placeholders for your CDM V4 and CDMV5 database/schema. In order to make this file work in your environment, you should plan to do a global "FIND AND REPLACE" on the conversion script to fill in the file with values that pertain to your environment. The following are the tokens you should use when doing your "FIND AND REPLACE" operation: + + * [SOURCE_CDMV4] + * [SOURCE_CDMV4].[SCHEMA] + * [TARGET_CDMV5] + * [TARGET_CDMV5].[SCHEMA] + +2. Run the resulting script on your target RDBDMS. + ** **NOTE** ** If you are running the Oracle script via Sql Developer or similar, you may need to alter the script to include the appropriate "/" symbols to mark the end of the anonymous code blocks. Quality Assurance