From c3e7bb7eeae4ef92997dcd484335fdb44a73308a Mon Sep 17 00:00:00 2001 From: Anthony Sena Date: Thu, 9 Jul 2015 08:47:18 -0400 Subject: [PATCH] Final commit before push --- .../OMOP CDMv4 to CDMv5 - templateSQL.sql | 212 +++++--- .../Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql | 501 ++++++++++-------- .../OMOP CDMv4 to CDMv5 - PostgreSQL.sql | 495 +++++++++-------- .../OMOP CDMv4 to CDMv5 - SQL Server.sql | 474 ++++++++++------- 4 files changed, 962 insertions(+), 720 deletions(-) diff --git a/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql b/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql index 80d83e5..347f4df 100644 --- a/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql +++ b/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - templateSQL.sql @@ -19,7 +19,7 @@ PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. -last revised: 01 July 2015 +last revised: 09 July 2015 author: Patrick Ryan, Chris Knoll editor: Anthony Sena @@ -34,7 +34,7 @@ instructions will be slightly different. General Assumptions ------------------- -This script assumes that your V4 and V5 database either located on the same +This script assumes that your V4 and V5 database are located on the same RDBMS server. Getting Started @@ -76,36 +76,12 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS ---{DEFAULT @SOURCE_CDMV4 = '[SOURCE_CDMV4]'} -- The CDMv4 database name ---{DEFAULT @SOURCE_CDMV4_SCHEMA = '[SOURCE_CDMV4].[SCHEMA]'} -- The CDMv4 database plus schema ---{DEFAULT @TARGET_CDMV5 = '[TARGET_CDMV5]'} -- The target CDMv5 database name ---{DEFAULT @TARGET_CDMV5_SCHEMA = '[TARGET_CDMV5].[SCHEMA]'} -- the target CDMv5 database plus schema -*/ -/* QA Settings -{DEFAULT @SOURCE_CDMV4 = 'CDM_TRUVEN_CCAE_6k'} -- The CDMv4 database name -{DEFAULT @SOURCE_CDMV4_SCHEMA = 'CDM_TRUVEN_CCAE_6k.dbo'} -- The CDMv4 database plus schema -{DEFAULT @TARGET_CDMV5 = 'CDMV5_Conversion_Target'} -- The target CDMv5 database name -{DEFAULT @TARGET_CDMV5_SCHEMA = 'CDMV5_Conversion_Target.dbo'} -- the target CDMv5 database plus schema - */ -/* PostgreSQL Settings -{DEFAULT @SOURCE_CDMV4 = 'sandbox'} -- The CDMv4 database name -{DEFAULT @SOURCE_CDMV4_SCHEMA = 'sandbox.cdmv4'} -- The CDMv4 database plus schema -{DEFAULT @TARGET_CDMV5 = 'sandbox'} -- The target CDMv5 database name -{DEFAULT @TARGET_CDMV5_SCHEMA = 'sandbox.cdmv5'} -- the target CDMv5 database plus schema -*/ -/* Oracle Settings -{DEFAULT @SOURCE_CDMV4 = 'CDMV4'} -- The CDMv4 database name -{DEFAULT @SOURCE_CDMV4_SCHEMA = 'CDMV4'} -- The CDMv4 database plus schema -{DEFAULT @TARGET_CDMV5 = 'CDMV5'} -- The target CDMv5 database name -{DEFAULT @TARGET_CDMV5_SCHEMA = 'CDMV5'} -- the target CDMv5 database plus schema -*/ -/* LOCAL SQL Server */ -{DEFAULT @SOURCE_CDMV4 = '[CDMV4]'} -- The CDMv4 database name -{DEFAULT @SOURCE_CDMV4_SCHEMA = '[CDMV4].[dbo]'} -- The CDMv4 database plus schema -{DEFAULT @TARGET_CDMV5 = '[CDMV5]'} -- The target CDMv5 database name -{DEFAULT @TARGET_CDMV5_SCHEMA = '[CDMV5].[dbo]'} -- the target CDMv5 database plus schema - +/* SCRIPT PARAMETERS */ +{DEFAULT @SOURCE_CDMV4 = '[SOURCE_CDMV4]'} -- The CDMv4 database name +{DEFAULT @SOURCE_CDMV4_SCHEMA = '[SOURCE_CDMV4].[SCHEMA]'} -- The CDMv4 database plus schema +{DEFAULT @TARGET_CDMV5 = '[TARGET_CDMV5]'} -- The target CDMv5 database name +{DEFAULT @TARGET_CDMV5_SCHEMA = '[TARGET_CDMV5].[SCHEMA]'} -- the target CDMv5 database plus schema + USE @TARGET_CDMV5; /* @@ -139,16 +115,14 @@ CREATE TABLE #concept_map ( source_concept_id int, target_concept_id int, - domain_id varchar(20), - source_concept_map_occurrence int + domain_id varchar(20) ); --standard concepts -INSERT INTO #concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) +INSERT INTO #concept_map (source_concept_id, target_concept_id, domain_id) select concept_id as source_concept_id, concept_id as target_concept_id, - domain_id, - 0 as source_concept_map_occurrence + domain_id from @TARGET_CDMV5_SCHEMA.concept where standard_concept = 'S' and invalid_reason is null @@ -159,7 +133,6 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( select concept_id @@ -186,7 +159,6 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( SELECT A.concept_id @@ -244,7 +216,6 @@ union SELECT DISTINCT c1.concept_id AS source_concept_id ,c2.concept_id AS target_concept_id ,c2.domain_id - ,0 FROM ( SELECT A.concept_id FROM @TARGET_CDMV5_SCHEMA.concept A @@ -303,40 +274,47 @@ WHERE c2.standard_concept = 'S' -- for the target tables when applicable /* UPDATE #concept_map - SET #concept_map.source_concept_map_occurrence = A.CountOfRows + SET #concept_map.source_concept_map_occurrence = A.targetConceptCount FROM #concept_map, ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) AS A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; UPDATE #concept_map -SET source_concept_map_occurrence = "CountOfRows" +SET source_concept_map_occurrence = "targetConceptCount" FROM ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; -*/ UPDATE #concept_map SET source_concept_map_occurrence = ( - select count(*) as "CountOfRows" + select count(*) as "targetConceptCount" from #concept_map A WHERE A.source_concept_id = #concept_map.source_concept_id AND A.domain_id = #concept_map.domain_id group by source_concept_id, domain_id ) +*/ IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL DROP TABLE #concept_map_distinct; - SELECT DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" - INTO #concept_map_distinct +CREATE TABLE #concept_map_distinct +( + source_concept_id int, + domain_id varchar(20), + targetConceptCount int +); + + INSERT INTO #concept_map_distinct (source_concept_id, domain_id, targetConceptCount) + SELECT source_concept_id, domain_id, COUNT(*) FROM #concept_map GROUP BY source_concept_id, domain_id; @@ -346,7 +324,7 @@ IF OBJECT_ID('@TARGET_CDMV5_SCHEMA.ETL_WARNINGS', 'U') IS NOT NULL CREATE TABLE @TARGET_CDMV5_SCHEMA.ETL_WARNINGS ( - WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(4000) ); /**** @@ -579,15 +557,48 @@ from @SOURCE_CDMV4_SCHEMA.provider -- *************************************************************************** -- AGS: Modifying this section to insert this information into the temp - -- table #procedure_occurrence_map but this may need to be revisited for + -- table #po_map but this may need to be revisited for -- performance tuning on APS as a large temp table may cause processing -- time issues. -- *************************************************************************** - IF OBJECT_ID('tempdb..#procedure_occurrence_map', 'U') IS NOT NULL - DROP TABLE #procedure_occurrence_map; + IF OBJECT_ID('tempdb..#po_map', 'U') IS NOT NULL + DROP TABLE #po_map; + +CREATE TABLE #po_map +( + procedure_occurrence_id int, + person_id int, + procedure_concept_id int, + procedure_date date, + procedure_type_concept_id int, + modifier_concept_id int, + quantity int, + provider_id int, + visit_occurrence_id int, + procedure_source_value varchar(50), + procedure_source_concept_id int, + qualifier_source_value varchar(50), + origional_drug_id bigint +); --find valid procedures from procedure table +INSERT INTO #po_map +( + procedure_occurrence_id, + person_id, + procedure_concept_id, + procedure_date, + procedure_type_concept_id, + modifier_concept_id, + quantity, + provider_id, + visit_occurrence_id, + procedure_source_value, + procedure_source_concept_id, + qualifier_source_value, + origional_drug_id +) SELECT procedure_occurrence_id, person_id, @@ -602,16 +613,17 @@ from @SOURCE_CDMV4_SCHEMA.provider CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - INTO #procedure_occurrence_map FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence = 1 + AND LOWER(cm1.domain_id) IN ('procedure') + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_map_occurrence = 1 + LEFT JOIN #concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 UNION ALL @@ -631,7 +643,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - WHERE procedure_concept_id = 0 + WHERE procedure_concept_id = 0 UNION ALL @@ -697,7 +709,8 @@ UNION ALL INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -791,7 +804,7 @@ SELECT ,procedure_source_value ,procedure_source_concept_id ,qualifier_source_value -FROM #procedure_occurrence_map; +FROM #po_map; --warnings of invalid records @@ -839,9 +852,60 @@ FROM #procedure_occurrence_map; ****/ --find valid drugs from drug_exposure table -IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL - DROP TABLE #drug_exposure_map; +IF OBJECT_ID('tempdb..#drgexp_map', 'U') IS NOT NULL + DROP TABLE #drgexp_map; +CREATE TABLE #drgexp_map +( + drug_exposure_id int, + person_id int, + drug_concept_id int, + drug_exposure_start_date date, + drug_exposure_end_date date, + drug_type_concept_id int, + stop_reason varchar(20), + refills int, + quantity float, + days_supply int, + sig varchar(max), + route_concept_id int, + effective_drug_dose float, + dose_unit_concept_id int, + lot_number varchar(50), + provider_id int, + visit_occurrence_id int, + drug_source_value varchar(50), + drug_source_concept_id int, + route_source_value varchar(50), + dose_unit_source_value varchar(50), + origional_procedure_id int +); + +INSERT INTO #drgexp_map +( + drug_exposure_id, + person_id, + drug_concept_id, + drug_exposure_start_date, + drug_exposure_end_date, + drug_type_concept_id, + stop_reason, + refills, + quantity, + days_supply, + sig, + route_concept_id, + effective_drug_dose, + dose_unit_concept_id, + lot_number, + provider_id, + visit_occurrence_id, + drug_source_value, + drug_source_concept_id, + route_source_value, + dose_unit_source_value, + origional_procedure_id +) SELECT drug_exposure_id, person_id, COALESCE(cm1.target_concept_id,0) as drug_concept_id, @@ -864,16 +928,17 @@ IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - INTO #drug_exposure_map FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -975,7 +1040,8 @@ FROM INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -1069,7 +1135,7 @@ SELECT ,drug_source_concept_id ,route_source_value ,dose_unit_source_value -FROM #drug_exposure_map; +FROM #drgexp_map; --warnings of invalid records @@ -1130,7 +1196,8 @@ FROM #drug_exposure_map; INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1197,7 +1264,8 @@ FROM ( INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1540,7 +1608,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as observation_source_value, @@ -1564,7 +1632,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value as observation_source_value, @@ -1667,7 +1735,7 @@ FROM ( FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE po join @SOURCE_CDMV4_SCHEMA.PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - JOIN #drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id + JOIN #drgexp_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id ) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM @SOURCE_CDMV4_SCHEMA.DRUG_COST) MAXROW ; @@ -1725,7 +1793,7 @@ SELECT FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE de join @SOURCE_CDMV4_SCHEMA.DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - JOIN #procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id + JOIN #po_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM @SOURCE_CDMV4_SCHEMA.DRUG_COST) MAXROW ; diff --git a/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql b/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql index 8cae4c9..b485f41 100644 --- a/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql +++ b/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql @@ -19,7 +19,7 @@ PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. -last revised: 01 July 2015 +last revised: 09 July 2015 author: Patrick Ryan, Chris Knoll editor: Anthony Sena @@ -34,7 +34,7 @@ instructions will be slightly different. General Assumptions ------------------- -This script assumes that your V4 and V5 database either located on the same +This script assumes that your V4 and V5 database are located on the same RDBMS server. Getting Started @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - ([CDMV4], [CDMV4].[dbo]) + ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) 2. Set parameter name of schema that contains CDMv5 instance - ([CDMV5], [CDMV5].[dbo]) + ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -76,37 +76,13 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS --- -- The CDMv4 database name --- -- The CDMv4 database plus schema --- -- The target CDMv5 database name --- -- the target CDMv5 database plus schema -*/ -/* QA Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - */ -/* PostgreSQL Settings +/* SCRIPT PARAMETERS */ -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema -*/ -/* Oracle Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema -*/ -/* LOCAL SQL Server */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -ALTER SESSION SET current_schema = [CDMV5]; + +ALTER SESSION SET current_schema = [TARGET_CDMV5]; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -146,17 +122,15 @@ CREATE GLOBAL TEMPORARY TABLE w3m16jd0concept_map ( source_concept_id int, target_concept_id int, - domain_id varchar(20), - source_concept_map_occurrence int + domain_id varchar(20) ) ON COMMIT PRESERVE ROWS; --standard concepts -INSERT INTO w3m16jd0concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) +INSERT INTO w3m16jd0concept_map (source_concept_id, target_concept_id, domain_id) SELECT concept_id as source_concept_id, concept_id as target_concept_id, - domain_id, - 0 as source_concept_map_occurrence - FROM [CDMV5].[dbo].concept + domain_id + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE standard_concept = 'S' and invalid_reason is null @@ -166,20 +140,19 @@ union SELECT distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 FROM ( select concept_id -from [CDMV5].[dbo].concept +from [TARGET_CDMV5].[SCHEMA].concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -[CDMV5].[dbo].concept_relationship cr1 +[TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -[CDMV5].[dbo].concept c2 +[TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' and c2.invalid_reason is null @@ -193,17 +166,16 @@ union SELECT distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 FROM ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -212,8 +184,8 @@ SELECT distinct c1.concept_id as source_concept_id, OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Maps to') @@ -228,8 +200,8 @@ SELECT distinct c1.concept_id as source_concept_id, ) AND B.concept_id IS NULL ) c1 -inner join [CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 -inner join [CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id +inner join [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 +inner join [TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' and c2.invalid_reason is null and cr1.relationship_id in ( @@ -251,16 +223,15 @@ union SELECT DISTINCT c1.concept_id AS source_concept_id ,c2.concept_id AS target_concept_id ,c2.domain_id - ,0 FROM ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -269,8 +240,8 @@ SELECT DISTINCT c1.concept_id AS source_concept_id OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ( @@ -296,8 +267,8 @@ SELECT DISTINCT c1.concept_id AS source_concept_id ) AND B.concept_id IS NULL ) c1 -INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Is a') @@ -310,34 +281,34 @@ INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id -- for the target tables when applicable /* UPDATE #concept_map - SET #concept_map.source_concept_map_occurrence = A.CountOfRows + SET #concept_map.source_concept_map_occurrence = A.targetConceptCount FROM #concept_map, ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) AS A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; UPDATE #concept_map -SET source_concept_map_occurrence = "CountOfRows" +SET source_concept_map_occurrence = "targetConceptCount" FROM ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; -*/ -UPDATE w3m16jd0concept_map +UPDATE #concept_map SET source_concept_map_occurrence = ( - select count(*) as "CountOfRows" - from w3m16jd0concept_map A - WHERE A.source_concept_id = w3m16jd0concept_map.source_concept_id AND A.domain_id = w3m16jd0concept_map.domain_id + select count(*) as "targetConceptCount" + from #concept_map A + WHERE A.source_concept_id = #concept_map.source_concept_id AND A.domain_id = #concept_map.domain_id group by source_concept_id, domain_id ) +*/ BEGIN EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0concept_map_distinct'; @@ -349,20 +320,22 @@ EXCEPTION END IF; END; - CREATE GLOBAL TEMPORARY TABLE w3m16jd0concept_map_distinct - ON COMMIT PRESERVE ROWS -AS -SELECT - DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" - -FROM - w3m16jd0concept_map +CREATE GLOBAL TEMPORARY TABLE w3m16jd0concept_map_distinct + ( + source_concept_id int, + domain_id varchar(20), + targetConceptCount int +) ON COMMIT PRESERVE ROWS; + + INSERT INTO w3m16jd0concept_map_distinct (source_concept_id, domain_id, targetConceptCount) + SELECT source_concept_id, domain_id, COUNT(*) + FROM w3m16jd0concept_map GROUP BY source_concept_id, domain_id; BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE [CDMV5].[dbo].ETL_WARNINGS'; - EXECUTE IMMEDIATE 'DROP TABLE [CDMV5].[dbo].ETL_WARNINGS'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS'; + EXECUTE IMMEDIATE 'DROP TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -370,9 +343,9 @@ EXCEPTION END IF; END; -CREATE TABLE [CDMV5].[dbo].ETL_WARNINGS +CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS ( - WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(4000) ); /**** @@ -381,9 +354,9 @@ CDM_SOURCE ****/ -INSERT INTO [CDMV5].[dbo].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -SELECT '[CDMV5]', 'V5', v.vocabulary_version, SYSDATE - FROM [CDMV5].[dbo].vocabulary v +INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +SELECT '[TARGET_CDMV5]', 'V5', v.vocabulary_version, SYSDATE + FROM [TARGET_CDMV5].[SCHEMA].vocabulary v WHERE vocabulary_id = 'Vocabulary'; /**** @@ -392,9 +365,9 @@ LOCATION ****/ -insert into [CDMV5].[dbo].location +insert into [TARGET_CDMV5].[SCHEMA].location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from [CDMV4].[dbo].LOCATION; +from [SOURCE_CDMV4].[SCHEMA].LOCATION; /**** @@ -402,9 +375,9 @@ CARE_SITE ****/ -insert into [CDMV5].[dbo].care_site +insert into [TARGET_CDMV5].[SCHEMA].care_site select care_site_id, TO_CHAR(null ) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from [CDMV4].[dbo].CARE_SITE; +from [SOURCE_CDMV4].[SCHEMA].CARE_SITE; /**** @@ -412,11 +385,11 @@ Provider ****/ -insert into [CDMV5].[dbo].provider +insert into [TARGET_CDMV5].[SCHEMA].provider select provider_id, TO_CHAR(null ) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, TO_CHAR(null ) as gender_source_value, cast(null as integer) as gender_source_concept_id -from [CDMV4].[dbo].provider +from [SOURCE_CDMV4].[SCHEMA].provider ; @@ -426,7 +399,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT into [CDMV5].[dbo].person + INSERT into [TARGET_CDMV5].[SCHEMA].person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -446,21 +419,21 @@ from [CDMV4].[dbo].provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM [CDMV4].[dbo].PERSON p + FROM [SOURCE_CDMV4].[SCHEMA].PERSON p left JOIN w3m16jd0concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN w3m16jd0concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN w3m16jd0concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -471,16 +444,16 @@ from [CDMV4].[dbo].provider ) warn ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -491,16 +464,16 @@ from [CDMV4].[dbo].provider ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -516,9 +489,9 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].observation_period + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM [CDMV4].[dbo].OBSERVATION_PERIOD; + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; /**** @@ -526,28 +499,28 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].death + INSERT INTO [TARGET_CDMV5].[SCHEMA].death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH LEFT JOIN w3m16jd0concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -564,7 +537,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].visit_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, TO_CHAR(null ) visit_start_time, @@ -573,20 +546,20 @@ from [CDMV4].[dbo].provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE LEFT JOIN w3m16jd0concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -605,14 +578,14 @@ from [CDMV4].[dbo].provider -- *************************************************************************** -- AGS: Modifying this section to insert this information into the temp - -- table #procedure_occurrence_map but this may need to be revisited for + -- table #po_map but this may need to be revisited for -- performance tuning on APS as a large temp table may cause processing -- time issues. -- *************************************************************************** BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0procedure_occurrence_map'; - EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0procedure_occurrence_map'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0po_map'; + EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0po_map'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -620,12 +593,41 @@ EXCEPTION END IF; END; +CREATE GLOBAL TEMPORARY TABLE w3m16jd0po_map + ( + procedure_occurrence_id int, + person_id int, + procedure_concept_id int, + procedure_date date, + procedure_type_concept_id int, + modifier_concept_id int, + quantity int, + provider_id int, + visit_occurrence_id int, + procedure_source_value varchar(50), + procedure_source_concept_id int, + qualifier_source_value varchar(50), + origional_drug_id NUMBER(19) +) ON COMMIT PRESERVE ROWS; + --find valid procedures from procedure table - CREATE GLOBAL TEMPORARY TABLE w3m16jd0procedure_occurrence_map - ON COMMIT PRESERVE ROWS -AS -SELECT - +INSERT INTO w3m16jd0po_map +( + procedure_occurrence_id, + person_id, + procedure_concept_id, + procedure_date, + procedure_type_concept_id, + modifier_concept_id, + quantity, + provider_id, + visit_occurrence_id, + procedure_source_value, + procedure_source_concept_id, + qualifier_source_value, + origional_drug_id +) + SELECT procedure_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as procedure_concept_id, @@ -639,17 +641,17 @@ SELECT CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id - -FROM - [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence = 1 + AND LOWER(cm1.domain_id) IN ('procedure') + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN w3m16jd0concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_map_occurrence = 1 + LEFT JOIN w3m16jd0concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 UNION ALL @@ -668,12 +670,12 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE FROM DUAL FROM DUAL -WHERE procedure_concept_id = 0 +WHERE procedure_concept_id = 0 UNION ALL @@ -692,7 +694,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE LEFT JOIN w3m16jd0concept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN w3m16jd0concept_map cm2 @@ -735,11 +737,12 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN w3m16jd0concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -761,7 +764,7 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -782,7 +785,7 @@ UNION ALL TO_CHAR(null ) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -798,16 +801,16 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, TO_CHAR(null ) qualifier_source_value, CAST(null as NUMBER(19)) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO [CDMV5].[dbo].procedure_occurrence +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence (procedure_occurrence_id ,person_id ,procedure_concept_id @@ -833,19 +836,19 @@ SELECT ,procedure_source_value ,procedure_source_concept_id ,qualifier_source_value -FROM w3m16jd0procedure_occurrence_map; +FROM w3m16jd0po_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -855,15 +858,15 @@ FROM w3m16jd0procedure_occurrence_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -882,8 +885,8 @@ FROM w3m16jd0procedure_occurrence_map; --find valid drugs from drug_exposure table BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0drug_exposure_map'; - EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0drug_exposure_map'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE w3m16jd0drgexp_map'; + EXECUTE IMMEDIATE 'DROP TABLE w3m16jd0drgexp_map'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN @@ -891,11 +894,58 @@ EXCEPTION END IF; END; - CREATE GLOBAL TEMPORARY TABLE w3m16jd0drug_exposure_map - ON COMMIT PRESERVE ROWS -AS -SELECT - drug_exposure_id, +CREATE GLOBAL TEMPORARY TABLE w3m16jd0drgexp_map + ( + drug_exposure_id int, + person_id int, + drug_concept_id int, + drug_exposure_start_date date, + drug_exposure_end_date date, + drug_type_concept_id int, + stop_reason varchar(20), + refills int, + quantity float, + days_supply int, + sig CLOB, + route_concept_id int, + effective_drug_dose float, + dose_unit_concept_id int, + lot_number varchar(50), + provider_id int, + visit_occurrence_id int, + drug_source_value varchar(50), + drug_source_concept_id int, + route_source_value varchar(50), + dose_unit_source_value varchar(50), + origional_procedure_id int +) ON COMMIT PRESERVE ROWS; + +INSERT INTO w3m16jd0drgexp_map +( + drug_exposure_id, + person_id, + drug_concept_id, + drug_exposure_start_date, + drug_exposure_end_date, + drug_type_concept_id, + stop_reason, + refills, + quantity, + days_supply, + sig, + route_concept_id, + effective_drug_dose, + dose_unit_concept_id, + lot_number, + provider_id, + visit_occurrence_id, + drug_source_value, + drug_source_concept_id, + route_source_value, + dose_unit_source_value, + origional_procedure_id +) + SELECT drug_exposure_id, person_id, COALESCE(cm1.target_concept_id,0) as drug_concept_id, drug_exposure_start_date, @@ -917,17 +967,17 @@ SELECT TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id - -FROM - [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN w3m16jd0concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_map_occurrence = 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -955,7 +1005,7 @@ UNION ALL TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -983,7 +1033,7 @@ UNION ALL TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE LEFT JOIN w3m16jd0concept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN w3m16jd0concept_map cm2 @@ -1025,11 +1075,12 @@ FROM TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN w3m16jd0concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -1044,7 +1095,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1059,7 +1110,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1073,13 +1124,13 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, TO_CHAR(null ) route_source_value, TO_CHAR(null ) dose_unit_source_value, CAST(null as NUMBER(19)) origional_procedure_id, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [CDMV4].[dbo].DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE) MAXROW ; -INSERT INTO [CDMV5].[dbo].drug_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure (drug_exposure_id ,person_id ,drug_concept_id @@ -1123,19 +1174,19 @@ SELECT ,drug_source_concept_id ,route_source_value ,dose_unit_source_value -FROM w3m16jd0drug_exposure_map; +FROM w3m16jd0drgexp_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1145,15 +1196,15 @@ FROM w3m16jd0drug_exposure_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1171,7 +1222,7 @@ FROM w3m16jd0drug_exposure_map; ****/ --find valid conditions from condition_occurrence table - INSERT INTO [CDMV5].[dbo].condition_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1180,11 +1231,12 @@ FROM w3m16jd0drug_exposure_map; COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN w3m16jd0concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1204,7 +1256,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1221,7 +1273,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE LEFT JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN w3m16jd0concept_map cm2 @@ -1247,11 +1299,12 @@ FROM ( stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN w3m16jd0concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN w3m16jd0concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1266,7 +1319,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1279,7 +1332,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1292,25 +1345,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [CDMV4].[dbo].CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1320,15 +1373,15 @@ FROM ( ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1347,7 +1400,7 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].device_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1361,7 +1414,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1375,7 +1428,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1389,7 +1442,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1403,7 +1456,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1418,7 +1471,7 @@ FROM ****/ --find valid measurements from observation table -INSERT INTO [CDMV5].[dbo].measurement +INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1443,7 +1496,7 @@ from unit_source_value AS unit_source_value, TO_CHAR(null ) as value_source_value, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN w3m16jd0concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1471,7 +1524,7 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1496,7 +1549,7 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1521,7 +1574,7 @@ from TO_CHAR(null ) as unit_source_value, TO_CHAR(null ) as value_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].drug_exposure + FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1540,7 +1593,7 @@ from --find valid observation from observation table - INSERT INTO [CDMV5].[dbo].observation + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation SELECT observation_id, person_id, @@ -1559,7 +1612,7 @@ from CAST(null as integer) observation_source_concept_id, unit_source_value, TO_CHAR(null ) qualifier_source_value - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM w3m16jd0concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) @@ -1594,7 +1647,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as observation_source_value, @@ -1602,7 +1655,7 @@ FROM null as unit_source_value, TO_CHAR(null ) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1618,7 +1671,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value as observation_source_value, @@ -1626,7 +1679,7 @@ FROM null as unit_source_value, TO_CHAR(null ) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN w3m16jd0concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1650,11 +1703,11 @@ FROM null as unit_source_value, TO_CHAR(null ) qualifier_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN w3m16jd0concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [CDMV4].[dbo].OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION) MAXROW ; @@ -1665,10 +1718,10 @@ FROM ****/ -INSERT INTO [CDMV5].[dbo].payer_plan_period +INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; +FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; /**** @@ -1678,15 +1731,15 @@ FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; ****/ -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM [CDMV4].[dbo].DRUG_COST dc +FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, drug_exposure_id, @@ -1718,11 +1771,11 @@ FROM ( CAST(null as float) as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE po - join [CDMV4].[dbo].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po + join [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - JOIN w3m16jd0drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN w3m16jd0drgexp_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1734,15 +1787,15 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM [CDMV4].[dbo].PROCEDURE_COST; +FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, procedure_occurrence_id, @@ -1776,11 +1829,11 @@ SELECT CAST(null as integer) as revenue_code_concept_id, CAST(null as integer) as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE de - join [CDMV4].[dbo].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de + join [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - JOIN w3m16jd0procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN w3m16jd0po_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1791,16 +1844,16 @@ Note: Eras derived from DRUG_EXPOSURE table, using 30d gap ****/ -- drop table dbo.drug_era -INSERT INTO [CDMV5].[dbo].drug_era +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_era WITH cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, DRUG_EXPOSURE_END_DATE, INGREDIENT_CONCEPT_ID) AS ( -- Normalize DRUG_EXPOSURE_END_DATE to either the existing drug exposure end date, or add days supply, or add 1 day to the start date select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, (DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), (DRUG_EXPOSURE_START_DATE + 1)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM [CDMV5].[dbo].DRUG_EXPOSURE d - join [CDMV5].[dbo].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join [CDMV5].[dbo].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d + join [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join [TARGET_CDMV5].[SCHEMA].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1880,7 +1933,7 @@ WITH cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, (CONDITION_START_DATE + 1)) as CONDITION_END_DATE - FROM [CDMV5].[dbo].CONDITION_OCCURRENCE co + FROM [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1935,7 +1988,7 @@ FROM GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO [CDMV5].[dbo].condition_era +INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era (condition_era_id ,person_id ,condition_concept_id diff --git a/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql b/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql index dc02cf5..156123f 100644 --- a/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql +++ b/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql @@ -19,7 +19,7 @@ PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. -last revised: 01 July 2015 +last revised: 09 July 2015 author: Patrick Ryan, Chris Knoll editor: Anthony Sena @@ -34,7 +34,7 @@ instructions will be slightly different. General Assumptions ------------------- -This script assumes that your V4 and V5 database either located on the same +This script assumes that your V4 and V5 database are located on the same RDBMS server. Getting Started @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - ([CDMV4], [CDMV4].[dbo]) + ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) 2. Set parameter name of schema that contains CDMv5 instance - ([CDMV5], [CDMV5].[dbo]) + ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -76,37 +76,13 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS --- -- The CDMv4 database name --- -- The CDMv4 database plus schema --- -- The target CDMv5 database name --- -- the target CDMv5 database plus schema -*/ -/* QA Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - */ -/* PostgreSQL Settings +/* SCRIPT PARAMETERS */ -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema -*/ -/* Oracle Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema -*/ -/* LOCAL SQL Server */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -SET search_path TO [CDMV5]; + +SET search_path TO [TARGET_CDMV5]; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -138,17 +114,15 @@ CREATE TEMP TABLE concept_map ( source_concept_id int, target_concept_id int, - domain_id varchar(20), - source_concept_map_occurrence int + domain_id varchar(20) ); --standard concepts -INSERT INTO concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) +INSERT INTO concept_map (source_concept_id, target_concept_id, domain_id) select concept_id as source_concept_id, concept_id as target_concept_id, - domain_id, - 0 as source_concept_map_occurrence -from [CDMV5].[dbo].concept + domain_id +from [TARGET_CDMV5].[SCHEMA].concept where standard_concept = 'S' and invalid_reason is null @@ -158,20 +132,19 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( select concept_id -from [CDMV5].[dbo].concept +from [TARGET_CDMV5].[SCHEMA].concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -[CDMV5].[dbo].concept_relationship cr1 +[TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -[CDMV5].[dbo].concept c2 +[TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -185,17 +158,16 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -204,8 +176,8 @@ from OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Maps to') @@ -220,8 +192,8 @@ from ) AND B.concept_id IS NULL ) c1 -inner join [CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 -inner join [CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id +inner join [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 +inner join [TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null and cr1.relationship_id in ( @@ -243,16 +215,15 @@ union SELECT DISTINCT c1.concept_id AS source_concept_id ,c2.concept_id AS target_concept_id ,c2.domain_id - ,0 FROM ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -261,8 +232,8 @@ FROM ( OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ( @@ -288,8 +259,8 @@ FROM ( ) AND B.concept_id IS NULL ) c1 -INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Is a') @@ -302,53 +273,55 @@ WHERE c2.standard_concept = 'S' -- for the target tables when applicable /* UPDATE #concept_map - SET #concept_map.source_concept_map_occurrence = A.CountOfRows + SET #concept_map.source_concept_map_occurrence = A.targetConceptCount FROM #concept_map, ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) AS A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; UPDATE #concept_map -SET source_concept_map_occurrence = "CountOfRows" +SET source_concept_map_occurrence = "targetConceptCount" FROM ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; -*/ -UPDATE concept_map +UPDATE #concept_map SET source_concept_map_occurrence = ( - select count(*) as "CountOfRows" - from concept_map A - WHERE A.source_concept_id = concept_map.source_concept_id AND A.domain_id = concept_map.domain_id + select count(*) as "targetConceptCount" + from #concept_map A + WHERE A.source_concept_id = #concept_map.source_concept_id AND A.domain_id = #concept_map.domain_id group by source_concept_id, domain_id ) +*/ DROP TABLE IF EXISTS concept_map_distinct; - CREATE TEMP TABLE concept_map_distinct - -AS -SELECT - DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" - -FROM - concept_map +CREATE TEMP TABLE concept_map_distinct + ( + source_concept_id int, + domain_id varchar(20), + targetConceptCount int +); + + INSERT INTO concept_map_distinct (source_concept_id, domain_id, targetConceptCount) + SELECT source_concept_id, domain_id, COUNT(*) + FROM concept_map GROUP BY source_concept_id, domain_id; -DROP TABLE IF EXISTS [CDMV5].[dbo].ETL_WARNINGS; +DROP TABLE IF EXISTS [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS; -CREATE TABLE [CDMV5].[dbo].ETL_WARNINGS +CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS ( - WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(4000) ); /**** @@ -357,9 +330,9 @@ CDM_SOURCE ****/ -INSERT INTO [CDMV5].[dbo].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -select '[CDMV5]', 'V5', v.vocabulary_version, CURRENT_DATE -from [CDMV5].[dbo].vocabulary v +INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +select '[TARGET_CDMV5]', 'V5', v.vocabulary_version, CURRENT_DATE +from [TARGET_CDMV5].[SCHEMA].vocabulary v where vocabulary_id = 'Vocabulary'; /**** @@ -368,9 +341,9 @@ LOCATION ****/ -insert into [CDMV5].[dbo].location +insert into [TARGET_CDMV5].[SCHEMA].location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from [CDMV4].[dbo].LOCATION; +from [SOURCE_CDMV4].[SCHEMA].LOCATION; /**** @@ -378,9 +351,9 @@ CARE_SITE ****/ -insert into [CDMV5].[dbo].care_site +insert into [TARGET_CDMV5].[SCHEMA].care_site select care_site_id, cast(null as varchar(255)) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from [CDMV4].[dbo].CARE_SITE; +from [SOURCE_CDMV4].[SCHEMA].CARE_SITE; /**** @@ -388,11 +361,11 @@ Provider ****/ -insert into [CDMV5].[dbo].provider +insert into [TARGET_CDMV5].[SCHEMA].provider select provider_id, cast(null as varchar(255)) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, cast(null as varchar(50)) as gender_source_value, cast(null as integer) as gender_source_concept_id -from [CDMV4].[dbo].provider +from [SOURCE_CDMV4].[SCHEMA].provider ; @@ -402,7 +375,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT into [CDMV5].[dbo].person + INSERT into [TARGET_CDMV5].[SCHEMA].person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -422,21 +395,21 @@ from [CDMV4].[dbo].provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM [CDMV4].[dbo].PERSON p + FROM [SOURCE_CDMV4].[SCHEMA].PERSON p left JOIN concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -447,16 +420,16 @@ from [CDMV4].[dbo].provider ) warn ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -467,16 +440,16 @@ from [CDMV4].[dbo].provider ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -492,9 +465,9 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].observation_period + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM [CDMV4].[dbo].OBSERVATION_PERIOD; + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; /**** @@ -502,28 +475,28 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].death + INSERT INTO [TARGET_CDMV5].[SCHEMA].death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH LEFT JOIN concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -540,7 +513,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].visit_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, CAST(null as varchar(10)) visit_start_time, @@ -549,20 +522,20 @@ from [CDMV4].[dbo].provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE LEFT JOIN concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -581,19 +554,48 @@ from [CDMV4].[dbo].provider -- *************************************************************************** -- AGS: Modifying this section to insert this information into the temp - -- table #procedure_occurrence_map but this may need to be revisited for + -- table #po_map but this may need to be revisited for -- performance tuning on APS as a large temp table may cause processing -- time issues. -- *************************************************************************** - DROP TABLE IF EXISTS procedure_occurrence_map; + DROP TABLE IF EXISTS po_map; + +CREATE TEMP TABLE po_map + ( + procedure_occurrence_id int, + person_id int, + procedure_concept_id int, + procedure_date date, + procedure_type_concept_id int, + modifier_concept_id int, + quantity int, + provider_id int, + visit_occurrence_id int, + procedure_source_value varchar(50), + procedure_source_concept_id int, + qualifier_source_value varchar(50), + origional_drug_id bigint +); --find valid procedures from procedure table - CREATE TEMP TABLE procedure_occurrence_map - -AS -SELECT - +INSERT INTO po_map +( + procedure_occurrence_id, + person_id, + procedure_concept_id, + procedure_date, + procedure_type_concept_id, + modifier_concept_id, + quantity, + provider_id, + visit_occurrence_id, + procedure_source_value, + procedure_source_concept_id, + qualifier_source_value, + origional_drug_id +) + SELECT procedure_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as procedure_concept_id, @@ -607,17 +609,17 @@ SELECT CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - -FROM - [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence = 1 + AND LOWER(cm1.domain_id) IN ('procedure') + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_map_occurrence = 1 + LEFT JOIN concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 UNION ALL @@ -636,8 +638,8 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE - WHERE procedure_concept_id = 0 + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + WHERE procedure_concept_id = 0 UNION ALL @@ -656,7 +658,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE LEFT JOIN concept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -699,11 +701,12 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -725,7 +728,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -746,7 +749,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -762,16 +765,16 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO [CDMV5].[dbo].procedure_occurrence +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence (procedure_occurrence_id ,person_id ,procedure_concept_id @@ -797,19 +800,19 @@ SELECT ,procedure_source_value ,procedure_source_concept_id ,qualifier_source_value -FROM procedure_occurrence_map; +FROM po_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -819,15 +822,15 @@ FROM procedure_occurrence_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -845,13 +848,60 @@ FROM procedure_occurrence_map; ****/ --find valid drugs from drug_exposure table -DROP TABLE IF EXISTS drug_exposure_map; +DROP TABLE IF EXISTS drgexp_map; - CREATE TEMP TABLE drug_exposure_map - -AS -SELECT - drug_exposure_id, +CREATE TEMP TABLE drgexp_map + ( + drug_exposure_id int, + person_id int, + drug_concept_id int, + drug_exposure_start_date date, + drug_exposure_end_date date, + drug_type_concept_id int, + stop_reason varchar(20), + refills int, + quantity NUMERIC, + days_supply int, + sig TEXT, + route_concept_id int, + effective_drug_dose NUMERIC, + dose_unit_concept_id int, + lot_number varchar(50), + provider_id int, + visit_occurrence_id int, + drug_source_value varchar(50), + drug_source_concept_id int, + route_source_value varchar(50), + dose_unit_source_value varchar(50), + origional_procedure_id int +); + +INSERT INTO drgexp_map +( + drug_exposure_id, + person_id, + drug_concept_id, + drug_exposure_start_date, + drug_exposure_end_date, + drug_type_concept_id, + stop_reason, + refills, + quantity, + days_supply, + sig, + route_concept_id, + effective_drug_dose, + dose_unit_concept_id, + lot_number, + provider_id, + visit_occurrence_id, + drug_source_value, + drug_source_concept_id, + route_source_value, + dose_unit_source_value, + origional_procedure_id +) + SELECT drug_exposure_id, person_id, COALESCE(cm1.target_concept_id,0) as drug_concept_id, drug_exposure_start_date, @@ -873,17 +923,17 @@ SELECT CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - -FROM - [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_map_occurrence = 1 + INNER JOIN concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -911,7 +961,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -939,7 +989,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE LEFT JOIN concept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -981,11 +1031,12 @@ FROM CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -1000,7 +1051,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1015,7 +1066,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1029,13 +1080,13 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [CDMV4].[dbo].DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE) MAXROW ; -INSERT INTO [CDMV5].[dbo].drug_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure (drug_exposure_id ,person_id ,drug_concept_id @@ -1079,19 +1130,19 @@ SELECT ,drug_source_concept_id ,route_source_value ,dose_unit_source_value -FROM drug_exposure_map; +FROM drgexp_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1101,15 +1152,15 @@ FROM drug_exposure_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1127,7 +1178,7 @@ FROM drug_exposure_map; ****/ --find valid conditions from condition_occurrence table - INSERT INTO [CDMV5].[dbo].condition_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1136,11 +1187,12 @@ FROM drug_exposure_map; COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1160,7 +1212,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1177,7 +1229,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE LEFT JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN concept_map cm2 @@ -1203,11 +1255,12 @@ FROM ( stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1222,7 +1275,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1235,7 +1288,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1248,25 +1301,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [CDMV4].[dbo].CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1276,15 +1329,15 @@ FROM ( ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1303,7 +1356,7 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].device_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1317,7 +1370,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1331,7 +1384,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1345,7 +1398,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1359,7 +1412,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1374,7 +1427,7 @@ FROM ****/ --find valid measurements from observation table -INSERT INTO [CDMV5].[dbo].measurement +INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1399,7 +1452,7 @@ from unit_source_value AS unit_source_value, cast(null as varchar(50)) as value_source_value, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1427,7 +1480,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1452,7 +1505,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1477,7 +1530,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].drug_exposure + FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1496,7 +1549,7 @@ from --find valid observation from observation table - INSERT INTO [CDMV5].[dbo].observation + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation SELECT observation_id, person_id, @@ -1515,7 +1568,7 @@ from CAST(null as integer) observation_source_concept_id, unit_source_value, cast(null as varchar(50)) qualifier_source_value - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) @@ -1550,7 +1603,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as observation_source_value, @@ -1558,7 +1611,7 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1574,7 +1627,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value as observation_source_value, @@ -1582,7 +1635,7 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1606,11 +1659,11 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [CDMV4].[dbo].OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION) MAXROW ; @@ -1621,10 +1674,10 @@ FROM ****/ -INSERT INTO [CDMV5].[dbo].payer_plan_period +INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; +FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; /**** @@ -1634,15 +1687,15 @@ FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; ****/ -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM [CDMV4].[dbo].DRUG_COST dc +FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, drug_exposure_id, @@ -1674,11 +1727,11 @@ FROM ( CAST(null as NUMERIC) as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE po - join [CDMV4].[dbo].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po + join [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - JOIN drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN drgexp_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1690,15 +1743,15 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM [CDMV4].[dbo].PROCEDURE_COST; +FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, procedure_occurrence_id, @@ -1732,11 +1785,11 @@ SELECT CAST(null as integer) as revenue_code_concept_id, CAST(null as integer) as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE de - join [CDMV4].[dbo].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de + join [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - JOIN procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN po_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1753,9 +1806,9 @@ with cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONC select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, (DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), (DRUG_EXPOSURE_START_DATE + 1)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM [CDMV5].[dbo].DRUG_EXPOSURE d - join [CDMV5].[dbo].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join [CDMV5].[dbo].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d + join [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join [TARGET_CDMV5].[SCHEMA].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1805,7 +1858,7 @@ GROUP BY d.PERSON_ID, d.DRUG_TYPE_CONCEPT_ID, d.DRUG_EXPOSURE_START_DATE ) -insert into [CDMV5].[dbo].drug_era +insert into [TARGET_CDMV5].[SCHEMA].drug_era select row_number() over (order by person_id) as drug_era_id, person_id, drug_concept_id, min(DRUG_EXPOSURE_START_DATE) as drug_era_start_date, drug_era_end_date, COUNT(*) as DRUG_EXPOSURE_COUNT, 30 as gap_days from cteDrugExposureEnds GROUP BY person_id, drug_concept_id, drug_type_concept_id, DRUG_ERA_END_DATE @@ -1828,7 +1881,7 @@ WITH cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, (CONDITION_START_DATE + 1)) as CONDITION_END_DATE - FROM [CDMV5].[dbo].CONDITION_OCCURRENCE co + FROM [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1883,7 +1936,7 @@ FROM GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO [CDMV5].[dbo].condition_era +INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era (condition_era_id ,person_id ,condition_concept_id diff --git a/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql b/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql index 41f4232..ca655a0 100644 --- a/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql +++ b/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql @@ -19,7 +19,7 @@ PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. -last revised: 01 July 2015 +last revised: 09 July 2015 author: Patrick Ryan, Chris Knoll editor: Anthony Sena @@ -34,7 +34,7 @@ instructions will be slightly different. General Assumptions ------------------- -This script assumes that your V4 and V5 database either located on the same +This script assumes that your V4 and V5 database are located on the same RDBMS server. Getting Started @@ -51,9 +51,9 @@ TemplateSQL File Instructions ----------------------------- 1. Set parameter name of schema that contains CDMv4 instance - ([CDMV4], [CDMV4].[dbo]) + ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) 2. Set parameter name of schema that contains CDMv5 instance - ([CDMV5], [CDMV5].[dbo]) + ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) 3. Run this script through SqlRender to produce a script that will work in your source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender 4. Run the script produced by SQL Render on your target RDBDMS. @@ -76,37 +76,13 @@ TemplateSQL File Instructions *********************************************************************************/ -/* SCRIPT PARAMETERS --- -- The CDMv4 database name --- -- The CDMv4 database plus schema --- -- The target CDMv5 database name --- -- the target CDMv5 database plus schema -*/ -/* QA Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - */ -/* PostgreSQL Settings +/* SCRIPT PARAMETERS */ -- The CDMv4 database name -- The CDMv4 database plus schema -- The target CDMv5 database name -- the target CDMv5 database plus schema -*/ -/* Oracle Settings - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema -*/ -/* LOCAL SQL Server */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -USE [CDMV5]; + +USE [TARGET_CDMV5]; /* * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids @@ -139,17 +115,15 @@ CREATE TABLE #concept_map ( source_concept_id int, target_concept_id int, - domain_id varchar(20), - source_concept_map_occurrence int + domain_id varchar(20) ); --standard concepts -INSERT INTO #concept_map (source_concept_id, target_concept_id, domain_id, source_concept_map_occurrence) +INSERT INTO #concept_map (source_concept_id, target_concept_id, domain_id) select concept_id as source_concept_id, concept_id as target_concept_id, - domain_id, - 0 as source_concept_map_occurrence -from [CDMV5].[dbo].concept + domain_id +from [TARGET_CDMV5].[SCHEMA].concept where standard_concept = 'S' and invalid_reason is null @@ -159,20 +133,19 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( select concept_id -from [CDMV5].[dbo].concept +from [TARGET_CDMV5].[SCHEMA].concept where ((standard_concept <> 'S' or standard_concept is null) or invalid_reason is not null ) ) c1 inner join -[CDMV5].[dbo].concept_relationship cr1 +[TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 inner join -[CDMV5].[dbo].concept c2 +[TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null @@ -186,17 +159,16 @@ union select distinct c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, c2.domain_id - ,0 from ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -205,8 +177,8 @@ from OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Maps to') @@ -221,8 +193,8 @@ from ) AND B.concept_id IS NULL ) c1 -inner join [CDMV5].[dbo].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 -inner join [CDMV5].[dbo].concept c2 on cr1.concept_id_2 = c2.concept_id +inner join [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 on c1.concept_id = cr1.concept_id_1 +inner join [TARGET_CDMV5].[SCHEMA].concept c2 on cr1.concept_id_2 = c2.concept_id where c2.standard_concept = 'S' and c2.invalid_reason is null and cr1.relationship_id in ( @@ -244,16 +216,15 @@ union SELECT DISTINCT c1.concept_id AS source_concept_id ,c2.concept_id AS target_concept_id ,c2.domain_id - ,0 FROM ( SELECT A.concept_id - FROM [CDMV5].[dbo].concept A + FROM [TARGET_CDMV5].[SCHEMA].concept A LEFT JOIN ( SELECT DISTINCT c1.concept_id FROM ( SELECT concept_id - FROM [CDMV5].[dbo].concept + FROM [TARGET_CDMV5].[SCHEMA].concept WHERE ( ( standard_concept <> 'S' @@ -262,8 +233,8 @@ FROM ( OR invalid_reason IS NOT NULL ) ) c1 - INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 + INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ( @@ -289,8 +260,8 @@ FROM ( ) AND B.concept_id IS NULL ) c1 -INNER JOIN [CDMV5].[dbo].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [CDMV5].[dbo].concept c2 ON cr1.concept_id_2 = c2.concept_id +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 +INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id WHERE c2.standard_concept = 'S' AND c2.invalid_reason IS NULL AND cr1.relationship_id IN ('Is a') @@ -303,50 +274,57 @@ WHERE c2.standard_concept = 'S' -- for the target tables when applicable /* UPDATE #concept_map - SET #concept_map.source_concept_map_occurrence = A.CountOfRows + SET #concept_map.source_concept_map_occurrence = A.targetConceptCount FROM #concept_map, ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) AS A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; UPDATE #concept_map -SET source_concept_map_occurrence = "CountOfRows" +SET source_concept_map_occurrence = "targetConceptCount" FROM ( - select source_concept_id, domain_id, count(*) as "CountOfRows" + select source_concept_id, domain_id, count(*) as "targetConceptCount" from #concept_map group by source_concept_id, domain_id ) A WHERE #concept_map.source_concept_id = A.source_concept_id AND #concept_map.domain_id = A.domain_id; -*/ UPDATE #concept_map SET source_concept_map_occurrence = ( - select count(*) as "CountOfRows" + select count(*) as "targetConceptCount" from #concept_map A WHERE A.source_concept_id = #concept_map.source_concept_id AND A.domain_id = #concept_map.domain_id group by source_concept_id, domain_id ) +*/ IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL DROP TABLE #concept_map_distinct; - SELECT DISTINCT source_concept_id, domain_id, COUNT(*) as "rowcount" - INTO #concept_map_distinct +CREATE TABLE #concept_map_distinct +( + source_concept_id int, + domain_id varchar(20), + targetConceptCount int +); + + INSERT INTO #concept_map_distinct (source_concept_id, domain_id, targetConceptCount) + SELECT source_concept_id, domain_id, COUNT(*) FROM #concept_map GROUP BY source_concept_id, domain_id; -IF OBJECT_ID('[CDMV5].[dbo].ETL_WARNINGS', 'U') IS NOT NULL - DROP TABLE [CDMV5].[dbo].ETL_WARNINGS; +IF OBJECT_ID('[TARGET_CDMV5].[SCHEMA].ETL_WARNINGS', 'U') IS NOT NULL + DROP TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS; -CREATE TABLE [CDMV5].[dbo].ETL_WARNINGS +CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS ( - WARNING_MESSAGE varchar(8000) + WARNING_MESSAGE varchar(4000) ); /**** @@ -355,9 +333,9 @@ CDM_SOURCE ****/ -INSERT INTO [CDMV5].[dbo].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) -select '[CDMV5]', 'V5', v.vocabulary_version, getDate() -from [CDMV5].[dbo].vocabulary v +INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source (cdm_source_name, cdm_version, vocabulary_version, cdm_release_date) +select '[TARGET_CDMV5]', 'V5', v.vocabulary_version, getDate() +from [TARGET_CDMV5].[SCHEMA].vocabulary v where vocabulary_id = 'Vocabulary'; /**** @@ -366,9 +344,9 @@ LOCATION ****/ -insert into [CDMV5].[dbo].location +insert into [TARGET_CDMV5].[SCHEMA].location select location_id, address_1, address_2, city, state, zip, county, location_source_value -from [CDMV4].[dbo].LOCATION; +from [SOURCE_CDMV4].[SCHEMA].LOCATION; /**** @@ -376,9 +354,9 @@ CARE_SITE ****/ -insert into [CDMV5].[dbo].care_site +insert into [TARGET_CDMV5].[SCHEMA].care_site select care_site_id, cast(null as varchar(255)) as care_site_name, place_of_service_concept_id, location_id, care_site_source_value, place_of_service_source_value -from [CDMV4].[dbo].CARE_SITE; +from [SOURCE_CDMV4].[SCHEMA].CARE_SITE; /**** @@ -386,11 +364,11 @@ Provider ****/ -insert into [CDMV5].[dbo].provider +insert into [TARGET_CDMV5].[SCHEMA].provider select provider_id, cast(null as varchar(255)) as provider_name, NPI, DEA, specialty_concept_id, care_site_id, cast(null as integer) as year_of_birth, cast(null as integer) as gender_concept_id, provider_source_value, specialty_source_value, 0 as specialty_source_concept_id, cast(null as varchar(50)) as gender_source_value, cast(null as integer) as gender_source_concept_id -from [CDMV4].[dbo].provider +from [SOURCE_CDMV4].[SCHEMA].provider ; @@ -400,7 +378,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT into [CDMV5].[dbo].person + INSERT into [TARGET_CDMV5].[SCHEMA].person SELECT person_id, coalesce(gender.target_concept_id, 0) as gender_concept_id, @@ -420,21 +398,21 @@ from [CDMV4].[dbo].provider CAST(null as integer) race_source_concept_id, ethnicity_source_value, CAST(null as integer) ethnicity_source_concept_id - FROM [CDMV4].[dbo].PERSON p + FROM [SOURCE_CDMV4].[SCHEMA].PERSON p left JOIN #concept_map gender on LOWER(gender.DOMAIN_ID) IN ('gender') and p.gender_concept_id = gender.source_concept_id left JOIN #concept_map race on LOWER(race.DOMAIN_ID) IN ('race') and p.race_concept_id = race.source_concept_id LEFT JOIN #concept_map ethnicity on LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') and p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE GENDER_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -445,16 +423,16 @@ from [CDMV4].[dbo].provider ) warn ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE RACE_CONCEPT_ID IS NOT NULL AND RACE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -465,16 +443,16 @@ from [CDMV4].[dbo].provider ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PERSON + FROM [SOURCE_CDMV4].[SCHEMA].PERSON WHERE ETHNICITY_CONCEPT_ID IS NOT NULL AND ETHNICITY_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -490,9 +468,9 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].observation_period + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period SELECT observation_period_id, person_id, observation_period_start_date, observation_period_end_date, 44814722 as period_type_concept_id - FROM [CDMV4].[dbo].OBSERVATION_PERIOD; + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; /**** @@ -500,28 +478,28 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].death + INSERT INTO [TARGET_CDMV5].[SCHEMA].death SELECT person_id, death_date, COALESCE(death_type_concept_id,0) AS death_type_concept_id, cause_of_death_concept_id as cause_concept_id, cause_of_death_source_value as cause_source_value, CAST(null as integer) as cause_source_concept_id - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH LEFT JOIN #concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID AND LOWER(DOMAIN_ID) IN ('death type'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DEATH: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DEATH + FROM [SOURCE_CDMV4].[SCHEMA].DEATH WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -538,7 +516,7 @@ from [CDMV4].[dbo].provider ****/ - INSERT INTO [CDMV5].[dbo].visit_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence SELECT visit_occurrence_id, person_id, COALESCE(cm1.target_concept_id,0) as visit_concept_id, visit_start_date, CAST(null as varchar(10)) visit_start_time, @@ -547,20 +525,20 @@ from [CDMV4].[dbo].provider CAST(null as integer) provider_id, care_site_id, place_of_service_source_value as visit_source_value, CAST(null as integer) visit_source_concept_id - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE LEFT JOIN #concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('visit'); - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'VISIT_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].VISIT_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -579,15 +557,48 @@ from [CDMV4].[dbo].provider -- *************************************************************************** -- AGS: Modifying this section to insert this information into the temp - -- table #procedure_occurrence_map but this may need to be revisited for + -- table #po_map but this may need to be revisited for -- performance tuning on APS as a large temp table may cause processing -- time issues. -- *************************************************************************** - IF OBJECT_ID('tempdb..#procedure_occurrence_map', 'U') IS NOT NULL - DROP TABLE #procedure_occurrence_map; + IF OBJECT_ID('tempdb..#po_map', 'U') IS NOT NULL + DROP TABLE #po_map; + +CREATE TABLE #po_map +( + procedure_occurrence_id int, + person_id int, + procedure_concept_id int, + procedure_date date, + procedure_type_concept_id int, + modifier_concept_id int, + quantity int, + provider_id int, + visit_occurrence_id int, + procedure_source_value varchar(50), + procedure_source_concept_id int, + qualifier_source_value varchar(50), + origional_drug_id bigint +); --find valid procedures from procedure table +INSERT INTO #po_map +( + procedure_occurrence_id, + person_id, + procedure_concept_id, + procedure_date, + procedure_type_concept_id, + modifier_concept_id, + quantity, + provider_id, + visit_occurrence_id, + procedure_source_value, + procedure_source_concept_id, + qualifier_source_value, + origional_drug_id +) SELECT procedure_occurrence_id, person_id, @@ -602,16 +613,17 @@ from [CDMV4].[dbo].provider CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - INTO #procedure_occurrence_map - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence = 1 + AND LOWER(cm1.domain_id) IN ('procedure') + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') - AND cm2.source_concept_map_occurrence = 1 + LEFT JOIN #concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 UNION ALL @@ -630,8 +642,8 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE - WHERE procedure_concept_id = 0 + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE + WHERE procedure_concept_id = 0 UNION ALL @@ -650,7 +662,7 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE LEFT JOIN #concept_map cm1 ON procedure_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -693,11 +705,12 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('procedure type') @@ -719,7 +732,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) origional_drug_id, condition_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -740,7 +753,7 @@ UNION ALL CAST(null as varchar(50)) qualifier_source_value, drug_exposure_id as origional_drug_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') @@ -756,16 +769,16 @@ UNION ALL CAST(null as integer) procedure_source_concept_id, CAST(null as varchar(50)) qualifier_source_value, CAST(null as bigint) as origional_drug_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE) MAXROW + ) OTHERS,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE) MAXROW ; -INSERT INTO [CDMV5].[dbo].procedure_occurrence +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence (procedure_occurrence_id ,person_id ,procedure_concept_id @@ -791,19 +804,19 @@ SELECT ,procedure_source_value ,procedure_source_concept_id ,qualifier_source_value -FROM #procedure_occurrence_map; +FROM #po_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -813,15 +826,15 @@ FROM #procedure_occurrence_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -839,9 +852,60 @@ FROM #procedure_occurrence_map; ****/ --find valid drugs from drug_exposure table -IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL - DROP TABLE #drug_exposure_map; +IF OBJECT_ID('tempdb..#drgexp_map', 'U') IS NOT NULL + DROP TABLE #drgexp_map; +CREATE TABLE #drgexp_map +( + drug_exposure_id int, + person_id int, + drug_concept_id int, + drug_exposure_start_date date, + drug_exposure_end_date date, + drug_type_concept_id int, + stop_reason varchar(20), + refills int, + quantity float, + days_supply int, + sig varchar(max), + route_concept_id int, + effective_drug_dose float, + dose_unit_concept_id int, + lot_number varchar(50), + provider_id int, + visit_occurrence_id int, + drug_source_value varchar(50), + drug_source_concept_id int, + route_source_value varchar(50), + dose_unit_source_value varchar(50), + origional_procedure_id int +); + +INSERT INTO #drgexp_map +( + drug_exposure_id, + person_id, + drug_concept_id, + drug_exposure_start_date, + drug_exposure_end_date, + drug_type_concept_id, + stop_reason, + refills, + quantity, + days_supply, + sig, + route_concept_id, + effective_drug_dose, + dose_unit_concept_id, + lot_number, + provider_id, + visit_occurrence_id, + drug_source_value, + drug_source_concept_id, + route_source_value, + dose_unit_source_value, + origional_procedure_id +) SELECT drug_exposure_id, person_id, COALESCE(cm1.target_concept_id,0) as drug_concept_id, @@ -864,16 +928,17 @@ IF OBJECT_ID('tempdb..#drug_exposure_map', 'U') IS NOT NULL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - INTO #drug_exposure_map - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') - AND cm2.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis2 + ON cm2.source_concept_id = cmdis2.source_concept_id AND cm2.domain_id = cmdis2.domain_id AND cmdis2.targetConceptCount = 1 WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 UNION ALL @@ -901,7 +966,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE drug_concept_id = 0 UNION ALL @@ -929,7 +994,7 @@ UNION ALL CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE LEFT JOIN #concept_map cm1 ON drug_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -971,11 +1036,12 @@ FROM CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('drug type') @@ -990,7 +1056,7 @@ FROM null as provider_id, visit_occurrence_id, condition_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1005,7 +1071,7 @@ FROM null as provider_id, visit_occurrence_id, procedure_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, procedure_occurrence_id as origional_procedure_id, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') @@ -1019,13 +1085,13 @@ FROM null as provider_id, visit_occurrence_id, observation_source_value as drug_source_value, CAST(null as integer) drug_source_concept_id, CAST(null as varchar(50)) route_source_value, CAST(null as varchar(50)) dose_unit_source_value, CAST(null as bigint) origional_procedure_id, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('drug') -) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [CDMV4].[dbo].DRUG_EXPOSURE) MAXROW +) OTHERS,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE) MAXROW ; -INSERT INTO [CDMV5].[dbo].drug_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure (drug_exposure_id ,person_id ,drug_concept_id @@ -1069,19 +1135,19 @@ SELECT ,drug_source_concept_id ,route_source_value ,dose_unit_source_value -FROM #drug_exposure_map; +FROM #drgexp_map; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1091,15 +1157,15 @@ FROM #drug_exposure_map; ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1117,7 +1183,7 @@ FROM #drug_exposure_map; ****/ --find valid conditions from condition_occurrence table - INSERT INTO [CDMV5].[dbo].condition_occurrence + INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence SELECT condition_occurrence_id, person_id, COALESCE(cm1.target_concept_id, 0) AS condition_concept_id, @@ -1126,11 +1192,12 @@ FROM #drug_exposure_map; COALESCE(cm2.target_concept_id,0) AS condition_type_concept_id, stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence = 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount = 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1150,7 +1217,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE condition_concept_id = 0 UNION ALL @@ -1167,7 +1234,7 @@ UNION ALL visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE LEFT JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id LEFT JOIN #concept_map cm2 @@ -1193,11 +1260,12 @@ FROM ( stop_reason, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value, CAST(null as integer) condition_source_concept_id, NULL as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') - AND cm1.source_concept_map_occurrence > 1 + INNER JOIN #concept_map_distinct cmdis + ON cm1.source_concept_id = cmdis.source_concept_id AND cm1.domain_id = cmdis.domain_id AND cmdis.targetConceptCount > 1 LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id AND LOWER(cm2.domain_id) IN ('condition type') @@ -1212,7 +1280,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, procedure_occurrence_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1225,7 +1293,7 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, prescribing_provider_id as provider_id, visit_occurrence_id, drug_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, drug_exposure_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') @@ -1238,25 +1306,25 @@ FROM ( 0 as condition_type_concept_id, null as stop_reason, associated_provider_id as provider_id, visit_occurrence_id, observation_source_value as condition_source_value, CAST(null as integer) condition_source_concept_id, observation_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('condition') -) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [CDMV4].[dbo].CONDITION_OCCURRENCE) MAXROW +) OTHERS,(SELECT MAX(condition_occurrence_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE) MAXROW ; --warnings of invalid records - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDITION_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR STANDARD_CONCEPT = 'S' @@ -1266,15 +1334,15 @@ FROM ( ; - INSERT INTO [CDMV5].[dbo].ETL_WARNINGS (WARNING_MESSAGE) + INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) SELECT 'CONDIITON_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' FROM ( SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( SELECT CONCEPT_ID - FROM [CDMV5].[dbo].CONCEPT + FROM [TARGET_CDMV5].[SCHEMA].CONCEPT WHERE CONCEPT_ID = 0 OR (STANDARD_CONCEPT = 'S' @@ -1293,7 +1361,7 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].device_exposure +INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure select row_number() over (order by OCCURRENCE_ID) AS device_exposure_id, person_id, device_concept_id, device_exposure_start_date, device_exposure_end_date, device_type_concept_id, unique_device_id, quantity, provider_id, visit_occurrence_id, device_source_value, device_source_concept_id @@ -1307,7 +1375,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, PROCEDURE_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1321,7 +1389,7 @@ FROM quantity, PRESCRIBING_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, DRUG_EXPOSURE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1335,7 +1403,7 @@ FROM CAST(NULL as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, CONDITION_OCCURRENCE_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1349,7 +1417,7 @@ FROM CAST(null as integer) quantity, ASSOCIATED_PROVIDER_ID AS PROVIDER_ID, VISIT_OCCURRENCE_ID, OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE, 0 as device_source_concept_id, OBSERVATION_ID as OCCURRENCE_ID - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('device') @@ -1364,7 +1432,7 @@ FROM ****/ --find valid measurements from observation table -INSERT INTO [CDMV5].[dbo].measurement +INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement SELECT row_number() over (order by occurrence_id) AS measurement_id, person_id, measurement_concept_id, measurement_date, measurement_time, measurement_type_concept_id, operator_concept_id, value_as_number, value_as_concept_id, unit_concept_id, range_low, range_high, provider_id, visit_occurrence_id, measurement_source_value, measurement_source_concept_id, unit_source_value, value_source_value @@ -1389,7 +1457,7 @@ from unit_source_value AS unit_source_value, cast(null as varchar(50)) as value_source_value, observation_id as occurrence_id - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1417,7 +1485,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1442,7 +1510,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1467,7 +1535,7 @@ from CAST(null as varchar(50)) as unit_source_value, CAST(null as varchar(50)) as value_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].drug_exposure + FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('measurement') @@ -1486,7 +1554,7 @@ from --find valid observation from observation table - INSERT INTO [CDMV5].[dbo].observation + INSERT INTO [TARGET_CDMV5].[SCHEMA].observation SELECT observation_id, person_id, @@ -1505,7 +1573,7 @@ from CAST(null as integer) observation_source_concept_id, unit_source_value, cast(null as varchar(50)) qualifier_source_value - FROM [CDMV4].[dbo].OBSERVATION + FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION WHERE observation_concept_id NOT IN (SELECT source_concept_id FROM #concept_map_distinct WHERE LOWER(domain_id) IN ('condition','drug','procedure','device','measurement')) @@ -1540,7 +1608,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, procedure_source_value as observation_source_value, @@ -1548,7 +1616,7 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, procedure_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1564,7 +1632,7 @@ FROM null as value_as_string, CAST(null as integer) as value_as_concept_id, CAST(null as integer) qualifier_concept_id, - null as unit_concept_id, + CAST(null as integer) as unit_concept_id, associated_provider_id as provider_id, visit_occurrence_id, condition_source_value as observation_source_value, @@ -1572,7 +1640,7 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, condition_occurrence_id as occurrence_id - FROM [CDMV4].[dbo].CONDITION_OCCURRENCE + FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') @@ -1596,11 +1664,11 @@ FROM null as unit_source_value, cast(null as varchar(50)) qualifier_source_value, drug_exposure_id as occurrence_id - FROM [CDMV4].[dbo].DRUG_EXPOSURE + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [CDMV4].[dbo].OBSERVATION) MAXROW + ) OTHERS,(SELECT MAX(OBSERVATION_ID) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION) MAXROW ; @@ -1611,10 +1679,10 @@ FROM ****/ -INSERT INTO [CDMV5].[dbo].payer_plan_period +INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period SELECT payer_plan_period_id, person_id, payer_plan_period_start_date, payer_plan_period_end_date, payer_source_value, plan_source_value, family_source_value -FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; +FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; /**** @@ -1624,15 +1692,15 @@ FROM [CDMV4].[dbo].PAYER_PLAN_PERIOD; ****/ -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost SELECT drug_cost_id, dc.drug_exposure_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, ingredient_cost, dispensing_fee, average_wholesale_price, payer_plan_period_id -FROM [CDMV4].[dbo].DRUG_COST dc +FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ; -- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [CDMV5].[dbo].drug_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost select CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS drug_cost_id, drug_exposure_id, @@ -1664,11 +1732,11 @@ FROM ( CAST(null as float) as average_wholesale_price, payer_plan_period_id, procedure_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].PROCEDURE_OCCURRENCE po - join [CDMV4].[dbo].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id + FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po + join [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc on po.procedure_occurrence_id = pc.procedure_occurrence_id --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - JOIN #drug_exposure_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id -) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN #drgexp_map de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id +) OTHERS ,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1680,15 +1748,15 @@ FROM ( ****/ -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT procedure_cost_id, procedure_occurrence_id, cast(null as integer) currency_concept_id, paid_copay, paid_coinsurance, paid_toward_deductible, paid_by_payer, paid_by_coordination_benefits, total_out_of_pocket, total_paid, payer_plan_period_id, revenue_code_concept_id, revenue_code_source_value -FROM [CDMV4].[dbo].PROCEDURE_COST; +FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; -- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [CDMV5].[dbo].procedure_cost +INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost SELECT CASE WHEN MAXROW.MAXROWID IS NULL THEN 0 ELSE MAXROW.MAXROWID END + row_number() over (order by OCCURRENCE_ID) AS procedure_cost_id, procedure_occurrence_id, @@ -1722,11 +1790,11 @@ SELECT CAST(null as integer) as revenue_code_concept_id, CAST(null as integer) as revenue_code_source_value, drug_cost_id as OCCURRENCE_ID - FROM [CDMV4].[dbo].DRUG_EXPOSURE de - join [CDMV4].[dbo].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id + FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de + join [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc on de.drug_exposure_id = dc.drug_exposure_id --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - JOIN #procedure_occurrence_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [CDMV4].[dbo].DRUG_COST) MAXROW + JOIN #po_map po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id + ) OTHERS,(SELECT MAX(drug_cost_id) AS MAXROWID FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST) MAXROW ; /**** @@ -1743,9 +1811,9 @@ with cteDrugTarget (DRUG_EXPOSURE_ID, PERSON_ID, DRUG_CONCEPT_ID, DRUG_TYPE_CONC select d.DRUG_EXPOSURE_ID, d. PERSON_ID, c.CONCEPT_ID, d.DRUG_TYPE_CONCEPT_ID, DRUG_EXPOSURE_START_DATE, COALESCE(DRUG_EXPOSURE_END_DATE, DATEADD(day,DAYS_SUPPLY,DRUG_EXPOSURE_START_DATE), DATEADD(day,1,DRUG_EXPOSURE_START_DATE)) as DRUG_EXPOSURE_END_DATE, c.CONCEPT_ID as INGREDIENT_CONCEPT_ID - FROM [CDMV5].[dbo].DRUG_EXPOSURE d - join [CDMV5].[dbo].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID - join [CDMV5].[dbo].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID + FROM [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d + join [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca on ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID + join [TARGET_CDMV5].[SCHEMA].CONCEPT c on ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID where c.VOCABULARY_ID = 'RxNorm' and c.CONCEPT_CLASS_ID = 'Ingredient' ), @@ -1795,7 +1863,7 @@ GROUP BY d.PERSON_ID, d.DRUG_TYPE_CONCEPT_ID, d.DRUG_EXPOSURE_START_DATE ) -insert into [CDMV5].[dbo].drug_era +insert into [TARGET_CDMV5].[SCHEMA].drug_era select row_number() over (order by person_id) as drug_era_id, person_id, drug_concept_id, min(DRUG_EXPOSURE_START_DATE) as drug_era_start_date, drug_era_end_date, COUNT(*) as DRUG_EXPOSURE_COUNT, 30 as gap_days from cteDrugExposureEnds GROUP BY person_id, drug_concept_id, drug_type_concept_id, DRUG_ERA_END_DATE @@ -1816,7 +1884,7 @@ with cteConditionTarget (PERSON_ID, CONDITION_CONCEPT_ID, CONDITION_START_DATE, -- create base eras from the concepts found in condition_occurrence select co.PERSON_ID, co.condition_concept_id, co.CONDITION_START_DATE, COALESCE(co.CONDITION_END_DATE, DATEADD(day,1,CONDITION_START_DATE)) as CONDITION_END_DATE - FROM [CDMV5].[dbo].CONDITION_OCCURRENCE co + FROM [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co ), cteEndDates (PERSON_ID, CONDITION_CONCEPT_ID, END_DATE) as -- the magic ( @@ -1869,7 +1937,7 @@ from cteConditionEnds GROUP BY person_id, CONDITION_CONCEPT_ID, ERA_END_DATE ; -INSERT INTO [CDMV5].[dbo].condition_era +INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era (condition_era_id ,person_id ,condition_concept_id