diff --git a/README.md b/README.md index be50412..8b7b3af 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,45 @@ Common-Data-Model ================= See full CDM specification file on our [Wiki](http://www.ohdsi.org/web/wiki/doku.php?id=documentation:cdm:single-page) or in the [CDM V5 PDF](https://github.com/OHDSI/CommonDataModel/blob/master/OMOP%20CDM%20v5.pdf) + +Release Notes +============= +This version is bases on this CDM working group [proposal](http://www.ohdsi.org/web/wiki/doku.php?id=documentation:next_cdm:time). The proposed and accepted changes include adding a datetime field to every table that had a date column. These were the new columns added: + +**PERSON** +birth_datetime, not required + +**SPECIMEN** +specimen_datetime, not required + +**DEATH** +death_datetime, not required + +**VISIT_OCCURRENCE** +visit_start_datetime, not required +visit_end_datetime, not required + +**PROCEDURE_OCCURRENCE** +procedure_datetime, not required + +**DRUG_EXPOSURE** +drug_exposure_start_datetime, not required +drug_exposure_end_datetime, not required + +**DEVICE_EXPOSURE** +device_exposure_start_datetime, not required +device_exposure_end_datetime, not required + +**CONDITION_OCCURRENCE** +condition_start_datetime, not required +condition_end_datetime, not required + +**MEASUREMENT** +measurement_datetime as time, not required + +**OBSERVATION** +observation_datetime, not required + +**NOTE** +note_datetime, not required + diff --git a/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - OHDSI-SQL.sql b/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - OHDSI-SQL.sql deleted file mode 100644 index 743316e..0000000 --- a/Version4 To Version5 Conversion/OMOP CDMv4 to CDMv5 - OHDSI-SQL.sql +++ /dev/null @@ -1,2358 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -********************************************************************************/ -/******************************************************************************* - -PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. - -last revised: 04 August 2015 -authors: Patrick Ryan, Chris Knoll, Anthony Sena - -!!!!!!!!!!!!!!!!!!!!! PLEASE READ THESE INSTRUCTIONS !!!!!!!!!!!!!!!!!!!!!!!!!!! - -This script was authored using OHDSI-SQL which will require you to run this -script through SqlRender to creat a version that is compatible with your target -RDBMS. We have pre-generated these scripts using SQL Render and have placed -them in folders for each RDBMS. Depending on which script you are viewing, your -instructions will be slightly different. - -General Assumptions -------------------- - -This script assumes that your V4 and V5 database are located on the same -RDBMS server. It also assumes that the V4 and V5 databases were created -using the standard data definition scripts for these databases. If you -altered your V4 database in any way, this script will likely require -some mo - -Getting Started ---------------- - -Before you can use this script, there are some prerequisites: - - 1. Create a target CDMv5 database on your database server using the - appropriate script from https://github.com/OHDSI/CommonDataModel - 2. Load VocabV5 into the target database/schema that will contain CDMv5 using - Athena: http://ohdsi.org/web/ATHENA - -OHDSI-SQL File Instructions ------------------------------ - - 1. Set parameter name of schema that contains CDMv4 instance - (@SOURCE_CDMV4, @SOURCE_CDMV4_SCHEMA) - 2. Set parameter name of schema that contains CDMv5 instance - (@TARGET_CDMV5, @TARGET_CDMV5_SCHEMA) - 3. Run this script through SqlRender to produce a script that will work in your - source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender - 4. Run the script produced by SQL Render on your target RDBDMS. - - File Instructions -------------------------- - - 1. This script will hold a number of placeholders for your CDM V4 and CDMV5 - database/schema. In order to make this file work in your environment, you - should plan to do a global "FIND AND REPLACE" on this file to fill in the - file with values that pertain to your environment. The following are the - tokens you should use when doing your "FIND AND REPLACE" operation: - - a. [SOURCE_CDMV4] - b. [SOURCE_CDMV4].[SCHEMA] - c. [TARGET_CDMV5] - d. [TARGET_CDMV5].[SCHEMA] - - 2. Run the resulting script on your target RDBDMS. - -*********************************************************************************/ -/* SCRIPT PARAMETERS */ -{DEFAULT @SOURCE_CDMV4 = '[SOURCE_CDMV4]' } -- The CDMv4 database name - {DEFAULT @SOURCE_CDMV4_SCHEMA = '[SOURCE_CDMV4].[SCHEMA]' } -- The CDMv4 database plus schema - {DEFAULT @TARGET_CDMV5 = '[TARGET_CDMV5]' } -- The target CDMv5 database name - {DEFAULT @TARGET_CDMV5_SCHEMA = '[TARGET_CDMV5].[SCHEMA]' } -- the target CDMv5 database plus schema - -USE @TARGET_CDMV5; - -/* - * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids - * and their respective domain_ids. As a result, the results of this table will have - * some source_concept_ids that map to multiple target_concept_ids which is expected. - * - * In some of the conversion queries below, we will use the distinct values from the - * #concept_map table since our need in those instances is to understand the domain_id - * that will provide the destination table in the target V5 database. To make the code - * easier to read, we dervied a #concept_map_distinct table that holds the distinct - * source_concept_id and target domain_id. - * - * As of the dateof this script, the following domains contain source_concept_ids that - * map to multiple target_concept_ids: - * - * Condition - * Device - * Drug - * Measurement - * Observation - * Procedure - * Spec Anatomic Site - * - * Also, as of the date which this script was authored, no source_concept_ids map to multiple domains - */ -IF OBJECT_ID('tempdb..#concept_map', 'U') IS NOT NULL - DROP TABLE #concept_map; - -/* / */ - -SELECT concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id AS domain_id -INTO #concept_map -FROM @TARGET_CDMV5_SCHEMA.concept -WHERE 1 = 0; - -/* / */ - -INSERT INTO #concept_map ---standard concepts -SELECT concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id -FROM @TARGET_CDMV5_SCHEMA.concept -WHERE standard_concept = 'S' - AND invalid_reason IS NULL - -UNION - ---concepts with 'map to' standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT concept_id - FROM @TARGET_CDMV5_SCHEMA.concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 -INNER JOIN @TARGET_CDMV5_SCHEMA.concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN @TARGET_CDMV5_SCHEMA.concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with another non 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT A.concept_id - FROM @TARGET_CDMV5_SCHEMA.concept A - LEFT JOIN ( - SELECT DISTINCT c1.concept_id - FROM ( - SELECT concept_id - FROM @TARGET_CDMV5_SCHEMA.concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN @TARGET_CDMV5_SCHEMA.concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN @TARGET_CDMV5_SCHEMA.concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - A.standard_concept <> 'S' - OR A.standard_concept IS NULL - ) - OR A.invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN @TARGET_CDMV5_SCHEMA.concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN @TARGET_CDMV5_SCHEMA.concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT A.concept_id - FROM @TARGET_CDMV5_SCHEMA.concept A - LEFT JOIN ( - SELECT DISTINCT c1.concept_id - FROM ( - SELECT concept_id - FROM @TARGET_CDMV5_SCHEMA.concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN @TARGET_CDMV5_SCHEMA.concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN @TARGET_CDMV5_SCHEMA.concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'Maps to' - ,'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN @TARGET_CDMV5_SCHEMA.concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN @TARGET_CDMV5_SCHEMA.concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Is a') - AND cr1.invalid_reason IS NULL; - -IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL - DROP TABLE #concept_map_distinct; - -/* / */ - -SELECT source_concept_id - ,domain_id - ,COUNT(*) AS targetConceptCount -INTO #concept_map_distinct -FROM #concept_map -WHERE 1 = 0 -GROUP BY source_concept_id - ,domain_id; - -/* / */ - -INSERT INTO #concept_map_distinct -SELECT source_concept_id - ,domain_id - ,COUNT(*) -FROM #concept_map -GROUP BY source_concept_id - ,domain_id; - -IF OBJECT_ID('@TARGET_CDMV5_SCHEMA.ETL_WARNINGS', 'U') IS NOT NULL - DROP TABLE @TARGET_CDMV5_SCHEMA.ETL_WARNINGS; - -/* / */ - -CREATE TABLE @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE VARCHAR(4000)); -/* / */ - -/**** - -CDM_SOURCE - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.cdm_source ( - cdm_source_name - ,cdm_version - ,vocabulary_version - ,cdm_release_date - ) -SELECT '@TARGET_CDMV5' - ,'V5' - ,v.vocabulary_version - ,getDate() -FROM @TARGET_CDMV5_SCHEMA.vocabulary v -WHERE vocabulary_id = 'Vocabulary'; - -/**** - -LOCATION - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.location -SELECT location_id - ,address_1 - ,address_2 - ,city - ,STATE - ,zip - ,county - ,location_source_value -FROM @SOURCE_CDMV4_SCHEMA.LOCATION; - -/**** - -CARE_SITE - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.care_site -SELECT care_site_id - ,cast(NULL AS VARCHAR(255)) AS care_site_name - ,place_of_service_concept_id - ,location_id - ,care_site_source_value - ,place_of_service_source_value -FROM @SOURCE_CDMV4_SCHEMA.CARE_SITE; - -/**** - -Provider - -****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.provider -SELECT provider_id - ,cast(NULL AS VARCHAR(255)) AS provider_name - ,NPI - ,DEA - ,specialty_concept_id - ,care_site_id - ,cast(NULL AS INT) AS year_of_birth - ,cast(NULL AS INT) AS gender_concept_id - ,provider_source_value - ,specialty_source_value - ,0 AS specialty_source_concept_id - ,cast(NULL AS VARCHAR(50)) AS gender_source_value - ,cast(NULL AS INT) AS gender_source_concept_id -FROM @SOURCE_CDMV4_SCHEMA.provider; - -/**** - - PERSON - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.person -SELECT person_id - ,coalesce(gender.target_concept_id, 0) AS gender_concept_id - ,year_of_birth - ,month_of_birth - ,day_of_birth - ,CAST(NULL AS VARCHAR(10)) time_of_birth - ,coalesce(race.target_concept_id, 0) AS race_concept_id - ,coalesce(ethnicity.target_concept_id, 0) AS ethnicity_concept_id - ,location_id - ,provider_id - ,care_site_id - ,person_source_value - ,gender_source_value - ,CAST(NULL AS INT) gender_source_concept_id - ,CAST(NULL AS INT) race_source_value - ,CAST(NULL AS INT) race_source_concept_id - ,ethnicity_source_value - ,CAST(NULL AS INT) ethnicity_source_concept_id -FROM @SOURCE_CDMV4_SCHEMA.PERSON p -LEFT JOIN #concept_map gender ON LOWER(gender.DOMAIN_ID) IN ('gender') - AND p.gender_concept_id = gender.source_concept_id -LEFT JOIN #concept_map race ON LOWER(race.DOMAIN_ID) IN ('race') - AND p.race_concept_id = race.source_concept_id -LEFT JOIN #concept_map ethnicity ON LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') - AND p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.PERSON - WHERE GENDER_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('gender') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.PERSON - WHERE RACE_CONCEPT_ID IS NOT NULL - AND RACE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('race') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.PERSON - WHERE ETHNICITY_CONCEPT_ID IS NOT NULL - AND ETHNICITY_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('ethnicity') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - OBSERVATION_PERIOD - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.observation_period -SELECT observation_period_id - ,person_id - ,observation_period_start_date - ,observation_period_end_date - ,44814722 AS period_type_concept_id -FROM @SOURCE_CDMV4_SCHEMA.OBSERVATION_PERIOD; - -/**** - - DEATH - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.death -SELECT person_id - ,death_date - ,COALESCE(death_type_concept_id, 0) AS death_type_concept_id - ,cause_of_death_concept_id AS cause_concept_id - ,cause_of_death_source_value AS cause_source_value - ,CAST(NULL AS INT) AS cause_source_concept_id -FROM @SOURCE_CDMV4_SCHEMA.DEATH -LEFT JOIN #concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept'); - -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DEATH: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.DEATH - WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - VISIT_OCCURRENCE - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.visit_occurrence -SELECT visit_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS visit_concept_id - ,visit_start_date - ,CAST(NULL AS VARCHAR(10)) visit_start_time - ,visit_end_date - ,CAST(NULL AS VARCHAR(10)) visit_end_time - ,44818517 AS visit_type_concept_id - ,CAST(NULL AS INT) provider_id - ,care_site_id - ,place_of_service_source_value AS visit_source_value - ,CAST(NULL AS INT) visit_source_concept_id -FROM @SOURCE_CDMV4_SCHEMA.VISIT_OCCURRENCE -LEFT JOIN #concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('visit'); - -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'VISIT_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.VISIT_OCCURRENCE - WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('visit') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - PROCEDURE_OCCURRENCE - - ****/ -IF OBJECT_ID('tempdb..#po_map', 'U') IS NOT NULL - DROP TABLE #po_map; - -/* / */ - -SELECT po.procedure_occurrence_id - ,po.person_id - ,po.procedure_concept_id - ,po.procedure_date - ,po.procedure_type_concept_id - ,po.modifier_concept_id - ,po.quantity - ,po.provider_id - ,po.visit_occurrence_id - ,po.procedure_source_value - ,po.procedure_source_concept_id - ,po.qualifier_source_value - ,de.drug_exposure_id AS origional_drug_id -INTO #po_map -FROM @TARGET_CDMV5_SCHEMA.procedure_occurrence po -LEFT JOIN @TARGET_CDMV5_SCHEMA.drug_exposure de ON 1 = 0 -WHERE 0 = 1; - -/* / */ - ---find valid procedures from procedure table -INSERT INTO #po_map -SELECT procedure_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE -INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') -INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -LEFT JOIN #concept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 - -UNION ALL - --- All procedures that did not map to a standard concept in V4 should also carry over to V5 -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE -WHERE procedure_concept_id = 0 - -UNION ALL - --- All PROCEDURE_OCCURRENCE that do not map to a standard concept in V5 should also carry over with procedure_concept_id = 0 -SELECT procedure_occurrence_id - ,person_id - ,0 AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE -LEFT JOIN #concept_map cm1 ON procedure_concept_id = cm1.source_concept_id -LEFT JOIN #concept_map cm2 ON procedure_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -WHERE procedure_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ,origional_drug_id -FROM ( - --find valid procedures from procedure table that map to more than 1 - --target concept in V5 - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') - - UNION ALL - - --find procedures that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,condition_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,condition_occurrence_id AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - - UNION ALL - - --find procedures that were previously classified as drug - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,drug_exposure_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,drug_exposure_id AS origional_drug_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - --find procedures that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,observation_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS - ,( - SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - ) MAXROW; - -INSERT INTO @TARGET_CDMV5_SCHEMA.procedure_occurrence ( - procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ) -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value -FROM #po_map; - ---warnings of invalid records -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - WHERE PROCEDURE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('procedure type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - DRUG_EXPOSURE - - ****/ ---find valid drugs from drug_exposure table -IF OBJECT_ID('tempdb..#drgexp_map', 'U') IS NOT NULL - DROP TABLE #drgexp_map; - -/* / */ - -SELECT de.drug_exposure_id - ,de.person_id - ,de.drug_concept_id - ,de.drug_exposure_start_date - ,de.drug_exposure_end_date - ,de.drug_type_concept_id - ,de.stop_reason - ,de.refills - ,de.quantity - ,de.days_supply - ,de.sig - ,de.route_concept_id - ,de.effective_drug_dose - ,de.dose_unit_concept_id - ,de.lot_number - ,de.provider_id - ,de.visit_occurrence_id - ,de.drug_source_value - ,de.drug_source_concept_id - ,de.route_source_value - ,de.dose_unit_source_value - ,po.procedure_occurrence_id AS origional_procedure_id -INTO #drgexp_map -FROM @TARGET_CDMV5_SCHEMA.drug_exposure de -LEFT JOIN @TARGET_CDMV5_SCHEMA.procedure_occurrence po ON 1 = 0 -WHERE 0 = 1; - -/* / */ - -INSERT INTO #drgexp_map -SELECT drug_exposure_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE -INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') -INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -INNER JOIN #concept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 -WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All drug exposures that did not map to a standard concept in V4 should also carry over to V5 -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE -WHERE drug_concept_id = 0 - -UNION ALL - --- All drug exposures that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT drug_exposure_id - ,person_id - ,0 - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE -LEFT JOIN #concept_map cm1 ON drug_concept_id = cm1.source_concept_id -LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -WHERE drug_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ,origional_procedure_id -FROM ( - --find valid drugs from drug_exposure table that map to > 1 target concept - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') - - UNION ALL - - --find drugs that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,condition_start_date AS drug_exposure_start_date - ,NULL AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,condition_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,condition_occurrence_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as procedure - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,procedure_date AS drug_exposure_start_date - ,CAST(NULL AS DATE) AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,procedure_occurrence_id AS origional_procedure_id - ,procedure_occurrence_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,observation_date AS drug_exposure_start_date - ,CAST(NULL AS DATE) AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,observation_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,observation_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - ) OTHERS - ,( - SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID - FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE - ) MAXROW; - -INSERT INTO @TARGET_CDMV5_SCHEMA.drug_exposure ( - drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ) -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value -FROM #drgexp_map; - ---warnings of invalid records -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE - WHERE DRUG_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE - WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('drug type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - CONDITION_OCCURRENCE - - ****/ ---find valid conditions from condition_occurrence table -INSERT INTO @TARGET_CDMV5_SCHEMA.condition_occurrence -SELECT condition_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE -INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') -INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') -WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All conditions that did not map to a standard concept in V4 should also carry over to V5 -SELECT condition_occurrence_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(condition_type_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE -WHERE condition_concept_id = 0 - -UNION ALL - --- All conditions that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT condition_occurrence_id - ,person_id - ,0 AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE -LEFT JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id -LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') -WHERE condition_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,condition_type_concept_id - ,stop_reason - ,provider_id - ,visit_occurrence_id - ,condition_source_value - ,condition_source_concept_id -FROM ( - --find valid conditions from condition_occurrence table that map to > 1 target concept - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,NULL AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') - WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - - UNION ALL - - --find conditions that were previously classified as procedure - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,procedure_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,procedure_occurrence_id AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as drug - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,drug_exposure_start_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,observation_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,observation_id AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - ) OTHERS - ,( - SELECT MAX(condition_occurrence_id) AS MAXROWID - FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE - ) MAXROW; - ---warnings of invalid records -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDITION_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE - WHERE CONDITION_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO @TARGET_CDMV5_SCHEMA.ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDIITON_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE - WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM @TARGET_CDMV5_SCHEMA.CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('condition type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - DEVICE_EXPOSURE - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.device_exposure -SELECT row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS device_exposure_id - ,person_id - ,device_concept_id - ,device_exposure_start_date - ,device_exposure_end_date - ,device_type_concept_id - ,unique_device_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,device_source_value - ,device_source_concept_id -FROM ( - --find devices that were previously classified as procedures - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,PROCEDURE_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,PROCEDURE_OCCURRENCE_ID AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as drug exposure - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,quantity - ,PRESCRIBING_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,DRUG_EXPOSURE_ID AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as conditions - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,CONDITION_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,CONDITION_OCCURRENCE_ID AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as observations - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,OBSERVATION_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - ) OTHERS; - -/**** - - MEASUREMENT - - ****/ ---find valid measurements from observation table -INSERT INTO @TARGET_CDMV5_SCHEMA.measurement -SELECT row_number() OVER ( - ORDER BY occurrence_id - ) AS measurement_id - ,person_id - ,measurement_concept_id - ,measurement_date - ,measurement_time - ,measurement_type_concept_id - ,operator_concept_id - ,value_as_number - ,value_as_concept_id - ,unit_concept_id - ,range_low - ,range_high - ,provider_id - ,visit_occurrence_id - ,measurement_source_value - ,measurement_source_concept_id - ,unit_source_value - ,value_source_value -FROM ( - --find mesaurements that were previously classified as observations - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,OBSERVATION_DATE AS measurement_date - ,CAST(OBSERVATION_TIME AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) operator_concept_id - ,value_as_number - ,value_as_concept_id - ,COALESCE(cm2.target_concept_id, 0) AS unit_concept_id - ,range_low - ,range_high - ,ASSOCIATED_PROVIDER_ID AS provider_id - ,visit_occurrence_id - ,OBSERVATION_SOURCE_VALUE AS measurement_source_value - ,CAST(NULL AS INT) measurement_source_concept_id - ,unit_source_value AS unit_source_value - ,cast(NULL AS VARCHAR(50)) AS value_source_value - ,observation_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - LEFT JOIN #concept_map cm2 ON observation.unit_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('unit') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,procedure_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,procedure_occurrence_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,condition_start_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,condition_occurrence_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,drug_exposure_start_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,drug_exposure_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.drug_exposure - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - ) OTHERS; - -/**** - - OBSERVATION - - ****/ ---find valid observation from observation table -INSERT INTO @TARGET_CDMV5_SCHEMA.observation -SELECT observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,CAST(observation_time AS VARCHAR(50)) AS observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value -FROM @SOURCE_CDMV4_SCHEMA.OBSERVATION -WHERE observation_concept_id NOT IN ( - SELECT source_concept_id - FROM #concept_map_distinct - WHERE LOWER(domain_id) IN ( - 'condition' - ,'drug' - ,'procedure' - ,'device' - ,'measurement' - ) - ) ---find observations that were previously classified as procedure - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,qualifier_concept_id - ,unit_concept_id - ,provider_id - ,visit_occurrence_id - ,observation_source_value - ,observation_source_concept_id - ,unit_source_value - ,qualifier_source_value -FROM ( - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,procedure_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,procedure_occurrence_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - --find observations that were previously classified as condition - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,condition_start_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,condition_occurrence_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find DRG observations that were previously classified as procedure_cost - SELECT po.person_id - ,cm1.target_concept_id AS observation_concept_id - ,po.procedure_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,po.associated_provider_id AS provider_id - ,po.visit_occurrence_id - ,pc.DISEASE_CLASS_SOURCE_VALUE AS observation_source_value - ,cm1.source_concept_id as observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,po.procedure_occurrence_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_COST pc - INNER JOIN @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE po ON pc.PROCEDURE_OCCURRENCE_ID = po.PROCEDURE_OCCURRENCE_ID - INNER JOIN #concept_map cm1 ON pc.disease_class_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find observations that were previously classified as drug exposure - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,drug_exposure_start_date AS observation_date - ,CAST(NULL AS VARCHAR(10)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,drug_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,drug_exposure_id AS occurrence_id - FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS - ,( - SELECT MAX(OBSERVATION_ID) AS MAXROWID - FROM @SOURCE_CDMV4_SCHEMA.OBSERVATION - ) MAXROW; - -/**** - - PAYER_PLAN_PERIOD - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.payer_plan_period -SELECT payer_plan_period_id - ,person_id - ,payer_plan_period_start_date - ,payer_plan_period_end_date - ,payer_source_value - ,plan_source_value - ,family_source_value -FROM @SOURCE_CDMV4_SCHEMA.PAYER_PLAN_PERIOD; - -/**** - - DRUG_COST - - note : if there were invalid drug concepts in DRUG_EXPOSURE, those records may not enter CDMv5 but costs will persist - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.drug_cost -SELECT drug_cost_id - ,dc.drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id -FROM @SOURCE_CDMV4_SCHEMA.DRUG_COST dc; - --- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO @TARGET_CDMV5_SCHEMA.drug_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_cost_id - ,drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id -FROM ( - SELECT drug_exposure_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS FLOAT) AS ingredient_cost - ,CAST(NULL AS FLOAT) AS dispensing_fee - ,CAST(NULL AS FLOAT) AS average_wholesale_price - ,payer_plan_period_id - ,procedure_cost_id AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_OCCURRENCE po - INNER JOIN @SOURCE_CDMV4_SCHEMA.PROCEDURE_COST pc ON po.procedure_occurrence_id = pc.procedure_occurrence_id - --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - INNER JOIN #drgexp_map de ON de.person_id = po.person_id - AND pc.procedure_occurrence_id = de.origional_procedure_id - ) OTHERS - ,( - SELECT MAX(drug_cost_id) AS MAXROWID - FROM @SOURCE_CDMV4_SCHEMA.DRUG_COST - ) MAXROW; - -/**** - - PROCEDURE_COST - - note : if there were invalid procedure concepts in PROCEDURE_OCCURRENCE, those records may not enter CDMv5 but costs will persist - - - ****/ -INSERT INTO @TARGET_CDMV5_SCHEMA.procedure_cost -SELECT procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value -FROM @SOURCE_CDMV4_SCHEMA.PROCEDURE_COST; - --- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO @TARGET_CDMV5_SCHEMA.procedure_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value -FROM ( - SELECT po.procedure_occurrence_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS FLOAT) AS ingredient_cost - ,CAST(NULL AS FLOAT) AS dispensing_fee - ,CAST(NULL AS FLOAT) AS average_wholesale_price - ,payer_plan_period_id - ,CAST(NULL AS INT) AS revenue_code_concept_id - ,CAST(NULL AS INT) AS revenue_code_source_value - ,drug_cost_id AS OCCURRENCE_ID - FROM @SOURCE_CDMV4_SCHEMA.DRUG_EXPOSURE de - INNER JOIN @SOURCE_CDMV4_SCHEMA.DRUG_COST dc ON de.drug_exposure_id = dc.drug_exposure_id - --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - INNER JOIN #po_map po ON de.person_id = po.person_id - AND de.drug_exposure_id = po.origional_drug_id - ) OTHERS - ,( - SELECT MAX(drug_cost_id) AS MAXROWID - FROM @SOURCE_CDMV4_SCHEMA.DRUG_COST - ) MAXROW; - -/**** - -DRUG ERA -Note: Eras derived from DRUG_EXPOSURE table, using 30d gap - - ****/ -IF OBJECT_ID('tempdb..#cteDrugTarget', 'U') IS NOT NULL - DROP TABLE #cteDrugTarget; - -/* / */ - --- Normalize DRUG_EXPOSURE_END_DATE to either the existing drug exposure end date, or add days supply, or add 1 day to the start date -SELECT d.DRUG_EXPOSURE_ID - ,d.PERSON_ID - ,c.CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE - ,COALESCE(DRUG_EXPOSURE_END_DATE, DATEADD(day, DAYS_SUPPLY, DRUG_EXPOSURE_START_DATE), DATEADD(day, 1, DRUG_EXPOSURE_START_DATE)) AS DRUG_EXPOSURE_END_DATE - ,c.CONCEPT_ID AS INGREDIENT_CONCEPT_ID -INTO #cteDrugTarget -FROM @TARGET_CDMV5_SCHEMA.DRUG_EXPOSURE d -INNER JOIN @TARGET_CDMV5_SCHEMA.CONCEPT_ANCESTOR ca ON ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID -INNER JOIN @TARGET_CDMV5_SCHEMA.CONCEPT c ON ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID -WHERE c.VOCABULARY_ID = 'RxNorm' - AND c.CONCEPT_CLASS_ID = 'Ingredient'; - -/* / */ - -IF OBJECT_ID('tempdb..#cteEndDates', 'U') IS NOT NULL - DROP TABLE #cteEndDates; - -/* / */ - -SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DATEADD(day, - 30, EVENT_DATE) AS END_DATE -- unpad the end date -INTO #cteEndDates -FROM ( - SELECT E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM ( - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM ( - -- select the start dates, assigning a row number to each - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,0 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM #cteDrugTarget - - UNION ALL - - -- add the end dates with NULL as the row number, padding the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DATEADD(day, 30, DRUG_EXPOSURE_END_DATE) - ,1 AS EVENT_TYPE - ,NULL - FROM #cteDrugTarget - ) RAWDATA - ) E1 - INNER JOIN ( - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM #cteDrugTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.INGREDIENT_CONCEPT_ID = E2.INGREDIENT_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E -WHERE 2 * E.START_ORDINAL - E.OVERALL_ORD = 0; - -/* / */ - -IF OBJECT_ID('tempdb..#cteDrugExpEnds', 'U') IS NOT NULL - DROP TABLE #cteDrugExpEnds; - -/* / */ - -SELECT d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE -INTO #cteDrugExpEnds -FROM #cteDrugTarget d -INNER JOIN #cteEndDates e ON d.PERSON_ID = e.PERSON_ID - AND d.INGREDIENT_CONCEPT_ID = e.INGREDIENT_CONCEPT_ID - AND e.END_DATE >= d.DRUG_EXPOSURE_START_DATE -GROUP BY d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE; - -/* / */ - -INSERT INTO @TARGET_CDMV5_SCHEMA.drug_era -SELECT row_number() OVER ( - ORDER BY person_id - ) AS drug_era_id - ,person_id - ,INGREDIENT_CONCEPT_ID - ,min(DRUG_EXPOSURE_START_DATE) AS drug_era_start_date - ,ERA_END_DATE - ,COUNT(*) AS DRUG_EXPOSURE_COUNT - ,30 AS gap_days -FROM #cteDrugExpEnds -GROUP BY person_id - ,INGREDIENT_CONCEPT_ID - ,drug_type_concept_id - ,ERA_END_DATE; - -/**** - -CONDITION ERA -Note: Eras derived from CONDITION_OCCURRENCE table, using 30d gap - - ****/ -IF OBJECT_ID('tempdb..#condition_era_phase_1', 'U') IS NOT NULL - DROP TABLE #condition_era_phase_1; - -/* / */ - -IF OBJECT_ID('tempdb..#cteConditionTarget', 'U') IS NOT NULL - DROP TABLE #cteConditionTarget; - -/* / */ - --- create base eras from the concepts found in condition_occurrence -SELECT co.PERSON_ID - ,co.condition_concept_id - ,co.CONDITION_START_DATE - ,COALESCE(co.CONDITION_END_DATE, DATEADD(day, 1, CONDITION_START_DATE)) AS CONDITION_END_DATE -INTO #cteConditionTarget -FROM @TARGET_CDMV5_SCHEMA.CONDITION_OCCURRENCE co; - -/* / */ - -IF OBJECT_ID('tempdb..#cteCondEndDates', 'U') IS NOT NULL - DROP TABLE #cteCondEndDates; - -/* / */ - -SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,DATEADD(day, - 30, EVENT_DATE) AS END_DATE -- unpad the end date -INTO #cteCondEndDates -FROM ( - SELECT E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM ( - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM ( - -- select the start dates, assigning a row number to each - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,- 1 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM #cteConditionTarget - - UNION ALL - - -- pad the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,DATEADD(day, 30, CONDITION_END_DATE) - ,1 AS EVENT_TYPE - ,NULL - FROM #cteConditionTarget - ) RAWDATA - ) E1 - INNER JOIN ( - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM #cteConditionTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.CONDITION_CONCEPT_ID = E2.CONDITION_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E -WHERE (2 * E.START_ORDINAL) - E.OVERALL_ORD = 0; - -/* / */ - -IF OBJECT_ID('tempdb..#cteConditionEnds', 'U') IS NOT NULL - DROP TABLE #cteConditionEnds; - -/* / */ - -SELECT c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE -INTO #cteConditionEnds -FROM #cteConditionTarget c -INNER JOIN #cteCondEndDates e ON c.PERSON_ID = e.PERSON_ID - AND c.CONDITION_CONCEPT_ID = e.CONDITION_CONCEPT_ID - AND e.END_DATE >= c.CONDITION_START_DATE -GROUP BY c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE; - -/* / */ - -INSERT INTO @TARGET_CDMV5_SCHEMA.condition_era ( - condition_era_id - ,person_id - ,condition_concept_id - ,condition_era_start_date - ,condition_era_end_date - ,condition_occurrence_count - ) -SELECT row_number() OVER ( - ORDER BY person_id - ) AS condition_era_id - ,person_id - ,CONDITION_CONCEPT_ID - ,min(CONDITION_START_DATE) AS CONDITION_ERA_START_DATE - ,ERA_END_DATE AS CONDITION_ERA_END_DATE - ,COUNT(*) AS CONDITION_OCCURRENCE_COUNT -FROM #cteConditionEnds -GROUP BY person_id - ,CONDITION_CONCEPT_ID - ,ERA_END_DATE; - -/**** - -QUALITY ASSURANCE OUTPUT - -Note: These queries are used to provide some basic stats around row counts between your V4 and V5 database - to ensure that all of the data has migrated as expected. - - ****/ - -IF OBJECT_ID('tempdb..#v5_stats', 'U') IS NOT NULL - DROP TABLE #v5_stats; - -/* / */ - -IF OBJECT_ID('tempdb..#v4_stats', 'U') IS NOT NULL - DROP TABLE #v4_stats; - -/* / */ - --- Get the row counts for each table that is in scope for the migration -SELECT * -INTO #v4_stats -FROM -( - SELECT '@SOURCE_CDMV4' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.care_site - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.condition_era - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.condition_occurrence - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.death - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.drug_cost - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.drug_era - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.drug_exposure - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.location - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.observation - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.observation_period - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.payer_plan_period - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.person - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.procedure_cost - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.procedure_occurrence - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.provider - UNION - SELECT '@SOURCE_CDMV4' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM @SOURCE_CDMV4_SCHEMA.visit_occurrence -) v4_stats; - -/* / */ - -SELECT * -INTO #v5_stats -FROM -( - SELECT '@TARGET_CDMV5' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.care_site - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.condition_era - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.condition_occurrence - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.death - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'device_exposure' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.device_exposure - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.drug_cost - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.drug_era - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.drug_exposure - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.location - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'measurement' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.measurement - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.observation - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.observation_period - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.payer_plan_period - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.person - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.procedure_cost - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.procedure_occurrence - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.provider - UNION - SELECT '@TARGET_CDMV5' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM @TARGET_CDMV5_SCHEMA.visit_occurrence -) v5_stats; - -/* / */ - --- Show the results -select - 'Rowcounts for each database and table', - ISNULL(V4.DBName, 'None') v4_database_name, - v4.TableName v4_table_name, - v4.row_count v4_row_count, - ISNULL(v5.DBName, 'None') v5_database_name, - v5.TableName v5_table_name, - v5.row_count v5_row_count, - ISNULL(v5.row_count, 0) - ISNULL(v4.row_count, 0) row_count_change -from #v4_stats v4 -full outer join #v5_stats v5 ON v4.TableName = v5.TableName -order by v5.TableName; - -/* - * Determine how the vocabulary/domains helped to map from the V4 source - * tables to the V5 destinations - */ -IF OBJECT_ID('tempdb..#classification_map', 'U') IS NOT NULL - DROP TABLE #classification_map; - -/* / */ - -SELECT * -INTO #classification_map -FROM -( - SELECT 'Condition_Occurrence' TableName, ISNULL(LOWER(cm.domain_id), 'condition') domain_id, COUNT(*) row_count - FROM @SOURCE_CDMV4_SCHEMA.Condition_Occurrence CO - LEFT JOIN #concept_map CM ON co.condition_concept_id = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'condition') - UNION - SELECT 'Drug_Exposure' TableName, ISNULL(LOWER(cm.domain_id), 'drug') domain_id, COUNT(*) row_count - FROM @SOURCE_CDMV4_SCHEMA.Drug_Exposure de - LEFT JOIN #concept_map CM ON de.drug_concept_id = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'drug') - UNION - SELECT 'Observation' TableName, ISNULL(LOWER(cm.domain_id), 'observation') domain_id, COUNT(*) row_count - FROM @SOURCE_CDMV4_SCHEMA.Observation o - LEFT JOIN #concept_map CM ON o.observation_concept_id = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'observation') - UNION - SELECT 'Procedure_Occurrence' TableName, ISNULL(LOWER(cm.domain_id), 'procedure') domain_id, COUNT(*) row_count - FROM @SOURCE_CDMV4_SCHEMA.Procedure_Occurrence po - LEFT JOIN #concept_map CM ON po.PROCEDURE_CONCEPT_ID = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'procedure') -) A -ORDER by A.TableName, A.domain_id; - -/* / */ - -select * -from #classification_map -order by tablename, domain_id; - -select domain_id, SUM(row_count) -from #classification_map -group by domain_id -order by domain_id; \ No newline at end of file diff --git a/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql b/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql deleted file mode 100644 index 4825f69..0000000 --- a/Version4 To Version5 Conversion/Oracle/OMOP CDMv4 to CDMv5 - Oracle.sql +++ /dev/null @@ -1,2452 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -********************************************************************************/ -/******************************************************************************* - -PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. - -last revised: 04 August 2015 -authors: Patrick Ryan, Chris Knoll, Anthony Sena - -!!!!!!!!!!!!!!!!!!!!! PLEASE READ THESE INSTRUCTIONS !!!!!!!!!!!!!!!!!!!!!!!!!!! - -This script was authored using OHDSI-SQL which will require you to run this -script through SqlRender to creat a version that is compatible with your target -RDBMS. We have pre-generated these scripts using SQL Render and have placed -them in folders for each RDBMS. Depending on which script you are viewing, your -instructions will be slightly different. - -General Assumptions -------------------- - -This script assumes that your V4 and V5 database are located on the same -RDBMS server. It also assumes that the V4 and V5 databases were created -using the standard data definition scripts for these databases. If you -altered your V4 database in any way, this script will likely require -some mo - -Getting Started ---------------- - -Before you can use this script, there are some prerequisites: - - 1. Create a target CDMv5 database on your database server using the - appropriate script from https://github.com/OHDSI/CommonDataModel - 2. Load VocabV5 into the target database/schema that will contain CDMv5 using - Athena: http://ohdsi.org/web/ATHENA - -OHDSI-SQL File Instructions ------------------------------ - - 1. Set parameter name of schema that contains CDMv4 instance - ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) - 2. Set parameter name of schema that contains CDMv5 instance - ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) - 3. Run this script through SqlRender to produce a script that will work in your - source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender - 4. Run the script produced by SQL Render on your target RDBDMS. - - File Instructions -------------------------- - - 1. This script will hold a number of placeholders for your CDM V4 and CDMV5 - database/schema. In order to make this file work in your environment, you - should plan to do a global "FIND AND REPLACE" on this file to fill in the - file with values that pertain to your environment. The following are the - tokens you should use when doing your "FIND AND REPLACE" operation: - - a. [SOURCE_CDMV4] - b. [SOURCE_CDMV4].[SCHEMA] - c. [TARGET_CDMV5] - d. [TARGET_CDMV5].[SCHEMA] - - 2. Run the resulting script on your target RDBDMS. - -*********************************************************************************/ -/* SCRIPT PARAMETERS */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -ALTER SESSION SET current_schema = [TARGET_CDMV5]; - -/* - * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids - * and their respective domain_ids. As a result, the results of this table will have - * some source_concept_ids that map to multiple target_concept_ids which is expected. - * - * In some of the conversion queries below, we will use the distinct values from the - * #concept_map table since our need in those instances is to understand the domain_id - * that will provide the destination table in the target V5 database. To make the code - * easier to read, we dervied a #concept_map_distinct table that holds the distinct - * source_concept_id and target domain_id. - * - * As of the dateof this script, the following domains contain source_concept_ids that - * map to multiple target_concept_ids: - * - * Condition - * Device - * Drug - * Measurement - * Observation - * Procedure - * Spec Anatomic Site - * - * Also, as of the date which this script was authored, no source_concept_ids map to multiple domains - */ -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmconcept_map'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmconcept_map'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -CREATE TABLE g8iauhqmconcept_map - AS -SELECT - concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id AS domain_id - -FROM - [TARGET_CDMV5].[SCHEMA].concept - WHERE 1 = 0 ; - -/* / */ - -INSERT INTO g8iauhqmconcept_map ---standard concepts -SELECT concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE standard_concept = 'S' - AND invalid_reason IS NULL - -UNION - ---concepts with 'map to' standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id - FROM (SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with another non 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id - FROM (SELECT A.concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept A - LEFT JOIN (SELECT DISTINCT c1.concept_id - FROM (SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - A.standard_concept <> 'S' - OR A.standard_concept IS NULL - ) - OR A.invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id - FROM (SELECT A.concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept A - LEFT JOIN (SELECT DISTINCT c1.concept_id - FROM (SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'Maps to' - ,'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Is a') - AND cr1.invalid_reason IS NULL ; - -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmconcept_map_distinct'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmconcept_map_distinct'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -CREATE TABLE g8iauhqmconcept_map_distinct - AS -SELECT - source_concept_id - ,domain_id - ,COUNT(*) AS targetConceptCount - -FROM - g8iauhqmconcept_map - WHERE 1 = 0 -GROUP BY source_concept_id - ,domain_id ; - -/* / */ - -INSERT INTO g8iauhqmconcept_map_distinct -SELECT source_concept_id - ,domain_id - ,COUNT(*) - FROM g8iauhqmconcept_map -GROUP BY source_concept_id - ,domain_id ; - -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS'; - EXECUTE IMMEDIATE 'DROP TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE VARCHAR(4000)); -/* / */ - -/**** - -CDM_SOURCE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source ( - cdm_source_name - ,cdm_version - ,vocabulary_version - ,cdm_release_date - ) -SELECT '[TARGET_CDMV5]' - ,'V5' - ,v.vocabulary_version - ,SYSDATE - FROM [TARGET_CDMV5].[SCHEMA].vocabulary v - WHERE vocabulary_id = 'Vocabulary' ; - -/**** - -LOCATION - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].location -SELECT location_id - ,address_1 - ,address_2 - ,city - ,STATE - ,zip - ,county - ,location_source_value - FROM [SOURCE_CDMV4].[SCHEMA].LOCATION ; - -/**** - -CARE_SITE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].care_site -SELECT care_site_id - ,TO_CHAR(NULL ) AS care_site_name - ,place_of_service_concept_id - ,location_id - ,care_site_source_value - ,place_of_service_source_value - FROM [SOURCE_CDMV4].[SCHEMA].CARE_SITE ; - -/**** - -Provider - -****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].provider -SELECT provider_id - ,TO_CHAR(NULL ) AS provider_name - ,NPI - ,DEA - ,specialty_concept_id - ,care_site_id - ,cast(NULL AS INT) AS year_of_birth - ,cast(NULL AS INT) AS gender_concept_id - ,provider_source_value - ,specialty_source_value - ,0 AS specialty_source_concept_id - ,TO_CHAR(NULL ) AS gender_source_value - ,cast(NULL AS INT) AS gender_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].provider ; - -/**** - - PERSON - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].person -SELECT person_id - ,coalesce(gender.target_concept_id, 0) AS gender_concept_id - ,year_of_birth - ,month_of_birth - ,day_of_birth - ,TO_CHAR(NULL ) time_of_birth - ,coalesce(race.target_concept_id, 0) AS race_concept_id - ,coalesce(ethnicity.target_concept_id, 0) AS ethnicity_concept_id - ,location_id - ,provider_id - ,care_site_id - ,person_source_value - ,gender_source_value - ,CAST(NULL AS INT) gender_source_concept_id - ,CAST(NULL AS INT) race_source_value - ,CAST(NULL AS INT) race_source_concept_id - ,ethnicity_source_value - ,CAST(NULL AS INT) ethnicity_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].PERSON p -LEFT JOIN g8iauhqmconcept_map gender ON LOWER(gender.DOMAIN_ID) IN ('gender') - AND p.gender_concept_id = gender.source_concept_id -LEFT JOIN g8iauhqmconcept_map race ON LOWER(race.DOMAIN_ID) IN ('race') - AND p.race_concept_id = race.source_concept_id -LEFT JOIN g8iauhqmconcept_map ethnicity ON LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') - AND p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id ; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE GENDER_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('gender') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE RACE_CONCEPT_ID IS NOT NULL - AND RACE_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('race') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE ETHNICITY_CONCEPT_ID IS NOT NULL - AND ETHNICITY_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('ethnicity') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -/**** - - OBSERVATION_PERIOD - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period -SELECT observation_period_id - ,person_id - ,observation_period_start_date - ,observation_period_end_date - ,44814722 AS period_type_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD ; - -/**** - - DEATH - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].death -SELECT person_id - ,death_date - ,COALESCE(death_type_concept_id, 0) AS death_type_concept_id - ,cause_of_death_concept_id AS cause_concept_id - ,cause_of_death_source_value AS cause_source_value - ,CAST(NULL AS INT) AS cause_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].DEATH -LEFT JOIN g8iauhqmconcept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept') ; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DEATH: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DEATH - WHERE DEATH_TYPE_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -/**** - - VISIT_OCCURRENCE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence -SELECT visit_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS visit_concept_id - ,visit_start_date - ,TO_CHAR(NULL ) visit_start_time - ,visit_end_date - ,TO_CHAR(NULL ) visit_end_time - ,44818517 AS visit_type_concept_id - ,CAST(NULL AS INT) provider_id - ,care_site_id - ,place_of_service_source_value AS visit_source_value - ,CAST(NULL AS INT) visit_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE -LEFT JOIN g8iauhqmconcept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('visit') ; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'VISIT_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE - WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('visit') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -/**** - - PROCEDURE_OCCURRENCE - - ****/ -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmpo_map'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmpo_map'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -CREATE TABLE g8iauhqmpo_map - AS -SELECT - po.procedure_occurrence_id - ,po.person_id - ,po.procedure_concept_id - ,po.procedure_date - ,po.procedure_type_concept_id - ,po.modifier_concept_id - ,po.quantity - ,po.provider_id - ,po.visit_occurrence_id - ,po.procedure_source_value - ,po.procedure_source_concept_id - ,po.qualifier_source_value - ,de.drug_exposure_id AS origional_drug_id - -FROM - [TARGET_CDMV5].[SCHEMA].procedure_occurrence po -LEFT JOIN [TARGET_CDMV5].[SCHEMA].drug_exposure de ON 1 = 0 - WHERE 0 = 1 ; - -/* / */ - ---find valid procedures from procedure table -INSERT INTO g8iauhqmpo_map -SELECT procedure_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -INNER JOIN g8iauhqmconcept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') -INNER JOIN g8iauhqmconcept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN g8iauhqmconcept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -LEFT JOIN g8iauhqmconcept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 - -UNION ALL - --- All procedures that did not map to a standard concept in V4 should also carry over to V5 -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE procedure_concept_id = 0 - -UNION ALL - --- All PROCEDURE_OCCURRENCE that do not map to a standard concept in V5 should also carry over with procedure_concept_id = 0 -SELECT procedure_occurrence_id - ,person_id - ,0 AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -LEFT JOIN g8iauhqmconcept_map cm1 ON procedure_concept_id = cm1.source_concept_id -LEFT JOIN g8iauhqmconcept_map cm2 ON procedure_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') - WHERE procedure_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ,origional_drug_id - FROM (SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - INNER JOIN g8iauhqmconcept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN g8iauhqmconcept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') - - UNION ALL - - --find procedures that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,condition_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,condition_occurrence_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - - UNION ALL - - --find procedures that were previously classified as drug - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,drug_exposure_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,drug_exposure_id AS origional_drug_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN g8iauhqmconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - --find procedures that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,observation_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN g8iauhqmconcept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS - ,(SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - ) MAXROW ; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence ( - procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ) -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - FROM g8iauhqmpo_map ; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE PROCEDURE_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('procedure type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -/**** - - DRUG_EXPOSURE - - ****/ ---find valid drugs from drug_exposure table -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmdrgexp_map'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmdrgexp_map'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -CREATE TABLE g8iauhqmdrgexp_map - AS -SELECT - de.drug_exposure_id - ,de.person_id - ,de.drug_concept_id - ,de.drug_exposure_start_date - ,de.drug_exposure_end_date - ,de.drug_type_concept_id - ,de.stop_reason - ,de.refills - ,de.quantity - ,de.days_supply - ,de.sig - ,de.route_concept_id - ,de.effective_drug_dose - ,de.dose_unit_concept_id - ,de.lot_number - ,de.provider_id - ,de.visit_occurrence_id - ,de.drug_source_value - ,de.drug_source_concept_id - ,de.route_source_value - ,de.dose_unit_source_value - ,po.procedure_occurrence_id AS origional_procedure_id - -FROM - [TARGET_CDMV5].[SCHEMA].drug_exposure de -LEFT JOIN [TARGET_CDMV5].[SCHEMA].procedure_occurrence po ON 1 = 0 - WHERE 0 = 1 ; - -/* / */ - -INSERT INTO g8iauhqmdrgexp_map -SELECT drug_exposure_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -INNER JOIN g8iauhqmconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') -INNER JOIN g8iauhqmconcept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN g8iauhqmconcept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -INNER JOIN g8iauhqmconcept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 - WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All drug exposures that did not map to a standard concept in V4 should also carry over to V5 -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE drug_concept_id = 0 - -UNION ALL - --- All drug exposures that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT drug_exposure_id - ,person_id - ,0 - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -LEFT JOIN g8iauhqmconcept_map cm1 ON drug_concept_id = cm1.source_concept_id -LEFT JOIN g8iauhqmconcept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') - WHERE drug_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ,origional_procedure_id - FROM (SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN g8iauhqmconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - INNER JOIN g8iauhqmconcept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN g8iauhqmconcept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') - - UNION ALL - - --find drugs that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,condition_start_date AS drug_exposure_start_date - ,NULL AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,condition_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as procedure - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,procedure_date AS drug_exposure_start_date - ,TO_DATE(NULL , 'yyyymmdd') AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,procedure_occurrence_id AS origional_procedure_id - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,observation_date AS drug_exposure_start_date - ,TO_DATE(NULL , 'yyyymmdd') AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,observation_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,observation_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN g8iauhqmconcept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - ) OTHERS - ,(SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - ) MAXROW ; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure ( - drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ) -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - FROM g8iauhqmdrgexp_map ; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE DRUG_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE DRUG_TYPE_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('drug type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -/**** - - CONDITION_OCCURRENCE - - ****/ ---find valid conditions from condition_occurrence table -INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence -SELECT condition_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -INNER JOIN g8iauhqmconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') -INNER JOIN g8iauhqmconcept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN g8iauhqmconcept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') - WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All conditions that did not map to a standard concept in V4 should also carry over to V5 -SELECT condition_occurrence_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(condition_type_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE condition_concept_id = 0 - -UNION ALL - --- All conditions that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT condition_occurrence_id - ,person_id - ,0 AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -LEFT JOIN g8iauhqmconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id -LEFT JOIN g8iauhqmconcept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') - WHERE condition_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,condition_type_concept_id - ,stop_reason - ,provider_id - ,visit_occurrence_id - ,condition_source_value - ,condition_source_concept_id - FROM (SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,NULL AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - INNER JOIN g8iauhqmconcept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN g8iauhqmconcept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') - WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - - UNION ALL - - --find conditions that were previously classified as procedure - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,procedure_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,procedure_occurrence_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as drug - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,drug_exposure_start_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN g8iauhqmconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,observation_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,observation_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN g8iauhqmconcept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - ) OTHERS - ,(SELECT MAX(condition_occurrence_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - ) MAXROW ; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDITION_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE CONDITION_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDIITON_OCCURRENCE: ' || TO_CHAR(NUM_INVALID_RECORDS ) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' - FROM (SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE CONDITION_TYPE_CONCEPT_ID NOT IN (SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('condition type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn ; - -/**** - - DEVICE_EXPOSURE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure -SELECT row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS device_exposure_id - ,person_id - ,device_concept_id - ,device_exposure_start_date - ,device_exposure_end_date - ,device_type_concept_id - ,unique_device_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,device_source_value - ,device_source_concept_id - FROM (SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,PROCEDURE_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,TO_CHAR(NULL ) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,PROCEDURE_OCCURRENCE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as drug exposure - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,TO_CHAR(NULL ) unique_device_id - ,quantity - ,PRESCRIBING_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,DRUG_EXPOSURE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN g8iauhqmconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as conditions - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,CONDITION_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,TO_CHAR(NULL ) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,CONDITION_OCCURRENCE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as observations - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,OBSERVATION_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,TO_CHAR(NULL ) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN g8iauhqmconcept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - ) OTHERS ; - -/**** - - MEASUREMENT - - ****/ ---find valid measurements from observation table -INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement -SELECT row_number() OVER ( - ORDER BY occurrence_id - ) AS measurement_id - ,person_id - ,measurement_concept_id - ,measurement_date - ,measurement_time - ,measurement_type_concept_id - ,operator_concept_id - ,value_as_number - ,value_as_concept_id - ,unit_concept_id - ,range_low - ,range_high - ,provider_id - ,visit_occurrence_id - ,measurement_source_value - ,measurement_source_concept_id - ,unit_source_value - ,value_source_value - FROM (SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,OBSERVATION_DATE AS measurement_date - ,TO_CHAR(OBSERVATION_TIME ) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) operator_concept_id - ,value_as_number - ,value_as_concept_id - ,COALESCE(cm2.target_concept_id, 0) AS unit_concept_id - ,range_low - ,range_high - ,ASSOCIATED_PROVIDER_ID AS provider_id - ,visit_occurrence_id - ,OBSERVATION_SOURCE_VALUE AS measurement_source_value - ,CAST(NULL AS INT) measurement_source_concept_id - ,unit_source_value AS unit_source_value - ,TO_CHAR(NULL ) AS value_source_value - ,observation_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN g8iauhqmconcept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - LEFT JOIN g8iauhqmconcept_map cm2 ON observation.unit_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('unit') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,procedure_date AS measurement_date - ,TO_CHAR(NULL ) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,TO_CHAR(NULL ) AS unit_source_value - ,TO_CHAR(NULL ) AS value_source_value - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,condition_start_date AS measurement_date - ,TO_CHAR(NULL ) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,TO_CHAR(NULL ) AS unit_source_value - ,TO_CHAR(NULL ) AS value_source_value - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,drug_exposure_start_date AS measurement_date - ,TO_CHAR(NULL ) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,TO_CHAR(NULL ) AS unit_source_value - ,TO_CHAR(NULL ) AS value_source_value - ,drug_exposure_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure - INNER JOIN g8iauhqmconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - ) OTHERS ; - -/**** - - OBSERVATION - - ****/ ---find valid observation from observation table -INSERT INTO [TARGET_CDMV5].[SCHEMA].observation -SELECT observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,TO_CHAR(observation_time ) AS observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,unit_source_value - ,TO_CHAR(NULL ) qualifier_source_value - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - WHERE observation_concept_id NOT IN (SELECT source_concept_id - FROM g8iauhqmconcept_map_distinct - WHERE LOWER(domain_id) IN ( - 'condition' - ,'drug' - ,'procedure' - ,'device' - ,'measurement' - ) - ) ---find observations that were previously classified as procedure - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,qualifier_concept_id - ,unit_concept_id - ,provider_id - ,visit_occurrence_id - ,observation_source_value - ,observation_source_concept_id - ,unit_source_value - ,qualifier_source_value - FROM (SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,procedure_date AS observation_date - ,TO_CHAR(NULL ) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,TO_CHAR(NULL ) qualifier_source_value - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - --find observations that were previously classified as condition - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,condition_start_date AS observation_date - ,TO_CHAR(NULL ) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,TO_CHAR(NULL ) qualifier_source_value - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN g8iauhqmconcept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find DRG observations that were previously classified as procedure_cost - SELECT po.person_id - ,cm1.target_concept_id AS observation_concept_id - ,po.procedure_date AS observation_date - ,TO_CHAR(NULL ) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,po.associated_provider_id AS provider_id - ,po.visit_occurrence_id - ,pc.DISEASE_CLASS_SOURCE_VALUE AS observation_source_value - ,cm1.source_concept_id as observation_source_concept_id - ,NULL AS unit_source_value - ,TO_CHAR(NULL ) qualifier_source_value - ,po.procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc - INNER JOIN [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po ON pc.PROCEDURE_OCCURRENCE_ID = po.PROCEDURE_OCCURRENCE_ID - INNER JOIN g8iauhqmconcept_map cm1 ON pc.disease_class_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find observations that were previously classified as drug exposure - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,drug_exposure_start_date AS observation_date - ,TO_CHAR(NULL ) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,drug_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,TO_CHAR(NULL ) qualifier_source_value - ,drug_exposure_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN g8iauhqmconcept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS - ,(SELECT MAX(OBSERVATION_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - ) MAXROW ; - -/**** - - PAYER_PLAN_PERIOD - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period -SELECT payer_plan_period_id - ,person_id - ,payer_plan_period_start_date - ,payer_plan_period_end_date - ,payer_source_value - ,plan_source_value - ,family_source_value - FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD ; - -/**** - - DRUG_COST - - note : if there were invalid drug concepts in DRUG_EXPOSURE, those records may not enter CDMv5 but costs will persist - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost -SELECT drug_cost_id - ,dc.drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ; - --- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_cost_id - ,drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id - FROM (SELECT drug_exposure_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS FLOAT) AS ingredient_cost - ,CAST(NULL AS FLOAT) AS dispensing_fee - ,CAST(NULL AS FLOAT) AS average_wholesale_price - ,payer_plan_period_id - ,procedure_cost_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po - INNER JOIN [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc ON po.procedure_occurrence_id = pc.procedure_occurrence_id - --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - INNER JOIN g8iauhqmdrgexp_map de ON de.person_id = po.person_id - AND pc.procedure_occurrence_id = de.origional_procedure_id - ) OTHERS - ,(SELECT MAX(drug_cost_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST - ) MAXROW ; - -/**** - - PROCEDURE_COST - - note : if there were invalid procedure concepts in PROCEDURE_OCCURRENCE, those records may not enter CDMv5 but costs will persist - - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost -SELECT procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST ; - --- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value - FROM (SELECT po.procedure_occurrence_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS FLOAT) AS ingredient_cost - ,CAST(NULL AS FLOAT) AS dispensing_fee - ,CAST(NULL AS FLOAT) AS average_wholesale_price - ,payer_plan_period_id - ,CAST(NULL AS INT) AS revenue_code_concept_id - ,CAST(NULL AS INT) AS revenue_code_source_value - ,drug_cost_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de - INNER JOIN [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ON de.drug_exposure_id = dc.drug_exposure_id - --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - INNER JOIN g8iauhqmpo_map po ON de.person_id = po.person_id - AND de.drug_exposure_id = po.origional_drug_id - ) OTHERS - ,(SELECT MAX(drug_cost_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST - ) MAXROW ; - -/**** - -DRUG ERA -Note: Eras derived from DRUG_EXPOSURE table, using 30d gap - - ****/ -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmcteDrugTarget'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmcteDrugTarget'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - --- Normalize DRUG_EXPOSURE_END_DATE to either the existing drug exposure end date, or add days supply, or add 1 day to the start date -CREATE TABLE g8iauhqmcteDrugTarget - AS -SELECT - d.DRUG_EXPOSURE_ID - ,d.PERSON_ID - ,c.CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE - ,COALESCE(DRUG_EXPOSURE_END_DATE, ( DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), ( DRUG_EXPOSURE_START_DATE + 1)) AS DRUG_EXPOSURE_END_DATE - ,c.CONCEPT_ID AS INGREDIENT_CONCEPT_ID - -FROM - [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d -INNER JOIN [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca ON ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID -INNER JOIN [TARGET_CDMV5].[SCHEMA].CONCEPT c ON ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID - WHERE c.VOCABULARY_ID = 'RxNorm' - AND c.CONCEPT_CLASS_ID = 'Ingredient' ; - -/* / */ - -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmcteEndDates'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmcteEndDates'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -CREATE TABLE g8iauhqmcteEndDates - AS -SELECT - PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,( EVENT_DATE + - 30) AS END_DATE -- unpad the end date - -FROM - (SELECT E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM (SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM (SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,0 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM g8iauhqmcteDrugTarget - - UNION ALL - - -- add the end dates with NULL as the row number, padding the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,( DRUG_EXPOSURE_END_DATE + 30) - ,1 AS EVENT_TYPE - ,NULL - FROM g8iauhqmcteDrugTarget - ) RAWDATA - ) E1 - INNER JOIN (SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM g8iauhqmcteDrugTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.INGREDIENT_CONCEPT_ID = E2.INGREDIENT_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E - WHERE 2 * E.START_ORDINAL - E.OVERALL_ORD = 0 ; - -/* / */ - -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmcteDrugExpEnds'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmcteDrugExpEnds'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -CREATE TABLE g8iauhqmcteDrugExpEnds - AS -SELECT - d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE - -FROM - g8iauhqmcteDrugTarget d -INNER JOIN g8iauhqmcteEndDates e ON d.PERSON_ID = e.PERSON_ID - AND d.INGREDIENT_CONCEPT_ID = e.INGREDIENT_CONCEPT_ID - AND e.END_DATE >= d.DRUG_EXPOSURE_START_DATE -GROUP BY d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE ; - -/* / */ - -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_era -SELECT row_number() OVER ( - ORDER BY person_id - ) AS drug_era_id - ,person_id - ,INGREDIENT_CONCEPT_ID - ,min(DRUG_EXPOSURE_START_DATE) AS drug_era_start_date - ,ERA_END_DATE - ,COUNT(*) AS DRUG_EXPOSURE_COUNT - ,30 AS gap_days - FROM g8iauhqmcteDrugExpEnds -GROUP BY person_id - ,INGREDIENT_CONCEPT_ID - ,drug_type_concept_id - ,ERA_END_DATE ; - -/**** - -CONDITION ERA -Note: Eras derived from CONDITION_OCCURRENCE table, using 30d gap - - ****/ -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmcondition_era_phase_1'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmcondition_era_phase_1'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmcteConditionTarget'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmcteConditionTarget'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - --- create base eras from the concepts found in condition_occurrence -CREATE TABLE g8iauhqmcteConditionTarget - AS -SELECT - co.PERSON_ID - ,co.condition_concept_id - ,co.CONDITION_START_DATE - ,COALESCE(co.CONDITION_END_DATE, ( CONDITION_START_DATE + 1)) AS CONDITION_END_DATE - -FROM - [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co ; - -/* / */ - -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmcteCondEndDates'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmcteCondEndDates'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -CREATE TABLE g8iauhqmcteCondEndDates - AS -SELECT - PERSON_ID - ,CONDITION_CONCEPT_ID - ,( EVENT_DATE + - 30) AS END_DATE -- unpad the end date - -FROM - (SELECT E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM (SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM (SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,- 1 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM g8iauhqmcteConditionTarget - - UNION ALL - - -- pad the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,( CONDITION_END_DATE + 30) - ,1 AS EVENT_TYPE - ,NULL - FROM g8iauhqmcteConditionTarget - ) RAWDATA - ) E1 - INNER JOIN (SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM g8iauhqmcteConditionTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.CONDITION_CONCEPT_ID = E2.CONDITION_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E - WHERE (2 * E.START_ORDINAL) - E.OVERALL_ORD = 0 ; - -/* / */ - -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmcteConditionEnds'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmcteConditionEnds'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -CREATE TABLE g8iauhqmcteConditionEnds - AS -SELECT - c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE - -FROM - g8iauhqmcteConditionTarget c -INNER JOIN g8iauhqmcteCondEndDates e ON c.PERSON_ID = e.PERSON_ID - AND c.CONDITION_CONCEPT_ID = e.CONDITION_CONCEPT_ID - AND e.END_DATE >= c.CONDITION_START_DATE -GROUP BY c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE ; - -/* / */ - -INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era ( - condition_era_id - ,person_id - ,condition_concept_id - ,condition_era_start_date - ,condition_era_end_date - ,condition_occurrence_count - ) -SELECT row_number() OVER ( - ORDER BY person_id - ) AS condition_era_id - ,person_id - ,CONDITION_CONCEPT_ID - ,min(CONDITION_START_DATE) AS CONDITION_ERA_START_DATE - ,ERA_END_DATE AS CONDITION_ERA_END_DATE - ,COUNT(*) AS CONDITION_OCCURRENCE_COUNT - FROM g8iauhqmcteConditionEnds -GROUP BY person_id - ,CONDITION_CONCEPT_ID - ,ERA_END_DATE ; - -/**** - -QUALITY ASSURANCE OUTPUT - -Note: These queries are used to provide some basic stats around row counts between your V4 and V5 database - to ensure that all of the data has migrated as expected. - - ****/ - -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmv5_stats'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmv5_stats'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmv4_stats'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmv4_stats'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - --- Get the row counts for each table that is in scope for the migration -CREATE TABLE g8iauhqmv4_stats - AS -SELECT - * - -FROM - -(SELECT '[SOURCE_CDMV4]' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].care_site - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].condition_era - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].condition_occurrence - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].death - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_cost - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_era - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].location - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].observation - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].observation_period - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].payer_plan_period - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].person - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].procedure_cost - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].procedure_occurrence - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].provider - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].visit_occurrence - ) v4_stats ; - -/* / */ - -CREATE TABLE g8iauhqmv5_stats - AS -SELECT - * - -FROM - -(SELECT '[TARGET_CDMV5]' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].care_site - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].condition_era - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].condition_occurrence - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].death - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'device_exposure' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].device_exposure - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_cost - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_era - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_exposure - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].location - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'measurement' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].measurement - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].observation - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].observation_period - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].payer_plan_period - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].person - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].procedure_cost - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].procedure_occurrence - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].provider - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].visit_occurrence - ) v5_stats ; - -/* / */ - --- Show the results -SELECT - 'Rowcounts for each database and table', - NVL(V4.DBName, 'None') v4_database_name, - v4.TableName v4_table_name, - v4.row_count v4_row_count, - NVL(v5.DBName, 'None') v5_database_name, - v5.TableName v5_table_name, - v5.row_count v5_row_count, - NVL(v5.row_count, 0) - NVL(v4.row_count, 0) row_count_change - FROM g8iauhqmv4_stats v4 -full outer join g8iauhqmv5_stats v5 ON v4.TableName = v5.TableName -order by v5.TableName ; - -/* - * Determine how the vocabulary/domains helped to map from the V4 source - * tables to the V5 destinations - */ -BEGIN - EXECUTE IMMEDIATE 'TRUNCATE TABLE g8iauhqmclassification_map'; - EXECUTE IMMEDIATE 'DROP TABLE g8iauhqmclassification_map'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -942 THEN - RAISE; - END IF; -END; - -/* / */ - -CREATE TABLE g8iauhqmclassification_map - AS -SELECT - * - -FROM - -(SELECT 'Condition_Occurrence' TableName, NVL(LOWER(cm.domain_id), 'condition') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Condition_Occurrence CO - LEFT JOIN g8iauhqmconcept_map CM ON co.condition_concept_id = cm.source_concept_id - GROUP BY NVL(LOWER(cm.domain_id), 'condition') - UNION - SELECT 'Drug_Exposure' TableName, NVL(LOWER(cm.domain_id), 'drug') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Drug_Exposure de - LEFT JOIN g8iauhqmconcept_map CM ON de.drug_concept_id = cm.source_concept_id - GROUP BY NVL(LOWER(cm.domain_id), 'drug') - UNION - SELECT 'Observation' TableName, NVL(LOWER(cm.domain_id), 'observation') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Observation o - LEFT JOIN g8iauhqmconcept_map CM ON o.observation_concept_id = cm.source_concept_id - GROUP BY NVL(LOWER(cm.domain_id), 'observation') - UNION - SELECT 'Procedure_Occurrence' TableName, NVL(LOWER(cm.domain_id), 'procedure') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Procedure_Occurrence po - LEFT JOIN g8iauhqmconcept_map CM ON po.PROCEDURE_CONCEPT_ID = cm.source_concept_id - GROUP BY NVL(LOWER(cm.domain_id), 'procedure') - ) A -ORDER by A.TableName, A.domain_id ; - -/* / */ - -SELECT * - FROM g8iauhqmclassification_map -order by tablename, domain_id ; - -SELECT domain_id, SUM(row_count) - FROM g8iauhqmclassification_map -group by domain_id -order by domain_id ; \ No newline at end of file diff --git a/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql b/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql deleted file mode 100644 index db54431..0000000 --- a/Version4 To Version5 Conversion/PostgreSQL/OMOP CDMv4 to CDMv5 - PostgreSQL.sql +++ /dev/null @@ -1,2408 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -********************************************************************************/ -/******************************************************************************* - -PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. - -last revised: 04 August 2015 -authors: Patrick Ryan, Chris Knoll, Anthony Sena - -!!!!!!!!!!!!!!!!!!!!! PLEASE READ THESE INSTRUCTIONS !!!!!!!!!!!!!!!!!!!!!!!!!!! - -This script was authored using OHDSI-SQL which will require you to run this -script through SqlRender to creat a version that is compatible with your target -RDBMS. We have pre-generated these scripts using SQL Render and have placed -them in folders for each RDBMS. Depending on which script you are viewing, your -instructions will be slightly different. - -General Assumptions -------------------- - -This script assumes that your V4 and V5 database are located on the same -RDBMS server. It also assumes that the V4 and V5 databases were created -using the standard data definition scripts for these databases. If you -altered your V4 database in any way, this script will likely require -some mo - -Getting Started ---------------- - -Before you can use this script, there are some prerequisites: - - 1. Create a target CDMv5 database on your database server using the - appropriate script from https://github.com/OHDSI/CommonDataModel - 2. Load VocabV5 into the target database/schema that will contain CDMv5 using - Athena: http://ohdsi.org/web/ATHENA - -OHDSI-SQL File Instructions ------------------------------ - - 1. Set parameter name of schema that contains CDMv4 instance - ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) - 2. Set parameter name of schema that contains CDMv5 instance - ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) - 3. Run this script through SqlRender to produce a script that will work in your - source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender - 4. Run the script produced by SQL Render on your target RDBDMS. - - File Instructions -------------------------- - - 1. This script will hold a number of placeholders for your CDM V4 and CDMV5 - database/schema. In order to make this file work in your environment, you - should plan to do a global "FIND AND REPLACE" on this file to fill in the - file with values that pertain to your environment. The following are the - tokens you should use when doing your "FIND AND REPLACE" operation: - - a. [SOURCE_CDMV4] - b. [SOURCE_CDMV4].[SCHEMA] - c. [TARGET_CDMV5] - d. [TARGET_CDMV5].[SCHEMA] - - 2. Run the resulting script on your target RDBDMS. - -*********************************************************************************/ -/* SCRIPT PARAMETERS */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -SET search_path TO [TARGET_CDMV5]; - -/* - * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids - * and their respective domain_ids. As a result, the results of this table will have - * some source_concept_ids that map to multiple target_concept_ids which is expected. - * - * In some of the conversion queries below, we will use the distinct values from the - * #concept_map table since our need in those instances is to understand the domain_id - * that will provide the destination table in the target V5 database. To make the code - * easier to read, we dervied a #concept_map_distinct table that holds the distinct - * source_concept_id and target domain_id. - * - * As of the dateof this script, the following domains contain source_concept_ids that - * map to multiple target_concept_ids: - * - * Condition - * Device - * Drug - * Measurement - * Observation - * Procedure - * Spec Anatomic Site - * - * Also, as of the date which this script was authored, no source_concept_ids map to multiple domains - */ -DROP TABLE IF EXISTS concept_map; - -/* / */ - -CREATE TEMP TABLE concept_map - -AS -SELECT - concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id AS domain_id - -FROM - [TARGET_CDMV5].[SCHEMA].concept -WHERE 1 = 0; - -/* / */ - -INSERT INTO concept_map ---standard concepts -SELECT concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id -FROM [TARGET_CDMV5].[SCHEMA].concept -WHERE standard_concept = 'S' - AND invalid_reason IS NULL - -UNION - ---concepts with 'map to' standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with another non 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT A.concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept A - LEFT JOIN ( - SELECT DISTINCT c1.concept_id - FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - A.standard_concept <> 'S' - OR A.standard_concept IS NULL - ) - OR A.invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT A.concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept A - LEFT JOIN ( - SELECT DISTINCT c1.concept_id - FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'Maps to' - ,'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Is a') - AND cr1.invalid_reason IS NULL; - -DROP TABLE IF EXISTS concept_map_distinct; - -/* / */ - -CREATE TEMP TABLE concept_map_distinct - -AS -SELECT - source_concept_id - ,domain_id - ,COUNT(*) AS targetConceptCount - -FROM - concept_map -WHERE 1 = 0 -GROUP BY source_concept_id - ,domain_id; - -/* / */ - -INSERT INTO concept_map_distinct -SELECT source_concept_id - ,domain_id - ,COUNT(*) -FROM concept_map -GROUP BY source_concept_id - ,domain_id; - -DROP TABLE IF EXISTS [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS; - -/* / */ - -CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE VARCHAR(4000)); -/* / */ - -/**** - -CDM_SOURCE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source ( - cdm_source_name - ,cdm_version - ,vocabulary_version - ,cdm_release_date - ) -SELECT '[TARGET_CDMV5]' - ,'V5' - ,v.vocabulary_version - ,CURRENT_DATE -FROM [TARGET_CDMV5].[SCHEMA].vocabulary v -WHERE vocabulary_id = 'Vocabulary'; - -/**** - -LOCATION - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].location -SELECT location_id - ,address_1 - ,address_2 - ,city - ,STATE - ,zip - ,county - ,location_source_value -FROM [SOURCE_CDMV4].[SCHEMA].LOCATION; - -/**** - -CARE_SITE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].care_site -SELECT care_site_id - ,cast(NULL AS VARCHAR(255)) AS care_site_name - ,place_of_service_concept_id - ,location_id - ,care_site_source_value - ,place_of_service_source_value -FROM [SOURCE_CDMV4].[SCHEMA].CARE_SITE; - -/**** - -Provider - -****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].provider -SELECT provider_id - ,cast(NULL AS VARCHAR(255)) AS provider_name - ,NPI - ,DEA - ,specialty_concept_id - ,care_site_id - ,cast(NULL AS INT) AS year_of_birth - ,cast(NULL AS INT) AS gender_concept_id - ,provider_source_value - ,specialty_source_value - ,0 AS specialty_source_concept_id - ,cast(NULL AS VARCHAR(50)) AS gender_source_value - ,cast(NULL AS INT) AS gender_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].provider; - -/**** - - PERSON - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].person -SELECT person_id - ,coalesce(gender.target_concept_id, 0) AS gender_concept_id - ,year_of_birth - ,month_of_birth - ,day_of_birth - ,CAST(NULL AS VARCHAR(10)) time_of_birth - ,coalesce(race.target_concept_id, 0) AS race_concept_id - ,coalesce(ethnicity.target_concept_id, 0) AS ethnicity_concept_id - ,location_id - ,provider_id - ,care_site_id - ,person_source_value - ,gender_source_value - ,CAST(NULL AS INT) gender_source_concept_id - ,CAST(NULL AS INT) race_source_value - ,CAST(NULL AS INT) race_source_concept_id - ,ethnicity_source_value - ,CAST(NULL AS INT) ethnicity_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].PERSON p -LEFT JOIN concept_map gender ON LOWER(gender.DOMAIN_ID) IN ('gender') - AND p.gender_concept_id = gender.source_concept_id -LEFT JOIN concept_map race ON LOWER(race.DOMAIN_ID) IN ('race') - AND p.race_concept_id = race.source_concept_id -LEFT JOIN concept_map ethnicity ON LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') - AND p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE GENDER_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('gender') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE RACE_CONCEPT_ID IS NOT NULL - AND RACE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('race') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE ETHNICITY_CONCEPT_ID IS NOT NULL - AND ETHNICITY_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('ethnicity') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - OBSERVATION_PERIOD - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period -SELECT observation_period_id - ,person_id - ,observation_period_start_date - ,observation_period_end_date - ,44814722 AS period_type_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; - -/**** - - DEATH - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].death -SELECT person_id - ,death_date - ,COALESCE(death_type_concept_id, 0) AS death_type_concept_id - ,cause_of_death_concept_id AS cause_concept_id - ,cause_of_death_source_value AS cause_source_value - ,CAST(NULL AS INT) AS cause_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].DEATH -LEFT JOIN concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept'); - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DEATH: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DEATH - WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - VISIT_OCCURRENCE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence -SELECT visit_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS visit_concept_id - ,visit_start_date - ,CAST(NULL AS VARCHAR(10)) visit_start_time - ,visit_end_date - ,CAST(NULL AS VARCHAR(10)) visit_end_time - ,44818517 AS visit_type_concept_id - ,CAST(NULL AS INT) provider_id - ,care_site_id - ,place_of_service_source_value AS visit_source_value - ,CAST(NULL AS INT) visit_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE -LEFT JOIN concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('visit'); - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'VISIT_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE - WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('visit') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - PROCEDURE_OCCURRENCE - - ****/ -DROP TABLE IF EXISTS po_map; - -/* / */ - -CREATE TEMP TABLE po_map - -AS -SELECT - po.procedure_occurrence_id - ,po.person_id - ,po.procedure_concept_id - ,po.procedure_date - ,po.procedure_type_concept_id - ,po.modifier_concept_id - ,po.quantity - ,po.provider_id - ,po.visit_occurrence_id - ,po.procedure_source_value - ,po.procedure_source_concept_id - ,po.qualifier_source_value - ,de.drug_exposure_id AS origional_drug_id - -FROM - [TARGET_CDMV5].[SCHEMA].procedure_occurrence po -LEFT JOIN [TARGET_CDMV5].[SCHEMA].drug_exposure de ON 1 = 0 -WHERE 0 = 1; - -/* / */ - ---find valid procedures from procedure table -INSERT INTO po_map -SELECT procedure_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') -INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -LEFT JOIN concept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 - -UNION ALL - --- All procedures that did not map to a standard concept in V4 should also carry over to V5 -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -WHERE procedure_concept_id = 0 - -UNION ALL - --- All PROCEDURE_OCCURRENCE that do not map to a standard concept in V5 should also carry over with procedure_concept_id = 0 -SELECT procedure_occurrence_id - ,person_id - ,0 AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -LEFT JOIN concept_map cm1 ON procedure_concept_id = cm1.source_concept_id -LEFT JOIN concept_map cm2 ON procedure_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -WHERE procedure_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ,origional_drug_id -FROM ( - --find valid procedures from procedure table that map to more than 1 - --target concept in V5 - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') - - UNION ALL - - --find procedures that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,condition_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,condition_occurrence_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - - UNION ALL - - --find procedures that were previously classified as drug - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,drug_exposure_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,drug_exposure_id AS origional_drug_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - --find procedures that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,observation_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS - ,( - SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - ) MAXROW; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence ( - procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ) -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value -FROM po_map; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE PROCEDURE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('procedure type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - DRUG_EXPOSURE - - ****/ ---find valid drugs from drug_exposure table -DROP TABLE IF EXISTS drgexp_map; - -/* / */ - -CREATE TEMP TABLE drgexp_map - -AS -SELECT - de.drug_exposure_id - ,de.person_id - ,de.drug_concept_id - ,de.drug_exposure_start_date - ,de.drug_exposure_end_date - ,de.drug_type_concept_id - ,de.stop_reason - ,de.refills - ,de.quantity - ,de.days_supply - ,de.sig - ,de.route_concept_id - ,de.effective_drug_dose - ,de.dose_unit_concept_id - ,de.lot_number - ,de.provider_id - ,de.visit_occurrence_id - ,de.drug_source_value - ,de.drug_source_concept_id - ,de.route_source_value - ,de.dose_unit_source_value - ,po.procedure_occurrence_id AS origional_procedure_id - -FROM - [TARGET_CDMV5].[SCHEMA].drug_exposure de -LEFT JOIN [TARGET_CDMV5].[SCHEMA].procedure_occurrence po ON 1 = 0 -WHERE 0 = 1; - -/* / */ - -INSERT INTO drgexp_map -SELECT drug_exposure_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS NUMERIC) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') -INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -INNER JOIN concept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 -WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All drug exposures that did not map to a standard concept in V4 should also carry over to V5 -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS NUMERIC) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -WHERE drug_concept_id = 0 - -UNION ALL - --- All drug exposures that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT drug_exposure_id - ,person_id - ,0 - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS NUMERIC) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -LEFT JOIN concept_map cm1 ON drug_concept_id = cm1.source_concept_id -LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -WHERE drug_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ,origional_procedure_id -FROM ( - --find valid drugs from drug_exposure table that map to > 1 target concept - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS NUMERIC) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') - - UNION ALL - - --find drugs that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,condition_start_date AS drug_exposure_start_date - ,NULL AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS NUMERIC) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS NUMERIC) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,condition_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as procedure - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,procedure_date AS drug_exposure_start_date - ,TO_DATE(NULL , 'yyyymmdd') AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS NUMERIC) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS NUMERIC) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,procedure_occurrence_id AS origional_procedure_id - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,observation_date AS drug_exposure_start_date - ,TO_DATE(NULL , 'yyyymmdd') AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS NUMERIC) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS NUMERIC) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,observation_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,observation_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - ) OTHERS - ,( - SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - ) MAXROW; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure ( - drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ) -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value -FROM drgexp_map; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE DRUG_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('drug type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - CONDITION_OCCURRENCE - - ****/ ---find valid conditions from condition_occurrence table -INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence -SELECT condition_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') -INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') -WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All conditions that did not map to a standard concept in V4 should also carry over to V5 -SELECT condition_occurrence_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(condition_type_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -WHERE condition_concept_id = 0 - -UNION ALL - --- All conditions that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT condition_occurrence_id - ,person_id - ,0 AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -LEFT JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id -LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') -WHERE condition_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,condition_type_concept_id - ,stop_reason - ,provider_id - ,visit_occurrence_id - ,condition_source_value - ,condition_source_concept_id -FROM ( - --find valid conditions from condition_occurrence table that map to > 1 target concept - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,NULL AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') - WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - - UNION ALL - - --find conditions that were previously classified as procedure - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,procedure_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,procedure_occurrence_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as drug - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,drug_exposure_start_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,observation_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,observation_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - ) OTHERS - ,( - SELECT MAX(condition_occurrence_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - ) MAXROW; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDITION_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE CONDITION_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDIITON_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('condition type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - DEVICE_EXPOSURE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure -SELECT row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS device_exposure_id - ,person_id - ,device_concept_id - ,device_exposure_start_date - ,device_exposure_end_date - ,device_type_concept_id - ,unique_device_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,device_source_value - ,device_source_concept_id -FROM ( - --find devices that were previously classified as procedures - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,PROCEDURE_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,PROCEDURE_OCCURRENCE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as drug exposure - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,quantity - ,PRESCRIBING_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,DRUG_EXPOSURE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as conditions - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,CONDITION_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,CONDITION_OCCURRENCE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as observations - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,OBSERVATION_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - ) OTHERS; - -/**** - - MEASUREMENT - - ****/ ---find valid measurements from observation table -INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement -SELECT row_number() OVER ( - ORDER BY occurrence_id - ) AS measurement_id - ,person_id - ,measurement_concept_id - ,measurement_date - ,measurement_time - ,measurement_type_concept_id - ,operator_concept_id - ,value_as_number - ,value_as_concept_id - ,unit_concept_id - ,range_low - ,range_high - ,provider_id - ,visit_occurrence_id - ,measurement_source_value - ,measurement_source_concept_id - ,unit_source_value - ,value_source_value -FROM ( - --find mesaurements that were previously classified as observations - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,OBSERVATION_DATE AS measurement_date - ,CAST(OBSERVATION_TIME AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) operator_concept_id - ,value_as_number - ,value_as_concept_id - ,COALESCE(cm2.target_concept_id, 0) AS unit_concept_id - ,range_low - ,range_high - ,ASSOCIATED_PROVIDER_ID AS provider_id - ,visit_occurrence_id - ,OBSERVATION_SOURCE_VALUE AS measurement_source_value - ,CAST(NULL AS INT) measurement_source_concept_id - ,unit_source_value AS unit_source_value - ,cast(NULL AS VARCHAR(50)) AS value_source_value - ,observation_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - LEFT JOIN concept_map cm2 ON observation.unit_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('unit') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,procedure_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,condition_start_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,drug_exposure_start_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,drug_exposure_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - ) OTHERS; - -/**** - - OBSERVATION - - ****/ ---find valid observation from observation table -INSERT INTO [TARGET_CDMV5].[SCHEMA].observation -SELECT observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,CAST(observation_time AS VARCHAR(50)) AS observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value -FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION -WHERE observation_concept_id NOT IN ( - SELECT source_concept_id - FROM concept_map_distinct - WHERE LOWER(domain_id) IN ( - 'condition' - ,'drug' - ,'procedure' - ,'device' - ,'measurement' - ) - ) ---find observations that were previously classified as procedure - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,qualifier_concept_id - ,unit_concept_id - ,provider_id - ,visit_occurrence_id - ,observation_source_value - ,observation_source_concept_id - ,unit_source_value - ,qualifier_source_value -FROM ( - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,procedure_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS NUMERIC) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - --find observations that were previously classified as condition - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,condition_start_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS NUMERIC) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find DRG observations that were previously classified as procedure_cost - SELECT po.person_id - ,cm1.target_concept_id AS observation_concept_id - ,po.procedure_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS NUMERIC) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,po.associated_provider_id AS provider_id - ,po.visit_occurrence_id - ,pc.DISEASE_CLASS_SOURCE_VALUE AS observation_source_value - ,cm1.source_concept_id as observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,po.procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc - INNER JOIN [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po ON pc.PROCEDURE_OCCURRENCE_ID = po.PROCEDURE_OCCURRENCE_ID - INNER JOIN concept_map cm1 ON pc.disease_class_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find observations that were previously classified as drug exposure - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,drug_exposure_start_date AS observation_date - ,CAST(NULL AS VARCHAR(10)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS NUMERIC) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,drug_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,drug_exposure_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS - ,( - SELECT MAX(OBSERVATION_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - ) MAXROW; - -/**** - - PAYER_PLAN_PERIOD - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period -SELECT payer_plan_period_id - ,person_id - ,payer_plan_period_start_date - ,payer_plan_period_end_date - ,payer_source_value - ,plan_source_value - ,family_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; - -/**** - - DRUG_COST - - note : if there were invalid drug concepts in DRUG_EXPOSURE, those records may not enter CDMv5 but costs will persist - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost -SELECT drug_cost_id - ,dc.drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc; - --- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_cost_id - ,drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id -FROM ( - SELECT drug_exposure_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS NUMERIC) AS ingredient_cost - ,CAST(NULL AS NUMERIC) AS dispensing_fee - ,CAST(NULL AS NUMERIC) AS average_wholesale_price - ,payer_plan_period_id - ,procedure_cost_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po - INNER JOIN [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc ON po.procedure_occurrence_id = pc.procedure_occurrence_id - --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - INNER JOIN drgexp_map de ON de.person_id = po.person_id - AND pc.procedure_occurrence_id = de.origional_procedure_id - ) OTHERS - ,( - SELECT MAX(drug_cost_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST - ) MAXROW; - -/**** - - PROCEDURE_COST - - note : if there were invalid procedure concepts in PROCEDURE_OCCURRENCE, those records may not enter CDMv5 but costs will persist - - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost -SELECT procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; - --- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value -FROM ( - SELECT po.procedure_occurrence_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS NUMERIC) AS ingredient_cost - ,CAST(NULL AS NUMERIC) AS dispensing_fee - ,CAST(NULL AS NUMERIC) AS average_wholesale_price - ,payer_plan_period_id - ,CAST(NULL AS INT) AS revenue_code_concept_id - ,CAST(NULL AS INT) AS revenue_code_source_value - ,drug_cost_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de - INNER JOIN [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ON de.drug_exposure_id = dc.drug_exposure_id - --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - INNER JOIN po_map po ON de.person_id = po.person_id - AND de.drug_exposure_id = po.origional_drug_id - ) OTHERS - ,( - SELECT MAX(drug_cost_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST - ) MAXROW; - -/**** - -DRUG ERA -Note: Eras derived from DRUG_EXPOSURE table, using 30d gap - - ****/ -DROP TABLE IF EXISTS cteDrugTarget; - -/* / */ - --- Normalize DRUG_EXPOSURE_END_DATE to either the existing drug exposure end date, or add days supply, or add 1 day to the start date -CREATE TEMP TABLE cteDrugTarget - -AS -SELECT - d.DRUG_EXPOSURE_ID - ,d.PERSON_ID - ,c.CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE - ,COALESCE(DRUG_EXPOSURE_END_DATE, ( DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), ( DRUG_EXPOSURE_START_DATE + 1)) AS DRUG_EXPOSURE_END_DATE - ,c.CONCEPT_ID AS INGREDIENT_CONCEPT_ID - -FROM - [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d -INNER JOIN [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca ON ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID -INNER JOIN [TARGET_CDMV5].[SCHEMA].CONCEPT c ON ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID -WHERE c.VOCABULARY_ID = 'RxNorm' - AND c.CONCEPT_CLASS_ID = 'Ingredient'; - -/* / */ - -DROP TABLE IF EXISTS cteEndDates; - -/* / */ - -CREATE TEMP TABLE cteEndDates - -AS -SELECT - PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,( EVENT_DATE + - 30) AS END_DATE -- unpad the end date - -FROM - ( - SELECT E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM ( - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM ( - -- select the start dates, assigning a row number to each - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,0 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM cteDrugTarget - - UNION ALL - - -- add the end dates with NULL as the row number, padding the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,( DRUG_EXPOSURE_END_DATE + 30) - ,1 AS EVENT_TYPE - ,NULL - FROM cteDrugTarget - ) RAWDATA - ) E1 - INNER JOIN ( - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM cteDrugTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.INGREDIENT_CONCEPT_ID = E2.INGREDIENT_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E -WHERE 2 * E.START_ORDINAL - E.OVERALL_ORD = 0; - -/* / */ - -DROP TABLE IF EXISTS cteDrugExpEnds; - -/* / */ - -CREATE TEMP TABLE cteDrugExpEnds - -AS -SELECT - d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE - -FROM - cteDrugTarget d -INNER JOIN cteEndDates e ON d.PERSON_ID = e.PERSON_ID - AND d.INGREDIENT_CONCEPT_ID = e.INGREDIENT_CONCEPT_ID - AND e.END_DATE >= d.DRUG_EXPOSURE_START_DATE -GROUP BY d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE; - -/* / */ - -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_era -SELECT row_number() OVER ( - ORDER BY person_id - ) AS drug_era_id - ,person_id - ,INGREDIENT_CONCEPT_ID - ,min(DRUG_EXPOSURE_START_DATE) AS drug_era_start_date - ,ERA_END_DATE - ,COUNT(*) AS DRUG_EXPOSURE_COUNT - ,30 AS gap_days -FROM cteDrugExpEnds -GROUP BY person_id - ,INGREDIENT_CONCEPT_ID - ,drug_type_concept_id - ,ERA_END_DATE; - -/**** - -CONDITION ERA -Note: Eras derived from CONDITION_OCCURRENCE table, using 30d gap - - ****/ -DROP TABLE IF EXISTS condition_era_phase_1; - -/* / */ - -DROP TABLE IF EXISTS cteConditionTarget; - -/* / */ - --- create base eras from the concepts found in condition_occurrence -CREATE TEMP TABLE cteConditionTarget - -AS -SELECT - co.PERSON_ID - ,co.condition_concept_id - ,co.CONDITION_START_DATE - ,COALESCE(co.CONDITION_END_DATE, ( CONDITION_START_DATE + 1)) AS CONDITION_END_DATE - -FROM - [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co; - -/* / */ - -DROP TABLE IF EXISTS cteCondEndDates; - -/* / */ - -CREATE TEMP TABLE cteCondEndDates - -AS -SELECT - PERSON_ID - ,CONDITION_CONCEPT_ID - ,( EVENT_DATE + - 30) AS END_DATE -- unpad the end date - -FROM - ( - SELECT E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM ( - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM ( - -- select the start dates, assigning a row number to each - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,- 1 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM cteConditionTarget - - UNION ALL - - -- pad the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,( CONDITION_END_DATE + 30) - ,1 AS EVENT_TYPE - ,NULL - FROM cteConditionTarget - ) RAWDATA - ) E1 - INNER JOIN ( - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM cteConditionTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.CONDITION_CONCEPT_ID = E2.CONDITION_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E -WHERE (2 * E.START_ORDINAL) - E.OVERALL_ORD = 0; - -/* / */ - -DROP TABLE IF EXISTS cteConditionEnds; - -/* / */ - -CREATE TEMP TABLE cteConditionEnds - -AS -SELECT - c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE - -FROM - cteConditionTarget c -INNER JOIN cteCondEndDates e ON c.PERSON_ID = e.PERSON_ID - AND c.CONDITION_CONCEPT_ID = e.CONDITION_CONCEPT_ID - AND e.END_DATE >= c.CONDITION_START_DATE -GROUP BY c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE; - -/* / */ - -INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era ( - condition_era_id - ,person_id - ,condition_concept_id - ,condition_era_start_date - ,condition_era_end_date - ,condition_occurrence_count - ) -SELECT row_number() OVER ( - ORDER BY person_id - ) AS condition_era_id - ,person_id - ,CONDITION_CONCEPT_ID - ,min(CONDITION_START_DATE) AS CONDITION_ERA_START_DATE - ,ERA_END_DATE AS CONDITION_ERA_END_DATE - ,COUNT(*) AS CONDITION_OCCURRENCE_COUNT -FROM cteConditionEnds -GROUP BY person_id - ,CONDITION_CONCEPT_ID - ,ERA_END_DATE; - -/**** - -QUALITY ASSURANCE OUTPUT - -Note: These queries are used to provide some basic stats around row counts between your V4 and V5 database - to ensure that all of the data has migrated as expected. - - ****/ - -DROP TABLE IF EXISTS v5_stats; - -/* / */ - -DROP TABLE IF EXISTS v4_stats; - -/* / */ - --- Get the row counts for each table that is in scope for the migration -CREATE TEMP TABLE v4_stats - -AS -SELECT - * - -FROM - -( - SELECT '[SOURCE_CDMV4]' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].care_site - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].condition_era - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].condition_occurrence - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].death - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_cost - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_era - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].location - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].observation - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].observation_period - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].payer_plan_period - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].person - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].procedure_cost - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].procedure_occurrence - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].provider - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].visit_occurrence -) v4_stats; - -/* / */ - -CREATE TEMP TABLE v5_stats - -AS -SELECT - * - -FROM - -( - SELECT '[TARGET_CDMV5]' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].care_site - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].condition_era - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].condition_occurrence - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].death - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'device_exposure' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].device_exposure - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_cost - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_era - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_exposure - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].location - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'measurement' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].measurement - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].observation - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].observation_period - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].payer_plan_period - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].person - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].procedure_cost - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].procedure_occurrence - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].provider - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].visit_occurrence -) v5_stats; - -/* / */ - --- Show the results -select - 'Rowcounts for each database and table', - COALESCE(V4.DBName, 'None') v4_database_name, - v4.TableName v4_table_name, - v4.row_count v4_row_count, - COALESCE(v5.DBName, 'None') v5_database_name, - v5.TableName v5_table_name, - v5.row_count v5_row_count, - COALESCE(v5.row_count, 0) - COALESCE(v4.row_count, 0) row_count_change -from v4_stats v4 -full outer join v5_stats v5 ON v4.TableName = v5.TableName -order by v5.TableName; - -/* - * Determine how the vocabulary/domains helped to map from the V4 source - * tables to the V5 destinations - */ -DROP TABLE IF EXISTS classification_map; - -/* / */ - -CREATE TEMP TABLE classification_map - -AS -SELECT - * - -FROM - -( - SELECT 'Condition_Occurrence' TableName, COALESCE(LOWER(cm.domain_id), 'condition') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Condition_Occurrence CO - LEFT JOIN concept_map CM ON co.condition_concept_id = cm.source_concept_id - GROUP BY COALESCE(LOWER(cm.domain_id), 'condition') - UNION - SELECT 'Drug_Exposure' TableName, COALESCE(LOWER(cm.domain_id), 'drug') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Drug_Exposure de - LEFT JOIN concept_map CM ON de.drug_concept_id = cm.source_concept_id - GROUP BY COALESCE(LOWER(cm.domain_id), 'drug') - UNION - SELECT 'Observation' TableName, COALESCE(LOWER(cm.domain_id), 'observation') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Observation o - LEFT JOIN concept_map CM ON o.observation_concept_id = cm.source_concept_id - GROUP BY COALESCE(LOWER(cm.domain_id), 'observation') - UNION - SELECT 'Procedure_Occurrence' TableName, COALESCE(LOWER(cm.domain_id), 'procedure') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Procedure_Occurrence po - LEFT JOIN concept_map CM ON po.PROCEDURE_CONCEPT_ID = cm.source_concept_id - GROUP BY COALESCE(LOWER(cm.domain_id), 'procedure') -) A -ORDER by A.TableName, A.domain_id; - -/* / */ - -select * -from classification_map -order by tablename, domain_id; - -select domain_id, SUM(row_count) -from classification_map -group by domain_id -order by domain_id; \ No newline at end of file diff --git a/Version4 To Version5 Conversion/QA-Results.xlsx b/Version4 To Version5 Conversion/QA-Results.xlsx deleted file mode 100644 index 6c17c3f..0000000 Binary files a/Version4 To Version5 Conversion/QA-Results.xlsx and /dev/null differ diff --git a/Version4 To Version5 Conversion/README.md b/Version4 To Version5 Conversion/README.md deleted file mode 100644 index 013fafe..0000000 --- a/Version4 To Version5 Conversion/README.md +++ /dev/null @@ -1,60 +0,0 @@ -Conversion from CDM v4 to CDM v5 -============================================================== - -The scripts in this directory will aid you in moving your data from the Common Data Model (CDM) version 4 to version 5. - -Overview -============================================================== - -The resources in this folder provide you with a means for converting your CDM V4 database to CDM V5. The goal of these scripts is to provide a path for converting your data to the CDM V5 to take advantage of the other tools that are being built to support research on CDM V5. These scripts are **NOT** designed to replace a proper ETL from your source data to CDM V5. - -One of the most important aspects to this conversion script is the use of the **[Standarized Vocabularies](http://www.ohdsi.org/web/wiki/doku.php?id=documentation:vocabulary:introduction "Standarized Vocabularies")** to map from tables in the V4 database to their cooresponding V5 table using the vocabulary **[domains](http://www.ohdsi.org/web/wiki/doku.php?id=documentation:vocabulary:domains "domains")**. At the beginning of the conversion script, we create a #concept\_map temporary table which holds a mapping from source_concept_id's to standard target_concept_ids for each of the domains. This table is then used throughout the remainder of the script to map rows from each of the source V4 tables (i.e. condition\_occurrence) to the proper table in the V5 data model. As a result, the number of rows in the V4 condition\_occurrence will not be the same as in V5 since some rows may be converted to a different table based on the standard concept mapping. - -Assumptions -============================================================== - -We have created a directory per Relational Database Management System (RDBMS) that contains the conversion script for that database platform. All of the scripts have the same assumptions: - -1. Your source CDM V4 database is on the same sever as your target CDM v5 database. -2. You have read rights to the CDM V4 database and database owner privileges on the target V5 database as this script will create an "ETL_WARNINGS" table in the process. -3. You have enough storage on your database server to perform the conversion. - -Usage -===== - -1. **Create your V5 Target Database:** Create a CDM V5 database on the same server as your CDM V4 database by using the **[Common Data Model Scripts](https://github.com/OHDSI/CommonDataModel "Common Data Model Scripts")** for your RDBMS. **NOTE: Please review the data types that exist on your V4 database and ensure you carry forward any data type changes from V4 to V5. For example, if you converted columns from an INT to a BIGINT to accommodate tables with > 2.1 Billion Rows, you will need to make the corresponding changes in your V5 Database and potentially to this conversion script** - -2. **Load the V5 Vocabulary**: Download the V5 vocabulary from **[Athena](http://www.ohdsi.org/web/athena/ "Athena")** and load them into the V5 database created in step 1 above. - -2. **Download the conversion script:** The **[CDM V4 to V5 Conversion](https://github.com/OHDSI/CommonDataModel/tree/master/Version4%20To%20Version5%20Conversion "CDM V4 to V5 Conversion Directory")** folder has subfolders with scripts that will work on each RDBMS. In order to make this file work in your environment, you will need to perform a global "FIND AND REPLACE" on the conversion script to fill in the file with values that pertain to your environment. The following are the tokens you should use when doing your "FIND AND REPLACE" operation: - - * [SOURCE_CDMV4] - Your V4 database name - * [SOURCE_CDMV4].[SCHEMA] - Your V4 database name + schema - * [TARGET_CDMV5] - Your V5 database name - * [TARGET_CDMV5].[SCHEMA] - Your V5 database name + schema - -3. Run the resulting script on your target RDBDMS. ** **NOTE** ** If you are running the Oracle script via Sql Developer or similar, you may need to alter the script to include the appropriate "/" symbols to mark the end of the anonymous code blocks. This has been done in the Oracle script that has been provided in this repository. -4. At the end of the conversion process, several tables will be produced that will help you to understand how your data has changed as a result of the conversion process. This is described in the Quality Assurance section below. - - -Quality Assurance -=================== - -At the end of the conversion script, there are 3 queries which will provide information on the conversion process. For reference, this section of the conversion script has a header comment: - -/**** QUALITY ASSURANCE OUTPUT ****/ - -The first query provides a means for comparing the table row counts between the V4 and V5 databases. As mentioned in the overview section above, table row counts will differ between V4 and V5 based on the way that the standard vocabulary maps the data. The next set of queries will help to tie out the row count changes in these tables. - -The second query shows the source V4 table (i.e condition\_occurrence) and how the row counts maps to the V5 domain. This table is useful to understand how the data from the V4 source was distributed into the V5 tables. As a note, 1 record in the V4 table could map to multiple records in V5 as some concepts will map to multiple standard domains. - -The third query uses the information from the second query and provides a summary for each V5 domain. This is useful for tying out the rowcounts we'd expect from the script with the actual results observed in the first query. - -We have included a spreadsheet called "QA-Results.xlsx" which provides an example of how to utilize these 3 result queries to understand the results of the conversion process. The results of the first query go into the "Rowcounts" worksheet. The results of the second and third queries go into the "Classification Map Results" worksheet. If the conversion process worked as expected, all of the "Difference" columns should equal 0 in the "Classification Map Results" worksheet. - -Getting Involved -============================================================== -Each script found in the RDBMS directory was generated from the OHDSI-SQL file: *OMOP CDMv4 to CDMv5 - OHDSI-SQL.sql* found in the root of this directory. If you would like to contribute to this script, we'd suggest you modify this script and use **[SqlRender](https://github.com/OHDSI/SqlRender "SqlRender")** to re-generate the specific RDBMS scripts. We have also supplied a basic R script in this directory to help re-generate the scripts using SqlRender. - -Developer questions/comments/feedback: OHDSI Forum -We use the GitHub issue tracker for all bugs/issues/enhancements \ No newline at end of file diff --git a/Version4 To Version5 Conversion/Redshift/OMOP CDMv4 to CDMv5 - Redshift.sql b/Version4 To Version5 Conversion/Redshift/OMOP CDMv4 to CDMv5 - Redshift.sql deleted file mode 100644 index b132f2e..0000000 --- a/Version4 To Version5 Conversion/Redshift/OMOP CDMv4 to CDMv5 - Redshift.sql +++ /dev/null @@ -1,2408 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -********************************************************************************/ -/******************************************************************************* - -PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. - -last revised: 04 August 2015 -authors: Patrick Ryan, Chris Knoll, Anthony Sena - -!!!!!!!!!!!!!!!!!!!!! PLEASE READ THESE INSTRUCTIONS !!!!!!!!!!!!!!!!!!!!!!!!!!! - -This script was authored using OHDSI-SQL which will require you to run this -script through SqlRender to creat a version that is compatible with your target -RDBMS. We have pre-generated these scripts using SQL Render and have placed -them in folders for each RDBMS. Depending on which script you are viewing, your -instructions will be slightly different. - -General Assumptions -------------------- - -This script assumes that your V4 and V5 database are located on the same -RDBMS server. It also assumes that the V4 and V5 databases were created -using the standard data definition scripts for these databases. If you -altered your V4 database in any way, this script will likely require -some mo - -Getting Started ---------------- - -Before you can use this script, there are some prerequisites: - - 1. Create a target CDMv5 database on your database server using the - appropriate script from https://github.com/OHDSI/CommonDataModel - 2. Load VocabV5 into the target database/schema that will contain CDMv5 using - Athena: http://ohdsi.org/web/ATHENA - -OHDSI-SQL File Instructions ------------------------------ - - 1. Set parameter name of schema that contains CDMv4 instance - ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) - 2. Set parameter name of schema that contains CDMv5 instance - ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) - 3. Run this script through SqlRender to produce a script that will work in your - source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender - 4. Run the script produced by SQL Render on your target RDBDMS. - - File Instructions -------------------------- - - 1. This script will hold a number of placeholders for your CDM V4 and CDMV5 - database/schema. In order to make this file work in your environment, you - should plan to do a global "FIND AND REPLACE" on this file to fill in the - file with values that pertain to your environment. The following are the - tokens you should use when doing your "FIND AND REPLACE" operation: - - a. [SOURCE_CDMV4] - b. [SOURCE_CDMV4].[SCHEMA] - c. [TARGET_CDMV5] - d. [TARGET_CDMV5].[SCHEMA] - - 2. Run the resulting script on your target RDBDMS. - -*********************************************************************************/ -/* SCRIPT PARAMETERS */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -SET search_path TO [TARGET_CDMV5]; - -/* - * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids - * and their respective domain_ids. As a result, the results of this table will have - * some source_concept_ids that map to multiple target_concept_ids which is expected. - * - * In some of the conversion queries below, we will use the distinct values from the - * #concept_map table since our need in those instances is to understand the domain_id - * that will provide the destination table in the target V5 database. To make the code - * easier to read, we dervied a #concept_map_distinct table that holds the distinct - * source_concept_id and target domain_id. - * - * As of the dateof this script, the following domains contain source_concept_ids that - * map to multiple target_concept_ids: - * - * Condition - * Device - * Drug - * Measurement - * Observation - * Procedure - * Spec Anatomic Site - * - * Also, as of the date which this script was authored, no source_concept_ids map to multiple domains - */ -DROP TABLE IF EXISTS concept_map; - -/* / */ - -CREATE TEMP TABLE concept_map - -AS -SELECT - concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id AS domain_id - -FROM - [TARGET_CDMV5].[SCHEMA].concept -WHERE 1 = 0; - -/* / */ - -INSERT INTO concept_map ---standard concepts -SELECT concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id -FROM [TARGET_CDMV5].[SCHEMA].concept -WHERE standard_concept = 'S' - AND invalid_reason IS NULL - -UNION - ---concepts with 'map to' standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with another non 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT A.concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept A - LEFT JOIN ( - SELECT DISTINCT c1.concept_id - FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - A.standard_concept <> 'S' - OR A.standard_concept IS NULL - ) - OR A.invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT A.concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept A - LEFT JOIN ( - SELECT DISTINCT c1.concept_id - FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'Maps to' - ,'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Is a') - AND cr1.invalid_reason IS NULL; - -DROP TABLE IF EXISTS concept_map_distinct; - -/* / */ - -CREATE TEMP TABLE concept_map_distinct - -AS -SELECT - source_concept_id - ,domain_id - ,COUNT(*) AS targetConceptCount - -FROM - concept_map -WHERE 1 = 0 -GROUP BY source_concept_id - ,domain_id; - -/* / */ - -INSERT INTO concept_map_distinct -SELECT source_concept_id - ,domain_id - ,COUNT(*) -FROM concept_map -GROUP BY source_concept_id - ,domain_id; - -DROP TABLE IF EXISTS [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS; - -/* / */ - -CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE VARCHAR(4000)); -/* / */ - -/**** - -CDM_SOURCE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source ( - cdm_source_name - ,cdm_version - ,vocabulary_version - ,cdm_release_date - ) -SELECT '[TARGET_CDMV5]' - ,'V5' - ,v.vocabulary_version - ,CURRENT_DATE -FROM [TARGET_CDMV5].[SCHEMA].vocabulary v -WHERE vocabulary_id = 'Vocabulary'; - -/**** - -LOCATION - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].location -SELECT location_id - ,address_1 - ,address_2 - ,city - ,STATE - ,zip - ,county - ,location_source_value -FROM [SOURCE_CDMV4].[SCHEMA].LOCATION; - -/**** - -CARE_SITE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].care_site -SELECT care_site_id - ,cast(NULL AS VARCHAR(255)) AS care_site_name - ,place_of_service_concept_id - ,location_id - ,care_site_source_value - ,place_of_service_source_value -FROM [SOURCE_CDMV4].[SCHEMA].CARE_SITE; - -/**** - -Provider - -****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].provider -SELECT provider_id - ,cast(NULL AS VARCHAR(255)) AS provider_name - ,NPI - ,DEA - ,specialty_concept_id - ,care_site_id - ,cast(NULL AS INT) AS year_of_birth - ,cast(NULL AS INT) AS gender_concept_id - ,provider_source_value - ,specialty_source_value - ,0 AS specialty_source_concept_id - ,cast(NULL AS VARCHAR(50)) AS gender_source_value - ,cast(NULL AS INT) AS gender_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].provider; - -/**** - - PERSON - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].person -SELECT person_id - ,coalesce(gender.target_concept_id, 0) AS gender_concept_id - ,year_of_birth - ,month_of_birth - ,day_of_birth - ,CAST(NULL AS VARCHAR(10)) time_of_birth - ,coalesce(race.target_concept_id, 0) AS race_concept_id - ,coalesce(ethnicity.target_concept_id, 0) AS ethnicity_concept_id - ,location_id - ,provider_id - ,care_site_id - ,person_source_value - ,gender_source_value - ,CAST(NULL AS INT) gender_source_concept_id - ,CAST(NULL AS INT) race_source_value - ,CAST(NULL AS INT) race_source_concept_id - ,ethnicity_source_value - ,CAST(NULL AS INT) ethnicity_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].PERSON p -LEFT JOIN concept_map gender ON LOWER(gender.DOMAIN_ID) IN ('gender') - AND p.gender_concept_id = gender.source_concept_id -LEFT JOIN concept_map race ON LOWER(race.DOMAIN_ID) IN ('race') - AND p.race_concept_id = race.source_concept_id -LEFT JOIN concept_map ethnicity ON LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') - AND p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE GENDER_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('gender') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE RACE_CONCEPT_ID IS NOT NULL - AND RACE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('race') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE ETHNICITY_CONCEPT_ID IS NOT NULL - AND ETHNICITY_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('ethnicity') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - OBSERVATION_PERIOD - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period -SELECT observation_period_id - ,person_id - ,observation_period_start_date - ,observation_period_end_date - ,44814722 AS period_type_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; - -/**** - - DEATH - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].death -SELECT person_id - ,death_date - ,COALESCE(death_type_concept_id, 0) AS death_type_concept_id - ,cause_of_death_concept_id AS cause_concept_id - ,cause_of_death_source_value AS cause_source_value - ,CAST(NULL AS INT) AS cause_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].DEATH -LEFT JOIN concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept'); - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DEATH: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DEATH - WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - VISIT_OCCURRENCE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence -SELECT visit_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS visit_concept_id - ,visit_start_date - ,CAST(NULL AS VARCHAR(10)) visit_start_time - ,visit_end_date - ,CAST(NULL AS VARCHAR(10)) visit_end_time - ,44818517 AS visit_type_concept_id - ,CAST(NULL AS INT) provider_id - ,care_site_id - ,place_of_service_source_value AS visit_source_value - ,CAST(NULL AS INT) visit_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE -LEFT JOIN concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('visit'); - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'VISIT_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE - WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('visit') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - PROCEDURE_OCCURRENCE - - ****/ -DROP TABLE IF EXISTS po_map; - -/* / */ - -CREATE TEMP TABLE po_map - -AS -SELECT - po.procedure_occurrence_id - ,po.person_id - ,po.procedure_concept_id - ,po.procedure_date - ,po.procedure_type_concept_id - ,po.modifier_concept_id - ,po.quantity - ,po.provider_id - ,po.visit_occurrence_id - ,po.procedure_source_value - ,po.procedure_source_concept_id - ,po.qualifier_source_value - ,de.drug_exposure_id AS origional_drug_id - -FROM - [TARGET_CDMV5].[SCHEMA].procedure_occurrence po -LEFT JOIN [TARGET_CDMV5].[SCHEMA].drug_exposure de ON 1 = 0 -WHERE 0 = 1; - -/* / */ - ---find valid procedures from procedure table -INSERT INTO po_map -SELECT procedure_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') -INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -LEFT JOIN concept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 - -UNION ALL - --- All procedures that did not map to a standard concept in V4 should also carry over to V5 -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -WHERE procedure_concept_id = 0 - -UNION ALL - --- All PROCEDURE_OCCURRENCE that do not map to a standard concept in V5 should also carry over with procedure_concept_id = 0 -SELECT procedure_occurrence_id - ,person_id - ,0 AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -LEFT JOIN concept_map cm1 ON procedure_concept_id = cm1.source_concept_id -LEFT JOIN concept_map cm2 ON procedure_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -WHERE procedure_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ,origional_drug_id -FROM ( - --find valid procedures from procedure table that map to more than 1 - --target concept in V5 - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') - - UNION ALL - - --find procedures that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,condition_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,condition_occurrence_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - - UNION ALL - - --find procedures that were previously classified as drug - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,drug_exposure_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,drug_exposure_id AS origional_drug_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - --find procedures that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,observation_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS - ,( - SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - ) MAXROW; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence ( - procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ) -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value -FROM po_map; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE PROCEDURE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('procedure type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - DRUG_EXPOSURE - - ****/ ---find valid drugs from drug_exposure table -DROP TABLE IF EXISTS drgexp_map; - -/* / */ - -CREATE TEMP TABLE drgexp_map - -AS -SELECT - de.drug_exposure_id - ,de.person_id - ,de.drug_concept_id - ,de.drug_exposure_start_date - ,de.drug_exposure_end_date - ,de.drug_type_concept_id - ,de.stop_reason - ,de.refills - ,de.quantity - ,de.days_supply - ,de.sig - ,de.route_concept_id - ,de.effective_drug_dose - ,de.dose_unit_concept_id - ,de.lot_number - ,de.provider_id - ,de.visit_occurrence_id - ,de.drug_source_value - ,de.drug_source_concept_id - ,de.route_source_value - ,de.dose_unit_source_value - ,po.procedure_occurrence_id AS origional_procedure_id - -FROM - [TARGET_CDMV5].[SCHEMA].drug_exposure de -LEFT JOIN [TARGET_CDMV5].[SCHEMA].procedure_occurrence po ON 1 = 0 -WHERE 0 = 1; - -/* / */ - -INSERT INTO drgexp_map -SELECT drug_exposure_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') -INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -INNER JOIN concept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 -WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All drug exposures that did not map to a standard concept in V4 should also carry over to V5 -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -WHERE drug_concept_id = 0 - -UNION ALL - --- All drug exposures that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT drug_exposure_id - ,person_id - ,0 - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -LEFT JOIN concept_map cm1 ON drug_concept_id = cm1.source_concept_id -LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -WHERE drug_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ,origional_procedure_id -FROM ( - --find valid drugs from drug_exposure table that map to > 1 target concept - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') - - UNION ALL - - --find drugs that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,condition_start_date AS drug_exposure_start_date - ,NULL AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,condition_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as procedure - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,procedure_date AS drug_exposure_start_date - ,TO_DATE(NULL , 'yyyymmdd') AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,procedure_occurrence_id AS origional_procedure_id - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,observation_date AS drug_exposure_start_date - ,TO_DATE(NULL , 'yyyymmdd') AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,observation_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,observation_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - ) OTHERS - ,( - SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - ) MAXROW; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure ( - drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ) -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value -FROM drgexp_map; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE DRUG_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('drug type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - CONDITION_OCCURRENCE - - ****/ ---find valid conditions from condition_occurrence table -INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence -SELECT condition_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') -INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') -WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All conditions that did not map to a standard concept in V4 should also carry over to V5 -SELECT condition_occurrence_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(condition_type_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -WHERE condition_concept_id = 0 - -UNION ALL - --- All conditions that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT condition_occurrence_id - ,person_id - ,0 AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -LEFT JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id -LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') -WHERE condition_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,condition_type_concept_id - ,stop_reason - ,provider_id - ,visit_occurrence_id - ,condition_source_value - ,condition_source_concept_id -FROM ( - --find valid conditions from condition_occurrence table that map to > 1 target concept - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,NULL AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - INNER JOIN concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') - WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - - UNION ALL - - --find conditions that were previously classified as procedure - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,procedure_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,procedure_occurrence_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as drug - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,drug_exposure_start_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,observation_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,observation_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - ) OTHERS - ,( - SELECT MAX(condition_occurrence_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - ) MAXROW; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDITION_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE CONDITION_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDIITON_OCCURRENCE: ' || CAST(NUM_INVALID_RECORDS AS VARCHAR) || ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('condition type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - DEVICE_EXPOSURE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure -SELECT row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS device_exposure_id - ,person_id - ,device_concept_id - ,device_exposure_start_date - ,device_exposure_end_date - ,device_type_concept_id - ,unique_device_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,device_source_value - ,device_source_concept_id -FROM ( - --find devices that were previously classified as procedures - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,PROCEDURE_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,PROCEDURE_OCCURRENCE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as drug exposure - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,quantity - ,PRESCRIBING_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,DRUG_EXPOSURE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as conditions - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,CONDITION_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,CONDITION_OCCURRENCE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as observations - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,OBSERVATION_DATE AS DEVICE_EXPOSURE_START_DATE - ,TO_DATE(NULL , 'yyyymmdd') AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - ) OTHERS; - -/**** - - MEASUREMENT - - ****/ ---find valid measurements from observation table -INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement -SELECT row_number() OVER ( - ORDER BY occurrence_id - ) AS measurement_id - ,person_id - ,measurement_concept_id - ,measurement_date - ,measurement_time - ,measurement_type_concept_id - ,operator_concept_id - ,value_as_number - ,value_as_concept_id - ,unit_concept_id - ,range_low - ,range_high - ,provider_id - ,visit_occurrence_id - ,measurement_source_value - ,measurement_source_concept_id - ,unit_source_value - ,value_source_value -FROM ( - --find mesaurements that were previously classified as observations - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,OBSERVATION_DATE AS measurement_date - ,CAST(OBSERVATION_TIME AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) operator_concept_id - ,value_as_number - ,value_as_concept_id - ,COALESCE(cm2.target_concept_id, 0) AS unit_concept_id - ,range_low - ,range_high - ,ASSOCIATED_PROVIDER_ID AS provider_id - ,visit_occurrence_id - ,OBSERVATION_SOURCE_VALUE AS measurement_source_value - ,CAST(NULL AS INT) measurement_source_concept_id - ,unit_source_value AS unit_source_value - ,cast(NULL AS VARCHAR(50)) AS value_source_value - ,observation_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - LEFT JOIN concept_map cm2 ON observation.unit_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('unit') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,procedure_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,condition_start_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,drug_exposure_start_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,drug_exposure_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - ) OTHERS; - -/**** - - OBSERVATION - - ****/ ---find valid observation from observation table -INSERT INTO [TARGET_CDMV5].[SCHEMA].observation -SELECT observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,CAST(observation_time AS VARCHAR(50)) AS observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value -FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION -WHERE observation_concept_id NOT IN ( - SELECT source_concept_id - FROM concept_map_distinct - WHERE LOWER(domain_id) IN ( - 'condition' - ,'drug' - ,'procedure' - ,'device' - ,'measurement' - ) - ) ---find observations that were previously classified as procedure - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,qualifier_concept_id - ,unit_concept_id - ,provider_id - ,visit_occurrence_id - ,observation_source_value - ,observation_source_concept_id - ,unit_source_value - ,qualifier_source_value -FROM ( - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,procedure_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - --find observations that were previously classified as condition - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,condition_start_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find DRG observations that were previously classified as procedure_cost - SELECT po.person_id - ,cm1.target_concept_id AS observation_concept_id - ,po.procedure_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,po.associated_provider_id AS provider_id - ,po.visit_occurrence_id - ,pc.DISEASE_CLASS_SOURCE_VALUE AS observation_source_value - ,cm1.source_concept_id as observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,po.procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc - INNER JOIN [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po ON pc.PROCEDURE_OCCURRENCE_ID = po.PROCEDURE_OCCURRENCE_ID - INNER JOIN concept_map cm1 ON pc.disease_class_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find observations that were previously classified as drug exposure - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,drug_exposure_start_date AS observation_date - ,CAST(NULL AS VARCHAR(10)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,drug_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,drug_exposure_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS - ,( - SELECT MAX(OBSERVATION_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - ) MAXROW; - -/**** - - PAYER_PLAN_PERIOD - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period -SELECT payer_plan_period_id - ,person_id - ,payer_plan_period_start_date - ,payer_plan_period_end_date - ,payer_source_value - ,plan_source_value - ,family_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; - -/**** - - DRUG_COST - - note : if there were invalid drug concepts in DRUG_EXPOSURE, those records may not enter CDMv5 but costs will persist - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost -SELECT drug_cost_id - ,dc.drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc; - --- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_cost_id - ,drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id -FROM ( - SELECT drug_exposure_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS FLOAT) AS ingredient_cost - ,CAST(NULL AS FLOAT) AS dispensing_fee - ,CAST(NULL AS FLOAT) AS average_wholesale_price - ,payer_plan_period_id - ,procedure_cost_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po - INNER JOIN [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc ON po.procedure_occurrence_id = pc.procedure_occurrence_id - --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - INNER JOIN drgexp_map de ON de.person_id = po.person_id - AND pc.procedure_occurrence_id = de.origional_procedure_id - ) OTHERS - ,( - SELECT MAX(drug_cost_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST - ) MAXROW; - -/**** - - PROCEDURE_COST - - note : if there were invalid procedure concepts in PROCEDURE_OCCURRENCE, those records may not enter CDMv5 but costs will persist - - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost -SELECT procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; - --- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value -FROM ( - SELECT po.procedure_occurrence_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS FLOAT) AS ingredient_cost - ,CAST(NULL AS FLOAT) AS dispensing_fee - ,CAST(NULL AS FLOAT) AS average_wholesale_price - ,payer_plan_period_id - ,CAST(NULL AS INT) AS revenue_code_concept_id - ,CAST(NULL AS INT) AS revenue_code_source_value - ,drug_cost_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de - INNER JOIN [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ON de.drug_exposure_id = dc.drug_exposure_id - --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - INNER JOIN po_map po ON de.person_id = po.person_id - AND de.drug_exposure_id = po.origional_drug_id - ) OTHERS - ,( - SELECT MAX(drug_cost_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST - ) MAXROW; - -/**** - -DRUG ERA -Note: Eras derived from DRUG_EXPOSURE table, using 30d gap - - ****/ -DROP TABLE IF EXISTS cteDrugTarget; - -/* / */ - --- Normalize DRUG_EXPOSURE_END_DATE to either the existing drug exposure end date, or add days supply, or add 1 day to the start date -CREATE TEMP TABLE cteDrugTarget - -AS -SELECT - d.DRUG_EXPOSURE_ID - ,d.PERSON_ID - ,c.CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE - ,COALESCE(DRUG_EXPOSURE_END_DATE, ( DRUG_EXPOSURE_START_DATE + DAYS_SUPPLY), ( DRUG_EXPOSURE_START_DATE + 1)) AS DRUG_EXPOSURE_END_DATE - ,c.CONCEPT_ID AS INGREDIENT_CONCEPT_ID - -FROM - [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d -INNER JOIN [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca ON ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID -INNER JOIN [TARGET_CDMV5].[SCHEMA].CONCEPT c ON ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID -WHERE c.VOCABULARY_ID = 'RxNorm' - AND c.CONCEPT_CLASS_ID = 'Ingredient'; - -/* / */ - -DROP TABLE IF EXISTS cteEndDates; - -/* / */ - -CREATE TEMP TABLE cteEndDates - -AS -SELECT - PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,( EVENT_DATE + - 30) AS END_DATE -- unpad the end date - -FROM - ( - SELECT E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM ( - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM ( - -- select the start dates, assigning a row number to each - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,0 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM cteDrugTarget - - UNION ALL - - -- add the end dates with NULL as the row number, padding the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,( DRUG_EXPOSURE_END_DATE + 30) - ,1 AS EVENT_TYPE - ,NULL - FROM cteDrugTarget - ) RAWDATA - ) E1 - INNER JOIN ( - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM cteDrugTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.INGREDIENT_CONCEPT_ID = E2.INGREDIENT_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E -WHERE 2 * E.START_ORDINAL - E.OVERALL_ORD = 0; - -/* / */ - -DROP TABLE IF EXISTS cteDrugExpEnds; - -/* / */ - -CREATE TEMP TABLE cteDrugExpEnds - -AS -SELECT - d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE - -FROM - cteDrugTarget d -INNER JOIN cteEndDates e ON d.PERSON_ID = e.PERSON_ID - AND d.INGREDIENT_CONCEPT_ID = e.INGREDIENT_CONCEPT_ID - AND e.END_DATE >= d.DRUG_EXPOSURE_START_DATE -GROUP BY d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE; - -/* / */ - -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_era -SELECT row_number() OVER ( - ORDER BY person_id - ) AS drug_era_id - ,person_id - ,INGREDIENT_CONCEPT_ID - ,min(DRUG_EXPOSURE_START_DATE) AS drug_era_start_date - ,ERA_END_DATE - ,COUNT(*) AS DRUG_EXPOSURE_COUNT - ,30 AS gap_days -FROM cteDrugExpEnds -GROUP BY person_id - ,INGREDIENT_CONCEPT_ID - ,drug_type_concept_id - ,ERA_END_DATE; - -/**** - -CONDITION ERA -Note: Eras derived from CONDITION_OCCURRENCE table, using 30d gap - - ****/ -DROP TABLE IF EXISTS condition_era_phase_1; - -/* / */ - -DROP TABLE IF EXISTS cteConditionTarget; - -/* / */ - --- create base eras from the concepts found in condition_occurrence -CREATE TEMP TABLE cteConditionTarget - -AS -SELECT - co.PERSON_ID - ,co.condition_concept_id - ,co.CONDITION_START_DATE - ,COALESCE(co.CONDITION_END_DATE, ( CONDITION_START_DATE + 1)) AS CONDITION_END_DATE - -FROM - [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co; - -/* / */ - -DROP TABLE IF EXISTS cteCondEndDates; - -/* / */ - -CREATE TEMP TABLE cteCondEndDates - -AS -SELECT - PERSON_ID - ,CONDITION_CONCEPT_ID - ,( EVENT_DATE + - 30) AS END_DATE -- unpad the end date - -FROM - ( - SELECT E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM ( - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM ( - -- select the start dates, assigning a row number to each - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,- 1 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM cteConditionTarget - - UNION ALL - - -- pad the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,( CONDITION_END_DATE + 30) - ,1 AS EVENT_TYPE - ,NULL - FROM cteConditionTarget - ) RAWDATA - ) E1 - INNER JOIN ( - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM cteConditionTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.CONDITION_CONCEPT_ID = E2.CONDITION_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E -WHERE (2 * E.START_ORDINAL) - E.OVERALL_ORD = 0; - -/* / */ - -DROP TABLE IF EXISTS cteConditionEnds; - -/* / */ - -CREATE TEMP TABLE cteConditionEnds - -AS -SELECT - c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE - -FROM - cteConditionTarget c -INNER JOIN cteCondEndDates e ON c.PERSON_ID = e.PERSON_ID - AND c.CONDITION_CONCEPT_ID = e.CONDITION_CONCEPT_ID - AND e.END_DATE >= c.CONDITION_START_DATE -GROUP BY c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE; - -/* / */ - -INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era ( - condition_era_id - ,person_id - ,condition_concept_id - ,condition_era_start_date - ,condition_era_end_date - ,condition_occurrence_count - ) -SELECT row_number() OVER ( - ORDER BY person_id - ) AS condition_era_id - ,person_id - ,CONDITION_CONCEPT_ID - ,min(CONDITION_START_DATE) AS CONDITION_ERA_START_DATE - ,ERA_END_DATE AS CONDITION_ERA_END_DATE - ,COUNT(*) AS CONDITION_OCCURRENCE_COUNT -FROM cteConditionEnds -GROUP BY person_id - ,CONDITION_CONCEPT_ID - ,ERA_END_DATE; - -/**** - -QUALITY ASSURANCE OUTPUT - -Note: These queries are used to provide some basic stats around row counts between your V4 and V5 database - to ensure that all of the data has migrated as expected. - - ****/ - -DROP TABLE IF EXISTS v5_stats; - -/* / */ - -DROP TABLE IF EXISTS v4_stats; - -/* / */ - --- Get the row counts for each table that is in scope for the migration -CREATE TEMP TABLE v4_stats - -AS -SELECT - * - -FROM - -( - SELECT '[SOURCE_CDMV4]' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].care_site - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].condition_era - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].condition_occurrence - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].death - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_cost - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_era - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].location - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].observation - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].observation_period - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].payer_plan_period - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].person - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].procedure_cost - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].procedure_occurrence - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].provider - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].visit_occurrence -) v4_stats; - -/* / */ - -CREATE TEMP TABLE v5_stats - -AS -SELECT - * - -FROM - -( - SELECT '[TARGET_CDMV5]' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].care_site - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].condition_era - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].condition_occurrence - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].death - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'device_exposure' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].device_exposure - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_cost - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_era - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_exposure - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].location - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'measurement' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].measurement - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].observation - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].observation_period - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].payer_plan_period - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].person - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].procedure_cost - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].procedure_occurrence - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].provider - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].visit_occurrence -) v5_stats; - -/* / */ - --- Show the results -select - 'Rowcounts for each database and table', - COALESCE(V4.DBName, 'None') v4_database_name, - v4.TableName v4_table_name, - v4.row_count v4_row_count, - COALESCE(v5.DBName, 'None') v5_database_name, - v5.TableName v5_table_name, - v5.row_count v5_row_count, - COALESCE(v5.row_count, 0) - COALESCE(v4.row_count, 0) row_count_change -from v4_stats v4 -full outer join v5_stats v5 ON v4.TableName = v5.TableName -order by v5.TableName; - -/* - * Determine how the vocabulary/domains helped to map from the V4 source - * tables to the V5 destinations - */ -DROP TABLE IF EXISTS classification_map; - -/* / */ - -CREATE TEMP TABLE classification_map - -AS -SELECT - * - -FROM - -( - SELECT 'Condition_Occurrence' TableName, COALESCE(LOWER(cm.domain_id), 'condition') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Condition_Occurrence CO - LEFT JOIN concept_map CM ON co.condition_concept_id = cm.source_concept_id - GROUP BY COALESCE(LOWER(cm.domain_id), 'condition') - UNION - SELECT 'Drug_Exposure' TableName, COALESCE(LOWER(cm.domain_id), 'drug') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Drug_Exposure de - LEFT JOIN concept_map CM ON de.drug_concept_id = cm.source_concept_id - GROUP BY COALESCE(LOWER(cm.domain_id), 'drug') - UNION - SELECT 'Observation' TableName, COALESCE(LOWER(cm.domain_id), 'observation') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Observation o - LEFT JOIN concept_map CM ON o.observation_concept_id = cm.source_concept_id - GROUP BY COALESCE(LOWER(cm.domain_id), 'observation') - UNION - SELECT 'Procedure_Occurrence' TableName, COALESCE(LOWER(cm.domain_id), 'procedure') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Procedure_Occurrence po - LEFT JOIN concept_map CM ON po.PROCEDURE_CONCEPT_ID = cm.source_concept_id - GROUP BY COALESCE(LOWER(cm.domain_id), 'procedure') -) A -ORDER by A.TableName, A.domain_id; - -/* / */ - -select * -from classification_map -order by tablename, domain_id; - -select domain_id, SUM(row_count) -from classification_map -group by domain_id -order by domain_id; \ No newline at end of file diff --git a/Version4 To Version5 Conversion/Sql Server PDW/OMOP CDMv4 to CDMv5 - Sql Server PDW.dsql b/Version4 To Version5 Conversion/Sql Server PDW/OMOP CDMv4 to CDMv5 - Sql Server PDW.dsql deleted file mode 100644 index f351b4f..0000000 --- a/Version4 To Version5 Conversion/Sql Server PDW/OMOP CDMv4 to CDMv5 - Sql Server PDW.dsql +++ /dev/null @@ -1,2388 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -********************************************************************************/ -/******************************************************************************* - -PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. - -last revised: 04 August 2015 -authors: Patrick Ryan, Chris Knoll, Anthony Sena - -!!!!!!!!!!!!!!!!!!!!! PLEASE READ THESE INSTRUCTIONS !!!!!!!!!!!!!!!!!!!!!!!!!!! - -This script was authored using OHDSI-SQL which will require you to run this -script through SqlRender to creat a version that is compatible with your target -RDBMS. We have pre-generated these scripts using SQL Render and have placed -them in folders for each RDBMS. Depending on which script you are viewing, your -instructions will be slightly different. - -General Assumptions -------------------- - -This script assumes that your V4 and V5 database are located on the same -RDBMS server. It also assumes that the V4 and V5 databases were created -using the standard data definition scripts for these databases. If you -altered your V4 database in any way, this script will likely require -some mo - -Getting Started ---------------- - -Before you can use this script, there are some prerequisites: - - 1. Create a target CDMv5 database on your database server using the - appropriate script from https://github.com/OHDSI/CommonDataModel - 2. Load VocabV5 into the target database/schema that will contain CDMv5 using - Athena: http://ohdsi.org/web/ATHENA - -OHDSI-SQL File Instructions ------------------------------ - - 1. Set parameter name of schema that contains CDMv4 instance - ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) - 2. Set parameter name of schema that contains CDMv5 instance - ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) - 3. Run this script through SqlRender to produce a script that will work in your - source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender - 4. Run the script produced by SQL Render on your target RDBDMS. - - File Instructions -------------------------- - - 1. This script will hold a number of placeholders for your CDM V4 and CDMV5 - database/schema. In order to make this file work in your environment, you - should plan to do a global "FIND AND REPLACE" on this file to fill in the - file with values that pertain to your environment. The following are the - tokens you should use when doing your "FIND AND REPLACE" operation: - - a. [SOURCE_CDMV4] - b. [SOURCE_CDMV4].[SCHEMA] - c. [TARGET_CDMV5] - d. [TARGET_CDMV5].[SCHEMA] - - 2. Run the resulting script on your target RDBDMS. - -*********************************************************************************/ -/* SCRIPT PARAMETERS */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -USE [TARGET_CDMV5]; - -/* - * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids - * and their respective domain_ids. As a result, the results of this table will have - * some source_concept_ids that map to multiple target_concept_ids which is expected. - * - * In some of the conversion queries below, we will use the distinct values from the - * #concept_map table since our need in those instances is to understand the domain_id - * that will provide the destination table in the target V5 database. To make the code - * easier to read, we dervied a #concept_map_distinct table that holds the distinct - * source_concept_id and target domain_id. - * - * As of the dateof this script, the following domains contain source_concept_ids that - * map to multiple target_concept_ids: - * - * Condition - * Device - * Drug - * Measurement - * Observation - * Procedure - * Spec Anatomic Site - * - * Also, as of the date which this script was authored, no source_concept_ids map to multiple domains - */ -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#concept_map', 'U') IS NOT NULL DROP TABLE #concept_map; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #concept_map - WITH (LOCATION = USER_DB, DISTRIBUTION = REPLICATE) AS -SELECT - concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id AS domain_id - -FROM - [TARGET_CDMV5].[SCHEMA].concept -WHERE 1 = 0; - -/* / */ - -INSERT INTO #concept_map ---standard concepts -SELECT concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id -FROM [TARGET_CDMV5].[SCHEMA].concept -WHERE standard_concept = 'S' - AND invalid_reason IS NULL - -UNION - ---concepts with 'map to' standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with another non 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT A.concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept A - LEFT JOIN ( - SELECT DISTINCT c1.concept_id - FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - A.standard_concept <> 'S' - OR A.standard_concept IS NULL - ) - OR A.invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT A.concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept A - LEFT JOIN ( - SELECT DISTINCT c1.concept_id - FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'Maps to' - ,'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Is a') - AND cr1.invalid_reason IS NULL; - -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL DROP TABLE #concept_map_distinct; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #concept_map_distinct - WITH (LOCATION = USER_DB, DISTRIBUTION = REPLICATE) AS -SELECT - source_concept_id - ,domain_id - ,COUNT(*) AS targetConceptCount - -FROM - #concept_map -WHERE 1 = 0 -GROUP BY source_concept_id - ,domain_id; - -/* / */ - -INSERT INTO #concept_map_distinct -SELECT source_concept_id - ,domain_id - ,COUNT(*) -FROM #concept_map -GROUP BY source_concept_id - ,domain_id; - -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('[TARGET_CDMV5].[SCHEMA].ETL_WARNINGS', 'U') IS NOT NULL DROP TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE VARCHAR(4000)) -WITH (DISTRIBUTION = REPLICATE); -/* / */ - -/**** - -CDM_SOURCE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source ( - cdm_source_name - ,cdm_version - ,vocabulary_version - ,cdm_release_date - ) -SELECT '[TARGET_CDMV5]' - ,'V5' - ,v.vocabulary_version - ,getDate() -FROM [TARGET_CDMV5].[SCHEMA].vocabulary v -WHERE vocabulary_id = 'Vocabulary'; - -/**** - -LOCATION - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].location -SELECT location_id - ,address_1 - ,address_2 - ,city - ,STATE - ,zip - ,county - ,location_source_value -FROM [SOURCE_CDMV4].[SCHEMA].LOCATION; - -/**** - -CARE_SITE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].care_site -SELECT care_site_id - ,cast(NULL AS VARCHAR(255)) AS care_site_name - ,place_of_service_concept_id - ,location_id - ,care_site_source_value - ,place_of_service_source_value -FROM [SOURCE_CDMV4].[SCHEMA].CARE_SITE; - -/**** - -Provider - -****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].provider -SELECT provider_id - ,cast(NULL AS VARCHAR(255)) AS provider_name - ,NPI - ,DEA - ,specialty_concept_id - ,care_site_id - ,cast(NULL AS INT) AS year_of_birth - ,cast(NULL AS INT) AS gender_concept_id - ,provider_source_value - ,specialty_source_value - ,0 AS specialty_source_concept_id - ,cast(NULL AS VARCHAR(50)) AS gender_source_value - ,cast(NULL AS INT) AS gender_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].provider; - -/**** - - PERSON - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].person -SELECT person_id - ,coalesce(gender.target_concept_id, 0) AS gender_concept_id - ,year_of_birth - ,month_of_birth - ,day_of_birth - ,CAST(NULL AS VARCHAR(10)) time_of_birth - ,coalesce(race.target_concept_id, 0) AS race_concept_id - ,coalesce(ethnicity.target_concept_id, 0) AS ethnicity_concept_id - ,location_id - ,provider_id - ,care_site_id - ,person_source_value - ,gender_source_value - ,CAST(NULL AS INT) gender_source_concept_id - ,CAST(NULL AS INT) race_source_value - ,CAST(NULL AS INT) race_source_concept_id - ,ethnicity_source_value - ,CAST(NULL AS INT) ethnicity_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].PERSON p -LEFT JOIN #concept_map gender ON LOWER(gender.DOMAIN_ID) IN ('gender') - AND p.gender_concept_id = gender.source_concept_id -LEFT JOIN #concept_map race ON LOWER(race.DOMAIN_ID) IN ('race') - AND p.race_concept_id = race.source_concept_id -LEFT JOIN #concept_map ethnicity ON LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') - AND p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE GENDER_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('gender') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE RACE_CONCEPT_ID IS NOT NULL - AND RACE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('race') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE ETHNICITY_CONCEPT_ID IS NOT NULL - AND ETHNICITY_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('ethnicity') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - OBSERVATION_PERIOD - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period -SELECT observation_period_id - ,person_id - ,observation_period_start_date - ,observation_period_end_date - ,44814722 AS period_type_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; - -/**** - - DEATH - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].death -SELECT person_id - ,death_date - ,COALESCE(death_type_concept_id, 0) AS death_type_concept_id - ,cause_of_death_concept_id AS cause_concept_id - ,cause_of_death_source_value AS cause_source_value - ,CAST(NULL AS INT) AS cause_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].DEATH -LEFT JOIN #concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept'); - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DEATH: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DEATH - WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - VISIT_OCCURRENCE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence -SELECT visit_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS visit_concept_id - ,visit_start_date - ,CAST(NULL AS VARCHAR(10)) visit_start_time - ,visit_end_date - ,CAST(NULL AS VARCHAR(10)) visit_end_time - ,44818517 AS visit_type_concept_id - ,CAST(NULL AS INT) provider_id - ,care_site_id - ,place_of_service_source_value AS visit_source_value - ,CAST(NULL AS INT) visit_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE -LEFT JOIN #concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('visit'); - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'VISIT_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE - WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('visit') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - PROCEDURE_OCCURRENCE - - ****/ -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#po_map', 'U') IS NOT NULL DROP TABLE #po_map; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #po_map - WITH (LOCATION = USER_DB, DISTRIBUTION = HASH(person_id)) AS -SELECT - po.procedure_occurrence_id - ,po. person_id, po.procedure_concept_id - ,po.procedure_date - ,po.procedure_type_concept_id - ,po.modifier_concept_id - ,po.quantity - ,po.provider_id - ,po.visit_occurrence_id - ,po.procedure_source_value - ,po.procedure_source_concept_id - ,po.qualifier_source_value - ,de.drug_exposure_id AS origional_drug_id - -FROM - [TARGET_CDMV5].[SCHEMA].procedure_occurrence po -LEFT JOIN [TARGET_CDMV5].[SCHEMA].drug_exposure de ON 1 = 0 -WHERE 0 = 1; - -/* / */ - ---find valid procedures from procedure table -INSERT INTO #po_map -SELECT procedure_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') -INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -LEFT JOIN #concept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 - -UNION ALL - --- All procedures that did not map to a standard concept in V4 should also carry over to V5 -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -WHERE procedure_concept_id = 0 - -UNION ALL - --- All PROCEDURE_OCCURRENCE that do not map to a standard concept in V5 should also carry over with procedure_concept_id = 0 -SELECT procedure_occurrence_id - ,person_id - ,0 AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -LEFT JOIN #concept_map cm1 ON procedure_concept_id = cm1.source_concept_id -LEFT JOIN #concept_map cm2 ON procedure_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -WHERE procedure_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ,origional_drug_id -FROM ( - --find valid procedures from procedure table that map to more than 1 - --target concept in V5 - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') - - UNION ALL - - --find procedures that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,condition_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,condition_occurrence_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - - UNION ALL - - --find procedures that were previously classified as drug - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,drug_exposure_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,drug_exposure_id AS origional_drug_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - --find procedures that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,observation_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS - ,( - SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - ) MAXROW; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence ( - procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ) -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value -FROM #po_map; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE PROCEDURE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('procedure type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - DRUG_EXPOSURE - - ****/ ---find valid drugs from drug_exposure table -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#drgexp_map', 'U') IS NOT NULL DROP TABLE #drgexp_map; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #drgexp_map - WITH (LOCATION = USER_DB, DISTRIBUTION = HASH(person_id)) AS -SELECT - de.drug_exposure_id - ,de. person_id, de.drug_concept_id - ,de.drug_exposure_start_date - ,de.drug_exposure_end_date - ,de.drug_type_concept_id - ,de.stop_reason - ,de.refills - ,de.quantity - ,de.days_supply - ,de.sig - ,de.route_concept_id - ,de.effective_drug_dose - ,de.dose_unit_concept_id - ,de.lot_number - ,de.provider_id - ,de.visit_occurrence_id - ,de.drug_source_value - ,de.drug_source_concept_id - ,de.route_source_value - ,de.dose_unit_source_value - ,po.procedure_occurrence_id AS origional_procedure_id - -FROM - [TARGET_CDMV5].[SCHEMA].drug_exposure de -LEFT JOIN [TARGET_CDMV5].[SCHEMA].procedure_occurrence po ON 1 = 0 -WHERE 0 = 1; - -/* / */ - -INSERT INTO #drgexp_map -SELECT drug_exposure_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') -INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -INNER JOIN #concept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 -WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All drug exposures that did not map to a standard concept in V4 should also carry over to V5 -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -WHERE drug_concept_id = 0 - -UNION ALL - --- All drug exposures that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT drug_exposure_id - ,person_id - ,0 - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -LEFT JOIN #concept_map cm1 ON drug_concept_id = cm1.source_concept_id -LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -WHERE drug_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ,origional_procedure_id -FROM ( - --find valid drugs from drug_exposure table that map to > 1 target concept - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') - - UNION ALL - - --find drugs that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,condition_start_date AS drug_exposure_start_date - ,NULL AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,condition_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as procedure - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,procedure_date AS drug_exposure_start_date - ,CAST(NULL AS DATE) AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,procedure_occurrence_id AS origional_procedure_id - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,observation_date AS drug_exposure_start_date - ,CAST(NULL AS DATE) AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,observation_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,observation_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - ) OTHERS - ,( - SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - ) MAXROW; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure ( - drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ) -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value -FROM #drgexp_map; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE DRUG_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('drug type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - CONDITION_OCCURRENCE - - ****/ ---find valid conditions from condition_occurrence table -INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence -SELECT condition_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') -INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') -WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All conditions that did not map to a standard concept in V4 should also carry over to V5 -SELECT condition_occurrence_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(condition_type_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -WHERE condition_concept_id = 0 - -UNION ALL - --- All conditions that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT condition_occurrence_id - ,person_id - ,0 AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -LEFT JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id -LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') -WHERE condition_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,condition_type_concept_id - ,stop_reason - ,provider_id - ,visit_occurrence_id - ,condition_source_value - ,condition_source_concept_id -FROM ( - --find valid conditions from condition_occurrence table that map to > 1 target concept - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,NULL AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') - WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - - UNION ALL - - --find conditions that were previously classified as procedure - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,procedure_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,procedure_occurrence_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as drug - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,drug_exposure_start_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,observation_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,observation_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - ) OTHERS - ,( - SELECT MAX(condition_occurrence_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - ) MAXROW; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDITION_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE CONDITION_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDIITON_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('condition type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - DEVICE_EXPOSURE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure -SELECT row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS device_exposure_id - ,person_id - ,device_concept_id - ,device_exposure_start_date - ,device_exposure_end_date - ,device_type_concept_id - ,unique_device_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,device_source_value - ,device_source_concept_id -FROM ( - --find devices that were previously classified as procedures - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,PROCEDURE_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,PROCEDURE_OCCURRENCE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as drug exposure - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,quantity - ,PRESCRIBING_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,DRUG_EXPOSURE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as conditions - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,CONDITION_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,CONDITION_OCCURRENCE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as observations - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,OBSERVATION_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - ) OTHERS; - -/**** - - MEASUREMENT - - ****/ ---find valid measurements from observation table -INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement -SELECT row_number() OVER ( - ORDER BY occurrence_id - ) AS measurement_id - ,person_id - ,measurement_concept_id - ,measurement_date - ,measurement_time - ,measurement_type_concept_id - ,operator_concept_id - ,value_as_number - ,value_as_concept_id - ,unit_concept_id - ,range_low - ,range_high - ,provider_id - ,visit_occurrence_id - ,measurement_source_value - ,measurement_source_concept_id - ,unit_source_value - ,value_source_value -FROM ( - --find mesaurements that were previously classified as observations - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,OBSERVATION_DATE AS measurement_date - ,CAST(OBSERVATION_TIME AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) operator_concept_id - ,value_as_number - ,value_as_concept_id - ,COALESCE(cm2.target_concept_id, 0) AS unit_concept_id - ,range_low - ,range_high - ,ASSOCIATED_PROVIDER_ID AS provider_id - ,visit_occurrence_id - ,OBSERVATION_SOURCE_VALUE AS measurement_source_value - ,CAST(NULL AS INT) measurement_source_concept_id - ,unit_source_value AS unit_source_value - ,cast(NULL AS VARCHAR(50)) AS value_source_value - ,observation_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - LEFT JOIN #concept_map cm2 ON observation.unit_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('unit') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,procedure_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,condition_start_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,drug_exposure_start_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,drug_exposure_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - ) OTHERS; - -/**** - - OBSERVATION - - ****/ ---find valid observation from observation table -INSERT INTO [TARGET_CDMV5].[SCHEMA].observation -SELECT observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,CAST(observation_time AS VARCHAR(50)) AS observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value -FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION -WHERE observation_concept_id NOT IN ( - SELECT source_concept_id - FROM #concept_map_distinct - WHERE LOWER(domain_id) IN ( - 'condition' - ,'drug' - ,'procedure' - ,'device' - ,'measurement' - ) - ) ---find observations that were previously classified as procedure - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,qualifier_concept_id - ,unit_concept_id - ,provider_id - ,visit_occurrence_id - ,observation_source_value - ,observation_source_concept_id - ,unit_source_value - ,qualifier_source_value -FROM ( - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,procedure_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - --find observations that were previously classified as condition - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,condition_start_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find DRG observations that were previously classified as procedure_cost - SELECT po.person_id - ,cm1.target_concept_id AS observation_concept_id - ,po.procedure_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,po.associated_provider_id AS provider_id - ,po.visit_occurrence_id - ,pc.DISEASE_CLASS_SOURCE_VALUE AS observation_source_value - ,cm1.source_concept_id as observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,po.procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc - INNER JOIN [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po ON pc.PROCEDURE_OCCURRENCE_ID = po.PROCEDURE_OCCURRENCE_ID - INNER JOIN #concept_map cm1 ON pc.disease_class_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find observations that were previously classified as drug exposure - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,drug_exposure_start_date AS observation_date - ,CAST(NULL AS VARCHAR(10)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,drug_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,drug_exposure_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS - ,( - SELECT MAX(OBSERVATION_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - ) MAXROW; - -/**** - - PAYER_PLAN_PERIOD - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period -SELECT payer_plan_period_id - ,person_id - ,payer_plan_period_start_date - ,payer_plan_period_end_date - ,payer_source_value - ,plan_source_value - ,family_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; - -/**** - - DRUG_COST - - note : if there were invalid drug concepts in DRUG_EXPOSURE, those records may not enter CDMv5 but costs will persist - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost -SELECT drug_cost_id - ,dc.drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc; - --- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_cost_id - ,drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id -FROM ( - SELECT drug_exposure_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS FLOAT) AS ingredient_cost - ,CAST(NULL AS FLOAT) AS dispensing_fee - ,CAST(NULL AS FLOAT) AS average_wholesale_price - ,payer_plan_period_id - ,procedure_cost_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po - INNER JOIN [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc ON po.procedure_occurrence_id = pc.procedure_occurrence_id - --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - INNER JOIN #drgexp_map de ON de.person_id = po.person_id - AND pc.procedure_occurrence_id = de.origional_procedure_id - ) OTHERS - ,( - SELECT MAX(drug_cost_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST - ) MAXROW; - -/**** - - PROCEDURE_COST - - note : if there were invalid procedure concepts in PROCEDURE_OCCURRENCE, those records may not enter CDMv5 but costs will persist - - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost -SELECT procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; - --- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value -FROM ( - SELECT po.procedure_occurrence_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS FLOAT) AS ingredient_cost - ,CAST(NULL AS FLOAT) AS dispensing_fee - ,CAST(NULL AS FLOAT) AS average_wholesale_price - ,payer_plan_period_id - ,CAST(NULL AS INT) AS revenue_code_concept_id - ,CAST(NULL AS INT) AS revenue_code_source_value - ,drug_cost_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de - INNER JOIN [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ON de.drug_exposure_id = dc.drug_exposure_id - --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - INNER JOIN #po_map po ON de.person_id = po.person_id - AND de.drug_exposure_id = po.origional_drug_id - ) OTHERS - ,( - SELECT MAX(drug_cost_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST - ) MAXROW; - -/**** - -DRUG ERA -Note: Eras derived from DRUG_EXPOSURE table, using 30d gap - - ****/ -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#cteDrugTarget', 'U') IS NOT NULL DROP TABLE #cteDrugTarget; - -/* / */ - --- Normalize DRUG_EXPOSURE_END_DATE to either the existing drug exposure end date, or add days supply, or add 1 day to the start date -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #cteDrugTarget - WITH (LOCATION = USER_DB, DISTRIBUTION = HASH(person_id)) AS -SELECT - d.DRUG_EXPOSURE_ID - ,d. person_id, c.CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE - ,COALESCE(DRUG_EXPOSURE_END_DATE, DATEADD(day, DAYS_SUPPLY, DRUG_EXPOSURE_START_DATE), DATEADD(day, 1, DRUG_EXPOSURE_START_DATE)) AS DRUG_EXPOSURE_END_DATE - ,c.CONCEPT_ID AS INGREDIENT_CONCEPT_ID - -FROM - [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d -INNER JOIN [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca ON ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID -INNER JOIN [TARGET_CDMV5].[SCHEMA].CONCEPT c ON ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID -WHERE c.VOCABULARY_ID = 'RxNorm' - AND c.CONCEPT_CLASS_ID = 'Ingredient'; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#cteEndDates', 'U') IS NOT NULL DROP TABLE #cteEndDates; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #cteEndDates - WITH (LOCATION = USER_DB, DISTRIBUTION = HASH(person_id)) AS -SELECT - person_id, INGREDIENT_CONCEPT_ID - ,DATEADD(day, - 30, EVENT_DATE) AS END_DATE -- unpad the end date - -FROM - ( - SELECT E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM ( - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM ( - -- select the start dates, assigning a row number to each - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,0 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM #cteDrugTarget - - UNION ALL - - -- add the end dates with NULL as the row number, padding the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DATEADD(day, 30, DRUG_EXPOSURE_END_DATE) - ,1 AS EVENT_TYPE - ,NULL - FROM #cteDrugTarget - ) RAWDATA - ) E1 - INNER JOIN ( - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM #cteDrugTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.INGREDIENT_CONCEPT_ID = E2.INGREDIENT_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E -WHERE 2 * E.START_ORDINAL - E.OVERALL_ORD = 0; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#cteDrugExpEnds', 'U') IS NOT NULL DROP TABLE #cteDrugExpEnds; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #cteDrugExpEnds - WITH (LOCATION = USER_DB, DISTRIBUTION = HASH(person_id)) AS -SELECT - d. person_id, d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE - -FROM - #cteDrugTarget d -INNER JOIN #cteEndDates e ON d.PERSON_ID = e.PERSON_ID - AND d.INGREDIENT_CONCEPT_ID = e.INGREDIENT_CONCEPT_ID - AND e.END_DATE >= d.DRUG_EXPOSURE_START_DATE -GROUP BY d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE; - -/* / */ - -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_era -SELECT row_number() OVER ( - ORDER BY person_id - ) AS drug_era_id - ,person_id - ,INGREDIENT_CONCEPT_ID - ,min(DRUG_EXPOSURE_START_DATE) AS drug_era_start_date - ,ERA_END_DATE - ,COUNT(*) AS DRUG_EXPOSURE_COUNT - ,30 AS gap_days -FROM #cteDrugExpEnds -GROUP BY person_id - ,INGREDIENT_CONCEPT_ID - ,drug_type_concept_id - ,ERA_END_DATE; - -/**** - -CONDITION ERA -Note: Eras derived from CONDITION_OCCURRENCE table, using 30d gap - - ****/ -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#condition_era_phase_1', 'U') IS NOT NULL DROP TABLE #condition_era_phase_1; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#cteConditionTarget', 'U') IS NOT NULL DROP TABLE #cteConditionTarget; - -/* / */ - --- create base eras from the concepts found in condition_occurrence -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #cteConditionTarget - WITH (LOCATION = USER_DB, DISTRIBUTION = HASH(person_id)) AS -SELECT - co. person_id, co.condition_concept_id - ,co.CONDITION_START_DATE - ,COALESCE(co.CONDITION_END_DATE, DATEADD(day, 1, CONDITION_START_DATE)) AS CONDITION_END_DATE - -FROM - [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#cteCondEndDates', 'U') IS NOT NULL DROP TABLE #cteCondEndDates; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #cteCondEndDates - WITH (LOCATION = USER_DB, DISTRIBUTION = HASH(person_id)) AS -SELECT - person_id, CONDITION_CONCEPT_ID - ,DATEADD(day, - 30, EVENT_DATE) AS END_DATE -- unpad the end date - -FROM - ( - SELECT E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM ( - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM ( - -- select the start dates, assigning a row number to each - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,- 1 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM #cteConditionTarget - - UNION ALL - - -- pad the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,DATEADD(day, 30, CONDITION_END_DATE) - ,1 AS EVENT_TYPE - ,NULL - FROM #cteConditionTarget - ) RAWDATA - ) E1 - INNER JOIN ( - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM #cteConditionTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.CONDITION_CONCEPT_ID = E2.CONDITION_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E -WHERE (2 * E.START_ORDINAL) - E.OVERALL_ORD = 0; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#cteConditionEnds', 'U') IS NOT NULL DROP TABLE #cteConditionEnds; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #cteConditionEnds - WITH (LOCATION = USER_DB, DISTRIBUTION = HASH(person_id)) AS -SELECT - c. person_id, c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE - -FROM - #cteConditionTarget c -INNER JOIN #cteCondEndDates e ON c.PERSON_ID = e.PERSON_ID - AND c.CONDITION_CONCEPT_ID = e.CONDITION_CONCEPT_ID - AND e.END_DATE >= c.CONDITION_START_DATE -GROUP BY c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE; - -/* / */ - -INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era ( - condition_era_id - ,person_id - ,condition_concept_id - ,condition_era_start_date - ,condition_era_end_date - ,condition_occurrence_count - ) -SELECT row_number() OVER ( - ORDER BY person_id - ) AS condition_era_id - ,person_id - ,CONDITION_CONCEPT_ID - ,min(CONDITION_START_DATE) AS CONDITION_ERA_START_DATE - ,ERA_END_DATE AS CONDITION_ERA_END_DATE - ,COUNT(*) AS CONDITION_OCCURRENCE_COUNT -FROM #cteConditionEnds -GROUP BY person_id - ,CONDITION_CONCEPT_ID - ,ERA_END_DATE; - -/**** - -QUALITY ASSURANCE OUTPUT - -Note: These queries are used to provide some basic stats around row counts between your V4 and V5 database - to ensure that all of the data has migrated as expected. - - ****/ - -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#v5_stats', 'U') IS NOT NULL DROP TABLE #v5_stats; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#v4_stats', 'U') IS NOT NULL DROP TABLE #v4_stats; - -/* / */ - --- Get the row counts for each table that is in scope for the migration -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #v4_stats - WITH (LOCATION = USER_DB, DISTRIBUTION = REPLICATE) AS -SELECT - * - -FROM - -( - SELECT '[SOURCE_CDMV4]' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].care_site - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].condition_era - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].condition_occurrence - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].death - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_cost - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_era - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].location - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].observation - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].observation_period - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].payer_plan_period - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].person - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].procedure_cost - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].procedure_occurrence - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].provider - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].visit_occurrence -) v4_stats; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #v5_stats - WITH (LOCATION = USER_DB, DISTRIBUTION = REPLICATE) AS -SELECT - * - -FROM - -( - SELECT '[TARGET_CDMV5]' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].care_site - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].condition_era - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].condition_occurrence - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].death - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'device_exposure' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].device_exposure - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_cost - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_era - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_exposure - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].location - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'measurement' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].measurement - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].observation - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].observation_period - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].payer_plan_period - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].person - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].procedure_cost - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].procedure_occurrence - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].provider - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].visit_occurrence -) v5_stats; - -/* / */ - --- Show the results -select - 'Rowcounts for each database and table', - ISNULL(V4.DBName, 'None') v4_database_name, - v4.TableName v4_table_name, - v4.row_count v4_row_count, - ISNULL(v5.DBName, 'None') v5_database_name, - v5.TableName v5_table_name, - v5.row_count v5_row_count, - ISNULL(v5.row_count, 0) - ISNULL(v4.row_count, 0) row_count_change -from #v4_stats v4 -full outer join #v5_stats v5 ON v4.TableName = v5.TableName -order by v5.TableName; - -/* - * Determine how the vocabulary/domains helped to map from the V4 source - * tables to the V5 destinations - */ -IF XACT_STATE() = 1 COMMIT; IF OBJECT_ID('tempdb..#classification_map', 'U') IS NOT NULL DROP TABLE #classification_map; - -/* / */ - -IF XACT_STATE() = 1 COMMIT; CREATE TABLE #classification_map - WITH (LOCATION = USER_DB, DISTRIBUTION = REPLICATE) AS -SELECT - * - -FROM - -( - SELECT 'Condition_Occurrence' TableName, ISNULL(LOWER(cm.domain_id), 'condition') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Condition_Occurrence CO - LEFT JOIN #concept_map CM ON co.condition_concept_id = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'condition') - UNION - SELECT 'Drug_Exposure' TableName, ISNULL(LOWER(cm.domain_id), 'drug') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Drug_Exposure de - LEFT JOIN #concept_map CM ON de.drug_concept_id = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'drug') - UNION - SELECT 'Observation' TableName, ISNULL(LOWER(cm.domain_id), 'observation') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Observation o - LEFT JOIN #concept_map CM ON o.observation_concept_id = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'observation') - UNION - SELECT 'Procedure_Occurrence' TableName, ISNULL(LOWER(cm.domain_id), 'procedure') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Procedure_Occurrence po - LEFT JOIN #concept_map CM ON po.PROCEDURE_CONCEPT_ID = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'procedure') -) A -ORDER by A.TableName, A.domain_id; - -/* / */ - -select * -from #classification_map -order by tablename, domain_id; - -select domain_id, SUM(row_count) -from #classification_map -group by domain_id -order by domain_id; \ No newline at end of file diff --git a/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql b/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql deleted file mode 100644 index 31c4076..0000000 --- a/Version4 To Version5 Conversion/Sql Server/OMOP CDMv4 to CDMv5 - SQL Server.sql +++ /dev/null @@ -1,2358 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -********************************************************************************/ -/******************************************************************************* - -PURPOSE: Use this script to convert your OMOP V4 common data model to CDM V5. - -last revised: 04 August 2015 -authors: Patrick Ryan, Chris Knoll, Anthony Sena - -!!!!!!!!!!!!!!!!!!!!! PLEASE READ THESE INSTRUCTIONS !!!!!!!!!!!!!!!!!!!!!!!!!!! - -This script was authored using OHDSI-SQL which will require you to run this -script through SqlRender to creat a version that is compatible with your target -RDBMS. We have pre-generated these scripts using SQL Render and have placed -them in folders for each RDBMS. Depending on which script you are viewing, your -instructions will be slightly different. - -General Assumptions -------------------- - -This script assumes that your V4 and V5 database are located on the same -RDBMS server. It also assumes that the V4 and V5 databases were created -using the standard data definition scripts for these databases. If you -altered your V4 database in any way, this script will likely require -some mo - -Getting Started ---------------- - -Before you can use this script, there are some prerequisites: - - 1. Create a target CDMv5 database on your database server using the - appropriate script from https://github.com/OHDSI/CommonDataModel - 2. Load VocabV5 into the target database/schema that will contain CDMv5 using - Athena: http://ohdsi.org/web/ATHENA - -OHDSI-SQL File Instructions ------------------------------ - - 1. Set parameter name of schema that contains CDMv4 instance - ([SOURCE_CDMV4], [SOURCE_CDMV4].[SCHEMA]) - 2. Set parameter name of schema that contains CDMv5 instance - ([TARGET_CDMV5], [TARGET_CDMV5].[SCHEMA]) - 3. Run this script through SqlRender to produce a script that will work in your - source dialect. SqlRender can be found here: https://github.com/OHDSI/SqlRender - 4. Run the script produced by SQL Render on your target RDBDMS. - - File Instructions -------------------------- - - 1. This script will hold a number of placeholders for your CDM V4 and CDMV5 - database/schema. In order to make this file work in your environment, you - should plan to do a global "FIND AND REPLACE" on this file to fill in the - file with values that pertain to your environment. The following are the - tokens you should use when doing your "FIND AND REPLACE" operation: - - a. [SOURCE_CDMV4] - b. [SOURCE_CDMV4].[SCHEMA] - c. [TARGET_CDMV5] - d. [TARGET_CDMV5].[SCHEMA] - - 2. Run the resulting script on your target RDBDMS. - -*********************************************************************************/ -/* SCRIPT PARAMETERS */ - -- The CDMv4 database name - -- The CDMv4 database plus schema - -- The target CDMv5 database name - -- the target CDMv5 database plus schema - -USE [TARGET_CDMV5]; - -/* - * The #concept_map table will hold the mapping of source_concept_ids to target_concept_ids - * and their respective domain_ids. As a result, the results of this table will have - * some source_concept_ids that map to multiple target_concept_ids which is expected. - * - * In some of the conversion queries below, we will use the distinct values from the - * #concept_map table since our need in those instances is to understand the domain_id - * that will provide the destination table in the target V5 database. To make the code - * easier to read, we dervied a #concept_map_distinct table that holds the distinct - * source_concept_id and target domain_id. - * - * As of the dateof this script, the following domains contain source_concept_ids that - * map to multiple target_concept_ids: - * - * Condition - * Device - * Drug - * Measurement - * Observation - * Procedure - * Spec Anatomic Site - * - * Also, as of the date which this script was authored, no source_concept_ids map to multiple domains - */ -IF OBJECT_ID('tempdb..#concept_map', 'U') IS NOT NULL - DROP TABLE #concept_map; - -/* / */ - -SELECT concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id AS domain_id -INTO #concept_map -FROM [TARGET_CDMV5].[SCHEMA].concept -WHERE 1 = 0; - -/* / */ - -INSERT INTO #concept_map ---standard concepts -SELECT concept_id AS source_concept_id - ,concept_id AS target_concept_id - ,domain_id -FROM [TARGET_CDMV5].[SCHEMA].concept -WHERE standard_concept = 'S' - AND invalid_reason IS NULL - -UNION - ---concepts with 'map to' standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with another non 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT A.concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept A - LEFT JOIN ( - SELECT DISTINCT c1.concept_id - FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Maps to') - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - A.standard_concept <> 'S' - OR A.standard_concept IS NULL - ) - OR A.invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - -UNION - ---concepts without 'map to' standard with 'is a' relation to standard -SELECT DISTINCT c1.concept_id AS source_concept_id - ,c2.concept_id AS target_concept_id - ,c2.domain_id -FROM ( - SELECT A.concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept A - LEFT JOIN ( - SELECT DISTINCT c1.concept_id - FROM ( - SELECT concept_id - FROM [TARGET_CDMV5].[SCHEMA].concept - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - ) c1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 - INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id - WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ( - 'Maps to' - ,'RxNorm replaced by' - ,'SNOMED replaced by' - ,'UCUM replaced by' - ,'Concept replaced by' - ,'ICD9P replaced by' - ,'LOINC replaced by' - ,'Concept same_as to' - ,'Concept was_a to' - ,'Concept alt_to to' - ) - AND cr1.invalid_reason IS NULL - ) B ON A.concept_id = B.concept_id - WHERE ( - ( - standard_concept <> 'S' - OR standard_concept IS NULL - ) - OR invalid_reason IS NOT NULL - ) - AND B.concept_id IS NULL - ) c1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept_relationship cr1 ON c1.concept_id = cr1.concept_id_1 -INNER JOIN [TARGET_CDMV5].[SCHEMA].concept c2 ON cr1.concept_id_2 = c2.concept_id -WHERE c2.standard_concept = 'S' - AND c2.invalid_reason IS NULL - AND cr1.relationship_id IN ('Is a') - AND cr1.invalid_reason IS NULL; - -IF OBJECT_ID('tempdb..#concept_map_distinct', 'U') IS NOT NULL - DROP TABLE #concept_map_distinct; - -/* / */ - -SELECT source_concept_id - ,domain_id - ,COUNT(*) AS targetConceptCount -INTO #concept_map_distinct -FROM #concept_map -WHERE 1 = 0 -GROUP BY source_concept_id - ,domain_id; - -/* / */ - -INSERT INTO #concept_map_distinct -SELECT source_concept_id - ,domain_id - ,COUNT(*) -FROM #concept_map -GROUP BY source_concept_id - ,domain_id; - -IF OBJECT_ID('[TARGET_CDMV5].[SCHEMA].ETL_WARNINGS', 'U') IS NOT NULL - DROP TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS; - -/* / */ - -CREATE TABLE [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE VARCHAR(4000)); -/* / */ - -/**** - -CDM_SOURCE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].cdm_source ( - cdm_source_name - ,cdm_version - ,vocabulary_version - ,cdm_release_date - ) -SELECT '[TARGET_CDMV5]' - ,'V5' - ,v.vocabulary_version - ,getDate() -FROM [TARGET_CDMV5].[SCHEMA].vocabulary v -WHERE vocabulary_id = 'Vocabulary'; - -/**** - -LOCATION - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].location -SELECT location_id - ,address_1 - ,address_2 - ,city - ,STATE - ,zip - ,county - ,location_source_value -FROM [SOURCE_CDMV4].[SCHEMA].LOCATION; - -/**** - -CARE_SITE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].care_site -SELECT care_site_id - ,cast(NULL AS VARCHAR(255)) AS care_site_name - ,place_of_service_concept_id - ,location_id - ,care_site_source_value - ,place_of_service_source_value -FROM [SOURCE_CDMV4].[SCHEMA].CARE_SITE; - -/**** - -Provider - -****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].provider -SELECT provider_id - ,cast(NULL AS VARCHAR(255)) AS provider_name - ,NPI - ,DEA - ,specialty_concept_id - ,care_site_id - ,cast(NULL AS INT) AS year_of_birth - ,cast(NULL AS INT) AS gender_concept_id - ,provider_source_value - ,specialty_source_value - ,0 AS specialty_source_concept_id - ,cast(NULL AS VARCHAR(50)) AS gender_source_value - ,cast(NULL AS INT) AS gender_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].provider; - -/**** - - PERSON - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].person -SELECT person_id - ,coalesce(gender.target_concept_id, 0) AS gender_concept_id - ,year_of_birth - ,month_of_birth - ,day_of_birth - ,CAST(NULL AS VARCHAR(10)) time_of_birth - ,coalesce(race.target_concept_id, 0) AS race_concept_id - ,coalesce(ethnicity.target_concept_id, 0) AS ethnicity_concept_id - ,location_id - ,provider_id - ,care_site_id - ,person_source_value - ,gender_source_value - ,CAST(NULL AS INT) gender_source_concept_id - ,CAST(NULL AS INT) race_source_value - ,CAST(NULL AS INT) race_source_concept_id - ,ethnicity_source_value - ,CAST(NULL AS INT) ethnicity_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].PERSON p -LEFT JOIN #concept_map gender ON LOWER(gender.DOMAIN_ID) IN ('gender') - AND p.gender_concept_id = gender.source_concept_id -LEFT JOIN #concept_map race ON LOWER(race.DOMAIN_ID) IN ('race') - AND p.race_concept_id = race.source_concept_id -LEFT JOIN #concept_map ethnicity ON LOWER(ethnicity.DOMAIN_ID) IN ('ethnicity') - AND p.ETHNICITY_CONCEPT_ID = ethnicity.source_concept_id; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid GENDER_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE GENDER_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('gender') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid RACE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE RACE_CONCEPT_ID IS NOT NULL - AND RACE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('race') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PERSON: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid ETHNICITY_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PERSON - WHERE ETHNICITY_CONCEPT_ID IS NOT NULL - AND ETHNICITY_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('ethnicity') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - OBSERVATION_PERIOD - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].observation_period -SELECT observation_period_id - ,person_id - ,observation_period_start_date - ,observation_period_end_date - ,44814722 AS period_type_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION_PERIOD; - -/**** - - DEATH - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].death -SELECT person_id - ,death_date - ,COALESCE(death_type_concept_id, 0) AS death_type_concept_id - ,cause_of_death_concept_id AS cause_concept_id - ,cause_of_death_source_value AS cause_source_value - ,CAST(NULL AS INT) AS cause_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].DEATH -LEFT JOIN #concept_map_distinct cm1 ON DEATH.DEATH_TYPE_CONCEPT_ID = CM1.SOURCE_CONCEPT_ID - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept'); - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DEATH: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DEATH_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DEATH - WHERE DEATH_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('death type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - VISIT_OCCURRENCE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].visit_occurrence -SELECT visit_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS visit_concept_id - ,visit_start_date - ,CAST(NULL AS VARCHAR(10)) visit_start_time - ,visit_end_date - ,CAST(NULL AS VARCHAR(10)) visit_end_time - ,44818517 AS visit_type_concept_id - ,CAST(NULL AS INT) provider_id - ,care_site_id - ,place_of_service_source_value AS visit_source_value - ,CAST(NULL AS INT) visit_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE -LEFT JOIN #concept_map cm1 ON VISIT_OCCURRENCE.PLACE_OF_SERVICE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('visit'); - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'VISIT_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid VISIT_CONCEPT_ID (from the CDMv4 PLACE_OF_SERVICE_CONCEPT_ID field)' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].VISIT_OCCURRENCE - WHERE PLACE_OF_SERVICE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('visit') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - PROCEDURE_OCCURRENCE - - ****/ -IF OBJECT_ID('tempdb..#po_map', 'U') IS NOT NULL - DROP TABLE #po_map; - -/* / */ - -SELECT po.procedure_occurrence_id - ,po.person_id - ,po.procedure_concept_id - ,po.procedure_date - ,po.procedure_type_concept_id - ,po.modifier_concept_id - ,po.quantity - ,po.provider_id - ,po.visit_occurrence_id - ,po.procedure_source_value - ,po.procedure_source_concept_id - ,po.qualifier_source_value - ,de.drug_exposure_id AS origional_drug_id -INTO #po_map -FROM [TARGET_CDMV5].[SCHEMA].procedure_occurrence po -LEFT JOIN [TARGET_CDMV5].[SCHEMA].drug_exposure de ON 1 = 0 -WHERE 0 = 1; - -/* / */ - ---find valid procedures from procedure table -INSERT INTO #po_map -SELECT procedure_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') -INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -LEFT JOIN #concept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 - -UNION ALL - --- All procedures that did not map to a standard concept in V4 should also carry over to V5 -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -WHERE procedure_concept_id = 0 - -UNION ALL - --- All PROCEDURE_OCCURRENCE that do not map to a standard concept in V5 should also carry over with procedure_concept_id = 0 -SELECT procedure_occurrence_id - ,person_id - ,0 AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE -LEFT JOIN #concept_map cm1 ON procedure_concept_id = cm1.source_concept_id -LEFT JOIN #concept_map cm2 ON procedure_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') -WHERE procedure_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ,origional_drug_id -FROM ( - --find valid procedures from procedure table that map to more than 1 - --target concept in V5 - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS procedure_concept_id - ,procedure_date - ,COALESCE(cm2.target_concept_id, 0) AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value - ,CAST(NULL AS INT) procedure_source_concept_id - ,NULL qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON PROCEDURE_OCCURRENCE.PROCEDURE_CONCEPT_ID = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN #concept_map cm2 ON PROCEDURE_OCCURRENCE.PROCEDURE_TYPE_CONCEPT_ID = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('procedure type', 'Type Concept') - - UNION ALL - - --find procedures that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,condition_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,condition_occurrence_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - - UNION ALL - - --find procedures that were previously classified as drug - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,drug_exposure_start_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,drug_exposure_id AS origional_drug_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - --find procedures that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS procedure_concept_id - ,observation_date AS procedure_date - ,0 AS procedure_type_concept_id - ,CAST(NULL AS INT) AS modifier_concept_id - ,CAST(NULL AS INT) AS quantity - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS procedure_source_value - ,CAST(NULL AS INT) AS procedure_source_concept_id - ,NULL AS qualifier_source_value - ,CAST(NULL AS INT) AS origional_drug_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('procedure') - ) OTHERS - ,( - SELECT MAX(PROCEDURE_OCCURRENCE_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - ) MAXROW; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_occurrence ( - procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value - ) -SELECT procedure_occurrence_id - ,person_id - ,procedure_concept_id - ,procedure_date - ,procedure_type_concept_id - ,modifier_concept_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,procedure_source_value - ,procedure_source_concept_id - ,qualifier_source_value -FROM #po_map; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE PROCEDURE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'PROCEDURE_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid PROCOEDURE_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - WHERE PROCEDURE_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('procedure type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - DRUG_EXPOSURE - - ****/ ---find valid drugs from drug_exposure table -IF OBJECT_ID('tempdb..#drgexp_map', 'U') IS NOT NULL - DROP TABLE #drgexp_map; - -/* / */ - -SELECT de.drug_exposure_id - ,de.person_id - ,de.drug_concept_id - ,de.drug_exposure_start_date - ,de.drug_exposure_end_date - ,de.drug_type_concept_id - ,de.stop_reason - ,de.refills - ,de.quantity - ,de.days_supply - ,de.sig - ,de.route_concept_id - ,de.effective_drug_dose - ,de.dose_unit_concept_id - ,de.lot_number - ,de.provider_id - ,de.visit_occurrence_id - ,de.drug_source_value - ,de.drug_source_concept_id - ,de.route_source_value - ,de.dose_unit_source_value - ,po.procedure_occurrence_id AS origional_procedure_id -INTO #drgexp_map -FROM [TARGET_CDMV5].[SCHEMA].drug_exposure de -LEFT JOIN [TARGET_CDMV5].[SCHEMA].procedure_occurrence po ON 1 = 0 -WHERE 0 = 1; - -/* / */ - -INSERT INTO #drgexp_map -SELECT drug_exposure_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') -INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -INNER JOIN #concept_map_distinct cmdis2 ON cm2.source_concept_id = cmdis2.source_concept_id - AND cm2.domain_id = cmdis2.domain_id - AND cmdis2.targetConceptCount = 1 -WHERE drug_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All drug exposures that did not map to a standard concept in V4 should also carry over to V5 -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -WHERE drug_concept_id = 0 - -UNION ALL - --- All drug exposures that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT drug_exposure_id - ,person_id - ,0 - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE -LEFT JOIN #concept_map cm1 ON drug_concept_id = cm1.source_concept_id -LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') -WHERE drug_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ,origional_procedure_id -FROM ( - --find valid drugs from drug_exposure table that map to > 1 target concept - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,COALESCE(cm2.target_concept_id, 0) drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,CAST(NULL AS INT) AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN #concept_map cm2 ON drug_exposure.drug_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('drug type', 'Type Concept') - - UNION ALL - - --find drugs that were previously classified as condition - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,condition_start_date AS drug_exposure_start_date - ,NULL AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,condition_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as procedure - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,procedure_date AS drug_exposure_start_date - ,CAST(NULL AS DATE) AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,procedure_occurrence_id AS origional_procedure_id - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - --find drugs that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS drug_concept_id - ,observation_date AS drug_exposure_start_date - ,CAST(NULL AS DATE) AS drug_exposure_end_date - ,0 AS drug_type_concept_id - ,NULL AS stop_reason - ,CAST(NULL AS INT) AS refills - ,CAST(NULL AS FLOAT) AS quantity - ,CAST(NULL AS INT) AS days_supply - ,NULL AS sig - ,CAST(NULL AS INT) AS route_concept_id - ,CAST(NULL AS FLOAT) AS effective_drug_dose - ,CAST(NULL AS INT) AS dose_unit_concept_id - ,NULL AS lot_number - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,observation_source_value AS drug_source_value - ,CAST(NULL AS INT) AS drug_source_concept_id - ,NULL AS route_source_value - ,NULL AS dose_unit_source_value - ,CAST(NULL AS INT) AS origional_procedure_id - ,observation_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('drug') - ) OTHERS - ,( - SELECT MAX(DRUG_EXPOSURE_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - ) MAXROW; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_exposure ( - drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value - ) -SELECT drug_exposure_id - ,person_id - ,drug_concept_id - ,drug_exposure_start_date - ,drug_exposure_end_date - ,drug_type_concept_id - ,stop_reason - ,refills - ,quantity - ,days_supply - ,sig - ,route_concept_id - ,effective_drug_dose - ,dose_unit_concept_id - ,lot_number - ,provider_id - ,visit_occurrence_id - ,drug_source_value - ,drug_source_concept_id - ,route_source_value - ,dose_unit_source_value -FROM #drgexp_map; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE DRUG_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'DRUG_EXPOSURE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid DRUG_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - WHERE DRUG_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('drug type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - CONDITION_OCCURRENCE - - ****/ ---find valid conditions from condition_occurrence table -INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_occurrence -SELECT condition_occurrence_id - ,person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') -INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount = 1 -LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') -WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - -UNION ALL - --- All conditions that did not map to a standard concept in V4 should also carry over to V5 -SELECT condition_occurrence_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(condition_type_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -WHERE condition_concept_id = 0 - -UNION ALL - --- All conditions that do not map to a standard concept in V5 should also carry over with condition_concept_id = 0 -SELECT condition_occurrence_id - ,person_id - ,0 AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id -FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE -LEFT JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id -LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') -WHERE condition_concept_id <> 0 - AND cm1.domain_id IS NULL - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_exposure_id - ,person_id - ,condition_concept_id - ,condition_start_date - ,condition_end_date - ,condition_type_concept_id - ,stop_reason - ,provider_id - ,visit_occurrence_id - ,condition_source_value - ,condition_source_concept_id -FROM ( - --find valid conditions from condition_occurrence table that map to > 1 target concept - SELECT person_id - ,COALESCE(cm1.target_concept_id, 0) AS condition_concept_id - ,condition_start_date - ,condition_end_date - ,COALESCE(cm2.target_concept_id, 0) AS condition_type_concept_id - ,stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,NULL AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - INNER JOIN #concept_map_distinct cmdis ON cm1.source_concept_id = cmdis.source_concept_id - AND cm1.domain_id = cmdis.domain_id - AND cmdis.targetConceptCount > 1 - LEFT JOIN #concept_map cm2 ON condition_occurrence.condition_type_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('condition type', 'Type Concept') - WHERE condition_concept_id > 0 -- This condition will map those concepts that were mapped to valid concepts in V4 - - UNION ALL - - --find conditions that were previously classified as procedure - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,procedure_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,procedure_occurrence_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as drug - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,drug_exposure_start_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,drug_exposure_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - --find conditions that were previously classified as observation - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS condition_concept_id - ,observation_date AS condition_start_date - ,NULL AS condition_end_date - ,0 AS condition_type_concept_id - ,NULL AS stop_reason - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value AS condition_source_value - ,CAST(NULL AS INT) condition_source_concept_id - ,observation_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('condition') - ) OTHERS - ,( - SELECT MAX(condition_occurrence_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - ) MAXROW; - ---warnings of invalid records -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDITION_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE CONDITION_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR STANDARD_CONCEPT = 'S' - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -INSERT INTO [TARGET_CDMV5].[SCHEMA].ETL_WARNINGS (WARNING_MESSAGE) -SELECT 'CONDIITON_OCCURRENCE: ' + CAST(NUM_INVALID_RECORDS AS VARCHAR) + ' records in the source CDMv4 database have invalid CONDITION_TYPE_CONCEPT_ID' -FROM ( - SELECT COUNT(PERSON_ID) AS NUM_INVALID_RECORDS - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - WHERE CONDITION_TYPE_CONCEPT_ID NOT IN ( - SELECT CONCEPT_ID - FROM [TARGET_CDMV5].[SCHEMA].CONCEPT - WHERE CONCEPT_ID = 0 - OR ( - STANDARD_CONCEPT = 'S' - AND LOWER(DOMAIN_ID) IN ('condition type', 'Type Concept') - ) - ) - HAVING COUNT(PERSON_ID) > 0 - ) warn; - -/**** - - DEVICE_EXPOSURE - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].device_exposure -SELECT row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS device_exposure_id - ,person_id - ,device_concept_id - ,device_exposure_start_date - ,device_exposure_end_date - ,device_type_concept_id - ,unique_device_id - ,quantity - ,provider_id - ,visit_occurrence_id - ,device_source_value - ,device_source_concept_id -FROM ( - --find devices that were previously classified as procedures - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,PROCEDURE_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,PROCEDURE_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,PROCEDURE_OCCURRENCE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as drug exposure - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,quantity - ,PRESCRIBING_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,DRUG_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,DRUG_EXPOSURE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as conditions - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,CONDITION_START_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,CONDITION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,CONDITION_OCCURRENCE_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - --find devices that were previously classified as observations - - UNION ALL - - SELECT PERSON_ID - ,cm1.target_concept_id AS DEVICE_CONCEPT_ID - ,OBSERVATION_DATE AS DEVICE_EXPOSURE_START_DATE - ,CAST(NULL AS DATE) AS DEVICE_EXPOSURE_END_DATE - ,0 AS DEVICE_TYPE_CONCEPT_ID - ,CAST(NULL AS VARCHAR(50)) unique_device_id - ,CAST(NULL AS INT) quantity - ,ASSOCIATED_PROVIDER_ID AS PROVIDER_ID - ,VISIT_OCCURRENCE_ID - ,OBSERVATION_SOURCE_VALUE AS DEVICE_SOURCE_VALUE - ,0 AS device_source_concept_id - ,OBSERVATION_ID AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('device') - ) OTHERS; - -/**** - - MEASUREMENT - - ****/ ---find valid measurements from observation table -INSERT INTO [TARGET_CDMV5].[SCHEMA].measurement -SELECT row_number() OVER ( - ORDER BY occurrence_id - ) AS measurement_id - ,person_id - ,measurement_concept_id - ,measurement_date - ,measurement_time - ,measurement_type_concept_id - ,operator_concept_id - ,value_as_number - ,value_as_concept_id - ,unit_concept_id - ,range_low - ,range_high - ,provider_id - ,visit_occurrence_id - ,measurement_source_value - ,measurement_source_concept_id - ,unit_source_value - ,value_source_value -FROM ( - --find mesaurements that were previously classified as observations - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,OBSERVATION_DATE AS measurement_date - ,CAST(OBSERVATION_TIME AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) operator_concept_id - ,value_as_number - ,value_as_concept_id - ,COALESCE(cm2.target_concept_id, 0) AS unit_concept_id - ,range_low - ,range_high - ,ASSOCIATED_PROVIDER_ID AS provider_id - ,visit_occurrence_id - ,OBSERVATION_SOURCE_VALUE AS measurement_source_value - ,CAST(NULL AS INT) measurement_source_concept_id - ,unit_source_value AS unit_source_value - ,cast(NULL AS VARCHAR(50)) AS value_source_value - ,observation_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - INNER JOIN #concept_map cm1 ON observation.observation_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - LEFT JOIN #concept_map cm2 ON observation.unit_concept_id = cm2.source_concept_id - AND LOWER(cm2.domain_id) IN ('unit') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,procedure_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,condition_start_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS measurement_concept_id - ,drug_exposure_start_date AS measurement_date - ,CAST(NULL AS VARCHAR(50)) AS measurement_time - ,0 AS measurement_type_concept_id - ,CAST(NULL AS INT) AS operator_concept_id - ,CAST(NULL AS INT) AS value_as_number - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS range_low - ,CAST(NULL AS INT) AS range_high - ,prescribing_provider_id AS provider_id - ,visit_occurrence_id - ,drug_source_value AS measurement_source_value - ,CAST(NULL AS INT) AS measurement_source_concept_id - ,CAST(NULL AS VARCHAR(50)) AS unit_source_value - ,CAST(NULL AS VARCHAR(50)) AS value_source_value - ,drug_exposure_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('measurement') - ) OTHERS; - -/**** - - OBSERVATION - - ****/ ---find valid observation from observation table -INSERT INTO [TARGET_CDMV5].[SCHEMA].observation -SELECT observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,CAST(observation_time AS VARCHAR(50)) AS observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value -FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION -WHERE observation_concept_id NOT IN ( - SELECT source_concept_id - FROM #concept_map_distinct - WHERE LOWER(domain_id) IN ( - 'condition' - ,'drug' - ,'procedure' - ,'device' - ,'measurement' - ) - ) ---find observations that were previously classified as procedure - -UNION ALL - -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS observation_id - ,person_id - ,observation_concept_id - ,observation_date - ,observation_time - ,observation_type_concept_id - ,value_as_number - ,value_as_string - ,value_as_concept_id - ,qualifier_concept_id - ,unit_concept_id - ,provider_id - ,visit_occurrence_id - ,observation_source_value - ,observation_source_concept_id - ,unit_source_value - ,qualifier_source_value -FROM ( - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,procedure_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,procedure_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE - INNER JOIN #concept_map cm1 ON procedure_occurrence.procedure_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - --find observations that were previously classified as condition - - UNION ALL - - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,condition_start_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,associated_provider_id AS provider_id - ,visit_occurrence_id - ,condition_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,condition_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].CONDITION_OCCURRENCE - INNER JOIN #concept_map cm1 ON condition_occurrence.condition_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find DRG observations that were previously classified as procedure_cost - SELECT po.person_id - ,cm1.target_concept_id AS observation_concept_id - ,po.procedure_date AS observation_date - ,CAST(NULL AS VARCHAR(50)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,po.associated_provider_id AS provider_id - ,po.visit_occurrence_id - ,pc.DISEASE_CLASS_SOURCE_VALUE AS observation_source_value - ,cm1.source_concept_id as observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,po.procedure_occurrence_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc - INNER JOIN [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po ON pc.PROCEDURE_OCCURRENCE_ID = po.PROCEDURE_OCCURRENCE_ID - INNER JOIN #concept_map cm1 ON pc.disease_class_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - - UNION ALL - - --find observations that were previously classified as drug exposure - SELECT person_id - ,cm1.target_concept_id AS observation_concept_id - ,drug_exposure_start_date AS observation_date - ,CAST(NULL AS VARCHAR(10)) AS observation_time - ,0 AS observation_type_concept_id - ,CAST(NULL AS FLOAT) AS value_as_number - ,NULL AS value_as_string - ,CAST(NULL AS INT) AS value_as_concept_id - ,CAST(NULL AS INT) qualifier_concept_id - ,CAST(NULL AS INT) AS unit_concept_id - ,CAST(NULL AS INT) AS provider_id - ,visit_occurrence_id - ,drug_source_value AS observation_source_value - ,CAST(NULL AS INT) observation_source_concept_id - ,NULL AS unit_source_value - ,cast(NULL AS VARCHAR(50)) qualifier_source_value - ,drug_exposure_id AS occurrence_id - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE - INNER JOIN #concept_map cm1 ON drug_exposure.drug_concept_id = cm1.source_concept_id - AND LOWER(cm1.domain_id) IN ('observation') - ) OTHERS - ,( - SELECT MAX(OBSERVATION_ID) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].OBSERVATION - ) MAXROW; - -/**** - - PAYER_PLAN_PERIOD - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].payer_plan_period -SELECT payer_plan_period_id - ,person_id - ,payer_plan_period_start_date - ,payer_plan_period_end_date - ,payer_source_value - ,plan_source_value - ,family_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PAYER_PLAN_PERIOD; - -/**** - - DRUG_COST - - note : if there were invalid drug concepts in DRUG_EXPOSURE, those records may not enter CDMv5 but costs will persist - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost -SELECT drug_cost_id - ,dc.drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id -FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc; - --- insert procedure costs for procedures that were inserted into the drug_exposure table -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS drug_cost_id - ,drug_exposure_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,ingredient_cost - ,dispensing_fee - ,average_wholesale_price - ,payer_plan_period_id -FROM ( - SELECT drug_exposure_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS FLOAT) AS ingredient_cost - ,CAST(NULL AS FLOAT) AS dispensing_fee - ,CAST(NULL AS FLOAT) AS average_wholesale_price - ,payer_plan_period_id - ,procedure_cost_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_OCCURRENCE po - INNER JOIN [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST pc ON po.procedure_occurrence_id = pc.procedure_occurrence_id - --JOIN dbo.drug_exposure de on de.person_id = po.person_id and pc.procedure_occurrence_id = de.origional_procedure_id - INNER JOIN #drgexp_map de ON de.person_id = po.person_id - AND pc.procedure_occurrence_id = de.origional_procedure_id - ) OTHERS - ,( - SELECT MAX(drug_cost_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST - ) MAXROW; - -/**** - - PROCEDURE_COST - - note : if there were invalid procedure concepts in PROCEDURE_OCCURRENCE, those records may not enter CDMv5 but costs will persist - - - ****/ -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost -SELECT procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value -FROM [SOURCE_CDMV4].[SCHEMA].PROCEDURE_COST; - --- insert drug costs for drugs that were inserted into the procedure_occurrence table -INSERT INTO [TARGET_CDMV5].[SCHEMA].procedure_cost -SELECT CASE - WHEN MAXROW.MAXROWID IS NULL - THEN 0 - ELSE MAXROW.MAXROWID - END + row_number() OVER ( - ORDER BY OCCURRENCE_ID - ) AS procedure_cost_id - ,procedure_occurrence_id - ,cast(NULL AS INT) currency_concept_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,payer_plan_period_id - ,revenue_code_concept_id - ,revenue_code_source_value -FROM ( - SELECT po.procedure_occurrence_id - ,po.person_id - ,paid_copay - ,paid_coinsurance - ,paid_toward_deductible - ,paid_by_payer - ,paid_by_coordination_benefits - ,total_out_of_pocket - ,total_paid - ,CAST(NULL AS FLOAT) AS ingredient_cost - ,CAST(NULL AS FLOAT) AS dispensing_fee - ,CAST(NULL AS FLOAT) AS average_wholesale_price - ,payer_plan_period_id - ,CAST(NULL AS INT) AS revenue_code_concept_id - ,CAST(NULL AS INT) AS revenue_code_source_value - ,drug_cost_id AS OCCURRENCE_ID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_EXPOSURE de - INNER JOIN [SOURCE_CDMV4].[SCHEMA].DRUG_COST dc ON de.drug_exposure_id = dc.drug_exposure_id - --JOIN dbo.procedure_occurrence po on de.person_id = po.person_id and de.drug_exposure_id = po.origional_drug_id - INNER JOIN #po_map po ON de.person_id = po.person_id - AND de.drug_exposure_id = po.origional_drug_id - ) OTHERS - ,( - SELECT MAX(drug_cost_id) AS MAXROWID - FROM [SOURCE_CDMV4].[SCHEMA].DRUG_COST - ) MAXROW; - -/**** - -DRUG ERA -Note: Eras derived from DRUG_EXPOSURE table, using 30d gap - - ****/ -IF OBJECT_ID('tempdb..#cteDrugTarget', 'U') IS NOT NULL - DROP TABLE #cteDrugTarget; - -/* / */ - --- Normalize DRUG_EXPOSURE_END_DATE to either the existing drug exposure end date, or add days supply, or add 1 day to the start date -SELECT d.DRUG_EXPOSURE_ID - ,d.PERSON_ID - ,c.CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE - ,COALESCE(DRUG_EXPOSURE_END_DATE, DATEADD(day, DAYS_SUPPLY, DRUG_EXPOSURE_START_DATE), DATEADD(day, 1, DRUG_EXPOSURE_START_DATE)) AS DRUG_EXPOSURE_END_DATE - ,c.CONCEPT_ID AS INGREDIENT_CONCEPT_ID -INTO #cteDrugTarget -FROM [TARGET_CDMV5].[SCHEMA].DRUG_EXPOSURE d -INNER JOIN [TARGET_CDMV5].[SCHEMA].CONCEPT_ANCESTOR ca ON ca.DESCENDANT_CONCEPT_ID = d.DRUG_CONCEPT_ID -INNER JOIN [TARGET_CDMV5].[SCHEMA].CONCEPT c ON ca.ANCESTOR_CONCEPT_ID = c.CONCEPT_ID -WHERE c.VOCABULARY_ID = 'RxNorm' - AND c.CONCEPT_CLASS_ID = 'Ingredient'; - -/* / */ - -IF OBJECT_ID('tempdb..#cteEndDates', 'U') IS NOT NULL - DROP TABLE #cteEndDates; - -/* / */ - -SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DATEADD(day, - 30, EVENT_DATE) AS END_DATE -- unpad the end date -INTO #cteEndDates -FROM ( - SELECT E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM ( - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM ( - -- select the start dates, assigning a row number to each - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,0 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM #cteDrugTarget - - UNION ALL - - -- add the end dates with NULL as the row number, padding the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DATEADD(day, 30, DRUG_EXPOSURE_END_DATE) - ,1 AS EVENT_TYPE - ,NULL - FROM #cteDrugTarget - ) RAWDATA - ) E1 - INNER JOIN ( - SELECT PERSON_ID - ,INGREDIENT_CONCEPT_ID - ,DRUG_EXPOSURE_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,INGREDIENT_CONCEPT_ID ORDER BY DRUG_EXPOSURE_START_DATE - ) AS START_ORDINAL - FROM #cteDrugTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.INGREDIENT_CONCEPT_ID = E2.INGREDIENT_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.INGREDIENT_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E -WHERE 2 * E.START_ORDINAL - E.OVERALL_ORD = 0; - -/* / */ - -IF OBJECT_ID('tempdb..#cteDrugExpEnds', 'U') IS NOT NULL - DROP TABLE #cteDrugExpEnds; - -/* / */ - -SELECT d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE -INTO #cteDrugExpEnds -FROM #cteDrugTarget d -INNER JOIN #cteEndDates e ON d.PERSON_ID = e.PERSON_ID - AND d.INGREDIENT_CONCEPT_ID = e.INGREDIENT_CONCEPT_ID - AND e.END_DATE >= d.DRUG_EXPOSURE_START_DATE -GROUP BY d.PERSON_ID - ,d.INGREDIENT_CONCEPT_ID - ,d.DRUG_TYPE_CONCEPT_ID - ,d.DRUG_EXPOSURE_START_DATE; - -/* / */ - -INSERT INTO [TARGET_CDMV5].[SCHEMA].drug_era -SELECT row_number() OVER ( - ORDER BY person_id - ) AS drug_era_id - ,person_id - ,INGREDIENT_CONCEPT_ID - ,min(DRUG_EXPOSURE_START_DATE) AS drug_era_start_date - ,ERA_END_DATE - ,COUNT(*) AS DRUG_EXPOSURE_COUNT - ,30 AS gap_days -FROM #cteDrugExpEnds -GROUP BY person_id - ,INGREDIENT_CONCEPT_ID - ,drug_type_concept_id - ,ERA_END_DATE; - -/**** - -CONDITION ERA -Note: Eras derived from CONDITION_OCCURRENCE table, using 30d gap - - ****/ -IF OBJECT_ID('tempdb..#condition_era_phase_1', 'U') IS NOT NULL - DROP TABLE #condition_era_phase_1; - -/* / */ - -IF OBJECT_ID('tempdb..#cteConditionTarget', 'U') IS NOT NULL - DROP TABLE #cteConditionTarget; - -/* / */ - --- create base eras from the concepts found in condition_occurrence -SELECT co.PERSON_ID - ,co.condition_concept_id - ,co.CONDITION_START_DATE - ,COALESCE(co.CONDITION_END_DATE, DATEADD(day, 1, CONDITION_START_DATE)) AS CONDITION_END_DATE -INTO #cteConditionTarget -FROM [TARGET_CDMV5].[SCHEMA].CONDITION_OCCURRENCE co; - -/* / */ - -IF OBJECT_ID('tempdb..#cteCondEndDates', 'U') IS NOT NULL - DROP TABLE #cteCondEndDates; - -/* / */ - -SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,DATEADD(day, - 30, EVENT_DATE) AS END_DATE -- unpad the end date -INTO #cteCondEndDates -FROM ( - SELECT E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,COALESCE(E1.START_ORDINAL, MAX(E2.START_ORDINAL)) START_ORDINAL - ,E1.OVERALL_ORD - FROM ( - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,EVENT_DATE - ,EVENT_TYPE - ,START_ORDINAL - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY EVENT_DATE - ,EVENT_TYPE - ) AS OVERALL_ORD -- this re-numbers the inner UNION so all rows are numbered ordered by the event date - FROM ( - -- select the start dates, assigning a row number to each - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,- 1 AS EVENT_TYPE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM #cteConditionTarget - - UNION ALL - - -- pad the end dates by 30 to allow a grace period for overlapping ranges. - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,DATEADD(day, 30, CONDITION_END_DATE) - ,1 AS EVENT_TYPE - ,NULL - FROM #cteConditionTarget - ) RAWDATA - ) E1 - INNER JOIN ( - SELECT PERSON_ID - ,CONDITION_CONCEPT_ID - ,CONDITION_START_DATE AS EVENT_DATE - ,ROW_NUMBER() OVER ( - PARTITION BY PERSON_ID - ,CONDITION_CONCEPT_ID ORDER BY CONDITION_START_DATE - ) AS START_ORDINAL - FROM #cteConditionTarget - ) E2 ON E1.PERSON_ID = E2.PERSON_ID - AND E1.CONDITION_CONCEPT_ID = E2.CONDITION_CONCEPT_ID - AND E2.EVENT_DATE <= E1.EVENT_DATE - GROUP BY E1.PERSON_ID - ,E1.CONDITION_CONCEPT_ID - ,E1.EVENT_DATE - ,E1.START_ORDINAL - ,E1.OVERALL_ORD - ) E -WHERE (2 * E.START_ORDINAL) - E.OVERALL_ORD = 0; - -/* / */ - -IF OBJECT_ID('tempdb..#cteConditionEnds', 'U') IS NOT NULL - DROP TABLE #cteConditionEnds; - -/* / */ - -SELECT c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE - ,MIN(e.END_DATE) AS ERA_END_DATE -INTO #cteConditionEnds -FROM #cteConditionTarget c -INNER JOIN #cteCondEndDates e ON c.PERSON_ID = e.PERSON_ID - AND c.CONDITION_CONCEPT_ID = e.CONDITION_CONCEPT_ID - AND e.END_DATE >= c.CONDITION_START_DATE -GROUP BY c.PERSON_ID - ,c.CONDITION_CONCEPT_ID - ,c.CONDITION_START_DATE; - -/* / */ - -INSERT INTO [TARGET_CDMV5].[SCHEMA].condition_era ( - condition_era_id - ,person_id - ,condition_concept_id - ,condition_era_start_date - ,condition_era_end_date - ,condition_occurrence_count - ) -SELECT row_number() OVER ( - ORDER BY person_id - ) AS condition_era_id - ,person_id - ,CONDITION_CONCEPT_ID - ,min(CONDITION_START_DATE) AS CONDITION_ERA_START_DATE - ,ERA_END_DATE AS CONDITION_ERA_END_DATE - ,COUNT(*) AS CONDITION_OCCURRENCE_COUNT -FROM #cteConditionEnds -GROUP BY person_id - ,CONDITION_CONCEPT_ID - ,ERA_END_DATE; - -/**** - -QUALITY ASSURANCE OUTPUT - -Note: These queries are used to provide some basic stats around row counts between your V4 and V5 database - to ensure that all of the data has migrated as expected. - - ****/ - -IF OBJECT_ID('tempdb..#v5_stats', 'U') IS NOT NULL - DROP TABLE #v5_stats; - -/* / */ - -IF OBJECT_ID('tempdb..#v4_stats', 'U') IS NOT NULL - DROP TABLE #v4_stats; - -/* / */ - --- Get the row counts for each table that is in scope for the migration -SELECT * -INTO #v4_stats -FROM -( - SELECT '[SOURCE_CDMV4]' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].care_site - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].condition_era - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].condition_occurrence - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].death - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_cost - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_era - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].drug_exposure - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].location - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].observation - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].observation_period - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].payer_plan_period - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].person - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].procedure_cost - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].procedure_occurrence - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].provider - UNION - SELECT '[SOURCE_CDMV4]' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM [SOURCE_CDMV4].[SCHEMA].visit_occurrence -) v4_stats; - -/* / */ - -SELECT * -INTO #v5_stats -FROM -( - SELECT '[TARGET_CDMV5]' AS DBName, 'care_site' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].care_site - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'condition_era' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].condition_era - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'condition_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].condition_occurrence - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'death' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].death - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'device_exposure' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].device_exposure - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_cost' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_cost - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_era' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_era - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'drug_exposure' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].drug_exposure - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'location' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].location - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'measurement' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].measurement - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'observation' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].observation - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'observation_period' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].observation_period - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'payer_plan_period' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].payer_plan_period - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'person' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].person - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'procedure_cost' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].procedure_cost - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'procedure_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].procedure_occurrence - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'provider' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].provider - UNION - SELECT '[TARGET_CDMV5]' AS DBName, 'visit_occurrence' AS TableName, COUNT(*) as row_count FROM [TARGET_CDMV5].[SCHEMA].visit_occurrence -) v5_stats; - -/* / */ - --- Show the results -select - 'Rowcounts for each database and table', - ISNULL(V4.DBName, 'None') v4_database_name, - v4.TableName v4_table_name, - v4.row_count v4_row_count, - ISNULL(v5.DBName, 'None') v5_database_name, - v5.TableName v5_table_name, - v5.row_count v5_row_count, - ISNULL(v5.row_count, 0) - ISNULL(v4.row_count, 0) row_count_change -from #v4_stats v4 -full outer join #v5_stats v5 ON v4.TableName = v5.TableName -order by v5.TableName; - -/* - * Determine how the vocabulary/domains helped to map from the V4 source - * tables to the V5 destinations - */ -IF OBJECT_ID('tempdb..#classification_map', 'U') IS NOT NULL - DROP TABLE #classification_map; - -/* / */ - -SELECT * -INTO #classification_map -FROM -( - SELECT 'Condition_Occurrence' TableName, ISNULL(LOWER(cm.domain_id), 'condition') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Condition_Occurrence CO - LEFT JOIN #concept_map CM ON co.condition_concept_id = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'condition') - UNION - SELECT 'Drug_Exposure' TableName, ISNULL(LOWER(cm.domain_id), 'drug') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Drug_Exposure de - LEFT JOIN #concept_map CM ON de.drug_concept_id = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'drug') - UNION - SELECT 'Observation' TableName, ISNULL(LOWER(cm.domain_id), 'observation') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Observation o - LEFT JOIN #concept_map CM ON o.observation_concept_id = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'observation') - UNION - SELECT 'Procedure_Occurrence' TableName, ISNULL(LOWER(cm.domain_id), 'procedure') domain_id, COUNT(*) row_count - FROM [SOURCE_CDMV4].[SCHEMA].Procedure_Occurrence po - LEFT JOIN #concept_map CM ON po.PROCEDURE_CONCEPT_ID = cm.source_concept_id - GROUP BY ISNULL(LOWER(cm.domain_id), 'procedure') -) A -ORDER by A.TableName, A.domain_id; - -/* / */ - -select * -from #classification_map -order by tablename, domain_id; - -select domain_id, SUM(row_count) -from #classification_map -group by domain_id -order by domain_id; \ No newline at end of file diff --git a/Version4/Oracle/CDM V4 DDL.sql b/Version4/Oracle/CDM V4 DDL.sql deleted file mode 100644 index 5c4b794..0000000 --- a/Version4/Oracle/CDM V4 DDL.sql +++ /dev/null @@ -1,310 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License.4 -********************************************************************************/ - -/************************ - - ####### # # ####### ###### ##### ###### # # ## ####### - # # ## ## # # # # # # # # ## ## # # # # # - # # # # # # # # # # # # # # # # # # # # # # - # # # # # # # ###### # # # # # # # # ####### ####### - # # # # # # # # # # # # # # # # - # # # # # # # # # # # # # # # # # # - ####### # # ####### # ##### ###### # # ## # # ##### - - -script to create OMOP common data model, version 5.0 for Oracle database - -last revised: 1 Mar 2015 - -author: Christian Reich - - -*************************/ - -CREATE TABLE concept -( - concept_id INTEGER NOT NULL, - concept_name VARCHAR2(256 BYTE) NOT NULL, - concept_level NUMBER NOT NULL, - concept_class VARCHAR2(60 BYTE) NOT NULL, - vocabulary_id INTEGER NOT NULL, - concept_code VARCHAR2(40 BYTE) NOT NULL, - valid_start_date DATE NOT NULL, - valid_end_date DATE DEFAULT '31-Dec-2099' NOT NULL, - invalid_reason CHAR(1 BYTE) -) NOLOGGING; - -COMMENT ON TABLE concept IS 'A list of all valid terminology concepts across domains and their attributes. Concepts are derived from existing standards.'; - -COMMENT ON COLUMN concept.concept_id IS 'A system-generated identifier to uniquely identify each concept across all concept types.'; - -COMMENT ON COLUMN concept.concept_name IS 'An unambiguous, meaningful and descriptive name for the concept.'; - -COMMENT ON COLUMN concept.concept_level IS 'The level of hierarchy associated with the concept. Different concept levels are assigned to concepts to depict their seniority in a clearly defined hierarchy, such as drugs, conditions, etc. A concept level of 0 is assigned to concepts that are not part of a standard vocabulary, but are part of the vocabulary for reference purposes (e.g. drug form).'; - -COMMENT ON COLUMN concept.concept_class IS 'The category or class of the concept along both the hierarchical tree as well as different domains within a vocabulary. Examples are ''Clinical Drug'', ''Ingredient'', ''Clinical Finding'' etc.'; - -COMMENT ON COLUMN concept.vocabulary_id IS 'A foreign key to the vocabulary table indicating from which source the concept has been adapted.'; - -COMMENT ON COLUMN concept.concept_code IS 'The concept code represents the identifier of the concept in the source data it originates from, such as SNOMED-CT concept IDs, RxNorm RXCUIs etc. Note that concept codes are not unique across vocabularies.'; - -COMMENT ON COLUMN concept.valid_start_date IS 'The date when the was first recorded.'; - -COMMENT ON COLUMN concept.valid_end_date IS 'The date when the concept became invalid because it was deleted or superseded (updated) by a new concept. The default value is 31-Dec-2099.'; - -COMMENT ON COLUMN concept.invalid_reason IS 'Concepts that are replaced with a new concept are designated "Updated" (U) and concepts that are removed without replacement are "Deprecated" (D).'; - -CREATE INDEX concept_code ON concept (concept_code, vocabulary_id); -CREATE UNIQUE INDEX XPKconcept ON concept (concept_id); - -ALTER TABLE concept ADD ( - CHECK ( invalid_reason IN ('D', 'U')) - ENABLE VALIDATE, - CONSTRAINT XPKCONCEPT - PRIMARY KEY - (concept_id) - USING INDEX XPKCONCEPT - ENABLE VALIDATE); - ---add table RELATIONSHIP - -CREATE TABLE relationship -( - relationship_id INTEGER NOT NULL, - relationship_name VARCHAR2(256 BYTE) NOT NULL, - is_hierarchical INTEGER NOT NULL, - defines_ancestry INTEGER DEFAULT 1 NOT NULL, - reverse_relationship INTEGER -) NOLOGGING; - -COMMENT ON TABLE relationship IS 'A list of relationship between concepts. Some of these relationships are generic (e.g. "Subsumes" relationship), others are domain-specific.'; - -COMMENT ON COLUMN relationship.relationship_id IS 'The type of relationship captured by the relationship record.'; - -COMMENT ON COLUMN relationship.relationship_name IS 'The text that describes the relationship type.'; - -COMMENT ON COLUMN relationship.is_hierarchical IS 'Defines whether a relationship defines concepts into classes or hierarchies. Values are Y for hierarchical relationship or NULL if not'; - -COMMENT ON COLUMN relationship.defines_ancestry IS 'Defines whether a hierarchical relationship contributes to the concept_ancestor table. These are subsets of the hierarchical relationships. Valid values are Y or NULL.'; - -COMMENT ON COLUMN relationship.reverse_relationship IS 'relationship ID of the reverse relationship to this one. Corresponding records of reverse relationships have their concept_id_1 and concept_id_2 swapped.'; - -CREATE UNIQUE INDEX XPKRELATIONHIP_TYPE ON relationship -(relationship_id); - -ALTER TABLE relationship ADD ( - CONSTRAINT xpkrelationship_type - PRIMARY KEY - (relationship_id) - USING INDEX xpkrelationship_type - ENABLE VALIDATE); - ---add table concept_relationship - -CREATE TABLE concept_relationship -( - concept_id_1 INTEGER NOT NULL, - concept_id_2 INTEGER NOT NULL, - relationship_id INTEGER NOT NULL, - valid_start_date DATE NOT NULL, - valid_end_date DATE DEFAULT '31-Dec-2099' NOT NULL, - invalid_reason CHAR(1 BYTE) -) NOLOGGING; - -COMMENT ON TABLE concept_relationship IS 'A list of relationship between concepts. Some of these relationships are generic (e.g. ''Subsumes'' relationship), others are domain-specific.'; - -COMMENT ON COLUMN concept_relationship.concept_id_1 IS 'A foreign key to the concept in the concept table associated with the relationship. relationships are directional, and this field represents the source concept designation.'; - -COMMENT ON COLUMN concept_relationship.concept_id_2 IS 'A foreign key to the concept in the concept table associated with the relationship. relationships are directional, and this field represents the destination concept designation.'; - -COMMENT ON COLUMN concept_relationship.relationship_id IS 'The type of relationship as defined in the relationship table.'; - -COMMENT ON COLUMN concept_relationship.valid_start_date IS 'The date when the the relationship was first recorded.'; - -COMMENT ON COLUMN concept_relationship.valid_end_date IS 'The date when the relationship became invalid because it was deleted or superseded (updated) by a new relationship. Default value is 31-Dec-2099.'; - -COMMENT ON COLUMN concept_relationship.invalid_reason IS 'Reason the relationship was invalidated. Possible values are D (deleted), U (replaced with an update) or NULL when valid_end_date has the default value.'; - -CREATE UNIQUE INDEX xpkconcept_relationship ON concept_relationship -(concept_id_1, concept_id_2, relationship_id); - - -ALTER TABLE concept_relationship ADD ( - CHECK ( invalid_reason IN ('D', 'U')) - ENABLE VALIDATE, - CHECK ( invalid_reason IN ('D', 'U')) - ENABLE VALIDATE, - CHECK (invalid_reason in ('D', 'U')) - ENABLE VALIDATE, - CONSTRAINT xpkconcept_relationship - PRIMARY KEY - (concept_id_1, concept_id_2, relationship_id) - USING INDEX xpkconcept_relationship - ENABLE VALIDATE); - - -ALTER TABLE concept_relationship ADD ( - CONSTRAINT concept_REL_CHILD_FK - FOREIGN KEY (concept_id_2) - REFERENCES concept (concept_id) - ENABLE VALIDATE, - CONSTRAINT concept_REL_PARENT_FK - FOREIGN KEY (concept_id_1) - REFERENCES concept (concept_id) - ENABLE VALIDATE, - CONSTRAINT concept_REL_REL_type_FK - FOREIGN KEY (relationship_id) - REFERENCES relationship (relationship_id) - ENABLE VALIDATE); - ---add table concept_ancestor - -CREATE TABLE concept_ancestor -( - ancestor_concept_id INTEGER NOT NULL, - descendant_concept_id INTEGER NOT NULL, - max_levels_of_separation NUMBER, - min_levels_of_separation NUMBER -) NOLOGGING; - -COMMENT ON TABLE concept_ancestor IS 'A specialized table containing only hierarchical relationship between concepts that may span several generations.'; - -COMMENT ON COLUMN concept_ancestor.ancestor_concept_id IS 'A foreign key to the concept code in the concept table for the higher-level concept that forms the ancestor in the relationship.'; - -COMMENT ON COLUMN concept_ancestor.descendant_concept_id IS 'A foreign key to the concept code in the concept table for the lower-level concept that forms the descendant in the relationship.'; - -COMMENT ON COLUMN concept_ancestor.max_levels_of_separation IS 'The maximum separation in number of levels of hierarchy between ancestor and descendant concepts. This is an optional attribute that is used to simplify hierarchic analysis. '; - -COMMENT ON COLUMN concept_ancestor.min_levels_of_separation IS 'The minimum separation in number of levels of hierarchy between ancestor and descendant concepts. This is an optional attribute that is used to simplify hierarchic analysis.'; - -CREATE UNIQUE INDEX xpkconcept_ancestor ON concept_ancestor -(ancestor_concept_id, descendant_concept_id); - -ALTER TABLE concept_ancestor ADD ( - CONSTRAINT xpkconcept_ancestor - PRIMARY KEY - (ancestor_concept_id, descendant_concept_id) - USING INDEX xpkconcept_ancestor - ENABLE VALIDATE); - -ALTER TABLE concept_ancestor ADD ( - CONSTRAINT concept_ancestor_FK - FOREIGN KEY (ancestor_concept_id) - REFERENCES concept (concept_id) - ENABLE VALIDATE, - CONSTRAINT concept_descendant_FK - FOREIGN KEY (descendant_concept_id) - REFERENCES concept (concept_id) - ENABLE VALIDATE); - ---add table concept_synonym - -CREATE TABLE concept_synonym -( - concept_synonym_id INTEGER NOT NULL, - concept_id INTEGER NOT NULL, - concept_synonym_name VARCHAR2(1000 BYTE) NOT NULL -) NOLOGGING; - -COMMENT ON TABLE concept_synonym IS 'A table with synonyms for concepts that have more than one valid name or description.'; - -COMMENT ON COLUMN concept_synonym.concept_synonym_id IS 'A system-generated unique identifier for each concept synonym.'; - -COMMENT ON COLUMN concept_synonym.concept_id IS 'A foreign key to the concept in the concept table. '; - -COMMENT ON COLUMN concept_synonym.concept_synonym_name IS 'The alternative name for the concept.'; - -CREATE UNIQUE INDEX xpkconcept_synonym ON concept_synonym -(concept_synonym_id); - -ALTER TABLE concept_synonym ADD ( - CONSTRAINT xpkconcept_synonym - PRIMARY KEY - (concept_synonym_id) - USING INDEX xpkconcept_synonym - ENABLE VALIDATE); - -ALTER TABLE concept_synonym ADD ( - CONSTRAINT concept_synonym_concept_FK - FOREIGN KEY (concept_id) - REFERENCES concept (concept_id) - ENABLE VALIDATE); - ---add table source_to_concept_map - -CREATE TABLE source_to_concept_map -( - source_code VARCHAR2(40 BYTE) NOT NULL, - source_vocabulary_id INTEGER NOT NULL, - source_code_description VARCHAR2(256 BYTE), - target_concept_id INTEGER NOT NULL, - target_vocabulary_id INTEGER NOT NULL, - mapping_type VARCHAR2(256 BYTE), - primary_map CHAR(1 BYTE), - valid_start_date DATE NOT NULL, - valid_end_date DATE NOT NULL, - invalid_reason CHAR(1 BYTE) -) NOLOGGING; - -CREATE INDEX SOURCE_TO_concept_SOURCE_idX ON source_to_concept_map -(SOURCE_CODE); - -CREATE UNIQUE INDEX xpksource_to_concept_map ON source_to_concept_map -(SOURCE_vocabulary_id, TARGET_concept_id, SOURCE_CODE, valid_end_date); - -ALTER TABLE source_to_concept_map ADD ( - CHECK (primary_map in ('Y')) - ENABLE VALIDATE, - CHECK (invalid_reason in ('D', 'U')) - ENABLE VALIDATE, - CONSTRAINT xpksource_to_concept_map - PRIMARY KEY - (SOURCE_vocabulary_id, TARGET_concept_id, SOURCE_CODE, valid_end_date) - USING INDEX xpksource_to_concept_map - ENABLE VALIDATE); - -ALTER TABLE source_to_concept_map ADD ( - CONSTRAINT SOURCE_TO_concept_concept - FOREIGN KEY (TARGET_concept_id) - REFERENCES concept (concept_id) - ENABLE VALIDATE); - ---add table drug_strength - -CREATE TABLE drug_strength -( - drug_concept_id INTEGER NOT NULL, - ingredient_concept_id INTEGER NOT NULL, - amount_value NUMBER, - amount_unit VARCHAR2 (60 BYTE), - concentration_value NUMBER, - concentration_enum_unit VARCHAR2 (60 BYTE), - concentration_denom_unit VARCHAR2 (60 BYTE), - valid_start_date DATE NOT NULL, - valid_end_date DATE NOT NULL, - invalid_reason VARCHAR2 (1 BYTE) -); - ---add table vocabulary - -CREATE TABLE VOCABULARY -( - VOCABULARY_ID INTEGER NOT NULL, - VOCABULARY_NAME VARCHAR2 (256 BYTE) NOT NULL -); - - \ No newline at end of file diff --git a/Version4/Oracle/README.md b/Version4/Oracle/README.md deleted file mode 100644 index 4b362b0..0000000 --- a/Version4/Oracle/README.md +++ /dev/null @@ -1,14 +0,0 @@ -Common-Data-Model / Oracle -================= - -This folder contains the SQL scripts for Oracle. - -In order to create your instantiation of the Common Data Model, we recommend following these steps: - -1. Create an empty schema. - -2. Execute the script `CDM V4 ddl.sql` to create the tables and fields. - -3. Load your data into the schema using the loading scripts in VocabImport - -Note: you could also apply the constraints and the indexes after loading the data, this will speed up the insertion of the data considerably. diff --git a/Version4/Oracle/VocabImport/OMOP CDM vocabulary load - Oracle.bat b/Version4/Oracle/VocabImport/OMOP CDM vocabulary load - Oracle.bat deleted file mode 100644 index 5d621d5..0000000 --- a/Version4/Oracle/VocabImport/OMOP CDM vocabulary load - Oracle.bat +++ /dev/null @@ -1,51 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License.4 -********************************************************************************/ - -/************************ - - ####### # # ####### ###### ##### ###### # # ## ####### - # # ## ## # # # # # # # # ## ## # # # # # - # # # # # # # # # # # # # # # # # # # # # # - # # # # # # # ###### # # # # # # # # ####### ####### - # # # # # # # # # # # # # # # # - # # # # # # # # # # # # # # # # # # - ####### # # ####### # ##### ###### # # ## # # ##### - - -script to load the Vocabulary related tables in the OMOP common data model, version 4.5 for Oracle database - -last revised: 19 Mar 2015 - -author: Lee Evans - -Notes - -1) This script assumes the CDM version 4.5 vocabulary zip file has been unzipped into the "C:\CDM" directory. -2) If you unzipped your CDM version 4.5 vocabulary files into a different directory then replace all file paths below, with your directory path. -3) If you have existing data in your CDM vocabulary tables then backup that data (if needed) and truncate those tables before loading - - -*************************/ - -sqlldr CDM/ CONTROL=CONCEPT.ctl LOG=C:\CDM\CONCEPT.log BAD=C:\CDM\CONCEPT.bad -sqlldr CDM/ CONTROL=CONCEPT_ANCESTOR.ctl LOG=C:\CDM\CONCEPT_ANCESTOR.log BAD=C:\CDM\CONCEPT_ANCESTOR.bad -sqlldr CDM/ CONTROL=CONCEPT_RELATIONSHIP.ctl LOG=C:\CDM\CONCEPT_RELATIONSHIP.log BAD=C:\CDM\CONCEPT_RELATIONSHIP.bad -sqlldr CDM/ CONTROL=CONCEPT_SYNONYM.ctl LOG=C:\CDM\CONCEPT_SYNONYM.log BAD=C:\CDM\CONCEPT_SYNONYM.bad -sqlldr CDM/ CONTROL=DRUG_STRENGTH.ctl LOG=C:\CDM\DRUG_STRENGTH.log BAD=C:\CDM\DRUG_STRENGTH.bad -sqlldr CDM/ CONTROL=RELATIONSHIP.ctl LOG=C:\CDM\RELATIONSHIP.log BAD=C:\CDM\RELATIONSHIP.bad -sqlldr CDM/ CONTROL=VOCABULARY.ctl LOG=C:\CDM\VOCABULARY.log BAD=C:\CDM\VOCABULARY.bad -sqlldr CDM/ CONTROL=SOURCE_TO_CONCEPT_MAP.ctl LOG=C:\CDM\SOURCE_TO_CONCEPT_MAP.log BAD=C:\CDM\SOURCE_TO_CONCEPT_MAP.bad diff --git a/Version4/Oracle/VocabImport/concept.ctl b/Version4/Oracle/VocabImport/concept.ctl deleted file mode 100644 index d18106a..0000000 --- a/Version4/Oracle/VocabImport/concept.ctl +++ /dev/null @@ -1,18 +0,0 @@ -options (skip=1) -load data -infile concept.csv -into table concept -replace -fields terminated by '\t' -trailing nullcols -( - concept_id, - concept_name CHAR(256), - concept_level, - concept_class, - vocabulary_id, - concept_code, - valid_start_date DATE 'YYYYMMDD', - valid_end_date DATE 'YYYYMMDD', - invalid_reason -) \ No newline at end of file diff --git a/Version4/Oracle/VocabImport/concept_ancestor.ctl b/Version4/Oracle/VocabImport/concept_ancestor.ctl deleted file mode 100644 index 350015a..0000000 --- a/Version4/Oracle/VocabImport/concept_ancestor.ctl +++ /dev/null @@ -1,13 +0,0 @@ -options (skip=1) -load data -infile concept_ancestor.csv -into table concept_ancestor -replace -fields terminated by '\t' -trailing nullcols -( - ancestor_concept_id, - descendant_concept_id, - min_levels_of_separation, - max_levels_of_separation -) \ No newline at end of file diff --git a/Version4/Oracle/VocabImport/concept_relationship.ctl b/Version4/Oracle/VocabImport/concept_relationship.ctl deleted file mode 100644 index d26ce3e..0000000 --- a/Version4/Oracle/VocabImport/concept_relationship.ctl +++ /dev/null @@ -1,15 +0,0 @@ -options (skip=1) -load data -infile concept_relationship.csv -into table concept_relationship -replace -fields terminated by '\t' -trailing nullcols -( - concept_id_1, - concept_id_2, - relationship_id, - valid_start_date DATE 'YYYYMMDD', - valid_end_date DATE 'YYYYMMDD', - invalid_reason -) \ No newline at end of file diff --git a/Version4/Oracle/VocabImport/concept_synonym.ctl b/Version4/Oracle/VocabImport/concept_synonym.ctl deleted file mode 100644 index a7ef6fa..0000000 --- a/Version4/Oracle/VocabImport/concept_synonym.ctl +++ /dev/null @@ -1,12 +0,0 @@ -options (skip=1) -load data -infile concept_synonym.csv -into table concept_synonym -replace -fields terminated by '\t' -trailing nullcols -( - concept_synonym_id, - concept_id, - concept_synonym_name CHAR(1000) -) \ No newline at end of file diff --git a/Version4/Oracle/VocabImport/drug_strength.ctl b/Version4/Oracle/VocabImport/drug_strength.ctl deleted file mode 100644 index dc858c3..0000000 --- a/Version4/Oracle/VocabImport/drug_strength.ctl +++ /dev/null @@ -1,19 +0,0 @@ -options (skip=1) -load data -infile drug_strength.csv -into table drug_strength -replace -fields terminated by '\t' -trailing nullcols -( - drug_concept_id, - ingredient_concept_id, - amount_value, - amount_unit, - concentration_value, - concentration_enum_unit, - concentration_denom_unit, - valid_start_date DATE 'YYYYMMDD', - valid_end_date DATE 'YYYYMMDD', - invalid_reason -) \ No newline at end of file diff --git a/Version4/Oracle/VocabImport/relationship.ctl b/Version4/Oracle/VocabImport/relationship.ctl deleted file mode 100644 index 40daf29..0000000 --- a/Version4/Oracle/VocabImport/relationship.ctl +++ /dev/null @@ -1,14 +0,0 @@ -options (skip=1) -load data -infile relationship.csv -into table relationship -replace -fields terminated by '\t' -trailing nullcols -( - relationship_id, - relationship_name, - is_hierarchical, - defines_ancestry, - reverse_relationship -) \ No newline at end of file diff --git a/Version4/Oracle/VocabImport/source_to_concept_map.ctl b/Version4/Oracle/VocabImport/source_to_concept_map.ctl deleted file mode 100644 index d9e0ef0..0000000 --- a/Version4/Oracle/VocabImport/source_to_concept_map.ctl +++ /dev/null @@ -1,19 +0,0 @@ -options (skip=1) -load data -infile source_to_concept_map.csv -into table source_to_concept_map -replace -fields terminated by '\t' -trailing nullcols -( - source_code, - source_vocabulary_id, - source_code_description CHAR(256), - target_concept_id, - target_vocabulary_id, - mapping_type, - primary_map, - valid_start_date DATE 'YYYYMMDD', - valid_end_date DATE 'YYYYMMDD', - invalid_reason -) \ No newline at end of file diff --git a/Version4/Oracle/VocabImport/vocabulary.ctl b/Version4/Oracle/VocabImport/vocabulary.ctl deleted file mode 100644 index 3f80f25..0000000 --- a/Version4/Oracle/VocabImport/vocabulary.ctl +++ /dev/null @@ -1,11 +0,0 @@ -options (skip=1) -load data -infile vocabulary.csv -into table vocabulary -replace -fields terminated by '\t' -trailing nullcols -( - vocabulary_id, - vocabulary_name -) \ No newline at end of file diff --git a/Version4/PostgreSQL/CDM V4 DDL.sql b/Version4/PostgreSQL/CDM V4 DDL.sql deleted file mode 100644 index 8586422..0000000 --- a/Version4/PostgreSQL/CDM V4 DDL.sql +++ /dev/null @@ -1,131 +0,0 @@ - -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License.4 -********************************************************************************/ - -/************************ - - ####### # # ####### ###### ##### ###### # # ## ####### - # # ## ## # # # # # # # # ## ## # # # # # - # # # # # # # # # # # # # # # # # # # # # # - # # # # # # # ###### # # # # # # # # ####### ####### - # # # # # # # # # # # # # # # # - # # # # # # # # # # # # # # # # # # - ####### # # ####### # ##### ###### # # ## # # ##### - - -script to create OMOP common data model, version 4.5 for PostgreSQL database - -last revised: 20 Mar 2015 - -author: Lee Evans - - -*************************/ - -CREATE TABLE concept ( - concept_id integer NOT NULL, - concept_name varchar(256) NOT NULL, - concept_level integer NOT NULL, - concept_class varchar(60) NOT NULL, - vocabulary_id integer NOT NULL, - concept_code varchar(40) NOT NULL, - valid_start_date date NOT NULL, - valid_end_date date NOT NULL DEFAULT '2099-12-31'::date, - invalid_reason varchar(1) -) -; - - -CREATE TABLE concept_ancestor ( - ancestor_concept_id INTEGER NOT NULL, - descendant_concept_id INTEGER NOT NULL, - min_levels_of_separation INTEGER NOT NULL, - max_levels_of_separation INTEGER NOT NULL -) -; - - -CREATE TABLE concept_relationship ( - concept_id_1 integer NOT NULL, - concept_id_2 integer NOT NULL, - relationship_id integer NOT NULL, - valid_start_date date NOT NULL, - valid_end_date date NOT NULL DEFAULT '2099-12-31'::date, - invalid_reason varchar(1) -) -; - - -CREATE TABLE concept_synonym ( - concept_synonym_id integer NOT NULL, - concept_id integer NOT NULL, - concept_synonym_name varchar(1000) NOT NULL -) -; - - - -CREATE TABLE drug_strength ( - drug_concept_id integer NOT NULL, - ingredient_concept_id integer NOT NULL, - amount_value double precision, - amount_unit varchar(60), - concentration_value double precision, - concentration_enum_unit character varying(60), - concentration_denom_unit character varying(60), - valid_start_date date NOT NULL, - valid_end_date date NOT NULL, - invalid_reason varchar(1) -) -; - - - - -CREATE TABLE relationship ( - relationship_id VARCHAR(20) NOT NULL, - relationship_name VARCHAR(256) NOT NULL, - is_hierarchical integer NOT NULL, - defines_ancestry integer DEFAULT 1, - reverse_relationship integer -) -; - - - -CREATE TABLE source_to_concept_map ( - source_code VARCHAR(40) NOT NULL, - source_vocabulary_id INTEGER NOT NULL, - source_code_description VARCHAR(256), - target_concept_id INTEGER NOT NULL, - target_vocabulary_id INTEGER NOT NULL, - mapping_type VARCHAR(256), - primary_map VARCHAR(1), - valid_start_date DATE NOT NULL, - valid_end_date DATE NOT NULL, - invalid_reason VARCHAR(1) NULL -) -; - - - - -CREATE TABLE vocabulary ( - vocabulary_id integer NOT NULL, - vocabulary_name VARCHAR(256) NOT NULL -) -; diff --git a/Version4/PostgreSQL/README.md b/Version4/PostgreSQL/README.md deleted file mode 100644 index 7e4699b..0000000 --- a/Version4/PostgreSQL/README.md +++ /dev/null @@ -1,13 +0,0 @@ -Common-Data-Model / PostgreSQL -================= - -This folder contains the SQL scripts for PostgreSQL. - -In order to create your instantiation of the Common Data Model, we recommend following these steps: - -1. Create an empty schema. - -2. Execute the script `CDM V4 ddl.sql` to create the tables and fields. - -3. Load your data into the schema. - diff --git a/Version4/PostgreSQL/VocabImport/OMOP CDM vocabulary load - PostgreSQL.sql b/Version4/PostgreSQL/VocabImport/OMOP CDM vocabulary load - PostgreSQL.sql deleted file mode 100644 index 238b003..0000000 --- a/Version4/PostgreSQL/VocabImport/OMOP CDM vocabulary load - PostgreSQL.sql +++ /dev/null @@ -1,54 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -********************************************************************************/ - -/************************ - - ####### # # ####### ###### ##### ###### # # ####### - # # ## ## # # # # # # # # ## ## # # # - # # # # # # # # # # # # # # # # # # # # - # # # # # # # ###### # # # # # # # # ###### - # # # # # # # # # # # # # # # - # # # # # # # # # # # # # # # # # - ####### # # ####### # ##### ###### # # ## ##### - - -Script to load the common data model, version 4.5 vocabulary tables for PostgreSQL database - -Notes - -1) This script assumes the CDM version 4.5 vocabulary zip file has been unzipped into the "C:\CDM" directory. -2) If you unzipped your CDM version 4.5 vocabulary files into a different directory then replace all file paths below, with your directory path. -3) Run this SQL query script in the database where you created your CDM Version 4.5 tables - -last revised: 20th March 2015 - -author: Lee Evans - - -*************************/ - -COPY CONCEPT FROM 'C:\CDM\CONCEPT.csv' WITH DELIMITER E'\t' CSV HEADER QUOTE E'\b' ; -COPY CONCEPT_ANCESTOR FROM 'C:\CDM\CONCEPT_ANCESTOR.csv' WITH DELIMITER E'\t' CSV HEADER QUOTE E'\b' ; -COPY CONCEPT_RELATIONSHIP FROM 'C:\CDM\CONCEPT_RELATIONSHIP.csv' WITH DELIMITER E'\t' CSV HEADER QUOTE E'\b' ; -COPY CONCEPT_SYNONYM FROM 'C:\CDM\CONCEPT_SYNONYM.csv' WITH DELIMITER E'\t' CSV HEADER QUOTE E'\b' ; -COPY DRUG_STRENGTH FROM 'C:\CDM\DRUG_STRENGTH.csv' WITH DELIMITER E'\t' CSV HEADER QUOTE E'\b' ; -COPY RELATIONSHIP FROM 'C:\CDM\RELATIONSHIP.csv' WITH DELIMITER E'\t' CSV HEADER QUOTE E'\b' ; -COPY SOURCE_TO_CONCEPT_MAP FROM 'C:\CDM\SOURCE_TO_CONCEPT_MAP.csv' WITH DELIMITER E'\t' CSV HEADER QUOTE E'\b' ; -COPY VOCABULARY FROM 'C:\CDM\VOCABULARY.csv' WITH DELIMITER E'\t' CSV HEADER QUOTE E'\b' ; - - - diff --git a/Version4/SQL Server/CDM V4 DDL.sql b/Version4/SQL Server/CDM V4 DDL.sql deleted file mode 100644 index 981cec8..0000000 --- a/Version4/SQL Server/CDM V4 DDL.sql +++ /dev/null @@ -1,117 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License.4 -********************************************************************************/ - -/************************ - - ####### # # ####### ###### ##### ###### # # ## ####### - # # ## ## # # # # # # # # ## ## # # # # # - # # # # # # # # # # # # # # # # # # # # # # - # # # # # # # ###### # # # # # # # # ####### ####### - # # # # # # # # # # # # # # # # - # # # # # # # # # # # # # # # # # # - ####### # # ####### # ##### ###### # # ## # # ##### - - -script to create OMOP common data model, version 4.5 for Sql Server database - -last revised: 20 Mar 2015 - -author: Lee Evans - - -*************************/ - - -CREATE TABLE CONCEPT( - CONCEPT_ID int NOT NULL, - CONCEPT_NAME varchar(256) NOT NULL, - CONCEPT_LEVEL int NOT NULL, - CONCEPT_CLASS varchar(60) NOT NULL, - VOCABULARY_ID int NOT NULL, - CONCEPT_CODE varchar(40) NOT NULL, - VALID_START_DATE date NOT NULL, - VALID_END_DATE date NOT NULL DEFAULT ('31-Dec-2099'), - INVALID_REASON char(1) NULL -); - - -CREATE TABLE CONCEPT_ANCESTOR( - ANCESTOR_CONCEPT_ID int NOT NULL, - DESCENDANT_CONCEPT_ID int NOT NULL, - MAX_LEVELS_OF_SEPARATION int NULL, - MIN_LEVELS_OF_SEPARATION int NULL -); - - -CREATE TABLE CONCEPT_RELATIONSHIP( - CONCEPT_ID_1 int NOT NULL, - CONCEPT_ID_2 int NOT NULL, - RELATIONSHIP_ID int NOT NULL, - VALID_START_DATE date NOT NULL, - VALID_END_DATE date NOT NULL DEFAULT ('31-Dec-2099'), - INVALID_REASON char(1) NULL -); - - -CREATE TABLE CONCEPT_SYNONYM( - CONCEPT_SYNONYM_ID int NOT NULL, - CONCEPT_ID int NOT NULL, - CONCEPT_SYNONYM_NAME varchar(1000) NOT NULL -); - - -CREATE TABLE DRUG_STRENGTH( - DRUG_CONCEPT_ID int NOT NULL, - INGREDIENT_CONCEPT_ID int NOT NULL, - AMOUNT_VALUE float NULL, - AMOUNT_UNIT varchar(60) NULL, - CONCENTRATION_VALUE float NULL, - CONCENTRATION_ENUM_UNIT varchar(60) NULL, - CONCENTRATION_DENOM_UNIT varchar(60) NULL, - VALID_START_DATE date NOT NULL, - VALID_END_DATE date NOT NULL, - INVALID_REASON varchar(1) NULL -); - - -CREATE TABLE RELATIONSHIP( - RELATIONSHIP_ID int NOT NULL, - RELATIONSHIP_NAME varchar(256) NOT NULL, - IS_HIERARCHICAL int NOT NULL, - DEFINES_ANCESTRY int NOT NULL DEFAULT ((1)), - REVERSE_RELATIONSHIP int NULL -); - - -CREATE TABLE SOURCE_TO_CONCEPT_MAP( - SOURCE_CODE varchar(40) NOT NULL, - SOURCE_VOCABULARY_ID int NOT NULL, - SOURCE_CODE_DESCRIPTION varchar(256) NULL, - TARGET_CONCEPT_ID int NOT NULL, - TARGET_VOCABULARY_ID int NOT NULL, - MAPPING_TYPE varchar(256) NULL, - PRIMARY_MAP char(1) NULL, - VALID_START_DATE date NOT NULL, - VALID_END_DATE date NOT NULL DEFAULT ('31-Dec-2099'), - INVALID_REASON char(1) NULL -); - - -CREATE TABLE VOCABULARY( - VOCABULARY_ID int NOT NULL, - VOCABULARY_NAME varchar(256) NOT NULL -); diff --git a/Version4/SQL Server/README.md b/Version4/SQL Server/README.md deleted file mode 100644 index 27af881..0000000 --- a/Version4/SQL Server/README.md +++ /dev/null @@ -1,14 +0,0 @@ -Common-Data-Model / SQL Server -================= - -This folder contains the SQL scripts for SQL Server. - -In order to create your instantiation of the Common Data Model, we recommend following these steps: - -1. Create an empty schema. - -2. Execute the script `CDM V4 ddl.sql` to create the tables and fields. - -3. Load your data into the schema using the loading scripts in VocabImport - -Note: you could also apply the constraints and the indexes after loading the data, this will speed up the insertion of the data considerably. diff --git a/Version4/SQL Server/VocabImport/OMOP CDM vocabulary load - SQL Server.sql b/Version4/SQL Server/VocabImport/OMOP CDM vocabulary load - SQL Server.sql deleted file mode 100644 index bc2218e..0000000 --- a/Version4/SQL Server/VocabImport/OMOP CDM vocabulary load - SQL Server.sql +++ /dev/null @@ -1,130 +0,0 @@ -/********************************************************************************* -# Copyright 2015 Observational Health Data Sciences and Informatics -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -********************************************************************************/ - -/************************ - - ####### # # ####### ###### ##### ###### # # ####### - # # ## ## # # # # # # # # ## ## # # # - # # # # # # # # # # # # # # # # # # # # - # # # # # # # ###### # # # # # # # # ###### - # # # # # # # # # # # # # # # - # # # # # # # # # # # # # # # # # - ####### # # ####### # ##### ###### # # ## ##### - - -Script to load the common data model, version 4.5 vocabulary tables for SQL Server database - -Notes - -1) This script assumes the CDM version 4.5 vocabulary zip file has been unzipped into the "C:\CDM" directory. -2) If you unzipped your CDM version 4.5 vocabulary files into a different directory then replace all file paths below, with your directory path. -3) Run this SQL query script in the database where you created your CDM Version 4.5 tables - -last revised: 20th March 2015 - -author: Lee Evans - - -*************************/ - -TRUNCATE TABLE CONCEPT; -BULK INSERT CONCEPT -FROM 'C:\CDM\CONCEPT.csv' -WITH ( -FIRSTROW = 2, -FIELDTERMINATOR = '\t', -ROWTERMINATOR = '0x0a', -ERRORFILE = 'C:\CDM\CONCEPT.bad', -TABLOCK -); - -TRUNCATE TABLE CONCEPT_ANCESTOR; -BULK INSERT CONCEPT_ANCESTOR -FROM 'C:\CDM\CONCEPT_ANCESTOR.csv' -WITH ( -FIRSTROW = 2, -FIELDTERMINATOR = '\t', -ROWTERMINATOR = '0x0a', -ERRORFILE = 'C:\CDM\CONCEPT_ANCESTOR.bad', -TABLOCK -); - -TRUNCATE TABLE CONCEPT_RELATIONSHIP; -BULK INSERT CONCEPT_RELATIONSHIP -FROM 'C:\CDM\CONCEPT_RELATIONSHIP.csv' -WITH ( -FIRSTROW = 2, -FIELDTERMINATOR = '\t', -ROWTERMINATOR = '0x0a', -ERRORFILE = 'C:\CDM\CONCEPT_RELATIONSHIP.bad', -TABLOCK -); - -TRUNCATE TABLE CONCEPT_SYNONYM; -BULK INSERT CONCEPT_SYNONYM -FROM 'C:\CDM\CONCEPT_SYNONYM.csv' -WITH ( -FIRSTROW = 2, -FIELDTERMINATOR = '\t', -ROWTERMINATOR = '0x0a', -ERRORFILE = 'C:\CDM\CONCEPT_SYNONYM.bad', -TABLOCK -); - -TRUNCATE TABLE DRUG_STRENGTH; -BULK INSERT DRUG_STRENGTH -FROM 'C:\CDM\DRUG_STRENGTH.csv' -WITH ( -FIRSTROW = 2, -FIELDTERMINATOR = '\t', -ROWTERMINATOR = '0x0a', -ERRORFILE = 'C:\CDM\DRUG_STRENGTH.bad', -TABLOCK -); - -TRUNCATE TABLE RELATIONSHIP; -BULK INSERT RELATIONSHIP -FROM 'C:\CDM\RELATIONSHIP.csv' -WITH ( -FIRSTROW = 2, -FIELDTERMINATOR = '\t', -ROWTERMINATOR = '0x0a', -ERRORFILE = 'C:\CDM\RELATIONSHIP.bad', -TABLOCK -); - -TRUNCATE TABLE SOURCE_TO_CONCEPT_MAP; -BULK INSERT SOURCE_TO_CONCEPT_MAP -FROM 'C:\CDM\SOURCE_TO_CONCEPT_MAP.csv' -WITH ( -FIRSTROW = 2, -FIELDTERMINATOR = '\t', -ROWTERMINATOR = '0x0a', -ERRORFILE = 'C:\CDM\SOURCE_TO_CONCEPT_MAP.bad', -TABLOCK -); - -TRUNCATE TABLE VOCABULARY; -BULK INSERT VOCABULARY -FROM 'C:\CDM\VOCABULARY.csv' -WITH ( -FIRSTROW = 2, -FIELDTERMINATOR = '\t', -ROWTERMINATOR = '0x0a', -ERRORFILE = 'C:\CDM\VOCABULARY.bad', -TABLOCK -);