Merge pull request #437 from OHDSI/v5.4_unittest

Add more unit tests
This commit is contained in:
clairblacketer 2021-08-31 13:48:45 -04:00 committed by GitHub
commit 9779c07b72
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
85 changed files with 357 additions and 172 deletions

BIN
.DS_Store vendored Normal file

Binary file not shown.

View File

@ -2,3 +2,4 @@
^\.Rproj\.user$ ^\.Rproj\.user$
^LICENSE\.md$ ^LICENSE\.md$
.github/* .github/*
extras/*

View File

@ -33,6 +33,18 @@ jobs:
CDMDDLBASE_POSTGRESQL_PASSWORD: ${{ secrets.CDMDDLBASE_POSTGRESQL_PASSWORD }} CDMDDLBASE_POSTGRESQL_PASSWORD: ${{ secrets.CDMDDLBASE_POSTGRESQL_PASSWORD }}
CDMDDLBASE_POSTGRESQL_SERVER: ${{ secrets.CDMDDLBASE_POSTGRESQL_SERVER }} CDMDDLBASE_POSTGRESQL_SERVER: ${{ secrets.CDMDDLBASE_POSTGRESQL_SERVER }}
CDMDDLBASE_POSTGRESQL_SCHEMA: ${{ secrets.CDMDDLBASE_POSTGRESQL_SCHEMA }} CDMDDLBASE_POSTGRESQL_SCHEMA: ${{ secrets.CDMDDLBASE_POSTGRESQL_SCHEMA }}
CDMDDLBASE_REDSHIFT_USER: ${{ secrets.CDMDDLBASE_REDSHIFT_USER }}
CDMDDLBASE_REDSHIFT_PASSWORD: ${{ secrets.CDMDDLBASE_REDSHIFT_PASSWORD }}
CDMDDLBASE_REDSHIFT_SERVER: ${{ secrets.CDMDDLBASE_REDSHIFT_SERVER }}
CDMDDLBASE_REDSHIFT_SCHEMA: ${{ secrets.CDMDDLBASE_REDSHIFT_SCHEMA }}
CDMDDLBASE_SQL_SERVER_USER: ${{ secrets.CDMDDLBASE_SQL_SERVER_USER }}
CDMDDLBASE_SQL_SERVER_PASSWORD: ${{ secrets.CDMDDLBASE_SQL_SERVER_PASSWORD }}
CDMDDLBASE_SQL_SERVER_SERVER: ${{ secrets.CDMDDLBASE_SQL_SERVER_SERVER }}
CDMDDLBASE_SQL_SERVER_CDM_SCHEMA: ${{ secrets.CDMDDLBASE_SQL_SERVER_CDM_SCHEMA }}
CDMDDLBASE_ORACLE_USER: ${{ secrets.CDMDDLBASE_ORACLE_USER }}
CDMDDLBASE_ORACLE_PASSWORD: ${{ secrets.CDMDDLBASE_ORACLE_PASSWORD }}
CDMDDLBASE_ORACLE_SERVER: ${{ secrets.CDMDDLBASE_ORACLE_SERVER }}
CDMDDLBASE_ORACLE_CDM_SCHEMA: ${{ secrets.CDMDDLBASE_ORACLE_CDM_SCHEMA }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

View File

@ -1,5 +1,6 @@
# Generated by roxygen2: do not edit by hand # Generated by roxygen2: do not edit by hand
export(buildRelease)
export(buildReleaseZip) export(buildReleaseZip)
export(createDdl) export(createDdl)
export(createForeignKeys) export(createForeignKeys)
@ -13,3 +14,6 @@ export(writeDdl)
export(writeForeignKeys) export(writeForeignKeys)
export(writeIndex) export(writeIndex)
export(writePrimaryKeys) export(writePrimaryKeys)
importFrom(utils,download.file)
importFrom(utils,read.csv)
importFrom(utils,write.csv)

View File

@ -23,7 +23,7 @@
#' #'
#' @param mdFilesLocation Path to the root folder of the Wiki repository. #' @param mdFilesLocation Path to the root folder of the Wiki repository.
#' @param output_file Path to where the output CSV file should be written. #' @param output_file Path to where the output CSV file should be written.
#' #' @importFrom utils write.csv
#' @export #' @export
parseWiki <- function(mdFilesLocation, output_file) { parseWiki <- function(mdFilesLocation, output_file) {
# mdFilesLocation <- "../CommonDataModel.wiki" # mdFilesLocation <- "../CommonDataModel.wiki"

View File

@ -19,25 +19,36 @@
#' Writes DDL, ForeignKey, PrimaryKey and index SQL files for given cdmVersion #' Writes DDL, ForeignKey, PrimaryKey and index SQL files for given cdmVersion
#' and targetDialect to the 'ddl' folder in current working directory. #' and targetDialect to the 'ddl' folder in current working directory.
#' #'
#' @param cdmVersion The version of the CDM you are creating, e.g. 5.3, 5.4. #' @param cdmVersions The versions of the CDM you are creating, e.g. 5.3, 5.4.
#' Defaults to all supported CDM versions. #' Defaults to all supported CDM versions.
#' @param targetDialect The target dialect #' @param targetDialects A character vector of target dialects.
#' #' Defaults to all supported dialects.
buildRelease <- function(cdmVersion = listSupportedVersions(), #' @param outputfolder The base folder where the SQL files will be written.
targetDialect = listSupportedDialects()){ #' Subfolders will be created for each cdmVersion and targetDialect.
for (cdmVersion in cdmVersion) { #' @export
for (targetDialect in targetDialect) { buildRelease <- function(cdmVersions = listSupportedVersions(),
targetDialects = listSupportedDialects(),
outputfolder = file.path(getwd(), "inst", "ddl")){
basefolder <- outputfolder
for (cdmVersion in cdmVersions) {
for (targetDialect in targetDialects) {
outputfolder <- file.path(basefolder, cdmVersion, gsub(" ", "_", targetDialect))
writeDdl(targetDialect = targetDialect, writeDdl(targetDialect = targetDialect,
cdmVersion = cdmVersion) cdmVersion = cdmVersion,
outputfolder = outputfolder)
writePrimaryKeys(targetDialect = targetDialect, writePrimaryKeys(targetDialect = targetDialect,
cdmVersion = cdmVersion) cdmVersion = cdmVersion,
outputfolder = outputfolder)
writeForeignKeys(targetDialect = targetDialect, writeForeignKeys(targetDialect = targetDialect,
cdmVersion = cdmVersion) cdmVersion = cdmVersion,
outputfolder = outputfolder)
writeIndex(targetDialect = targetDialect, writeIndex(targetDialect = targetDialect,
cdmVersion = cdmVersion) cdmVersion = cdmVersion,
outputfolder = outputfolder)
} }
} }
} }
@ -62,7 +73,7 @@ buildRelease <- function(cdmVersion = listSupportedVersions(),
#' #'
buildReleaseZip <- function(cdmVersion, buildReleaseZip <- function(cdmVersion,
targetDialect = listSupportedDialects(), targetDialect = listSupportedDialects(),
outputfolder = "output"){ outputfolder = file.path(getwd(), "inst", "ddl")){
# argument checks # argument checks
stopifnot(is.character(cdmVersion), length(cdmVersion) == 1, cdmVersion %in% listSupportedVersions()) stopifnot(is.character(cdmVersion), length(cdmVersion) == 1, cdmVersion %in% listSupportedVersions())
@ -72,8 +83,8 @@ buildReleaseZip <- function(cdmVersion,
files <- c() files <- c()
for (dialect in targetDialect) { for (dialect in targetDialect) {
buildRelease(cdmVersion, dialect) buildRelease(cdmVersion, dialect, outputfolder = outputfolder)
files <- c(files, list.files(file.path('ddl', cdmVersion, gsub(" ", "_", dialect)), files <- c(files, list.files(file.path(outputfolder, cdmVersion, gsub(" ", "_", dialect)),
pattern = ".*\\.sql$", pattern = ".*\\.sql$",
full.names = TRUE)) full.names = TRUE))
} }

View File

@ -26,6 +26,7 @@
#' #'
#' @param cdmVersion The version of the CDM you are creating, e.g. 5.3, 5.4 #' @param cdmVersion The version of the CDM you are creating, e.g. 5.3, 5.4
#' @return A character string containing the OHDSQL DDL #' @return A character string containing the OHDSQL DDL
#' @importFrom utils read.csv
#' @export #' @export
#' @examples #' @examples
#' ddl <- createDdl("5.4") #' ddl <- createDdl("5.4")
@ -138,3 +139,63 @@ createForeignKeys <- function(cdmVersion){
} }
return(paste0(sql_result, collapse = "")) return(paste0(sql_result, collapse = ""))
} }
# A helper function that will return a character string with the omop ascii art given a major and minor cdm version
# example: cat(createAsciiHeader(5, 3))
createAsciiHeader <- function(major, minor) {
stopifnot(is.numeric(major), is.numeric(minor), length(major) == 1, length(minor) == 1)
stopifnot(major %in% 0:99, minor %in% 0:99)
# An inner function that returns an ascii art matrix for any number between 0 and 99
numberMatrix <- function(num){
stopifnot(is.numeric(num), num %in% 0:99)
# An inner function that returns a 7x7 matrix of number ascii art for the number 0 through 9
# for the number 1 a 7x5 matrix is returned because 1 is narrower than other numbers.
singleDigit <- function(num) {
nums <- c(' ### # ##### ##### # ####### ##### ####### ##### ##### # # ## # ## ## # # # ## # # ## ## # # # # ## # # # # # ## ## # # ##### ##### # # ###### ###### # ##### ####### # # # ######## ## # # # # # # # # # # # # # ## # # # ## # ### ##### ####### ##### # ##### ##### # ##### ##### ')
numsMatrix <- matrix(data = strsplit(nums, character(0))[[1]], nrow = 7, byrow = T)
cols <- seq(num*7+1, num*7+7, by = 1)
out <- numsMatrix[1:7, cols]
# the number 1 is narrower than the other numbers
if(num == 1) out<- out[1:7, 2:6]
out
}
if(num < 10){
return(singleDigit(num))
} else {
space <- matrix(rep(" ", 7), nrow = 7)
return(cbind(singleDigit(floor(num/10)), space, singleDigit(num %% 10)))
}
}
omop <- c('.
####### # # ####### ###### ##### ###### # # .
# # ## ## # # # # # # # # ## ## # #.
# # # # # # # # # # # # # # # # # # #.
# # # # # # # ###### # # # # # # # #.
# # # # # # # # # # # # # #.
# # # # # # # # # # # # # # # .
####### # # ####### # ##### ###### # # ## ')
# convert to matrix and remove first column
omop <- matrix(strsplit(omop, character(0))[[1]], nrow = 7, byrow = TRUE)
omop <- omop[,c(-1, -2)]
dot <- matrix(c(rep(" ", 3*4), rep("#", 3*3)), nrow = 7, byrow = TRUE)
space <- matrix(rep(" ", 7), nrow = 7)
newline <- matrix(rep("\n", 7, nrow = 7))
header <- character(0)
headerMatrix <- cbind(omop, space, numberMatrix(major), space, dot, space, numberMatrix(minor), newline)
for(i in 1:7) {
header <- c(header, as.character(headerMatrix[i,]))
}
header <- paste(header, collapse = "")
return(header)
}

View File

@ -14,7 +14,7 @@
#' downloadCurrentDdl("OHDSI/CommonDataModel", #' downloadCurrentDdl("OHDSI/CommonDataModel",
#' pathToCsv="Sql%20Server/OMOP%20CDM%20sql%20server%20ddl.txt") #' pathToCsv="Sql%20Server/OMOP%20CDM%20sql%20server%20ddl.txt")
#' } #' }
#' #' @importFrom utils download.file
#' @export #' @export
downloadCurrentDdl <- function(githubPath="OHDSI/CommonDataModel", downloadCurrentDdl <- function(githubPath="OHDSI/CommonDataModel",

View File

@ -42,16 +42,16 @@ executeDdl <- function(connectionDetails,
executeForeignKey = TRUE, executeForeignKey = TRUE,
...) { ...) {
outputpath <- tempdir(check = TRUE) outputfolder <- tempdir(check = TRUE)
if(executeDdl) { if(executeDdl) {
filename <- writeDdl(targetDialect = connectionDetails$dbms, filename <- writeDdl(targetDialect = connectionDetails$dbms,
cdmVersion = cdmVersion, cdmVersion = cdmVersion,
cdmDatabaseSchema = cdmDatabaseSchema, cdmDatabaseSchema = cdmDatabaseSchema,
outputpath = outputpath) outputfolder = outputfolder)
sql <- readr::read_file(file.path(outputpath, filename)) sql <- readr::read_file(file.path(outputfolder, filename))
} else { } else {
sql <- "" sql <- ""
} }
@ -60,18 +60,18 @@ executeDdl <- function(connectionDetails,
filename <- writePrimaryKeys(targetDialect = connectionDetails$dbms, filename <- writePrimaryKeys(targetDialect = connectionDetails$dbms,
cdmVersion = cdmVersion, cdmVersion = cdmVersion,
cdmDatabaseSchema = cdmDatabaseSchema, cdmDatabaseSchema = cdmDatabaseSchema,
outputpath = outputpath) outputfolder = outputfolder)
sql <- paste(sql, readr::read_file(file.path(outputpath, filename)), sep = "\n") sql <- paste(sql, readr::read_file(file.path(outputfolder, filename)), sep = "\n")
} }
if(executeForeignKey) { if(executeForeignKey) {
filename <- writeForeignKeys(targetDialect = connectionDetails$dbms, filename <- writeForeignKeys(targetDialect = connectionDetails$dbms,
cdmVersion = cdmVersion, cdmVersion = cdmVersion,
cdmDatabaseSchema = cdmDatabaseSchema, cdmDatabaseSchema = cdmDatabaseSchema,
outputpath = outputpath) outputfolder = outputfolder)
sql <- paste(sql, readr::read_file(file.path(outputpath, filename)), sep = "\n") sql <- paste(sql, readr::read_file(file.path(outputfolder, filename)), sep = "\n")
} }
con <- DatabaseConnector::connect(connectionDetails = connectionDetails) con <- DatabaseConnector::connect(connectionDetails = connectionDetails)

View File

@ -1,7 +1,7 @@
#' List CDM versions supported by this package #' List CDM versions supported by this package
#' #'
#' @return A character vector containing the support CDM versions in {major}.{minor} format.
#' @export #' @export
listSupportedVersions <- function() { listSupportedVersions <- function() {
supportedVersions <- c("5.3", "5.4") supportedVersions <- c("5.3", "5.4")
return(supportedVersions) return(supportedVersions)

View File

@ -22,95 +22,95 @@
#' #'
#' @param targetDialect The dialect of the target database. Choices are "oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server" #' @param targetDialect The dialect of the target database. Choices are "oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server"
#' @param cdmVersion The version of the CDM you are creating, e.g. 5.3, 5.4 #' @param cdmVersion The version of the CDM you are creating, e.g. 5.3, 5.4
#' @param outputpath The directory or folder where the SQL file should be saved. #' @param outputfolder The directory or folder where the SQL file should be saved.
#' @param cdmDatabaseSchema The schema of the CDM instance where the DDL will be run. For example, this would be "ohdsi.dbo" when testing on sql server. #' @param cdmDatabaseSchema The schema of the CDM instance where the DDL will be run. For example, this would be "ohdsi.dbo" when testing on sql server.
#' Defaults to "@cdmDatabaseSchema" #' Defaults to "@cdmDatabaseSchema"
#' #'
#' @export #' @export
writeDdl <- function(targetDialect, cdmVersion, outputpath, cdmDatabaseSchema = "@cdmDatabaseSchema") { writeDdl <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchema = "@cdmDatabaseSchema") {
# argument checks # argument checks
stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server")) stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server"))
stopifnot(cdmVersion %in% listSupportedVersions()) stopifnot(cdmVersion %in% listSupportedVersions())
stopifnot(is.character(cdmDatabaseSchema)) stopifnot(is.character(cdmDatabaseSchema))
if(missing(outputpath)) { if(missing(outputfolder)) {
outputpath <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect)) outputfolder <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
} }
if(!dir.exists(outputpath)) dir.create(outputpath, showWarnings = FALSE, recursive = TRUE) if(!dir.exists(outputfolder)) dir.create(outputfolder, showWarnings = FALSE, recursive = TRUE)
sql <- createDdl(cdmVersion) sql <- createDdl(cdmVersion)
sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect) sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect)
sql <- SqlRender::translate(sql, targetDialect = targetDialect) sql <- SqlRender::translate(sql, targetDialect = targetDialect)
filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "ddl.sql", sep = "_") filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "ddl.sql", sep = "_")
SqlRender::writeSql(sql = sql, targetFile = file.path(outputpath, filename)) SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
invisible(filename) invisible(filename)
} }
#' @describeIn writeDdl writePrimaryKeys Write the SQL code that creates the primary keys to a file. #' @describeIn writeDdl writePrimaryKeys Write the SQL code that creates the primary keys to a file.
#' @export #' @export
writePrimaryKeys <- function(targetDialect, cdmVersion, outputpath, cdmDatabaseSchema = "@cdmDatabaseSchema") { writePrimaryKeys <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchema = "@cdmDatabaseSchema") {
# argument checks # argument checks
stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server")) stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server"))
stopifnot(cdmVersion %in% listSupportedVersions()) stopifnot(cdmVersion %in% listSupportedVersions())
stopifnot(is.character(cdmDatabaseSchema)) stopifnot(is.character(cdmDatabaseSchema))
if(missing(outputpath)) { if(missing(outputfolder)) {
outputpath <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect)) outputfolder <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
} }
if(!dir.exists(outputpath)) dir.create(outputpath, showWarnings = FALSE, recursive = TRUE) if(!dir.exists(outputfolder)) dir.create(outputfolder, showWarnings = FALSE, recursive = TRUE)
sql <- createPrimaryKeys(cdmVersion) sql <- createPrimaryKeys(cdmVersion)
sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect) sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect)
sql <- SqlRender::translate(sql, targetDialect = targetDialect) sql <- SqlRender::translate(sql, targetDialect = targetDialect)
filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "primary", "keys.sql", sep = "_") filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "primary", "keys.sql", sep = "_")
SqlRender::writeSql(sql = sql, targetFile = file.path(outputpath, filename)) SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
invisible(filename) invisible(filename)
} }
#' @describeIn writeDdl writeForeignKeys Write the SQL code that creates the foreign keys to a file. #' @describeIn writeDdl writeForeignKeys Write the SQL code that creates the foreign keys to a file.
#' @export #' @export
writeForeignKeys <- function(targetDialect, cdmVersion, outputpath, cdmDatabaseSchema = "@cdmDatabaseSchema") { writeForeignKeys <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchema = "@cdmDatabaseSchema") {
# argument checks # argument checks
stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server")) stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server"))
stopifnot(cdmVersion %in% listSupportedVersions()) stopifnot(cdmVersion %in% listSupportedVersions())
stopifnot(is.character(cdmDatabaseSchema)) stopifnot(is.character(cdmDatabaseSchema))
if(missing(outputpath)) { if(missing(outputfolder)) {
outputpath <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect)) outputfolder <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
} }
if(!dir.exists(outputpath)) dir.create(outputpath, showWarnings = FALSE, recursive = TRUE) if(!dir.exists(outputfolder)) dir.create(outputfolder, showWarnings = FALSE, recursive = TRUE)
sql <- createForeignKeys(cdmVersion) sql <- createForeignKeys(cdmVersion)
sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect) sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect)
sql <- SqlRender::translate(sql, targetDialect = targetDialect) sql <- SqlRender::translate(sql, targetDialect = targetDialect)
filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "constraints.sql", sep = "_") filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "constraints.sql", sep = "_")
SqlRender::writeSql(sql = sql, targetFile = file.path(outputpath, filename)) SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
invisible(filename) invisible(filename)
} }
#' @describeIn writeDdl writeIndex Write the rendered and translated sql that creates recommended indexes to a file. #' @describeIn writeDdl writeIndex Write the rendered and translated sql that creates recommended indexes to a file.
#' @export #' @export
writeIndex <- function(targetDialect, cdmVersion, outputpath, cdmDatabaseSchema = "@cdmDatabaseSchema") { writeIndex <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchema = "@cdmDatabaseSchema") {
# argument checks # argument checks
stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server")) stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server"))
stopifnot(cdmVersion %in% listSupportedVersions()) stopifnot(cdmVersion %in% listSupportedVersions())
stopifnot(is.character(cdmDatabaseSchema)) stopifnot(is.character(cdmDatabaseSchema))
if(missing(outputpath)) { if(missing(outputfolder)) {
outputpath <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect)) outputfolder <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
} }
if(!dir.exists(outputpath)) dir.create(outputpath, showWarnings = FALSE, recursive = TRUE) if(!dir.exists(outputfolder)) dir.create(outputfolder, showWarnings = FALSE, recursive = TRUE)
sqlFilename <- paste0("OMOP_CDM_indices_v", cdmVersion, ".sql") sqlFilename <- paste0("OMOP_CDM_indices_v", cdmVersion, ".sql")
sql <- readr::read_file(system.file(file.path("sql", "sql_server", sqlFilename), package = "CommonDataModel")) sql <- readr::read_file(system.file(file.path("sql", "sql_server", sqlFilename), package = "CommonDataModel"))
@ -118,6 +118,6 @@ writeIndex <- function(targetDialect, cdmVersion, outputpath, cdmDatabaseSchema
sql <- SqlRender::translate(sql, targetDialect = targetDialect) sql <- SqlRender::translate(sql, targetDialect = targetDialect)
filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "indices.sql", sep = "_") filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "indices.sql", sep = "_")
SqlRender::writeSql(sql = sql, targetFile = file.path(outputpath, filename)) SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
invisible(filename) invisible(filename)
} }

View File

@ -5,15 +5,20 @@
\title{Create OMOP CDM SQL files} \title{Create OMOP CDM SQL files}
\usage{ \usage{
buildRelease( buildRelease(
cdmVersion = listSupportedVersions(), cdmVersions = listSupportedVersions(),
targetDialect = listSupportedDialects() targetDialects = listSupportedDialects(),
outputfolder = file.path(getwd(), "inst", "ddl")
) )
} }
\arguments{ \arguments{
\item{cdmVersion}{The version of the CDM you are creating, e.g. 5.3, 5.4. \item{cdmVersions}{The versions of the CDM you are creating, e.g. 5.3, 5.4.
Defaults to all supported CDM versions.} Defaults to all supported CDM versions.}
\item{targetDialect}{The target dialect} \item{targetDialects}{A character vector of target dialects.
Defaults to all supported dialects.}
\item{outputfolder}{The base folder where the SQL files will be written.
Subfolders will be created for each cdmVersion and targetDialect.}
} }
\description{ \description{
Writes DDL, ForeignKey, PrimaryKey and index SQL files for given cdmVersion Writes DDL, ForeignKey, PrimaryKey and index SQL files for given cdmVersion

View File

@ -7,7 +7,7 @@
buildReleaseZip( buildReleaseZip(
cdmVersion, cdmVersion,
targetDialect = listSupportedDialects(), targetDialect = listSupportedDialects(),
outputfolder = "output" outputfolder = file.path(getwd(), "inst", "ddl")
) )
} }
\arguments{ \arguments{

View File

@ -30,5 +30,4 @@ The default location is
downloadCurrentDdl("OHDSI/CommonDataModel", downloadCurrentDdl("OHDSI/CommonDataModel",
pathToCsv="Sql\%20Server/OMOP\%20CDM\%20sql\%20server\%20ddl.txt") pathToCsv="Sql\%20Server/OMOP\%20CDM\%20sql\%20server\%20ddl.txt")
} }
} }

View File

@ -6,6 +6,9 @@
\usage{ \usage{
listSupportedVersions() listSupportedVersions()
} }
\value{
A character vector containing the support CDM versions in {major}.{minor} format.
}
\description{ \description{
List CDM versions supported by this package List CDM versions supported by this package
} }

View File

@ -1,5 +1,5 @@
% Generated by roxygen2: do not edit by hand % Generated by roxygen2: do not edit by hand
% Please edit documentation in R/writeDDL.R % Please edit documentation in R/writeDdl.R
\name{writeDdl} \name{writeDdl}
\alias{writeDdl} \alias{writeDdl}
\alias{writePrimaryKeys} \alias{writePrimaryKeys}
@ -10,28 +10,28 @@
writeDdl( writeDdl(
targetDialect, targetDialect,
cdmVersion, cdmVersion,
outputpath, outputfolder,
cdmDatabaseSchema = "@cdmDatabaseSchema" cdmDatabaseSchema = "@cdmDatabaseSchema"
) )
writePrimaryKeys( writePrimaryKeys(
targetDialect, targetDialect,
cdmVersion, cdmVersion,
outputpath, outputfolder,
cdmDatabaseSchema = "@cdmDatabaseSchema" cdmDatabaseSchema = "@cdmDatabaseSchema"
) )
writeForeignKeys( writeForeignKeys(
targetDialect, targetDialect,
cdmVersion, cdmVersion,
outputpath, outputfolder,
cdmDatabaseSchema = "@cdmDatabaseSchema" cdmDatabaseSchema = "@cdmDatabaseSchema"
) )
writeIndex( writeIndex(
targetDialect, targetDialect,
cdmVersion, cdmVersion,
outputpath, outputfolder,
cdmDatabaseSchema = "@cdmDatabaseSchema" cdmDatabaseSchema = "@cdmDatabaseSchema"
) )
} }
@ -40,7 +40,7 @@ writeIndex(
\item{cdmVersion}{The version of the CDM you are creating, e.g. 5.3, 5.4} \item{cdmVersion}{The version of the CDM you are creating, e.g. 5.3, 5.4}
\item{outputpath}{The directory or folder where the SQL file should be saved.} \item{outputfolder}{The directory or folder where the SQL file should be saved.}
\item{cdmDatabaseSchema}{The schema of the CDM instance where the DDL will be run. For example, this would be "ohdsi.dbo" when testing on sql server. \item{cdmDatabaseSchema}{The schema of the CDM instance where the DDL will be run. For example, this would be "ohdsi.dbo" when testing on sql server.
Defaults to "@cdmDatabaseSchema"} Defaults to "@cdmDatabaseSchema"}

View File

@ -1,21 +0,0 @@
# Download the JDBC drivers used in the tests
# oldJarFolder <- Sys.getenv("DATABASECONNECTOR_JAR_FOLDER")
driverPath <- file.path(Sys.getenv("HOME"), "drivers")
if(!dir.exists(driverPath)) dir.create(driverPath)
Sys.setenv("DATABASECONNECTOR_JAR_FOLDER" = driverPath)
DatabaseConnector::downloadJdbcDrivers("postgresql", pathToDriver = driverPath)
print(Sys.getenv("DATABASECONNECTOR_JAR_FOLDER"))
print(list.files(driverPath))
# if(Sys.getenv("DATABASECONNECTOR_JAR_FOLDER") == "") {
# driverPath <- file.path(Sys.getenv("HOME"), "drivers")
# }
# downloadJdbcDrivers("sql server")
# downloadJdbcDrivers("oracle")
# withr::defer({
# unlink(Sys.getenv("DATABASECONNECTOR_JAR_FOLDER"), recursive = TRUE, force = TRUE)
# Sys.setenv("DATABASECONNECTOR_JAR_FOLDER" = oldJarFolder)
# }, testthat::teardown_env())

113
tests/testthat/setup.R Normal file
View File

@ -0,0 +1,113 @@
# Download the JDBC drivers used in the tests
library(DatabaseConnector)
driverPath <- file.path(Sys.getenv("HOME"), "drivers")
if(!dir.exists(driverPath)) dir.create(driverPath)
if(Sys.getenv("LOCAL_TEST") != "TRUE") {
cat("\ndownloading drivers\n")
downloadJdbcDrivers("redshift", pathToDriver = driverPath)
downloadJdbcDrivers("postgresql", pathToDriver = driverPath)
downloadJdbcDrivers("oracle", pathToDriver = driverPath)
downloadJdbcDrivers("sql server", pathToDriver = driverPath)
}
# Helper functions used in tests -----------------------------------------------
getConnectionDetails <- function(dbms) {
switch (dbms,
"postgresql" = createConnectionDetails(
dbms = "postgresql",
user = Sys.getenv("CDMDDLBASE_POSTGRESQL_USER"),
password = Sys.getenv("CDMDDLBASE_POSTGRESQL_PASSWORD"),
server = Sys.getenv("CDMDDLBASE_POSTGRESQL_SERVER"),
pathToDriver = file.path(Sys.getenv("HOME"), "drivers")
),
"redshift" = createConnectionDetails(
dbms = "redshift",
user = Sys.getenv("CDMDDLBASE_REDSHIFT_USER"),
password = Sys.getenv("CDMDDLBASE_REDSHIFT_PASSWORD"),
server = Sys.getenv("CDMDDLBASE_REDSHIFT_SERVER"),
pathToDriver = file.path(Sys.getenv("HOME"), "drivers")
),
"sql server" = createConnectionDetails(
dbms = "sql server",
user = Sys.getenv("CDMDDLBASE_SQL_SERVER_USER"),
password = Sys.getenv("CDMDDLBASE_SQL_SERVER_PASSWORD"),
server = Sys.getenv("CDMDDLBASE_SQL_SERVER_SERVER"),
pathToDriver = file.path(Sys.getenv("HOME"), "drivers")
),
"oracle" = createConnectionDetails(
dbms = "oracle",
user = Sys.getenv("CDMDDLBASE_ORACLE_USER"),
password = Sys.getenv("CDMDDLBASE_ORACLE_PASSWORD"),
server = Sys.getenv("CDMDDLBASE_ORACLE_SERVER"),
pathToDriver = file.path(Sys.getenv("HOME"), "drivers")
)
)
}
getSchema <- function(dbms) {
switch (dbms,
"postgresql" = Sys.getenv("CDMDDLBASE_POSTGRESQL_SCHEMA"),
"redshift" = Sys.getenv("CDMDDLBASE_REDSHIFT_SCHEMA"),
"sql server" = Sys.getenv("CDMDDLBASE_SQL_SERVER_CDM_SCHEMA"),
"oracle" = Sys.getenv("CDMDDLBASE_ORACLE_CDM_SCHEMA")
)
}
listTablesInSchema <- function(connectionDetails, schema) {
stopifnot(class(connectionDetails) == "connectionDetails", class(schema) == "character", length(schema) == 1)
stopifnot(connectionDetails$dbms %in% c("postgresql", "redshift", "sql server", "oracle"))
con <- DatabaseConnector::connect(connectionDetails)
on.exit(DatabaseConnector::disconnect(con))
dbms <- connectionDetails$dbms
if(dbms %in% c("postgresql", "redshift", "sql server")) {
tables <- dbGetQuery(con, paste0("select table_name from information_schema.tables where table_schema = '", schema, "'"))[[1]]
} else if(dbms == "oracle") {
query <- paste0("select table_name from all_tables where owner = '", toupper(schema), "' and tablespace_name = 'USERS'")
tables <- dbGetQuery(con, query)[[1]]
}
return(tables)
}
dropAllTablesFromSchema <- function(connectionDetails, schema) {
stopifnot(class(connectionDetails) == "connectionDetails", class(schema) == "character", length(schema) == 1)
stopifnot(connectionDetails$dbms %in% c("postgresql", "redshift", "sql server", "oracle"))
tables <- listTablesInSchema(connectionDetails, schema)
con <- DatabaseConnector::connect(connectionDetails)
on.exit(DatabaseConnector::disconnect(con))
dbms <- connectionDetails$dbms
if(dbms %in% c("redshift", "postgresql", "sql server")) {
for(table in tables) {
DBI::dbRemoveTable(con, name = DBI::SQL(paste(schema, table, sep = ".")))
}
} else if(dbms == "oracle") {
for(table in tables) {
DBI::dbRemoveTable(con, name = table)
}
}
}
# .removeConstraintsPostgresql <- function(connectionDetails, schema) {
# # the order of removal of constraints matters!
# con <- DatabaseConnector::connect(connectionDetails)
# constraints <- DBI::dbGetQuery(con,
# "SELECT con.conname, rel.relname as relname
# FROM pg_catalog.pg_constraint con
# INNER JOIN pg_catalog.pg_class rel
# ON rel.oid = con.conrelid
# INNER JOIN pg_catalog.pg_namespace nsp
# ON nsp.oid = connamespace
# WHERE nsp.nspname = 'cdmddlbase';")
#
#
# constraints <- dplyr::mutate(constraints, sql = paste0("alter table ", schema, ".", relname, " drop constraint if exists ", conname, ";\n" ))
#
# sql <- paste(rev(constraints$sql), collapse = "")
# executeSql(con, sql)
#
# disconnect(con)
#
# }

View File

@ -0,0 +1,30 @@
test_that("buildRelease() output matches current ddl folder", {
tempfolder <- tempdir(check = TRUE)
generatedBaseFolder <- file.path(tempdir(check = TRUE), "ddl")
currentBaseFolder <- system.file("ddl", package = "CommonDataModel", mustWork = TRUE)
# build all of the releases in a temp folder
buildRelease(outputfolder = generatedBaseFolder)
# compare to the files in the current ddl folder of the package
generatedDirectories <- list.dirs(generatedBaseFolder, full.names = F)
currentDirectories <- list.dirs(currentBaseFolder, full.names = F)
expect_gt(length(currentDirectories), 1)
expect_setequal(generatedDirectories, currentDirectories)
# compare filenames
generatedFilenames <- list.files(generatedBaseFolder, recursive = TRUE)
currentFilenames <- list.files(generatedBaseFolder, recursive = TRUE)
expect_gt(length(currentFilenames), 1)
expect_setequal(generatedFilenames, currentFilenames)
# compare file contents using md5 hash
generatedChecksums <- tools::md5sum(file.path(generatedBaseFolder, generatedFilenames))
currentChecksums <- tools::md5sum(file.path(currentBaseFolder, currentFilenames))
names(generatedChecksums) <- NULL
names(currentChecksums) <- NULL
expect_gt(length(currentChecksums), 1)
expect_setequal(generatedChecksums, currentChecksums)
})

View File

@ -1,84 +1,51 @@
library(DatabaseConnector) # To run tests for one dbms and one cdmVersion use the variables below
# dbms <- "postgresql"; cdmVersion <- "5.4"
connectionDetails <- createConnectionDetails( testDatabases <- c("oracle", "postgresql", "sql server", "redshift")
dbms = "postgresql", # testDatabases <- c("redshift")
user = Sys.getenv("CDMDDLBASE_POSTGRESQL_USER"),
password = Sys.getenv("CDMDDLBASE_POSTGRESQL_PASSWORD"),
server = Sys.getenv("CDMDDLBASE_POSTGRESQL_SERVER"),
pathToDriver = file.path(Sys.getenv("HOME"), "drivers")
)
# Helper functions used in tests test_that("getConnectionDetails works", {
.listTablesInSchema <- function(connectionDetails, schema) { for(dbms in testDatabases) {
con <- DatabaseConnector::connect(connectionDetails) expect_s3_class(getConnectionDetails(dbms), "connectionDetails")
tables <- DBI::dbListObjects(con, prefix = "cdmddlbase")
DatabaseConnector::disconnect(con)
tables <- subset(tables, is_prefix == FALSE)
tables <- subset(tables, grepl("table", table))$table
tables <- vapply(tables, function(.) .@name, FUN.VALUE = character(1))
return(tables)
}
.dropAllTablesFromSchema <- function(connectionDetails, schema) {
tables <- .listTablesInSchema(connectionDetails, schema)
con <- DatabaseConnector::connect(connectionDetails)
for(table in tables) {
DBI::dbRemoveTable(con, name = DBI::SQL(paste(schema, table, sep = ".")))
} }
DatabaseConnector::disconnect(con)
}
.removeConstraintsPostgresql <- function(connectionDetails, schema) {
# the order of removal of constraints matters!
con <- DatabaseConnector::connect(connectionDetails)
constraints <- DBI::dbGetQuery(con,
"SELECT con.conname, rel.relname as relname
FROM pg_catalog.pg_constraint con
INNER JOIN pg_catalog.pg_class rel
ON rel.oid = con.conrelid
INNER JOIN pg_catalog.pg_namespace nsp
ON nsp.oid = connamespace
WHERE nsp.nspname = 'cdmddlbase';")
constraints <- dplyr::mutate(constraints, sql = paste0("alter table ", schema, ".", relname, " drop constraint if exists ", conname, ";\n" ))
sql <- paste(rev(constraints$sql), collapse = "")
executeSql(con, sql)
disconnect(con)
}
test_that("Database can be connected to", {
expect_error(con <- connect(connectionDetails), NA)
disconnect(con)
}) })
test_that("Execute DDL on Postgres", { test_that("getSchema works", {
# make sure the schema is cleared out for(dbms in testDatabases) {
cdmDatabaseSchema <- Sys.getenv("CDMDDLBASE_POSTGRESQL_SCHEMA") expect_type(getSchema(dbms), "character")
cdmVersion <- "5.4" }
# .removeConstraintsPostgresql(connectionDetails, cdmDatabaseSchema)
.dropAllTablesFromSchema(connectionDetails, cdmDatabaseSchema)
cat(paste("Connecting to schema", cdmDatabaseSchema, "\n"))
executeDdl(connectionDetails,
cdmVersion = cdmVersion,
cdmDatabaseSchema = cdmDatabaseSchema,
executeDdl = TRUE,
executePrimaryKey = FALSE,
executeForeignKey = FALSE)
tables <- .listTablesInSchema(connectionDetails, schema = cdmDatabaseSchema)
cdmTableCsvLoc <- system.file(file.path("csv", paste0("OMOP_CDMv", cdmVersion, "_Table_Level.csv")), package = "CommonDataModel", mustWork = TRUE)
tableSpecs <- read.csv(cdmTableCsvLoc, stringsAsFactors = FALSE)$cdmTableName
# check that the tables in the database match the tables in the specification
expect_equal(sort(tables), sort(tableSpecs))
# clear schema
# .removeConstraintsPostgresql(connectionDetails, cdmDatabaseSchema)
.dropAllTablesFromSchema(connectionDetails, cdmDatabaseSchema)
}) })
for(dbms in testDatabases) {
connectionDetails <- getConnectionDetails(dbms)
cdmDatabaseSchema <- getSchema(dbms)
test_that(paste("Can connect to", dbms), {
expect_error(con <- connect(connectionDetails), NA)
expect_error(disconnect(con), NA)
})
for(cdmVersion in listSupportedVersions()) {
test_that(paste("DDL", cdmVersion, "runs on", dbms), {
dropAllTablesFromSchema(connectionDetails, cdmDatabaseSchema)
tables <- listTablesInSchema(connectionDetails, cdmDatabaseSchema)
expect_equal(tables, character(0))
executeDdl(connectionDetails,
cdmVersion = cdmVersion,
cdmDatabaseSchema = cdmDatabaseSchema,
executeDdl = TRUE,
executePrimaryKey = TRUE,
executeForeignKey = FALSE)
tables <- listTablesInSchema(connectionDetails, cdmDatabaseSchema)
cdmTableCsvLoc <- system.file(file.path("csv", paste0("OMOP_CDMv", cdmVersion, "_Table_Level.csv")), package = "CommonDataModel", mustWork = TRUE)
tableSpecs <- read.csv(cdmTableCsvLoc, stringsAsFactors = FALSE)$cdmTableName
# check that the tables in the database match the tables in the specification
expect_equal(sort(tolower(tables)), sort(tolower(tableSpecs)))
dropAllTablesFromSchema(connectionDetails, cdmDatabaseSchema)
})
}
}

View File

@ -1,10 +1,10 @@
test_that("writeDdl works", { test_that("writeDdl works", {
outputpath <- tempdir(TRUE) outputfolder <- tempdir(TRUE)
filename <- writeDdl(targetDialect = "postgresql", cdmVersion = "5.4", outputpath = outputpath) filename <- writeDdl(targetDialect = "postgresql", cdmVersion = "5.4", outputfolder = outputfolder)
expect_true(file.exists(file.path(outputpath, filename))) expect_true(file.exists(file.path(outputfolder, filename)))
sql <- readr::read_file(file.path(outputpath, filename)) sql <- readr::read_file(file.path(outputfolder, filename))
expect_type(sql, "character") expect_type(sql, "character")
expect_gt(nchar(sql), 10) expect_gt(nchar(sql), 10)
@ -13,11 +13,11 @@ test_that("writeDdl works", {
test_that("writePrimaryKeys works", { test_that("writePrimaryKeys works", {
outputpath <- tempdir(TRUE) outputfolder <- tempdir(TRUE)
filename <- writePrimaryKeys(targetDialect = "postgresql", cdmVersion = "5.4", outputpath = outputpath) filename <- writePrimaryKeys(targetDialect = "postgresql", cdmVersion = "5.4", outputfolder = outputfolder)
expect_true(file.exists(file.path(outputpath, filename))) expect_true(file.exists(file.path(outputfolder, filename)))
sql <- readr::read_file(file.path(outputpath, filename)) sql <- readr::read_file(file.path(outputfolder, filename))
expect_type(sql, "character") expect_type(sql, "character")
expect_gt(nchar(sql), 10) expect_gt(nchar(sql), 10)
@ -26,11 +26,11 @@ test_that("writePrimaryKeys works", {
test_that("writeForeignKeys works", { test_that("writeForeignKeys works", {
outputpath <- tempdir(TRUE) outputfolder <- tempdir(TRUE)
filename <- writeForeignKeys(targetDialect = "postgresql", cdmVersion = "5.4", outputpath = outputpath) filename <- writeForeignKeys(targetDialect = "postgresql", cdmVersion = "5.4", outputfolder = outputfolder)
expect_true(file.exists(file.path(outputpath, filename))) expect_true(file.exists(file.path(outputfolder, filename)))
sql <- readr::read_file(file.path(outputpath, filename)) sql <- readr::read_file(file.path(outputfolder, filename))
expect_type(sql, "character") expect_type(sql, "character")
expect_gt(nchar(sql), 10) expect_gt(nchar(sql), 10)
@ -39,11 +39,11 @@ test_that("writeForeignKeys works", {
test_that("writeIndex works", { test_that("writeIndex works", {
outputpath <- tempdir(TRUE) outputfolder <- tempdir(TRUE)
filename <- writeIndex(targetDialect = "postgresql", cdmVersion = "5.4", outputpath = outputpath) filename <- writeIndex(targetDialect = "postgresql", cdmVersion = "5.4", outputfolder = outputfolder)
expect_true(file.exists(file.path(outputpath, filename))) expect_true(file.exists(file.path(outputfolder, filename)))
sql <- readr::read_file(file.path(outputpath, filename)) sql <- readr::read_file(file.path(outputfolder, filename))
expect_type(sql, "character") expect_type(sql, "character")
expect_gt(nchar(sql), 10) expect_gt(nchar(sql), 10)