Add test for buildRelease (#432)

* Add github actions workflow to build package and run tests.

* update Description file

* rename .Rproj file.

* Consolidate 'create' functions into one file.

* Add tests for create functions.

* update description

* removed spaces in file and folder names. Regenerated ddl output. Tried to fix Field_Level.csv file.

* consolidate write functions into one file. Add execute function.

* update docs

* add tests for write and execute functions

* update documentation

* Add windows and linux runners in github actions.

* update github actions

* download drivers before running tests

* fix small error in setup test file.

* debug github actions

* debug github actions

* debug github actions

* debug github actions

* fix tiny bug

* comment out execute ddl test

* fix bug in test

* Add execute test back in

* revert accidental change in description

* add print statement for debugging schema error on github actions.

* Fix schema environment variable name

* Add comment to github actions workflow file.

* remove placeholder text in function documentation.

* Rename createdDdl.R to createDdl.R

* Hack-a-thon updates

Closes #81, #387, #239, #412, #391, #330, #408, #365, #306, #264

* Changed bigint to integer for consistency

* Updated DDLs

* Add tests for redshift. Clean up test setup file.

* Foreign key fixes

* Add imports and update docs.

* Fix bug in setup test script.

* update setup file

* Add tests for oracle and sql server. Move setup.R file.

* fix bug in setup

* debug tests on github

* debug github actions

* debug actions.

* debug actions

* debug actions.

* Add missing secrets to yaml!!

* debug actions

* test connection on all platforms

* add ddl execution

* add windows and linux runners

* Allow user to specify output location in buildRelease

* replace outputpath with outputfolder for consitent argument names in the package.

* Add test for buildRelease.

* replace outputpath with outputfolder for consistency. update documentation.

* move ddl folder to inst so it is accessible from tests

* update documentation

Co-authored-by: Adam Black <adam.black@odysseusinc.com>
Co-authored-by: Clair Blacketer <mblacke@its.jnj.com>

NAMESPACE

@@ -1,5 +1,6 @@
 # Generated by roxygen2: do not edit by hand
 
+export(buildRelease)
 export(buildReleaseZip)
 export(createDdl)
 export(createForeignKeys)

R/buildRelease.R

@@ -19,25 +19,36 @@
 #' Writes DDL, ForeignKey, PrimaryKey and index SQL files for given cdmVersion
 #' and targetDialect to the 'ddl' folder in current working directory.
 #'
-#' @param cdmVersion The version of the CDM you are creating, e.g. 5.3, 5.4.
+#' @param cdmVersions The versions of the CDM you are creating, e.g. 5.3, 5.4.
 #'                   Defaults to all supported CDM versions.
-#' @param targetDialect The target dialect
-#'
-buildRelease <- function(cdmVersion = listSupportedVersions(),
-                         targetDialect = listSupportedDialects()){
-  for (cdmVersion in cdmVersion) {
-    for (targetDialect in targetDialect) {
+#' @param targetDialects A character vector of target dialects.
+#'                      Defaults to all supported dialects.
+#' @param outputfolder The base folder where the SQL files will be written.
+#'                     Subfolders will be created for each cdmVersion and targetDialect.
+#' @export
+buildRelease <- function(cdmVersions = listSupportedVersions(),
+                         targetDialects = listSupportedDialects(),
+                         outputfolder = file.path(getwd(), "inst", "ddl")){
+  basefolder <- outputfolder
+  for (cdmVersion in cdmVersions) {
+    for (targetDialect in targetDialects) {
+      outputfolder <- file.path(basefolder, cdmVersion, gsub(" ", "_", targetDialect))
+
       writeDdl(targetDialect = targetDialect,
-               cdmVersion = cdmVersion)
+               cdmVersion = cdmVersion,
+               outputfolder = outputfolder)
 
       writePrimaryKeys(targetDialect = targetDialect,
-                       cdmVersion = cdmVersion)
+                       cdmVersion = cdmVersion,
+                       outputfolder = outputfolder)
 
       writeForeignKeys(targetDialect = targetDialect,
-                       cdmVersion = cdmVersion)
+                       cdmVersion = cdmVersion,
+                       outputfolder = outputfolder)
 
       writeIndex(targetDialect = targetDialect,
-                 cdmVersion = cdmVersion)
+                 cdmVersion = cdmVersion,
+                 outputfolder = outputfolder)
     }
   }
 }
@@ -62,7 +73,7 @@ buildRelease <- function(cdmVersion = listSupportedVersions(),
 #'
 buildReleaseZip <- function(cdmVersion,
                             targetDialect = listSupportedDialects(),
-                            outputfolder = "output"){
+                            outputfolder = file.path(getwd(), "inst", "ddl")){
   # argument checks
   stopifnot(is.character(cdmVersion), length(cdmVersion) == 1, cdmVersion %in% listSupportedVersions())
 
@@ -72,8 +83,8 @@ buildReleaseZip <- function(cdmVersion,
 
   files <- c()
   for (dialect in targetDialect) {
-    buildRelease(cdmVersion, dialect)
-    files <- c(files, list.files(file.path('ddl', cdmVersion, gsub(" ", "_", dialect)),
+    buildRelease(cdmVersion, dialect, outputfolder = outputfolder)
+    files <- c(files, list.files(file.path(outputfolder, cdmVersion, gsub(" ", "_", dialect)),
                                  pattern = ".*\\.sql$",
                                  full.names = TRUE))
   }

R/executeDdl.R

@@ -42,16 +42,16 @@ executeDdl <- function(connectionDetails,
                        executeForeignKey = TRUE,
                        ...) {
 
-  outputpath <- tempdir(check = TRUE)
+  outputfolder <- tempdir(check = TRUE)
 
 
   if(executeDdl) {
     filename <- writeDdl(targetDialect = connectionDetails$dbms,
                          cdmVersion = cdmVersion,
                          cdmDatabaseSchema = cdmDatabaseSchema,
-                         outputpath = outputpath)
+                         outputfolder = outputfolder)
 
-    sql <- readr::read_file(file.path(outputpath, filename))
+    sql <- readr::read_file(file.path(outputfolder, filename))
   } else {
     sql <- ""
   }
@@ -60,18 +60,18 @@ executeDdl <- function(connectionDetails,
     filename <- writePrimaryKeys(targetDialect = connectionDetails$dbms,
                                  cdmVersion = cdmVersion,
                                  cdmDatabaseSchema = cdmDatabaseSchema,
-                                 outputpath = outputpath)
+                                 outputfolder = outputfolder)
 
-    sql <- paste(sql, readr::read_file(file.path(outputpath, filename)), sep = "\n")
+    sql <- paste(sql, readr::read_file(file.path(outputfolder, filename)), sep = "\n")
   }
 
   if(executeForeignKey) {
     filename <- writeForeignKeys(targetDialect = connectionDetails$dbms,
                                  cdmVersion = cdmVersion,
                                  cdmDatabaseSchema = cdmDatabaseSchema,
-                                 outputpath = outputpath)
+                                 outputfolder = outputfolder)
 
-    sql <- paste(sql, readr::read_file(file.path(outputpath, filename)), sep = "\n")
+    sql <- paste(sql, readr::read_file(file.path(outputfolder, filename)), sep = "\n")
   }
 
   con <- DatabaseConnector::connect(connectionDetails = connectionDetails)

R/writeDDL.R

@@ -22,95 +22,95 @@
 #'
 #' @param targetDialect  The dialect of the target database. Choices are "oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server"
 #' @param cdmVersion The version of the CDM you are creating, e.g. 5.3, 5.4
-#' @param outputpath The directory or folder where the SQL file should be saved.
+#' @param outputfolder The directory or folder where the SQL file should be saved.
 #' @param cdmDatabaseSchema The schema of the CDM instance where the DDL will be run. For example, this would be "ohdsi.dbo" when testing on sql server.
 #'                          Defaults to "@cdmDatabaseSchema"
 #'
 #' @export
-writeDdl <- function(targetDialect, cdmVersion, outputpath, cdmDatabaseSchema = "@cdmDatabaseSchema") {
+writeDdl <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchema = "@cdmDatabaseSchema") {
 
   # argument checks
   stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server"))
   stopifnot(cdmVersion %in% listSupportedVersions())
   stopifnot(is.character(cdmDatabaseSchema))
 
-  if(missing(outputpath)) {
-    outputpath <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
+  if(missing(outputfolder)) {
+    outputfolder <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
   }
 
-  if(!dir.exists(outputpath)) dir.create(outputpath, showWarnings = FALSE, recursive = TRUE)
+  if(!dir.exists(outputfolder)) dir.create(outputfolder, showWarnings = FALSE, recursive = TRUE)
 
   sql <- createDdl(cdmVersion)
   sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect)
   sql <- SqlRender::translate(sql, targetDialect = targetDialect)
 
   filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "ddl.sql", sep = "_")
-  SqlRender::writeSql(sql = sql, targetFile = file.path(outputpath, filename))
+  SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
   invisible(filename)
 }
 
 #' @describeIn writeDdl writePrimaryKeys Write the SQL code that creates the primary keys to a file.
 #' @export
-writePrimaryKeys <- function(targetDialect, cdmVersion, outputpath, cdmDatabaseSchema = "@cdmDatabaseSchema") {
+writePrimaryKeys <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchema = "@cdmDatabaseSchema") {
 
   # argument checks
   stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server"))
   stopifnot(cdmVersion %in% listSupportedVersions())
   stopifnot(is.character(cdmDatabaseSchema))
 
-  if(missing(outputpath)) {
-    outputpath <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
+  if(missing(outputfolder)) {
+    outputfolder <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
   }
 
-  if(!dir.exists(outputpath)) dir.create(outputpath, showWarnings = FALSE, recursive = TRUE)
+  if(!dir.exists(outputfolder)) dir.create(outputfolder, showWarnings = FALSE, recursive = TRUE)
 
   sql <- createPrimaryKeys(cdmVersion)
   sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect)
   sql <- SqlRender::translate(sql, targetDialect = targetDialect)
 
   filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "primary", "keys.sql", sep = "_")
-  SqlRender::writeSql(sql = sql, targetFile = file.path(outputpath, filename))
+  SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
   invisible(filename)
 }
 
 #' @describeIn writeDdl writeForeignKeys Write the SQL code that creates the foreign keys to a file.
 #' @export
-writeForeignKeys <- function(targetDialect, cdmVersion, outputpath, cdmDatabaseSchema = "@cdmDatabaseSchema") {
+writeForeignKeys <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchema = "@cdmDatabaseSchema") {
 
   # argument checks
   stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server"))
   stopifnot(cdmVersion %in% listSupportedVersions())
   stopifnot(is.character(cdmDatabaseSchema))
 
-  if(missing(outputpath)) {
-    outputpath <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
+  if(missing(outputfolder)) {
+    outputfolder <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
   }
 
-  if(!dir.exists(outputpath)) dir.create(outputpath, showWarnings = FALSE, recursive = TRUE)
+  if(!dir.exists(outputfolder)) dir.create(outputfolder, showWarnings = FALSE, recursive = TRUE)
 
   sql <- createForeignKeys(cdmVersion)
   sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect)
   sql <- SqlRender::translate(sql, targetDialect = targetDialect)
 
   filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "constraints.sql", sep = "_")
-  SqlRender::writeSql(sql = sql, targetFile = file.path(outputpath, filename))
+  SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
   invisible(filename)
 }
 
 #' @describeIn writeDdl writeIndex Write the rendered and translated sql that creates recommended indexes to a file.
 #' @export
-writeIndex <- function(targetDialect, cdmVersion, outputpath, cdmDatabaseSchema  = "@cdmDatabaseSchema") {
+writeIndex <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchema  = "@cdmDatabaseSchema") {
 
   # argument checks
   stopifnot(targetDialect %in% c("oracle", "postgresql", "pdw", "redshift", "impala", "netezza", "bigquery", "sql server"))
   stopifnot(cdmVersion %in% listSupportedVersions())
   stopifnot(is.character(cdmDatabaseSchema))
 
-  if(missing(outputpath)) {
-    outputpath <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
+  if(missing(outputfolder)) {
+    outputfolder <- file.path("ddl", cdmVersion, gsub(" ", "_", targetDialect))
   }
 
-  if(!dir.exists(outputpath)) dir.create(outputpath, showWarnings = FALSE, recursive = TRUE)
+  if(!dir.exists(outputfolder)) dir.create(outputfolder, showWarnings = FALSE, recursive = TRUE)
 
   sqlFilename <- paste0("OMOP_CDM_indices_v", cdmVersion, ".sql")
   sql <- readr::read_file(system.file(file.path("sql", "sql_server", sqlFilename), package = "CommonDataModel"))
@@ -118,6 +118,6 @@ writeIndex <- function(targetDialect, cdmVersion, outputpath, cdmDatabaseSchema
   sql <- SqlRender::translate(sql, targetDialect = targetDialect)
 
   filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "indices.sql", sep = "_")
-  SqlRender::writeSql(sql = sql, targetFile = file.path(outputpath, filename))
+  SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
   invisible(filename)
 }

man/buildRelease.Rd

@@ -5,15 +5,20 @@
 \title{Create OMOP CDM SQL files}
 \usage{
 buildRelease(
-  cdmVersion = listSupportedVersions(),
-  targetDialect = listSupportedDialects()
+  cdmVersions = listSupportedVersions(),
+  targetDialects = listSupportedDialects(),
+  outputfolder = file.path(getwd(), "inst", "ddl")
 )
 }
 \arguments{
-\item{cdmVersion}{The version of the CDM you are creating, e.g. 5.3, 5.4.
+\item{cdmVersions}{The versions of the CDM you are creating, e.g. 5.3, 5.4.
 Defaults to all supported CDM versions.}
 
-\item{targetDialect}{The target dialect}
+\item{targetDialects}{A character vector of target dialects.
+Defaults to all supported dialects.}
+
+\item{outputfolder}{The base folder where the SQL files will be written.
+Subfolders will be created for each cdmVersion and targetDialect.}
 }
 \description{
 Writes DDL, ForeignKey, PrimaryKey and index SQL files for given cdmVersion

man/buildReleaseZip.Rd

@@ -7,7 +7,7 @@
 buildReleaseZip(
   cdmVersion,
   targetDialect = listSupportedDialects(),
-  outputfolder = "output"
+  outputfolder = file.path(getwd(), "inst", "ddl")
 )
 }
 \arguments{

man/writeDdl.Rd

@@ -10,28 +10,28 @@
 writeDdl(
   targetDialect,
   cdmVersion,
-  outputpath,
+  outputfolder,
   cdmDatabaseSchema = "@cdmDatabaseSchema"
 )
 
 writePrimaryKeys(
   targetDialect,
   cdmVersion,
-  outputpath,
+  outputfolder,
   cdmDatabaseSchema = "@cdmDatabaseSchema"
 )
 
 writeForeignKeys(
   targetDialect,
   cdmVersion,
-  outputpath,
+  outputfolder,
   cdmDatabaseSchema = "@cdmDatabaseSchema"
 )
 
 writeIndex(
   targetDialect,
   cdmVersion,
-  outputpath,
+  outputfolder,
   cdmDatabaseSchema = "@cdmDatabaseSchema"
 )
 }
@@ -40,7 +40,7 @@ writeIndex(
 
 \item{cdmVersion}{The version of the CDM you are creating, e.g. 5.3, 5.4}
 
-\item{outputpath}{The directory or folder where the SQL file should be saved.}
+\item{outputfolder}{The directory or folder where the SQL file should be saved.}
 
 \item{cdmDatabaseSchema}{The schema of the CDM instance where the DDL will be run. For example, this would be "ohdsi.dbo" when testing on sql server.
 Defaults to "@cdmDatabaseSchema"}

tests/testthat/test-buildRelease.R

@@ -0,0 +1,30 @@
+test_that("buildRelease() output matches current ddl folder", {
+
+  tempfolder <- tempdir(check = TRUE)
+  generatedBaseFolder <- file.path(tempdir(check = TRUE), "ddl")
+  currentBaseFolder <- system.file("ddl", package = "CommonDataModel", mustWork = TRUE)
+
+  # build all of the releases in a temp folder
+  buildRelease(outputfolder = generatedBaseFolder)
+
+  # compare to the files in the current ddl folder of the package
+  generatedDirectories <- list.dirs(generatedBaseFolder, full.names = F)
+  currentDirectories <- list.dirs(currentBaseFolder, full.names = F)
+  expect_gt(length(currentDirectories), 1)
+  expect_setequal(generatedDirectories, currentDirectories)
+
+  # compare filenames
+  generatedFilenames <- list.files(generatedBaseFolder, recursive = TRUE)
+  currentFilenames <- list.files(generatedBaseFolder, recursive = TRUE)
+  expect_gt(length(currentFilenames), 1)
+  expect_setequal(generatedFilenames, currentFilenames)
+
+  # compare file contents using md5 hash
+  generatedChecksums <- tools::md5sum(file.path(generatedBaseFolder, generatedFilenames))
+  currentChecksums <- tools::md5sum(file.path(currentBaseFolder, currentFilenames))
+  names(generatedChecksums) <- NULL
+  names(currentChecksums) <- NULL
+  expect_gt(length(currentChecksums), 1)
+  expect_setequal(generatedChecksums, currentChecksums)
+
+})

tests/testthat/test-writeDdl.R

@@ -1,10 +1,10 @@
 test_that("writeDdl works", {
 
-  outputpath <- tempdir(TRUE)
-  filename <- writeDdl(targetDialect = "postgresql", cdmVersion = "5.4", outputpath = outputpath)
+  outputfolder <- tempdir(TRUE)
+  filename <- writeDdl(targetDialect = "postgresql", cdmVersion = "5.4", outputfolder = outputfolder)
 
-  expect_true(file.exists(file.path(outputpath, filename)))
-  sql <- readr::read_file(file.path(outputpath, filename))
+  expect_true(file.exists(file.path(outputfolder, filename)))
+  sql <- readr::read_file(file.path(outputfolder, filename))
 
   expect_type(sql, "character")
   expect_gt(nchar(sql), 10)
@@ -13,11 +13,11 @@ test_that("writeDdl works", {
 
 test_that("writePrimaryKeys works", {
 
-  outputpath <- tempdir(TRUE)
-  filename <- writePrimaryKeys(targetDialect = "postgresql", cdmVersion = "5.4", outputpath = outputpath)
+  outputfolder <- tempdir(TRUE)
+  filename <- writePrimaryKeys(targetDialect = "postgresql", cdmVersion = "5.4", outputfolder = outputfolder)
 
-  expect_true(file.exists(file.path(outputpath, filename)))
-  sql <- readr::read_file(file.path(outputpath, filename))
+  expect_true(file.exists(file.path(outputfolder, filename)))
+  sql <- readr::read_file(file.path(outputfolder, filename))
 
   expect_type(sql, "character")
   expect_gt(nchar(sql), 10)
@@ -26,11 +26,11 @@ test_that("writePrimaryKeys works", {
 
 test_that("writeForeignKeys works", {
 
-  outputpath <- tempdir(TRUE)
-  filename <- writeForeignKeys(targetDialect = "postgresql", cdmVersion = "5.4", outputpath = outputpath)
+  outputfolder <- tempdir(TRUE)
+  filename <- writeForeignKeys(targetDialect = "postgresql", cdmVersion = "5.4", outputfolder = outputfolder)
 
-  expect_true(file.exists(file.path(outputpath, filename)))
-  sql <- readr::read_file(file.path(outputpath, filename))
+  expect_true(file.exists(file.path(outputfolder, filename)))
+  sql <- readr::read_file(file.path(outputfolder, filename))
 
   expect_type(sql, "character")
   expect_gt(nchar(sql), 10)
@@ -39,11 +39,11 @@ test_that("writeForeignKeys works", {
 
 test_that("writeIndex works", {
 
-  outputpath <- tempdir(TRUE)
-  filename <- writeIndex(targetDialect = "postgresql", cdmVersion = "5.4", outputpath = outputpath)
+  outputfolder <- tempdir(TRUE)
+  filename <- writeIndex(targetDialect = "postgresql", cdmVersion = "5.4", outputfolder = outputfolder)
 
-  expect_true(file.exists(file.path(outputpath, filename)))
-  sql <- readr::read_file(file.path(outputpath, filename))
+  expect_true(file.exists(file.path(outputfolder, filename)))
+  sql <- readr::read_file(file.path(outputfolder, filename))
 
   expect_type(sql, "character")
   expect_gt(nchar(sql), 10)