Skip to content

Commit 558d8f6

Browse files
authored
Merge pull request #282 from USEPA/198-update-github-actions-to-include-dev-branch
Large update to unit test suite including mocking invitrodb and CTX API data
2 parents d6f5645 + f489c04 commit 558d8f6

40 files changed

+3914
-1197
lines changed

Diff for: .github/workflows/test-on-PR.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
33
on:
44
pull_request:
5-
branches: [main]
5+
branches: [main, dev]
66

77
name: test-coverage
88

Diff for: R/data.R

+255-1
Large diffs are not rendered by default.

Diff for: R/tcplConf.R

+3-3
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,9 @@ tcplConf <- function (drvr = NULL, user = NULL, pass = NULL, host = NULL,
7474

7575
if (drvr == "API") {
7676
options("TCPL_DRVR" = "API")
77-
if (is.null(pass)) stop("'API' driver requires an API-key, supply it to
78-
the 'pass' parameter. To request a key, send an
79-
email to [email protected].")
77+
if (is.null(pass)) stop("'API' driver requires an API-key, supply it to ",
78+
"the 'pass' parameter. To request a key, send an ",
79+
"email to [email protected].")
8080
if (is.null(host)) options("TCPL_HOST" = "https://api-ccte.epa.gov/bioactivity")
8181
register_ctx_api_key(key = pass)
8282
}

Diff for: R/tcplLoadChem.R

+3
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ tcplLoadChem <- function(field = NULL, val = NULL, exact = TRUE,
5959
if (tolower(field) != "spid") stop("When drvr option is set to 'API', only 'spid' is a valid 'field' value.")
6060
if (!exact) exact <- TRUE
6161
dat <- tcplQueryAPI(resource = "data", fld = "spid", val = val, return_flds = c("spid", "chid", "casn", "chnm", "dsstox_substance_id"))
62+
if (!length(colnames(dat))) {
63+
return(dat)
64+
}
6265
setorder(dat, "spid")
6366
} else {
6467
tbl <- c("chemical", "sample")

Diff for: R/tcplLoadConcUnit.R

+4-2
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@ tcplLoadConcUnit <- function(spid) {
2121

2222
if (getOption("TCPL_DRVR") == "API") {
2323
dat <- tcplQueryAPI(resource = "data", fld = "spid", val = spid, return_flds = c("spid", "tested_conc_unit"))
24-
setnames(dat, "tested_conc_unit", "conc_unit")
25-
setorder(dat, "spid")
24+
if (length(colnames(dat))) {
25+
setnames(dat, "tested_conc_unit", "conc_unit")
26+
setorder(dat, "spid")
27+
}
2628
return(unique(dat, by = c("spid", "conc_unit")))
2729
}
2830

Diff for: R/tcplLoadData.R

+30-18
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ tcplLoadData <- function(lvl, fld = NULL, val = NULL, type = "mc", add.fld = TRU
108108
}
109109
else if (lvl == 1L) {
110110
sc1 <- sc_vignette[["sc1"]]
111-
sc1 <- sc1[,c("s0id","s1id","spid","acid","aeid","apid","rowi","coli","wllt","logc","resp")]
111+
sc1 <- sc1[,c("s0id","s1id","spid","acid","aeid","apid","rowi","coli","wllt","conc","resp")]
112112
return(sc1)
113113
}
114114

@@ -121,7 +121,7 @@ tcplLoadData <- function(lvl, fld = NULL, val = NULL, type = "mc", add.fld = TRU
121121
sc1 <- sc_vignette[["sc1"]]
122122
sc2 <- sc_vignette[["sc2"]]
123123
agg <- sc1[sc2, on = c("spid","aeid")]
124-
agg <- agg[,c("aeid","s2id","s1id","s0id","logc","resp")]
124+
agg <- agg[,c("aeid","s2id","s1id","s0id","conc","resp")]
125125
return(agg)
126126
}
127127
else stop("example tables for sc0, sc1, sc2, agg available.")
@@ -146,14 +146,17 @@ tcplLoadData <- function(lvl, fld = NULL, val = NULL, type = "mc", add.fld = TRU
146146
}
147147
else if (lvl == 3L) {
148148
mc3 <- mc_vignette[["mc3"]]
149-
mc3 <- mc3[,c("m0id","m1id","m2id","m3id","spid","aeid","logc","resp","cndx","wllt","apid","rowi","coli","repi")]
149+
mc3 <- mc3[,c("m0id","m1id","m2id","m3id","spid","aeid","conc","resp","cndx","wllt","apid","rowi","coli","repi")]
150150
return(mc3)
151151
}
152152
else if (lvl == 4L) {
153153
mc4 <- mc_vignette[["mc4"]]
154154
if (!add.fld) {
155-
mc4 <- mc4[,c("m4id","aeid","spid","bmad","resp_max","resp_min","max_mean","max_mean_conc","max_med","max_med_conc",
156-
"logc_max","logc_min","nconc","npts","nrep","nmed_gtbl")]
155+
mc4 <- mc4[,c("m4id", "aeid", "spid", "bmad", "resp_max", "resp_min",
156+
"max_mean", "max_mean_conc", "min_mean", "min_mean_conc",
157+
"max_med", "max_med_conc", "min_med", "min_med_conc",
158+
"max_med_diff", "max_med_diff_conc", "conc_max", "conc_min",
159+
"nconc", "npts", "nrep", "nmed_gtbl_pos", "nmed_gtbl_neg")]
157160
} else {
158161
mc4 <- mc4[,!c("chid","casn","chnm","dsstox_substance_id","code","aenm","resp_unit","conc_unit")]
159162
setcolorder(mc4, c("m4id", "aeid", "spid"))
@@ -163,10 +166,15 @@ tcplLoadData <- function(lvl, fld = NULL, val = NULL, type = "mc", add.fld = TRU
163166
else if (lvl == 5L) {
164167
mc5 <- mc_vignette[["mc5"]]
165168
if (!add.fld){
166-
mc5 <- mc5[,c("m5id","m4id","aeid","spid","bmad","resp_max","resp_min","max_mean","max_mean_conc","max_med",
167-
"max_med_conc","logc_max","logc_min","nconc","npts","nrep","nmed_gtbl","hitc","modl","fitc","coff")]
169+
mc5 <- mc5[,c("m5id","m4id", "aeid", "spid", "bmad", "resp_max", "resp_min",
170+
"max_mean", "max_mean_conc", "min_mean", "min_mean_conc",
171+
"max_med", "max_med_conc", "min_med", "min_med_conc",
172+
"max_med_diff", "max_med_diff_conc", "conc_max", "conc_min",
173+
"nconc", "npts", "nrep", "nmed_gtbl_pos", "nmed_gtbl_neg",
174+
"hitc", "modl", "fitc", "coff")]
168175
} else {
169-
mc5 <- mc5[,!c("chid","casn","chnm","dsstox_substance_id","code","aenm","resp_unit","conc_unit","tp","ga","q","la","ac50_loss")]
176+
mc5 <- mc5[,!c("chid","casn","chnm","dsstox_substance_id","code","aenm",
177+
"resp_unit","conc_unit","tp","ga","q","la","ac50_loss")]
170178
setcolorder(mc5, c("m5id", "m4id","aeid", "spid"))
171179
}
172180
return(mc5)
@@ -175,7 +183,8 @@ tcplLoadData <- function(lvl, fld = NULL, val = NULL, type = "mc", add.fld = TRU
175183
mc3 <- mc_vignette[["mc3"]]
176184
mc4 <- mc_vignette[["mc4"]]
177185
agg <- mc3[mc4, on = c("spid","aeid")]
178-
agg <- agg[, c("aeid", "m4id", "m3id", "m2id", "m1id", "m0id", "spid", "logc", "resp")]
186+
agg <- agg[, c("aeid", "m4id", "m3id", "m2id", "m1id", "m0id", "spid",
187+
"conc", "resp")]
179188
return(agg)
180189

181190
}
@@ -204,15 +213,18 @@ tcplLoadData <- function(lvl, fld = NULL, val = NULL, type = "mc", add.fld = TRU
204213
# query the API
205214
dat <- tcplQueryAPI(fld = fld, val = val, return_flds = cols)
206215

207-
if (lvl == 3) {
208-
dat$resp <- lapply(dat$resp, unlist)
209-
dat$logc <- lapply(dat$logc, unlist)
210-
dat <- unnest_longer(dat, c(conc, logc, resp)) %>% as.data.table()
211-
}
212-
213-
if (lvl == 6) {
214-
dat$flag <- lapply(dat$flag, unlist)
215-
dat <- unnest_longer(dat, flag) %>% filter(flag != "NULL") %>% as.data.table()
216+
if (length(colnames(dat))) {
217+
if (lvl == 3 | lvl == "agg") {
218+
dat$resp <- lapply(dat$resp, unlist)
219+
dat$logc <- lapply(dat$logc, unlist)
220+
if (lvl == 3) dat <- unnest_longer(dat, c(conc, logc, resp)) %>% as.data.table()
221+
else dat <- unnest_longer(dat, c(logc, resp)) %>% as.data.table()
222+
}
223+
224+
if (lvl == 6) {
225+
dat$flag <- lapply(dat$flag, unlist)
226+
dat <- unnest_longer(dat, flag) %>% filter(flag != "NULL") %>% as.data.table()
227+
}
216228
}
217229

218230
return(dat)

Diff for: R/tcplPlot.R

+8-6
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ tcplPlotlyPlot <- function(dat, lvl = 5){
202202
# extract range from level 3 data for creating plotting all the functions
203203
# increase resolution to get smoother curves
204204
resolution <- 100
205-
x_min_max <- range(l3_dat_both$conc)
205+
x_min_max <- range(l3_dat_both$conc, na.rm=TRUE)
206206
#if the overall minimum conc is greater than 0 (test wells)
207207
if (x_min_max[1] > 0) {
208208
hline_range <- 10^(seq(from = log10(x_min_max[1]/100), to = log10(x_min_max[2]*100), length.out = resolution))
@@ -413,11 +413,13 @@ tcplPlotlyPlot <- function(dat, lvl = 5){
413413
}
414414

415415
# compare data
416-
if (!is.null(compare.dat$coff) && compare.dat$max_med < 0) {
417-
compare.dat$coff <- compare.dat$coff * -1
418-
}
419-
if (!is.null(compare.dat$coff) && !is.null(compare.dat$hitc) && compare.dat$hitc < 0) {
420-
compare.dat$coff <- compare.dat$coff * -1
416+
if (nrow(compare.dat) > 0) {
417+
if (!is.null(compare.dat$coff) && compare.dat$max_med < 0) {
418+
compare.dat$coff <- compare.dat$coff * -1
419+
}
420+
if (!is.null(compare.dat$coff) && !is.null(compare.dat$hitc) && compare.dat$hitc < 0) {
421+
compare.dat$coff <- compare.dat$coff * -1
422+
}
421423
}
422424
}
423425

Diff for: R/tcplPlotUtils.R

+2-1
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,15 @@ tcplPlotSetYRange <- function(dat,yuniform,yrange,type){
3838
}
3939

4040

41-
tcplPlotValidate <- function(type = "mc",flags = NULL,output = "none",multi = FALSE,verbose = FALSE){
41+
tcplPlotValidate <- function(type = "mc",flags = NULL,output = "none",multi = NULL,verbose = FALSE){
4242

4343
# set lvl based on type
4444
lvl <- 5
4545
if (type == "sc") {
4646
lvl <- 2
4747
if (flags == TRUE) {
4848
warning("'flags' was set to TRUE - no flags exist for plotting single concentration")
49+
flags = FALSE
4950
}
5051
}
5152

Diff for: data-raw/mc_test.R

+161
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
#==============================================================================#
2+
# NOTE: This script is written such that it is run from 'top' to 'bottom'
3+
# or programmatically via the Terminal.
4+
# ('R CMD BATCH --vanilla <script.R>'.)
5+
# Please do not jump around when running this script.
6+
#==============================================================================#
7+
# NOTE: You MUST temporarily update tcplQuery() by adding a line at the top of the
8+
# function: print(query).
9+
# This is because the queries will be captured from output and saved as
10+
# part of the mocking data.
11+
#==============================================================================#
12+
## r packages
13+
devtools::load_all()
14+
15+
library(here)
16+
library(dplyr)
17+
library(stringr)
18+
#---------------------------#
19+
## code to prepare `mc_test` dataset goes here
20+
# source the user ID, password, host, and database information for connection
21+
# - NOTE: To replicate one will need to save their own 'db_cred.R', including
22+
# the 'userid', 'userpwd', 'host', and DB collection via 'ivtdb'.
23+
source(file = here::here("data-raw/db_cred.R"),verbose = FALSE)
24+
# connect to the DB
25+
tcplConf(user = userid,
26+
pass = userpwd,
27+
host = host,
28+
db = ivtdb,
29+
drvr = "MySQL")
30+
31+
# pick endpoints and ids
32+
# load the number of rows and max hitc per aeid
33+
mc5_counts <- tcplQuery("SELECT DISTINCT aeid,
34+
COUNT( aeid ) as n,
35+
max(hitc) as max_hitc
36+
FROM invitrodb.mc5 GROUP BY aeid")
37+
# filter to only include where at least one sample is active and n < 10
38+
mc5_counts <- mc5_counts %>% filter(max_hitc > 0.9 & n == 2)
39+
# pick one aeid
40+
aeid <- selected <- mc5_counts[sample(1:nrow(mc5_counts),size = 1,replace = FALSE),aeid]
41+
# obtain the acid for the example dataset
42+
acid <- tcplLoadAcid(fld = 'aeid',val = aeid)$acid
43+
# pick one sample/row from each level (lvl 3 contains ids back to lvl 0 and lvl 6 does back to lvl 4)
44+
l3 <- tcplLoadData(lvl = 3, fld = "acid", val = acid)
45+
l3_sample1 <- l3[sample(1:nrow(l3),size = 1,replace = FALSE)]
46+
l3_sample2 <- l3[sample(1:nrow(l3),size = 2,replace = FALSE)]
47+
l5 <- tcplLoadData(lvl = 5, fld = "aeid", val = aeid, add.fld = FALSE)
48+
l5_sample1 <- l5[sample(1:nrow(l5),size = 1,replace = FALSE)]
49+
l5_sample2 <- l5[sample(1:nrow(l5),size = 2,replace = FALSE)]
50+
l6 <- tcplLoadData(lvl = 6, fld = "aeid", val = aeid, add.fld = FALSE)
51+
l6_sample1 <- l6[sample(1:nrow(l6),size = 1,replace = FALSE)]
52+
l6_sample2 <- l6[sample(1:nrow(l6),size = 2,replace = FALSE)]
53+
l7 <- tcplLoadData(lvl = 7, fld = "aeid", val = aeid, add.fld = FALSE)
54+
l7_sample1 <- l7[sample(1:nrow(l7),size = 1,replace = FALSE)]
55+
l7_sample2 <- l7[sample(1:nrow(l7),size = 2,replace = FALSE)]
56+
# pick compare.val endpoints and ids
57+
# be sure to only allow to choose from endpoints with the same number of samples
58+
mc5_counts <- filter(mc5_counts, n == mc5_counts[aeid == selected]$n & aeid != selected)
59+
compare.aeid <- mc5_counts[sample(1:nrow(mc5_counts),size = 1,replace = FALSE),aeid]
60+
compare.l5 <- tcplLoadData(lvl = 5, fld = "aeid", val = compare.aeid)
61+
compare.l5_sample1 <- compare.l5[sample(1:nrow(compare.l5),size = 1,replace = FALSE)]
62+
compare.l5_sample2 <- compare.l5[sample(1:nrow(compare.l5),size = 2,replace = FALSE)]
63+
64+
65+
get_query_data <- function(lvl, fld, val, compare.val = NULL, add.fld = TRUE, func = "tcplLoadData") {
66+
message(compare.val)
67+
if (func == "tcplLoadData") {
68+
# IMPORTANT || MUST ADD TEMPORARY LINE TO TCPLQUERY --------------------------
69+
# add temporary line to top of tcplQuery to get the query string: print(query)
70+
query_strings <- capture.output(result<-tcplLoadData(lvl = lvl, fld = fld, val = val, add.fld = add.fld))
71+
} else if (func == "tcplPlot") {
72+
query_strings <- capture.output(result<-tcplPlot(type = "mc", fld = fld,
73+
val = val, compare.val = compare.val,
74+
output = "pdf", multi = TRUE, flags = TRUE,
75+
fileprefix = "temp_tcplPlot"))
76+
file.remove(stringr::str_subset(list.files(), "^temp_tcplPlot")) # clean up
77+
}
78+
79+
query_strings <- unique(gsub("\\\\", "\\\"", gsub("\"", "", gsub("\\\\n", "\\\n", gsub("\\[1\\] ", "", query_strings)))))
80+
81+
# use queries to save data
82+
dat <- lapply(query_strings, function(query_string) {
83+
return(tcplQuery(query_string))
84+
})
85+
names(dat) <- query_strings
86+
87+
# also store fld and val in list object for use in test case
88+
dat[fld] <- val
89+
if (!is.null(compare.val)) dat[sprintf("compare.%s", fld)] <- compare.val
90+
return(dat)
91+
92+
}
93+
94+
95+
# to add more tests with new/different data to test-tcplLoadData.R, add lines below and run script
96+
mc_test <- list(
97+
tcplConfQuery = tcplQuery("SHOW VARIABLES LIKE 'max_allowed_packet'"),
98+
mc0_by_m0id = get_query_data(lvl = 0, fld = "m0id", val = l3_sample1$m0id),
99+
mc0_by_acid = get_query_data(lvl = 0, fld = "acid", val = acid),
100+
mc1_by_m1id = get_query_data(lvl = 1, fld = "m1id", val = l3_sample1$m1id),
101+
mc1_by_acid = get_query_data(lvl = 1, fld = "acid", val = acid),
102+
mc2_by_m2id = get_query_data(lvl = 2, fld = "m2id", val = l3_sample1$m2id),
103+
mc2_by_acid = get_query_data(lvl = 2, fld = "acid", val = acid),
104+
mc3_by_m3id = get_query_data(lvl = 3, fld = "m3id", val = l3_sample1$m3id),
105+
mc3_by_aeid = get_query_data(lvl = 3, fld = "aeid", val = aeid),
106+
mc4_by_m4id = get_query_data(lvl = 4, fld = "m4id", val = l5_sample1$m4id),
107+
mc4_by_aeid = get_query_data(lvl = 4, fld = "aeid", val = aeid, add.fld = FALSE),
108+
mc5_by_m5id = get_query_data(lvl = 5, fld = "m5id", val = l5_sample1$m5id),
109+
mc5_by_aeid = get_query_data(lvl = 5, fld = "aeid", val = aeid, add.fld = FALSE),
110+
mc6_by_m6id = get_query_data(lvl = 6, fld = "m6id", val = l6_sample1$m6id),
111+
mc6_by_aeid = get_query_data(lvl = 6, fld = "aeid", val = aeid),
112+
mc7_by_m7id = get_query_data(lvl = 7, fld = "m7id", val = l7_sample1$m7id),
113+
mc7_by_aeid = get_query_data(lvl = 7, fld = "aeid", val = aeid),
114+
mcagg_by_aeid = get_query_data(lvl = "agg", fld = "aeid", val = aeid),
115+
plot_single_m4id = get_query_data(fld = "m4id",
116+
val = l5_sample1$m4id,
117+
func = "tcplPlot"),
118+
plot_multiple_m4id = get_query_data(fld = "m4id",
119+
val = list(l5_sample2$m4id),
120+
func = "tcplPlot"),
121+
plot_single_aeid = get_query_data(fld = "aeid",
122+
val = aeid,
123+
func = "tcplPlot"),
124+
plot_multiple_aeid = get_query_data(fld = "aeid",
125+
val = list(c(aeid, compare.aeid)),
126+
func = "tcplPlot"),
127+
plot_single_spid = get_query_data(fld = c("spid", "aeid"),
128+
val = list(l5_sample1$spid, aeid),
129+
func = "tcplPlot"),
130+
plot_multiple_spid = get_query_data(fld = c("spid", "aeid"),
131+
val = list(l5_sample2$spid, aeid),
132+
func = "tcplPlot"),
133+
plot_single_m4id_compare = get_query_data(fld = "m4id",
134+
val = l5_sample1$m4id,
135+
compare.val = compare.l5_sample1$m4id,
136+
func = "tcplPlot"),
137+
plot_multiple_m4id_compare = get_query_data(fld = "m4id",
138+
val = list(l5_sample2$m4id),
139+
compare.val = list(compare.l5_sample2$m4id),
140+
func = "tcplPlot"),
141+
plot_single_aeid_compare = get_query_data(fld = "aeid",
142+
val = aeid,
143+
compare.val = compare.aeid,
144+
func = "tcplPlot"),
145+
plot_multiple_aeid_compare = get_query_data(fld = "aeid",
146+
val = list(c(aeid, compare.aeid)),
147+
compare.val = list(c(compare.aeid, aeid)),
148+
func = "tcplPlot"),
149+
plot_single_spid_compare = get_query_data(fld = c("spid", "aeid"),
150+
val = list(l5_sample1$spid, aeid),
151+
compare.val = list(compare.l5_sample1$spid, compare.aeid),
152+
func = "tcplPlot"),
153+
plot_multiple_spid_compare = get_query_data(fld = c("spid", "aeid"),
154+
val = list(l5_sample2$spid, aeid),
155+
compare.val = list(compare.l5_sample2$spid, compare.aeid),
156+
func = "tcplPlot")
157+
)
158+
#---------------------------#
159+
## save the data
160+
usethis::use_data(mc_test, overwrite = TRUE)
161+
#---------------------------#

0 commit comments

Comments
 (0)