Merge pull request #19 from Sage-Bionetworks/develop

Kenneth Daily · web-flow · commit bdbc81157333 · 2017-09-01T15:24:59.000-07:00
Develop
diff --git a/inst/lib.R b/inst/lib.R
@@ -11,11 +11,11 @@ mytheme <- ggplot2::theme_bw() + ggplot2::theme(axis.text=ggplot2::element_text(
 # doQuery <- function(con, template, projectId, beginTimestamp, endTimestamp) {
 #   q.browse <- sprintf(template, projectId, beginTimestamp, endTimestamp)
 
-doQuery <- function(con, template, projectId, date) {
+doQuery <- function(conn, template, projectId, date) {
   message(sprintf("%s", date))
   q.browse <- sprintf(template, date, date %m+% months(1))
 
-  DBI::dbGetQuery(conn = con, statement=q.browse)
+  DBI::dbGetQuery(conn = conn, statement=q.browse)
   
 }
 
@@ -40,27 +40,47 @@ processQuery <- function(data) {
   queryData
 }
 
-getData <- function(con, qTemplate, projectId, timestampBreaksDf) {
+createTempTable <- function(conn, projectId, parentIds=NULL) {
 
+  if (is.null(parentIds)) {
+    q.create_temp <- "CREATE TEMPORARY TABLE PROJECT_STATS SELECT ID, MAX(TIMESTAMP) AS TIMESTAMP FROM NODE_SNAPSHOT WHERE PROJECT_ID = %s GROUP BY ID;"
+    statement <- sprintf(q.create_temp, projectId)
+  } else {
+    parentIdsSQL <- sprintf("(%s)", paste(parentIds, collapse=","))
+    q.create_temp <- "CREATE TEMPORARY TABLE PROJECT_STATS SELECT ID, MAX(TIMESTAMP) AS TIMESTAMP FROM NODE_SNAPSHOT WHERE PROJECT_ID = %s AND PARENT_ID IN %s GROUP BY ID;"
+    statement <- sprintf(q.create_temp, projectId, parentIdsSQL)
+  }
+  
+  create <- DBI::dbSendQuery(conn=conn,
+                             statement=statement)
+}
+
+dropTempTable <- function(conn) {
+  DBI::dbSendQuery(conn=conn, statement='DROP TABLE PROJECT_STATS;')
+}
+
+getData <- function(conn, qTemplate, projectId, timestampBreaksDf, parentIds=NULL) {
   maxDate <- max(timestampBreaksDf$date)
   
-  q.create_temp <- "CREATE TEMPORARY TABLE PROJECT_STATS SELECT ID, MAX(TIMESTAMP) AS TIMESTAMP FROM NODE_SNAPSHOT WHERE PROJECT_ID = %s GROUP BY ID;"
-  create <- DBI::dbSendQuery(conn=con,
-                             statement=sprintf(q.create_temp, projectId))
+  # q.create_temp <- "CREATE TEMPORARY TABLE PROJECT_STATS SELECT ID, MAX(TIMESTAMP) AS TIMESTAMP FROM NODE_SNAPSHOT WHERE PROJECT_ID = %s GROUP BY ID;"
+  # create <- DBI::dbSendQuery(conn=con,
+  #                            statement=sprintf(q.create_temp, projectId))
+  create <- createTempTable(conn=conn, projectId=projectId, parentIds=parentIds)
+  res <- tryCatch(plyr::ddply(timestampBreaksDf, plyr::.(month, year),
+                              function (x) doQuery(conn=conn,
+                                                   template=qTemplate, 
+                                                   projectId=projectId, 
+                                                   date=x$date)),
+                  error=function(e) dropTempTable(conn=conn))
   
-  res <- plyr::ddply(timestampBreaksDf, plyr::.(month, year),
-                     function (x) doQuery(con=con,
-                                          template=qTemplate, 
-                                          projectId=projectId, 
-					  date=x$date))
+  dropTempTable(conn=conn)
   
-  foo <- DBI::dbSendQuery(conn=con, statement='DROP TABLE PROJECT_STATS;')
+  # foo <- DBI::dbSendQuery(conn=con, statement='DROP TABLE PROJECT_STATS;')
   
   res
 }
 
 getTeamMemberDF <- function(teamId) {
-  
   totalNumberOfResults <- 1000
   offset <- 0
   limit <- 50
@@ -307,4 +327,21 @@ topNEntities <- function(queryData, allUsers, topN=20) {
     dplyr::top_n(topN, n) %>% 
     dplyr::arrange(-n) %>% 
     dplyr::select(n)
-}
+}
+
+REPORT_TEMPLATES <- c("report"="../report.Rmd")
+
+generateAndStore <- function(myParams, reportType) {
+  htmlFileName <- paste0(myParams[['projectId']], "_", reportType, "_",
+                         lubridate::today(), ".html")
+  
+  outputFileName <- paste0(tempdir(), "/", htmlFileName)
+  
+  rmarkdown::render(input=REPORT_TEMPLATES[[reportType]],
+                    output_file=outputFileName,
+                    params=myParams)
+  
+  synStore(File(outputFileName,
+                name="Usage Statistics", 
+                parentId=parentId))
+}
diff --git a/inst/report.Rmd b/inst/report.Rmd
@@ -8,6 +8,7 @@ params:
   useTeamGrouping: !r TRUE
   aclTeamOrder: !r c('syn2775243')
   queryDataFile: !r NA
+  dataOutput: !r NA
 ---
 
 ```{r libs, echo=FALSE, warning=FALSE, message=FALSE}
@@ -29,11 +30,11 @@ knitr::opts_chunk$set(
   tidy = FALSE,
   fig.width=20)
 
-qPageviewTemplate <- 'select CLIENT,ENTITY_ID,CONVERT(AR.TIMESTAMP, CHAR) AS TIMESTAMP,DATE,USER_ID,NODE_TYPE,N.NAME from ACCESS_RECORD AR, PROCESSED_ACCESS_RECORD PAR, NODE_SNAPSHOT N, PROJECT_STATS NODE where AR.RESPONSE_STATUS=200 AND AR.TIMESTAMP > unix_timestamp("%s")*1000 AND AR.TIMESTAMP < unix_timestamp("%s")*1000 AND AR.SESSION_ID = PAR.SESSION_ID and AR.TIMESTAMP = PAR.TIMESTAMP and PAR.ENTITY_ID = NODE.ID AND N.ID = NODE.ID and N.TIMESTAMP = NODE.TIMESTAMP and CLIENT = "WEB" AND (PAR.NORMALIZED_METHOD_SIGNATURE IN ("GET /entity/#/bundle", "GET /entity/#/version/#/bundle", "GET /entity/#/wiki2", "GET /entity/#/wiki2/#"));'
-
-qDownloadTemplate <- 'select CLIENT,ENTITY_ID,CONVERT(AR.TIMESTAMP, CHAR) AS TIMESTAMP,DATE,USER_ID,NODE_TYPE,N.NAME from ACCESS_RECORD AR, PROCESSED_ACCESS_RECORD PAR, NODE_SNAPSHOT N, PROJECT_STATS NODE where AR.TIMESTAMP > unix_timestamp("%s")*1000 AND AR.TIMESTAMP < unix_timestamp("%s")*1000 and (AR.RESPONSE_STATUS IN (200, 307)) AND AR.SESSION_ID = PAR.SESSION_ID and AR.TIMESTAMP = PAR.TIMESTAMP and PAR.ENTITY_ID = NODE.ID and N.ID = NODE.ID AND N.TIMESTAMP = NODE.TIMESTAMP and (PAR.NORMALIZED_METHOD_SIGNATURE IN ("GET /entity/#/file", "GET /entity/#/version/#/file"));'
+qPageviewTemplate <- 'select ENTITY_ID,CONVERT(AR.TIMESTAMP, CHAR) AS TIMESTAMP,DATE,USER_ID,NODE_TYPE,N.NAME from ACCESS_RECORD AR, PROCESSED_ACCESS_RECORD PAR, NODE_SNAPSHOT N, PROJECT_STATS NODE where AR.RESPONSE_STATUS=200 AND AR.TIMESTAMP > unix_timestamp("%s")*1000 AND AR.TIMESTAMP < unix_timestamp("%s")*1000 AND AR.SESSION_ID = PAR.SESSION_ID and AR.TIMESTAMP = PAR.TIMESTAMP and PAR.ENTITY_ID = NODE.ID AND N.ID = NODE.ID and N.TIMESTAMP = NODE.TIMESTAMP and CLIENT = "WEB" AND (PAR.NORMALIZED_METHOD_SIGNATURE IN ("GET /entity/#/bundle", "GET /entity/#/version/#/bundle", "GET /entity/#/wiki2", "GET /entity/#/wiki2/#"));'
 
+qDownloadTemplate <- 'select ENTITY_ID,CONVERT(AR.TIMESTAMP, CHAR) AS TIMESTAMP,DATE,USER_ID,NODE_TYPE,N.NAME from ACCESS_RECORD AR, PROCESSED_ACCESS_RECORD PAR, NODE_SNAPSHOT N, PROJECT_STATS NODE where AR.TIMESTAMP > unix_timestamp("%s")*1000 AND AR.TIMESTAMP < unix_timestamp("%s")*1000 and (AR.RESPONSE_STATUS IN (200, 307)) AND AR.SESSION_ID = PAR.SESSION_ID and AR.TIMESTAMP = PAR.TIMESTAMP and PAR.ENTITY_ID = NODE.ID and N.ID = NODE.ID AND N.TIMESTAMP = NODE.TIMESTAMP and (PAR.NORMALIZED_METHOD_SIGNATURE IN ("GET /entity/#/file", "GET /entity/#/version/#/file"));'
 
+qFDRTemplate <- 'SELECT FDR.ASSOCIATION_OBJECT_ID AS ENTITY_ID, CONVERT(FDR.TIMESTAMP , CHAR) AS TIMESTAMP, DATE_FORMAT(from_unixtime(PROJECT_STATS.TIMESTAMP / 1000), "%%Y-%%m-%%e") AS DATE, FDR.USER_ID, N.NODE_TYPE, N.NAME FROM FILE_DOWNLOAD_RECORD FDR, NODE_SNAPSHOT N, PROJECT_STATS WHERE FDR.TIMESTAMP > unix_timestamp("%s")*1000 AND FDR.TIMESTAMP < unix_timestamp("%s")*1000 AND N.ID = PROJECT_STATS.ID AND PROJECT_STATS.ID = FDR.ASSOCIATION_OBJECT_ID AND FDR.ASSOCIATION_OBJECT_TYPE = "FileEntity" AND N.TIMESTAMP = PROJECT_STATS.TIMESTAMP;'
 ```
 
 ```{r config}
@@ -67,26 +68,35 @@ timestampBreaksDf <- makeDateBreaks(nMonths) %>% dplyr::arrange(date)
 ```
 
 ```{r domysqlquery}
-queryDataPageviews <- getData(con=con, qTemplate=qPageviewTemplate, 
+queryDataPageviews <- getData(conn=con, qTemplate=qPageviewTemplate, 
                               projectId=projectId,
                               timestampBreaksDf=timestampBreaksDf)
 
-
 queryDataPageviewsProcessed <- queryDataPageviews %>% 
   dplyr::mutate(recordType='pageview') %>% 
   processQuery()
 
-queryDataDownloads <- getData(con=con, qTemplate=qDownloadTemplate, 
+queryDataDownloads <- getData(conn=con, qTemplate=qDownloadTemplate, 
                               projectId=projectId,
                               timestampBreaksDf=timestampBreaksDf)
 
 queryDataDownloadsProcessed <- queryDataDownloads %>% 
   dplyr::mutate(recordType='download') %>% 
   processQuery()
 
-queryData <- rbind(queryDataPageviewsProcessed, queryDataDownloadsProcessed)
+queryDataFDR <- getData(conn=con, qTemplate=qFDRTemplate, 
+                        projectId=projectId,
+                        timestampBreaksDf=timestampBreaksDf)
+
+queryDataFDRProcessed <- queryDataFDR %>% 
+  dplyr::mutate(recordType='download') %>% 
+  processQuery()
 
+queryData <- rbind(queryDataPageviewsProcessed, queryDataDownloadsProcessed, queryDataFDRProcessed)
 
+if (!is.na(params$dataOutput)) {
+  write.csv(queryData, file=params$dataOutput, row.names = FALSE)
+}
 ```
 ### Activity on Synapse project `r proj@properties$name` (`r params$projectId`) from `r min(timestampBreaksDf$date)` to `r max(timestampBreaksDf$date)`.
 
diff --git a/inst/scripts/manual_AMPAD.R b/inst/scripts/manual_AMPAD.R
@@ -20,4 +20,5 @@ rmarkdown::render(input=templates[[reportType]],
                   params = myParams)
 
 htmlFile <- synStore(File(paste0("../", htmlFileName),
+                          name="Usage Statistics",
                           parentId=parentId))
diff --git a/inst/scripts/manual_CSBC_all.R b/inst/scripts/manual_CSBC_all.R
@@ -0,0 +1,31 @@
+library(synapseClient)
+synapseLogin()
+
+templates <- c("report"="../report.Rmd")
+reportType <- "report"
+
+res <- synTableQuery("select id from syn10142562")
+
+parentId <- 'syn10322050'
+
+for (id in res@values$id) {
+  
+  message(sprintf("Processing Project %s", id))
+  
+  projectId <- id
+  
+  myParams <- list(projectId=projectId, 
+                   nMonths=NA,
+                   aclTeamOrder=c(),
+                   useTeamGrouping=FALSE)
+  
+  htmlFileName <- paste0(myParams[['projectId']], "_", reportType, "_",
+                         lubridate::today(), ".html")
+  
+  rmarkdown::render(input=templates[[reportType]],
+                    output_file=htmlFileName,
+                    params = myParams)
+  
+  htmlFile <- synStore(File(paste0("../", htmlFileName),
+                            parentId=parentId))
+}
diff --git a/inst/scripts/manual_PCBC.R b/inst/scripts/manual_PCBC.R
@@ -1,9 +1,8 @@
 library(synapseClient)
 synapseLogin()
+source("../lib.R")
 
-templates <- c("report"="../report.Rmd")
 reportType <- "report"
-
 projectId <- 'syn1773109'
 parentId <- 'syn4892835'
 
@@ -12,12 +11,4 @@ myParams <- list(projectId=projectId,
                  aclTeamOrder=c(2224090, 3319054, 273957, projectId), 
                  useTeamGrouping=FALSE)
 
-htmlFileName <- paste0(myParams[['projectId']], "_", reportType, "_",
-                       lubridate::today(), ".html")
-
-rmarkdown::render(input=templates[[reportType]],
-                  output_file=htmlFileName,
-                  params = myParams)
-
-htmlFile <- synStore(File(paste0("../", htmlFileName), 
-                          parentId=parentId))
+generateAndStore(myParams, reportType)
