Merge pull request #34 from Sage-Bionetworks/develop

Develop

Author: Kenneth Daily

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: synapseusagereports
 Title: Generate a Reports of Synapse Project Usage
-Version: 0.3
+Version: 0.3.1
 Authors@R: person("Kenneth", "Daily", email = "kenneth.daily@sagebase.org", role = c("aut", "cre"))
 Description: Generate a RMarkdown report of Synapse Project usage, including users, page views and downloads.
 Depends: R (>= 3.3.1)

NAMESPACE

@@ -1,7 +1,5 @@
 # Generated by roxygen2: do not edit by hand
 
-export(aclToMemberList)
-export(aclToUserList)
 export(countByDay)
 export(countByMonth)
 export(doQuery)
@@ -13,7 +11,6 @@ export(makeDateBreaks)
 export(makeDateBreaksStartEnd)
 export(multiMonthVisits)
 export(plotByDay)
-export(processAclUserList)
 export(processQuery)
 export(processTeamMemberList)
 export(render_report)

R/lib.R

@@ -13,13 +13,13 @@ mytheme <- ggplot2::theme_bw() + ggplot2::theme(axis.text=ggplot2::element_text(
 #   q.browse <- sprintf(template, projectId, beginTimestamp, endTimestamp)
 
 #' @export
-render_report <- function(project_id, acl_team_order, data_file, reportType="report") {
+render_report <- function(project_id, team_order, data_file, reportType="report") {
 
   templates <- c("report"=system.file("templates", "report.Rmd",
                                       package = "synapseusagereports"))
 
   myParams <- list(projectId=project_id,
-                   aclTeamOrder=acl_team_order,
+                   teamOrder=team_order,
                    queryDataFile=data_file)
 
   htmlFileName <- paste0(myParams[['projectId']], "_", reportType, "_",
@@ -146,56 +146,6 @@ getTeamMemberDF <- function(teamId) {
 
 }
 
-#' @export
-aclToMemberList <- function(acl) {
-  aclMemberList <- plyr::ldply(acl$resourceAccess,
-                               function(x) data.frame(principalId=as.character(x$principalId),
-                                                      teamId=acl$id))
-
-  accessUsers <- plyr::llply(chunk(aclMemberList$principalId, 50),
-                             function(x) synapser::synRestGET(sprintf("/userGroupHeaders/batch?ids=%s",
-                                                                      paste(x, collapse=",")))$children)
-
-  userGroupHeaders <- do.call(c, accessUsers)
-
-  plyr::ldply(userGroupHeaders, as.data.frame)
-
-}
-
-#' @export
-aclToUserList <- function(acl) {
-  aclMemberList <- aclToMemberList(acl)
-  aclMemberList$teamId <- acl$id
-
-  userList <- plyr::ldply(aclMemberList$ownerId, getTeamMemberDF)
-
-  userList2 <- aclMemberList %>%
-    dplyr::filter(isIndividual) %>%
-    dplyr::rename(userId=ownerId)
-
-  rbind(userList2[, c("userId", "teamId")], userList)
-
-}
-
-#' @export
-processAclUserList <- function(projectId, aclTeamOrder) {
-  # Get users at project level and select the team
-  # they are on dependent on the ordering in aclTeamOrder
-  acl <- synapser::synRestGET(sprintf('/entity/%s/acl', paste0("syn", projectId)))
-  aclUserList <- aclToUserList(acl)
-  aclUserList$teamId <- factor(aclUserList$teamId,
-                               levels=aclTeamOrder,
-                               ordered=TRUE)
-
-  aclUserList <- aclUserList %>%
-    dplyr::group_by(userId) %>%
-    dplyr::arrange(teamId) %>%
-    dplyr::slice(1) %>%
-    dplyr::ungroup()
-
-  aclUserList
-}
-
 #' @export
 processTeamMemberList <- function(teamIds) {
   # Get users from provided teamIds
@@ -218,7 +168,7 @@ processTeamMemberList <- function(teamIds) {
 chunk <- function(d, n) split(d, ceiling(seq_along(d)/n))
 
 #' @export
-getQueryUserProfiles <- function(queryData, useTeamGrouping, aclUserList) {
+getQueryUserProfiles <- function(queryData, useTeamGrouping, userList) {
   # Get user profile info for users in data download records
 
 
@@ -233,7 +183,7 @@ getQueryUserProfiles <- function(queryData, useTeamGrouping, aclUserList) {
     dplyr::select(userId, userName)
 
   if (useTeamGrouping) {
-    allUsers <- dplyr::left_join(allUsersList, aclUserList)
+    allUsers <- dplyr::left_join(allUsersList, userList)
   } else{
     allUsers <- allUsersList
     allUsers$teamId <- "Registered Synapse User"

README.md

@@ -8,3 +8,20 @@ Generate HTML usage reports for Synapse projects.
 devtools::install_github("Sage-Bionetworks/synapseusagereports")
 ```
 
+## Usage
+
+Scripts are in the [inst/scripts](inst/scripts) directory.
+
+See [example-db-config.yml](example-db-config.yml) for an example configuration file to connect to the warehouse database.
+
+### Query the warehouse
+
+```
+report_data_query.R --project_id=syn1234567 --start_date=2018-01-01 --end_date=2018-03-01 --config_file=example-db-config.yml > output.csv
+```
+
+### Generate a report
+
+```
+render_report.R  --project_id=syn1234567 --team_order='273957' output.csv
+```

data/test_query_data.RData

@@ -0,0 +1,5 @@
+host: mysql.hostname.com
+port: '3306'
+db: warehouse
+username: yourusername
+password: yourpassword

example-db-config.yml

@@ -0,0 +1,48 @@
+userId,id,DATE,TIMESTAMP,NODE_TYPE,NAME,recordType,date,dateGrouping,monthYear
+3323072,11417482,2018-08-17,1534538158006,file,A,pageview,2018-08-17,2018-08-01,Aug 2018
+3323072,11417482,2018-08-17,1534538160976,file,A,pageview,2018-08-17,2018-08-01,Aug 2018
+3323072,11417482,2018-08-17,1534538171612,file,A,pageview,2018-08-17,2018-08-01,Aug 2018
+3323072,10242922,2018-08-30,1535663050650,table,T,pageview,2018-08-30,2018-08-01,Aug 2018
+3323072,10242922,2018-08-30,1535663052000,table,T,pageview,2018-08-30,2018-08-01,Aug 2018
+3323072,10242922,2018-08-30,1535663085899,table,T,pageview,2018-08-30,2018-08-01,Aug 2018
+3323072,10242922,2018-08-30,1535672186254,table,T,pageview,2018-08-30,2018-08-01,Aug 2018
+3323072,10242922,2018-08-30,1535672187051,table,T,pageview,2018-08-30,2018-08-01,Aug 2018
+3323072,10242922,2018-08-30,1535672202787,table,T,pageview,2018-08-30,2018-08-01,Aug 2018
+3323072,10242922,2018-08-31,1535731808005,table,T,pageview,2018-08-31,2018-08-01,Aug 2018
+3323072,12083426,2018-08-08,1533686511458,file,N,pageview,2018-08-08,2018-08-01,Aug 2018
+3323072,12083426,2018-08-08,1533755485501,file,N,pageview,2018-08-08,2018-08-01,Aug 2018
+3323072,12083426,2018-08-10,1533942892281,file,N,pageview,2018-08-10,2018-08-01,Aug 2018
+3323072,12083426,2018-08-13,1534200198589,file,N,pageview,2018-08-13,2018-08-01,Aug 2018
+3323072,12083426,2018-08-14,1534264981565,file,N,pageview,2018-08-14,2018-08-01,Aug 2018
+3323072,12083426,2018-08-17,1534538144361,file,N,pageview,2018-08-17,2018-08-01,Aug 2018
+3323072,12083426,2018-08-17,1534539168935,file,N,pageview,2018-08-17,2018-08-01,Aug 2018
+3323072,12083426,2018-08-17,1534542678643,file,N,pageview,2018-08-17,2018-08-01,Aug 2018
+3323072,12083426,2018-08-17,1534542767694,file,N,pageview,2018-08-17,2018-08-01,Aug 2018
+3323072,12083426,2018-08-17,1534542775674,file,N,pageview,2018-08-17,2018-08-01,Aug 2018
+3323072,12083426,2018-08-22,1534976335572,file,N,pageview,2018-08-22,2018-08-01,Aug 2018
+3323072,12083426,2018-08-22,1534976355920,file,N,pageview,2018-08-22,2018-08-01,Aug 2018
+3323072,12083426,2018-08-24,1535132491648,file,N,pageview,2018-08-24,2018-08-01,Aug 2018
+3323072,12083426,2018-08-24,1535147256497,file,N,pageview,2018-08-24,2018-08-01,Aug 2018
+3335327,11417482,2018-08-17,1534538186435,file,P,download,2018-08-17,2018-08-01,Aug 2018
+3335327,11417482,2018-08-17,1534538311833,file,P,download,2018-08-17,2018-08-01,Aug 2018
+3335327,11417482,2018-08-17,1534538320137,file,P,download,2018-08-17,2018-08-01,Aug 2018
+3335327,12179981,2018-07-10,1531259582326,file,H,download,2018-07-10,2018-07-01,Jul 2018
+3323072,12179981,2018-07-10,1531259582326,file,H,download,2018-07-10,2018-07-01,Jul 2018
+3323072,11363590,2018-07-11,1531337575668,file,P,download,2018-07-11,2018-07-01,Jul 2018
+3323072,11363590,2018-07-13,1531499135882,file,P,download,2018-07-13,2018-07-01,Jul 2018
+3323072,11363590,2018-07-13,1531499136133,file,P,download,2018-07-13,2018-07-01,Jul 2018
+3323072,11363590,2018-07-18,1531938199869,file,P,download,2018-07-18,2018-07-01,Jul 2018
+3335327,11363975,2018-07-10,1531180905867,file,P,download,2018-07-10,2018-07-01,Jul 2018
+3335327,11363975,2018-07-20,1532110034215,file,P,download,2018-07-20,2018-07-01,Jul 2018
+3335327,11363975,2018-07-20,1532111346920,file,P,download,2018-07-20,2018-07-01,Jul 2018
+3323072,8474877,2018-07-03,1530659245321,folder,O,pageview,2018-07-03,2018-07-01,Jul 2018
+3323072,8474877,2018-07-04,1530662923446,folder,O,pageview,2018-07-04,2018-07-01,Jul 2018
+3323072,8474877,2018-07-05,1530808584022,folder,O,pageview,2018-07-05,2018-07-01,Jul 2018
+3323072,8474877,2018-07-06,1530899426741,folder,O,pageview,2018-07-06,2018-07-01,Jul 2018
+3323072,8474877,2018-07-06,1530899468952,folder,O,pageview,2018-07-06,2018-07-01,Jul 2018
+273950,11185668,2018-07-05,1530808541212,folder,E,pageview,2018-07-05,2018-07-01,Jul 2018
+273950,11185668,2018-07-06,1530906236092,folder,E,pageview,2018-07-06,2018-07-01,Jul 2018
+273950,11363590,2018-07-01,1530432183518,file,E,pageview,2018-07-01,2018-07-01,Jul 2018
+273950,11363590,2018-07-01,1530432195463,file,E,pageview,2018-07-01,2018-07-01,Jul 2018
+273950,11363590,2018-07-01,1530437240350,file,E,pageview,2018-07-01,2018-07-01,Jul 2018
+273950,11363590,2018-07-01,1530470603919,file,E,pageview,2018-07-01,2018-07-01,Jul 2018

inst/config.R

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 project_id="syn2580853"
-acl_team_order='3377637,3346847'
+team_order='273957,3377637,3346847,3320424'
 
 # 6 months ago
 start_date=$(date --date='-6 months' -I -u)
@@ -10,4 +10,4 @@ start_date=$(date --date='-6 months' -I -u)
 end_date=`date -u -I`
 
 ./report_data_query.R --project_id=${project_id} --start_date=${start_date} --end_date=${end_date} --config_file=~/datawarehouse-before-aug.yml > /tmp/${project_id}_${end_date}.csv
-./render_report.R  --project_id=${project_id} --acl_team_order=${acl_team_order} /tmp/${project_id}_${end_date}.csv
+./render_report.R  --project_id=${project_id} --team_order=${team_order} /tmp/${project_id}_${end_date}.csv