CORE-282 top groups (#1641)

Author: dvoet

src/main/resources/swagger/api-docs.yaml

@@ -155,10 +155,17 @@ paths:
       parameters:
         - name: email
           in: path
-          description: Email address of user to check
+          description: Email address (or user id) of user to check
           required: true
           schema:
             type: string
+        - name: groupsContributingToMostMembershipsLimit
+          in: query
+          description: The number of groups to return that contribute to the most google group memberships. Default is 0.
+          schema:
+            type: integer
+            minimum: 1
+            maximum: 100
       responses:
         200:
           description: user summary information
@@ -3427,6 +3434,14 @@ paths:
         - Users
       summary: Gets a combined SamUser, attributes, allowances, terms of service, enterprise features state of the user, used to reduce api calls.
       operationId: getSamUserCombinedState
+      parameters:
+        - name: groupsContributingToMostMembershipsLimit
+          in: query
+          description: The number of groups to return that contribute to the most google group memberships. Default is 0.
+          schema:
+            type: integer
+            minimum: 1
+            maximum: 100
       responses:
         200:
           description: user exists
@@ -4992,6 +5007,20 @@ components:
             additional details about the user, such as enterprise features they have access to example: '{ "enterpriseFeatures": ["feature1", "feature2"] }'
         groupMemberhipCounts:
           $ref: '#/components/schemas/GroupMembershipCounts'
+        groupsContributingToMostMemberships:
+          type: array
+          items:
+            $ref: '#/components/schemas/GroupMembershipCount'
+    GroupMembershipCount:
+      type: object
+      properties:
+        group:
+          description: unlikely to be used. Name of an internal sam group, not a managed-group which is what most users usually belong to, those will show up in policies
+          type: string
+        policy:
+          $ref: '#/components/schemas/PolicyIdentifiers'
+        attributedMembershipCount:
+          type: integer
     SamUserRegistrationRequest:
       type: object
       properties:

src/main/scala/org/broadinstitute/dsde/workbench/sam/api/AdminRoutes.scala

@@ -53,8 +53,10 @@ trait AdminRoutes extends SecurityDirectives with SamRequestContextDirectives wi
           requireAdminResourceAction(SamResourceActions.adminReadSummaryInformation, userType, samUser, samRequestContext) {
             val workbenchEmail = WorkbenchEmail(email)
             getWithTelemetry(samRequestContext, emailParam(workbenchEmail)) {
-              complete {
-                userService.getSamUserCombinedState(workbenchEmail, samRequestContext, resourceService)
+              parameters("groupsContributingToMostMembershipsLimit".as[Int].?) { topGroupsLimit =>
+                complete {
+                  userService.getSamUserCombinedState(workbenchEmail, topGroupsLimit.getOrElse(0), samRequestContext, resourceService)
+                }
               }
             }
           }

src/main/scala/org/broadinstitute/dsde/workbench/sam/api/UserRoutesV2.scala

@@ -80,8 +80,10 @@ trait UserRoutesV2 extends SamUserDirectives with SamRequestContextDirectives wi
                 val samRequestContext = samRequestContextWithoutUser.copy(samUser = Some(samUser))
                 pathEndOrSingleSlash {
                   get {
-                    complete {
-                      userService.getSamUserCombinedState(samUser, samRequestContext, resourceService)
+                    parameters("groupsContributingToMostMembershipsLimit".as[Int].?) { topGroupsLimit =>
+                      complete {
+                        userService.getSamUserCombinedState(samUser, topGroupsLimit.getOrElse(0), samRequestContext, resourceService)
+                      }
                     }
                   }
                 }

src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/DirectoryDAO.scala

@@ -207,4 +207,15 @@ trait DirectoryDAO {
       resourceTypeName: ResourceTypeName,
       samRequestContext: SamRequestContext
   ): IO[Set[FullyQualifiedResourceId]]
+
+  /** List the top groups contributing to the most memberships for a user.
+    * @param samUser
+    *   the user to list groups for
+    * @param limit
+    *   the maximum number of groups to return
+    * @param samRequestContext
+    * @return
+    *   a map of group to membership count
+    */
+  def listGroupsContributingToMostMemberships(samUser: SamUser, limit: Int, samRequestContext: SamRequestContext): IO[Map[WorkbenchGroupIdentity, Int]]
 }

src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAO.scala

@@ -732,6 +732,48 @@ class PostgresDirectoryDAO(protected val writeDbRef: DbReference, protected val
     }
   }
 
+  override def listGroupsContributingToMostMemberships(
+      samUser: SamUser,
+      limit: Int,
+      samRequestContext: SamRequestContext
+  ): IO[Map[WorkbenchGroupIdentity, Int]] = if (limit <= 0) {
+    IO.pure(Map.empty)
+  } else {
+    readOnlyTransaction("listGroupsContributingToMostMemberships", samRequestContext) { implicit session =>
+      val f = GroupMemberFlatTable.syntax("f")
+      val g = GroupTable.syntax("g")
+      val p = PolicyTable.syntax("p")
+      val r = ResourceTable.syntax("r")
+      val rt = ResourceTypeTable.syntax("rt")
+
+      val query = samsql"""with group_counts as (
+                            select ${f.lastGroupMembershipElement} as group_id, count(${f.groupId}) AS membership_count
+                            from ${GroupMemberFlatTable as f}
+                            join ${GroupTable as g} on ${f.groupId} = ${g.id}
+                            where ${g.synchronizedDate} is not null
+                            and ${f.memberUserId} = ${samUser.id}
+                            group by ${f.lastGroupMembershipElement}
+                            order by membership_count desc
+                            limit $limit
+                            )
+                            select gc.membership_count, ${g.result.name}, ${p.result.name}, ${r.result.name}, ${rt.result.name}
+                            from group_counts gc
+                            join ${GroupTable as g} on gc.group_id = ${g.id}
+                            left join ${PolicyTable as p} on ${p.groupId} = ${g.id}
+                            left join ${ResourceTable as r} on ${p.resourceId} = ${r.id}
+                            left join ${ResourceTypeTable as rt} on ${r.resourceTypeId} = ${rt.id}
+                            """
+
+      query
+        .map { rs =>
+          resultSetToGroupIdentity(rs, g, p, r, rt) -> rs.int("membership_count")
+        }
+        .list()
+        .apply()
+        .toMap
+    }
+  }
+
   override def countDirectSynchronizedGroupMemberships(samUser: SamUser, samRequestContext: SamRequestContext): IO[Int] =
     readOnlyTransaction("countDirectSynchronizedGroupMemberships", samRequestContext) { implicit session =>
       val query = samsql"""select count(distinct g.id) directMembershipCount

src/main/scala/org/broadinstitute/dsde/workbench/sam/model/api/SamUserCombinedStateResponse.scala

@@ -1,6 +1,8 @@
 package org.broadinstitute.dsde.workbench.sam.model.api
 
-import org.broadinstitute.dsde.workbench.sam.model.{FullyQualifiedResourceId, TermsOfServiceDetails}
+import org.broadinstitute.dsde.workbench.model.WorkbenchIdentityJsonSupport.WorkbenchGroupNameFormat
+import org.broadinstitute.dsde.workbench.model.{WorkbenchGroupIdentity, WorkbenchGroupName}
+import org.broadinstitute.dsde.workbench.sam.model.{FullyQualifiedPolicyId, FullyQualifiedResourceId, PolicyIdentifiers, TermsOfServiceDetails}
 import spray.json.DefaultJsonProtocol._
 import spray.json._
 import org.broadinstitute.dsde.workbench.sam.model.api.GroupMembershipCounts.GroupMembershipCountsFormat
@@ -9,14 +11,35 @@ import org.broadinstitute.dsde.workbench.sam.model.api.SamUserAttributes.SamUser
 import org.broadinstitute.dsde.workbench.sam.model.api.SamJsonSupport._
 
 object SamUserCombinedStateResponse {
-  implicit val SamUserResponseFormat: RootJsonFormat[SamUserCombinedStateResponse] = jsonFormat7(SamUserCombinedStateResponse.apply)
+  implicit val GroupMembershipCountFormat: RootJsonFormat[GroupMembershipCount] = jsonFormat3(GroupMembershipCount.apply)
+  implicit val SamUserResponseFormat: RootJsonFormat[SamUserCombinedStateResponse] = jsonFormat8(SamUserCombinedStateResponse.apply)
 }
+
+final case class GroupMembershipCount(
+    group: Option[WorkbenchGroupName],
+    policy: Option[PolicyIdentifiers],
+    attributedMembershipCount: Int
+)
+object GroupMembershipCount {
+  def apply(group: WorkbenchGroupIdentity, attributedMembershipCount: Int): GroupMembershipCount =
+    group match {
+      case groupName: WorkbenchGroupName => GroupMembershipCount(Option(groupName), None, attributedMembershipCount)
+      case policyId: FullyQualifiedPolicyId =>
+        GroupMembershipCount(
+          None,
+          Option(PolicyIdentifiers(policyId.accessPolicyName, policyId.resource.resourceTypeName, policyId.resource.resourceId)),
+          attributedMembershipCount
+        )
+    }
+}
+
 final case class SamUserCombinedStateResponse(
     samUser: SamUser,
     allowances: SamUserAllowances,
     attributes: Option[SamUserAttributes],
     termsOfServiceDetails: TermsOfServiceDetails,
     groupMembershipCounts: GroupMembershipCounts,
     additionalDetails: Map[String, JsValue],
-    favoriteResources: Set[FullyQualifiedResourceId]
+    favoriteResources: Set[FullyQualifiedResourceId],
+    groupsContributingToMostMemberships: Option[List[GroupMembershipCount]]
 )

src/main/scala/org/broadinstitute/dsde/workbench/sam/service/UserService.scala

@@ -2,6 +2,7 @@ package org.broadinstitute.dsde.workbench.sam
 package service
 
 import akka.http.scaladsl.model.StatusCodes
+import cats.data.OptionT
 import cats.effect.IO
 import com.typesafe.scalalogging.LazyLogging
 import org.apache.commons.codec.binary.Hex
@@ -519,24 +520,35 @@ class UserService(
   def countIndirectPublicGroupMemberships(samUser: SamUser, samRequestContext: SamRequestContext): IO[Int] =
     directoryDAO.countIndirectPublicGroupMemberships(samUser, samRequestContext)
 
+  def listGroupsContributingToMostMemberships(samUser: SamUser, limit: Int, samRequestContext: SamRequestContext): IO[Map[WorkbenchGroupIdentity, Int]] =
+    directoryDAO.listGroupsContributingToMostMemberships(samUser, limit, samRequestContext)
+
   def getSamUserCombinedState(
       workbenchEmail: WorkbenchEmail,
+      topGroupsLimit: Int,
       samRequestContext: SamRequestContext,
       resourceService: ResourceService
   ): IO[SamUserCombinedStateResponse] =
     for {
-      samUserOption <- getUserFromEmail(workbenchEmail, samRequestContext)
-      samUser = samUserOption.getOrElse(throw new WorkbenchException("user not found"))
-      combinedState <- getSamUserCombinedState(samUser, samRequestContext, resourceService)
+      samUser <- OptionT(getUserFromEmail(workbenchEmail, samRequestContext))
+        .orElseF(directoryDAO.loadUser(WorkbenchUserId(workbenchEmail.value), samRequestContext))
+        .getOrRaise(new WorkbenchExceptionWithErrorReport(ErrorReport(StatusCodes.NotFound, "user not found")))
+      combinedState <- getSamUserCombinedState(samUser, topGroupsLimit, samRequestContext, resourceService)
     } yield combinedState
 
-  def getSamUserCombinedState(samUser: SamUser, samRequestContext: SamRequestContext, resourceService: ResourceService): IO[SamUserCombinedStateResponse] =
+  def getSamUserCombinedState(
+      samUser: SamUser,
+      topGroupsLimit: Int,
+      samRequestContext: SamRequestContext,
+      resourceService: ResourceService
+  ): IO[SamUserCombinedStateResponse] =
     for {
       allowances <- getUserAllowances(samUser, samRequestContext)
       maybeAttributes <- getUserAttributes(samUser.id, samRequestContext)
       directGroupMemberships <- countDirectSynchronizedGroupMemberships(samUser, samRequestContext)
       indirectGroupMemberships <- countIndirectSynchronizedGroupMemberships(samUser, samRequestContext)
       indirectPublicMemberships <- countIndirectPublicGroupMemberships(samUser, samRequestContext)
+      topGroups <- listGroupsContributingToMostMemberships(samUser, topGroupsLimit, samRequestContext)
       termsOfServiceDetails <- tosService.getTermsOfServiceDetailsForUser(samUser.id, samRequestContext)
       enterpriseFeatures <- resourceService
         .listResourcesFlat(
@@ -560,7 +572,11 @@ class UserService(
         indirectPublic = indirectPublicMemberships
       ),
       Map("enterpriseFeatures" -> enterpriseFeatures.toJson),
-      favoriteResources
+      favoriteResources,
+      topGroups.map { case (group, count) => GroupMembershipCount(group, count) }.toList match {
+        case Nil => None
+        case list => Option(list)
+      }
     )
 
 }

src/test/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/MockDirectoryDAO.scala

@@ -520,4 +520,10 @@ class MockDirectoryDAO(val groups: mutable.Map[WorkbenchGroupIdentity, Workbench
       samRequestContext: SamRequestContext
   ): IO[Set[FullyQualifiedResourceId]] =
     IO.pure(userFavoriteResources.getOrElse(userId, Set.empty).filter(_.resourceTypeName == resourceTypeName))
+
+  override def listGroupsContributingToMostMemberships(
+      samUser: SamUser,
+      limit: Int,
+      samRequestContext: SamRequestContext
+  ): IO[Map[WorkbenchGroupIdentity, Int]] = ???
 }

src/test/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAOSpec.scala

@@ -13,8 +13,9 @@ import org.broadinstitute.dsde.workbench.sam.db.tables.TosTable
 import org.broadinstitute.dsde.workbench.sam.matchers.TimeMatchers
 import org.broadinstitute.dsde.workbench.sam.model._
 import org.broadinstitute.dsde.workbench.sam.model.api.{AdminUpdateUserRequest, SamUser, SamUserAttributes}
-import org.broadinstitute.dsde.workbench.sam.{Generator, RetryableAnyFreeSpec, TestSupport}
+import org.broadinstitute.dsde.workbench.sam.{Generator, TestSupport}
 import org.scalatest.Inside.inside
+import org.scalatest.freespec.AnyFreeSpec
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.{BeforeAndAfterEach, OptionValues}
 
@@ -23,7 +24,7 @@ import java.time.temporal.ChronoUnit
 import java.util.{Date, UUID}
 import scala.concurrent.duration._
 
-class PostgresDirectoryDAOSpec extends RetryableAnyFreeSpec with Matchers with BeforeAndAfterEach with TimeMatchers with OptionValues {
+class PostgresDirectoryDAOSpec extends AnyFreeSpec with Matchers with BeforeAndAfterEach with TimeMatchers with OptionValues {
   val dao = new PostgresDirectoryDAO(TestSupport.dbRef, TestSupport.dbRef)
   val policyDAO = new PostgresAccessPolicyDAO(TestSupport.dbRef, TestSupport.dbRef)
   val azureManagedResourceGroupDAO = new PostgresAzureManagedResourceGroupDAO(TestSupport.dbRef, TestSupport.dbRef)
@@ -2222,5 +2223,85 @@ class PostgresDirectoryDAOSpec extends RetryableAnyFreeSpec with Matchers with B
         loadedFavoriteResources should contain theSameElementsAs Set(otherResource.fullyQualifiedId)
       }
     }
+
+    "listGroupsContributingToMostMemberships" - {
+      "list groups contributing to most memberships" in {
+        assume(databaseEnabled, databaseEnabledClue)
+        val otherUser = Generator.genWorkbenchUserBoth.sample.get
+        dao.createUser(defaultUser, samRequestContext).unsafeRunSync()
+        dao.createUser(otherUser, samRequestContext).unsafeRunSync()
+
+        /*
+        create `drop` + `limit` groups each containing defaultUser, these are leaf groups
+        for each leaf group, create `multiplier` * `index` parent groups, these are parent groups
+        the result of the list call using `limit` should exclude the lowest `drop` indexed groups
+        the counts for the remaining groups should be 1 (for the leaf group) + `index` * `multiplier` (for the parent groups)
+         */
+        val multiplier = 5
+        val limit = 3
+        val drop = 3
+        val testGroups = for {
+          index <- 1 to limit + drop
+        } yield {
+          val leafUuid = UUID.randomUUID().toString
+          val leafGroup = BasicWorkbenchGroup(WorkbenchGroupName(leafUuid), Set(defaultUser.id), WorkbenchEmail(leafUuid))
+          dao.createGroup(leafGroup, samRequestContext = samRequestContext).unsafeRunSync()
+          dao.updateSynchronizedDateAndVersion(leafGroup, samRequestContext).unsafeRunSync()
+          for (_ <- 1 to multiplier * index) {
+            val parentUuid = UUID.randomUUID().toString
+            val parentGroup = BasicWorkbenchGroup(WorkbenchGroupName(parentUuid), Set(leafGroup.id), WorkbenchEmail(parentUuid))
+            dao.createGroup(parentGroup, samRequestContext = samRequestContext).unsafeRunSync()
+            dao.updateSynchronizedDateAndVersion(parentGroup, samRequestContext).unsafeRunSync()
+          }
+          (leafGroup, index)
+        }
+
+        dao.addGroupMember(testGroups.last._1.id, otherUser.id, samRequestContext).unsafeRunSync()
+
+        val result = dao.listGroupsContributingToMostMemberships(defaultUser, limit, samRequestContext).unsafeRunSync()
+        result should be(testGroups.drop(drop).map { case (group, index) => group.id -> (1 + index * multiplier) }.toMap)
+
+        // check that another user gives different results
+        val otherResult = dao.listGroupsContributingToMostMemberships(otherUser, limit, samRequestContext).unsafeRunSync()
+        otherResult should be(Map(testGroups.last._1.id -> (1 + (limit + drop) * multiplier)))
+      }
+
+      "list policies and groups" in {
+        assume(databaseEnabled, databaseEnabledClue)
+        dao.createUser(defaultUser, samRequestContext).unsafeRunSync()
+
+        val policy = defaultPolicy.copy(members = Set(defaultUser.id))
+        policyDAO.createResourceType(resourceType, samRequestContext).unsafeRunSync()
+        policyDAO.createResource(defaultResource, samRequestContext).unsafeRunSync()
+        policyDAO.createPolicy(policy, samRequestContext).unsafeRunSync()
+        dao.updateSynchronizedDateAndVersion(policy, samRequestContext).unsafeRunSync()
+
+        val groupUuid = UUID.randomUUID().toString
+        val group = BasicWorkbenchGroup(WorkbenchGroupName(groupUuid), Set(defaultUser.id), WorkbenchEmail(groupUuid))
+        dao.createGroup(group, samRequestContext = samRequestContext).unsafeRunSync()
+        dao.updateSynchronizedDateAndVersion(group, samRequestContext).unsafeRunSync()
+
+        val result = dao.listGroupsContributingToMostMemberships(defaultUser, 10, samRequestContext).unsafeRunSync()
+        result should be(Map(policy.id -> 1, group.id -> 1))
+      }
+
+      "ignores unsynchronized groups" in {
+        assume(databaseEnabled, databaseEnabledClue)
+        dao.createUser(defaultUser, samRequestContext).unsafeRunSync()
+
+        val leafUuid = UUID.randomUUID().toString
+        val leafGroup = BasicWorkbenchGroup(WorkbenchGroupName(leafUuid), Set(defaultUser.id), WorkbenchEmail(leafUuid))
+        dao.createGroup(leafGroup, samRequestContext = samRequestContext).unsafeRunSync()
+        dao.updateSynchronizedDateAndVersion(leafGroup, samRequestContext).unsafeRunSync()
+
+        val parentUuid = UUID.randomUUID().toString
+        val parentGroup = BasicWorkbenchGroup(WorkbenchGroupName(parentUuid), Set(leafGroup.id), WorkbenchEmail(parentUuid))
+        dao.createGroup(parentGroup, samRequestContext = samRequestContext).unsafeRunSync()
+        // not synchronized
+
+        val result = dao.listGroupsContributingToMostMemberships(defaultUser, 10, samRequestContext).unsafeRunSync()
+        result should be(Map(leafGroup.id -> 1))
+      }
+    }
   }
 }