diff --git a/env/local.env b/env/local.env index d35ac44bb9..7d3b14e968 100644 --- a/env/local.env +++ b/env/local.env @@ -28,6 +28,7 @@ export LEONARDO_PET_SERVICE_ACCOUNT="leonardo-dev@broad-dsde-dev.iam.gserviceacc export OIDC_AUTHORITY_ENDPOINT="https://terradevb2c.b2clogin.com/terradevb2c.onmicrosoft.com/v2.0?p=b2c_1a_signup_signin_dev" export GOOGLE_TRACE_SAMPLING_PROBABILITY=0 export GOOGLE_TRACE_ENABLED="false" +export PET_SIGNING_ACCOUNTS_ENABLED="true" export POSTGRES_PASSWORD="sam-test" export POSTGRES_READ_URL="jdbc:postgresql://localhost:5432/testdb" export POSTGRES_WRITE_URL="jdbc:postgresql://localhost:5432/testdb" diff --git a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/MockTestSupport.scala b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/MockTestSupport.scala index 5e3d9e603b..e7ed9ae97a 100644 --- a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/MockTestSupport.scala +++ b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/MockTestSupport.scala @@ -158,7 +158,7 @@ object MockTestSupport extends MockTestSupport { policyDAO, googleExt, FakeOpenIDConnectConfiguration, - azureService:Option[AzureService] + azureService: Option[AzureService] ) } diff --git a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/provider/SamProviderSpec.scala b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/provider/SamProviderSpec.scala index e972c241c6..c239ddb6e0 100644 --- a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/provider/SamProviderSpec.scala +++ b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/provider/SamProviderSpec.scala @@ -12,7 +12,7 @@ import org.broadinstitute.dsde.workbench.sam.MockTestSupport.genSamRoutes import org.broadinstitute.dsde.workbench.sam.api.StandardSamUserDirectives import org.broadinstitute.dsde.workbench.sam.azure.AzureService import org.broadinstitute.dsde.workbench.sam.dataAccess.{AccessPolicyDAO, DirectoryDAO, StatefulMockAccessPolicyDaoBuilder} -import org.broadinstitute.dsde.workbench.sam.google.GoogleExtensions +import org.broadinstitute.dsde.workbench.sam.google.{GoogleExtensions, PetServiceAccounts} import org.broadinstitute.dsde.workbench.sam.model._ import org.broadinstitute.dsde.workbench.sam.model.api.SamUser import org.broadinstitute.dsde.workbench.sam.service._ @@ -95,6 +95,7 @@ class SamProviderSpec // The following services are mocked for now val googleExt: GoogleExtensions = mock[GoogleExtensions] + val mockPetServiceAccountExt = mock[PetServiceAccounts] val mockManagedGroupService: ManagedGroupService = mock[ManagedGroupService] val tosService: TosService = MockTosServiceBuilder().withAllAccepted().build val azureService: AzureService = mock[AzureService] @@ -135,11 +136,18 @@ class SamProviderSpec private def mockGetArbitraryPetServiceAccountToken(): IO[Unit] = for { _ <- IO( when { - googleExt.getArbitraryPetServiceAccountToken(any[SamUser], any[Set[String]], any[SamRequestContext]) + mockPetServiceAccountExt.getArbitraryPetServiceAccountToken(any[SamUser], any[Set[String]], any[SamRequestContext]) } thenReturn { Future.successful("aToken") } ) + _ <- IO( + when { + googleExt.petServiceAccounts + } thenReturn { + mockPetServiceAccountExt + } + ) } yield () private def mockResourceActionPermission(action: ResourceAction, hasPermission: Boolean): IO[Unit] = for { diff --git a/src/main/resources/org/broadinstitute/dsde/sam/liquibase/changelog.xml b/src/main/resources/org/broadinstitute/dsde/sam/liquibase/changelog.xml index 578a27770c..a2dfa4f572 100644 --- a/src/main/resources/org/broadinstitute/dsde/sam/liquibase/changelog.xml +++ b/src/main/resources/org/broadinstitute/dsde/sam/liquibase/changelog.xml @@ -31,4 +31,5 @@ + diff --git a/src/main/resources/org/broadinstitute/dsde/sam/liquibase/changesets/20240812_action_service_accounts.xml b/src/main/resources/org/broadinstitute/dsde/sam/liquibase/changesets/20240812_action_service_accounts.xml new file mode 100644 index 0000000000..6c4b9e5cb5 --- /dev/null +++ b/src/main/resources/org/broadinstitute/dsde/sam/liquibase/changesets/20240812_action_service_accounts.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/sam.conf b/src/main/resources/sam.conf index ee98fd0267..ba8b7cf142 100644 --- a/src/main/resources/sam.conf +++ b/src/main/resources/sam.conf @@ -117,6 +117,7 @@ googleServices { samplingProbability = ${?OPENCENSUS_SAMPLING_PROBABILITY} # for backwards compatibility samplingProbability = ${?GOOGLE_TRACE_SAMPLING_PROBABILITY} } + petSigningAccountsEnabled = ${?PET_SIGNING_ACCOUNTS_ENABLED} } db { diff --git a/src/main/resources/swagger/api-docs.yaml b/src/main/resources/swagger/api-docs.yaml index e7ef83d592..8ad348d928 100755 --- a/src/main/resources/swagger/api-docs.yaml +++ b/src/main/resources/swagger/api-docs.yaml @@ -1331,41 +1331,6 @@ paths: schema: $ref: '#/components/schemas/ErrorReport' x-codegen-request-body-name: scopes - /api/google/v1/user/petServiceAccount/{project}/signedUrlForBlob: - post: - tags: - - Google - summary: Gets a signed URL for the given blob, signed by the Pet Service account of the calling user. - The signed URL is active for 1 hour and scoped to the permissions of the signing Pet Service Account. - Sam will provide a signed URL for any object path, even if that object does not exist. - operationId: getSignedUrlForBlob - parameters: - - name: project - in: path - description: Google project of the pet - required: true - schema: - type: string - requestBody: - description: bucketName and blobName of the object to get a signed URL for - content: - 'application/json': - schema: - $ref: '#/components/schemas/SignedUrlRequest' - required: true - responses: - 200: - description: signed URL for the blob, signed by the Pet Service Account key - content: - application/json: - schema: - type: string - 500: - description: Internal Server Error - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorReport' /api/google/v1/user/proxyGroup/{email}: get: tags: @@ -1477,6 +1442,109 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorReport' + /api/google/v2/actionServiceAccount/{project}/{resourceTypeName}/{resourceId}/{action}: + post: + tags: + - Google + summary: Gets/creates an Action Service Account in a Google Project for an action on a resource + operationId: getActionServiceAccount + parameters: + - name: project + in: path + description: Google project of the the resource + required: true + schema: + type: string + - name: resourceTypeName + in: path + description: Type of resource + required: true + schema: + type: string + - name: resourceId + in: path + description: Id of resource + required: true + schema: + type: string + - name: action + in: path + description: Name of action to create an Action Service Account for + required: true + schema: + type: string + requestBody: + content: + 'application/json': + schema: + type: object + responses: + 200: + description: Successfully created Action Service Account + content: + text/plain: + schema: + type: string + 500: + description: Internal Server Error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReport' + /api/google/v2/actionServiceAccount/{project}/{resourceTypeName}/{resourceId}/{action}/signedUrlForBlob: + post: + tags: + - Google + summary: Gets a signed URL for the given blob, signed by the Action Service Account for the given + action, delegated to by the user's Pet Signing Account + The signed URL is active for 1 hour and scoped to the permissions of the Action Service Account. + Sam will provide a signed URL for any object path, even if that object does not exist. + operationId: getActionServiceAccountSignedUrlForBlob + parameters: + - name: project + in: path + description: project of the GCP Resource + required: true + schema: + type: string + - name: resourceTypeName + in: path + description: Resource type name + required: true + schema: + type: string + - name: resourceId + in: path + description: Resource ID + required: true + schema: + type: string + - name: action + in: path + description: Action on the Resource Type to use + required: true + schema: + type: string + requestBody: + description: gspath of the object to get a signed URL for + content: + 'application/json': + schema: + $ref: '#/components/schemas/RequesterPaysSignedUrlRequest' + required: true + responses: + 200: + description: signed URL for the blob, signed by the Pet Signing Account key via the Action Service Account + content: + application/json: + schema: + type: string + 500: + description: Internal Server Error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReport' /api/azure/v1/billingProfile/{billingProfileId}/managedResourceGroup: post: tags: @@ -3895,6 +3963,21 @@ components: requesterPaysProject: type: string description: Optional Google Project to use for billing. + ActionServiceAccountSignedUrlRequest: + type: object + required: + - gsPath + properties: + gsPath: + type: string + description: GS Path to the blob + duration: + type: number + description: Optional validity duration of the link in minutes. Defaults to 1 hour. + default: 60 + requesterPaysProject: + type: string + description: Optional Google Project to use for billing. CreateResourceRequest: required: - policies diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/config/GoogleServicesConfig.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/config/GoogleServicesConfig.scala index cdb0d5d50f..9a21d27337 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/sam/config/GoogleServicesConfig.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/config/GoogleServicesConfig.scala @@ -33,7 +33,8 @@ final case class GoogleServicesConfig( adminSdkServiceAccountPaths: Option[NonEmptyList[String]], googleKms: GoogleKmsConfig, terraGoogleOrgNumber: String, - traceExporter: TraceExporterConfig + traceExporter: TraceExporterConfig, + petSigningAccountsEnabled: Boolean ) object GoogleServicesConfig { @@ -98,7 +99,8 @@ object GoogleServicesConfig { config.as[Option[NonEmptyList[String]]]("adminSdkServiceAccountPaths"), config.as[GoogleKmsConfig]("kms"), config.getString("terraGoogleOrgNumber"), - config.as[TraceExporterConfig]("traceExporter") + config.as[TraceExporterConfig]("traceExporter"), + config.as[Option[Boolean]]("petSigningAccountsEnabled").getOrElse(false) ) } } diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/DirectoryDAO.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/DirectoryDAO.scala index 3d03f77dac..4cbf10fc7d 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/DirectoryDAO.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/DirectoryDAO.scala @@ -13,6 +13,8 @@ import org.broadinstitute.dsde.workbench.sam.azure.{ } import org.broadinstitute.dsde.workbench.sam.model.api.{AdminUpdateUserRequest, SamUser, SamUserAttributes} import org.broadinstitute.dsde.workbench.sam.model.{BasicWorkbenchGroup, FullyQualifiedResourceId, ResourceAction, SamUserTos} +import org.broadinstitute.dsde.workbench.sam.model.api.{ActionServiceAccount, ActionServiceAccountId} +import org.broadinstitute.dsde.workbench.sam.model.ResourceId import org.broadinstitute.dsde.workbench.sam.util.SamRequestContext import java.time.Instant @@ -116,6 +118,31 @@ trait DirectoryDAO { def updatePetServiceAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[PetServiceAccount] + def createActionServiceAccount(actionServiceAccount: ActionServiceAccount, samRequestContext: SamRequestContext): IO[ActionServiceAccount] + + def loadActionServiceAccount(actionServiceAccountId: ActionServiceAccountId, samRequestContext: SamRequestContext): IO[Option[ActionServiceAccount]] + + def updateActionServiceAccount(actionServiceAccount: ActionServiceAccount, samRequestContext: SamRequestContext): IO[ActionServiceAccount] + + def deleteActionServiceAccount(actionServiceAccountId: ActionServiceAccountId, samRequestContext: SamRequestContext): IO[Unit] + + def getAllActionServiceAccountsForResource( + resourceId: ResourceId, + samRequestContext: SamRequestContext + ): IO[Seq[ActionServiceAccount]] + + def deleteAllActionServiceAccountsForResource(resourceId: ResourceId, samRequestContext: SamRequestContext): IO[Unit] + + def createPetSigningAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[PetServiceAccount] + + def loadPetSigningAccount(petServiceAccountId: PetServiceAccountId, samRequestContext: SamRequestContext): IO[Option[PetServiceAccount]] + + def updatePetSigningAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[PetServiceAccount] + + def loadUserPetSigningAccount(userId: WorkbenchUserId, samRequestContext: SamRequestContext): IO[Option[PetServiceAccount]] + + def deletePetSigningAccount(petServiceAccountId: PetServiceAccountId, samRequestContext: SamRequestContext): IO[Unit] + def getManagedGroupAccessInstructions(groupName: WorkbenchGroupName, samRequestContext: SamRequestContext): IO[Option[String]] def setManagedGroupAccessInstructions(groupName: WorkbenchGroupName, accessInstructions: String, samRequestContext: SamRequestContext): IO[Unit] @@ -174,7 +201,6 @@ trait DirectoryDAO { def setUserRegisteredAt(userId: WorkbenchUserId, registeredAt: Instant, samRequestContext: SamRequestContext): IO[Unit] def getUserAttributes(userId: WorkbenchUserId, samRequestContext: SamRequestContext): IO[Option[SamUserAttributes]] - def setUserAttributes(samUserAttributes: SamUserAttributes, samRequestContext: SamRequestContext): IO[Unit] def listParentGroups(groupName: WorkbenchGroupName, samRequestContext: SamRequestContext): IO[Set[WorkbenchGroupName]] diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAO.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAO.scala index 140d2a501b..76923dddd2 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAO.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAO.scala @@ -4,7 +4,7 @@ import akka.http.scaladsl.model.StatusCodes import cats.effect.{IO, Temporal} import com.typesafe.scalalogging.LazyLogging import org.broadinstitute.dsde.workbench.model._ -import org.broadinstitute.dsde.workbench.model.google.{GoogleProject, ServiceAccount, ServiceAccountSubjectId} +import org.broadinstitute.dsde.workbench.model.google.{GoogleProject, ServiceAccount, ServiceAccountDisplayName, ServiceAccountSubjectId} import org.broadinstitute.dsde.workbench.sam._ import org.broadinstitute.dsde.workbench.sam.azure.{ ActionManagedIdentity, @@ -25,6 +25,7 @@ import org.broadinstitute.dsde.workbench.sam.db._ import org.broadinstitute.dsde.workbench.sam.db.tables._ import org.broadinstitute.dsde.workbench.sam.model._ import org.broadinstitute.dsde.workbench.sam.model.api.{AdminUpdateUserRequest, SamUser, SamUserAttributes} +import org.broadinstitute.dsde.workbench.sam.model.api.{ActionServiceAccount, ActionServiceAccountId} import org.broadinstitute.dsde.workbench.sam.util.{DatabaseSupport, SamRequestContext} import org.postgresql.util.PSQLException import scalikejdbc._ @@ -896,6 +897,209 @@ class PostgresDirectoryDAO(protected val writeDbRef: DbReference, protected val ServiceAccount(petRecord.googleSubjectId, petRecord.email, petRecord.displayName) ) + override def createActionServiceAccount(actionServiceAccount: ActionServiceAccount, samRequestContext: SamRequestContext): IO[ActionServiceAccount] = + serializableWriteTransaction("createActionServiceAccount", samRequestContext) { implicit session => + val actionServiceAccountColumn = ActionServiceAccountTable.column + + samsql"""insert into ${ActionServiceAccountTable.table} + ( + ${actionServiceAccountColumn.resourceId}, + ${actionServiceAccountColumn.resourceActionId}, + ${actionServiceAccountColumn.project}, + ${actionServiceAccountColumn.googleSubjectId}, + ${actionServiceAccountColumn.email}, + ${actionServiceAccountColumn.displayName} + ) + values ( + (select ${ResourceTable.column.id} from ${ResourceTable.table} where ${ResourceTable.column.name} = ${actionServiceAccount.id.resourceId}), + (select ${ResourceActionTable.column.id} from ${ResourceActionTable.table} where ${ResourceActionTable.column.action} = ${actionServiceAccount.id.action}), + ${actionServiceAccount.id.project}, + ${actionServiceAccount.serviceAccount.subjectId}, + ${actionServiceAccount.serviceAccount.email}, + ${actionServiceAccount.serviceAccount.displayName} + )""" + .update() + .apply() + actionServiceAccount + } + + type TableSyntax[A] = scalikejdbc.QuerySQLSyntaxProvider[scalikejdbc.SQLSyntaxSupport[A], A] + + override def loadActionServiceAccount( + actionServiceAccountId: ActionServiceAccountId, + samRequestContext: SamRequestContext + ): IO[Option[ActionServiceAccount]] = + readOnlyTransaction("loadActionServiceAccount", samRequestContext) { implicit session => + implicit val actionServiceAccountTable: TableSyntax[ActionServiceAccountRecord] = ActionServiceAccountTable.syntax + implicit val resourceActionTable: TableSyntax[ResourceActionRecord] = ResourceActionTable.syntax + implicit val resourceTable: TableSyntax[ResourceRecord] = ResourceTable.syntax + + val loadActionServiceAccountQuery = + samsql"""select ${resourceTable.result.name}, ${resourceActionTable.result.action}, ${actionServiceAccountTable.result.project}, ${actionServiceAccountTable.result.googleSubjectId}, ${actionServiceAccountTable.result.email}, ${actionServiceAccountTable.result.displayName} + from ${ActionServiceAccountTable as actionServiceAccountTable} + left join ${ResourceActionTable as resourceActionTable} + on ${actionServiceAccountTable.resourceActionId} = ${resourceActionTable.id} + left join ${ResourceTable as resourceTable} + on ${actionServiceAccountTable.resourceId} = ${resourceTable.id} + where ${resourceTable.name} = ${actionServiceAccountId.resourceId} + and ${actionServiceAccountTable.project} = ${actionServiceAccountId.project} + and ${resourceActionTable.action} = ${actionServiceAccountId.action}""" + + loadActionServiceAccountQuery.map(unmarshalActionServiceAccount).single().apply() + } + + override def updateActionServiceAccount(actionServiceAccount: ActionServiceAccount, samRequestContext: SamRequestContext): IO[ActionServiceAccount] = + serializableWriteTransaction("updateActionServiceAccount", samRequestContext) { implicit session => + val actionServiceAccountColumn = ActionServiceAccountTable.column + val updateAsaQuery = + samsql""" + update ${ActionServiceAccountTable.table} + set + ${actionServiceAccountColumn.googleSubjectId} = ${actionServiceAccount.serviceAccount.subjectId}, + ${actionServiceAccountColumn.email} = ${actionServiceAccount.serviceAccount.email}, + ${actionServiceAccountColumn.displayName} = ${actionServiceAccount.serviceAccount.displayName} + where + ${actionServiceAccountColumn.resourceId} = (select ${ResourceTable.column.id} from ${ResourceTable.table} where ${ResourceTable.column.name} = ${actionServiceAccount.id.resourceId}) + and ${actionServiceAccountColumn.resourceActionId} = (select ${ResourceActionTable.column.id} from ${ResourceActionTable.table} where ${ResourceActionTable.column.action} = ${actionServiceAccount.id.action}) + and ${actionServiceAccountColumn.project} = ${actionServiceAccountColumn.project}""" + val updated = updateAsaQuery.update().apply() + if (updated != 1) { + throw new WorkbenchException(s"Update cannot be applied because ${actionServiceAccount.id} does not exist") + } + + actionServiceAccount + } + + override def deleteActionServiceAccount(actionServiceAccountId: ActionServiceAccountId, samRequestContext: SamRequestContext): IO[Unit] = + serializableWriteTransaction("deleteActionServiceAccount", samRequestContext) { implicit session => + val actionServiceAccountTable = ActionServiceAccountTable.syntax + val deleteActionServiceAccountQuery = + samsql"""delete from ${ActionServiceAccountTable.table} + where ${actionServiceAccountTable.resourceId} = (select ${ResourceTable.column.id} from ${ResourceTable.table} where ${ResourceTable.column.name} = ${actionServiceAccountId.resourceId}) + and ${actionServiceAccountTable.project} = ${actionServiceAccountId.project} + and ${actionServiceAccountTable.resourceActionId} = (select ${ResourceActionTable.column.id} from ${ResourceActionTable.table} where ${ResourceActionTable.column.action} = ${actionServiceAccountId.action})""" + if (deleteActionServiceAccountQuery.update().apply() != 1) { + throw new WorkbenchException(s"${actionServiceAccountId} cannot be deleted because it already does not exist") + } + } + + override def getAllActionServiceAccountsForResource( + resourceId: ResourceId, + samRequestContext: SamRequestContext + ): IO[Seq[ActionServiceAccount]] = + readOnlyTransaction("loadActionServiceAccountsForResource", samRequestContext) { implicit session => + implicit val actionServiceAccountTable: TableSyntax[ActionServiceAccountRecord] = ActionServiceAccountTable.syntax + implicit val resourceActionTable: TableSyntax[ResourceActionRecord] = ResourceActionTable.syntax + implicit val resourceTable: TableSyntax[ResourceRecord] = ResourceTable.syntax + + val listActionServiceAccountsQuery = + samsql"""select ${resourceTable.result.name}, ${resourceActionTable.result.action}, ${actionServiceAccountTable.result.project}, ${actionServiceAccountTable.result.googleSubjectId}, ${actionServiceAccountTable.result.email}, ${actionServiceAccountTable.result.displayName} + from ${ActionServiceAccountTable as actionServiceAccountTable} + left join ${ResourceActionTable as resourceActionTable} + on ${actionServiceAccountTable.resourceActionId} = ${resourceActionTable.id} + left join ${ResourceTable as resourceTable} + on ${actionServiceAccountTable.resourceId} = ${resourceTable.id} + where ${resourceTable.name} = ${resourceId}""" + + listActionServiceAccountsQuery.map(unmarshalActionServiceAccount).list().apply() + } + + override def deleteAllActionServiceAccountsForResource(resourceId: ResourceId, samRequestContext: SamRequestContext): IO[Unit] = + serializableWriteTransaction("deleteAllActionServiceAccountsForResource", samRequestContext) { implicit session => + val actionServiceAccountTable = ActionServiceAccountTable.syntax + val deleteActionServiceAccountQuery = + samsql"""delete from ${ActionServiceAccountTable.table} + where ${actionServiceAccountTable.resourceId} = (select ${ResourceTable.column.id} from ${ResourceTable.table} where ${ResourceTable.column.name} = ${resourceId})""" + deleteActionServiceAccountQuery.update().apply() + } + + private def unmarshalActionServiceAccount(rs: WrappedResultSet)(implicit + resourceTable: TableSyntax[ResourceRecord], + resourceActionTable: TableSyntax[ResourceActionRecord], + actionServiceAccountTable: TableSyntax[ActionServiceAccountRecord] + ) = + ActionServiceAccount( + ActionServiceAccountId( + rs.get[ResourceId](resourceTable.resultName.name), + rs.get[ResourceAction](resourceActionTable.resultName.action), + rs.get[GoogleProject](actionServiceAccountTable.resultName.project) + ), + ServiceAccount( + rs.get[ServiceAccountSubjectId](actionServiceAccountTable.resultName.googleSubjectId), + rs.get[WorkbenchEmail](actionServiceAccountTable.resultName.email), + rs.get[ServiceAccountDisplayName](actionServiceAccountTable.resultName.displayName) + ) + ) + + override def createPetSigningAccount(petSigningAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[PetServiceAccount] = + serializableWriteTransaction("createPetSigningAccount", samRequestContext) { implicit session => + val petSigningAccountColumn = PetSigningAccountTable.column + + samsql"""insert into ${PetSigningAccountTable.table} (${petSigningAccountColumn.samUserId}, ${petSigningAccountColumn.project}, ${petSigningAccountColumn.googleSubjectId}, ${petSigningAccountColumn.email}, ${petSigningAccountColumn.displayName}) + values (${petSigningAccount.id.userId}, ${petSigningAccount.id.project}, ${petSigningAccount.serviceAccount.subjectId}, ${petSigningAccount.serviceAccount.email}, ${petSigningAccount.serviceAccount.displayName})""" + .update() + .apply() + petSigningAccount + } + + override def loadPetSigningAccount(petSigningAccountId: PetServiceAccountId, samRequestContext: SamRequestContext): IO[Option[PetServiceAccount]] = + readOnlyTransaction("loadPetSigningAccount", samRequestContext) { implicit session => + val petSigningAccountTable = PetSigningAccountTable.syntax + + val loadPetQuery = + samsql"""select ${petSigningAccountTable.resultAll} + from ${PetSigningAccountTable as petSigningAccountTable} + where ${petSigningAccountTable.samUserId} = ${petSigningAccountId.userId} and ${petSigningAccountTable.project} = ${petSigningAccountId.project}""" + + val petRecordOpt = loadPetQuery.map(PetSigningAccountTable(petSigningAccountTable)).single().apply() + petRecordOpt.map(unmarshalPetSigningAccountRecord) + } + + def loadUserPetSigningAccount(userId: WorkbenchUserId, samRequestContext: SamRequestContext): IO[Option[PetServiceAccount]] = + readOnlyTransaction("loadUserPetSigningAccount", samRequestContext) { implicit session => + val petSigningAccountTable = PetSigningAccountTable.syntax + + val loadPetQuery = + samsql"""select ${petSigningAccountTable.resultAll} + from ${PetSigningAccountTable as petSigningAccountTable} + where ${petSigningAccountTable.samUserId} = ${userId}""" + + val petRecordOpt = loadPetQuery.map(PetSigningAccountTable(petSigningAccountTable)).single().apply() + petRecordOpt.map(unmarshalPetSigningAccountRecord) + } + + override def updatePetSigningAccount(petSigningAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[PetServiceAccount] = + serializableWriteTransaction("updatePetSigningAccount", samRequestContext) { implicit session => + val petSigningAccountColumn = PetSigningAccountTable.column + val updatePetQuery = + samsql"""update ${PetServiceAccountTable.table} set + ${petSigningAccountColumn.googleSubjectId} = ${petSigningAccount.serviceAccount.subjectId}, + ${petSigningAccountColumn.email} = ${petSigningAccount.serviceAccount.email}, + ${petSigningAccountColumn.displayName} = ${petSigningAccount.serviceAccount.displayName} + where ${petSigningAccountColumn.samUserId} = ${petSigningAccount.id.userId} and ${petSigningAccountColumn.project} = ${petSigningAccount.id.project}""" + + if (updatePetQuery.update().apply() != 1) { + throw new WorkbenchException(s"Update cannot be applied because ${petSigningAccount.id} does not exist") + } + + petSigningAccount + } + + override def deletePetSigningAccount(petSigningAccountId: PetServiceAccountId, samRequestContext: SamRequestContext): IO[Unit] = + serializableWriteTransaction("deletePetSigningAccount", samRequestContext) { implicit session => + val petSigningAccountTable = PetSigningAccountTable.syntax + val deletePetQuery = + samsql"delete from ${PetSigningAccountTable.table} where ${petSigningAccountTable.samUserId} = ${petSigningAccountId.userId} and ${petSigningAccountTable.project} = ${petSigningAccountId.project}" + if (deletePetQuery.update().apply() != 1) { + throw new WorkbenchException(s"${petSigningAccountId} cannot be deleted because it already does not exist") + } + } + + private def unmarshalPetSigningAccountRecord(petRecord: PetSigningAccountRecord): PetServiceAccount = + PetServiceAccount( + PetServiceAccountId(petRecord.samUserId, petRecord.project), + ServiceAccount(petRecord.googleSubjectId, petRecord.email, petRecord.displayName) + ) case class SubjectConglomerate( userId: Option[WorkbenchUserId], groupName: Option[WorkbenchGroupName], @@ -1069,8 +1273,6 @@ class PostgresDirectoryDAO(protected val writeDbRef: DbReference, protected val actionManagedIdentity } - type TableSyntax[A] = scalikejdbc.QuerySQLSyntaxProvider[scalikejdbc.SQLSyntaxSupport[A], A] - override def loadActionManagedIdentity( actionManagedIdentityId: ActionManagedIdentityId, samRequestContext: SamRequestContext diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/db/tables/ActionServiceAccountTable.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/db/tables/ActionServiceAccountTable.scala new file mode 100644 index 0000000000..f96be7dcbe --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/db/tables/ActionServiceAccountTable.scala @@ -0,0 +1,31 @@ +package org.broadinstitute.dsde.workbench.sam.db.tables + +import org.broadinstitute.dsde.workbench.model.google.{GoogleProject, ServiceAccountDisplayName, ServiceAccountSubjectId} +import org.broadinstitute.dsde.workbench.model.WorkbenchEmail +import org.broadinstitute.dsde.workbench.sam.db.SamTypeBinders +import scalikejdbc._ + +final case class ActionServiceAccountRecord( + resourceId: ResourcePK, + resourceActionId: ResourceActionPK, + project: GoogleProject, + googleSubjectId: ServiceAccountSubjectId, + email: WorkbenchEmail, + displayName: ServiceAccountDisplayName +) + +object ActionServiceAccountTable extends SQLSyntaxSupportWithDefaultSamDB[ActionServiceAccountRecord] { + override def tableName: String = "SAM_ACTION_SERVICE_ACCOUNT" + + import SamTypeBinders._ + def apply(e: ResultName[ActionServiceAccountRecord])(rs: WrappedResultSet): ActionServiceAccountRecord = ActionServiceAccountRecord( + rs.get(e.resourceId), + rs.get(e.resourceActionId), + rs.get(e.project), + rs.get(e.googleSubjectId), + rs.get(e.email), + rs.get(e.displayName) + ) + + def apply(p: SyntaxProvider[ActionServiceAccountRecord])(rs: WrappedResultSet): ActionServiceAccountRecord = apply(p.resultName)(rs) +} diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/db/tables/PetSigningAccountTable.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/db/tables/PetSigningAccountTable.scala new file mode 100644 index 0000000000..bb2a83303d --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/db/tables/PetSigningAccountTable.scala @@ -0,0 +1,29 @@ +package org.broadinstitute.dsde.workbench.sam.db.tables + +import org.broadinstitute.dsde.workbench.model.google.{GoogleProject, ServiceAccountDisplayName, ServiceAccountSubjectId} +import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchUserId} +import org.broadinstitute.dsde.workbench.sam.db.SamTypeBinders +import scalikejdbc._ + +final case class PetSigningAccountRecord( + samUserId: WorkbenchUserId, + project: GoogleProject, + googleSubjectId: ServiceAccountSubjectId, + email: WorkbenchEmail, + displayName: ServiceAccountDisplayName +) + +object PetSigningAccountTable extends SQLSyntaxSupportWithDefaultSamDB[PetSigningAccountRecord] { + override def tableName: String = "SAM_PET_SIGNING_ACCOUNT" + + import SamTypeBinders._ + def apply(e: ResultName[PetSigningAccountRecord])(rs: WrappedResultSet): PetSigningAccountRecord = PetSigningAccountRecord( + rs.get(e.samUserId), + rs.get(e.project), + rs.get(e.googleSubjectId), + rs.get(e.email), + rs.get(e.displayName) + ) + + def apply(p: SyntaxProvider[PetSigningAccountRecord])(rs: WrappedResultSet): PetSigningAccountRecord = apply(p.resultName)(rs) +} diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/ActionServiceAccounts.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/ActionServiceAccounts.scala new file mode 100644 index 0000000000..7dca4b6daf --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/ActionServiceAccounts.scala @@ -0,0 +1,127 @@ +package org.broadinstitute.dsde.workbench.sam.google + +import akka.actor.ActorSystem +import cats.effect.IO +import cats.implicits.{catsSyntaxApplicativeId, catsSyntaxTuple2Parallel, toTraverseOps} +import org.broadinstitute.dsde.workbench.google.{GoogleIamDAO, GoogleProjectDAO} +import org.broadinstitute.dsde.workbench.model.google._ +import org.broadinstitute.dsde.workbench.sam.config.GoogleServicesConfig +import org.broadinstitute.dsde.workbench.sam.dataAccess.{DirectoryDAO, LockDetails, PostgresDistributedLockDAO} +import org.broadinstitute.dsde.workbench.sam.model.{FullyQualifiedResourceId, ResourceAction, ResourceId} +import org.broadinstitute.dsde.workbench.sam.model.api.{ActionServiceAccount, ActionServiceAccountId} +import org.broadinstitute.dsde.workbench.sam.util.SamRequestContext + +import scala.concurrent.duration._ +import scala.concurrent.ExecutionContext + +class ActionServiceAccounts( + distributedLock: PostgresDistributedLockDAO[IO], + googleIamDAO: GoogleIamDAO, + googleProjectDAO: GoogleProjectDAO, + directoryDAO: DirectoryDAO, + googleServicesConfig: GoogleServicesConfig +)(implicit override val system: ActorSystem, executionContext: ExecutionContext) + extends ServiceAccounts(googleProjectDAO, googleServicesConfig) { + + private[google] def createActionServiceAccount( + resource: FullyQualifiedResourceId, + project: GoogleProject, + action: ResourceAction, + samRequestContext: SamRequestContext + ): IO[ActionServiceAccount] = { + // The normal situation is that the pet either exists in both the database and google or neither. + // Sometimes, especially in tests, the asa may be removed from the database, but not google or the other way around. + // This code is a little extra complicated to detect the cases when a asa does not exist in google, the database or + // both and do the right thing. + val (asaName, asaDisplayName) = actionServiceAccountName(resource, action) + val createAsa = for { + (maybeAsa, maybeServiceAccount) <- retrieveAsaAndSa(resource, project, action, samRequestContext) + serviceAccount <- maybeServiceAccount match { + // SA does not exist in google, create it and add it to the proxy group + case None => + for { + _ <- assertProjectInTerraOrg(project) + sa <- IO.fromFuture(IO(googleIamDAO.createServiceAccount(project, asaName, asaDisplayName))) + } yield sa + // SA already exists in google, use it + case Some(sa) => IO.pure(sa) + } + pet <- (maybeAsa, maybeServiceAccount) match { + // pet does not exist in the database, create it and enable the identity + case (None, _) => + for { + p <- directoryDAO.createActionServiceAccount( + ActionServiceAccount(ActionServiceAccountId(resource.resourceId, action, project), serviceAccount), + samRequestContext + ) + } yield p + // pet already exists in the database, but a new SA was created so update the database with new SA info + case (Some(p), None) => + for { + p <- directoryDAO.updateActionServiceAccount(p.copy(serviceAccount = serviceAccount), samRequestContext) + } yield p + + // everything already existed + case (Some(p), Some(_)) => IO.pure(p) + } + } yield pet + + val lock = LockDetails(s"${project.value}-createAsa", s"${resource.resourceId}-${action}", 30 seconds) + + for { + (asa, sa) <- retrieveAsaAndSa(resource, project, action, samRequestContext) // I'm loving better-monadic-for + shouldLock = !(asa.isDefined && sa.isDefined) // if either is not defined, we need to lock and potentially create them; else we return the pet + p <- if (shouldLock) distributedLock.withLock(lock).use(_ => createAsa) else asa.get.pure[IO] + } yield p + } + + private def actionServiceAccountName(resource: FullyQualifiedResourceId, action: ResourceAction) = { + /* + * Service account IDs must be: + * 1. between 6 and 30 characters + * 2. lower case alphanumeric separated by hyphens + * 3. must start with a lower case letter + * + * + * So, the Action name, truncated to 9 chars, prepended to the Resource Id truncated to 20 chars, with "-" is 30 chars. + */ + + val serviceAccountName = s"${action.value.take(9)}-${resource.resourceId.value.take(20)}" + val displayName = s"ASA [${action.value} on ${resource.resourceTypeName} ${resource.resourceId.value}]" + + // Display names have a max length of 100 characters + (ServiceAccountName(serviceAccountName), ServiceAccountDisplayName(displayName.take(100))) + } + + private def retrieveAsaAndSa( + resource: FullyQualifiedResourceId, + project: GoogleProject, + action: ResourceAction, + samRequestContext: SamRequestContext + ): IO[(Option[ActionServiceAccount], Option[ServiceAccount])] = { + val asaName = actionServiceAccountName(resource, action)._1 + val serviceAccount = IO.fromFuture(IO(googleIamDAO.findServiceAccount(project, asaName))) + val asaId = ActionServiceAccountId(resource.resourceId, action, project) + val asa = directoryDAO.loadActionServiceAccount(asaId, samRequestContext) + (asa, serviceAccount).parTupled + } + + private[google] def removeActionServiceAccount(actionServiceAccount: ActionServiceAccount, samRequestContext: SamRequestContext): IO[Unit] = + for { + // remove the record for the pet service account + _ <- directoryDAO.deleteActionServiceAccount(actionServiceAccount.id, samRequestContext) + // remove the service account itself in Google + _ <- IO.fromFuture(IO(googleIamDAO.removeServiceAccount(actionServiceAccount.id.project, toAccountName(actionServiceAccount.serviceAccount.email)))) + } yield () + + private[google] def forAllActionServiceAccounts[T](resourceId: ResourceId, samRequestContext: SamRequestContext)( + f: ActionServiceAccount => IO[T] + ): IO[Seq[T]] = + for { + actionServiceAccounts <- directoryDAO.getAllActionServiceAccountsForResource(resourceId, samRequestContext) + a <- actionServiceAccounts.traverse { asa => + f(asa) + } + } yield a + +} diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensionRoutes.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensionRoutes.scala index 641e73f6d4..cbaf27ffbd 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensionRoutes.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensionRoutes.scala @@ -20,6 +20,7 @@ import org.broadinstitute.dsde.workbench.sam.api.{ import org.broadinstitute.dsde.workbench.sam.model._ import org.broadinstitute.dsde.workbench.sam.model.api.SamJsonSupport._ import org.broadinstitute.dsde.workbench.sam.model.api.SamUser +import org.broadinstitute.dsde.workbench.sam.model.api.ActionServiceAccount._ import org.broadinstitute.dsde.workbench.sam.service.CloudExtensions import org.broadinstitute.dsde.workbench.sam.util.SamRequestContext import spray.json.DefaultJsonProtocol._ @@ -47,7 +48,7 @@ trait GoogleExtensionRoutes extends ExtensionRoutes with SamUserDirectives with getWithTelemetry(samRequestContext, "userEmail" -> email) { complete { import spray.json._ - googleExtensions.getArbitraryPetServiceAccountKey(email, samRequestContext) map { + googleExtensions.petServiceAccounts.getArbitraryPetServiceAccountKey(email, samRequestContext) map { // parse json to ensure it is json and tells akka http the right content-type case Some(key) => StatusCodes.OK -> key.parseJson case None => @@ -62,7 +63,7 @@ trait GoogleExtensionRoutes extends ExtensionRoutes with SamUserDirectives with getWithTelemetry(samRequestContext, "userEmail" -> email, "googleProject" -> googleProject) { complete { import spray.json._ - googleExtensions.getPetServiceAccountKey(email, googleProject, samRequestContext) map { + googleExtensions.petServiceAccounts.getPetServiceAccountKey(email, googleProject, samRequestContext) map { // parse json to ensure it is json and tells akka http the right content-type case Some(key) => StatusCodes.OK -> key.parseJson case None => @@ -80,7 +81,7 @@ trait GoogleExtensionRoutes extends ExtensionRoutes with SamUserDirectives with getWithTelemetry(samRequestContext) { complete { import spray.json._ - googleExtensions + googleExtensions.petServiceAccounts .getArbitraryPetServiceAccountKey(samUser, samRequestContext) .map(key => StatusCodes.OK -> key.parseJson) } @@ -92,7 +93,7 @@ trait GoogleExtensionRoutes extends ExtensionRoutes with SamUserDirectives with postWithTelemetry(samRequestContext) { entity(as[Set[String]]) { scopes => complete { - googleExtensions.getArbitraryPetServiceAccountToken(samUser, scopes, samRequestContext).map { token => + googleExtensions.petServiceAccounts.getArbitraryPetServiceAccountToken(samUser, scopes, samRequestContext).map { token => StatusCodes.OK -> JsString(token) } } @@ -113,7 +114,7 @@ trait GoogleExtensionRoutes extends ExtensionRoutes with SamUserDirectives with complete { import spray.json._ // parse json to ensure it is json and tells akka http the right content-type - googleExtensions + googleExtensions.petServiceAccounts .getPetServiceAccountKey(samUser, GoogleProject(project), samRequestContext) .map { key => StatusCodes.OK -> key.parseJson @@ -125,7 +126,7 @@ trait GoogleExtensionRoutes extends ExtensionRoutes with SamUserDirectives with val serviceAccountKeyId = ServiceAccountKeyId(keyId) deleteWithTelemetry(samRequestContext, "googleProject" -> projectResourceId, "keyId" -> serviceAccountKeyId) { complete { - googleExtensions + googleExtensions.petServiceAccounts .removePetServiceAccountKey(samUser.id, GoogleProject(project), serviceAccountKeyId, samRequestContext) .map(_ => StatusCodes.NoContent) } @@ -142,7 +143,7 @@ trait GoogleExtensionRoutes extends ExtensionRoutes with SamUserDirectives with postWithTelemetry(samRequestContext, "googleProject" -> projectResourceId) { entity(as[Set[String]]) { scopes => complete { - googleExtensions + googleExtensions.petServiceAccounts .getPetServiceAccountToken(samUser, GoogleProject(project), scopes, samRequestContext) .map { token => StatusCodes.OK -> JsString(token) @@ -189,7 +190,7 @@ trait GoogleExtensionRoutes extends ExtensionRoutes with SamUserDirectives with ) { getWithTelemetry(samRequestContext, "googleProject" -> projectResourceId) { complete { - googleExtensions.createUserPetServiceAccount(samUser, GoogleProject(project), samRequestContext).map { petSA => + googleExtensions.petServiceAccounts.createUserPetServiceAccount(samUser, GoogleProject(project), samRequestContext).map { petSA => StatusCodes.OK -> petSA.serviceAccount.email } } @@ -261,5 +262,55 @@ trait GoogleExtensionRoutes extends ExtensionRoutes with SamUserDirectives with } } } - } + } ~ + pathPrefix("google" / "v2") { + pathPrefix("actionServiceAccount") { + path(Segment / Segment / Segment / Segment) { (project, resourceTypeName, resourceId, action) => + val resource = FullyQualifiedResourceId(ResourceTypeName(resourceTypeName), ResourceId(resourceId)) + val googleProject = GoogleProject(project) + val resourceAction = ResourceAction(action) + + withNonAdminResourceType(resource.resourceTypeName) { resourceType => + if (!resourceType.actionPatterns.map(ap => ResourceAction(ap.value)).contains(resourceAction)) { + throw new WorkbenchExceptionWithErrorReport(ErrorReport(StatusCodes.NotFound, s"action $action not found")) + } + pathEndOrSingleSlash { + post { + complete { + googleExtensions.actionServiceAccounts.createActionServiceAccount(resource, googleProject, resourceAction, samRequestContext).map { + StatusCodes.OK -> _ + } + } + } + } ~ + pathPrefix("signedUrlForBlob") { + pathEndOrSingleSlash { + post { + requireAction(resource, resourceAction, samUser.id, samRequestContext) { + entity(as[RequesterPaysSignedUrlRequest]) { request => + complete { + googleExtensions + .getRequesterPaysSignedUrl( + samUser, + resource.resourceId, + resourceAction, + googleProject, + request.gsPath, + request.duration, + request.requesterPaysProject.map(GoogleProject), + samRequestContext + ) + .map { signedUrl => + StatusCodes.OK -> JsString(signedUrl.toString) + } + } + } + } + } + } + } + } + } + } + } } diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensions.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensions.scala index a90e54e4d6..ea888b05b1 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensions.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensions.scala @@ -2,16 +2,13 @@ package org.broadinstitute.dsde.workbench.sam.google import akka.actor.ActorSystem import akka.http.scaladsl.model.StatusCodes -import cats.effect.unsafe.implicits.global import cats.effect.{Clock, IO} import cats.implicits._ import com.google.api.client.googleapis.json.GoogleJsonResponseException -import com.google.api.client.http.HttpResponseException import com.google.api.gax.rpc.AlreadyExistsException import com.google.auth.oauth2.ServiceAccountCredentials import com.google.cloud.storage.BlobId import com.google.protobuf.{Duration, Timestamp} -import com.google.rpc.Code import com.typesafe.scalalogging.LazyLogging import net.logstash.logback.argument.StructuredArguments import org.broadinstitute.dsde.workbench.dataaccess.NotificationDAO @@ -24,10 +21,10 @@ import org.broadinstitute.dsde.workbench.model._ import org.broadinstitute.dsde.workbench.model.google._ import org.broadinstitute.dsde.workbench.sam._ import org.broadinstitute.dsde.workbench.sam.config.{GoogleServicesConfig, PetServiceAccountConfig} -import org.broadinstitute.dsde.workbench.sam.dataAccess.{AccessPolicyDAO, DirectoryDAO, LockDetails, PostgresDistributedLockDAO} +import org.broadinstitute.dsde.workbench.sam.dataAccess.{AccessPolicyDAO, DirectoryDAO, PostgresDistributedLockDAO} import org.broadinstitute.dsde.workbench.sam.model.api.SamJsonSupport._ import org.broadinstitute.dsde.workbench.sam.model._ -import org.broadinstitute.dsde.workbench.sam.model.api.SamUser +import org.broadinstitute.dsde.workbench.sam.model.api.{ActionServiceAccount, ActionServiceAccountId, SamUser} import org.broadinstitute.dsde.workbench.sam.service.UserService._ import org.broadinstitute.dsde.workbench.sam.service.{CloudExtensions, CloudExtensionsInitializer, ManagedGroupService, SamApplication} import org.broadinstitute.dsde.workbench.sam.util.SamRequestContext @@ -39,7 +36,6 @@ import java.io.ByteArrayInputStream import java.net.URL import java.util.Date import java.util.concurrent.TimeUnit -import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ @@ -68,15 +64,22 @@ class GoogleExtensions( val resourceTypes: Map[ResourceTypeName, ResourceType], val superAdminsGroup: WorkbenchEmail )(implicit val system: ActorSystem, executionContext: ExecutionContext, clock: Clock[IO]) - extends LazyLogging + extends ProxyEmailSupport(googleServicesConfig) + with LazyLogging with FutureSupport with CloudExtensions with Retry { + val petServiceAccounts: PetServiceAccounts = + new PetServiceAccounts(distributedLock, googleIamDAO, googleDirectoryDAO, googleProjectDAO, googleKeyCache, directoryDAO, googleServicesConfig) + val petSigningAccounts: PetSigningAccounts = + new PetSigningAccounts(distributedLock, googleIamDAO, googleDirectoryDAO, googleProjectDAO, googleKeyCache, directoryDAO, googleServicesConfig) + val actionServiceAccounts: ActionServiceAccounts = + new ActionServiceAccounts(distributedLock, googleIamDAO, googleProjectDAO, directoryDAO, googleServicesConfig) + private val maxGroupEmailLength = 64 - private[google] def toProxyFromUser(userId: WorkbenchUserId): WorkbenchEmail = - WorkbenchEmail(s"${googleServicesConfig.resourceNamePrefix.getOrElse("")}PROXY_${userId.value}@${googleServicesConfig.appsDomain}") + private val excludeFromPetSigningAccount: Set[WorkbenchEmail] = Set(googleServicesConfig.serviceAccountClientEmail) override val emailDomain = googleServicesConfig.appsDomain @@ -84,6 +87,10 @@ class GoogleExtensions( s"${googleServicesConfig.resourceNamePrefix.getOrElse("")}GROUP_${CloudExtensions.allUsersGroupName.value}@$emailDomain" ) + private[google] val allPetSigningAccountsGroupEmail = WorkbenchEmail( + s"${googleServicesConfig.resourceNamePrefix.getOrElse("")}GROUP_${CloudExtensions.allPetSingingAccountsGroupName.value}@$emailDomain" + ) + private val userProjectQueryParam = "userProject" private val requestedByQueryParam = "requestedBy" private val defaultSignedUrlDuration = 60L @@ -112,6 +119,31 @@ class GoogleExtensions( } yield allUsersGroup } + def getOrCreateAllPetSigningAccountsGroup(directoryDAO: DirectoryDAO, samRequestContext: SamRequestContext): IO[WorkbenchGroup] = { + val allPetSigningAccountsGroupStub = BasicWorkbenchGroup(CloudExtensions.allPetSingingAccountsGroupName, Set.empty, allPetSigningAccountsGroupEmail) + for { + existingGroup <- directoryDAO.loadGroup(allPetSigningAccountsGroupStub.id, samRequestContext = samRequestContext) + allPetSigningAccountsGroup <- existingGroup match { + case None => directoryDAO.createGroup(allPetSigningAccountsGroupStub, samRequestContext = samRequestContext) + case Some(group) => IO.pure(group) + } + existingGoogleGroup <- IO.fromFuture(IO(googleDirectoryDAO.getGoogleGroup(allPetSigningAccountsGroup.email))) + _ <- existingGoogleGroup match { + case None => + IO.fromFuture( + IO( + googleDirectoryDAO + .createGroup(allPetSigningAccountsGroup.id.toString, allPetSigningAccountsGroup.email, Option(googleDirectoryDAO.lockedDownGroupSettings)) + ) + ) recover { + case e: GoogleJsonResponseException if e.getDetails.getCode == StatusCodes.Conflict.intValue => () + } + case Some(_) => IO.unit + } + + } yield allPetSigningAccountsGroup + } + override def isWorkbenchAdmin(memberEmail: WorkbenchEmail): Future[Boolean] = googleDirectoryDAO.isGroupMember(WorkbenchEmail(s"fc-admins@${googleServicesConfig.appsDomain}"), memberEmail) recoverWith { case t => throw new WorkbenchException("Unable to query for admin status.", t) @@ -147,6 +179,8 @@ class GoogleExtensions( } } + _ <- getOrCreateAllPetSigningAccountsGroup(directoryDAO, samRequestContext) + _ <- googleKms.createKeyRing( googleServicesConfig.googleKms.project, googleServicesConfig.googleKms.location, @@ -290,7 +324,14 @@ class GoogleExtensions( } allUsersGroup <- getOrCreateAllUsersGroup(directoryDAO, samRequestContext) _ <- IO.fromFuture(IO(googleDirectoryDAO.addMemberToGroup(allUsersGroup.email, proxyEmail))) - + _ <- + if (excludeFromPetSigningAccount.contains(user.email) || !googleServicesConfig.petSigningAccountsEnabled) IO.none + else + for { + petSigningAccount <- petSigningAccounts.createPetSigningAccountForUser(user, samRequestContext) + allPetSigningAccountsGroup <- getOrCreateAllPetSigningAccountsGroup(directoryDAO, samRequestContext) + _ <- IO.fromFuture(IO(googleDirectoryDAO.addMemberToGroup(allPetSigningAccountsGroup.email, petSigningAccount.serviceAccount.email))) + } yield () } yield () } @@ -300,27 +341,21 @@ class GoogleExtensions( case None => IO.pure(false) } - /** Evaluate a future for each pet in parallel. - */ - private def forAllPets[T](userId: WorkbenchUserId, samRequestContext: SamRequestContext)(f: PetServiceAccount => IO[T]): IO[Seq[T]] = - for { - pets <- directoryDAO.getAllPetServiceAccountsForUser(userId, samRequestContext) - a <- pets.traverse { pet => - f(pet) - } - } yield a - override def onUserEnable(user: SamUser, samRequestContext: SamRequestContext): IO[Unit] = for { _ <- withProxyEmail(user.id) { proxyEmail => IO.fromFuture(IO(googleDirectoryDAO.addMemberToGroup(proxyEmail, WorkbenchEmail(user.email.value)))) } - _ <- forAllPets(user.id, samRequestContext)((petServiceAccount: PetServiceAccount) => enablePetServiceAccount(petServiceAccount, samRequestContext)) + _ <- petServiceAccounts.forAllPets(user.id, samRequestContext)((petServiceAccount: PetServiceAccount) => + petServiceAccounts.enablePetServiceAccount(petServiceAccount, samRequestContext) + ) } yield () override def onUserDisable(user: SamUser, samRequestContext: SamRequestContext): IO[Unit] = for { - _ <- forAllPets(user.id, samRequestContext)((petServiceAccount: PetServiceAccount) => disablePetServiceAccount(petServiceAccount, samRequestContext)) + _ <- petServiceAccounts.forAllPets(user.id, samRequestContext)((petServiceAccount: PetServiceAccount) => + petServiceAccounts.disablePetServiceAccount(petServiceAccount, samRequestContext) + ) _ <- withProxyEmail(user.id) { proxyEmail => IO.fromFuture(IO(googleDirectoryDAO.removeMemberFromGroup(proxyEmail, WorkbenchEmail(user.email.value)))) } @@ -328,233 +363,23 @@ class GoogleExtensions( override def onUserDelete(userId: WorkbenchUserId, samRequestContext: SamRequestContext): IO[Unit] = for { - _ <- forAllPets(userId, samRequestContext)((petServiceAccount: PetServiceAccount) => removePetServiceAccount(petServiceAccount, samRequestContext)) + _ <- petServiceAccounts.forAllPets(userId, samRequestContext)((petServiceAccount: PetServiceAccount) => + petServiceAccounts.removePetServiceAccount(petServiceAccount, samRequestContext) + ) + _ <- petSigningAccounts.removePetSigningAccount(userId, samRequestContext) _ <- withProxyEmail(userId)(email => IO.fromFuture(IO(googleDirectoryDAO.deleteGroup(email)))) } yield () override def onGroupDelete(groupEmail: WorkbenchEmail): IO[Unit] = IO.fromFuture(IO(googleDirectoryDAO.deleteGroup(groupEmail))) - def deleteUserPetServiceAccount(userId: WorkbenchUserId, project: GoogleProject, samRequestContext: SamRequestContext): IO[Boolean] = + override def onResourceDelete(resourceId: ResourceId, samRequestContext: SamRequestContext): IO[Unit] = for { - maybePet <- directoryDAO.loadPetServiceAccount(PetServiceAccountId(userId, project), samRequestContext) - deletedSomething <- maybePet match { - case Some(pet) => removePetServiceAccount(pet, samRequestContext).map(_ => true) - case None => IO.pure(false) // didn't find the pet, nothing to delete - } - } yield deletedSomething - - def createUserPetServiceAccount(user: SamUser, project: GoogleProject, samRequestContext: SamRequestContext): IO[PetServiceAccount] = { - val (petSaName, petSaDisplayName) = toPetSAFromUser(user) - // The normal situation is that the pet either exists in both the database and google or neither. - // Sometimes, especially in tests, the pet may be removed from the database, but not google or the other way around. - // This code is a little extra complicated to detect the cases when a pet does not exist in google, the database or - // both and do the right thing. - val createPet = for { - (maybePet, maybeServiceAccount) <- retrievePetAndSA(user.id, petSaName, project, samRequestContext) - serviceAccount <- maybeServiceAccount match { - // SA does not exist in google, create it and add it to the proxy group - case None => - for { - _ <- assertProjectInTerraOrg(project) - _ <- assertProjectIsActive(project) - sa <- IO.fromFuture(IO(googleIamDAO.createServiceAccount(project, petSaName, petSaDisplayName))) - _ <- withProxyEmail(user.id) { proxyEmail => - // Add group member by uniqueId instead of email to avoid race condition - // See: https://broadworkbench.atlassian.net/browse/CA-1005 - IO.fromFuture(IO(googleDirectoryDAO.addServiceAccountToGroup(proxyEmail, sa))) - } - _ <- IO.fromFuture(IO(googleIamDAO.addServiceAccountUserRoleForUser(project, sa.email, sa.email))) - } yield sa - // SA already exists in google, use it - case Some(sa) => IO.pure(sa) - } - pet <- (maybePet, maybeServiceAccount) match { - // pet does not exist in the database, create it and enable the identity - case (None, _) => - for { - p <- directoryDAO.createPetServiceAccount(PetServiceAccount(PetServiceAccountId(user.id, project), serviceAccount), samRequestContext) - _ <- directoryDAO.enableIdentity(p.id, samRequestContext) - } yield p - // pet already exists in the database, but a new SA was created so update the database with new SA info - case (Some(p), None) => - for { - p <- directoryDAO.updatePetServiceAccount(p.copy(serviceAccount = serviceAccount), samRequestContext) - } yield p - - // everything already existed - case (Some(p), Some(_)) => IO.pure(p) - } - } yield pet - - val lock = LockDetails(s"${project.value}-createPet", user.id.value, 30 seconds) - - for { - (pet, sa) <- retrievePetAndSA(user.id, petSaName, project, samRequestContext) // I'm loving better-monadic-for - shouldLock = !(pet.isDefined && sa.isDefined) // if either is not defined, we need to lock and potentially create them; else we return the pet - p <- if (shouldLock) distributedLock.withLock(lock).use(_ => createPet) else pet.get.pure[IO] - } yield p - } - - private def assertProjectInTerraOrg(project: GoogleProject): IO[Unit] = { - val validOrg = IO - .fromFuture(IO(googleProjectDAO.getAncestry(project.value).map { ancestry => - ancestry.exists { ancestor => - ancestor.getResourceId.getType == GoogleResourceTypes.Organization.value && ancestor.getResourceId.getId == googleServicesConfig.terraGoogleOrgNumber - } - })) - .recoverWith { - // if the getAncestry call results in a 403 error the project can't be in the right org - case e: HttpResponseException if e.getStatusCode == StatusCodes.Forbidden.intValue => - IO.raiseError( - new WorkbenchExceptionWithErrorReport( - ErrorReport(StatusCodes.BadRequest, s"Access denied from google accessing project ${project.value}, is it a Terra project?", e) - ) - ) - } - - validOrg.flatMap { - case true => IO.unit - case false => - IO.raiseError(new WorkbenchExceptionWithErrorReport(ErrorReport(StatusCodes.BadRequest, s"Project ${project.value} must be in Terra Organization"))) - } - } - - private def assertProjectIsActive(project: GoogleProject): IO[Unit] = - for { - projectIsActive <- IO.fromFuture(IO(googleProjectDAO.isProjectActive(project.value))) - _ <- IO.raiseUnless(projectIsActive)( - new WorkbenchExceptionWithErrorReport(ErrorReport(StatusCodes.BadRequest, s"Project ${project.value} is inactive")) + _ <- actionServiceAccounts.forAllActionServiceAccounts(resourceId, samRequestContext)((actionServiceAccount: ActionServiceAccount) => + actionServiceAccounts.removeActionServiceAccount(actionServiceAccount, samRequestContext) ) } yield () - private def retrievePetAndSA( - userId: WorkbenchUserId, - petServiceAccountName: ServiceAccountName, - project: GoogleProject, - samRequestContext: SamRequestContext - ): IO[(Option[PetServiceAccount], Option[ServiceAccount])] = { - val serviceAccount = IO.fromFuture(IO(googleIamDAO.findServiceAccount(project, petServiceAccountName))) - val pet = directoryDAO.loadPetServiceAccount(PetServiceAccountId(userId, project), samRequestContext) - (pet, serviceAccount).parTupled - } - - def getPetServiceAccountKey(userEmail: WorkbenchEmail, project: GoogleProject, samRequestContext: SamRequestContext): IO[Option[String]] = - for { - subject <- directoryDAO.loadSubjectFromEmail(userEmail, samRequestContext) - key <- subject match { - case Some(userId: WorkbenchUserId) => - getPetServiceAccountKey(SamUser(userId, None, userEmail, None, false), project, samRequestContext).map(Option(_)) - case _ => IO.pure(None) - } - } yield key - - def getPetServiceAccountKey(user: SamUser, project: GoogleProject, samRequestContext: SamRequestContext): IO[String] = - for { - pet <- createUserPetServiceAccount(user, project, samRequestContext) - key <- googleKeyCache.getKey(pet) - } yield key - - def getPetServiceAccountToken(user: SamUser, project: GoogleProject, scopes: Set[String], samRequestContext: SamRequestContext): Future[String] = - getPetServiceAccountKey(user, project, samRequestContext).unsafeToFuture().flatMap { key => - getAccessTokenUsingJson(key, scopes) - } - - def getArbitraryPetServiceAccountKey(userEmail: WorkbenchEmail, samRequestContext: SamRequestContext): IO[Option[String]] = - for { - subject <- directoryDAO.loadSubjectFromEmail(userEmail, samRequestContext) - key <- subject match { - case Some(userId: WorkbenchUserId) => - IO.fromFuture(IO(getArbitraryPetServiceAccountKey(SamUser(userId, None, userEmail, None, false), samRequestContext))).map(Option(_)) - case _ => IO.none - } - } yield key - - def getArbitraryPetServiceAccountKey(user: SamUser, samRequestContext: SamRequestContext): Future[String] = - getDefaultServiceAccountForShellProject(user, samRequestContext) - - def getArbitraryPetServiceAccountToken(user: SamUser, scopes: Set[String], samRequestContext: SamRequestContext): Future[String] = - getArbitraryPetServiceAccountKey(user, samRequestContext).flatMap { key => - getAccessTokenUsingJson(key, scopes) - } - - private def getDefaultServiceAccountForShellProject(user: SamUser, samRequestContext: SamRequestContext): Future[String] = { - val projectName = - s"fc-${googleServicesConfig.environment.substring(0, Math.min(googleServicesConfig.environment.length(), 5))}-${user.id.value}" // max 30 characters. subject ID is 21 - for { - creationOperationId <- googleProjectDAO - .createProject(projectName, googleServicesConfig.terraGoogleOrgNumber, GoogleResourceTypes.Organization) - .map(opId => Option(opId)) recover { - case gjre: GoogleJsonResponseException if gjre.getDetails.getCode == StatusCodes.Conflict.intValue => None - } - _ <- creationOperationId match { - case Some(opId) => pollShellProjectCreation(opId) // poll until it's created - case None => Future.successful(()) - } - key <- getPetServiceAccountKey(user, GoogleProject(projectName), samRequestContext).unsafeToFuture() - } yield key - } - - private def pollShellProjectCreation(operationId: String): Future[Boolean] = { - def whenCreating(throwable: Throwable): Boolean = - throwable match { - case t: WorkbenchException => throw t - case t: Exception => true - case _ => false - } - - retryExponentially(whenCreating) { () => - googleProjectDAO.pollOperation(operationId).map { operation => - if (operation.getDone && Option(operation.getError).exists(_.getCode.intValue() == Code.ALREADY_EXISTS.getNumber)) true - else if (operation.getDone && Option(operation.getError).isEmpty) true - else if (operation.getDone && Option(operation.getError).isDefined) - throw new WorkbenchException(s"project creation failed with error ${operation.getError.getMessage}") - else throw new Exception("project still creating...") - } - } - } - - def getAccessTokenUsingJson(saKey: String, desiredScopes: Set[String]): Future[String] = Future { - val keyStream = new ByteArrayInputStream(saKey.getBytes) - val credential = ServiceAccountCredentials.fromStream(keyStream).createScoped(desiredScopes.asJava) - credential.refreshAccessToken.getTokenValue - } - - def removePetServiceAccountKey(userId: WorkbenchUserId, project: GoogleProject, keyId: ServiceAccountKeyId, samRequestContext: SamRequestContext): IO[Unit] = - for { - maybePet <- directoryDAO.loadPetServiceAccount(PetServiceAccountId(userId, project), samRequestContext) - result <- maybePet match { - case Some(pet) => googleKeyCache.removeKey(pet, keyId) - case None => IO.unit - } - } yield result - - private def enablePetServiceAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[Unit] = - for { - _ <- directoryDAO.enableIdentity(petServiceAccount.id, samRequestContext) - _ <- withProxyEmail(petServiceAccount.id.userId) { proxyEmail => - IO.fromFuture(IO(googleDirectoryDAO.addMemberToGroup(proxyEmail, petServiceAccount.serviceAccount.email))) - } - } yield () - - private def disablePetServiceAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[Unit] = - for { - _ <- directoryDAO.disableIdentity(petServiceAccount.id, samRequestContext) - _ <- withProxyEmail(petServiceAccount.id.userId) { proxyEmail => - IO.fromFuture(IO(googleDirectoryDAO.removeMemberFromGroup(proxyEmail, petServiceAccount.serviceAccount.email))) - } - } yield () - - private def removePetServiceAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[Unit] = - for { - // disable the pet service account - _ <- disablePetServiceAccount(petServiceAccount, samRequestContext) - // remove the record for the pet service account - _ <- directoryDAO.deletePetServiceAccount(petServiceAccount.id, samRequestContext) - // remove the service account itself in Google - _ <- IO.fromFuture(IO(googleIamDAO.removeServiceAccount(petServiceAccount.id.project, toAccountName(petServiceAccount.serviceAccount.email)))) - } yield () - def getSynchronizedState(groupId: WorkbenchGroupIdentity, samRequestContext: SamRequestContext): IO[Option[GroupSyncResponse]] = { val groupDate = getSynchronizedDate(groupId, samRequestContext) val groupEmail = getSynchronizedEmail(groupId, samRequestContext) @@ -574,22 +399,6 @@ class GoogleExtensions( def getSynchronizedEmail(groupId: WorkbenchGroupIdentity, samRequestContext: SamRequestContext): IO[Option[WorkbenchEmail]] = directoryDAO.getSynchronizedEmail(groupId, samRequestContext) - private[google] def toPetSAFromUser(user: SamUser): (ServiceAccountName, ServiceAccountDisplayName) = { - /* - * Service account IDs must be: - * 1. between 6 and 30 characters - * 2. lower case alphanumeric separated by hyphens - * 3. must start with a lower case letter - * - * Subject IDs are 22 numeric characters, so "pet-${subjectId}" fulfills these requirements. - */ - val serviceAccountName = s"${googleServicesConfig.resourceNamePrefix.getOrElse("")}pet-${user.id.value}" - val displayName = s"Pet Service Account for user [${user.email.value}]" - - // Display names have a max length of 100 characters - (ServiceAccountName(serviceAccountName), ServiceAccountDisplayName(displayName.take(100))) - } - override def fireAndForgetNotifications[T <: Notification](notifications: Set[T]): Unit = notificationDAO.fireAndForgetNotifications(notifications) @@ -600,16 +409,6 @@ class GoogleExtensions( case _ => IO.pure(None) } - private[google] def getUserProxy(userId: WorkbenchUserId): IO[Option[WorkbenchEmail]] = - IO.pure(Some(toProxyFromUser(userId))) - - private def withProxyEmail[T](userId: WorkbenchUserId)(f: WorkbenchEmail => IO[T]): IO[T] = - getUserProxy(userId) flatMap { - case Some(e) => f(e) - case None => - throw new WorkbenchExceptionWithErrorReport(ErrorReport(StatusCodes.InternalServerError, s"Proxy group does not exist for subject ID: $userId")) - } - override def checkStatus: Map[Subsystems.Subsystem, Future[SubsystemStatus]] = { import HealthMonitor._ @@ -690,12 +489,46 @@ class GoogleExtensions( val bucket = GcsBucketName(blobId.getBucket) val objectName = GcsBlobName(blobId.getName) for { - petKey <- IO.fromFuture(IO(getArbitraryPetServiceAccountKey(samUser, samRequestContext))) + petKey <- IO.fromFuture(IO(petServiceAccounts.getArbitraryPetServiceAccountKey(samUser, samRequestContext))) serviceAccountCredentials = ServiceAccountCredentials.fromStream(new ByteArrayInputStream(petKey.getBytes())) url <- getSignedUrl(samUser, bucket, objectName, duration, urlParamsMap, serviceAccountCredentials) } yield url } + def getRequesterPaysSignedUrl( + samUser: SamUser, + resourceId: ResourceId, + resourceAction: ResourceAction, + googleProject: GoogleProject, + gsPath: String, + duration: Option[Long], + requesterPaysProject: Option[GoogleProject], + samRequestContext: SamRequestContext + ): IO[URL] = { + val urlParamsMap: Map[String, String] = requesterPaysProject.map(p => Map(userProjectQueryParam -> p.value)).getOrElse(Map.empty) + val blobId = fromGsPath(gsPath) + val bucket = GcsBucketName(blobId.getBucket) + val objectName = GcsBlobName(blobId.getName) + directoryDAO + .loadActionServiceAccount(ActionServiceAccountId(resourceId, resourceAction, googleProject), samRequestContext) + .flatMap { + case Some(actionServiceAccount) => + for { + petSigningAccountKey <- petSigningAccounts.getUserPetSigningAccountKey(samUser, samRequestContext) + } yield Some((actionServiceAccount, petSigningAccountKey)) + case None => IO.none[(ActionServiceAccount, String)] + } + .flatMap { + case Some((actionServiceAccount, petSigningAccountKey)) => + val serviceAccountCredentials = ServiceAccountCredentials + .fromStream(new ByteArrayInputStream(petSigningAccountKey.getBytes())) + .createDelegated(actionServiceAccount.serviceAccount.email.value) + .asInstanceOf[ServiceAccountCredentials] + getSignedUrl(samUser, bucket, objectName, duration, urlParamsMap, serviceAccountCredentials) + case None => getRequesterPaysSignedUrl(samUser, gsPath, duration, requesterPaysProject, samRequestContext) + } + } + def getSignedUrl( samUser: SamUser, project: GoogleProject, @@ -707,7 +540,7 @@ class GoogleExtensions( ): IO[URL] = { val urlParamsMap: Map[String, String] = if (requesterPays) Map(userProjectQueryParam -> project.value) else Map.empty for { - petServiceAccount <- createUserPetServiceAccount(samUser, project, samRequestContext) + petServiceAccount <- petServiceAccounts.createUserPetServiceAccount(samUser, project, samRequestContext) petKey <- googleKeyCache.getKey(petServiceAccount) serviceAccountCredentials = ServiceAccountCredentials.fromStream(new ByteArrayInputStream(petKey.getBytes())) url <- getSignedUrl(samUser, bucket, name, duration, urlParamsMap, serviceAccountCredentials) @@ -739,6 +572,8 @@ class GoogleExtensions( override val allSubSystems: Set[Subsystems.Subsystem] = Set(Subsystems.GoogleGroups, Subsystems.GooglePubSub, Subsystems.GoogleIam) + override def deleteUserPetServiceAccount(userId: WorkbenchUserId, project: GoogleProject, samRequestContext: SamRequestContext): IO[Boolean] = + petServiceAccounts.deleteUserPetServiceAccount(userId, project, samRequestContext) } case class GoogleExtensionsInitializer(cloudExtensions: GoogleExtensions, googleGroupSynchronizer: GoogleGroupSynchronizer) extends CloudExtensionsInitializer { diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/PetServiceAccounts.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/PetServiceAccounts.scala new file mode 100644 index 0000000000..f6340c4a2d --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/PetServiceAccounts.scala @@ -0,0 +1,223 @@ +package org.broadinstitute.dsde.workbench.sam.google + +import akka.actor.ActorSystem +import akka.http.scaladsl.model.StatusCodes +import cats.effect.unsafe.implicits.global +import cats.effect.IO +import cats.implicits.{catsSyntaxApplicativeId, catsSyntaxTuple2Parallel, toTraverseOps} +import com.google.api.client.googleapis.json.GoogleJsonResponseException +import org.broadinstitute.dsde.workbench.google.{GoogleDirectoryDAO, GoogleIamDAO, GoogleProjectDAO} +import org.broadinstitute.dsde.workbench.model.{PetServiceAccount, PetServiceAccountId, WorkbenchEmail, WorkbenchUserId} +import org.broadinstitute.dsde.workbench.model.google._ +import org.broadinstitute.dsde.workbench.sam.config.GoogleServicesConfig +import org.broadinstitute.dsde.workbench.sam.dataAccess.{DirectoryDAO, LockDetails, PostgresDistributedLockDAO} +import org.broadinstitute.dsde.workbench.sam.model.api.SamUser +import org.broadinstitute.dsde.workbench.sam.util.SamRequestContext + +import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.duration._ + +class PetServiceAccounts( + distributedLock: PostgresDistributedLockDAO[IO], + googleIamDAO: GoogleIamDAO, + googleDirectoryDAO: GoogleDirectoryDAO, + googleProjectDAO: GoogleProjectDAO, + googleKeyCache: GoogleKeyCache, + directoryDAO: DirectoryDAO, + googleServicesConfig: GoogleServicesConfig +)(implicit override val system: ActorSystem, executionContext: ExecutionContext) + extends ServiceAccounts(googleProjectDAO, googleServicesConfig) { + + private[google] def toPetSAFromUser(user: SamUser): (ServiceAccountName, ServiceAccountDisplayName) = { + /* + * Service account IDs must be: + * 1. between 6 and 30 characters + * 2. lower case alphanumeric separated by hyphens + * 3. must start with a lower case letter + * + * Subject IDs are 22 numeric characters, so "pet-${subjectId}" fulfills these requirements. + */ + val serviceAccountName = s"${googleServicesConfig.resourceNamePrefix.getOrElse("")}pet-${user.id.value}" + val displayName = s"Pet Service Account for user [${user.email.value}]" + + // Display names have a max length of 100 characters + (ServiceAccountName(serviceAccountName), ServiceAccountDisplayName(displayName.take(100))) + } + + private def retrievePetAndSA( + userId: WorkbenchUserId, + petServiceAccountName: ServiceAccountName, + project: GoogleProject, + samRequestContext: SamRequestContext + ): IO[(Option[PetServiceAccount], Option[ServiceAccount])] = { + val serviceAccount = IO.fromFuture(IO(googleIamDAO.findServiceAccount(project, petServiceAccountName))) + val pet = directoryDAO.loadPetServiceAccount(PetServiceAccountId(userId, project), samRequestContext) + (pet, serviceAccount).parTupled + } + def createUserPetServiceAccount(user: SamUser, project: GoogleProject, samRequestContext: SamRequestContext): IO[PetServiceAccount] = { + val (petSaName, petSaDisplayName) = toPetSAFromUser(user) + // The normal situation is that the pet either exists in both the database and google or neither. + // Sometimes, especially in tests, the pet may be removed from the database, but not google or the other way around. + // This code is a little extra complicated to detect the cases when a pet does not exist in google, the database or + // both and do the right thing. + val createPet = for { + (maybePet, maybeServiceAccount) <- retrievePetAndSA(user.id, petSaName, project, samRequestContext) + serviceAccount <- maybeServiceAccount match { + // SA does not exist in google, create it and add it to the proxy group + case None => + for { + _ <- assertProjectInTerraOrg(project) + _ <- assertProjectIsActive(project) + sa <- IO.fromFuture(IO(googleIamDAO.createServiceAccount(project, petSaName, petSaDisplayName))) + _ <- withProxyEmail(user.id) { proxyEmail => + // Add group member by uniqueId instead of email to avoid race condition + // See: https://broadworkbench.atlassian.net/browse/CA-1005 + IO.fromFuture(IO(googleDirectoryDAO.addServiceAccountToGroup(proxyEmail, sa))) + } + _ <- IO.fromFuture(IO(googleIamDAO.addServiceAccountUserRoleForUser(project, sa.email, sa.email))) + } yield sa + // SA already exists in google, use it + case Some(sa) => IO.pure(sa) + } + pet <- (maybePet, maybeServiceAccount) match { + // pet does not exist in the database, create it and enable the identity + case (None, _) => + for { + p <- directoryDAO.createPetServiceAccount(PetServiceAccount(PetServiceAccountId(user.id, project), serviceAccount), samRequestContext) + _ <- directoryDAO.enableIdentity(p.id, samRequestContext) + } yield p + // pet already exists in the database, but a new SA was created so update the database with new SA info + case (Some(p), None) => + for { + p <- directoryDAO.updatePetServiceAccount(p.copy(serviceAccount = serviceAccount), samRequestContext) + } yield p + + // everything already existed + case (Some(p), Some(_)) => IO.pure(p) + } + } yield pet + + val lock = LockDetails(s"${project.value}-createPet", user.id.value, 30 seconds) + + for { + (pet, sa) <- retrievePetAndSA(user.id, petSaName, project, samRequestContext) // I'm loving better-monadic-for + shouldLock = !(pet.isDefined && sa.isDefined) // if either is not defined, we need to lock and potentially create them; else we return the pet + p <- if (shouldLock) distributedLock.withLock(lock).use(_ => createPet) else pet.get.pure[IO] + } yield p + } + + def getPetServiceAccountKey(userEmail: WorkbenchEmail, project: GoogleProject, samRequestContext: SamRequestContext): IO[Option[String]] = + for { + subject <- directoryDAO.loadSubjectFromEmail(userEmail, samRequestContext) + key <- subject match { + case Some(userId: WorkbenchUserId) => + getPetServiceAccountKey(SamUser(userId, None, userEmail, None, false), project, samRequestContext).map(Option(_)) + case _ => IO.pure(None) + } + } yield key + + def getPetServiceAccountKey(user: SamUser, project: GoogleProject, samRequestContext: SamRequestContext): IO[String] = + for { + pet <- createUserPetServiceAccount(user, project, samRequestContext) + key <- googleKeyCache.getKey(pet) + } yield key + + def deleteUserPetServiceAccount(userId: WorkbenchUserId, project: GoogleProject, samRequestContext: SamRequestContext): IO[Boolean] = + for { + maybePet <- directoryDAO.loadPetServiceAccount(PetServiceAccountId(userId, project), samRequestContext) + deletedSomething <- maybePet match { + case Some(pet) => removePetServiceAccount(pet, samRequestContext).map(_ => true) + case None => IO.pure(false) // didn't find the pet, nothing to delete + } + } yield deletedSomething + + private[google] def removePetServiceAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[Unit] = + for { + // disable the pet service account + _ <- disablePetServiceAccount(petServiceAccount, samRequestContext) + // remove the record for the pet service account + _ <- directoryDAO.deletePetServiceAccount(petServiceAccount.id, samRequestContext) + // remove the service account itself in Google + _ <- IO.fromFuture(IO(googleIamDAO.removeServiceAccount(petServiceAccount.id.project, toAccountName(petServiceAccount.serviceAccount.email)))) + } yield () + + private[google] def disablePetServiceAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[Unit] = + for { + _ <- directoryDAO.disableIdentity(petServiceAccount.id, samRequestContext) + _ <- withProxyEmail(petServiceAccount.id.userId) { proxyEmail => + IO.fromFuture(IO(googleDirectoryDAO.removeMemberFromGroup(proxyEmail, petServiceAccount.serviceAccount.email))) + } + } yield () + + def getPetServiceAccountToken(user: SamUser, project: GoogleProject, scopes: Set[String], samRequestContext: SamRequestContext): Future[String] = + getPetServiceAccountKey(user, project, samRequestContext).unsafeToFuture().flatMap { key => + getAccessTokenUsingJson(key, scopes) + } + + def getArbitraryPetServiceAccountKey(userEmail: WorkbenchEmail, samRequestContext: SamRequestContext): IO[Option[String]] = + for { + subject <- directoryDAO.loadSubjectFromEmail(userEmail, samRequestContext) + key <- subject match { + case Some(userId: WorkbenchUserId) => + IO.fromFuture(IO(getArbitraryPetServiceAccountKey(SamUser(userId, None, userEmail, None, false), samRequestContext))).map(Option(_)) + case _ => IO.none + } + } yield key + + def getArbitraryPetServiceAccountKey(user: SamUser, samRequestContext: SamRequestContext): Future[String] = + getDefaultServiceAccountForShellProject(user, samRequestContext) + + def getArbitraryPetServiceAccountToken(user: SamUser, scopes: Set[String], samRequestContext: SamRequestContext): Future[String] = + getArbitraryPetServiceAccountKey(user, samRequestContext).flatMap { key => + getAccessTokenUsingJson(key, scopes) + } + + private def getDefaultServiceAccountForShellProject(user: SamUser, samRequestContext: SamRequestContext): Future[String] = { + val projectName = + s"fc-${googleServicesConfig.environment.substring(0, Math.min(googleServicesConfig.environment.length(), 5))}-${user.id.value}" // max 30 characters. subject ID is 21 + for { + creationOperationId <- googleProjectDAO + .createProject(projectName, googleServicesConfig.terraGoogleOrgNumber, GoogleResourceTypes.Organization) + .map(opId => Option(opId)) recover { + case gjre: GoogleJsonResponseException if gjre.getDetails.getCode == StatusCodes.Conflict.intValue => None + } + _ <- creationOperationId match { + case Some(opId) => pollShellProjectCreation(opId) // poll until it's created + case None => Future.successful(()) + } + key <- getPetServiceAccountKey(user, GoogleProject(projectName), samRequestContext).unsafeToFuture() + } yield key + } + + private[google] def enablePetServiceAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[Unit] = + for { + _ <- directoryDAO.enableIdentity(petServiceAccount.id, samRequestContext) + _ <- withProxyEmail(petServiceAccount.id.userId) { proxyEmail => + IO.fromFuture(IO(googleDirectoryDAO.addMemberToGroup(proxyEmail, petServiceAccount.serviceAccount.email))) + } + } yield () + + private[google] def removePetServiceAccountKey( + userId: WorkbenchUserId, + project: GoogleProject, + keyId: ServiceAccountKeyId, + samRequestContext: SamRequestContext + ): IO[Unit] = + for { + maybePet <- directoryDAO.loadPetServiceAccount(PetServiceAccountId(userId, project), samRequestContext) + result <- maybePet match { + case Some(pet) => googleKeyCache.removeKey(pet, keyId) + case None => IO.unit + } + } yield result + + /** Evaluate a future for each pet in parallel. + */ + private[google] def forAllPets[T](userId: WorkbenchUserId, samRequestContext: SamRequestContext)(f: PetServiceAccount => IO[T]): IO[Seq[T]] = + for { + pets <- directoryDAO.getAllPetServiceAccountsForUser(userId, samRequestContext) + a <- pets.traverse { pet => + f(pet) + } + } yield a +} diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/PetSigningAccounts.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/PetSigningAccounts.scala new file mode 100644 index 0000000000..b850f77805 --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/PetSigningAccounts.scala @@ -0,0 +1,162 @@ +package org.broadinstitute.dsde.workbench.sam.google + +import akka.actor.ActorSystem +import akka.http.scaladsl.model.StatusCodes +import cats.effect.unsafe.implicits.global +import cats.effect.IO +import cats.implicits.{catsSyntaxApplicativeId, catsSyntaxTuple2Parallel} +import com.google.api.client.googleapis.json.GoogleJsonResponseException +import org.broadinstitute.dsde.workbench.google.{GoogleDirectoryDAO, GoogleIamDAO, GoogleProjectDAO} +import org.broadinstitute.dsde.workbench.model.google._ +import org.broadinstitute.dsde.workbench.model.{PetServiceAccount, PetServiceAccountId, WorkbenchEmail, WorkbenchException, WorkbenchUserId} +import org.broadinstitute.dsde.workbench.sam.config.GoogleServicesConfig +import org.broadinstitute.dsde.workbench.sam.dataAccess.{DirectoryDAO, LockDetails, PostgresDistributedLockDAO} +import org.broadinstitute.dsde.workbench.sam.model.api.SamUser +import org.broadinstitute.dsde.workbench.sam.service.CloudExtensions +import org.broadinstitute.dsde.workbench.sam.util.SamRequestContext + +import scala.concurrent.duration._ +import scala.concurrent.{ExecutionContext, Future} + +class PetSigningAccounts( + distributedLock: PostgresDistributedLockDAO[IO], + googleIamDAO: GoogleIamDAO, + googleDirectoryDAO: GoogleDirectoryDAO, + googleProjectDAO: GoogleProjectDAO, + googleKeyCache: GoogleKeyCache, + directoryDAO: DirectoryDAO, + googleServicesConfig: GoogleServicesConfig +)(implicit override val system: ActorSystem, executionContext: ExecutionContext) + extends ServiceAccounts(googleProjectDAO, googleServicesConfig) { + + private val emailDomain = googleServicesConfig.appsDomain + + private[google] val allPetSigningAccountsGroupEmail = WorkbenchEmail( + s"${googleServicesConfig.resourceNamePrefix.getOrElse("")}GROUP_${CloudExtensions.allPetSingingAccountsGroupName.value}@$emailDomain" + ) + + private[google] def createPetSigningAccountForUser(user: SamUser, samRequestContext: SamRequestContext): IO[PetServiceAccount] = { + val googleProject = petServiceAccountProject(user) + for { + _ <- getUserPetSigningAccountKey(user, samRequestContext) + account <- directoryDAO.loadPetSigningAccount(PetServiceAccountId(user.id, googleProject), samRequestContext) + } yield account.getOrElse(throw new WorkbenchException(s"Failed to create Pet Signing Account for ${user}")) + } + + private[google] def retrievePetSigningAccountAndSA( + userId: WorkbenchUserId, + petServiceAccountName: ServiceAccountName, + project: GoogleProject, + samRequestContext: SamRequestContext + ): IO[(Option[PetServiceAccount], Option[ServiceAccount])] = { + val serviceAccount = IO.fromFuture(IO(googleIamDAO.findServiceAccount(project, petServiceAccountName))) + val pet = directoryDAO.loadPetSigningAccount(PetServiceAccountId(userId, project), samRequestContext) + (pet, serviceAccount).parTupled + } + + private[google] def petServiceAccountProject(samUser: SamUser): GoogleProject = + GoogleProject( + s"fc-${googleServicesConfig.environment.substring(0, Math.min(googleServicesConfig.environment.length(), 5))}-${samUser.id}" + ) // max 30 characters. subject ID is 21 + + private[google] def getUserPetSigningAccountKey(user: SamUser, samRequestContext: SamRequestContext): IO[String] = { + val googleProject = petServiceAccountProject(user) + val (petSaName, petSaDisplayName) = toPetSigningAccountFromUser(user) + + val keyFuture = for { + creationOperationId <- googleProjectDAO + .createProject(googleProject.value, googleServicesConfig.terraGoogleOrgNumber, GoogleResourceTypes.Organization) + .map(opId => Option(opId)) recover { + case gjre: GoogleJsonResponseException if gjre.getDetails.getCode == StatusCodes.Conflict.intValue => None + } + _ <- creationOperationId match { + case Some(opId) => pollShellProjectCreation(opId) // poll until it's created + case None => Future.successful(()) + } + serviceAccount <- createPetSigningAccount(user, petSaName, petSaDisplayName, googleProject, samRequestContext).unsafeToFuture() + key <- googleKeyCache.getKey(serviceAccount).unsafeToFuture() + } yield key + IO.fromFuture(IO(keyFuture)) + } + + private def createPetSigningAccount( + user: SamUser, + petSaName: ServiceAccountName, + petSaDisplayName: ServiceAccountDisplayName, + project: GoogleProject, + samRequestContext: SamRequestContext + ): IO[PetServiceAccount] = { + // The normal situation is that the pet either exists in both the database and google or neither. + // Sometimes, especially in tests, the pet may be removed from the database, but not google or the other way around. + // This code is a little extra complicated to detect the cases when a pet does not exist in google, the database or + // both and do the right thing. + val createPet = for { + (maybePet, maybeServiceAccount) <- retrievePetSigningAccountAndSA(user.id, petSaName, project, samRequestContext) + serviceAccount <- maybeServiceAccount match { + // SA does not exist in google, create it and add it to the proxy group + case None => + for { + _ <- assertProjectInTerraOrg(project) + sa <- IO.fromFuture(IO(googleIamDAO.createServiceAccount(project, petSaName, petSaDisplayName))) + _ <- IO.fromFuture(IO(googleDirectoryDAO.addServiceAccountToGroup(allPetSigningAccountsGroupEmail, sa))) + _ <- IO.fromFuture(IO(googleIamDAO.addServiceAccountUserRoleForUser(project, sa.email, sa.email))) + } yield sa + // SA already exists in google, use it + case Some(sa) => IO.pure(sa) + } + pet <- (maybePet, maybeServiceAccount) match { + // pet does not exist in the database, create it and enable the identity + case (None, _) => + for { + p <- directoryDAO.createPetSigningAccount(PetServiceAccount(PetServiceAccountId(user.id, project), serviceAccount), samRequestContext) + } yield p + // pet already exists in the database, but a new SA was created so update the database with new SA info + case (Some(p), None) => + for { + p <- directoryDAO.updatePetSigningAccount(p.copy(serviceAccount = serviceAccount), samRequestContext) + } yield p + + // everything already existed + case (Some(p), Some(_)) => IO.pure(p) + } + } yield pet + + val lock = LockDetails(s"${project.value}-createPetSigning", user.id.value, 30 seconds) + + for { + (pet, sa) <- retrievePetSigningAccountAndSA(user.id, petSaName, project, samRequestContext) // I'm loving better-monadic-for + shouldLock = !(pet.isDefined && sa.isDefined) // if either is not defined, we need to lock and potentially create them; else we return the pet + p <- if (shouldLock) distributedLock.withLock(lock).use(_ => createPet) else pet.get.pure[IO] + } yield p + } + + private[google] def toPetSigningAccountFromUser(user: SamUser): (ServiceAccountName, ServiceAccountDisplayName) = { + /* + * Service account IDs must be: + * 1. between 6 and 30 characters + * 2. lower case alphanumeric separated by hyphens + * 3. must start with a lower case letter + * + * Subject IDs are 22 numeric characters, so "sign-${subjectId}" fulfills these requirements. + */ + val serviceAccountName = s"sign-${user.id.value}" + val displayName = s"Pet Signing Account [${user.email.value}]" + + // Display names have a max length of 100 characters + (ServiceAccountName(serviceAccountName), ServiceAccountDisplayName(displayName.take(100))) + } + + private[google] def removePetSigningAccount(userId: WorkbenchUserId, samRequestContext: SamRequestContext): IO[Unit] = + for { + petSigningAccount <- directoryDAO.loadUserPetSigningAccount(userId, samRequestContext) + _ <- petSigningAccount.map(account => directoryDAO.deletePetSigningAccount(account.id, samRequestContext)).getOrElse(IO.unit) + _ <- IO.fromFuture( + IO( + petSigningAccount + .map(account => googleIamDAO.removeServiceAccount(account.id.project, toAccountName(account.serviceAccount.email))) + .getOrElse(Future.successful(())) + ) + ) + } yield () + +} diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/ProxyEmailSupport.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/ProxyEmailSupport.scala new file mode 100644 index 0000000000..617ceb9e1a --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/ProxyEmailSupport.scala @@ -0,0 +1,24 @@ +package org.broadinstitute.dsde.workbench.sam.google + +import org.broadinstitute.dsde.workbench.sam._ + +import akka.http.scaladsl.model.StatusCodes +import cats.effect.IO +import org.broadinstitute.dsde.workbench.model.{ErrorReport, WorkbenchEmail, WorkbenchExceptionWithErrorReport, WorkbenchUserId} +import org.broadinstitute.dsde.workbench.sam.config.GoogleServicesConfig + +abstract class ProxyEmailSupport(googleServicesConfig: GoogleServicesConfig) { + private[google] def getUserProxy(userId: WorkbenchUserId): IO[Option[WorkbenchEmail]] = + IO.pure(Some(toProxyFromUser(userId))) + + private[google] def withProxyEmail[T](userId: WorkbenchUserId)(f: WorkbenchEmail => IO[T]): IO[T] = + getUserProxy(userId) flatMap { + case Some(e) => f(e) + case None => + throw new WorkbenchExceptionWithErrorReport(ErrorReport(StatusCodes.InternalServerError, s"Proxy group does not exist for subject ID: $userId")) + } + + private[google] def toProxyFromUser(userId: WorkbenchUserId): WorkbenchEmail = + WorkbenchEmail(s"${googleServicesConfig.resourceNamePrefix.getOrElse("")}PROXY_${userId.value}@${googleServicesConfig.appsDomain}") + +} diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/ServiceAccounts.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/ServiceAccounts.scala new file mode 100644 index 0000000000..3175599e45 --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/google/ServiceAccounts.scala @@ -0,0 +1,85 @@ +package org.broadinstitute.dsde.workbench.sam.google + +import akka.actor.ActorSystem +import akka.http.scaladsl.model.StatusCodes +import cats.effect.IO +import com.google.api.client.http.HttpResponseException +import com.google.auth.oauth2.ServiceAccountCredentials +import com.google.rpc.Code +import com.typesafe.scalalogging.LazyLogging +import org.broadinstitute.dsde.workbench.google.GoogleProjectDAO +import org.broadinstitute.dsde.workbench.model.{ErrorReport, WorkbenchException, WorkbenchExceptionWithErrorReport} +import org.broadinstitute.dsde.workbench.model.google.{GoogleProject, GoogleResourceTypes} +import org.broadinstitute.dsde.workbench.sam._ +import org.broadinstitute.dsde.workbench.sam.config.GoogleServicesConfig +import org.broadinstitute.dsde.workbench.util.{FutureSupport, Retry} + +import java.io.ByteArrayInputStream +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ + +class ServiceAccounts(googleProjectDAO: GoogleProjectDAO, googleServicesConfig: GoogleServicesConfig)(implicit + val system: ActorSystem, + executionContext: ExecutionContext +) extends ProxyEmailSupport(googleServicesConfig) + with LazyLogging + with FutureSupport + with Retry { + private[google] def assertProjectInTerraOrg(project: GoogleProject): IO[Unit] = { + val validOrg = IO + .fromFuture(IO(googleProjectDAO.getAncestry(project.value).map { ancestry => + ancestry.exists { ancestor => + ancestor.getResourceId.getType == GoogleResourceTypes.Organization.value && ancestor.getResourceId.getId == googleServicesConfig.terraGoogleOrgNumber + } + })) + .recoverWith { + // if the getAncestry call results in a 403 error the project can't be in the right org + case e: HttpResponseException if e.getStatusCode == StatusCodes.Forbidden.intValue => + IO.raiseError( + new WorkbenchExceptionWithErrorReport( + ErrorReport(StatusCodes.BadRequest, s"Access denied from google accessing project ${project.value}, is it a Terra project?", e) + ) + ) + } + + validOrg.flatMap { + case true => IO.unit + case false => + IO.raiseError(new WorkbenchExceptionWithErrorReport(ErrorReport(StatusCodes.BadRequest, s"Project ${project.value} must be in Terra Organization"))) + } + } + + private[google] def assertProjectIsActive(project: GoogleProject): IO[Unit] = + for { + projectIsActive <- IO.fromFuture(IO(googleProjectDAO.isProjectActive(project.value))) + _ <- IO.raiseUnless(projectIsActive)( + new WorkbenchExceptionWithErrorReport(ErrorReport(StatusCodes.BadRequest, s"Project ${project.value} is inactive")) + ) + } yield () + + def getAccessTokenUsingJson(saKey: String, desiredScopes: Set[String]): Future[String] = Future { + val keyStream = new ByteArrayInputStream(saKey.getBytes) + val credential = ServiceAccountCredentials.fromStream(keyStream).createScoped(desiredScopes.asJava) + credential.refreshAccessToken.getTokenValue + } + + private[google] def pollShellProjectCreation(operationId: String): Future[Boolean] = { + def whenCreating(throwable: Throwable): Boolean = + throwable match { + case t: WorkbenchException => throw t + case t: Exception => true + case _ => false + } + + retryExponentially(whenCreating) { () => + googleProjectDAO.pollOperation(operationId).map { operation => + if (operation.getDone && Option(operation.getError).exists(_.getCode.intValue() == Code.ALREADY_EXISTS.getNumber)) true + else if (operation.getDone && Option(operation.getError).isEmpty) true + else if (operation.getDone && Option(operation.getError).isDefined) + throw new WorkbenchException(s"project creation failed with error ${operation.getError.getMessage}") + else throw new Exception("project still creating...") + } + } + } + +} diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/model/api/ActionServiceAccount.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/model/api/ActionServiceAccount.scala new file mode 100644 index 0000000000..632566e344 --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/model/api/ActionServiceAccount.scala @@ -0,0 +1,19 @@ +package org.broadinstitute.dsde.workbench.sam.model.api + +import org.broadinstitute.dsde.workbench.model.google.GoogleModelJsonSupport._ +import org.broadinstitute.dsde.workbench.model.google.{GoogleProject, ServiceAccount} +import org.broadinstitute.dsde.workbench.sam.model.{ResourceAction, ResourceId} +import org.broadinstitute.dsde.workbench.sam.model.api.SamJsonSupport._ +import spray.json.DefaultJsonProtocol._ +import spray.json.RootJsonFormat + +// This should live in wb-libs +object ActionServiceAccountId { + implicit val ActionServiceAccountIdFormat: RootJsonFormat[ActionServiceAccountId] = jsonFormat3(ActionServiceAccountId.apply) +} +case class ActionServiceAccountId(resourceId: ResourceId, action: ResourceAction, project: GoogleProject) + +object ActionServiceAccount { + implicit val ActionServiceAccountFormat: RootJsonFormat[ActionServiceAccount] = jsonFormat2(ActionServiceAccount.apply) +} +case class ActionServiceAccount(id: ActionServiceAccountId, serviceAccount: ServiceAccount) diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/service/CloudExtensions.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/service/CloudExtensions.scala index ce96e83d8a..91b222fe0f 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/sam/service/CloudExtensions.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/service/CloudExtensions.scala @@ -11,7 +11,7 @@ import org.broadinstitute.dsde.workbench.model.google.GoogleProject import org.broadinstitute.dsde.workbench.sam.api.ExtensionRoutes import org.broadinstitute.dsde.workbench.sam.dataAccess.DirectoryDAO import org.broadinstitute.dsde.workbench.sam.model.api.SamUser -import org.broadinstitute.dsde.workbench.sam.model.{BasicWorkbenchGroup, ResourceTypeName} +import org.broadinstitute.dsde.workbench.sam.model.{BasicWorkbenchGroup, ResourceId, ResourceTypeName} import org.broadinstitute.dsde.workbench.sam.util.SamRequestContext import org.broadinstitute.dsde.workbench.util.health.SubsystemStatus import org.broadinstitute.dsde.workbench.util.health.Subsystems.Subsystem @@ -21,6 +21,7 @@ import scala.concurrent.{ExecutionContext, Future} object CloudExtensions { val resourceTypeName = ResourceTypeName("cloud-extension") val allUsersGroupName = WorkbenchGroupName("All_Users") + val allPetSingingAccountsGroupName = WorkbenchGroupName("All_Signing_Accounts") } trait CloudExtensions { @@ -53,6 +54,7 @@ trait CloudExtensions { def deleteUserPetServiceAccount(userId: WorkbenchUserId, project: GoogleProject, samRequestContext: SamRequestContext): IO[Boolean] + def onResourceDelete(resourceId: ResourceId, samRequestContext: SamRequestContext): IO[Unit] def getUserProxy(userEmail: WorkbenchEmail, samRequestContext: SamRequestContext): IO[Option[WorkbenchEmail]] def fireAndForgetNotifications[T <: Notification](notifications: Set[T]): Unit @@ -99,7 +101,7 @@ trait NoExtensions extends CloudExtensions { override def onUserDelete(userId: WorkbenchUserId, samRequestContext: SamRequestContext): IO[Unit] = IO.unit override def deleteUserPetServiceAccount(userId: WorkbenchUserId, project: GoogleProject, samRequestContext: SamRequestContext): IO[Boolean] = IO.pure(true) - + def onResourceDelete(resourceId: ResourceId, samRequestContext: SamRequestContext): IO[Unit] = IO.unit override def getUserProxy(userEmail: WorkbenchEmail, samRequestContext: SamRequestContext): IO[Option[WorkbenchEmail]] = IO.pure(Option(userEmail)) diff --git a/src/main/scala/org/broadinstitute/dsde/workbench/sam/service/ResourceService.scala b/src/main/scala/org/broadinstitute/dsde/workbench/sam/service/ResourceService.scala index 707d63cff5..a277561c0d 100644 --- a/src/main/scala/org/broadinstitute/dsde/workbench/sam/service/ResourceService.scala +++ b/src/main/scala/org/broadinstitute/dsde/workbench/sam/service/ResourceService.scala @@ -376,8 +376,8 @@ class ResourceService( // remove from cloud first so a failure there does not leave sam in a bad state _ <- cloudDeletePolicies(resource, samRequestContext) + _ <- cloudExtensions.onResourceDelete(resource.resourceId, samRequestContext) _ <- deleteActionManagedIdentitiesForResource(resource, samRequestContext) - // leave a tomb stone if the resource type does not allow reuse leaveTombStone = !resourceTypes(resource.resourceTypeName).reuseIds _ <- accessPolicyDAO.deleteResource(resource, leaveTombStone, samRequestContext) diff --git a/src/test/scala/org/broadinstitute/dsde/workbench/sam/TestSupport.scala b/src/test/scala/org/broadinstitute/dsde/workbench/sam/TestSupport.scala index 5744681dc2..02551a1a33 100644 --- a/src/test/scala/org/broadinstitute/dsde/workbench/sam/TestSupport.scala +++ b/src/test/scala/org/broadinstitute/dsde/workbench/sam/TestSupport.scala @@ -230,6 +230,8 @@ object TestSupport extends TestSupport { if (databaseEnabled) { dbRef.inLocalTransaction { implicit session => val tables = List( + ActionServiceAccountTable, + PetSigningAccountTable, ActionManagedIdentityTable, PolicyActionTable, PolicyRoleTable, diff --git a/src/test/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/MockDirectoryDAO.scala b/src/test/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/MockDirectoryDAO.scala index 835bdc6fb2..ce4bac92af 100644 --- a/src/test/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/MockDirectoryDAO.scala +++ b/src/test/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/MockDirectoryDAO.scala @@ -17,6 +17,8 @@ import org.broadinstitute.dsde.workbench.sam.azure.{ import org.broadinstitute.dsde.workbench.sam.db.tables.TosTable import org.broadinstitute.dsde.workbench.sam.model.api.{AdminUpdateUserRequest, SamUser, SamUserAttributes} import org.broadinstitute.dsde.workbench.sam.model.{AccessPolicy, BasicWorkbenchGroup, FullyQualifiedResourceId, ResourceAction, SamUserTos} +import org.broadinstitute.dsde.workbench.sam.model.api.{ActionServiceAccount, ActionServiceAccountId} +import org.broadinstitute.dsde.workbench.sam.model.ResourceId import org.broadinstitute.dsde.workbench.sam.util.SamRequestContext import java.time.Instant @@ -305,6 +307,43 @@ class MockDirectoryDAO(val groups: mutable.Map[WorkbenchGroupIdentity, Workbench petServiceAccount } + override def createActionServiceAccount(actionServiceAccount: ActionServiceAccount, samRequestContext: SamRequestContext): IO[ActionServiceAccount] = ??? + + override def loadActionServiceAccount( + actionServiceAccountId: ActionServiceAccountId, + samRequestContext: SamRequestContext + ): IO[Option[ActionServiceAccount]] = ??? + + override def updateActionServiceAccount(actionServiceAccount: ActionServiceAccount, samRequestContext: SamRequestContext): IO[ActionServiceAccount] = ??? + override def deleteActionServiceAccount(actionServiceAccountId: ActionServiceAccountId, samRequestContext: SamRequestContext): IO[Unit] = ??? + + override def getAllActionServiceAccountsForResource( + resourceId: ResourceId, + samRequestContext: SamRequestContext + ): IO[Seq[ActionServiceAccount]] = ??? + override def deleteAllActionServiceAccountsForResource(resourceId: ResourceId, samRequestContext: SamRequestContext): IO[Unit] = + ??? + + override def createPetSigningAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[PetServiceAccount] = { + if (petServiceAccountsByUser.keySet.contains(petServiceAccount.id)) { + IO.raiseError(new WorkbenchExceptionWithErrorReport(ErrorReport(StatusCodes.Conflict, s"pet service account ${petServiceAccount.id} already exists"))) + } + petServiceAccountsByUser += petServiceAccount.id -> petServiceAccount + petsWithEmails += petServiceAccount.serviceAccount.email -> petServiceAccount.id + usersWithGoogleSubjectIds += GoogleSubjectId(petServiceAccount.serviceAccount.subjectId.value) -> petServiceAccount.id + IO.pure(petServiceAccount) + } + + override def loadPetSigningAccount(petServiceAccountId: PetServiceAccountId, samRequestContext: SamRequestContext): IO[Option[PetServiceAccount]] = IO { + petServiceAccountsByUser.get(petServiceAccountId) + } + def loadUserPetSigningAccount(userId: WorkbenchUserId, samRequestContext: SamRequestContext): IO[Option[PetServiceAccount]] = ??? + + override def updatePetSigningAccount(petServiceAccount: PetServiceAccount, samRequestContext: SamRequestContext): IO[PetServiceAccount] = IO { + petServiceAccountsByUser.update(petServiceAccount.id, petServiceAccount) + petServiceAccount + } + override def deletePetSigningAccount(petServiceAccountId: PetServiceAccountId, samRequestContext: SamRequestContext): IO[Unit] = ??? override def getManagedGroupAccessInstructions(groupName: WorkbenchGroupName, samRequestContext: SamRequestContext): IO[Option[String]] = if (groups.contains(groupName)) IO.pure(groupAccessInstructions.get(groupName)) diff --git a/src/test/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAOSpec.scala b/src/test/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAOSpec.scala index 7c04091faa..df18bfc79c 100644 --- a/src/test/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAOSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAOSpec.scala @@ -13,6 +13,7 @@ import org.broadinstitute.dsde.workbench.sam.db.tables.TosTable import org.broadinstitute.dsde.workbench.sam.matchers.TimeMatchers import org.broadinstitute.dsde.workbench.sam.model._ import org.broadinstitute.dsde.workbench.sam.model.api.{AdminUpdateUserRequest, SamUser, SamUserAttributes} +import org.broadinstitute.dsde.workbench.sam.model.api.{ActionServiceAccount, ActionServiceAccountId} import org.broadinstitute.dsde.workbench.sam.{Generator, RetryableAnyFreeSpec, TestSupport} import org.scalatest.Inside.inside import org.scalatest.matchers.should.Matchers @@ -29,10 +30,11 @@ class PostgresDirectoryDAOSpec extends RetryableAnyFreeSpec with Matchers with B val azureManagedResourceGroupDAO = new PostgresAzureManagedResourceGroupDAO(TestSupport.dbRef, TestSupport.dbRef) val defaultGroupName: WorkbenchGroupName = WorkbenchGroupName("group") + val defaultGoogleProject: GoogleProject = GoogleProject("testProject") val defaultGroup: BasicWorkbenchGroup = BasicWorkbenchGroup(defaultGroupName, Set.empty, WorkbenchEmail("foo@bar.com")) val defaultUser: SamUser = Generator.genWorkbenchUserBoth.sample.get val defaultPetSA: PetServiceAccount = PetServiceAccount( - PetServiceAccountId(defaultUser.id, GoogleProject("testProject")), + PetServiceAccountId(defaultUser.id, defaultGoogleProject), ServiceAccount(ServiceAccountSubjectId("testGoogleSubjectId"), WorkbenchEmail("test@pet.co"), ServiceAccountDisplayName("whoCares")) ) val defaultPetMI: PetManagedIdentity = PetManagedIdentity( @@ -67,6 +69,23 @@ class PostgresDirectoryDAOSpec extends RetryableAnyFreeSpec with Matchers with B public = false ) + val defaultActionServiceAccounts: Set[ActionServiceAccount] = Set(readAction, writeAction).map(action => + ActionServiceAccount( + ActionServiceAccountId(defaultResource.resourceId, action, defaultGoogleProject), + ServiceAccount( + ServiceAccountSubjectId(s"testGoogleSubjectId-$action"), + WorkbenchEmail(s"test-$action@asa.co"), + ServiceAccountDisplayName(s"whoCares-$action") + ) + ) + ) + + val defaultUserGoogleAccount: GoogleProject = GoogleProject("testUserProject") + val defaultPetSigningAccount: PetServiceAccount = PetServiceAccount( + PetServiceAccountId(defaultUser.id, defaultUserGoogleAccount), + ServiceAccount(ServiceAccountSubjectId("testGoogleSubjectId"), WorkbenchEmail("test@petsigning.co"), ServiceAccountDisplayName("whoCares-signing")) + ) + val defaultTenantId = TenantId("testTenant") val defaultSubscriptionId = SubscriptionId(UUID.randomUUID().toString) val defaultManagedResourceGroupName = ManagedResourceGroupName("mrg-test") @@ -1945,7 +1964,6 @@ class PostgresDirectoryDAOSpec extends RetryableAnyFreeSpec with Matchers with B retrievedAttributes should be(Some(upsertedAttributes)) } } - "Action Managed Identities" - { "can be individually created, read, updated, and deleted" in { assume(databaseEnabled, databaseEnabledClue) @@ -2018,6 +2036,96 @@ class PostgresDirectoryDAOSpec extends RetryableAnyFreeSpec with Matchers with B } } + "Action Service Accounts" - { + "can be individually created, read, updated, and deleted" in { + assume(databaseEnabled, databaseEnabledClue) + policyDAO.createResourceType(resourceType, samRequestContext).unsafeRunSync() + policyDAO.createResource(defaultResource, samRequestContext).unsafeRunSync() + + defaultActionServiceAccounts.map(dao.createActionServiceAccount(_, samRequestContext).unsafeRunSync()) + + val readActionServiceAccount = defaultActionServiceAccounts.find(_.id.action == readAction) + val loadedReadActionServiceAccount = dao.loadActionServiceAccount(readActionServiceAccount.get.id, samRequestContext).unsafeRunSync() + loadedReadActionServiceAccount should be(readActionServiceAccount) + + val writeActionServiceAccount = defaultActionServiceAccounts.find(_.id.action == writeAction) + val loadedWriteActionServiceAccount = dao.loadActionServiceAccount(writeActionServiceAccount.get.id, samRequestContext).unsafeRunSync() + loadedWriteActionServiceAccount should be(writeActionServiceAccount) + + val updatedActionServiceAccount = writeActionServiceAccount.get.copy(serviceAccount = + writeActionServiceAccount.get.serviceAccount.copy(displayName = ServiceAccountDisplayName("new name"), email = WorkbenchEmail("newEmail@asa.co")) + ) + dao.updateActionServiceAccount(updatedActionServiceAccount, samRequestContext).unsafeRunSync() + + val loadedUpdatedActionServiceAccount = dao.loadActionServiceAccount(updatedActionServiceAccount.id, samRequestContext).unsafeRunSync() + loadedUpdatedActionServiceAccount should be(Some(updatedActionServiceAccount)) + + dao.deleteActionServiceAccount(readActionServiceAccount.get.id, samRequestContext).unsafeRunSync() + dao.deleteActionServiceAccount(writeActionServiceAccount.get.id, samRequestContext).unsafeRunSync() + + dao.loadActionServiceAccount(readActionServiceAccount.get.id, samRequestContext).unsafeRunSync() should be(None) + dao.loadActionServiceAccount(writeActionServiceAccount.get.id, samRequestContext).unsafeRunSync() should be(None) + } + + "can be read, and deleted en mass for a resource" in { + assume(databaseEnabled, databaseEnabledClue) + policyDAO.createResourceType(resourceType, samRequestContext).unsafeRunSync() + policyDAO.createResource(defaultResource, samRequestContext).unsafeRunSync() + + defaultActionServiceAccounts.map(dao.createActionServiceAccount(_, samRequestContext).unsafeRunSync()) + + val bothLoadedServiceAccounts = + dao.getAllActionServiceAccountsForResource(defaultResource.resourceId, samRequestContext).unsafeRunSync().toSet + bothLoadedServiceAccounts should be(defaultActionServiceAccounts) + + dao.deleteAllActionServiceAccountsForResource(defaultResource.resourceId, samRequestContext).unsafeRunSync() + + dao.getAllActionServiceAccountsForResource(defaultResource.resourceId, samRequestContext).unsafeRunSync() should be(Seq.empty) + } + } + "Pet Signing Accounts" - { + "can be individually created, read, and deleted" in { + assume(databaseEnabled, databaseEnabledClue) + + dao.createUser(defaultUser, samRequestContext).unsafeRunSync() + + dao.createPetServiceAccount(defaultPetSA, samRequestContext).unsafeRunSync() + val createdPetSigningAccount = dao.createPetSigningAccount(defaultPetSigningAccount, samRequestContext).unsafeRunSync() + + val loadedPetSigningAccount = dao.loadPetSigningAccount(defaultPetSigningAccount.id, samRequestContext).unsafeRunSync() + loadedPetSigningAccount should not be None + loadedPetSigningAccount should be(Some(createdPetSigningAccount)) + + dao.deletePetSigningAccount(defaultPetSigningAccount.id, samRequestContext).unsafeRunSync() + + dao.loadPetSigningAccount(defaultPetSigningAccount.id, samRequestContext).unsafeRunSync() should be(None) + } + + "are distinct from Pet Service Accounts" in { + assume(databaseEnabled, databaseEnabledClue) + + dao.createUser(defaultUser, samRequestContext).unsafeRunSync() + + dao.createPetServiceAccount(defaultPetSA, samRequestContext).unsafeRunSync() + dao.createPetSigningAccount(defaultPetSigningAccount, samRequestContext).unsafeRunSync() + + val loadedPetServiceAccounts = dao.getAllPetServiceAccountsForUser(defaultUser.id, samRequestContext).unsafeRunSync() + + loadedPetServiceAccounts should be(Seq(defaultPetSA)) + } + + "can be loaded per-user" in { + assume(databaseEnabled, databaseEnabledClue) + + dao.createUser(defaultUser, samRequestContext).unsafeRunSync() + + dao.createPetSigningAccount(defaultPetSigningAccount, samRequestContext).unsafeRunSync() + + val userPetSigningAccount = dao.loadUserPetSigningAccount(defaultUser.id, samRequestContext).unsafeRunSync() + userPetSigningAccount should be(Some(defaultPetSigningAccount)) + } + } + "listParentGroups" - { "list all of the parent groups of a group" in { assume(databaseEnabled, databaseEnabledClue) diff --git a/src/test/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensionSpec.scala b/src/test/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensionSpec.scala index 3456f2e38f..3cf68f1143 100644 --- a/src/test/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensionSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/workbench/sam/google/GoogleExtensionSpec.scala @@ -413,7 +413,7 @@ class GoogleExtensionSpec(_system: ActorSystem) // create a pet service account val googleProject = GoogleProject("testproject") - val petServiceAccount = googleExtensions.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() + val petServiceAccount = googleExtensions.petServiceAccounts.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() petServiceAccount.serviceAccount.email.value should endWith(s"@${googleProject.value}.iam.gserviceaccount.com") @@ -425,7 +425,7 @@ class GoogleExtensionSpec(_system: ActorSystem) mockGoogleDirectoryDAO.groups(defaultUserProxyEmail) shouldBe Set(defaultUser.email, petServiceAccount.serviceAccount.email) // create one again, it should work - val petSaResponse2 = googleExtensions.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() + val petSaResponse2 = googleExtensions.petServiceAccounts.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() petSaResponse2 shouldBe petServiceAccount // delete the pet service account @@ -503,7 +503,7 @@ class GoogleExtensionSpec(_system: ActorSystem) ) = initPetTest val googleProject = GoogleProject("testproject") - val (saName, saDisplayName) = googleExtensions.toPetSAFromUser(defaultUser) + val (saName, saDisplayName) = googleExtensions.petServiceAccounts.toPetSAFromUser(defaultUser) val serviceAccount = mockGoogleIamDAO.createServiceAccount(googleProject, saName, saDisplayName).futureValue // create a user @@ -511,7 +511,7 @@ class GoogleExtensionSpec(_system: ActorSystem) newUser shouldBe UserStatus(UserStatusDetails(defaultUser.id, defaultUser.email), TestSupport.enabledMapTosAccepted) // create a pet service account - val petServiceAccount = googleExtensions.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() + val petServiceAccount = googleExtensions.petServiceAccounts.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() petServiceAccount.serviceAccount shouldBe serviceAccount } @@ -536,13 +536,13 @@ class GoogleExtensionSpec(_system: ActorSystem) // create a pet service account val googleProject = GoogleProject("testproject") - val petServiceAccount = googleExtensions.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() + val petServiceAccount = googleExtensions.petServiceAccounts.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() import org.broadinstitute.dsde.workbench.model.google.toAccountName mockGoogleIamDAO.removeServiceAccount(googleProject, toAccountName(petServiceAccount.serviceAccount.email)).futureValue mockGoogleIamDAO.findServiceAccount(googleProject, petServiceAccount.serviceAccount.email).futureValue shouldBe None - val petServiceAccount2 = googleExtensions.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() + val petServiceAccount2 = googleExtensions.petServiceAccounts.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() petServiceAccount.serviceAccount shouldNot equal(petServiceAccount2.serviceAccount) val res = dirDAO.loadPetServiceAccount(petServiceAccount.id, samRequestContext).unsafeRunSync() res shouldBe Some(petServiceAccount2) @@ -1288,12 +1288,12 @@ class GoogleExtensionSpec(_system: ActorSystem) // create a pet service account val googleProject = GoogleProject("testproject") - val petServiceAccount = googleExtensions.createUserPetServiceAccount(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() + val petServiceAccount = googleExtensions.petServiceAccounts.createUserPetServiceAccount(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() // get a key, which should create a brand new one - val firstKey = googleExtensions.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() + val firstKey = googleExtensions.petServiceAccounts.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() // get a key again, which should return the original cached key created above - val secondKey = googleExtensions.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() + val secondKey = googleExtensions.petServiceAccounts.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() assert(firstKey == secondKey) } @@ -1313,23 +1313,23 @@ class GoogleExtensionSpec(_system: ActorSystem) // create a pet service account val googleProject = GoogleProject("testproject") - val petServiceAccount = googleExtensions.createUserPetServiceAccount(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() + val petServiceAccount = googleExtensions.petServiceAccounts.createUserPetServiceAccount(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() // get a key, which should create a brand new one - val firstKey = googleExtensions.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() + val firstKey = googleExtensions.petServiceAccounts.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() // remove the key we just created runAndWait(for { keys <- googleExtensions.googleIamDAO.listServiceAccountKeys(googleProject, petServiceAccount.serviceAccount.email) _ <- keys.toList .parTraverse { key => - googleExtensions.removePetServiceAccountKey(createDefaultUser.id, googleProject, key.id, samRequestContext) + googleExtensions.petServiceAccounts.removePetServiceAccountKey(createDefaultUser.id, googleProject, key.id, samRequestContext) } .unsafeToFuture() } yield ()) // get a key again, which should once again create a brand new one because we've deleted the cached one - val secondKey = googleExtensions.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() + val secondKey = googleExtensions.petServiceAccounts.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() assert(firstKey != secondKey) } @@ -1350,10 +1350,10 @@ class GoogleExtensionSpec(_system: ActorSystem) // create a pet service account val googleProject = GoogleProject("testproject") - val petServiceAccount = googleExtensions.createUserPetServiceAccount(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() + val petServiceAccount = googleExtensions.petServiceAccounts.createUserPetServiceAccount(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() // get a key, which should create a brand new one - val firstKey = googleExtensions.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() + val firstKey = googleExtensions.petServiceAccounts.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() // remove the key we just created behind the scenes val removedKeyObjects = (for { @@ -1375,7 +1375,7 @@ class GoogleExtensionSpec(_system: ActorSystem) // get a key again, which should once again create a brand new one because we've deleted the cached one // and all the keys removed should have been removed from google - val secondKey = googleExtensions.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() + val secondKey = googleExtensions.petServiceAccounts.getPetServiceAccountKey(createDefaultUser, googleProject, samRequestContext).unsafeRunSync() // assert that keys have been removed from service account assert(removedKeyObjects.forall { removed => @@ -2017,7 +2017,7 @@ class GoogleExtensionSpec(_system: ActorSystem) val googleProject = GoogleProject("testproject") val report = intercept[WorkbenchExceptionWithErrorReport] { - googleExtensions.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() + googleExtensions.petServiceAccounts.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() } report.errorReport.statusCode shouldEqual Some(StatusCodes.BadRequest) @@ -2061,7 +2061,7 @@ class GoogleExtensionSpec(_system: ActorSystem) val googleProject = GoogleProject("testproject") val report = intercept[WorkbenchExceptionWithErrorReport] { - googleExtensions.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() + googleExtensions.petServiceAccounts.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() } report.errorReport.statusCode shouldEqual Some(StatusCodes.BadRequest) @@ -2105,7 +2105,7 @@ class GoogleExtensionSpec(_system: ActorSystem) val googleProject = GoogleProject("testproject") val report = intercept[WorkbenchExceptionWithErrorReport] { - googleExtensions.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() + googleExtensions.petServiceAccounts.createUserPetServiceAccount(defaultUser, googleProject, samRequestContext).unsafeRunSync() } report.errorReport.statusCode shouldEqual Some(StatusCodes.BadRequest) diff --git a/src/test/scala/org/broadinstitute/dsde/workbench/sam/google/NewGoogleExtensionsSpec.scala b/src/test/scala/org/broadinstitute/dsde/workbench/sam/google/NewGoogleExtensionsSpec.scala index c25faefbea..ae0e412986 100644 --- a/src/test/scala/org/broadinstitute/dsde/workbench/sam/google/NewGoogleExtensionsSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/workbench/sam/google/NewGoogleExtensionsSpec.scala @@ -90,10 +90,16 @@ class NewGoogleExtensionsSpec(_system: ActorSystem) ) ) + val petServiceAccounts = mock[PetServiceAccounts](RETURNS_SMART_NULLS) + doReturn(IO.pure(petServiceAccount)) - .when(googleExtensions) + .when(petServiceAccounts) .createUserPetServiceAccount(eqTo(newGoogleUser), eqTo(googleProject), any[SamRequestContext]) + doReturn(petServiceAccounts) + .when(googleExtensions) + .petServiceAccounts + doReturn(IO.pure(petServiceAccountKey)) .when(mockGoogleKeyCache) .getKey(eqTo(petServiceAccount)) @@ -117,7 +123,7 @@ class NewGoogleExtensionsSpec(_system: ActorSystem) signedUrl should be(expectedUrl) - verify(googleExtensions).createUserPetServiceAccount(eqTo(newGoogleUser), eqTo(googleProject), any[SamRequestContext]) + verify(petServiceAccounts).createUserPetServiceAccount(eqTo(newGoogleUser), eqTo(googleProject), any[SamRequestContext]) verify(mockGoogleKeyCache).getKey(eqTo(petServiceAccount)) @@ -167,9 +173,11 @@ class NewGoogleExtensionsSpec(_system: ActorSystem) val arbitraryPetServiceAccountKey = RealKeyMockGoogleIamDAO.generateNewRealKey(arbitraryPetServiceAccount.serviceAccount.email)._2 doReturn(Future.successful(arbitraryPetServiceAccountKey)) - .when(googleExtensions) + .when(petServiceAccounts) .getArbitraryPetServiceAccountKey(eqTo(newGoogleUser), any[SamRequestContext]) + doReturn(petServiceAccounts).when(googleExtensions).petServiceAccounts + "includes the requester pays user project if provided" in { runAndWait( googleExtensions.getRequesterPaysSignedUrl( diff --git a/src/test/scala/org/broadinstitute/dsde/workbench/sam/service/ResourceServiceUnitSpec.scala b/src/test/scala/org/broadinstitute/dsde/workbench/sam/service/ResourceServiceUnitSpec.scala index 2487bc3d63..ec41af5f11 100644 --- a/src/test/scala/org/broadinstitute/dsde/workbench/sam/service/ResourceServiceUnitSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/workbench/sam/service/ResourceServiceUnitSpec.scala @@ -30,6 +30,16 @@ class ResourceServiceUnitSpec extends AnyFlatSpec with Matchers with ScalaFuture private val nothingRoleName = ResourceRoleName("cantDoNuthin") private val authDomainGroup1 = WorkbenchGroupName("authDomain1") private val authDomainGroup2 = WorkbenchGroupName("authDomain2") + private val resourceType = ResourceType( + resourceTypeName, + Set(ResourceActionPattern(readAction.value, "read action", false), ResourceActionPattern(writeAction.value, "write action", false)), + Set( + ResourceRole(readerRoleName, Set(readAction), Set.empty, Map.empty), + ResourceRole(ownerRoleName, Set(readAction, writeAction), Set.empty, Map.empty), + ResourceRole(nothingRoleName, Set.empty, Set.empty, Map.empty) + ), + ownerRoleName + ) val testResourceId = ResourceId(UUID.randomUUID().toString) val testResourceId2 = ResourceId(UUID.randomUUID().toString) @@ -160,12 +170,14 @@ class ResourceServiceUnitSpec extends AnyFlatSpec with Matchers with ScalaFuture ) .thenReturn(IO.pure(dbResult)) + val mockCloudExtensions = spy[CloudExtensions](NoExtensions, true) + val resourceService = new ResourceService( - Map.empty, + Map(resourceTypeName -> resourceType), mock[PolicyEvaluatorService], mockAccessPolicyDAO, mock[DirectoryDAO], - NoExtensions, + mockCloudExtensions, emailDomain, Set("test.firecloud.org") ) @@ -221,4 +233,20 @@ class ResourceServiceUnitSpec extends AnyFlatSpec with Matchers with ScalaFuture authDomainResource.authDomainGroups should be(Set(authDomainGroup1, authDomainGroup2)) authDomainResource.missingAuthDomainGroups should be(Set(authDomainGroup2)) } + + it should "delete Action Service Accounts associated with a resource on resource delete" in { + when(mockAccessPolicyDAO.listResourceChildren(any[FullyQualifiedResourceId], any[SamRequestContext])) + .thenReturn(IO.pure(Set.empty)) + when(mockAccessPolicyDAO.listAccessPolicies(any[FullyQualifiedResourceId], any[SamRequestContext])) + .thenReturn(IO.pure(LazyList.empty)) + when(mockAccessPolicyDAO.deleteResource(any[FullyQualifiedResourceId], any[Boolean], any[SamRequestContext])) + .thenReturn(IO.unit) + when(mockAccessPolicyDAO.deleteResourceParent(any[FullyQualifiedResourceId], any[SamRequestContext])) + .thenReturn(IO.pure(true)) + when(mockAccessPolicyDAO.checkPolicyGroupsInUse(any[FullyQualifiedResourceId], any[SamRequestContext])) + .thenReturn(IO.pure(List.empty)) + + resourceService.deleteResource(FullyQualifiedResourceId(resourceTypeName, testResourceId), samRequestContext).unsafeRunSync() + verify(mockCloudExtensions).onResourceDelete(testResourceId, samRequestContext) + } }