Add a user-database and constrain access to known users. (unitycatalog#387)

**PR Checklist**

- [X] A description of the changes is added to the description of this
PR.
- [x] If there is a related issue, make sure it is linked to this PR.
- [x] If you've fixed a bug or added code that should be tested, add
tests!
- [ ] If you've added or modified a feature, documentation in `docs` is
updated

**Description of changes**

This enhancement adds a user database to the platform, a control API for
user management, CLI commands to manage users, and when authentication
is enabled, will constrain access to only the users in the user
database.

Related Issue unitycatalog#229

The user record is simple for this first implementation
- id - unique identifier for the user
- name - The user's full name
- email - The user's primary email address
- picture_url - A link to an avatar for the user
- external_id - An identifier for external identity providers to track
the user
- created_at - the time the record was created
- updated_at - the last time the record was updated

There is the beginning of a Control API which is meant to house
operations that are outside of the core of the Unity Catalog
specification.

The Control API initially provides endpoints to do user management. The
endpoints are SCIM 2.0 compatible, as such they should allow integration
with third party identity providers for syncing users between systems.

The CLI enhancements add commands to add, update and remove users, see
command line help for details, but as an example, to add a user
```
uc user create --name "Iggy Pop" --email [email protected] 
```

Finally, when the recently added OAuth authorization functionality is
enabled, users must exist in the user database in order to gain access
to the server operations.

---------

Signed-off-by: Mocker <[email protected]>
Co-authored-by: Xiang Xu <[email protected]>
Co-authored-by: Ramesh Chandra <[email protected]>

Author: 3 people

api/control.yaml

@@ -0,0 +1,261 @@
+openapi: 3.0.0
+servers:
+  - url: http://localhost:8080/api/1.0/unity-control
+    description: Localhost reference server
+tags:
+  - name: Users
+    description: |
+      A user is the identity recognized by Unity Catalog.
+
+paths:
+  /scim2/Users:
+    post:
+      tags:
+        - Users
+      operationId: createUser
+      summary: Create a user
+      description: |
+        Creates a new user.
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/UserResource'
+      responses:
+        '200':
+          description: The new user was successfully created.
+          content:
+            application/scim+json:
+              schema:
+                $ref: '#/components/schemas/UserResource'
+    get:
+      tags:
+        - Users
+      parameters:
+        - name: filter
+          in: query
+          description: Query by which the results have to be filtered.
+          schema:
+            type: string
+          required: false
+        - name: startIndex
+          in: query
+          description: Specifies the index of the first result. First item is number 1.
+          schema:
+            type: integer
+            format: int32
+          required: false
+        - name: count
+          in: query
+          description: Desired number of results per page. If no count is provided, it defaults to 50.
+          schema:
+            type: integer
+            format: int32
+          required: false
+      operationId: listUsers
+      summary: List users
+      description: |
+        Gets details for all the users.
+      responses:
+        '200':
+          description: The user list was successfully retrieved.
+          content:
+            application/scim+json:
+              schema:
+                $ref: '#/components/schemas/UserResourceList'
+  /scim2/Users/{id}:
+    parameters:
+      - name: id
+        in: path
+        description: The id of the user.
+        required: true
+        schema:
+          type: string
+    get:
+      tags:
+        - Users
+      operationId: getUser
+      summary: Get a user
+      description: |
+        Gets the specified user.
+      responses:
+        '200':
+          description: The user was successfully retrieved.
+          content:
+            application/scim+json:
+              schema:
+                $ref: '#/components/schemas/UserResource'
+    put:
+      tags:
+        - Users
+      operationId: updateUser
+      summary: Update a user
+      description: |
+        Updates the user that matches the supplied id.
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/UserResource'
+      responses:
+        '200':
+          description: The user was successfully updated.
+          content:
+            application/scim+json:
+              schema:
+                $ref: '#/components/schemas/UserResource'
+    delete:
+      tags:
+        - Users
+      operationId: deleteUser
+      summary: Delete a user
+      description: |
+        Deletes the user that matches the supplied id.
+      responses:
+        '200':
+          description: The user was successfully deleted.
+          content:
+            application/scim+json:
+              schema: {}
+  /scim2/Users/self:
+    get:
+      tags:
+        - Users
+      operationId: getSelf
+      summary: Get the current user
+      description: |
+        Gets the user from the jwt token provided.
+      responses:
+        '200':
+          description: The user was successfully retrieved.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/UserResource'
+
+components:
+  schemas:
+    User:
+      properties:
+        id:
+          description: The unique identifier of the user.
+          type: string
+        name:
+          description: The name of the user.
+          type: string
+        email:
+          description: The email address of the user.
+          type: string
+        external_id:
+          description: The external identifier of the user.
+          type: string
+        state:
+          description: The state of the account.
+          type: string
+          enum:
+            - ENABLED
+            - DISABLED
+        picture_url:
+          description: The URL of the user's profile picture.
+          type: string
+        created_at:
+          description: The time the user was created.
+          type: integer
+          format: int64
+        updated_at:
+          description: The time the user was last updated.
+          type: integer
+          format: int64
+      type: object
+      required:
+        - id
+        - name
+        - email
+    UserResourceList:
+      description: List of SCIM User resources.
+      type: object
+      properties:
+        totalResults:
+          description: The total number of results.
+          type: integer
+          format: int32
+        itemsPerPage:
+            description: The number of items per page.
+            type: integer
+            format: int32
+        startIndex:
+            description: The index of the first result.
+            type: integer
+            format: int32
+        Resources:
+            description: The list of User resources.
+            type: array
+            items:
+                $ref: '#/components/schemas/UserResource'
+        id:
+            description: User list id metadata.
+            type: string
+        externalId:
+            description: User list external id metadata.
+            type: string
+        meta:
+          description: The metadata of the user.
+          type: object
+          properties:
+            resourceType:
+              type: string
+            created:
+              type: string
+            lastModified:
+              type: string
+    UserResource:
+      description: SCIM provides a resource type for "User" resources.
+      type: object
+      properties:
+        id:
+          description: The id of the user.
+          type: string
+        displayName:
+          description: The name of the user.
+          type: string
+        externalId:
+          description: The SCIM external id.
+          type: string
+        emails:
+          description: E-mail addresses for the user.
+          type: array
+          items:
+            $ref: '#/components/schemas/Email'
+        active:
+            description: The active status of the user.
+            type: boolean
+        meta:
+          description: The metadata of the user.
+          type: object
+          properties:
+            created:
+              type: string
+            lastModified:
+              type: string
+        photos:
+          description: The photos of the user.
+          type: array
+          items:
+            type: object
+            properties:
+              value:
+                description: The url of the user's photo.
+                type: string
+    Email:
+      description: SCIM email for a user.
+      type: object
+      properties:
+        value:
+          description: The email of the user.
+          type: string
+        primary:
+          description: If the email is primary.
+          type: boolean
+info:
+  title: Unity Control API
+  version: '0.1'

build.sbt

@@ -79,6 +79,8 @@ lazy val commonSettings = Seq(
     // https://en.wikipedia.org/wiki/GNU_Lesser_General_Public_License
     // We can use and distribute the, but not modify the source code
     case DepModuleInfo("org.hibernate.orm", _, _) => true
+    case DepModuleInfo("com.unboundid.scim2", _, _) => true
+    case DepModuleInfo("com.unboundid.product.scim2", _, _) => true
     // Duo license:
     //  - Eclipse Public License 2.0
     //  - GNU General Public License, version 2 with the GNU Classpath Exception
@@ -112,9 +114,52 @@ def javafmtCheckSettings() = Seq(
   (Compile / compile) := ((Compile / compile) dependsOn (Compile / javafmtCheckAll)).value
 )
 
+lazy val controlApi = (project in file("target/control/java"))
+  .enablePlugins(OpenApiGeneratorPlugin)
+  .disablePlugins(JavaFormatterPlugin)
+  .settings(
+    name := s"$artifactNamePrefix-controlapi",
+    commonSettings,
+    javaOnlyReleaseSettings,
+    libraryDependencies ++= Seq(
+      "jakarta.annotation" % "jakarta.annotation-api" % "3.0.0" % Provided,
+      "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion,
+      "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
+      "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion,
+    ),
+    (Compile / compile) := ((Compile / compile) dependsOn generate).value,
+
+    // OpenAPI generation specs
+    openApiInputSpec := (file(".") / "api" / "control.yaml").toString,
+    openApiGeneratorName := "java",
+    openApiOutputDir := (file("target") / "control" / "java").toString,
+    openApiApiPackage := s"$orgName.control.api",
+    openApiModelPackage := s"$orgName.control.model",
+    openApiAdditionalProperties := Map(
+      "library" -> "native",
+      "useJakartaEe" -> "true",
+      "hideGenerationTimestamp" -> "true",
+      "openApiNullable" -> "false"),
+    openApiGenerateApiTests := SettingDisabled,
+    openApiGenerateModelTests := SettingDisabled,
+    openApiGenerateApiDocumentation := SettingDisabled,
+    openApiGenerateModelDocumentation := SettingDisabled,
+    // Define the simple generate command to generate full client codes
+    generate := {
+      val _ = openApiGenerate.value
+
+      // Delete the generated build.sbt file so that it is not used for our sbt config
+      val buildSbtFile = file(openApiOutputDir.value) / "build.sbt"
+      if (buildSbtFile.exists()) {
+        buildSbtFile.delete()
+      }
+    }
+  )
+
 lazy val client = (project in file("target/clients/java"))
   .enablePlugins(OpenApiGeneratorPlugin)
   .disablePlugins(JavaFormatterPlugin)
+  .dependsOn(controlApi % "compile->compile")
   .settings(
     name := s"$artifactNamePrefix-client",
     commonSettings,
@@ -206,8 +251,6 @@ lazy val server = (project in file("server"))
       "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
       "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonVersion,
       "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion,
-      "com.auth0" % "java-jwt" % "4.4.0",
-      "com.auth0" % "jwks-rsa" % "0.22.1",
 
       "com.google.code.findbugs" % "jsr305" % "3.0.2",
       "com.h2database" %  "h2" % "2.2.224",
@@ -242,6 +285,12 @@ lazy val server = (project in file("server"))
       "io.vertx" % "vertx-web" % "4.3.5",
       "io.vertx" % "vertx-web-client" % "4.3.5",
 
+      // Auth dependencies
+      "com.unboundid.product.scim2" % "scim2-sdk-common" % "3.1.0",
+      "org.springframework" % "spring-expression" % "6.1.11",
+      "com.auth0" % "java-jwt" % "4.4.0",
+      "com.auth0" % "jwks-rsa" % "0.22.1",
+
       // Test dependencies
       "org.junit.jupiter" %  "junit-jupiter" % "5.10.3" % Test,
       "org.mockito" % "mockito-core" % "5.11.0" % Test,
@@ -278,6 +327,7 @@ lazy val server = (project in file("server"))
 lazy val serverModels = (project in file("server") / "target" / "models")
   .enablePlugins(OpenApiGeneratorPlugin)
   .disablePlugins(JavaFormatterPlugin)
+  .dependsOn(controlModels % "compile->compile")
   .settings(
     name := s"$artifactNamePrefix-servermodels",
     commonSettings,
@@ -310,6 +360,41 @@ lazy val serverModels = (project in file("server") / "target" / "models")
     }
   )
 
+lazy val controlModels = (project in file("server") / "target" / "controlmodels")
+  .enablePlugins(OpenApiGeneratorPlugin)
+  .disablePlugins(JavaFormatterPlugin)
+  .settings(
+    name := s"$artifactNamePrefix-controlmodels",
+    commonSettings,
+    (Compile / compile) := ((Compile / compile) dependsOn generate).value,
+    Compile / compile / javacOptions ++= javacRelease17,
+    libraryDependencies ++= Seq(
+      "jakarta.annotation" % "jakarta.annotation-api" % "3.0.0" % Provided,
+      "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion,
+    ),
+    // OpenAPI generation configs for generating model codes from the spec
+    openApiInputSpec := (file(".") / "api" / "control.yaml").toString,
+    openApiGeneratorName := "java",
+    openApiOutputDir := (file("server") / "target" / "controlmodels").toString,
+    openApiValidateSpec := SettingEnabled,
+    openApiGenerateMetadata := SettingDisabled,
+    openApiModelPackage := s"$orgName.control.model",
+    openApiAdditionalProperties := Map(
+      "library" -> "resteasy", // resteasy generates the most minimal models
+      "useJakartaEe" -> "true",
+      "hideGenerationTimestamp" -> "true"
+    ),
+    openApiGlobalProperties := Map("models" -> ""),
+    openApiGenerateApiTests := SettingDisabled,
+    openApiGenerateModelTests := SettingDisabled,
+    openApiGenerateApiDocumentation := SettingDisabled,
+    openApiGenerateModelDocumentation := SettingDisabled,
+    // Define the simple generate command to generate model codes
+    generate := {
+      val _ = openApiGenerate.value
+    }
+  )
+
 lazy val cli = (project in file("examples") / "cli")
   .dependsOn(server % "test->test")
   .dependsOn(client % "compile->compile;test->test")