Fix mongo indexes (#6122)

Signed-off-by: Andrey Sobolev <[email protected]>

Author: haiodo

Diff for: dev/tool/src/index.ts

@@ -46,7 +46,7 @@ import {
   createStorageBackupStorage,
   restore
 } from '@hcengineering/server-backup'
-import serverClientPlugin, { BlobClient, getTransactorEndpoint } from '@hcengineering/server-client'
+import serverClientPlugin, { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client'
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
 import toolPlugin from '@hcengineering/server-tool'
 
@@ -244,13 +244,29 @@ export function devTool (
       const { mongodbUri } = prepareTools()
       await withDatabase(mongodbUri, async (db, client) => {
         console.log(`assigning user ${email} to ${workspace}...`)
-        const workspaceInfo = await getWorkspaceById(db, productId, workspace)
-        if (workspaceInfo === null) {
-          throw new Error(`workspace ${workspace} not found`)
-        }
-        console.log('assigning to workspace', workspaceInfo)
         try {
-          await assignWorkspace(toolCtx, db, productId, null, email, workspaceInfo.workspace, AccountRole.User)
+          const workspaceInfo = await getWorkspaceById(db, productId, workspace)
+          if (workspaceInfo === null) {
+            throw new Error(`workspace ${workspace} not found`)
+          }
+          const token = generateToken(systemAccountEmail, { name: workspaceInfo.workspace, productId })
+          const endpoint = await getTransactorEndpoint(token, 'external')
+          console.log('assigning to workspace', workspaceInfo, endpoint)
+          const client = await createClient(endpoint, token)
+          console.log('assigning to workspace connected', workspaceInfo, endpoint)
+          await assignWorkspace(
+            toolCtx,
+            db,
+            productId,
+            null,
+            email,
+            workspaceInfo.workspace,
+            AccountRole.User,
+            undefined,
+            undefined,
+            client
+          )
+          await client.close()
         } catch (err: any) {
           console.error(err)
         }
@@ -328,7 +344,16 @@ export function devTool (
       const { mongodbUri } = prepareTools()
       console.log(`set user ${email} role for ${workspace}...`)
       await withDatabase(mongodbUri, async (db) => {
-        await setRole(toolCtx, db, email, workspace, productId, role)
+        const workspaceInfo = await getWorkspaceById(db, productId, workspace)
+        if (workspaceInfo === null) {
+          throw new Error(`workspace ${workspace} not found`)
+        }
+        console.log('assigning to workspace', workspaceInfo)
+        const token = generateToken(systemAccountEmail, { name: workspaceInfo.workspace, productId })
+        const endpoint = await getTransactorEndpoint(token, 'external')
+        const client = await createClient(endpoint, token)
+        await setRole(toolCtx, db, email, workspace, productId, role, client)
+        await client.close()
       })
     })
 

Diff for: models/activity/src/index.ts

@@ -362,10 +362,7 @@ export function createModel (builder: Builder): void {
 
   builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
     domain: DOMAIN_ACTIVITY,
-    indexes: [
-      { attachedTo: 1, createdOn: 1 },
-      { attachedTo: 1, createdOn: -1 }
-    ],
+    indexes: [{ keys: { attachedTo: 1, createdOn: 1 } }, { keys: { attachedTo: 1, createdOn: -1 } }],
     disabled: [
       { modifiedOn: 1 },
       { createdOn: -1 },

Diff for: models/core/src/core.ts

@@ -36,7 +36,7 @@ import {
   type DomainIndexConfiguration,
   type Enum,
   type EnumOf,
-  type FieldIndex,
+  type FieldIndexConfig,
   type FullTextSearchContext,
   type IndexStageState,
   type IndexingConfiguration,
@@ -134,7 +134,7 @@ export class TAttachedDoc extends TDoc implements AttachedDoc {
 export class TBlob extends TDoc implements Blob {
   @Prop(TypeString(), core.string.Blob)
   @ReadOnly()
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
     provider!: string
 
   @Prop(TypeString(), core.string.BlobContentType)
@@ -340,7 +340,6 @@ export class TDocIndexState extends TDoc implements DocIndexState {
     stages!: Record<string, boolean | string>
 
   @Prop(TypeString(), getEmbeddedLabel('Generation'))
-  @Index(IndexKind.Indexed)
   @Hidden()
     generationId?: string
 }
@@ -371,7 +370,7 @@ export class TConfiguration extends TDoc implements Configuration {
 
 @MMixin(core.mixin.IndexConfiguration, core.class.Class)
 export class TIndexConfiguration<T extends Doc = Doc> extends TClass implements IndexingConfiguration<T> {
-  indexes!: FieldIndex<T>[]
+  indexes!: (string | FieldIndexConfig<T>)[]
   searchDisabled!: boolean
 }
 

Diff for: models/core/src/index.ts

@@ -194,26 +194,32 @@ export function createModel (builder: Builder): void {
     core.class.Class,
     core.mixin.IndexConfiguration,
     {
-      indexes: [
-        'tx.objectId',
-        'tx.operations.attachedTo',
-        'space',
-        {
+      indexes: ['tx.objectId', 'tx.operations.attachedTo']
+    }
+  )
+  builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
+    domain: DOMAIN_TX,
+    disabled: [
+      { space: 1 },
+      { objectClass: 1 },
+      { createdBy: 1 },
+      { createdBy: -1 },
+      { createdOn: -1 },
+      { modifiedBy: 1 },
+      { objectSpace: 1 }
+    ],
+    indexes: [
+      {
+        keys: {
           objectSpace: 1,
           _id: 1,
           modifiedOn: 1
         },
-        {
-          objectSpace: 1,
-          modifiedBy: 1,
-          objectClass: 1
+        filter: {
+          objectSpace: core.space.Model
         }
-      ]
-    }
-  )
-  builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
-    domain: DOMAIN_TX,
-    disabled: [{ space: 1 }, { objectClass: 1 }, { createdBy: 1 }, { createdBy: -1 }, { createdOn: -1 }]
+      }
+    ]
   })
 
   builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
@@ -299,20 +305,13 @@ export function createModel (builder: Builder): void {
     {
       indexes: [
         {
-          _class: 1,
-          stages: 1,
-          _id: 1,
-          modifiedOn: 1
-        },
-        {
-          _class: 1,
-          _id: 1,
-          modifiedOn: 1
-        },
-        {
-          _class: 1,
-          _id: 1,
-          objectClass: 1
+          keys: {
+            _class: 1,
+            stages: 1,
+            _id: 1,
+            modifiedOn: 1
+          },
+          sparse: true
         }
       ]
     }

Diff for: models/core/src/tx.ts

@@ -46,7 +46,7 @@ import { TDoc } from './core'
 @Model(core.class.Tx, core.class.Doc, DOMAIN_TX)
 export class TTx extends TDoc implements Tx {
   @Prop(TypeRef(core.class.Space), core.string.Space)
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
   @Hidden()
     objectSpace!: Ref<Space>
 }
@@ -62,7 +62,7 @@ export class TTxCUD<T extends Doc> extends TTx implements TxCUD<T> {
     objectId!: Ref<T>
 
   @Prop(TypeRef(core.class.Class), core.string.ClassLabel)
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
   @Hidden()
     objectClass!: Ref<Class<T>>
 }

Diff for: models/notification/src/index.ts

@@ -619,9 +619,10 @@ export function createModel (builder: Builder): void {
     },
     presenter: notification.component.ReactionNotificationPresenter
   })
+
   builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
     domain: DOMAIN_NOTIFICATION,
-    indexes: [{ user: 1, archived: 1 }],
+    indexes: [{ keys: { user: 1, archived: 1 } }],
     disabled: [{ modifiedOn: 1 }, { modifiedBy: 1 }, { createdBy: 1 }, { isViewed: 1 }, { hidden: 1 }]
   })
 

Diff for: packages/core/src/classes.ts

@@ -15,6 +15,7 @@
 //
 
 import type { Asset, IntlString, Plugin } from '@hcengineering/platform'
+import type { DocumentQuery } from './storage'
 
 /**
  * @public
@@ -122,6 +123,7 @@ export enum IndexKind {
    * Also mean to include into Elastic search.
    */
   Indexed,
+
   // Same as indexed but for descending
   IndexedDsc
 }
@@ -623,14 +625,20 @@ export type FieldIndex<T extends Doc> = {
   [P in keyof T]?: IndexOrder
 } & Record<string, IndexOrder>
 
+export interface FieldIndexConfig<T extends Doc> {
+  sparse?: boolean
+  filter?: Omit<DocumentQuery<T>, '$search'>
+  keys: FieldIndex<T> | string
+}
+
 /**
  * @public
  *
  * Mixin for extra indexing fields.
  */
 export interface IndexingConfiguration<T extends Doc> extends Class<Doc> {
   // Define a list of extra index definitions.
-  indexes: (FieldIndex<T> | string)[]
+  indexes: (string | FieldIndexConfig<T>)[]
   searchDisabled?: boolean
 }
 
@@ -643,7 +651,7 @@ export interface DomainIndexConfiguration extends Doc {
   disabled?: (FieldIndex<Doc> | string)[]
 
   // Additional indexes we could like to enabled
-  indexes?: (FieldIndex<Doc> | string)[]
+  indexes?: (FieldIndexConfig<Doc> | string)[]
 
   skip?: string[]
 }

Diff for: packages/core/src/measurements/context.ts

@@ -11,12 +11,12 @@ export class MeasureMetricsContext implements MeasureContext {
   private readonly params: ParamsType
   logger: MeasureLogger
   metrics: Metrics
-  private readonly done: (value?: number) => void
+  private readonly done: (value?: number, override?: boolean) => void
 
   constructor (
     name: string,
     params: ParamsType,
-    fullParams: FullParamsType = {},
+    fullParams: FullParamsType | (() => FullParamsType) = {},
     metrics: Metrics = newMetrics(),
     logger?: MeasureLogger,
     readonly parent?: MeasureContext,
@@ -25,8 +25,21 @@ export class MeasureMetricsContext implements MeasureContext {
     this.name = name
     this.params = params
     this.metrics = metrics
+    this.metrics.namedParams = this.metrics.namedParams ?? {}
+    for (const [k, v] of Object.entries(params)) {
+      if (this.metrics.namedParams[k] !== v) {
+        this.metrics.namedParams[k] = v
+      } else {
+        this.metrics.namedParams[k] = '*'
+      }
+    }
     this.done = measure(metrics, params, fullParams, (spend) => {
-      this.logger.logOperation(this.name, spend, { ...params, ...fullParams, ...(this.logParams ?? {}) })
+      this.logger.logOperation(this.name, spend, {
+        ...params,
+        ...(typeof fullParams === 'function' ? fullParams() : fullParams),
+        ...fullParams,
+        ...(this.logParams ?? {})
+      })
     })
 
     const errorPrinter = ({ message, stack, ...rest }: Error): object => ({
@@ -63,12 +76,17 @@ export class MeasureMetricsContext implements MeasureContext {
     }
   }
 
-  measure (name: string, value: number): void {
+  measure (name: string, value: number, override?: boolean): void {
     const c = new MeasureMetricsContext('#' + name, {}, {}, childMetrics(this.metrics, ['#' + name]), this.logger, this)
-    c.done(value)
+    c.done(value, override)
   }
 
-  newChild (name: string, params: ParamsType, fullParams?: FullParamsType, logger?: MeasureLogger): MeasureContext {
+  newChild (
+    name: string,
+    params: ParamsType,
+    fullParams?: FullParamsType | (() => FullParamsType),
+    logger?: MeasureLogger
+  ): MeasureContext {
     return new MeasureMetricsContext(
       name,
       params,
@@ -84,7 +102,7 @@ export class MeasureMetricsContext implements MeasureContext {
     name: string,
     params: ParamsType,
     op: (ctx: MeasureContext) => T | Promise<T>,
-    fullParams?: ParamsType
+    fullParams?: ParamsType | (() => FullParamsType)
   ): Promise<T> {
     const c = this.newChild(name, params, fullParams, this.logger)
     try {