diff --git a/.vscode/launch.json b/.vscode/launch.json index d56a15e20fdd..0b0c8558a465 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -627,7 +627,7 @@ ] }, { - "name": "[perftests] Changeset PerfTests (Offline)", + "name": "[perftests] Sqlite Changeset Reader and ChangesetECAdaptor PerfTests (Offline)", "presentation": { "group": "Perftests" }, @@ -637,7 +637,24 @@ "runtimeExecutable": "npm", "runtimeArgs": [ "run", - "perftest:changesetPerformance" + "perftest:sqliteChangesetReaderAndChangesetECAdaptorPerformance" + ], + "outFiles": [ + "${workspaceFolder}/full-stack-tests/*/lib/**/*.js" + ] + }, + { + "name": "[perftests] ChangesetReader PerfTests (Offline)", + "presentation": { + "group": "Perftests" + }, + "cwd": "${workspaceFolder}/full-stack-tests/backend", + "type": "node", + "request": "launch", + "runtimeExecutable": "npm", + "runtimeArgs": [ + "run", + "perftest:changesetReaderPerformance" ], "outFiles": [ "${workspaceFolder}/full-stack-tests/*/lib/**/*.js" diff --git a/common/api/core-backend.api.md b/common/api/core-backend.api.md index 7596529bc4a5..22de98232d47 100644 --- a/common/api/core-backend.api.md +++ b/common/api/core-backend.api.md @@ -858,6 +858,14 @@ export class CategorySelector extends DefinitionElement { } // @beta +export interface ChangeCache extends Disposable { + all(): IterableIterator; + count(): number; + get(key: string): ChangeInstance | undefined; + set(key: string, value: ChangeInstance): void; +} + +// @beta @deprecated export interface ChangedECInstance { // (undocumented) $meta?: ChangeMetaData; @@ -899,6 +907,12 @@ export interface ChangeFormatArgs { includeTableName?: true; } +// @beta +export interface ChangeInstance { + $meta: ChangeMeta; + [key: string]: any; +} + // @beta export interface ChangeInstanceKey { changeType: "inserted" | "updated" | "deleted"; @@ -907,6 +921,19 @@ export interface ChangeInstanceKey { } // @beta +export interface ChangeMeta { + changeFetchedPropNames: string[]; + changeIndexes: number[]; + instanceKey: string; + isIndirectChange: boolean; + op: SqliteChangeOp; + propFilter: PropertyFilter; + rowOptions?: RowFormatOptions; + stage: SqliteValueStage; + tables: string[]; +} + +// @beta @deprecated export interface ChangeMetaData { changeIndexes: number[]; classFullName?: string; @@ -922,7 +949,7 @@ export interface ChangesetArg extends IModelIdArg { readonly changeset: ChangesetIndexOrId; } -// @beta +// @beta @deprecated export class ChangesetECAdaptor implements Disposable { [Symbol.dispose](): void; constructor(reader: SqliteChangesetReader, disableMetaData?: boolean); @@ -960,6 +987,58 @@ export interface ChangesetRangeArg extends IModelIdArg { readonly range?: ChangesetRange; } +// @beta +export class ChangesetReader implements Disposable, ChangeSource { + [Symbol.dispose](): void; + clearClassNameFilters(): void; + clearOpCodeFilters(): void; + clearTableNameFilters(): void; + close(): void; + readonly db: AnyDb; + deleted?: ChangeInstance; + inserted?: ChangeInstance; + get isECTable(): boolean; + get isIndirectChange(): boolean; + get op(): SqliteChangeOp; + static openFile(args: { + readonly fileName: string; + } & ChangesetReaderArgs): ChangesetReader; + static openGroup(args: { + readonly changesetFiles: string[]; + } & ChangesetReaderArgs): ChangesetReader; + static openInMemoryChanges(args: Omit & { + db: IModelDb; + }): ChangesetReader; + static openLocalChanges(args: Omit & { + db: IModelDb; + includeInMemoryChanges?: boolean; + }): ChangesetReader; + static openTxn(args: Omit & { + db: IModelDb; + txnId: Id64String; + }): ChangesetReader; + setClassNameFilters(classNames: Set): void; + setOpCodeFilters(ops: Set): void; + setTableNameFilters(tableNames: Set): void; + step(): boolean; + get tableName(): string; +} + +// @beta +export interface ChangesetReaderArgs { + readonly db: AnyDb; + readonly invert?: boolean; + readonly propFilter?: PropertyFilter; + readonly rowOptions?: RowFormatOptions; +} + +// @beta +export interface ChangeSource { + readonly deleted?: ChangeInstance; + readonly inserted?: ChangeInstance; + readonly op: SqliteChangeOp; +} + // @beta export interface ChangeSummary { // (undocumented) @@ -994,6 +1073,12 @@ export class ChangeSummaryManager { static queryInstanceChange(iModel: BriefcaseDb, instanceChangeId: Id64String): InstanceChange; } +// @beta (undocumented) +export namespace ChangeUnifierCache { + export function createInMemoryCache(): ChangeCache; + export function createSqliteBackedCache(bufferedReadInstanceSizeInBytes?: number): ChangeCache; +} + // @beta export interface ChannelControl { // @internal (undocumented) @@ -2248,7 +2333,7 @@ export abstract class DriverBundleElement extends InformationContentElement { static get className(): string; } -// @beta +// @beta @deprecated export interface ECChangeUnifierCache extends Disposable { all(): IterableIterator; count(): number; @@ -2256,7 +2341,7 @@ export interface ECChangeUnifierCache extends Disposable { set(key: string, value: ChangedECInstance): void; } -// @beta (undocumented) +// @beta @deprecated (undocumented) export namespace ECChangeUnifierCache { export function createInMemoryCache(): ECChangeUnifierCache; export function createSqliteBackedCache(db: AnyDb, bufferedReadInstanceSizeInBytes?: number): ECChangeUnifierCache; @@ -5423,6 +5508,15 @@ export class OrthographicViewDefinition extends SpatialViewDefinition { export function parseTextAnnotationData(json: string | undefined): VersionedJSON | undefined; // @beta +export class PartialChangeUnifier implements Disposable { + [Symbol.dispose](): void; + constructor(_cache?: ChangeCache); + appendFrom(source: ChangeSource): void; + get instanceCount(): number; + get instances(): IterableIterator; +} + +// @beta @deprecated export class PartialECChangeUnifier implements Disposable { [Symbol.dispose](): void; constructor(_db: AnyDb, _cache?: ECChangeUnifierCache); @@ -5609,6 +5703,13 @@ export interface ProjectInformationRecordCreateArgs extends ProjectInformation { parentSubjectId: Id64String; } +// @beta +export enum PropertyFilter { + All = 0, + BisCoreElement = 1, + InstanceKey = 2 +} + // @public @preview export type PropertyHandler = (name: string, property: Property) => void; @@ -5970,6 +6071,13 @@ export class RoleModel extends Model { static get className(): string; } +// @beta +export interface RowFormatOptions { + abbreviateBlobs?: boolean; + classIdsToClassNames?: boolean; + useJsName?: boolean; +} + // @public export class RpcTrace { static get currentActivity(): RpcActivity | undefined; diff --git a/common/api/summary/core-backend.exports.csv b/common/api/summary/core-backend.exports.csv index 289045163c73..2ca420c514de 100644 --- a/common/api/summary/core-backend.exports.csv +++ b/common/api/summary/core-backend.exports.csv @@ -38,17 +38,27 @@ preview;class;Category public;class;CategoryOwnsSubCategories public;class;CategorySelector preview;class;CategorySelector +beta;interface;ChangeCache beta;interface;ChangedECInstance +deprecated;interface;ChangedECInstance internal;class;ChangedElementsDb beta;interface;ChangeFormatArgs +beta;interface;ChangeInstance beta;interface;ChangeInstanceKey +beta;interface;ChangeMeta beta;interface;ChangeMetaData +deprecated;interface;ChangeMetaData public;interface;ChangesetArg beta;class;ChangesetECAdaptor +deprecated;class;ChangesetECAdaptor internal;interface;ChangesetIndexArg public;interface;ChangesetRangeArg +beta;class;ChangesetReader +beta;interface;ChangesetReaderArgs +beta;interface;ChangeSource beta;interface;ChangeSummary beta;class;ChangeSummaryManager +beta;namespace;ChangeUnifierCache beta;interface;ChannelControl beta;namespace;ChannelControl beta;type;ChannelKey @@ -170,7 +180,9 @@ public;class;DrawingViewDefinition preview;class;DrawingViewDefinition beta;class;DriverBundleElement beta;interface;ECChangeUnifierCache +deprecated;interface;ECChangeUnifierCache beta;namespace;ECChangeUnifierCache +deprecated;namespace;ECChangeUnifierCache public;class;ECDb public;enum;ECDbOpenMode public;interface;ECEnumValue @@ -439,7 +451,9 @@ public;type;OpenBriefcaseArgs public;class;OrthographicViewDefinition preview;class;OrthographicViewDefinition internal;function;parseTextAnnotationData +beta;class;PartialChangeUnifier beta;class;PartialECChangeUnifier +deprecated;class;PartialECChangeUnifier public;class;PhysicalElement preview;class;PhysicalElement public;class;PhysicalElementAssemblesElements @@ -466,6 +480,7 @@ public;type;ProgressFunction public;enum;ProgressStatus beta;class;ProjectInformationRecord beta;interface;ProjectInformationRecordCreateArgs +beta;enum;PropertyFilter public;type;PropertyHandler preview;type;PropertyHandler beta;namespace;PropertyStore @@ -498,6 +513,7 @@ public;class;RoleElement preview;class;RoleElement public;class;RoleModel preview;class;RoleModel +beta;interface;RowFormatOptions public;class;RpcTrace beta;class;RunLayout public;class;Schema diff --git a/common/changes/@itwin/core-backend/soham-native-ec-reader_2026-04-23-15-29.json b/common/changes/@itwin/core-backend/soham-native-ec-reader_2026-04-23-15-29.json new file mode 100644 index 000000000000..c6843f29a914 --- /dev/null +++ b/common/changes/@itwin/core-backend/soham-native-ec-reader_2026-04-23-15-29.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@itwin/core-backend", + "comment": "Added ChangesetReader api(along with new unifier apis) and deprecated ChangesetECAdaptor apis(including unifier apis)", + "type": "none" + } + ], + "packageName": "@itwin/core-backend" +} \ No newline at end of file diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 92a339227eae..04436686fa06 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -27,8 +27,8 @@ importers: specifier: ^12.28.0 version: 12.28.0 '@bentley/imodeljs-native': - specifier: 5.9.11 - version: 5.9.11 + specifier: 5.9.14 + version: 5.9.14 '@itwin/object-storage-azure': specifier: ^3.0.4 version: 3.0.4 @@ -4757,8 +4757,8 @@ packages: '@bentley/icons-generic@1.0.34': resolution: {integrity: sha512-IIs1wDcY2oZ8tJ3EZRw0U51M+0ZL3MvwoDYYmhUXaa9/UZqpFoOyLBGaxjirQteWXqTIMm3mFvmC+Nbn1ok4Iw==} - '@bentley/imodeljs-native@5.9.11': - resolution: {integrity: sha512-L0317CpTSulu8et2Iul+vvwCGVswCLTBsjmWyZh+uAeG02nHHaCYWnrox6KffQB3VBeZfZJDtDkXtFzFnciMnA==} + '@bentley/imodeljs-native@5.9.14': + resolution: {integrity: sha512-7qPq0peUidZ6527et0jMEq3haz4JuIdpEUFfT6CiYDF2CZ76m74wW9qW+V3R4AwWgH9pkYe9pdcajD78BEINsQ==} '@bentley/linear-referencing-schema@2.0.3': resolution: {integrity: sha512-2pFIEN4BS7alIDhGous6N2icKAS8ZhVBfoWB8WvSFaYnneGv5YwbbXl46qATDdPO5jUFezkW6uVxdpp1eOgUHQ==} @@ -11358,7 +11358,7 @@ snapshots: '@bentley/icons-generic@1.0.34': {} - '@bentley/imodeljs-native@5.9.11': {} + '@bentley/imodeljs-native@5.9.14': {} '@bentley/linear-referencing-schema@2.0.3': {} diff --git a/core/backend/package.json b/core/backend/package.json index 085be2b6e3e5..59277a88dcdb 100644 --- a/core/backend/package.json +++ b/core/backend/package.json @@ -113,7 +113,7 @@ "webpack": "^5.97.1" }, "dependencies": { - "@bentley/imodeljs-native": "5.9.11", + "@bentley/imodeljs-native": "5.9.14", "@itwin/object-storage-azure": "^3.0.4", "@azure/storage-blob": "^12.28.0", "form-data": "^4.0.4", diff --git a/core/backend/src/BriefcaseManager.ts b/core/backend/src/BriefcaseManager.ts index a39e5cccd1fb..53d918a78167 100644 --- a/core/backend/src/BriefcaseManager.ts +++ b/core/backend/src/BriefcaseManager.ts @@ -49,7 +49,7 @@ interface PatchInstanceKey { */ interface ChangedInstanceForSemanticRebase { isIndirect: boolean; - instance: ChangedECInstance; + instance: ChangedECInstance; // eslint-disable-line @typescript-eslint/no-deprecated } /** The argument for patch instances during high level rebase application @@ -921,16 +921,19 @@ export class BriefcaseManager { const reader = SqliteChangesetReader.openTxn({ txnId, db, disableSchemaCheck: true }); + // eslint-disable-next-line @typescript-eslint/no-deprecated const adaptor = new ChangesetECAdaptor(reader); + // eslint-disable-next-line @typescript-eslint/no-deprecated using indirectUnifier = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); + // eslint-disable-next-line @typescript-eslint/no-deprecated using directUnifier = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); while (adaptor.step()) { if (adaptor.reader.isIndirect) - indirectUnifier.appendFrom(adaptor); + indirectUnifier.appendFrom(adaptor); // eslint-disable-line @typescript-eslint/no-deprecated else - directUnifier.appendFrom(adaptor); + directUnifier.appendFrom(adaptor); // eslint-disable-line @typescript-eslint/no-deprecated } - return [...Array.from(directUnifier.instances).map((instance) => ({ isIndirect: false, instance })), ...Array.from(indirectUnifier.instances).map((instance) => ({ isIndirect: true, instance }))]; + return [...Array.from(directUnifier.instances).map((instance) => ({ isIndirect: false, instance })), ...Array.from(indirectUnifier.instances).map((instance) => ({ isIndirect: true, instance }))]; // eslint-disable-line @typescript-eslint/no-deprecated } /** diff --git a/core/backend/src/ChangesetECAdaptor.ts b/core/backend/src/ChangesetECAdaptor.ts index 5c2146ac15c7..8c82db1c2e56 100644 --- a/core/backend/src/ChangesetECAdaptor.ts +++ b/core/backend/src/ChangesetECAdaptor.ts @@ -11,6 +11,8 @@ import { Base64EncodedString } from "@itwin/core-common"; import { ECDb } from "./ECDb"; import { _nativeDb } from "./internal/Symbols"; +/* eslint-disable @typescript-eslint/no-deprecated */ // This file is marked as deprecated and will be removed subsequently, so we can allow usage of deprecated APIs within it. + interface IClassRef { classId: Id64String; classFullName: string; @@ -352,6 +354,7 @@ class ECDbMap { /** * Record meta data for the change. * @beta + * @deprecated Use [ChangeMeta]($backend) with [ChangesetReader]($backend) instead. * */ export interface ChangeMetaData { /** list of tables making up this EC change */ @@ -371,6 +374,7 @@ export interface ChangeMetaData { /** * Represent EC change derived from low level sqlite change * @beta + * @deprecated Use [ChangeInstance]($backend) with [ChangesetReader]($backend) instead. */ export interface ChangedECInstance { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -384,6 +388,7 @@ export interface ChangedECInstance { /** * Helper function to convert between JS DateTime & SQLite JulianDay values. * @beta + * @deprecated The DateTime namespace is deprecated and will be removed in a future release. * */ namespace DateTime { /** @@ -411,6 +416,7 @@ namespace DateTime { /** * Represents a cache for unifying EC changes. * @beta + * @deprecated Use [ChangeCache]($backend) with [ChangesetReader]($backend) instead. */ export interface ECChangeUnifierCache extends Disposable { /** @@ -439,7 +445,10 @@ export interface ECChangeUnifierCache extends Disposable { */ count(): number; } -/** @beta */ +/** + * @beta + * @deprecated Use [ChangeUnifierCache.createInMemoryCache]($backend) / [ChangeUnifierCache.createSqliteBackedCache]($backend) instead. +*/ export namespace ECChangeUnifierCache { /** * Creates and returns a new in-memory cache for EC change unification. @@ -648,6 +657,7 @@ class SqliteBackedInstanceCache implements ECChangeUnifierCache { * Partial changes is per table and a single instance can * span multiple tables. * @beta + * @deprecated Use [PartialChangeUnifier]($backend) with [ChangesetReader]($backend) instead. */ export class PartialECChangeUnifier implements Disposable { private _readonly = false; @@ -815,6 +825,7 @@ export class PartialECChangeUnifier implements Disposable { * it is per table while a single instance can span multiple table. * @note PrimitiveArray and StructArray are not supported types. * @beta + * @deprecated Use [ChangesetReader]($backend) instead. * */ export class ChangesetECAdaptor implements Disposable { diff --git a/core/backend/src/ChangesetReader.ts b/core/backend/src/ChangesetReader.ts new file mode 100644 index 000000000000..7cb244aa4826 --- /dev/null +++ b/core/backend/src/ChangesetReader.ts @@ -0,0 +1,412 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ +/** @packageDocumentation + * @module ECDb + */ +import { DbChangeStage, DbOpcode, Id64String, IModelStatus } from "@itwin/core-bentley"; +import { IModelError } from "@itwin/core-common"; +import { IModelDb } from "./IModelDb"; +import { IModelNative } from "./internal/NativePlatform"; +import { _nativeDb } from "./internal/Symbols"; +import { IModelJsNative } from "@bentley/imodeljs-native"; +import { ChangeInstance, ChangesetReaderArgs, ChangeSource, PropertyFilter, RowFormatOptions } from "./ChangesetReaderTypes"; +import { AnyDb, SqliteChangeOp } from "./SqliteChangesetReader"; + + +// --------------------------------------------------------------------------- +// ChangesetReader +// --------------------------------------------------------------------------- + +/** + * Reads EC-typed changeset data natively from a changeset file, changeset group, + * in-memory transaction, or local un-pushed changes. + * + * Implements [ChangeSource]($backend) so rows can be fed directly into + * [PartialChangeUnifier]($backend) to merge partial (per-table) instances into + * complete EC instances. + * + * When the current row is a non-EC internal SQLite table, [[isECTable]] is `false` + * and both [[inserted]] and [[deleted]] remain `undefined`. + * + * @note The native reader operates one SQLite table-row at a time. Multi-table EC + * instances must be merged using [PartialChangeUnifier]($backend). + * @beta + */ +export class ChangesetReader implements Disposable, ChangeSource { + private readonly _nativeReader: IModelJsNative.ChangesetReader = new IModelNative.platform.ChangesetReader(); + // Internal options — keep ECClassId as raw Id so the unifier can use it as-is. + private _rowOptions?: RowFormatOptions; + private _propFilter: PropertyFilter = PropertyFilter.All; + private _changeIndex = 0; + private _op?: SqliteChangeOp; + private _isECTable?: boolean; + private _tableName?: string; + private _isIndirectChange?: boolean; + + /** The db used for EC schema resolution. */ + public readonly db: AnyDb; + + /** + * `true` when the current row belongs to an EC-mapped table. + * Valid only after a successful call to [[step]]. + * @beta + */ + public get isECTable(): boolean { + if (this._isECTable === undefined) + throw new IModelError(IModelStatus.BadRequest, "ChangesetReader.isECTable is only valid after step() has positioned the reader on a current valid change."); + return this._isECTable; + } + + /** + * Name of the SQLite table for the current change row. + * Valid only after a successful call to [[step]]. + * @beta + */ + public get tableName(): string { + if (this._tableName === undefined) + throw new IModelError(IModelStatus.BadRequest, "ChangesetReader.tableName is only valid after step() has positioned the reader on a current valid change."); + return this._tableName; + } + + /** + * `true` when the current change was applied indirectly + * Valid only after a successful call to [[step]]. + * @beta + */ + public get isIndirectChange(): boolean { + if (this._isIndirectChange === undefined) + throw new IModelError(IModelStatus.BadRequest, "ChangesetReader.isIndirectChange is only valid after step() has positioned the reader on a current valid change."); + return this._isIndirectChange; + } + + /** + * Post-change (inserted or updated-new) EC instance, populated after each [[step]] call. + * `undefined` when the current row is a Delete or a non-EC table row or [[step]] returned false. + * @beta + */ + public inserted?: ChangeInstance; + + /** + * Pre-change (deleted or updated-old) EC instance, populated after each [[step]] call. + * `undefined` when the current row is an Insert or a non-EC table row or [[step]] returned false. + * @beta + */ + public deleted?: ChangeInstance; + + // Private — callers use static factory methods. + private constructor(db: AnyDb) { + this.db = db; + } + + /** Map public RowFormatOptions to the native adaptor options. + * @internal */ + private toNativeRowOptions(opts: RowFormatOptions): IModelJsNative.ECSqlRowAdaptorOptions { + return { + abbreviateBlobs: opts.abbreviateBlobs, + classIdsToClassNames: opts.classIdsToClassNames, + useJsName: opts.useJsName, + }; + } + + // --------------------------------------------------------------------------- + // Static factory methods + // --------------------------------------------------------------------------- + + /** + * Open a changeset file from disk. + * @param args.fileName Absolute path to the changeset file. + * @param args.db Database at or after the changeset's ending state, used for schema resolution. + * @param args.invert When `true`, invert all operations (Insert↔Delete). + * @param args.valueOptions Row adaptor options controlling how EC property values are formatted. + * @param args.propFilter Controls which properties are included. Defaults to `All`. + * @beta + */ + public static openFile(args: { readonly fileName: string } & ChangesetReaderArgs): ChangesetReader { + const reader = new ChangesetReader(args.db); + reader._rowOptions = args.rowOptions; + const propFilter = args.propFilter ?? PropertyFilter.All; + reader._propFilter = propFilter; + try { + reader._nativeReader.openFile(args.db[_nativeDb], args.fileName, args.invert ?? false, reader._propFilter); + } + catch (e) { + reader.close(); + throw e; + } + return reader; + } + + /** + * Concatenate multiple changeset files and read them as a single logical stream. + * @param args.changesetFiles Ordered list of changeset file paths. + * @param args.db Database with schema at or ahead of the last changeset. + * @param args.valueOptions Row adaptor options controlling how EC property values are formatted. + * @param args.propFilter Controls which properties are included. Defaults to `All`. + * @beta + */ + public static openGroup(args: { readonly changesetFiles: string[] } & ChangesetReaderArgs): ChangesetReader { + if (args.changesetFiles.length === 0) + throw new Error("changesetFiles must contain at least one file."); + const reader = new ChangesetReader(args.db); + reader._rowOptions = args.rowOptions; + const propFilter = args.propFilter ?? PropertyFilter.All; + reader._propFilter = propFilter; + try { + reader._nativeReader.openGroup(args.db[_nativeDb], args.changesetFiles, args.invert ?? false, reader._propFilter); + } + catch (e) { + reader.close(); + throw e; + } + return reader; + } + + /** + * Read pending (not yet pushed) local changes from an open IModelDb. + * @param args.db Must be an [IModelDb]($backend) (not [ECDb]($backend)). + * @param args.includeInMemoryChanges Also include in-memory (not yet saved to disk) changes. + * @param args.valueOptions Row adaptor options controlling how EC property values are formatted. + * @param args.propFilter Controls which properties are included. Defaults to `All`. + * @beta + */ + public static openLocalChanges( + args: Omit & { db: IModelDb; includeInMemoryChanges?: boolean }, + ): ChangesetReader { + const reader = new ChangesetReader(args.db); + reader._rowOptions = args.rowOptions; + const propFilter = args.propFilter ?? PropertyFilter.All; + reader._propFilter = propFilter; + try { + reader._nativeReader.openLocalChanges(args.db[_nativeDb], args.includeInMemoryChanges ?? false, args.invert ?? false, reader._propFilter); + } catch (e) { + reader.close(); + throw e; + } + return reader; + } + + /** + * Read the in-memory (not yet saved to disk) changes of an open IModelDb. + * @param args.db Must be an [IModelDb]($backend). + * @param args.valueOptions Row adaptor options controlling how EC property values are formatted. + * @param args.propFilter Controls which properties are included. Defaults to `All`. + * @beta + */ + public static openInMemoryChanges( + args: Omit & { db: IModelDb }, + ): ChangesetReader { + const reader = new ChangesetReader(args.db); + reader._rowOptions = args.rowOptions; + const propFilter = args.propFilter ?? PropertyFilter.All; + reader._propFilter = propFilter; + try { + reader._nativeReader.openInMemoryChanges(args.db[_nativeDb], args.invert ?? false, reader._propFilter); + } catch (e) { + reader.close(); + throw e; + } + return reader; + } + + /** + * Read a single saved transaction by its id. + * @param args.db Must be an [IModelDb]($backend) ([ECDb]($backend) does not support transactions). + * @param args.txnId The id of the saved transaction to read. + * @param args.valueOptions Row adaptor options controlling how EC property values are formatted. + * @param args.propFilter Controls which properties are included. Defaults to `All`. + * @beta + */ + public static openTxn( + args: Omit & { db: IModelDb; txnId: Id64String }, + ): ChangesetReader { + const reader = new ChangesetReader(args.db); + reader._rowOptions = args.rowOptions ?? {}; + const propFilter = args.propFilter ?? PropertyFilter.All; + reader._propFilter = propFilter; + try { + reader._nativeReader.openTxn(args.db[_nativeDb], args.txnId, args.invert ?? false, reader._propFilter); + } catch (e) { + reader.close(); + throw e; + } + return reader; + } + + // --------------------------------------------------------------------------- + // Filtering + // --------------------------------------------------------------------------- + + /** + * Restrict iteration to changes from the named SQLite tables. + * That means the rows for changes from other tables will be skipped entirely and won't be visible through the reader. + * @param tableNames SQLite table names to include. + * Note: Table names must be provided in the correct case for proper filtering. + * @beta + */ + public setTableNameFilters(tableNames: Set): void { + this._nativeReader.setTableNameFilters(Array.from(tableNames)); + } + + /** + * Restrict iteration to changes with the given operation types. + * That means the rows for changes with other operation types will be skipped entirely and won't be visible through the reader. + * @param ops Operations to include. + * @beta + */ + public setOpCodeFilters(ops: Set): void { + this._nativeReader.setOpCodeFilters(Array.from(ops)); + } + + /** + * Restrict iteration to changes for the given EC class names. + * That means the rows for changes from other EC classes will be skipped entirely and won't be visible through the reader. + * @param classNames EC class names to include. The classNames should be in the full name format(i.e. "SchemaName:ClassName"). + * Note: Schema names and class names must be provided in the correct case for proper filtering. Derived classes are not automatically included, so they must be specified explicitly if needed. + * @beta + */ + public setClassNameFilters(classNames: Set): void { + this._nativeReader.setClassNameFilters(Array.from(classNames)); + } + + /** + * Remove the table-name filters + * @beta + */ + public clearTableNameFilters(): void { + this._nativeReader.clearTableNameFilters(); + } + + /** + * Remove the op-code filters + * @beta + */ + public clearOpCodeFilters(): void { + this._nativeReader.clearOpCodeFilters(); + } + + /** + * Remove the class-name filters + * @beta + */ + public clearClassNameFilters(): void { + this._nativeReader.clearClassNameFilters(); + } + + // --------------------------------------------------------------------------- + // Iteration + // --------------------------------------------------------------------------- + + /** + * Advance to the next change and populate `inserted` and/or `deleted`. + * @returns `true` while positioned on a valid change; `false` when the stream is exhausted. + * @beta + */ + public step(): boolean { + this.inserted = undefined; + this.deleted = undefined; + this._op = undefined; + this._isECTable = undefined; + this._tableName = undefined; + this._isIndirectChange = undefined; + + if (this._nativeReader.step()) { + this._changeIndex++; + const meta = this._nativeReader.getChangeMetadata(); + const nativeOp = meta.opCode; + const op: SqliteChangeOp = nativeOp === DbOpcode.Insert ? "Inserted" : nativeOp === DbOpcode.Update ? "Updated" : "Deleted"; + this._op = op; + + this._tableName = meta.tableName; + this._isIndirectChange = meta.isIndirectChange; + this._isECTable = meta.isECTable; + + const nativeRowOpts = this._rowOptions ? this.toNativeRowOptions(this._rowOptions) : {}; + + if (op === "Inserted" || op === "Updated") { + const rowValue = this._nativeReader.getValue(DbChangeStage.New, nativeRowOpts); + if (rowValue !== undefined) { + this.inserted = { + ...rowValue.data, + $meta: { + op, + tables: [this._tableName], + changeIndexes: [this._changeIndex], + stage: "New", + instanceKey: rowValue.key, + propFilter: this._propFilter, + changeFetchedPropNames: rowValue.changeFetchedPropNames, + rowOptions: this._rowOptions, + isIndirectChange: this._isIndirectChange, + }, + }; + } + } + + if (op === "Deleted" || op === "Updated") { + const rowValue = this._nativeReader.getValue(DbChangeStage.Old, nativeRowOpts); + if (rowValue !== undefined) { + this.deleted = { + ...rowValue.data, + $meta: { + op, + tables: [this._tableName], + changeIndexes: [this._changeIndex], + stage: "Old", + instanceKey: rowValue.key, + propFilter: this._propFilter, + changeFetchedPropNames: rowValue.changeFetchedPropNames, + rowOptions: this._rowOptions, + isIndirectChange: this._isIndirectChange, + }, + }; + } + } + + return true; + } + + return false; + } + + /** + * SQLite opcode of the current change. + * Valid only after a successful call to [[step]]. + * @beta + */ + public get op(): SqliteChangeOp { + if (this._op === undefined) + throw new IModelError(IModelStatus.BadRequest, "ChangesetReader.op is only valid after step() has positioned the reader on a current valid change."); + return this._op; + } + + // --------------------------------------------------------------------------- + // Lifecycle + // --------------------------------------------------------------------------- + + /** + * Close the reader and release all native resources. + * @beta + */ + public close(): void { + this._changeIndex = 0; + this._op = undefined; + this._isECTable = undefined; + this._tableName = undefined; + this._isIndirectChange = undefined; + this.inserted = undefined; + this.deleted = undefined; + this._nativeReader.close(); + } + + /** + * Implements the `Disposable` contract — calls [[close]]. + * @beta + */ + public [Symbol.dispose](): void { + this.close(); + } +} + + diff --git a/core/backend/src/ChangesetReaderTypes.ts b/core/backend/src/ChangesetReaderTypes.ts new file mode 100644 index 000000000000..830d529ae509 --- /dev/null +++ b/core/backend/src/ChangesetReaderTypes.ts @@ -0,0 +1,141 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ +/** @packageDocumentation + * @module ECDb + */ +import { AnyDb, SqliteChangeOp, SqliteValueStage } from "./SqliteChangesetReader"; + +// --------------------------------------------------------------------------- +// Type aliases +// --------------------------------------------------------------------------- + +/** + * Controls which properties are included in the output of [ChangesetReader]($backend). + * @beta + */ +export enum PropertyFilter { + /** All EC properties mapped to changed tables. */ + All = 0, + /** For classes whose base class is `BisCore:Element`, only `BisCore:Element` properties + * mapped to changed tables are returned. If no `BisCore:Element` class property changed, + * only `ECInstanceId` and `ECClassId` are returned. For other classes all mapped properties + * are returned. */ + BisCoreElement = 1, + /** Only `ECInstanceId` and `ECClassId`. */ + InstanceKey = 2, +} + +/** + * Row-formatting options for [ChangesetReader]($backend) factory methods. + * Controls how EC property values are represented in the returned instances. + * @beta + */ +export interface RowFormatOptions { + /** + * When `false`, binary properties are returned as full `Uint8Array` values. + * When `true` (or omitted), binary properties are summarized as `{ bytes: N }`. + */ + abbreviateBlobs?: boolean; + /** + * When `true`, `ECClassId` and `RelECClassId` values are converted from hex strings + * to fully-qualified class names (e.g. `"BisCore.DrawingModel"`). + */ + classIdsToClassNames?: boolean; + /** + * When `true`, all property keys and struct sub-keys are returned in camelCase + * (e.g. `id`, `className`, `lastMod`). Navigation property sub-keys use + * `{ id, relClassName }` instead of `{ Id, RelECClassId }`. + */ + useJsName?: boolean; +} + +// --------------------------------------------------------------------------- +// Public interfaces +// --------------------------------------------------------------------------- + +/** + * Metadata attached to every [[ChangeInstance]]. + * @beta + */ +export interface ChangeMeta { + /** SQLite tables that contributed columns to this change row. */ + tables: string[]; + /** Operation that produced this change. */ + op: SqliteChangeOp; + /** Whether this is the pre-change (`"Old"`) or post-change (`"New"`) snapshot. */ + stage: SqliteValueStage; + /** Change-stream index positions. */ + changeIndexes: number[]; + /** + * ECInstanceId and class Id in format "-". + */ + instanceKey: string; + /** Reader property filter that was active when this change row was captured. */ + propFilter: PropertyFilter; + /** EC property names fetched from the current row of changeset or transaction or any other change stream. + For compound data properties like point2d, point3d or navigation properties, + the full name of the property is returned in case all the components of the property are fetched from the change. + If all of the components are not fetched from the changes(meaning they did not change), + then the individual component names which changed are returned smartly by using `.` as a separator (e.g. "MyPoint.X", "MyPoint.Y" for a point3d property "MyPoint" if only X and Y changed). + For struct properties the property names are always returned in the "StructProp.MemberName" format. + So if only X changed for a point2d property named "Myp2d" inide a struct "CustomStruct", the returned property name will be "CustomStruct.Myp2d.X". + Similaly if both X and Y changed for the same point2d property, the returned property name will be "CustomStruct.Myp2d". */ + changeFetchedPropNames: string[]; + /** Row adaptor options that were active when this change row was captured. */ + rowOptions?: RowFormatOptions; + /** `true` when the change was applied indirectly */ + isIndirectChange: boolean; +} + +/** + * An EC instance produced by [ChangesetReader]($backend) after each `step()`. + * Contains the EC property bag plus mandatory `$meta` metadata. + * @beta + */ +export interface ChangeInstance { + /** Metadata describing the origin and identity of this change. */ + $meta: ChangeMeta; + /** EC property bag (ECClassId, ECInstanceId, user-defined properties, ...). */ + [key: string]: any; +} + +/** + * Contract for any reader that produces EC-typed changed instances compatible with + * [PartialChangeUnifier]($backend). + * @beta + */ +export interface ChangeSource { + /** The SQLite opcode of the current change row. */ + readonly op: SqliteChangeOp; + /** + * The newly-inserted or post-update EC instance. + * `undefined` when the current row is a Delete, or when `isECTable` is `false`. + */ + readonly inserted?: ChangeInstance; + /** + * The deleted or pre-update EC instance. + * `undefined` when the current row is an Insert, or when `isECTable` is `false`. + */ + readonly deleted?: ChangeInstance; +} + +// --------------------------------------------------------------------------- +// ChangesetReader args / options +// --------------------------------------------------------------------------- + +/** + * Arguments common to all [ChangesetReader]($backend) `open*` factory methods. + * @beta + */ +export interface ChangesetReaderArgs { + /** The db used to resolve EC schema. Must be at or ahead of the changeset being read. */ + readonly db: AnyDb; + /** invert the changeset operations */ + readonly invert?: boolean; + /** Row adaptor options controlling how EC property values are formatted. */ + readonly rowOptions?: RowFormatOptions; + /** Controls which properties are included in the change output. Defaults to PropertyFilter.All. */ + readonly propFilter?: PropertyFilter; +} diff --git a/core/backend/src/PartialChangeUnifier.ts b/core/backend/src/PartialChangeUnifier.ts new file mode 100644 index 000000000000..8ee2529f7b44 --- /dev/null +++ b/core/backend/src/PartialChangeUnifier.ts @@ -0,0 +1,282 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ +/** @packageDocumentation + * @module ECDb + */ +import { DbResult, Guid, OpenMode } from "@itwin/core-bentley"; +import { Base64EncodedString } from "@itwin/core-common"; +import { SqliteStatement } from "./SqliteStatement"; +import { ChangeInstance, ChangeSource } from "./ChangesetReaderTypes"; +import { _nativeDb } from "./internal/Symbols"; +import { SQLiteDb } from "./SQLiteDb"; + +// --------------------------------------------------------------------------- +// ChangeCache — interface + factory +// --------------------------------------------------------------------------- + +/** + * Cache used by [[PartialChangeUnifier]] to accumulate and merge + * partial EC change instances. + * @beta + */ +export interface ChangeCache extends Disposable { + /** Retrieve a cached instance by key, or `undefined` if absent. */ + get(key: string): ChangeInstance | undefined; + /** Insert or replace a cached instance. */ + set(key: string, value: ChangeInstance): void; + /** Iterate over all cached instances. */ + all(): IterableIterator; + /** Number of instances currently in the cache. */ + count(): number; +} + +/** @beta */ +export namespace ChangeUnifierCache { + /** + * Creates an in-memory cache backed by a `Map`. + * Fast, but may exhaust memory for very large changesets. + * @returns An [[ChangeCache]] backed by an in-memory `Map`. + * @beta + */ + export function createInMemoryCache(): ChangeCache { + return new InMemoryCache(); + } + + /** + * Creates a SQLite-backed cache stored in a temporary SQlite database. + * Slower than in-memory but useful in handling large changesets. + * Temporary SQlite database is first created in memory and + * parts of a temporary database might be flushed to disk if the database becomes large or + * if SQLite comes under memory pressure. + * @param bufferedReadInstanceSizeInBytes Read-batch size in bytes (default 10 MB). + * @returns An [[ChangeCache]] backed by a SQLite temp table. + * @beta + */ + export function createSqliteBackedCache( + bufferedReadInstanceSizeInBytes = 1024 * 1024 * 10, + ): ChangeCache { + return new SqliteBackedCache(bufferedReadInstanceSizeInBytes); + } +} + +// --------------------------------------------------------------------------- +// Private: InMemoryCache +// --------------------------------------------------------------------------- + +class InMemoryCache implements ChangeCache { + private readonly _cache = new Map(); + + public get(key: string): ChangeInstance | undefined { + return this._cache.get(key); + } + + public set(key: string, value: ChangeInstance): void { + // Remove undefined meta keys to keep serialised form compact. + const meta = value.$meta as any; + if (meta) { + Object.keys(meta).forEach((k) => meta[k] === undefined && delete meta[k]); + } + this._cache.set(key, value); + } + + public *all(): IterableIterator { + for (const key of Array.from(this._cache.keys()).sort()) { + const instance = this._cache.get(key); + if (instance) + yield instance; + } + } + + public count(): number { + return this._cache.size; + } + + public [Symbol.dispose](): void { + this._cache.clear(); + } +} + +// --------------------------------------------------------------------------- +// Private: NativeSqliteBackedInstanceCache +// --------------------------------------------------------------------------- + +class SqliteBackedCache implements ChangeCache { + private readonly _cacheTable = `[${Guid.createValue()}]`; + public static readonly defaultBufferSize = 1024 * 1024 * 10; // 10 MB + private _db: SQLiteDb; + public constructor( + public readonly bufferedReadInstanceSizeInBytes: number = SqliteBackedCache.defaultBufferSize, + ) { + this._db = new SQLiteDb(); + this._db.openDb("", { skipFileCheck: true, rawSQLite: true, openMode: OpenMode.ReadWrite }); // creating temp sqlite db https://sqlite.org/inmemorydb.html#:~:text=Temporary%20Databases,under%20the%20default%20SQLite%20configuration. + if (bufferedReadInstanceSizeInBytes <= 0) + throw new Error("bufferedReadInstanceSizeInBytes must be greater than 0"); + this.createTempTable(); + } + + private createTempTable(): void { + this._db.withSqliteStatement(`CREATE TABLE ${this._cacheTable} ([key] text primary key, [value] text)`, (stmt: SqliteStatement) => { + if (DbResult.BE_SQLITE_DONE !== stmt.step()) + throw new Error("unable to create temp cache table"); + }); + } + + public get(key: string): ChangeInstance | undefined { + return this._db.withPreparedSqliteStatement( + `SELECT [value] FROM ${this._cacheTable} WHERE [key]=?`, + (stmt: SqliteStatement) => { + stmt.reset(); + stmt.clearBindings(); + stmt.bindString(1, key); + if (stmt.step() === DbResult.BE_SQLITE_ROW) + return JSON.parse(stmt.getValueString(0), Base64EncodedString.reviver) as ChangeInstance; + return undefined; + }, + ); + } + + public set(key: string, value: ChangeInstance): void { + const shallowCopy = Object.assign({}, value); + this._db.withPreparedSqliteStatement( + `INSERT INTO ${this._cacheTable} ([key], [value]) VALUES (?, ?) ON CONFLICT ([key]) DO UPDATE SET [value] = [excluded].[value]`, + (stmt: SqliteStatement) => { + stmt.reset(); + stmt.clearBindings(); + stmt.bindString(1, key); + stmt.bindString(2, JSON.stringify(shallowCopy, Base64EncodedString.replacer)); + stmt.step(); + }, + ); + } + + public *all(): IterableIterator { + const sql = ` + SELECT JSON_GROUP_ARRAY(JSON([value])) + FROM ( + SELECT [value], + SUM(LENGTH([value])) OVER (ORDER BY [key] ROWS UNBOUNDED PRECEDING) / ${this.bufferedReadInstanceSizeInBytes} AS [bucket] + FROM ${this._cacheTable} + ) + GROUP BY [bucket]`; + + const stmt = this._db.prepareSqliteStatement(sql); + try { + while (stmt.step() === DbResult.BE_SQLITE_ROW) { + const bucket = JSON.parse(stmt.getValueString(0), Base64EncodedString.reviver) as ChangeInstance[]; + for (const instance of bucket) + yield instance; + } + } finally { + stmt[Symbol.dispose](); + } + } + + public count(): number { + return this._db.withPreparedSqliteStatement( + `SELECT COUNT(*) FROM ${this._cacheTable}`, + (stmt: SqliteStatement) => { + stmt.reset(); + if (stmt.step() === DbResult.BE_SQLITE_ROW) + return stmt.getValue(0).getInteger(); + return 0; + }, + ); + } + + public [Symbol.dispose](): void { + this._db.closeDb(); + } +} + +// --------------------------------------------------------------------------- +// PartialChangeUnifier +// --------------------------------------------------------------------------- + +/** + * Combines partial EC change instances (one per SQLite table row) into complete + * instances that span all tables mapping to a single EC entity. + * + * The merge key is derived from the `instanceKey` and `stage` stored in `$meta.instanceKey` and `$meta.stage`. + * + * **Usage:** + * ```ts + * using reader = ChangesetReader.openFile({ fileName, db }); + * using unifier = new PartialChangeUnifier(); + * while (reader.step()) { + * unifier.appendFrom(reader); + * } + * for (const instance of unifier.instances) { ... } + * ``` + * @beta + */ +export class PartialChangeUnifier implements Disposable { + public constructor( + private readonly _cache: ChangeCache = new InMemoryCache(), + ) { } + + /** Releases the underlying cache. */ + public [Symbol.dispose](): void { + this._cache[Symbol.dispose](); + } + + /** Number of complete (merged) instances currently accumulated. */ + public get instanceCount(): number { + return this._cache.count(); + } + + /** + * Append partial changes from the current reader row and merge them into the cache. + * + * @param source Any [ChangeSource]($backend) positioned on a valid row. + * @beta + */ + public appendFrom(source: ChangeSource): void { + if (source.op === "Updated") { + if (source.inserted) + this.combine(source.inserted); + if (source.deleted) + this.combine(source.deleted); + } else if (source.op === "Inserted" && source.inserted) { + this.combine(source.inserted); + } else if (source.op === "Deleted" && source.deleted) { + this.combine(source.deleted); + } + } + + /** + * Iterator over all fully-merged EC change instances. + * @beta + */ + public get instances(): IterableIterator { + return this._cache.all(); + } + + // --------------------------------------------------------------------------- + // Private helpers + // --------------------------------------------------------------------------- + + private buildKey(instance: ChangeInstance): string { + const { instanceKey, stage } = instance.$meta; + return `${instanceKey}-${stage}`.toLowerCase(); + } + + private combine(rhs: ChangeInstance): void { + const key = this.buildKey(rhs); + const lhs = this._cache.get(key); + if (lhs) { + // Merge data fields — rhs wins for any overlapping columns. + const { $meta: _rhsMeta, ...rhsData } = rhs as any; + Object.assign(lhs, rhsData); + // Accumulate per-table metadata lists. + lhs.$meta.tables = [...lhs.$meta.tables, ...rhs.$meta.tables]; + lhs.$meta.changeIndexes = [...lhs.$meta.changeIndexes, ...rhs.$meta.changeIndexes]; + // ECInstanceId will be part of changeset fetchedProps for every table, so we should not include multiple of those in the final list + lhs.$meta.changeFetchedPropNames = [...new Set([...lhs.$meta.changeFetchedPropNames, ...rhs.$meta.changeFetchedPropNames])]; + this._cache.set(key, lhs); + } else { + this._cache.set(key, rhs); + } + } +} diff --git a/core/backend/src/SqliteChangesetReader.ts b/core/backend/src/SqliteChangesetReader.ts index 6bf01d666197..a65628dd9423 100644 --- a/core/backend/src/SqliteChangesetReader.ts +++ b/core/backend/src/SqliteChangesetReader.ts @@ -84,7 +84,7 @@ export interface SqliteChange { * @beta */ export class SqliteChangesetReader implements Disposable { - private readonly _nativeReader = new IModelNative.platform.ChangesetReader(); + private readonly _nativeReader = new IModelNative.platform.SqliteChangesetReader(); private _schemaCache = new Map(); private _disableSchemaCheck = false; private _changeIndex = 0; diff --git a/core/backend/src/core-backend.ts b/core/backend/src/core-backend.ts index 21d0a98c2224..9d5619e20263 100644 --- a/core/backend/src/core-backend.ts +++ b/core/backend/src/core-backend.ts @@ -85,6 +85,9 @@ export * from "./workspace/Workspace"; export * from "./workspace/WorkspaceEditor"; export * from "./SqliteChangesetReader"; export * from "./ChangesetECAdaptor"; +export * from "./ChangesetReader"; +export * from "./ChangesetReaderTypes"; +export * from "./PartialChangeUnifier"; export * from "./internal/cross-package"; diff --git a/core/backend/src/test/hubaccess/Rebase.test.ts b/core/backend/src/test/hubaccess/Rebase.test.ts index 9e1bf70a0584..e949b4652357 100644 --- a/core/backend/src/test/hubaccess/Rebase.test.ts +++ b/core/backend/src/test/hubaccess/Rebase.test.ts @@ -1139,7 +1139,7 @@ for (const enableSemanticRebase of [false, true]) { }, recompute: async (txn: TxnProps): Promise => { const reader = SqliteChangesetReader.openTxn({ txnId: txn.id, db: b2, disableSchemaCheck: true }); - const adaptor = new ChangesetECAdaptor(reader); + const adaptor = new ChangesetECAdaptor(reader); // eslint-disable-line @typescript-eslint/no-deprecated adaptor.acceptClass("TestDomain:a1"); const ids = new Set(); while (adaptor.step()) { diff --git a/core/backend/src/test/standalone/ChangesetReader.test.ts b/core/backend/src/test/standalone/ChangesetReader.test.ts index 7819a0a1fb64..64a6c09a8577 100644 --- a/core/backend/src/test/standalone/ChangesetReader.test.ts +++ b/core/backend/src/test/standalone/ChangesetReader.test.ts @@ -2,23 +2,43 @@ * Copyright (c) Bentley Systems, Incorporated. All rights reserved. * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ -import { DbResult, GuidString, Id64, Id64String } from "@itwin/core-bentley"; -import { Code, ColorDef, GeometryStreamProps, IModel, QueryBinder, QueryRowFormat, SubCategoryAppearance } from "@itwin/core-common"; +import { DbResult, Id64, Id64String } from "@itwin/core-bentley"; +import { Code, ColorDef, GeometryStreamProps, IModel, SubCategoryAppearance } from "@itwin/core-common"; import { Arc3d, IModelJson, Point3d } from "@itwin/core-geometry"; -import * as chai from "chai"; import { assert, expect } from "chai"; -import * as path from "node:path"; import { DrawingCategory } from "../../Category"; -import { ChangedECInstance, ChangesetECAdaptor, ChangesetECAdaptor as ECChangesetAdaptor, ECChangeUnifierCache, PartialECChangeUnifier } from "../../ChangesetECAdaptor"; -import { _nativeDb, ChannelControl, GraphicalElement2d, Subject, SubjectOwnsSubjects } from "../../core-backend"; -import { BriefcaseDb, SnapshotDb } from "../../IModelDb"; +import { _nativeDb, BriefcaseDb, ChannelControl } from "../../core-backend"; import { HubMock } from "../../internal/HubMock"; -import { SqliteChangeOp, SqliteChangesetReader } from "../../SqliteChangesetReader"; +import { ChangesetReader } from "../../ChangesetReader"; +import * as path from "node:path"; +import * as fs from "fs"; import { HubWrappers, IModelTestUtils } from "../IModelTestUtils"; import { KnownTestLocations } from "../KnownTestLocations"; +import { ChangeUnifierCache, PartialChangeUnifier } from "../../PartialChangeUnifier"; +import { ChangeInstance, PropertyFilter, RowFormatOptions } from "../../ChangesetReaderTypes"; import { EditTxn } from "../../EditTxn"; -function startTestTxn(iModel: BriefcaseDb | SnapshotDb, description = "changeset reader"): EditTxn { +/* eslint-disable @typescript-eslint/naming-convention */ // disabling it because the property names are not in camelcase, and we want to test them as-is + +/** Open a txn, drive the unifier, log and return all merged instances. */ +function readTxn( + db: BriefcaseDb, + txnId: string, + propFilter?: PropertyFilter, + rowOptions?: RowFormatOptions, + invert?: boolean, + useInMemoryUnifierCache?: boolean, +): ChangeInstance[] { + using reader = ChangesetReader.openTxn({ db, txnId, propFilter, rowOptions, invert }); + const inMemCache = useInMemoryUnifierCache ?? true; + using pcu = new PartialChangeUnifier(inMemCache ? ChangeUnifierCache.createInMemoryCache() : ChangeUnifierCache.createSqliteBackedCache()); + while (reader.step()) + pcu.appendFrom(reader); + const instances = Array.from(pcu.instances); + return instances; +} + +function startTestTxn(iModel: BriefcaseDb, description = "changeset reader"): EditTxn { const txn = new EditTxn(iModel, description); txn.start(); return txn; @@ -30,1475 +50,3890 @@ async function importSchemaStrings(txn: EditTxn, schemas: string[]): Promise { - let iTwinId: GuidString; +describe("ChangesetReader insert-full", () => { + let rwIModel: BriefcaseDb; + let fullElementId: Id64String; + let drawingModelId: Id64String; + let drawingCategoryId: Id64String; + let txnId: string; + let txn: EditTxn; - before(() => { - HubMock.startup("ChangesetReaderTest", KnownTestLocations.outputDir); - iTwinId = HubMock.iTwinId; - }); - after(() => HubMock.shutdown()); - it("Able to recover from when ExclusiveRootClassId is NULL for overflow table", async () => { - /** - * 1. Import schema with class that span overflow table. - * 2. Insert a element for the class. - * 3. Push changes to hub. - * 4. Update the element. - * 5. Push changes to hub. - * 6. Delete the element. - * 7. Set ExclusiveRootClassId to NULL for overflow table. (Simulate the issue) - * 8. ECChangesetAdaptor should be able to read the changeset 2 in which element is updated against latest imodel where element is deleted. - */ + before(async () => { + HubMock.startup("ECChangesetInsertFull", KnownTestLocations.outputDir); const adminToken = "super manager token"; - const iModelName = "test"; - const nProps = 36; - const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(rwIModelId); - const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); - const txn = startTestTxn(rwIModel, "recover overflow table changeset reader"); - // 1. Import schema with class that span overflow table. + const iTwinId = HubMock.iTwinId; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "insertFull", description: "insertFull", accessToken: adminToken }); + rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + txn = startTestTxn(rwIModel, "ChangesetReader insert-full setup"); + // Txn 1: import schema + drawing model setup, then push const schema = ` - - - - bis:GraphicalElement2d - ${Array(nProps).fill(undefined).map((_, i) => ``).join("\n")} - - `; + + + + + + + + + + + + + + + + + + bis:GraphicalElement2d + + + + + + + + + + + + + + + + + + + + `; await importSchemaStrings(txn, [schema]); rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - // Create drawing model and category await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); const codeProps = Code.createEmpty(); - codeProps.value = "DrawingModel"; - const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); - if (undefined === drawingCategoryId) - drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + codeProps.value = "DrillDownDrawing"; + [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - // Insert element with 100 properties - const geomArray: Arc3d[] = [ + const foundCat = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "DrillDownCategory"); + drawingCategoryId = foundCat ?? DrawingCategory.insert(txn, IModel.dictionaryId, "DrillDownCategory", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + + txn.saveChanges("setup"); + + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Txn 2: insert FULL element every EC primitive type populated + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + const geom: GeometryStreamProps = [ Arc3d.createXY(Point3d.create(0, 0), 5), Arc3d.createXY(Point3d.create(5, 5), 2), Arc3d.createXY(Point3d.create(-5, -5), 20), - ]; - const geometryStream: GeometryStreamProps = []; - for (const geom of geomArray) { - const arcData = IModelJson.Writer.toIModelJson(geom); - geometryStream.push(arcData); - } - const props = Array(nProps).fill(undefined).map((_, i) => { - return { [`p${i}`]: `test_${i}` }; - }).reduce((acc, curr) => { - return { ...acc, ...curr }; - }, {}); - - const geomElement = { - classFullName: `TestDomain:Test2dElement`, + ].map((a) => IModelJson.Writer.toIModelJson(a)); + + fullElementId = txn.insertElement({ + classFullName: "TestDomain:Test2dElement", model: drawingModelId, category: drawingCategoryId, code: Code.createEmpty(), - geom: geometryStream, - ...props, - }; + geom, + StrProp: "hello", + IntProp: 42, + LongProp: 9_007_199_254_740_991, // Number.MAX_SAFE_INTEGER + DblProp: 3.14159265358979, + BoolProp: true, + DtProp: "2024-01-15T12:00:00.000", + BinProp: new Uint8Array([1, 2, 3, 4]), + Pt2dProp: { x: 1.5, y: 2.5 }, + Pt3dProp: { x: 3.0, y: 4.0, z: 5.0 }, + StructProp: { + X: 1.0, Y: 2.0, Z: 3.0, Label: "origin", + Pt2d: { x: 0.5, y: 0.5 }, + Pt3d: { x: 1.0, y: 2.0, z: 3.0 }, + }, + IntArrProp: [10, 20, 30], + StrArrProp: ["alpha", "beta", "gamma"], + StructArrProp: [ + { X: 0.0, Y: 1.0, Z: 2.0, Label: "a", Pt2d: { x: 0.0, y: 0.0 }, Pt3d: { x: 0.0, y: 0.0, z: 0.0 } }, + { X: 3.0, Y: 4.0, Z: 5.0, Label: "b", Pt2d: { x: 1.0, y: 1.0 }, Pt3d: { x: 1.0, y: 1.0, z: 1.0 } }, + ], + RelatedElem: { id: drawingCategoryId, relClassName: "TestDomain:Test2dUsesElement" }, + } as any); + txn.saveChanges("insert full element"); + txnId = rwIModel.txns.getLastSavedTxnProps()!.id; + }); - // 2. Insert a element for the class. - const id = txn.insertElement(geomElement); - assert.isTrue(Id64.isValidId64(id), "insert worked"); - txn.saveChanges(); + after(() => { + txn.end(); + rwIModel?.close(); + HubMock.shutdown(); + }); - // 3. Push changes to hub. - await rwIModel.pushChanges({ description: "insert element", accessToken: adminToken }); + it("txn1 insert-full | All_Properties | default rowOptions", () => { + const instances = readTxn(rwIModel, txnId); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + // Object.keys + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + // $meta keys + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.deepEqual([...modelNew!.$meta.tables].sort(), ["bis_Model"].sort()); + assert.deepEqual([...modelNew!.$meta.changeIndexes].sort(), [3].sort()); + assert.isString(modelNew!.$meta.instanceKey); + assert.equal(modelNew!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + // Object.keys + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + // $meta keys + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.deepEqual([...modelOld!.$meta.tables].sort(), ["bis_Model"].sort()); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Inserted New --- + const elem = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, fullElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.equal(elem!.Model.Id, drawingModelId); + assert.equal(rwIModel.getClassNameFromId(elem!.Model.RelECClassId), "BisCore:ModelContainsElements"); + assert.isString(elem!.LastMod); + assert.equal(elem!.CodeSpec.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeSpec.RelECClassId), "BisCore:CodeSpecSpecifiesCode"); + + assert.equal(elem!.CodeScope.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeScope.RelECClassId), "BisCore:ElementScopesCode"); + assert.isString(elem!.FederationGuid); + + assert.equal(elem!.Category.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.Category.RelECClassId), "BisCore:GeometricElement2dIsInCategory"); + assert.deepEqual(elem!.Origin, { X: 0, Y: 0 }); + assert.equal(elem!.Rotation, 0); + assert.deepEqual(elem!.BBoxLow, { X: -25, Y: -25 }); + assert.deepEqual(elem!.BBoxHigh, { X: 15, Y: 15 }); + assert.include(String(elem!.GeometryStream), "\"bytes\""); + assert.include(String(elem!.BinProp), "\"bytes\""); + assert.equal(elem!.StrProp, "hello"); + assert.equal(elem!.IntProp, 42); + assert.equal(elem!.LongProp, 9007199254740991); + assert.closeTo(elem!.DblProp as number, 3.14159265358979, 1e-10); + assert.equal(elem!.BoolProp, true); + assert.equal(elem!.DtProp, "2024-01-15T12:00:00.000"); + assert.deepEqual(elem!.Pt2dProp, { X: 1.5, Y: 2.5 }); + assert.deepEqual(elem!.Pt3dProp, { X: 3, Y: 4, Z: 5 }); + assert.deepEqual(elem!.StructProp, { X: 1, Y: 2, Z: 3, Label: "origin", Pt2d: { X: 0.5, Y: 0.5 }, Pt3d: { X: 1, Y: 2, Z: 3 } }); + assert.deepEqual(elem!.IntArrProp, [10, 20, 30]); + assert.deepEqual(elem!.StrArrProp, ["alpha", "beta", "gamma"]); + assert.deepEqual(elem!.StructArrProp, [ + { X: 0, Y: 1, Z: 2, Label: "a", Pt2d: { X: 0, Y: 0 }, Pt3d: { X: 0, Y: 0, Z: 0 } }, + { X: 3, Y: 4, Z: 5, Label: "b", Pt2d: { X: 1, Y: 1 }, Pt3d: { X: 1, Y: 1, Z: 1 } }, + ]); + assert.equal(elem!.RelatedElem.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.RelatedElem.RelECClassId), "TestDomain:Test2dUsesElement"); + // Object.keys + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", + "Category", "Origin", "Rotation", "BBoxLow", "BBoxHigh", "GeometryStream", + "StrProp", "IntProp", "LongProp", "DblProp", "BoolProp", "DtProp", + "Pt2dProp", "Pt3dProp", "StructProp", "IntArrProp", "StrArrProp", "StructArrProp", "RelatedElem", "BinProp" + ].sort()); + // $meta keys + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), [ + 'BBoxHigh', 'BBoxLow', 'BinProp', 'BoolProp', 'Category.Id', 'CodeScope.Id', + 'CodeSpec.Id', 'CodeValue', 'DblProp', 'DtProp', 'ECClassId', 'ECInstanceId', + 'FederationGuid', 'GeometryStream', 'IntArrProp', 'IntProp', 'JsonProperties', + 'LastMod', 'LongProp', 'Model.Id', 'Origin', 'Parent', 'Pt2dProp', 'Pt3dProp', + 'RelatedElem', 'Rotation', 'StrArrProp', 'StrProp', 'StructArrProp', 'StructProp.Label', + 'StructProp.Pt2d', 'StructProp.Pt3d', 'StructProp.X', 'StructProp.Y', 'StructProp.Z', + 'TypeDefinition', 'UserLabel' + ].sort()); + assert.deepEqual(elem!.$meta.rowOptions, {}); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - // 4. Update the element. - const updatedElementProps = Object.assign( - rwIModel.elements.getElementProps(id), - Array(nProps).fill(undefined).map((_, i) => { - return { [`p${i}`]: `updated_${i}` }; - }).reduce((acc, curr) => { - return { ...acc, ...curr }; - }, {})); + it("txn1 insert-full | All_Properties | default rowOptions | invert", () => { + const instances = readTxn(rwIModel, txnId, undefined, undefined, true); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.isUndefined(modelNew!.LastMod); + assert.isUndefined(modelNew!.GeometryGuid); + // Object.keys + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + // $meta keys + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.deepEqual([...modelNew!.$meta.tables].sort(), ["bis_Model"].sort()); + assert.deepEqual([...modelNew!.$meta.changeIndexes].sort(), [3].sort()); + assert.isString(modelNew!.$meta.instanceKey); + assert.equal(modelNew!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.isString(modelOld!.LastMod); + assert.isString(modelOld!.GeometryGuid); + // Object.keys + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta", "LastMod", "GeometryGuid"].sort()); + // $meta keys + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.deepEqual([...modelOld!.$meta.tables].sort(), ["bis_Model"].sort()); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Inserted New --- + const elem = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "Old"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, fullElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.equal(elem!.Model.Id, drawingModelId); + assert.equal(rwIModel.getClassNameFromId(elem!.Model.RelECClassId), "BisCore:ModelContainsElements"); + assert.isString(elem!.LastMod); + assert.equal(elem!.CodeSpec.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeSpec.RelECClassId), "BisCore:CodeSpecSpecifiesCode"); + + assert.equal(elem!.CodeScope.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeScope.RelECClassId), "BisCore:ElementScopesCode"); + assert.isString(elem!.FederationGuid); + + assert.equal(elem!.Category.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.Category.RelECClassId), "BisCore:GeometricElement2dIsInCategory"); + assert.deepEqual(elem!.Origin, { X: 0, Y: 0 }); + assert.equal(elem!.Rotation, 0); + assert.deepEqual(elem!.BBoxLow, { X: -25, Y: -25 }); + assert.deepEqual(elem!.BBoxHigh, { X: 15, Y: 15 }); + assert.include(String(elem!.GeometryStream), "\"bytes\""); + assert.include(String(elem!.BinProp), "\"bytes\""); + assert.equal(elem!.StrProp, "hello"); + assert.equal(elem!.IntProp, 42); + assert.equal(elem!.LongProp, 9007199254740991); + assert.closeTo(elem!.DblProp as number, 3.14159265358979, 1e-10); + assert.equal(elem!.BoolProp, true); + assert.equal(elem!.DtProp, "2024-01-15T12:00:00.000"); + assert.deepEqual(elem!.Pt2dProp, { X: 1.5, Y: 2.5 }); + assert.deepEqual(elem!.Pt3dProp, { X: 3, Y: 4, Z: 5 }); + assert.deepEqual(elem!.StructProp, { X: 1, Y: 2, Z: 3, Label: "origin", Pt2d: { X: 0.5, Y: 0.5 }, Pt3d: { X: 1, Y: 2, Z: 3 } }); + assert.deepEqual(elem!.IntArrProp, [10, 20, 30]); + assert.deepEqual(elem!.StrArrProp, ["alpha", "beta", "gamma"]); + assert.deepEqual(elem!.StructArrProp, [ + { X: 0, Y: 1, Z: 2, Label: "a", Pt2d: { X: 0, Y: 0 }, Pt3d: { X: 0, Y: 0, Z: 0 } }, + { X: 3, Y: 4, Z: 5, Label: "b", Pt2d: { X: 1, Y: 1 }, Pt3d: { X: 1, Y: 1, Z: 1 } }, + ]); + assert.equal(elem!.RelatedElem.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.RelatedElem.RelECClassId), "TestDomain:Test2dUsesElement"); + // Object.keys + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", + "Category", "Origin", "Rotation", "BBoxLow", "BBoxHigh", "GeometryStream", + "StrProp", "IntProp", "LongProp", "DblProp", "BoolProp", "DtProp", + "Pt2dProp", "Pt3dProp", "StructProp", "IntArrProp", "StrArrProp", "StructArrProp", "RelatedElem", "BinProp" + ].sort()); + // $meta keys + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Deleted"); + assert.equal(elem!.$meta.stage, "Old"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), [ + 'BBoxHigh', 'BBoxLow', 'BinProp', 'BoolProp', 'Category.Id', 'CodeScope.Id', + 'CodeSpec.Id', 'CodeValue', 'DblProp', 'DtProp', 'ECClassId', 'ECInstanceId', + 'FederationGuid', 'GeometryStream', 'IntArrProp', 'IntProp', 'JsonProperties', + 'LastMod', 'LongProp', 'Model.Id', 'Origin', 'Parent', 'Pt2dProp', 'Pt3dProp', + 'RelatedElem', 'Rotation', 'StrArrProp', 'StrProp', 'StructArrProp', 'StructProp.Label', + 'StructProp.Pt2d', 'StructProp.Pt3d', 'StructProp.X', 'StructProp.Y', 'StructProp.Z', + 'TypeDefinition', 'UserLabel' + ].sort()); + assert.deepEqual(elem!.$meta.rowOptions, {}); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - await rwIModel.locks.acquireLocks({ exclusive: id }); - txn.updateElement(updatedElementProps); - txn.saveChanges(); + it("txn1 insert-full | Bis_Element_Properties", () => { + const instances = readTxn(rwIModel, txnId, PropertyFilter.BisCoreElement, { classIdsToClassNames: true }); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal(modelNew!.ECClassId, "BisCore.DrawingModel"); + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal(modelOld!.ECClassId, "BisCore.DrawingModel"); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Inserted New (Bis_Element_Properties: no custom props) --- + const elem = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, fullElementId); + assert.equal(elem!.ECClassId, "TestDomain.Test2dElement"); + // No custom domain props + assert.isUndefined(elem!.StrProp); + assert.isUndefined(elem!.IntProp); + assert.isUndefined(elem!.LongProp); + assert.isUndefined(elem!.DblProp); + assert.isUndefined(elem!.BoolProp); + assert.isUndefined(elem!.DtProp); + assert.isUndefined(elem!.Pt2dProp); + assert.isUndefined(elem!.Pt3dProp); + assert.isUndefined(elem!.StructProp); + assert.isUndefined(elem!.IntArrProp); + assert.isUndefined(elem!.StrArrProp); + assert.isUndefined(elem!.StructArrProp); + assert.isUndefined(elem!.RelatedElem); + expect(elem!.Model).to.exist; + expect(elem!.CodeScope).to.exist; + expect(elem!.CodeSpec).to.exist; + expect(elem!.FederationGuid).to.exist; + expect(elem!.LastMod).to.exist; + assert.deepEqual(Object.keys(elem!).sort(), ["ECInstanceId", "ECClassId", "$meta", "CodeScope", "CodeSpec", "FederationGuid", "LastMod", "Model"].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "ECClassId", "CodeScope.Id", "CodeSpec.Id", + "CodeValue", "FederationGuid", "JsonProperties", "LastMod", "Model.Id", "Parent", "UserLabel"].sort()); + assert.deepEqual(elem!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - // 5. Push changes to hub. - await rwIModel.pushChanges({ description: "update element", accessToken: adminToken }); + it("txn1 insert-full | Instance_Key", () => { + const instances = readTxn(rwIModel, txnId, PropertyFilter.InstanceKey); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.deepEqual([...modelNew!.$meta.tables].sort(), ["bis_Model"].sort()); + assert.deepEqual([...modelNew!.$meta.changeIndexes].sort(), [3].sort()); + assert.isString(modelNew!.$meta.instanceKey); + assert.equal(modelNew!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Inserted New (only ECInstanceId + ECClassId) --- + const elem = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, fullElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.isUndefined(elem!.StrProp); + assert.isUndefined(elem!.Model); + assert.isUndefined(elem!.Category); + assert.isUndefined(elem!.LastMod); + assert.deepEqual(Object.keys(elem!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "ECClassId"].sort()); + assert.deepEqual(elem!.$meta.rowOptions, {}); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - await rwIModel.locks.acquireLocks({ exclusive: id }); + it("txn1 | rowOptions: classIdsToClassNames", () => { + const instances = readTxn(rwIModel, txnId, undefined, { classIdsToClassNames: true }); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New (ECClassId is now class name) --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal(modelNew!.ECClassId, "BisCore.DrawingModel"); + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal(modelOld!.ECClassId, "BisCore.DrawingModel"); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + + // --- instances[2]: Test2dElement Inserted New (ECClassId + all RelECClassId = class names) --- + const elem = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, fullElementId); + assert.equal("TestDomain.Test2dElement", elem!.ECClassId); + assert.deepEqual(elem!.Model, { Id: drawingModelId, RelECClassId: "BisCore.ModelContainsElements" }); + assert.isString(elem!.LastMod); + assert.deepEqual(elem!.CodeSpec, { Id: "0x1", RelECClassId: "BisCore.CodeSpecSpecifiesCode" }); + assert.deepEqual(elem!.CodeScope, { Id: "0x1", RelECClassId: "BisCore.ElementScopesCode" }); + assert.isString(elem!.FederationGuid); + assert.deepEqual(elem!.Category, { Id: drawingCategoryId, RelECClassId: "BisCore.GeometricElement2dIsInCategory" }); + assert.deepEqual(elem!.Origin, { X: 0, Y: 0 }); + assert.equal(elem!.Rotation, 0); + assert.deepEqual(elem!.BBoxLow, { X: -25, Y: -25 }); + assert.deepEqual(elem!.BBoxHigh, { X: 15, Y: 15 }); + assert.include(String(elem!.GeometryStream), "\"bytes\""); + assert.include(String(elem!.BinProp), "\"bytes\""); + assert.equal(elem!.StrProp, "hello"); + assert.equal(elem!.IntProp, 42); + assert.equal(elem!.LongProp, 9007199254740991); + assert.closeTo(elem!.DblProp as number, 3.14159265358979, 1e-10); + assert.equal(elem!.BoolProp, true); + assert.equal(elem!.DtProp, "2024-01-15T12:00:00.000"); + assert.deepEqual(elem!.Pt2dProp, { X: 1.5, Y: 2.5 }); + assert.deepEqual(elem!.Pt3dProp, { X: 3, Y: 4, Z: 5 }); + assert.deepEqual(elem!.StructProp, { X: 1, Y: 2, Z: 3, Label: "origin", Pt2d: { X: 0.5, Y: 0.5 }, Pt3d: { X: 1, Y: 2, Z: 3 } }); + assert.deepEqual(elem!.IntArrProp, [10, 20, 30]); + assert.deepEqual(elem!.StrArrProp, ["alpha", "beta", "gamma"]); + assert.equal(elem!.StructArrProp.length, 2); + assert.deepEqual(elem!.RelatedElem, { Id: drawingCategoryId, RelECClassId: "TestDomain.Test2dUsesElement" }); + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", + "Category", "Origin", "Rotation", "BBoxLow", "BBoxHigh", "GeometryStream", + "StrProp", "IntProp", "LongProp", "DblProp", "BoolProp", "DtProp", + "Pt2dProp", "Pt3dProp", "StructProp", "IntArrProp", "StrArrProp", "StructArrProp", "RelatedElem", "BinProp" + ].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), + ["ECInstanceId", "ECClassId", "Model.Id", "LastMod", "CodeSpec.Id", "CodeScope.Id", + "CodeValue", "UserLabel", "Parent", "FederationGuid", "JsonProperties", "Category.Id", + "Origin", "Rotation", "BBoxLow", "BBoxHigh", "GeometryStream", "TypeDefinition", "StrProp", + "IntProp", "LongProp", "DblProp", "BoolProp", "DtProp", "BinProp", "Pt2dProp", "Pt3dProp", + "StructProp.X", "StructProp.Y", "StructProp.Z", "StructProp.Label", "StructProp.Pt2d", + "StructProp.Pt3d", "IntArrProp", "StrArrProp", "StructArrProp", "RelatedElem"].sort()); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - // 6. Delete the element. - txn.deleteElement(id); - txn.saveChanges(); - await rwIModel.pushChanges({ description: "delete element", accessToken: adminToken }); + it("txn1 | rowOptions: useJsName", () => { + const instances = readTxn(rwIModel, txnId, undefined, { useJsName: true }); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New (all keys camelCase) --- + const modelNew = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.id, drawingModelId); + assert.equal(modelNew!.className, "BisCore.DrawingModel"); + assert.isString(modelNew!.lastMod); + assert.isString(modelNew!.geometryGuid); + assert.isUndefined(modelNew!.ECInstanceId); + assert.isUndefined(modelNew!.ECClassId); + assert.deepEqual(Object.keys(modelNew!).sort(), ["id", "className", "lastMod", "geometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, { useJsName: true }); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.id, drawingModelId); + assert.equal(modelOld!.className, "BisCore.DrawingModel"); + assert.isUndefined(modelOld!.ECInstanceId); + assert.deepEqual(Object.keys(modelOld!).sort(), ["id", "className", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, { useJsName: true }); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Inserted New (camelCase keys + class names for nav) --- + const elem = instances.find((i) => i.id === fullElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.id, fullElementId); + assert.equal(elem!.className, "TestDomain.Test2dElement"); + assert.isUndefined(elem!.ECInstanceId); + assert.isUndefined(elem!.ECClassId); + assert.isUndefined(elem!.StrProp); + assert.deepEqual(elem!.model, { id: drawingModelId, relClassName: "BisCore.ModelContainsElements" }); + assert.isString(elem!.lastMod); + assert.deepEqual(elem!.codeSpec, { id: "0x1", relClassName: "BisCore.CodeSpecSpecifiesCode" }); + assert.deepEqual(elem!.codeScope, { id: "0x1", relClassName: "BisCore.ElementScopesCode" }); + assert.isString(elem!.federationGuid); + assert.deepEqual(elem!.category, { id: drawingCategoryId, relClassName: "BisCore.GeometricElement2dIsInCategory" }); + assert.deepEqual(elem!.origin, { x: 0, y: 0 }); + assert.equal(elem!.rotation, 0); + assert.deepEqual(elem!.bBoxLow, { x: -25, y: -25 }); + assert.deepEqual(elem!.bBoxHigh, { x: 15, y: 15 }); + assert.include(String(elem!.geometryStream), "\"bytes\""); + assert.include(String(elem!.binProp), "\"bytes\""); + assert.equal(elem!.strProp, "hello"); + assert.equal(elem!.intProp, 42); + assert.equal(elem!.longProp, 9007199254740991); + assert.closeTo(elem!.dblProp as number, 3.14159265358979, 1e-10); + assert.equal(elem!.boolProp, true); + assert.equal(elem!.dtProp, "2024-01-15T12:00:00.000"); + assert.deepEqual(elem!.pt2dProp, { x: 1.5, y: 2.5 }); + assert.deepEqual(elem!.pt3dProp, { x: 3, y: 4, z: 5 }); + assert.deepEqual(elem!.structProp, { x: 1, y: 2, z: 3, label: "origin", pt2d: { x: 0.5, y: 0.5 }, pt3d: { x: 1, y: 2, z: 3 } }); + assert.deepEqual(elem!.intArrProp, [10, 20, 30]); + assert.deepEqual(elem!.strArrProp, ["alpha", "beta", "gamma"]); + assert.equal(elem!.structArrProp.length, 2); + assert.deepEqual(elem!.structArrProp[0], { x: 0, y: 1, z: 2, label: "a", pt2d: { x: 0, y: 0 }, pt3d: { x: 0, y: 0, z: 0 } }); + assert.deepEqual(elem!.relatedElem, { id: drawingCategoryId, relClassName: "TestDomain.Test2dUsesElement" }); + assert.deepEqual(Object.keys(elem!).sort(), [ + "id", "className", "model", "lastMod", "codeSpec", "codeScope", "federationGuid", "$meta", + "category", "origin", "rotation", "bBoxLow", "bBoxHigh", "geometryStream", + "strProp", "intProp", "longProp", "dblProp", "boolProp", "dtProp", + "pt2dProp", "pt3dProp", "structProp", "intArrProp", "strArrProp", "structArrProp", "relatedElem", "binProp" + ].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), [ + 'BBoxHigh', 'BBoxLow', 'BinProp', 'BoolProp', 'Category.Id', 'CodeScope.Id', + 'CodeSpec.Id', 'CodeValue', 'DblProp', 'DtProp', 'ECClassId', + 'ECInstanceId', 'FederationGuid', 'GeometryStream', 'IntArrProp', 'IntProp', + 'JsonProperties', 'LastMod', 'LongProp', 'Model.Id', 'Origin', 'Parent', 'Pt2dProp', + 'Pt3dProp', 'RelatedElem', 'Rotation', 'StrArrProp', 'StrProp', 'StructArrProp', 'StructProp.Label', + 'StructProp.Pt2d', 'StructProp.Pt3d', 'StructProp.X', 'StructProp.Y', + 'StructProp.Z', 'TypeDefinition', 'UserLabel' + ].sort()); + assert.deepEqual(elem!.$meta.rowOptions, { useJsName: true }); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); - const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); - const reader = SqliteChangesetReader.openFile({ fileName: changesets[1].pathname, db: rwIModel, disableSchemaCheck: true }); - - // Set ExclusiveRootClassId to NULL for overflow table to simulate the issue - expect(rwIModel[_nativeDb].executeSql("UPDATE ec_Table SET ExclusiveRootClassId=NULL WHERE Name='bis_GeometricElement2d_Overflow'")).to.be.eq(DbResult.BE_SQLITE_OK); - - const adaptor = new ECChangesetAdaptor(reader); - let assertOnOverflowTable = false; - const classId = getClassIdByName(rwIModel, "GeometricElement2d"); - - while (adaptor.step()) { - if (adaptor.op === "Updated" && adaptor.inserted?.$meta?.tables[0] === "bis_GeometricElement2d_Overflow") { - - assert.isUndefined(adaptor.inserted.ECClassId); - assert.equal(adaptor.inserted.ECInstanceId, ""); - assert.deepEqual(adaptor.inserted.$meta?.tables, ["bis_GeometricElement2d_Overflow"]); - assert.equal(adaptor.inserted.$meta?.op, "Updated"); - assert.equal(adaptor.inserted.$meta?.classFullName, "BisCore:GeometricElement2d"); - assert.equal(adaptor.inserted.$meta.fallbackClassId, classId); - assert.deepEqual(adaptor.inserted.$meta?.changeIndexes, [3]); - assert.equal(adaptor.inserted.$meta?.stage, "New"); - - assert.equal(adaptor.deleted!.ECInstanceId, ""); - assert.isUndefined(adaptor.deleted!.ECClassId); - assert.deepEqual(adaptor.deleted!.$meta?.tables, ["bis_GeometricElement2d_Overflow"]); - assert.equal(adaptor.deleted!.$meta?.op, "Updated"); - assert.equal(adaptor.deleted!.$meta?.classFullName, "BisCore:GeometricElement2d"); - assert.equal(adaptor.deleted!.$meta!.fallbackClassId, classId); - assert.deepEqual(adaptor.deleted!.$meta?.changeIndexes, [3]); - assert.equal(adaptor.deleted!.$meta?.stage, "Old"); - - assertOnOverflowTable = true; - } - } + it("txn1 | rowOptions: abbreviateBlobs", () => { + const instances = readTxn(rwIModel, txnId, undefined, { abbreviateBlobs: true }); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); // still raw hex + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, { abbreviateBlobs: true }); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, { abbreviateBlobs: true }); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Inserted New --- + const elem = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, fullElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); // still raw hex (no classIdsToClassNames) + assert.equal(elem!.Model.Id, drawingModelId); + assert.equal(rwIModel.getClassNameFromId(elem!.Model.RelECClassId), "BisCore:ModelContainsElements"); + assert.isString(elem!.LastMod); + assert.equal(elem!.CodeSpec.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeSpec.RelECClassId), "BisCore:CodeSpecSpecifiesCode"); + + assert.equal(elem!.CodeScope.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeScope.RelECClassId), "BisCore:ElementScopesCode"); + assert.isString(elem!.FederationGuid); + assert.equal(elem!.Category.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.Category.RelECClassId), "BisCore:GeometricElement2dIsInCategory"); + assert.deepEqual(elem!.Origin, { X: 0, Y: 0 }); + assert.equal(elem!.Rotation, 0); + assert.deepEqual(elem!.BBoxLow, { X: -25, Y: -25 }); + assert.deepEqual(elem!.BBoxHigh, { X: 15, Y: 15 }); + // GeometryStream is a blob — abbreviated to {"bytes": N} + assert.include(String(elem!.GeometryStream), "\"bytes\""); + assert.include(String(elem!.BinProp), "\"bytes\""); + assert.equal(elem!.StrProp, "hello"); + assert.equal(elem!.IntProp, 42); + assert.equal(elem!.LongProp, 9007199254740991); + assert.closeTo(elem!.DblProp as number, 3.14159265358979, 1e-10); + assert.equal(elem!.BoolProp, true); + assert.equal(elem!.DtProp, "2024-01-15T12:00:00.000"); + assert.deepEqual(elem!.Pt2dProp, { X: 1.5, Y: 2.5 }); + assert.deepEqual(elem!.Pt3dProp, { X: 3, Y: 4, Z: 5 }); + assert.deepEqual(elem!.StructProp, { X: 1, Y: 2, Z: 3, Label: "origin", Pt2d: { X: 0.5, Y: 0.5 }, Pt3d: { X: 1, Y: 2, Z: 3 } }); + assert.deepEqual(elem!.IntArrProp, [10, 20, 30]); + assert.deepEqual(elem!.StrArrProp, ["alpha", "beta", "gamma"]); + assert.equal(elem!.StructArrProp.length, 2); + assert.equal(elem!.RelatedElem.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.RelatedElem.RelECClassId), "TestDomain:Test2dUsesElement"); + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", + "Category", "Origin", "Rotation", "BBoxLow", "BBoxHigh", "GeometryStream", + "StrProp", "IntProp", "LongProp", "DblProp", "BoolProp", "DtProp", + "Pt2dProp", "Pt3dProp", "StructProp", "IntArrProp", "StrArrProp", "StructArrProp", "RelatedElem", "BinProp" + ].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { abbreviateBlobs: true }); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - assert.isTrue(assertOnOverflowTable); - txn.end(); - rwIModel.close(); + it("txn1 | rowOptions: abbreviateBlobs: false", () => { + const instances = readTxn(rwIModel, txnId, undefined, { abbreviateBlobs: false }, undefined, false); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); // still raw hex + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, { abbreviateBlobs: false }); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, { abbreviateBlobs: false }); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Inserted New --- + const elem = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, fullElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); // still raw hex (no classIdsToClassNames) + assert.equal(elem!.Model.Id, drawingModelId); + assert.equal(rwIModel.getClassNameFromId(elem!.Model.RelECClassId), "BisCore:ModelContainsElements"); + assert.isString(elem!.LastMod); + assert.equal(elem!.CodeSpec.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeSpec.RelECClassId), "BisCore:CodeSpecSpecifiesCode"); + + assert.equal(elem!.CodeScope.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeScope.RelECClassId), "BisCore:ElementScopesCode"); + assert.isString(elem!.FederationGuid); + assert.equal(elem!.Category.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.Category.RelECClassId), "BisCore:GeometricElement2dIsInCategory"); + assert.deepEqual(elem!.Origin, { X: 0, Y: 0 }); + assert.equal(elem!.Rotation, 0); + assert.deepEqual(elem!.BBoxLow, { X: -25, Y: -25 }); + assert.deepEqual(elem!.BBoxHigh, { X: 15, Y: 15 }); + // GeometryStream is a blob — abbreviated to {"bytes": N} + assert.deepEqual(elem!.GeometryStream, new Uint8Array([ + 1, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 112, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 88, 0, 8, 0, 32, 0, 56, 0, 0, 0, 80, 0, 7, 0, + 16, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 20, 64, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 20, 64, 0, 0, 0, 0, 0, 0, 0, 0, + 24, 45, 68, 84, 251, 33, 25, 64, 7, 0, 0, 0, 112, 0, 0, 0, + 24, 0, 0, 0, 0, 0, 0, 0, 16, 0, 88, 0, 8, 0, 32, 0, + 56, 0, 0, 0, 80, 0, 7, 0, 16, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 20, 64, 0, 0, 0, 0, 0, 0, 20, 64, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, + 0, 0, 0, 0, 0, 0, 0, 0, 24, 45, 68, 84, 251, 33, 25, 64, + 7, 0, 0, 0, 112, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 88, 0, 8, 0, 32, 0, 56, 0, 0, 0, 80, 0, 7, 0, + 16, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 20, 192, + 0, 0, 0, 0, 0, 0, 20, 192, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 0, 0, + 24, 45, 68, 84, 251, 33, 25, 64 + ])); + assert.deepEqual(elem!.BinProp, new Uint8Array([1, 2, 3, 4])); + assert.equal(elem!.StrProp, "hello"); + assert.equal(elem!.IntProp, 42); + assert.equal(elem!.LongProp, 9007199254740991); + assert.closeTo(elem!.DblProp as number, 3.14159265358979, 1e-10); + assert.equal(elem!.BoolProp, true); + assert.equal(elem!.DtProp, "2024-01-15T12:00:00.000"); + assert.deepEqual(elem!.Pt2dProp, { X: 1.5, Y: 2.5 }); + assert.deepEqual(elem!.Pt3dProp, { X: 3, Y: 4, Z: 5 }); + assert.deepEqual(elem!.StructProp, { X: 1, Y: 2, Z: 3, Label: "origin", Pt2d: { X: 0.5, Y: 0.5 }, Pt3d: { X: 1, Y: 2, Z: 3 } }); + assert.deepEqual(elem!.IntArrProp, [10, 20, 30]); + assert.deepEqual(elem!.StrArrProp, ["alpha", "beta", "gamma"]); + assert.equal(elem!.StructArrProp.length, 2); + assert.equal(elem!.RelatedElem.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.RelatedElem.RelECClassId), "TestDomain:Test2dUsesElement"); + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", + "Category", "Origin", "Rotation", "BBoxLow", "BBoxHigh", "GeometryStream", + "StrProp", "IntProp", "LongProp", "DblProp", "BoolProp", "DtProp", + "Pt2dProp", "Pt3dProp", "StructProp", "IntArrProp", "StrArrProp", "StructArrProp", "RelatedElem", "BinProp" + ].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { abbreviateBlobs: false }); + assert.equal(elem!.$meta.isIndirectChange, false); }); - function getClassIdByName(iModel: BriefcaseDb, className: string): Id64String { - // eslint-disable-next-line @typescript-eslint/no-deprecated - return iModel.withPreparedStatement(`SELECT ECInstanceId from meta.ECClassDef where Name=?`, (stmt) => { - stmt.bindString(1, className); - assert.equal(stmt.step(), DbResult.BE_SQLITE_ROW); - return stmt.getValue(0).getId(); - }); - } + it("txn1 | rowOptions: classIdsToClassNames + useJsName", () => { + const instances = readTxn(rwIModel, txnId, undefined, { classIdsToClassNames: true, useJsName: true }, undefined, false); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.id, drawingModelId); + assert.equal(modelNew!.className, "BisCore.DrawingModel"); + assert.isString(modelNew!.lastMod); + assert.isString(modelNew!.geometryGuid); + assert.isUndefined(modelNew!.ECInstanceId); + assert.deepEqual(Object.keys(modelNew!).sort(), ["id", "className", "lastMod", "geometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.id, drawingModelId); + assert.equal(modelOld!.className, "BisCore.DrawingModel"); + assert.isUndefined(modelOld!.ECInstanceId); + assert.deepEqual(Object.keys(modelOld!).sort(), ["id", "className", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Inserted New (camelCase + class names) --- + const elem = instances.find((i) => i.id === fullElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.id, fullElementId); + assert.equal(elem!.className, "TestDomain.Test2dElement"); + assert.isUndefined(elem!.ECInstanceId); + assert.isUndefined(elem!.ECClassId); + assert.isUndefined(elem!.StrProp); + assert.deepEqual(elem!.model, { id: drawingModelId, relClassName: "BisCore.ModelContainsElements" }); + assert.isString(elem!.lastMod); + assert.deepEqual(elem!.codeSpec, { id: "0x1", relClassName: "BisCore.CodeSpecSpecifiesCode" }); + assert.deepEqual(elem!.codeScope, { id: "0x1", relClassName: "BisCore.ElementScopesCode" }); + assert.isString(elem!.federationGuid); + assert.deepEqual(elem!.category, { id: drawingCategoryId, relClassName: "BisCore.GeometricElement2dIsInCategory" }); + assert.deepEqual(elem!.origin, { x: 0, y: 0 }); + assert.equal(elem!.rotation, 0); + assert.deepEqual(elem!.bBoxLow, { x: -25, y: -25 }); + assert.deepEqual(elem!.bBoxHigh, { x: 15, y: 15 }); + assert.include(String(elem!.geometryStream), "\"bytes\""); + assert.include(String(elem!.binProp), "\"bytes\""); + assert.equal(elem!.strProp, "hello"); + assert.equal(elem!.intProp, 42); + assert.equal(elem!.longProp, 9007199254740991); + assert.closeTo(elem!.dblProp as number, 3.14159265358979, 1e-10); + assert.equal(elem!.boolProp, true); + assert.equal(elem!.dtProp, "2024-01-15T12:00:00.000"); + assert.deepEqual(elem!.pt2dProp, { x: 1.5, y: 2.5 }); + assert.deepEqual(elem!.pt3dProp, { x: 3, y: 4, z: 5 }); + assert.deepEqual(elem!.structProp, { x: 1, y: 2, z: 3, label: "origin", pt2d: { x: 0.5, y: 0.5 }, pt3d: { x: 1, y: 2, z: 3 } }); + assert.deepEqual(elem!.intArrProp, [10, 20, 30]); + assert.deepEqual(elem!.strArrProp, ["alpha", "beta", "gamma"]); + assert.equal(elem!.structArrProp.length, 2); + assert.deepEqual(elem!.relatedElem, { id: drawingCategoryId, relClassName: "TestDomain.Test2dUsesElement" }); + assert.deepEqual(Object.keys(elem!).sort(), [ + "id", "className", "model", "lastMod", "codeSpec", "codeScope", "federationGuid", "$meta", + "category", "origin", "rotation", "bBoxLow", "bBoxHigh", "geometryStream", + "strProp", "intProp", "longProp", "dblProp", "boolProp", "dtProp", + "pt2dProp", "pt3dProp", "structProp", "intArrProp", "strArrProp", "structArrProp", "relatedElem", "binProp" + ].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), [ + 'BBoxHigh', 'BBoxLow', 'BinProp', 'BoolProp', 'Category.Id', + 'CodeScope.Id', 'CodeSpec.Id', + 'CodeValue', 'DblProp', 'DtProp', + 'ECClassId', 'ECInstanceId', 'FederationGuid', + 'GeometryStream', 'IntArrProp', 'IntProp', + 'JsonProperties', 'LastMod', 'LongProp', + 'Model.Id', 'Origin', 'Parent', 'Pt2dProp', 'Pt3dProp', 'RelatedElem', 'Rotation', 'StrArrProp', + 'StrProp', 'StructArrProp', 'StructProp.Label', 'StructProp.Pt2d', 'StructProp.Pt3d', + 'StructProp.X', 'StructProp.Y', 'StructProp.Z', 'TypeDefinition', 'UserLabel' + ].sort()); + assert.deepEqual(elem!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - async function getClassNameById(iModel: BriefcaseDb, classId: string): Promise { - const reader = iModel.createQueryReader(`select ec_classname(${classId});`); + it("txn1 insert-full | All_Properties | inMemoryCache vs sqliteBackedCache", () => { + const inMemoryInstances = readTxn(rwIModel, txnId); + const sqliteBackedInstances = readTxn(rwIModel, txnId, undefined, undefined, undefined, false); + assert.equal(inMemoryInstances.length, 3); + assert.equal(sqliteBackedInstances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const inMemoryModelNew = inMemoryInstances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + const sqliteBackedModelNew = sqliteBackedInstances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(inMemoryModelNew).to.exist; + expect(sqliteBackedModelNew).to.exist; + assert.equal(inMemoryModelNew!.ECInstanceId, sqliteBackedModelNew!.ECInstanceId); + assert.equal(sqliteBackedModelNew!.ECClassId, inMemoryModelNew!.ECClassId); + assert.equal(inMemoryModelNew!.LastMod, sqliteBackedModelNew!.LastMod); + assert.equal(inMemoryModelNew!.GeometryGuid, sqliteBackedModelNew!.GeometryGuid); + // Object.keys + assert.deepEqual(Object.keys(inMemoryModelNew!).sort(), Object.keys(sqliteBackedModelNew!).sort()); + // $meta keys + assert.deepEqual(Object.keys(inMemoryModelNew!.$meta).sort(), Object.keys(sqliteBackedModelNew!.$meta).sort()); + assert.equal(inMemoryModelNew!.$meta.op, sqliteBackedModelNew!.$meta.op); + assert.equal(inMemoryModelNew!.$meta.stage, sqliteBackedModelNew!.$meta.stage); + assert.deepEqual([...inMemoryModelNew!.$meta.tables].sort(), [...sqliteBackedModelNew!.$meta.tables].sort()); + assert.deepEqual([...inMemoryModelNew!.$meta.changeIndexes].sort(), [...sqliteBackedModelNew!.$meta.changeIndexes].sort()); + assert.isString(inMemoryModelNew!.$meta.instanceKey); + assert.equal(inMemoryModelNew!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(inMemoryModelNew!.$meta.propFilter, sqliteBackedModelNew!.$meta.propFilter); + assert.deepEqual([...inMemoryModelNew!.$meta.changeFetchedPropNames].sort(), [...sqliteBackedModelNew!.$meta.changeFetchedPropNames].sort()); + assert.deepEqual(inMemoryModelNew!.$meta.rowOptions, sqliteBackedModelNew!.$meta.rowOptions); + assert.equal(inMemoryModelNew!.$meta.isIndirectChange, sqliteBackedModelNew!.$meta.isIndirectChange); + + // --- instances[1]: DrawingModel Updated Old --- + const inMemoryModelOld = inMemoryInstances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + const sqliteBackedModelOld = sqliteBackedInstances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(inMemoryModelOld).to.exist; + expect(sqliteBackedModelOld).to.exist; + assert.equal(inMemoryModelOld!.ECInstanceId, sqliteBackedModelOld!.ECInstanceId); + assert.equal(sqliteBackedModelOld!.ECClassId, inMemoryModelOld!.ECClassId); + // Object.keys + assert.deepEqual(Object.keys(inMemoryModelOld!).sort(), Object.keys(sqliteBackedModelOld!).sort()); + // $meta keys + assert.deepEqual(Object.keys(inMemoryModelOld!.$meta).sort(), Object.keys(sqliteBackedModelOld!.$meta).sort()); + assert.equal(inMemoryModelOld!.$meta.op, sqliteBackedModelOld!.$meta.op); + assert.equal(inMemoryModelOld!.$meta.stage, sqliteBackedModelOld!.$meta.stage); + assert.deepEqual([...inMemoryModelOld!.$meta.tables].sort(), [...sqliteBackedModelOld!.$meta.tables].sort()); + assert.equal(inMemoryModelOld!.$meta.propFilter, sqliteBackedModelOld!.$meta.propFilter); + assert.deepEqual([...inMemoryModelOld!.$meta.changeFetchedPropNames].sort(), [...sqliteBackedModelOld!.$meta.changeFetchedPropNames].sort()); + assert.deepEqual(inMemoryModelOld!.$meta.rowOptions, sqliteBackedModelOld!.$meta.rowOptions); + assert.equal(inMemoryModelOld!.$meta.isIndirectChange, sqliteBackedModelOld!.$meta.isIndirectChange); + + // --- instances[2]: Test2dElement Inserted New --- + const inMemoryElem = inMemoryInstances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + const sqliteBackedElem = sqliteBackedInstances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(inMemoryElem).to.exist; + expect(sqliteBackedElem).to.exist; + assert.equal(inMemoryElem!.ECInstanceId, sqliteBackedElem!.ECInstanceId); + assert.equal(sqliteBackedElem!.ECClassId, inMemoryElem!.ECClassId); + assert.equal(inMemoryElem!.Model.Id, sqliteBackedElem!.Model.Id); + assert.equal(inMemoryElem!.Model.RelECClassId, sqliteBackedElem!.Model.RelECClassId); + assert.equal(inMemoryElem!.LastMod, sqliteBackedElem!.LastMod); + assert.equal(inMemoryElem!.CodeSpec.Id, sqliteBackedElem!.CodeSpec.Id); + assert.equal(inMemoryElem!.CodeSpec.RelECClassId, sqliteBackedElem!.CodeSpec.RelECClassId); + + assert.equal(inMemoryElem!.CodeScope.Id, sqliteBackedElem!.CodeScope.Id); + assert.equal(inMemoryElem!.CodeScope.RelECClassId, sqliteBackedElem!.CodeScope.RelECClassId); + assert.equal(inMemoryElem!.FederationGuid, sqliteBackedElem!.FederationGuid); + + assert.equal(inMemoryElem!.Category.Id, sqliteBackedElem!.Category.Id); + assert.equal(inMemoryElem!.Category.RelECClassId, sqliteBackedElem!.Category.RelECClassId); + assert.deepEqual(inMemoryElem!.Origin, sqliteBackedElem!.Origin); + assert.equal(inMemoryElem!.Rotation, sqliteBackedElem!.Rotation); + assert.deepEqual(inMemoryElem!.BBoxLow, sqliteBackedElem!.BBoxLow); + assert.deepEqual(inMemoryElem!.BBoxHigh, sqliteBackedElem!.BBoxHigh); + assert.equal(inMemoryElem!.GeometryStream, sqliteBackedElem!.GeometryStream); + assert.equal(inMemoryElem!.BinProp, sqliteBackedElem!.BinProp); + assert.equal(inMemoryElem!.StrProp, sqliteBackedElem!.StrProp); + assert.equal(inMemoryElem!.IntProp, sqliteBackedElem!.IntProp); + assert.equal(inMemoryElem!.LongProp, sqliteBackedElem!.LongProp); + assert.closeTo(inMemoryElem!.DblProp as number, sqliteBackedElem!.DblProp as number, 1e-10); + assert.equal(inMemoryElem!.BoolProp, sqliteBackedElem!.BoolProp); + assert.equal(inMemoryElem!.DtProp, sqliteBackedElem!.DtProp); + assert.deepEqual(inMemoryElem!.Pt2dProp, sqliteBackedElem!.Pt2dProp); + assert.deepEqual(inMemoryElem!.Pt3dProp, sqliteBackedElem!.Pt3dProp); + assert.deepEqual(inMemoryElem!.StructProp, sqliteBackedElem!.StructProp); + assert.deepEqual(inMemoryElem!.IntArrProp, sqliteBackedElem!.IntArrProp); + assert.deepEqual(inMemoryElem!.StrArrProp, sqliteBackedElem!.StrArrProp); + assert.deepEqual(inMemoryElem!.StructArrProp, sqliteBackedElem!.StructArrProp); + assert.equal(inMemoryElem!.RelatedElem.Id, sqliteBackedElem!.RelatedElem.Id); + assert.equal(inMemoryElem!.RelatedElem.RelECClassId, sqliteBackedElem!.RelatedElem.RelECClassId); + // Object.keys + assert.deepEqual(Object.keys(inMemoryElem!).sort(), Object.keys(sqliteBackedElem!).sort()); + // $meta keys + assert.deepEqual(Object.keys(inMemoryElem!.$meta).sort(), Object.keys(sqliteBackedElem!.$meta).sort()); + assert.equal(inMemoryElem!.$meta.op, sqliteBackedElem!.$meta.op); + assert.equal(inMemoryElem!.$meta.stage, sqliteBackedElem!.$meta.stage); + assert.deepEqual([...inMemoryElem!.$meta.tables].sort(), [...sqliteBackedElem!.$meta.tables].sort()); + assert.deepEqual([...inMemoryElem!.$meta.changeIndexes].sort(), [...sqliteBackedElem!.$meta.changeIndexes].sort()); + assert.equal(inMemoryElem!.$meta.instanceKey, sqliteBackedElem!.$meta.instanceKey); + assert.deepEqual([...inMemoryElem!.$meta.changeFetchedPropNames].sort(), [...sqliteBackedElem!.$meta.changeFetchedPropNames].sort()); + assert.deepEqual(inMemoryElem!.$meta.rowOptions, sqliteBackedElem!.$meta.rowOptions); + assert.equal(inMemoryElem!.$meta.isIndirectChange, sqliteBackedElem!.$meta.isIndirectChange); + }); + it("should throw error if tried to fetch changeset metadata values before stepping", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + expect(() => reader.isECTable).to.throw(); + expect(() => reader.isIndirectChange).to.throw(); + expect(() => reader.tableName).to.throw(); + expect(() => reader.op).to.throw(); + assert.isUndefined(reader.inserted); + assert.isUndefined(reader.deleted); + reader.close(); + }); + it("should throw error if tried to fetch changeset metadata values after stepping past the end", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + while (reader.step()) { } + assert.equal(reader.step(), false); + expect(() => reader.isECTable).to.throw(); + expect(() => reader.isIndirectChange).to.throw(); + expect(() => reader.tableName).to.throw(); + expect(() => reader.op).to.throw(); + assert.isUndefined(reader.inserted); + assert.isUndefined(reader.deleted); + reader.close(); + }); - if (await reader.step()) - return reader.current[0] as string; + it("Checking non unified values", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + + assert.isTrue(reader.step()); + assert.isTrue(reader.isECTable); + assert.isFalse(reader.isIndirectChange); + assert.equal(reader.tableName, "bis_Element"); + assert.equal(reader.op, "Inserted"); + assert.isDefined(reader.inserted); + assert.isUndefined(reader.deleted); + + assert.isTrue(reader.step()); + assert.isTrue(reader.isECTable); + assert.isFalse(reader.isIndirectChange); + assert.equal(reader.tableName, "bis_GeometricElement2d"); + assert.equal(reader.op, "Inserted"); + assert.isDefined(reader.inserted); + assert.isUndefined(reader.deleted); + + assert.isTrue(reader.step()); + assert.isTrue(reader.isECTable); + assert.isTrue(reader.isIndirectChange); + assert.equal(reader.tableName, "bis_Model"); + assert.equal(reader.op, "Updated"); + assert.isDefined(reader.inserted); + assert.isDefined(reader.deleted); + + assert.isFalse(reader.step()); + }); +}); - return undefined; - } +describe("ChangesetReader insert-partial", () => { + let rwIModel: BriefcaseDb; + let partialElementId: Id64String; + let drawingModelId: Id64String; + let drawingCategoryId: Id64String; + let txnId: string; + let txn: EditTxn; - it("Changeset reader / EC adaptor", async () => { + before(async () => { + HubMock.startup("ECChangesetInsertPartial", KnownTestLocations.outputDir); const adminToken = "super manager token"; - const iModelName = "test"; - const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(rwIModelId); - const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); - const txn = startTestTxn(rwIModel, "changeset reader EC adaptor"); - + const iTwinId = HubMock.iTwinId; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "insertPartial", description: "insertPartial", accessToken: adminToken }); + rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + txn = startTestTxn(rwIModel, "ChangesetReader insert-partial"); + // Txn 1: import schema + drawing model setup, then push const schema = ` - - - - bis:GraphicalElement2d - - - `; + + + + + + + + + + + + + + + + + + bis:GraphicalElement2d + + + + + + + + + + + + + + + + + + + + `; await importSchemaStrings(txn, [schema]); - if (true || "push changes") { - // Push the changes to the hub - const prePushChangeSetId = rwIModel.changeset.id; - await rwIModel.pushChanges({ description: "push schema changeset", accessToken: adminToken }); - const postPushChangeSetId = rwIModel.changeset.id; - assert(!!postPushChangeSetId); - expect(prePushChangeSetId !== postPushChangeSetId); - rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - } + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); const codeProps = Code.createEmpty(); - codeProps.value = "DrawingModel"; - let totalEl = 0; - const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); - if (undefined === drawingCategoryId) - drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + codeProps.value = "DrillDownDrawing"; + [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - txn.saveChanges("user 1: create drawing partition"); - if (true || "push changes") { - // Push the changes to the hub - const prePushChangeSetId = rwIModel.changeset.id; - await rwIModel.pushChanges({ description: "user 1: create drawing partition", accessToken: adminToken }); - const postPushChangeSetId = rwIModel.changeset.id; - assert(!!postPushChangeSetId); - expect(prePushChangeSetId !== postPushChangeSetId); - } + const foundCat = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "DrillDownCategory"); + drawingCategoryId = foundCat ?? DrawingCategory.insert(txn, IModel.dictionaryId, "DrillDownCategory", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + + txn.saveChanges("setup"); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Txn 2: insert PARTIAL element only mandatory props await rwIModel.locks.acquireLocks({ shared: drawingModelId }); - const insertElements = (className: string = "Test2dElement", noOfElements: number = 10, userProp: (n: number) => object) => { - for (let m = 0; m < noOfElements; ++m) { - const geomArray: Arc3d[] = [ - Arc3d.createXY(Point3d.create(0, 0), 5), - Arc3d.createXY(Point3d.create(5, 5), 2), - Arc3d.createXY(Point3d.create(-5, -5), 20), - ]; - const geometryStream: GeometryStreamProps = []; - for (const geom of geomArray) { - const arcData = IModelJson.Writer.toIModelJson(geom); - geometryStream.push(arcData); - } - const prop = userProp(++totalEl); - // Create props - const geomElement = { - classFullName: `TestDomain:${className}`, - model: drawingModelId, - category: drawingCategoryId, - code: Code.createEmpty(), - geom: geometryStream, - ...prop, - }; - const id = txn.insertElement(geomElement); - assert.isTrue(Id64.isValidId64(id), "insert worked"); - } - }; - const generatedStr = new Array(10).join("x"); - insertElements("Test2dElement", 1, () => { - return { s: generatedStr }; - }); + partialElementId = txn.insertElement({ + classFullName: "TestDomain:Test2dElement", + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + // StrProp, IntProp, LongProp, DblProp, BoolProp, DtProp, BinProp intentionally absent + } as any); + txn.saveChanges("insert partial element"); + txnId = rwIModel.txns.getLastSavedTxnProps()!.id; + }); - const updatedElements = async () => { - await rwIModel.locks.acquireLocks({ exclusive: "0x20000000004" }); - const updatedElement = rwIModel.elements.getElementProps("0x20000000004"); - (updatedElement as any).s = "updated property"; - txn.updateElement(updatedElement); - txn.saveChanges("user 1: updated data"); - await rwIModel.pushChanges({ description: "user 1: update property id=0x20000000004", accessToken: adminToken }); - }; - - txn.saveChanges("user 1: data"); - - if (true || "test local changes") { - const testChanges = async (changes: ChangedECInstance[]) => { - assert.equal(changes.length, 3); - - assert.equal(changes[0].ECInstanceId, "0x20000000001"); - assert.equal(changes[0].$meta?.classFullName, "BisCore:DrawingModel"); - assert.equal(changes[0].$meta?.op, "Updated"); - assert.equal(changes[0].$meta?.stage, "New"); - assert.isNotNull(changes[0].LastMod); - assert.isNotNull(changes[0].GeometryGuid); - - assert.equal(changes[1].ECInstanceId, "0x20000000001"); - assert.equal(changes[1].$meta?.classFullName, "BisCore:DrawingModel"); - assert.equal(changes[1].$meta?.op, "Updated"); - assert.equal(changes[1].$meta?.stage, "Old"); - assert.isNull(changes[1].LastMod); - assert.isNull(changes[1].GeometryGuid); - - assert.equal(changes[2].ECInstanceId, "0x20000000004"); - assert.equal(changes[2].$meta?.classFullName, "TestDomain:Test2dElement"); - assert.equal(changes[2].$meta?.op, "Inserted"); - assert.equal(changes[2].$meta?.stage, "New"); - - const el = changes.filter((x) => x.ECInstanceId === "0x20000000004")[0]; - assert.equal(el.Rotation, 0); - // eslint-disable-next-line @typescript-eslint/naming-convention - assert.deepEqual(el.Origin, { X: 0, Y: 0 }); - // eslint-disable-next-line @typescript-eslint/naming-convention - assert.deepEqual(el.BBoxLow, { X: -25, Y: -25 }); - // eslint-disable-next-line @typescript-eslint/naming-convention - assert.deepEqual(el.BBoxHigh, { X: 15, Y: 15 }); - - assert.equal(el.Category.Id, "0x20000000002"); - assert.isNotEmpty(el.Category.RelECClassId); - - const categoryRelClass = await getClassNameById(rwIModel, el.Category.RelECClassId); - assert.equal("BisCore:GeometricElement2dIsInCategory", categoryRelClass); - assert.equal(el.s, "xxxxxxxxx"); - assert.isNull(el.CodeValue); - assert.isNull(el.UserLabel); - assert.isNull(el.JsonProperties); - assert.instanceOf(el.GeometryStream, Uint8Array); - assert.typeOf(el.FederationGuid, "string"); - assert.typeOf(el.LastMod, "string"); - // eslint-disable-next-line @typescript-eslint/naming-convention - assert.deepEqual(el.Parent, { Id: null, RelECClassId: null }); - // eslint-disable-next-line @typescript-eslint/naming-convention - assert.deepEqual(el.TypeDefinition, { Id: null, RelECClassId: null }); - - assert.equal(el.CodeSpec.Id, "0x1"); - assert.isNotEmpty(el.CodeSpec.RelECClassId); - - const codeSpecRelClass = await getClassNameById(rwIModel, el.CodeSpec.RelECClassId); - assert.equal("BisCore:CodeSpecSpecifiesCode", codeSpecRelClass); - - assert.equal(el.CodeScope.Id, "0x1"); - assert.isNotEmpty(el.CodeScope.RelECClassId); - - const codeScopeRelClass = await getClassNameById(rwIModel, el.CodeScope.RelECClassId); - assert.equal("BisCore:ElementScopesCode", codeScopeRelClass); - - assert.deepEqual(el.$meta, { - tables: [ - "bis_GeometricElement2d", - "bis_Element", - ], - op: "Inserted", - classFullName: "TestDomain:Test2dElement", - changeIndexes: [ - 2, - 1, - ], - stage: "New", - }); - } + after(() => { + txn.end(); + rwIModel?.close(); + HubMock.shutdown(); + }); - if (true || "test with InMemoryInstanceCache") { - using reader = SqliteChangesetReader.openLocalChanges({ db: rwIModel, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - await testChanges(Array.from(pcu.instances)); - } + it("txn2 insert-partial | All_Properties | default rowOptions", () => { + const instances = readTxn(rwIModel, txnId); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + // Model Old has LastMod and GeometryGuid when previous txn's model New values survive + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement (partial) Inserted New --- + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, partialElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.equal(elem!.Model.Id, drawingModelId); + assert.equal(rwIModel.getClassNameFromId(elem!.Model.RelECClassId), "BisCore:ModelContainsElements"); + assert.isString(elem!.LastMod); + assert.equal(elem!.CodeSpec.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeSpec.RelECClassId), "BisCore:CodeSpecSpecifiesCode"); + + assert.equal(elem!.CodeScope.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeScope.RelECClassId), "BisCore:ElementScopesCode"); + assert.isString(elem!.FederationGuid); + assert.equal(elem!.Category.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.Category.RelECClassId), "BisCore:GeometricElement2dIsInCategory"); + // No custom props on the partial element + assert.isUndefined(elem!.StrProp); + assert.isUndefined(elem!.IntProp); + assert.isUndefined(elem!.LongProp); + assert.isUndefined(elem!.DblProp); + assert.isUndefined(elem!.BoolProp); + assert.isUndefined(elem!.DtProp); + assert.isUndefined(elem!.Pt2dProp); + assert.isUndefined(elem!.Pt3dProp); + assert.isUndefined(elem!.StructProp); + assert.isUndefined(elem!.IntArrProp); + assert.isUndefined(elem!.StrArrProp); + assert.isUndefined(elem!.StructArrProp); + assert.isUndefined(elem!.RelatedElem); + assert.isUndefined(elem!.BinProp); + // Object.keys — partial insert: only BIS columns present (no 2d geometry columns either) + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", + "Category", + ].sort()); + // $meta keys + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), [ + 'ECInstanceId', 'ECClassId', 'Model.Id', 'LastMod', 'CodeSpec.Id', + 'CodeScope.Id', 'CodeValue', 'UserLabel', 'Parent', 'FederationGuid', + 'JsonProperties', 'Category.Id', 'Origin', 'Rotation', 'BBoxLow', 'BBoxHigh', + 'GeometryStream', 'TypeDefinition', 'StrProp', 'IntProp', 'LongProp', + 'DblProp', 'BoolProp', 'DtProp', 'BinProp', 'Pt2dProp', 'Pt3dProp', + 'StructProp.X', 'StructProp.Y', 'StructProp.Z', 'StructProp.Label', + 'StructProp.Pt2d', 'StructProp.Pt3d', 'IntArrProp', 'StrArrProp', + 'StructArrProp', 'RelatedElem' + ].sort()); + assert.deepEqual(elem!.$meta.rowOptions, {}); + assert.isUndefined(elem!.BinProp); + assert.equal(elem!.$meta.isIndirectChange, false); - if (true || "test with SqliteBackedInstanceCache") { - using reader = SqliteChangesetReader.openLocalChanges({ db: rwIModel, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - await testChanges(Array.from(pcu.instances)); - } - } + }); - const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); - await rwIModel.pushChanges({ description: "schema changeset", accessToken: adminToken }); + it("txn2 insert-partial | All_Properties | default rowOptions | invert", () => { + const instances = readTxn(rwIModel, txnId, undefined, undefined, true); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.isUndefined(modelNew!.LastMod); + assert.isUndefined(modelNew!.GeometryGuid); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.isString(modelOld!.LastMod); + assert.isString(modelOld!.GeometryGuid); + // Model Old has LastMod and GeometryGuid when previous txn's model New values survive + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "Old"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, partialElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.equal(elem!.Model.Id, drawingModelId); + assert.equal(rwIModel.getClassNameFromId(elem!.Model.RelECClassId), "BisCore:ModelContainsElements"); + assert.isString(elem!.LastMod); + assert.equal(elem!.CodeSpec.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeSpec.RelECClassId), "BisCore:CodeSpecSpecifiesCode"); + + assert.equal(elem!.CodeScope.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeScope.RelECClassId), "BisCore:ElementScopesCode"); + assert.isString(elem!.FederationGuid); + assert.equal(elem!.Category.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.Category.RelECClassId), "BisCore:GeometricElement2dIsInCategory"); + // No custom props on the partial element + assert.isUndefined(elem!.StrProp); + assert.isUndefined(elem!.IntProp); + assert.isUndefined(elem!.LongProp); + assert.isUndefined(elem!.DblProp); + assert.isUndefined(elem!.BoolProp); + assert.isUndefined(elem!.DtProp); + assert.isUndefined(elem!.Pt2dProp); + assert.isUndefined(elem!.Pt3dProp); + assert.isUndefined(elem!.StructProp); + assert.isUndefined(elem!.IntArrProp); + assert.isUndefined(elem!.StrArrProp); + assert.isUndefined(elem!.StructArrProp); + assert.isUndefined(elem!.RelatedElem); + assert.isUndefined(elem!.BinProp); + // Object.keys — partial insert: only BIS columns present (no 2d geometry columns either) + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", + "Category", + ].sort()); + // $meta keys + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Deleted"); + assert.equal(elem!.$meta.stage, "Old"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), [ + 'ECInstanceId', 'ECClassId', 'Model.Id', 'LastMod', 'CodeSpec.Id', + 'CodeScope.Id', 'CodeValue', 'UserLabel', 'Parent', 'FederationGuid', + 'JsonProperties', 'Category.Id', 'Origin', 'Rotation', 'BBoxLow', 'BBoxHigh', + 'GeometryStream', 'TypeDefinition', 'StrProp', 'IntProp', 'LongProp', + 'DblProp', 'BoolProp', 'DtProp', 'BinProp', 'Pt2dProp', 'Pt3dProp', + 'StructProp.X', 'StructProp.Y', 'StructProp.Z', 'StructProp.Label', + 'StructProp.Pt2d', 'StructProp.Pt3d', 'IntArrProp', 'StrArrProp', + 'StructArrProp', 'RelatedElem' + ].sort()); + assert.deepEqual(elem!.$meta.rowOptions, {}); + assert.isUndefined(elem!.BinProp); + assert.equal(elem!.$meta.isIndirectChange, false); - await updatedElements(); + }); - const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); - if (true || "updated element") { - const testChanges = (changes: ChangedECInstance[]) => { - assert.equal(changes.length, 4); - - const classId: Id64String = getClassIdByName(rwIModel, "Test2dElement"); - - // new value - assert.equal(changes[2].ECInstanceId, "0x20000000004"); - assert.equal(changes[2].ECClassId, classId); - assert.equal(changes[2].s, "updated property"); - assert.equal(changes[2].$meta?.classFullName, "TestDomain:Test2dElement"); - assert.equal(changes[2].$meta?.op, "Updated"); - assert.equal(changes[2].$meta?.stage, "New"); - - // old value - assert.equal(changes[3].ECInstanceId, "0x20000000004"); - assert.equal(changes[3].ECClassId, classId); - assert.equal(changes[3].s, "xxxxxxxxx"); - assert.equal(changes[3].$meta?.classFullName, "TestDomain:Test2dElement"); - assert.equal(changes[3].$meta?.op, "Updated"); - assert.equal(changes[3].$meta?.stage, "Old"); - }; - - if (true || "test with InMemoryInstanceCache") { - using reader = SqliteChangesetReader.openFile({ fileName: changesets[3].pathname, db: rwIModel, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - testChanges(Array.from(pcu.instances)); - } + it("txn2 insert-partial | Bis_Element_Properties", () => { + const instances = readTxn(rwIModel, txnId, PropertyFilter.BisCoreElement, { classIdsToClassNames: true }); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal(modelNew!.ECClassId, "BisCore.DrawingModel"); + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore.DrawingModel", modelOld!.ECClassId); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement (partial) Inserted New --- + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, partialElementId); + assert.equal(elem!.ECClassId, "TestDomain.Test2dElement"); + assert.isUndefined(elem!.StrProp); + assert.isUndefined(elem!.IntProp); + expect(elem!.Model).to.exist; + expect(elem!.LastMod).to.exist; + expect(elem!.CodeSpec).to.exist; + expect(elem!.CodeScope).to.exist; + expect(elem!.FederationGuid).to.exist; + assert.deepEqual(Object.keys(elem!).sort(), ["ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "ECClassId", "CodeScope.Id", + "CodeSpec.Id", "CodeValue", "FederationGuid", "JsonProperties", "LastMod", "Model.Id", "Parent", "UserLabel"].sort()); + assert.deepEqual(elem!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - if (true || "test with SqliteBackedInstanceCache") { - using reader = SqliteChangesetReader.openFile({ fileName: changesets[3].pathname, db: rwIModel, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - testChanges(Array.from(pcu.instances)); - } - } + it("txn2 insert-partial | Instance_Key", () => { + const instances = readTxn(rwIModel, txnId, PropertyFilter.InstanceKey); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement (partial) Inserted New --- + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, partialElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.isUndefined(elem!.StrProp); + assert.isUndefined(elem!.Model); + assert.isUndefined(elem!.LastMod); + assert.deepEqual(Object.keys(elem!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "ECClassId"].sort()); + assert.deepEqual(elem!.$meta.rowOptions, {}); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - if (true || "updated element when no classId") { - const otherDb = SnapshotDb.openFile(IModelTestUtils.resolveAssetFile("test.bim")); - const testChanges = (changes: ChangedECInstance[]) => { - assert.equal(changes.length, 4); - - // new value - assert.equal(changes[2].ECInstanceId, "0x20000000004"); - assert.isUndefined(changes[2].ECClassId); - assert.isDefined(changes[2].$meta?.fallbackClassId); - assert.equal(changes[2].$meta?.fallbackClassId, "0x3d"); - assert.isUndefined(changes[2].s); - assert.equal(changes[2].$meta?.classFullName, "BisCore:GeometricElement2d"); - assert.equal(changes[2].$meta?.op, "Updated"); - assert.equal(changes[2].$meta?.stage, "New"); - - // old value - assert.equal(changes[3].ECInstanceId, "0x20000000004"); - assert.isUndefined(changes[3].ECClassId); - assert.isDefined(changes[3].$meta?.fallbackClassId); - assert.equal(changes[3].$meta?.fallbackClassId, "0x3d"); - assert.isUndefined(changes[3].s); - assert.equal(changes[3].$meta?.classFullName, "BisCore:GeometricElement2d"); - assert.equal(changes[3].$meta?.op, "Updated"); - assert.equal(changes[3].$meta?.stage, "Old"); - }; - - if (true || "test with InMemoryInstanceCache") { - using reader = SqliteChangesetReader.openFile({ fileName: changesets[3].pathname, db: otherDb, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - testChanges(Array.from(pcu.instances)); - } + it("txn2 insert-partial | rowOptions: classIdsToClassNames", () => { + const instances = readTxn(rwIModel, txnId, undefined, { classIdsToClassNames: true }); + assert.equal(instances.length, 3); + + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECClassId, "BisCore.DrawingModel"); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true }); + + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECClassId, "BisCore.DrawingModel"); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true }); + + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECClassId, "TestDomain.Test2dElement"); + assert.deepEqual(elem!.Model, { Id: drawingModelId, RelECClassId: "BisCore.ModelContainsElements" }); + assert.deepEqual(elem!.CodeSpec, { Id: "0x1", RelECClassId: "BisCore.CodeSpecSpecifiesCode" }); + assert.deepEqual(elem!.CodeScope, { Id: "0x1", RelECClassId: "BisCore.ElementScopesCode" }); + assert.deepEqual(elem!.Category, { Id: drawingCategoryId, RelECClassId: "BisCore.GeometricElement2dIsInCategory" }); + assert.isUndefined(elem!.StrProp); + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", "Category", + ].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { classIdsToClassNames: true }); + }); - if (true || "test with SqliteBackedInstanceCache") { - using reader = SqliteChangesetReader.openFile({ fileName: changesets[3].pathname, db: otherDb, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - testChanges(Array.from(pcu.instances)); - } - } + it("txn2 insert-partial | rowOptions: useJsName", () => { + const instances = readTxn(rwIModel, txnId, undefined, { useJsName: true }); + assert.equal(instances.length, 3); + + const modelNew = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.className, "BisCore.DrawingModel"); + assert.isUndefined(modelNew!.ECInstanceId); + assert.deepEqual(modelNew!.$meta.rowOptions, { useJsName: true }); + + const modelOld = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.className, "BisCore.DrawingModel"); + assert.deepEqual(modelOld!.$meta.rowOptions, { useJsName: true }); + + const elem = instances.find((i) => i.id === partialElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.className, "TestDomain.Test2dElement"); + assert.isUndefined(elem!.ECInstanceId); + assert.isUndefined(elem!.ECClassId); + assert.deepEqual(elem!.model, { id: drawingModelId, relClassName: "BisCore.ModelContainsElements" }); + assert.deepEqual(elem!.codeSpec, { id: "0x1", relClassName: "BisCore.CodeSpecSpecifiesCode" }); + assert.deepEqual(elem!.codeScope, { id: "0x1", relClassName: "BisCore.ElementScopesCode" }); + assert.deepEqual(elem!.category, { id: drawingCategoryId, relClassName: "BisCore.GeometricElement2dIsInCategory" }); + assert.isUndefined(elem!.strProp); + assert.deepEqual(Object.keys(elem!).sort(), [ + "id", "className", "model", "lastMod", "codeSpec", "codeScope", "federationGuid", "$meta", "category", + ].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { useJsName: true }); + }); - if (true || "test changeset file") { - const testChanges = async (changes: ChangedECInstance[]) => { - assert.equal(changes.length, 3); - - assert.equal(changes[0].ECInstanceId, "0x20000000001"); - assert.equal(changes[0].$meta?.classFullName, "BisCore:DrawingModel"); - assert.equal(changes[0].$meta?.op, "Updated"); - assert.equal(changes[0].$meta?.stage, "New"); - assert.isNotNull(changes[0].LastMod); - assert.isNotNull(changes[0].GeometryGuid); - - assert.equal(changes[1].ECInstanceId, "0x20000000001"); - assert.equal(changes[1].$meta?.classFullName, "BisCore:DrawingModel"); - assert.equal(changes[1].$meta?.op, "Updated"); - assert.equal(changes[1].$meta?.stage, "Old"); - assert.isNull(changes[1].LastMod); - assert.isNull(changes[1].GeometryGuid); - - assert.equal(changes[2].ECInstanceId, "0x20000000004"); - assert.equal(changes[2].$meta?.classFullName, "TestDomain:Test2dElement"); - assert.equal(changes[2].$meta?.op, "Inserted"); - assert.equal(changes[2].$meta?.stage, "New"); - - const el = changes.filter((x) => x.ECInstanceId === "0x20000000004")[0]; - assert.equal(el.Rotation, 0); - // eslint-disable-next-line @typescript-eslint/naming-convention - assert.deepEqual(el.Origin, { X: 0, Y: 0 }); - // eslint-disable-next-line @typescript-eslint/naming-convention - assert.deepEqual(el.BBoxLow, { X: -25, Y: -25 }); - // eslint-disable-next-line @typescript-eslint/naming-convention - assert.deepEqual(el.BBoxHigh, { X: 15, Y: 15 }); - - assert.equal(el.Category.Id, "0x20000000002"); - assert.isNotEmpty(el.Category.RelECClassId); - - const categoryRelClass = await getClassNameById(rwIModel, el.Category.RelECClassId); - assert.equal("BisCore:GeometricElement2dIsInCategory", categoryRelClass); - assert.equal(el.s, "xxxxxxxxx"); - assert.isNull(el.CodeValue); - assert.isNull(el.UserLabel); - assert.isNull(el.JsonProperties); - assert.instanceOf(el.GeometryStream, Uint8Array); - assert.typeOf(el.FederationGuid, "string"); - assert.typeOf(el.LastMod, "string"); - // eslint-disable-next-line @typescript-eslint/naming-convention - assert.deepEqual(el.Parent, { Id: null, RelECClassId: null }); - // eslint-disable-next-line @typescript-eslint/naming-convention - assert.deepEqual(el.TypeDefinition, { Id: null, RelECClassId: null }); - - assert.equal(el.CodeSpec.Id, "0x1"); - assert.isNotEmpty(el.CodeSpec.RelECClassId); - - const codeSpecRelClass = await getClassNameById(rwIModel, el.CodeSpec.RelECClassId); - assert.equal("BisCore:CodeSpecSpecifiesCode", codeSpecRelClass); - - assert.equal(el.CodeScope.Id, "0x1"); - assert.isNotEmpty(el.CodeScope.RelECClassId); - - const codeScopeRelClass = await getClassNameById(rwIModel, el.CodeScope.RelECClassId); - assert.equal("BisCore:ElementScopesCode", codeScopeRelClass); - - assert.deepEqual(el.$meta, { - tables: [ - "bis_GeometricElement2d", - "bis_Element", - ], - op: "Inserted", - classFullName: "TestDomain:Test2dElement", - changeIndexes: [ - 2, - 1, - ], - stage: "New", - }); - } - if (true || "test with InMemoryInstanceCache") { - using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - await testChanges(Array.from(pcu.instances)); - } + it("txn2 insert-partial | rowOptions: abbreviateBlobs", () => { + const instances = readTxn(rwIModel, txnId, undefined, { abbreviateBlobs: true }); + assert.equal(instances.length, 3); + + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.deepEqual(modelNew!.$meta.rowOptions, { abbreviateBlobs: true }); + + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.deepEqual(modelOld!.$meta.rowOptions, { abbreviateBlobs: true }); + + // Partial element has no blob props; ECClassId stays as raw hex + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.equal(elem!.Model.Id, drawingModelId); + assert.equal(rwIModel.getClassNameFromId(elem!.Model.RelECClassId), "BisCore:ModelContainsElements"); + assert.isUndefined(elem!.StrProp); + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", "Category", + ].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { abbreviateBlobs: true }); + }); - if (true || "test with SqliteBackedInstanceCache") { - using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - await testChanges(Array.from(pcu.instances)); - } - } - if (true || "test ChangesetAdaptor.acceptClass()") { - const testChanges = (changes: ChangedECInstance[]) => { - assert.equal(changes.length, 1); - assert.equal(changes[0].$meta?.classFullName, "TestDomain:Test2dElement"); - }; - if (true || "test with InMemoryInstanceCache") { - using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - adaptor.acceptClass("TestDomain.Test2dElement"); - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - testChanges(Array.from(pcu.instances)); - } + it("txn2 insert-partial | rowOptions: classIdsToClassNames + useJsName", () => { + const instances = readTxn(rwIModel, txnId, undefined, { classIdsToClassNames: true, useJsName: true }); + assert.equal(instances.length, 3); + + const modelNew = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.className, "BisCore.DrawingModel"); + assert.isUndefined(modelNew!.ECInstanceId); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + + const modelOld = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.className, "BisCore.DrawingModel"); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + + const elem = instances.find((i) => i.id === partialElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.className, "TestDomain.Test2dElement"); + assert.isUndefined(elem!.ECInstanceId); + assert.deepEqual(elem!.model, { id: drawingModelId, relClassName: "BisCore.ModelContainsElements" }); + assert.deepEqual(elem!.codeSpec, { id: "0x1", relClassName: "BisCore.CodeSpecSpecifiesCode" }); + assert.deepEqual(elem!.codeScope, { id: "0x1", relClassName: "BisCore.ElementScopesCode" }); + assert.deepEqual(elem!.category, { id: drawingCategoryId, relClassName: "BisCore.GeometricElement2dIsInCategory" }); + assert.isUndefined(elem!.strProp); + assert.deepEqual(Object.keys(elem!).sort(), [ + "id", "className", "model", "lastMod", "codeSpec", "codeScope", "federationGuid", "$meta", "category", + ].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + }); + it("should throw error if tried to fetch changeset metadata values before stepping", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + expect(() => reader.isECTable).to.throw(); + expect(() => reader.isIndirectChange).to.throw(); + expect(() => reader.tableName).to.throw(); + expect(() => reader.op).to.throw(); + assert.isUndefined(reader.inserted); + assert.isUndefined(reader.deleted); + reader.close(); + }); + it("should throw error if tried to fetch changeset metadata values after stepping past the end", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + while (reader.step()) { } + assert.equal(reader.step(), false); + expect(() => reader.isECTable).to.throw(); + expect(() => reader.isIndirectChange).to.throw(); + expect(() => reader.tableName).to.throw(); + expect(() => reader.op).to.throw(); + assert.isUndefined(reader.inserted); + assert.isUndefined(reader.deleted); + reader.close(); + }); - if (true || "test with SqliteBackedInstanceCache") { - using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - adaptor.acceptClass("TestDomain.Test2dElement"); - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - testChanges(Array.from(pcu.instances)); - } - } - if (true || "test ChangesetAdaptor.adaptor()") { - const testChanges = (changes: ChangedECInstance[]) => { - assert.equal(changes.length, 2); - assert.equal(changes[0].ECInstanceId, "0x20000000001"); - assert.equal(changes[0].$meta?.classFullName, "BisCore:DrawingModel"); - assert.equal(changes[0].$meta?.op, "Updated"); - assert.equal(changes[0].$meta?.stage, "New"); - assert.equal(changes[1].ECInstanceId, "0x20000000001"); - assert.equal(changes[1].$meta?.classFullName, "BisCore:DrawingModel"); - assert.equal(changes[1].$meta?.op, "Updated"); - assert.equal(changes[1].$meta?.stage, "Old"); - }; - if (true || "test with InMemoryInstanceCache") { - using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - adaptor.acceptOp("Updated") - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - testChanges(Array.from(pcu.instances)); - } +}); - if (true || "test with SqliteBackedInstanceCache") { - using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); - using adaptor = new ECChangesetAdaptor(reader); - adaptor.acceptOp("Updated") - using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); - while (adaptor.step()) { - pcu.appendFrom(adaptor); - } - testChanges(Array.from(pcu.instances)); - } - } - txn.end(); - rwIModel.close(); - }); - it("revert timeline changes", async () => { + +describe("ChangesetReader update-full", () => { + let rwIModel: BriefcaseDb; + let fullElementId: Id64String; + let partialElementId: Id64String; + let drawingModelId: Id64String; + let drawingCategoryId: Id64String; + let txnId: string; + let txn: EditTxn; + + before(async () => { + HubMock.startup("ECChangesetUpdateFull", KnownTestLocations.outputDir); const adminToken = "super manager token"; - const iModelName = "test"; - const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(rwIModelId); - const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); - const txn = startTestTxn(rwIModel, "revert timeline changes"); - let nProps = 0; - // 1. Import schema with class that span overflow table. - const addPropertyAndImportSchema = async () => { - await rwIModel.acquireSchemaLock(); - ++nProps; - const schema = ` - - - - bis:GraphicalElement2d - ${Array(nProps).fill(undefined).map((_, i) => ``).join("\n")} - - `; - await importSchemaStrings(txn, [schema]); - }; - await addPropertyAndImportSchema(); + const iTwinId = HubMock.iTwinId; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "updateFull", description: "updateFull", accessToken: adminToken }); + rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + txn = startTestTxn(rwIModel, "ChangesetReader update-full setup"); + // Txn 1: import schema + drawing model setup, then push + const schema = ` + + + + + + + + + + + + + + + + + + bis:GraphicalElement2d + + + + + + + + + + + + + + + + + + + + `; + await importSchemaStrings(txn, [schema]); rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - // Create drawing model and category await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); const codeProps = Code.createEmpty(); - codeProps.value = "DrawingModel"; - const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); - if (undefined === drawingCategoryId) - drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + codeProps.value = "DrillDownDrawing"; + [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - txn.saveChanges(); - await rwIModel.pushChanges({ description: "setup category", accessToken: adminToken }); - - const createEl = async (args: { [key: string]: any }) => { - await rwIModel.locks.acquireLocks({ exclusive: drawingModelId }); - const geomArray: Arc3d[] = [ - Arc3d.createXY(Point3d.create(0, 0), 5), - Arc3d.createXY(Point3d.create(5, 5), 2), - Arc3d.createXY(Point3d.create(-5, -5), 20), - ]; - - const geometryStream: GeometryStreamProps = []; - for (const geom of geomArray) { - const arcData = IModelJson.Writer.toIModelJson(geom); - geometryStream.push(arcData); - } + const foundCat = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "DrillDownCategory"); + drawingCategoryId = foundCat ?? DrawingCategory.insert(txn, IModel.dictionaryId, "DrillDownCategory", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); - const e1 = { - classFullName: `TestDomain:Test2dElement`, - model: drawingModelId, - category: drawingCategoryId, - code: Code.createEmpty(), - geom: geometryStream, - ...args, - }; - return txn.insertElement(e1);; - }; - const updateEl = async (id: Id64String, args: { [key: string]: any }) => { - await rwIModel.locks.acquireLocks({ exclusive: id }); - const updatedElementProps = Object.assign(rwIModel.elements.getElementProps(id), args); - txn.updateElement(updatedElementProps); - }; - - const deleteEl = async (id: Id64String) => { - await rwIModel.locks.acquireLocks({ exclusive: id }); - txn.deleteElement(id); - }; - const getChanges = async () => { - return HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir: path.join(KnownTestLocations.outputDir, rwIModelId, "changesets") }); - }; - - const findEl = (id: Id64String) => { - try { - return rwIModel.elements.getElementProps(id); - } catch { - return undefined; - } - }; - // 2. Insert a element for the class - const el1 = await createEl({ p1: "test1" }); - const el2 = await createEl({ p1: "test2" }); - txn.saveChanges(); - await rwIModel.pushChanges({ description: "insert 2 elements" }); + txn.saveChanges("setup"); - // 3. Update the element. - await updateEl(el1, { p1: "test3" }); - txn.saveChanges(); - await rwIModel.pushChanges({ description: "update element 1" }); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the full insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Txn 2: insert FULL element (needed as update target) + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + const geom: GeometryStreamProps = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + Arc3d.createXY(Point3d.create(5, 5), 2), + Arc3d.createXY(Point3d.create(-5, -5), 20), + ].map((a) => IModelJson.Writer.toIModelJson(a)); - // 4. Delete the element. - await deleteEl(el2); - const el3 = await createEl({ p1: "test4" }); - txn.saveChanges(); - await rwIModel.pushChanges({ description: "delete element 2" }); + fullElementId = txn.insertElement({ + classFullName: "TestDomain:Test2dElement", + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + geom, + StrProp: "hello", + IntProp: 42, + LongProp: 9_007_199_254_740_991, // Number.MAX_SAFE_INTEGER + DblProp: 3.14159265358979, + BoolProp: true, + DtProp: "2024-01-15T12:00:00.000", + BinProp: new Uint8Array([1, 2, 3, 5]), + Pt2dProp: { x: 1.5, y: 2.5 }, + Pt3dProp: { x: 3.0, y: 4.0, z: 5.0 }, + StructProp: { + X: 1.0, Y: 2.0, Z: 3.0, Label: "origin", + Pt2d: { x: 0.5, y: 0.5 }, + Pt3d: { x: 1.0, y: 2.0, z: 3.0 }, + }, + IntArrProp: [10, 20, 30], + StrArrProp: ["alpha", "beta", "gamma"], + StructArrProp: [ + { X: 0.0, Y: 1.0, Z: 2.0, Label: "a", Pt2d: { x: 0.0, y: 0.0 }, Pt3d: { x: 0.0, y: 0.0, z: 0.0 } }, + { X: 3.0, Y: 4.0, Z: 5.0, Label: "b", Pt2d: { x: 1.0, y: 1.0 }, Pt3d: { x: 1.0, y: 1.0, z: 1.0 } }, + ], + RelatedElem: { id: drawingCategoryId, relClassName: "TestDomain:Test2dUsesElement" }, + } as any); + txn.saveChanges("insert full element"); + txnId = rwIModel.txns.getLastSavedTxnProps()!.id; + + // Wait so that LastMod on bis_Model gets a distinct timestamp before the partial insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Txn 3: insert PARTIAL element (needed as RelatedElem target in the update) + partialElementId = txn.insertElement({ + classFullName: "TestDomain:Test2dElement", + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + // StrProp, IntProp, LongProp, DblProp, BoolProp, DtProp, BinProp intentionally absent + } as any); + txn.saveChanges("insert partial element"); + txnId = rwIModel.txns.getLastSavedTxnProps()!.id; + + // Wait so that LastMod on bis_Model gets a distinct timestamp before the update txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Txn 4: update FULL element this is the txn under test + // Txn 3: update FULL element — change several property types + await rwIModel.locks.acquireLocks({ exclusive: fullElementId }); + txn.updateElement({ + ...rwIModel.elements.getElementProps(fullElementId), + StrProp: "updated", + IntProp: 99, + LongProp: 0, + DblProp: 2.71828182845904, + BoolProp: false, + DtProp: "2025-06-01T08:30:00.000", + BinProp: new Uint8Array([10, 20, 30, 40, 50]), + Pt2dProp: { x: 9.0, y: 8.0 }, + Pt3dProp: { x: 7.0, y: 6.0, z: 5.0 }, + StructProp: { + X: 9.0, Y: 8.0, Z: 7.0, Label: "updated-origin", + Pt2d: { x: 9.5, y: 8.5 }, + Pt3d: { x: 9.0, y: 8.0, z: 7.0 }, + }, + IntArrProp: [100, 200], + StrArrProp: ["delta", "epsilon"], + StructArrProp: [ + { X: 5.0, Y: 6.0, Z: 7.0, Label: "c", Pt2d: { x: 5.0, y: 5.0 }, Pt3d: { x: 5.0, y: 5.0, z: 5.0 } }, + { X: 7.0, Y: 8.0, Z: 9.0, Label: "d", Pt2d: { x: 7.0, y: 7.0 }, Pt3d: { x: 7.0, y: 7.0, z: 7.0 } }, + { X: 9.0, Y: 10.0, Z: 11.0, Label: "e", Pt2d: { x: 9.0, y: 9.0 }, Pt3d: { x: 9.0, y: 9.0, z: 9.0 } }, + ], + RelatedElem: { id: partialElementId, relClassName: "TestDomain:Test2dUsesElement" }, + }); + txn.saveChanges("update full element"); + txnId = rwIModel.txns.getLastSavedTxnProps()!.id; + }); - // 5. import schema and insert element 4 & update element 3 - await addPropertyAndImportSchema(); - const el4 = await createEl({ p1: "test5", p2: "test6" }); - await updateEl(el3, { p1: "test7", p2: "test8" }); - txn.saveChanges(); - await rwIModel.pushChanges({ description: "import schema, insert element 4 & update element 3" }); - - assert.isDefined(findEl(el1)); - assert.isUndefined(findEl(el2)); - assert.isDefined(findEl(el3)); - assert.isDefined(findEl(el4)); - // eslint-disable-next-line @typescript-eslint/no-deprecated - assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1", "p2"]); - // 6. Revert to timeline 2 - await rwIModel.revertAndPushChanges({ toIndex: 2, description: "revert to timeline 2" }); - assert.equal((await getChanges()).at(-1)!.description, "revert to timeline 2"); - - assert.isUndefined(findEl(el1)); - assert.isUndefined(findEl(el2)); - assert.isUndefined(findEl(el3)); - assert.isUndefined(findEl(el4)); - // eslint-disable-next-line @typescript-eslint/no-deprecated - assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1"]); - - await rwIModel.revertAndPushChanges({ toIndex: 6, description: "reinstate last reverted changeset" }); - assert.equal((await getChanges()).at(-1)!.description, "reinstate last reverted changeset"); - assert.isDefined(findEl(el1)); - assert.isUndefined(findEl(el2)); - assert.isDefined(findEl(el3)); - assert.isDefined(findEl(el4)); - // eslint-disable-next-line @typescript-eslint/no-deprecated - assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1", "p2"]); - - await addPropertyAndImportSchema(); - const el5 = await createEl({ p1: "test9", p2: "test10", p3: "test11" }); - await updateEl(el1, { p1: "test12", p2: "test13", p3: "test114" }); - txn.saveChanges(); - await rwIModel.pushChanges({ description: "import schema, insert element 5 & update element 1" }); - // eslint-disable-next-line @typescript-eslint/no-deprecated - assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1", "p2", "p3"]); - - // skip schema changes & auto generated comment - await rwIModel.revertAndPushChanges({ toIndex: 1, skipSchemaChanges: true }); - assert.equal((await getChanges()).at(-1)!.description, "Reverted changes from 8 to 1 (schema changes skipped)"); - assert.isUndefined(findEl(el1)); - assert.isUndefined(findEl(el2)); - assert.isUndefined(findEl(el3)); - assert.isUndefined(findEl(el4)); - assert.isUndefined(findEl(el5)); - // eslint-disable-next-line @typescript-eslint/no-deprecated - assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1", "p2", "p3"]); - - await rwIModel.revertAndPushChanges({ toIndex: 9 }); - assert.equal((await getChanges()).at(-1)!.description, "Reverted changes from 9 to 9"); - assert.isDefined(findEl(el1)); - assert.isUndefined(findEl(el2)); - assert.isDefined(findEl(el3)); - assert.isDefined(findEl(el4)); - assert.isDefined(findEl(el5)); - // eslint-disable-next-line @typescript-eslint/no-deprecated - assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1", "p2", "p3"]); + after(() => { txn.end(); - rwIModel.close(); + rwIModel?.close(); + HubMock.shutdown(); }); - it("openGroup() & writeToFile()", async () => { + + it("txn3 update-full | All_Properties | default rowOptions", () => { + const instances = readTxn(rwIModel, txnId, undefined, undefined, undefined, false); + assert.equal(instances.length, 4); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.isString(modelNew!.LastMod); + assert.isUndefined(modelNew!.GeometryGuid); // no GeometryGuid in update txn model row + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.isString(modelOld!.LastMod); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Updated New --- + const elemNew = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elemNew).to.exist; + assert.equal(elemNew!.ECInstanceId, fullElementId); + assert.equal(elemNew!.ECClassId, "0x176"); + assert.equal(elemNew!.StrProp, "updated"); + assert.equal(elemNew!.IntProp, 99); + assert.equal(elemNew!.LongProp, 0); + assert.include(String(elemNew!.BinProp), "\"bytes\""); + assert.closeTo(elemNew!.DblProp as number, 2.71828182845904, 1e-10); + assert.equal(elemNew!.BoolProp, false); + assert.equal(elemNew!.DtProp, "2025-06-01T08:30:00.000"); + assert.deepEqual(elemNew!.Pt2dProp, { X: 9, Y: 8 }); + assert.deepEqual(elemNew!.Pt3dProp, { X: 7, Y: 6, Z: 5 }); + assert.deepEqual(elemNew!.StructProp, { X: 9, Y: 8, Z: 7, Label: "updated-origin", Pt2d: { X: 9.5, Y: 8.5 }, Pt3d: { X: 9, Y: 8, Z: 7 } }); + assert.deepEqual(elemNew!.IntArrProp, [100, 200]); + assert.deepEqual(elemNew!.StrArrProp, ["delta", "epsilon"]); + assert.deepEqual(elemNew!.StructArrProp, [ + { X: 5, Y: 6, Z: 7, Label: "c", Pt2d: { X: 5, Y: 5 }, Pt3d: { X: 5, Y: 5, Z: 5 } }, + { X: 7, Y: 8, Z: 9, Label: "d", Pt2d: { X: 7, Y: 7 }, Pt3d: { X: 7, Y: 7, Z: 7 } }, + { X: 9, Y: 10, Z: 11, Label: "e", Pt2d: { X: 9, Y: 9 }, Pt3d: { X: 9, Y: 9, Z: 9 } }, + ]); + assert.equal(elemNew!.RelatedElem.Id, partialElementId); + assert.equal(rwIModel.getClassNameFromId(elemNew!.RelatedElem.RelECClassId), "TestDomain:Test2dUsesElement"); + assert.isString(elemNew!.LastMod); + // Object.keys — update row: custom-prop columns first, then $meta and LastMod at the end + assert.deepEqual(Object.keys(elemNew!).sort(), [ + "ECInstanceId", "ECClassId", + "StrProp", "IntProp", "LongProp", "DblProp", "BoolProp", "DtProp", + "Pt2dProp", "Pt3dProp", "StructProp", "IntArrProp", "StrArrProp", "StructArrProp", "RelatedElem", + "$meta", "LastMod", "BinProp" + ].sort()); + assert.deepEqual(Object.keys(elemNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elemNew!.$meta.op, "Updated"); + assert.equal(elemNew!.$meta.stage, "New"); + assert.deepEqual([...elemNew!.$meta.tables].sort(), ["bis_GeometricElement2d", "bis_Element"].sort()); + assert.deepEqual([...elemNew!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elemNew!.$meta.instanceKey); + assert.equal(elemNew!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elemNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), [ + "BoolProp", "DblProp", "DtProp", "ECInstanceId", "IntArrProp", "IntProp", "LastMod", + "LongProp", "Pt2dProp", "Pt3dProp.X", "Pt3dProp.Y", "RelatedElem.Id", "StrArrProp", + "StrProp", "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", + "StructProp.Y", "StructProp.Z", "BinProp" + ].sort()); + assert.deepEqual(elemNew!.$meta.rowOptions, {}); + assert.equal(elemNew!.$meta.isIndirectChange, false); + + // --- instances[3]: Test2dElement Updated Old --- + const elemOld = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "Old"); + expect(elemOld).to.exist; + assert.equal(elemOld!.ECInstanceId, fullElementId); + assert.equal(elemOld!.ECClassId, "0x176"); + assert.equal(elemOld!.StrProp, "hello"); + assert.equal(elemOld!.IntProp, 42); + assert.equal(elemOld!.LongProp, 9007199254740991); + assert.closeTo(elemOld!.DblProp as number, 3.14159265358979, 1e-10); + assert.equal(elemOld!.BoolProp, true); + assert.include(String(elemOld!.BinProp), "\"bytes\""); + assert.equal(elemOld!.DtProp, "2024-01-15T12:00:00.000"); + assert.deepEqual(elemOld!.Pt2dProp, { X: 1.5, Y: 2.5 }); + assert.deepEqual(elemOld!.Pt3dProp, { X: 3, Y: 4, Z: 5 }); + assert.deepEqual(elemOld!.StructProp, { X: 1, Y: 2, Z: 3, Label: "origin", Pt2d: { X: 0.5, Y: 0.5 }, Pt3d: { X: 1, Y: 2, Z: 3 } }); + assert.deepEqual(elemOld!.IntArrProp, [10, 20, 30]); + assert.deepEqual(elemOld!.StrArrProp, ["alpha", "beta", "gamma"]); + assert.deepEqual(elemOld!.StructArrProp, [ + { X: 0, Y: 1, Z: 2, Label: "a", Pt2d: { X: 0, Y: 0 }, Pt3d: { X: 0, Y: 0, Z: 0 } }, + { X: 3, Y: 4, Z: 5, Label: "b", Pt2d: { X: 1, Y: 1 }, Pt3d: { X: 1, Y: 1, Z: 1 } }, + ]); + assert.equal(elemOld!.RelatedElem.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elemOld!.RelatedElem.RelECClassId), "TestDomain:Test2dUsesElement"); + assert.isString(elemOld!.LastMod); + assert.deepEqual(Object.keys(elemOld!).sort(), [ + "ECInstanceId", "ECClassId", + "StrProp", "IntProp", "LongProp", "DblProp", "BoolProp", "DtProp", + "Pt2dProp", "Pt3dProp", "StructProp", "IntArrProp", "StrArrProp", "StructArrProp", "RelatedElem", + "$meta", "LastMod", "BinProp" + ].sort()); + assert.deepEqual(Object.keys(elemOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elemOld!.$meta.op, "Updated"); + assert.equal(elemOld!.$meta.stage, "Old"); + assert.deepEqual([...elemOld!.$meta.tables].sort(), ["bis_GeometricElement2d", "bis_Element"].sort()); + assert.equal(elemOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...elemOld!.$meta.changeFetchedPropNames].sort(), [ + "BoolProp", "DblProp", "DtProp", "ECInstanceId", "IntArrProp", "IntProp", "LastMod", + "LongProp", "Pt2dProp", "Pt3dProp.X", "Pt3dProp.Y", "RelatedElem.Id", "StrArrProp", + "StrProp", "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", + "StructProp.Y", "StructProp.Z", "BinProp" + ].sort()); + assert.deepEqual(elemOld!.$meta.rowOptions, {}); + assert.equal(elemOld!.$meta.isIndirectChange, false); + }); + + it("txn3 update-full | Bis_Element_Properties", () => { + const instances = readTxn(rwIModel, txnId, PropertyFilter.BisCoreElement, { classIdsToClassNames: true }, undefined, false); + assert.equal(instances.length, 4); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal(modelNew!.ECClassId, "BisCore.DrawingModel"); + assert.isString(modelNew!.LastMod); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal(modelOld!.ECClassId, "BisCore.DrawingModel"); + assert.isString(modelOld!.LastMod); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Updated New (no custom props) --- + const elemNew = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elemNew).to.exist; + assert.equal(elemNew!.ECInstanceId, fullElementId); + assert.equal(elemNew!.ECClassId, "TestDomain.Test2dElement"); + assert.isUndefined(elemNew!.StrProp); + assert.isUndefined(elemNew!.IntProp); + assert.isUndefined(elemNew!.Model); + assert.isUndefined(elemNew!.Category); + expect(elemNew!.LastMod).to.exist; + assert.deepEqual(Object.keys(elemNew!).sort(), ["ECInstanceId", "ECClassId", "$meta", "LastMod"].sort()); + assert.deepEqual(Object.keys(elemNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elemNew!.$meta.op, "Updated"); + assert.equal(elemNew!.$meta.stage, "New"); + assert.deepEqual([...elemNew!.$meta.tables].sort(), ["bis_GeometricElement2d", "bis_Element"].sort()); + assert.deepEqual([...elemNew!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elemNew!.$meta.instanceKey); + assert.equal(elemNew!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elemNew!.$meta.propFilter, PropertyFilter.BisCoreElement); + + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod"].sort()); + assert.deepEqual(elemNew!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(elemNew!.$meta.isIndirectChange, false); + + // --- instances[3]: Test2dElement Updated Old --- + const elemOld = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "Old"); + expect(elemOld).to.exist; + assert.equal(elemOld!.ECInstanceId, fullElementId); + assert.equal(elemOld!.ECClassId, "TestDomain.Test2dElement"); + assert.isUndefined(elemOld!.StrProp); + assert.isUndefined(elemOld!.IntProp); + assert.isUndefined(elemOld!.Model); + assert.deepEqual(Object.keys(elemOld!).sort(), ["ECInstanceId", "ECClassId", "$meta", "LastMod"].sort()); + assert.deepEqual(Object.keys(elemOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elemOld!.$meta.op, "Updated"); + assert.equal(elemOld!.$meta.stage, "Old"); + assert.deepEqual([...elemOld!.$meta.tables].sort(), ["bis_GeometricElement2d", "bis_Element"].sort()); + assert.equal(elemOld!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...elemOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod"].sort()); + assert.deepEqual(elemOld!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(elemOld!.$meta.isIndirectChange, false); + }); + + it("txn3 update-full | Instance_Key", () => { + const instances = readTxn(rwIModel, txnId, PropertyFilter.InstanceKey, undefined, undefined, false); + assert.equal(instances.length, 4); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement Updated New --- + const elemNew = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elemNew).to.exist; + assert.equal(elemNew!.ECInstanceId, fullElementId); + assert.equal(elemNew!.ECClassId, "0x176"); + assert.isUndefined(elemNew!.StrProp); + assert.isUndefined(elemNew!.Model); + assert.isUndefined(elemNew!.LastMod); + assert.deepEqual(Object.keys(elemNew!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(elemNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elemNew!.$meta.op, "Updated"); + assert.equal(elemNew!.$meta.stage, "New"); + assert.deepEqual([...elemNew!.$meta.tables].sort(), ["bis_GeometricElement2d", "bis_Element"].sort()); + assert.deepEqual([...elemNew!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elemNew!.$meta.instanceKey); + assert.equal(elemNew!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elemNew!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId"].sort()); + assert.deepEqual(elemNew!.$meta.rowOptions, {}); + assert.equal(elemNew!.$meta.isIndirectChange, false); + + // --- instances[3]: Test2dElement Updated Old --- + const elemOld = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "Old"); + expect(elemOld).to.exist; + assert.equal(elemOld!.ECInstanceId, fullElementId); + assert.equal(elemOld!.ECClassId, "0x176"); + assert.isUndefined(elemOld!.StrProp); + assert.isUndefined(elemOld!.Model); + assert.deepEqual(Object.keys(elemOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(elemOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elemOld!.$meta.op, "Updated"); + assert.equal(elemOld!.$meta.stage, "Old"); + assert.deepEqual([...elemOld!.$meta.tables].sort(), ["bis_GeometricElement2d", "bis_Element"].sort()); + assert.equal(elemOld!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...elemOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId"].sort()); + assert.deepEqual(elemOld!.$meta.rowOptions, {}); + assert.equal(elemOld!.$meta.isIndirectChange, false); + }); + + it("txn3 update-full | rowOptions: useJsName", () => { + const instances = readTxn(rwIModel, txnId, undefined, { useJsName: true }, undefined, false); + assert.equal(instances.length, 4); + + const modelNew = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.className, "BisCore.DrawingModel"); + assert.isUndefined(modelNew!.ECInstanceId); + assert.deepEqual(modelNew!.$meta.rowOptions, { useJsName: true }); + + const modelOld = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.className, "BisCore.DrawingModel"); + assert.deepEqual(modelOld!.$meta.rowOptions, { useJsName: true }); + + const elemNew = instances.find((i) => i.id === fullElementId && i.$meta.stage === "New"); + expect(elemNew).to.exist; + assert.equal(elemNew!.className, "TestDomain.Test2dElement"); + assert.isUndefined(elemNew!.ECInstanceId); + assert.equal(elemNew!.strProp, "updated"); + assert.equal(elemNew!.intProp, 99); + assert.include(String(elemNew!.binProp), "\"bytes\""); + assert.deepEqual(elemNew!.pt2dProp, { x: 9, y: 8 }); + assert.deepEqual(elemNew!.pt3dProp, { x: 7, y: 6, z: 5 }); + assert.deepEqual(elemNew!.relatedElem, { id: partialElementId, relClassName: "TestDomain.Test2dUsesElement" }); + assert.equal(elemNew!.$meta.op, "Updated"); + assert.equal(elemNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elemNew!.$meta.rowOptions, { useJsName: true }); + assert.deepEqual(Object.keys(elemNew!).sort(), ["$meta", "binProp", "boolProp", "className", + "dblProp", "dtProp", "id", "intArrProp", "intProp", "lastMod", "longProp", "pt2dProp", + "pt3dProp", "relatedElem", "strArrProp", "strProp", "structArrProp", "structProp"].sort()); + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ["BinProp", "BoolProp", + "DblProp", "DtProp", "ECInstanceId", "IntArrProp", "IntProp", "LastMod", + "LongProp", "Pt2dProp", "Pt3dProp.X", "Pt3dProp.Y", "RelatedElem.Id", "StrArrProp", "StrProp", + "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", + "StructProp.Y", "StructProp.Z"].sort()); + + const elemOld = instances.find((i) => i.id === fullElementId && i.$meta.stage === "Old"); + expect(elemOld).to.exist; + assert.equal(elemOld!.className, "TestDomain.Test2dElement"); + assert.isUndefined(elemOld!.ECInstanceId); + assert.equal(elemOld!.strProp, "hello"); + assert.include(String(elemOld!.binProp), "\"bytes\""); + assert.equal(elemOld!.intProp, 42); + assert.deepEqual(elemOld!.relatedElem, { id: drawingCategoryId, relClassName: "TestDomain.Test2dUsesElement" }); + assert.deepEqual(elemOld!.$meta.rowOptions, { useJsName: true }); + assert.deepEqual(Object.keys(elemOld!).sort(), ["$meta", "binProp", "boolProp", "className", + "dblProp", "dtProp", "id", "intArrProp", "intProp", "lastMod", "longProp", "pt2dProp", + "pt3dProp", "relatedElem", "strArrProp", "strProp", "structArrProp", "structProp"].sort()); + assert.deepEqual([...elemOld!.$meta.changeFetchedPropNames].sort(), ["BinProp", "BoolProp", + "DblProp", "DtProp", "ECInstanceId", "IntArrProp", "IntProp", "LastMod", + "LongProp", "Pt2dProp", "Pt3dProp.X", "Pt3dProp.Y", "RelatedElem.Id", "StrArrProp", "StrProp", + "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", + "StructProp.Y", "StructProp.Z"].sort()); + }); + + it("txn3 update-full | rowOptions: abbreviateBlobs", () => { + const instances = readTxn(rwIModel, txnId, undefined, { abbreviateBlobs: true }, undefined, false); + assert.equal(instances.length, 4); + + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.deepEqual(modelNew!.$meta.rowOptions, { abbreviateBlobs: true }); + + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.deepEqual(modelOld!.$meta.rowOptions, { abbreviateBlobs: true }); + + const elemNew = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elemNew).to.exist; + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elemNew!.ECClassId)); + assert.equal(elemNew!.StrProp, "updated"); + // BinProp is a blob should be abbreviated to { bytes: N } + assert.include(String(elemNew!.BinProp), "bytes"); + assert.equal(elemNew!.$meta.op, "Updated"); + assert.deepEqual(elemNew!.$meta.rowOptions, { abbreviateBlobs: true }); + assert.deepEqual(Object.keys(elemNew!).sort(), ["$meta", "BinProp", "BoolProp", + "DblProp", "DtProp", "IntArrProp", "IntProp", "LastMod", "LongProp", + "Pt2dProp", "Pt3dProp", "RelatedElem", "StrArrProp", "StrProp", "StructArrProp", "StructProp", "ECClassId", "ECInstanceId"].sort()); + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ["BinProp", "BoolProp", + "DblProp", "DtProp", "ECInstanceId", "IntArrProp", "IntProp", "LastMod", + "LongProp", "Pt2dProp", "Pt3dProp.X", "Pt3dProp.Y", "RelatedElem.Id", "StrArrProp", "StrProp", + "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", + "StructProp.Y", "StructProp.Z"].sort()); + + const elemOld = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "Old"); + expect(elemOld).to.exist; + assert.equal(elemOld!.ECClassId, "0x176"); + assert.equal(elemOld!.StrProp, "hello"); + assert.include(String(elemOld!.BinProp), "bytes"); + assert.deepEqual(elemOld!.$meta.rowOptions, { abbreviateBlobs: true }); + assert.deepEqual(Object.keys(elemOld!).sort(), ["$meta", "BinProp", "BoolProp", + "DblProp", "DtProp", "IntArrProp", "IntProp", "LastMod", "LongProp", + "Pt2dProp", "Pt3dProp", "RelatedElem", "StrArrProp", "StrProp", "StructArrProp", "StructProp", "ECClassId", "ECInstanceId"].sort()); + assert.deepEqual([...elemOld!.$meta.changeFetchedPropNames].sort(), ["BinProp", "BoolProp", + "DblProp", "DtProp", "ECInstanceId", "IntArrProp", "IntProp", "LastMod", + "LongProp", "Pt2dProp", "Pt3dProp.X", "Pt3dProp.Y", "RelatedElem.Id", "StrArrProp", "StrProp", + "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", + "StructProp.Y", "StructProp.Z"].sort()); + }); + + it("txn3 update-full | rowOptions: classIdsToClassNames + useJsName", () => { + const instances = readTxn(rwIModel, txnId, undefined, { classIdsToClassNames: true, useJsName: true }, undefined, false); + assert.equal(instances.length, 4); + + const modelNew = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.className, "BisCore.DrawingModel"); + assert.isUndefined(modelNew!.ECInstanceId); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + + const modelOld = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.className, "BisCore.DrawingModel"); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + + const elemNew = instances.find((i) => i.id === fullElementId && i.$meta.stage === "New"); + expect(elemNew).to.exist; + assert.equal(elemNew!.className, "TestDomain.Test2dElement"); + assert.isUndefined(elemNew!.ECInstanceId); + assert.equal(elemNew!.strProp, "updated"); + assert.equal(elemNew!.intProp, 99); + assert.include(String(elemNew!.binProp), "\"bytes\""); + assert.deepEqual(elemNew!.relatedElem, { id: partialElementId, relClassName: "TestDomain.Test2dUsesElement" }); + assert.equal(elemNew!.$meta.op, "Updated"); + assert.deepEqual(elemNew!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + assert.deepEqual(Object.keys(elemNew!).sort(), ["$meta", "binProp", "boolProp", "className", + "dblProp", "dtProp", "id", "intArrProp", "intProp", "lastMod", "longProp", "pt2dProp", + "pt3dProp", "relatedElem", "strArrProp", "strProp", "structArrProp", "structProp"].sort()); + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ["BinProp", "BoolProp", + "DblProp", "DtProp", "ECInstanceId", "IntArrProp", "IntProp", "LastMod", + "LongProp", "Pt2dProp", "Pt3dProp.X", "Pt3dProp.Y", "RelatedElem.Id", "StrArrProp", "StrProp", + "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", + "StructProp.Y", "StructProp.Z"].sort()); + + const elemOld = instances.find((i) => i.id === fullElementId && i.$meta.stage === "Old"); + expect(elemOld).to.exist; + assert.equal(elemOld!.className, "TestDomain.Test2dElement"); + assert.isUndefined(elemOld!.ECInstanceId); + assert.equal(elemOld!.strProp, "hello"); + assert.include(String(elemOld!.binProp), "\"bytes\""); + assert.deepEqual(elemOld!.relatedElem, { id: drawingCategoryId, relClassName: "TestDomain.Test2dUsesElement" }); + assert.deepEqual(elemOld!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + assert.deepEqual(Object.keys(elemOld!).sort(), ["$meta", "binProp", "boolProp", "className", + "dblProp", "dtProp", "id", "intArrProp", "intProp", "lastMod", "longProp", "pt2dProp", + "pt3dProp", "relatedElem", "strArrProp", "strProp", "structArrProp", "structProp"].sort()); + assert.deepEqual([...elemOld!.$meta.changeFetchedPropNames].sort(), ["BinProp", "BoolProp", + "DblProp", "DtProp", "ECInstanceId", "IntArrProp", "IntProp", "LastMod", + "LongProp", "Pt2dProp", "Pt3dProp.X", "Pt3dProp.Y", "RelatedElem.Id", "StrArrProp", "StrProp", + "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", + "StructProp.Y", "StructProp.Z"].sort()); + }); + + it("txn3 update-full | rowOptions: classIdsToClassNames", () => { + + const instances = readTxn(rwIModel, txnId, undefined, { classIdsToClassNames: true }, undefined, false); + assert.equal(instances.length, 4); + + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECClassId, "BisCore.DrawingModel"); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true }); + + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECClassId, "BisCore.DrawingModel"); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true }); + + const elemNew = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elemNew).to.exist; + assert.equal(elemNew!.ECClassId, "TestDomain.Test2dElement"); + assert.equal(elemNew!.StrProp, "updated"); + assert.equal(elemNew!.IntProp, 99); + assert.include(String(elemNew!.BinProp), "\"bytes\""); + assert.deepEqual(elemNew!.RelatedElem, { Id: partialElementId, RelECClassId: "TestDomain.Test2dUsesElement" }); + assert.equal(elemNew!.$meta.op, "Updated"); + assert.equal(elemNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elemNew!.$meta.rowOptions, { classIdsToClassNames: true }); + + assert.deepEqual(Object.keys(elemNew!).sort(), ["$meta", "BinProp", "BoolProp", + "DblProp", "DtProp", "IntArrProp", "IntProp", "LastMod", "LongProp", + "Pt2dProp", "Pt3dProp", "RelatedElem", "StrArrProp", "StrProp", "StructArrProp", "StructProp", "ECClassId", "ECInstanceId"].sort()); + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ["BinProp", "BoolProp", + "DblProp", "DtProp", "ECInstanceId", "IntArrProp", "IntProp", "LastMod", + "LongProp", "Pt2dProp", "Pt3dProp.X", "Pt3dProp.Y", "RelatedElem.Id", "StrArrProp", "StrProp", + "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", + "StructProp.Y", "StructProp.Z"].sort()); + + const elemOld = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "Old"); + expect(elemOld).to.exist; + assert.equal(elemOld!.ECClassId, "TestDomain.Test2dElement"); + assert.equal(elemOld!.StrProp, "hello"); + assert.equal(elemOld!.IntProp, 42); + assert.include(String(elemOld!.BinProp), "\"bytes\""); + assert.deepEqual(elemOld!.RelatedElem, { Id: drawingCategoryId, RelECClassId: "TestDomain.Test2dUsesElement" }); + assert.equal(elemOld!.$meta.op, "Updated"); + assert.deepEqual(elemOld!.$meta.rowOptions, { classIdsToClassNames: true }); + + assert.deepEqual(Object.keys(elemOld!).sort(), ["$meta", "BinProp", "BoolProp", + "DblProp", "DtProp", "IntArrProp", "IntProp", "LastMod", "LongProp", + "Pt2dProp", "Pt3dProp", "RelatedElem", "StrArrProp", "StrProp", "StructArrProp", "StructProp", "ECClassId", "ECInstanceId"].sort()); + assert.deepEqual([...elemOld!.$meta.changeFetchedPropNames].sort(), ["BinProp", "BoolProp", + "DblProp", "DtProp", "ECInstanceId", "IntArrProp", "IntProp", "LastMod", + "LongProp", "Pt2dProp", "Pt3dProp.X", "Pt3dProp.Y", "RelatedElem.Id", "StrArrProp", "StrProp", + "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", + "StructProp.Y", "StructProp.Z"].sort()); + }); + it("should throw error if tried to fetch changeset metadata values before stepping", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + expect(() => reader.isECTable).to.throw(); + expect(() => reader.isIndirectChange).to.throw(); + expect(() => reader.tableName).to.throw(); + expect(() => reader.op).to.throw(); + assert.isUndefined(reader.inserted); + assert.isUndefined(reader.deleted); + reader.close(); + }); + it("should throw error if tried to fetch changeset metadata values after stepping past the end", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + while (reader.step()) { } + assert.equal(reader.step(), false); + assert.equal(reader.step(), false); // trying multiple steps past the end should still return false and not throw, but metadata access should throw + assert.equal(reader.step(), false); + expect(() => reader.isECTable).to.throw(); + expect(() => reader.isIndirectChange).to.throw(); + expect(() => reader.tableName).to.throw(); + expect(() => reader.op).to.throw(); + assert.isUndefined(reader.inserted); + assert.isUndefined(reader.deleted); + reader.close(); + }); +}); + +describe("ChangesetReader delete-partial", () => { + let rwIModel: BriefcaseDb; + let partialElementId: Id64String; + let drawingModelId: Id64String; + let drawingCategoryId: Id64String; + let txnId: string; + let txn: EditTxn; + + before(async () => { + HubMock.startup("ECChangesetDeletePartial", KnownTestLocations.outputDir); const adminToken = "super manager token"; - const iModelName = "test"; - const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(rwIModelId); - const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); - const txn = startTestTxn(rwIModel, "openGroup writeToFile"); - // 1. Import schema with class that span overflow table. + const iTwinId = HubMock.iTwinId; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "deletePartial", description: "deletePartial", accessToken: adminToken }); + rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + txn = startTestTxn(rwIModel, "ChangesetReader delete-partial"); + // Txn 1: import schema + drawing model setup, then push const schema = ` - - - - bis:GraphicalElement2d - - - `; + + + + + + + + + + + + + + + + + + bis:GraphicalElement2d + + + + + + + + + + + + + + + + + + + + `; await importSchemaStrings(txn, [schema]); rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - // Create drawing model and category await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); const codeProps = Code.createEmpty(); - codeProps.value = "DrawingModel"; - const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); - if (undefined === drawingCategoryId) - drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); - - txn.saveChanges(); - await rwIModel.pushChanges({ description: "setup category", accessToken: adminToken }); - const geomArray: Arc3d[] = [ - Arc3d.createXY(Point3d.create(0, 0), 5), - Arc3d.createXY(Point3d.create(5, 5), 2), - Arc3d.createXY(Point3d.create(-5, -5), 20), - ]; - - const geometryStream: GeometryStreamProps = []; - for (const geom of geomArray) { - const arcData = IModelJson.Writer.toIModelJson(geom); - geometryStream.push(arcData); - } - - const e1 = { - classFullName: `TestDomain:Test2dElement`, + codeProps.value = "DrillDownDrawing"; + [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); + + const foundCat = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "DrillDownCategory"); + drawingCategoryId = foundCat ?? DrawingCategory.insert(txn, IModel.dictionaryId, "DrillDownCategory", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + + txn.saveChanges("setup"); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Txn 2: insert PARTIAL element + partialElementId = txn.insertElement({ + classFullName: "TestDomain:Test2dElement", model: drawingModelId, category: drawingCategoryId, code: Code.createEmpty(), - geom: geometryStream, - ...{ p1: "test1" }, - }; - - // 2. Insert a element for the class - await rwIModel.locks.acquireLocks({ shared: drawingModelId }); - const e1id = txn.insertElement(e1); - assert.isTrue(Id64.isValidId64(e1id), "insert worked"); - txn.saveChanges(); - await rwIModel.pushChanges({ description: "insert element", accessToken: adminToken }); - - // 3. Update the element. - const updatedElementProps = Object.assign(rwIModel.elements.getElementProps(e1id), { p1: "test2" }); - await rwIModel.locks.acquireLocks({ exclusive: e1id }); - txn.updateElement(updatedElementProps); - txn.saveChanges(); - await rwIModel.pushChanges({ description: "update element", accessToken: adminToken }); + // StrProp, IntProp, LongProp, DblProp, BoolProp, DtProp, BinProp intentionally absent + } as any); + txn.saveChanges("insert partial element"); + txnId = rwIModel.txns.getLastSavedTxnProps()!.id; + // Wait so that LastMod on bis_Model gets a distinct timestamp before the delete txn + await new Promise((resolve) => setTimeout(resolve, 300)); + await rwIModel.locks.acquireLocks({ exclusive: partialElementId }); + txn.deleteElement(partialElementId); + txn.saveChanges("delete partial element"); + txnId = rwIModel.txns.getLastSavedTxnProps()!.id; + }); - // 4. Delete the element. - await rwIModel.locks.acquireLocks({ exclusive: e1id }); - txn.deleteElement(e1id); - txn.saveChanges(); - await rwIModel.pushChanges({ description: "delete element", accessToken: adminToken }); + after(() => { + txn.end(); + rwIModel?.close(); + HubMock.shutdown(); + }); - const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); - const changesets = (await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir })).slice(1); - - const testElementClassId: Id64String = getClassIdByName(rwIModel, "Test2dElement"); - const drawingModelClassId: Id64String = getClassIdByName(rwIModel, "DrawingModel"); - - if (true || "Grouping changeset [2,3,4] should not contain TestDomain:Test2dElement as insert+update+delete=noop") { - const reader = SqliteChangesetReader.openGroup({ changesetFiles: changesets.map((c) => c.pathname), db: rwIModel, disableSchemaCheck: true }); - const adaptor = new ECChangesetAdaptor(reader); - const instances: ({ id: string, classId?: string, op: SqliteChangeOp, classFullName?: string })[] = []; - while (adaptor.step()) { - if (adaptor.inserted) { - instances.push({ id: adaptor.inserted?.ECInstanceId, classId: adaptor.inserted.ECClassId, op: adaptor.op, classFullName: adaptor.inserted.$meta?.classFullName }); - } else if (adaptor.deleted) { - instances.push({ id: adaptor.deleted?.ECInstanceId, classId: adaptor.deleted.ECClassId, op: adaptor.op, classFullName: adaptor.deleted.$meta?.classFullName }); - } - } + it("txn4 delete-partial | All_Properties | default rowOptions", () => { + const instances = readTxn(rwIModel, txnId); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.isString(modelOld!.LastMod); + assert.isString(modelOld!.GeometryGuid); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement (partial) Deleted Old --- + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "Old"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, partialElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.equal(elem!.Model.Id, drawingModelId); + assert.equal(rwIModel.getClassNameFromId(elem!.Model.RelECClassId), "BisCore:ModelContainsElements"); + assert.isString(elem!.LastMod); + assert.equal(elem!.CodeSpec.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeSpec.RelECClassId), "BisCore:CodeSpecSpecifiesCode"); + + assert.equal(elem!.CodeScope.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeScope.RelECClassId), "BisCore:ElementScopesCode"); + assert.isString(elem!.FederationGuid); + assert.equal(elem!.Category.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.Category.RelECClassId), "BisCore:GeometricElement2dIsInCategory"); + // Partial element had no custom props at time of delete + assert.isUndefined(elem!.StrProp); + assert.isUndefined(elem!.IntProp); + assert.isUndefined(elem!.Pt2dProp); + assert.isUndefined(elem!.StructProp); + assert.isUndefined(elem!.IntArrProp); + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", + "Category", + ].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Deleted"); + assert.equal(elem!.$meta.stage, "Old"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), ["BBoxHigh", "BBoxLow", "BinProp", "BoolProp", + "Category.Id", "CodeScope.Id", "CodeSpec.Id", "CodeValue", "DblProp", "DtProp", "ECClassId", "ECInstanceId", "FederationGuid", "GeometryStream", + "IntArrProp", "IntProp", "JsonProperties", "LastMod", "LongProp", "Model.Id", "Origin", "Parent", "Pt2dProp", "Pt3dProp", "RelatedElem", "Rotation", "StrArrProp", "StrProp", + "StructArrProp", "StructProp.Label", "StructProp.Pt2d", "StructProp.Pt3d", "StructProp.X", "StructProp.Y", "StructProp.Z", "TypeDefinition", "UserLabel" + ].sort()); + assert.deepEqual(elem!.$meta.rowOptions, {}); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - expect(instances.length).to.eq(1); - expect(instances[0].id).to.eq("0x20000000001"); - expect(instances[0].classId).to.eq(drawingModelClassId); - expect(instances[0].op).to.eq("Updated"); - expect(instances[0].classFullName).to.eq("BisCore:DrawingModel"); - } + it("txn4 delete-partial | Bis_Element_Properties", () => { + const instances = readTxn(rwIModel, txnId, PropertyFilter.BisCoreElement, { classIdsToClassNames: true }); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal(modelNew!.ECClassId, "BisCore.DrawingModel"); + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal(modelOld!.ECClassId, "BisCore.DrawingModel"); + assert.isString(modelOld!.LastMod); + assert.isString(modelOld!.GeometryGuid); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement (partial) Deleted Old --- + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "Old"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, partialElementId); + assert.equal(elem!.ECClassId, "TestDomain.Test2dElement"); + assert.isUndefined(elem!.StrProp); + assert.isUndefined(elem!.Category); + expect(elem!.Model).to.exist; + expect(elem!.LastMod).to.exist; + expect(elem!.CodeSpec).to.exist; + expect(elem!.CodeScope).to.exist; + expect(elem!.FederationGuid).to.exist; + assert.deepEqual(Object.keys(elem!).sort(), ["ECInstanceId", "ECClassId", "$meta", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid"].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Deleted"); + assert.equal(elem!.$meta.stage, "Old"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.BisCoreElement); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), ["CodeScope.Id", "CodeSpec.Id", "CodeValue", "ECClassId", "ECInstanceId", + "FederationGuid", "JsonProperties", "LastMod", "Model.Id", "Parent", "UserLabel"].sort()); + assert.deepEqual(elem!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - if (true || "Grouping changeset [3,4] should contain update+delete=delete TestDomain:Test2dElement") { - const reader = SqliteChangesetReader.openGroup({ changesetFiles: changesets.slice(1).map((c) => c.pathname), db: rwIModel, disableSchemaCheck: true }); - const adaptor = new ECChangesetAdaptor(reader); - const instances: ({ id: string, classId?: string, op: SqliteChangeOp, classFullName?: string })[] = []; - while (adaptor.step()) { - if (adaptor.inserted) { - instances.push({ id: adaptor.inserted?.ECInstanceId, classId: adaptor.inserted.ECClassId, op: adaptor.op, classFullName: adaptor.inserted.$meta?.classFullName }); - } else if (adaptor.deleted) { - instances.push({ id: adaptor.deleted?.ECInstanceId, classId: adaptor.deleted.ECClassId, op: adaptor.op, classFullName: adaptor.deleted.$meta?.classFullName }); - } - } - expect(instances.length).to.eq(3); - expect(instances[0]).deep.eq({ - id: "0x20000000004", - classId: testElementClassId, - op: "Deleted", - classFullName: "TestDomain:Test2dElement", - }); - expect(instances[1]).deep.eq({ - id: "0x20000000004", - classId: testElementClassId, - op: "Deleted", - classFullName: "TestDomain:Test2dElement", - }); - expect(instances[2]).deep.eq({ - id: "0x20000000001", - classId: drawingModelClassId, - op: "Updated", - classFullName: "BisCore:DrawingModel", - }); - } + it("txn4 delete-partial | Instance_Key", () => { + const instances = readTxn(rwIModel, txnId, PropertyFilter.InstanceKey); + assert.equal(instances.length, 3); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // --- instances[2]: Test2dElement (partial) Deleted Old --- + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "Old"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, partialElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.isUndefined(elem!.StrProp); + assert.isUndefined(elem!.Model); + assert.isUndefined(elem!.LastMod); + assert.deepEqual(Object.keys(elem!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Deleted"); + assert.equal(elem!.$meta.stage, "Old"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.InstanceKey); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), ['ECClassId', 'ECInstanceId'].sort()); + assert.deepEqual(elem!.$meta.rowOptions, {}); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - const groupCsFile = path.join(KnownTestLocations.outputDir, "changeset_grouping.ec"); - if (true || "Grouping changeset [2,3] should contain insert+update=insert TestDomain:Test2dElement") { - const reader = SqliteChangesetReader.openGroup({ changesetFiles: changesets.slice(0, 2).map((c) => c.pathname), db: rwIModel, disableSchemaCheck: true }); - const adaptor = new ECChangesetAdaptor(reader); - const instances: ({ id: string, classId?: string, op: SqliteChangeOp, classFullName?: string })[] = []; - while (adaptor.step()) { - if (adaptor.inserted) { - instances.push({ id: adaptor.inserted?.ECInstanceId, classId: adaptor.inserted.ECClassId, op: adaptor.op, classFullName: adaptor.inserted.$meta?.classFullName }); - } else if (adaptor.deleted) { - instances.push({ id: adaptor.deleted?.ECInstanceId, classId: adaptor.deleted.ECClassId, op: adaptor.op, classFullName: adaptor.deleted.$meta?.classFullName }); - } - } - expect(instances.length).to.eq(3); - expect(instances[0]).deep.eq({ - id: "0x20000000004", - classId: testElementClassId, - op: "Inserted", - classFullName: "TestDomain:Test2dElement", - }); - expect(instances[1]).deep.eq({ - id: "0x20000000004", - classId: testElementClassId, - op: "Inserted", - classFullName: "TestDomain:Test2dElement", - }); - expect(instances[2]).deep.eq({ - id: "0x20000000001", - classId: drawingModelClassId, - op: "Updated", - classFullName: "BisCore:DrawingModel", - }); - - reader.writeToFile({ fileName: groupCsFile, containsSchemaChanges: false, overwriteFile: true }); - } - if (true || "writeToFile() test") { - const reader = SqliteChangesetReader.openFile({ fileName: groupCsFile, db: rwIModel, disableSchemaCheck: true }); - const adaptor = new ECChangesetAdaptor(reader); - const instances: ({ id: string, classId?: string, op: SqliteChangeOp, classFullName?: string })[] = []; - while (adaptor.step()) { - if (adaptor.inserted) { - instances.push({ id: adaptor.inserted?.ECInstanceId, classId: adaptor.inserted.ECClassId, op: adaptor.op, classFullName: adaptor.inserted.$meta?.classFullName }); - } else if (adaptor.deleted) { - instances.push({ id: adaptor.deleted?.ECInstanceId, classId: adaptor.deleted.ECClassId, op: adaptor.op, classFullName: adaptor.deleted.$meta?.classFullName }); - } - } - expect(instances.length).to.eq(3); - expect(instances[0]).deep.eq({ - id: "0x20000000004", - classId: testElementClassId, - op: "Inserted", - classFullName: "TestDomain:Test2dElement", - }); - expect(instances[1]).deep.eq({ - id: "0x20000000004", - classId: testElementClassId, - op: "Inserted", - classFullName: "TestDomain:Test2dElement", - }); - expect(instances[2]).deep.eq({ - id: "0x20000000001", - classId: drawingModelClassId, - op: "Updated", - classFullName: "BisCore:DrawingModel", - }); - } - txn.end(); - rwIModel.close(); + it("txn4 delete-partial | rowOptions: classIdsToClassNames", () => { + const instances = readTxn(rwIModel, txnId, undefined, { classIdsToClassNames: true }); + assert.equal(instances.length, 3); + + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECClassId, "BisCore.DrawingModel"); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true }); + + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECClassId, "BisCore.DrawingModel"); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true }); + + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "Old"); + expect(elem).to.exist; + assert.equal(elem!.ECClassId, "TestDomain.Test2dElement"); + assert.deepEqual(elem!.Model, { Id: drawingModelId, RelECClassId: "BisCore.ModelContainsElements" }); + assert.deepEqual(elem!.CodeSpec, { Id: "0x1", RelECClassId: "BisCore.CodeSpecSpecifiesCode" }); + assert.deepEqual(elem!.CodeScope, { Id: "0x1", RelECClassId: "BisCore.ElementScopesCode" }); + assert.deepEqual(elem!.Category, { Id: drawingCategoryId, RelECClassId: "BisCore.GeometricElement2dIsInCategory" }); + assert.isUndefined(elem!.StrProp); + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", "Category", + ].sort()); + assert.equal(elem!.$meta.op, "Deleted"); + assert.equal(elem!.$meta.stage, "Old"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { classIdsToClassNames: true }); }); - it("Delete class FK constraint violation in cache table", async () => { - // Helper to check if TestClass exists in schema and cache table for both briefcases - function checkClass(firstBriefcase: BriefcaseDb, isClassInFirst: boolean, secondBriefcase: BriefcaseDb, isClassInSecond: boolean) { - const firstItems = firstBriefcase.getSchemaProps("TestSchema").items; - assert.equal(isClassInFirst, !!firstItems?.TestClass); + it("txn4 delete-partial | rowOptions: useJsName", () => { + const instances = readTxn(rwIModel, txnId, undefined, { useJsName: true }); + assert.equal(instances.length, 3); + + const modelNew = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.className, "BisCore.DrawingModel"); + assert.isUndefined(modelNew!.ECInstanceId); + assert.deepEqual(modelNew!.$meta.rowOptions, { useJsName: true }); + + const modelOld = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.className, "BisCore.DrawingModel"); + assert.deepEqual(modelOld!.$meta.rowOptions, { useJsName: true }); + + const elem = instances.find((i) => i.id === partialElementId && i.$meta.stage === "Old"); + expect(elem).to.exist; + assert.equal(elem!.className, "TestDomain.Test2dElement"); + assert.isUndefined(elem!.ECInstanceId); + assert.isUndefined(elem!.ECClassId); + assert.deepEqual(elem!.model, { id: drawingModelId, relClassName: "BisCore.ModelContainsElements" }); + assert.deepEqual(elem!.codeSpec, { id: "0x1", relClassName: "BisCore.CodeSpecSpecifiesCode" }); + assert.deepEqual(elem!.codeScope, { id: "0x1", relClassName: "BisCore.ElementScopesCode" }); + assert.deepEqual(elem!.category, { id: drawingCategoryId, relClassName: "BisCore.GeometricElement2dIsInCategory" }); + assert.isUndefined(elem!.strProp); + assert.deepEqual(Object.keys(elem!).sort(), [ + "id", "className", "model", "lastMod", "codeSpec", "codeScope", "federationGuid", "$meta", "category", + ].sort()); + assert.equal(elem!.$meta.op, "Deleted"); + assert.equal(elem!.$meta.stage, "Old"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { useJsName: true }); + }); - const secondItems = secondBriefcase.getSchemaProps("TestSchema").items; - assert.equal(isClassInSecond, !!secondItems?.TestClass); + it("txn4 delete-partial | rowOptions: abbreviateBlobs", () => { + const instances = readTxn(rwIModel, txnId, undefined, { abbreviateBlobs: true }); + assert.equal(instances.length, 3); + + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.deepEqual(modelNew!.$meta.rowOptions, { abbreviateBlobs: true }); + + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + assert.deepEqual(modelOld!.$meta.rowOptions, { abbreviateBlobs: true }); + + // Partial element had no blobs; ECClassId still raw hex + const elem = instances.find((i) => i.ECInstanceId === partialElementId && i.$meta.stage === "Old"); + expect(elem).to.exist; + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.equal(elem!.Model.Id, drawingModelId); + assert.equal(rwIModel.getClassNameFromId(elem!.Model.RelECClassId), "BisCore:ModelContainsElements"); + assert.isUndefined(elem!.StrProp); + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", "Category", + ].sort()); + assert.equal(elem!.$meta.op, "Deleted"); + assert.equal(elem!.$meta.stage, "Old"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { abbreviateBlobs: true }); + }); - const sql = `SELECT ch.classId FROM ec_cache_ClassHierarchy ch JOIN ec_Class c ON ch.classId = c.Id WHERE c.Name = 'TestClass'`; - const firstStmt = firstBriefcase.prepareSqliteStatement(sql); - assert.equal(firstStmt.step(), isClassInFirst ? DbResult.BE_SQLITE_ROW : DbResult.BE_SQLITE_DONE); - firstStmt[Symbol.dispose](); + it("txn4 delete-partial | rowOptions: classIdsToClassNames + useJsName", () => { + const instances = readTxn(rwIModel, txnId, undefined, { classIdsToClassNames: true, useJsName: true }); + assert.equal(instances.length, 3); + + const modelNew = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.className, "BisCore.DrawingModel"); + assert.isUndefined(modelNew!.ECInstanceId); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + + const modelOld = instances.find((i) => i.id === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.className, "BisCore.DrawingModel"); + assert.deepEqual(modelOld!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + + const elem = instances.find((i) => i.id === partialElementId && i.$meta.stage === "Old"); + expect(elem).to.exist; + assert.equal(elem!.className, "TestDomain.Test2dElement"); + assert.isUndefined(elem!.ECInstanceId); + assert.deepEqual(elem!.model, { id: drawingModelId, relClassName: "BisCore.ModelContainsElements" }); + assert.deepEqual(elem!.codeSpec, { id: "0x1", relClassName: "BisCore.CodeSpecSpecifiesCode" }); + assert.deepEqual(elem!.codeScope, { id: "0x1", relClassName: "BisCore.ElementScopesCode" }); + assert.deepEqual(elem!.category, { id: drawingCategoryId, relClassName: "BisCore.GeometricElement2dIsInCategory" }); + assert.isUndefined(elem!.strProp); + assert.deepEqual(Object.keys(elem!).sort(), [ + "id", "className", "model", "lastMod", "codeSpec", "codeScope", "federationGuid", "$meta", "category", + ].sort()); + assert.equal(elem!.$meta.op, "Deleted"); + assert.equal(elem!.$meta.stage, "Old"); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elem!.$meta.rowOptions, { classIdsToClassNames: true, useJsName: true }); + }); + it("should throw error if tried to fetch changeset metadata values before stepping", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + expect(() => reader.isECTable).to.throw(); + expect(() => reader.isIndirectChange).to.throw(); + expect(() => reader.tableName).to.throw(); + expect(() => reader.op).to.throw(); + assert.isUndefined(reader.inserted); + assert.isUndefined(reader.deleted); + reader.close(); + }); + it("should throw error if tried to fetch changeset metadata values after stepping past the end", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + while (reader.step()) { } + assert.equal(reader.step(), false); + assert.equal(reader.step(), false); // try stepping again after already stepping past the end + assert.equal(reader.step(), false); + expect(() => reader.isECTable).to.throw(); + expect(() => reader.isIndirectChange).to.throw(); + expect(() => reader.tableName).to.throw(); + expect(() => reader.op).to.throw(); + assert.isUndefined(reader.inserted); + assert.isUndefined(reader.deleted); + reader.close(); + }); +}); - const secondStmt = secondBriefcase.prepareSqliteStatement(sql); - assert.equal(secondStmt.step(), isClassInSecond ? DbResult.BE_SQLITE_ROW : DbResult.BE_SQLITE_DONE); - secondStmt[Symbol.dispose](); - } +describe("ChangesetReader filters", () => { + let rwIModel: BriefcaseDb; + let fullElementId: Id64String; + let drawingModelId: Id64String; + let drawingCategoryId: Id64String; + let txnId: string; + let txn: EditTxn; + before(async () => { + HubMock.startup("ECChangesetInsertFull", KnownTestLocations.outputDir); const adminToken = "super manager token"; - const iModelName = "test"; - const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(rwIModelId); + const iTwinId = HubMock.iTwinId; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "insertFull", description: "insertFull", accessToken: adminToken }); + rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + txn = startTestTxn(rwIModel, "ChangesetReader filters"); + // Txn 1: import schema + drawing model setup, then push + const schema = ` + + + + + + + + + + + + + + + + + + bis:GraphicalElement2d + + + + + + + + + + + + + + + + + + + + `; + await importSchemaStrings(txn, [schema]); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - // Open two briefcases for the same iModel - const [firstBriefCase, secondBriefCase] = await Promise.all([ - HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }), - HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }) - ]); - const firstTxn = startTestTxn(firstBriefCase, "delete class FK constraint setup"); + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "DrillDownDrawing"; + [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - // Enable shared channel for both - [firstBriefCase, secondBriefCase].forEach(briefcase => briefcase.channels.addAllowedChannel(ChannelControl.sharedChannelName)); + const foundCat = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "DrillDownCategory"); + drawingCategoryId = foundCat ?? DrawingCategory.insert(txn, IModel.dictionaryId, "DrillDownCategory", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + + txn.saveChanges("setup"); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Txn 2: insert FULL element every EC primitive type populated + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + const geom: GeometryStreamProps = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + Arc3d.createXY(Point3d.create(5, 5), 2), + Arc3d.createXY(Point3d.create(-5, -5), 20), + ].map((a) => IModelJson.Writer.toIModelJson(a)); - await importSchemaStrings(firstTxn, [` - - - + fullElementId = txn.insertElement({ + classFullName: "TestDomain:Test2dElement", + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + geom, + StrProp: "hello", + IntProp: 42, + LongProp: 9_007_199_254_740_991, // Number.MAX_SAFE_INTEGER + DblProp: 3.14159265358979, + BoolProp: true, + DtProp: "2024-01-15T12:00:00.000", + BinProp: new Uint8Array([1, 2, 3, 4]), + Pt2dProp: { x: 1.5, y: 2.5 }, + Pt3dProp: { x: 3.0, y: 4.0, z: 5.0 }, + StructProp: { + X: 1.0, Y: 2.0, Z: 3.0, Label: "origin", + Pt2d: { x: 0.5, y: 0.5 }, + Pt3d: { x: 1.0, y: 2.0, z: 3.0 }, + }, + IntArrProp: [10, 20, 30], + StrArrProp: ["alpha", "beta", "gamma"], + StructArrProp: [ + { X: 0.0, Y: 1.0, Z: 2.0, Label: "a", Pt2d: { x: 0.0, y: 0.0 }, Pt3d: { x: 0.0, y: 0.0, z: 0.0 } }, + { X: 3.0, Y: 4.0, Z: 5.0, Label: "b", Pt2d: { x: 1.0, y: 1.0 }, Pt3d: { x: 1.0, y: 1.0, z: 1.0 } }, + ], + RelatedElem: { id: drawingCategoryId, relClassName: "TestDomain:Test2dUsesElement" }, + } as any); + txn.saveChanges("insert full element"); + txnId = rwIModel.txns.getLastSavedTxnProps()!.id; + }); - - - + after(() => { + txn.end(); + rwIModel?.close(); + HubMock.shutdown(); + }); - - bis:PhysicalElement - - `]); + it("txn1 insert-full | filter by table name", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + reader.setTableNameFilters(new Set(["bis_Model"])); + while (reader.step()) + pcu.appendFrom(reader); + const instances = Array.from(pcu.instances); + assert.equal(instances.length, 2); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + // Object.keys + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + // $meta keys + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.deepEqual([...modelNew!.$meta.tables].sort(), ["bis_Model"].sort()); + assert.deepEqual([...modelNew!.$meta.changeIndexes].sort(), [1].sort()); + assert.isString(modelNew!.$meta.instanceKey); + assert.equal(modelNew!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + // Object.keys + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + // $meta keys + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.deepEqual([...modelOld!.$meta.tables].sort(), ["bis_Model"].sort()); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + }); - // Push the changes to the hub - await firstBriefCase.pushChanges({ description: "push initial schema changeset", accessToken: adminToken }); + it("txn1 insert-full | filter by operation name", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + reader.setOpCodeFilters(new Set(["Inserted"])); + while (reader.step()) + pcu.appendFrom(reader); + const instances = Array.from(pcu.instances); + assert.equal(instances.length, 1); + + const elem = instances.find((i) => i.ECInstanceId === fullElementId && i.$meta.stage === "New"); + expect(elem).to.exist; + assert.equal(elem!.ECInstanceId, fullElementId); + assert.equal("TestDomain:Test2dElement", rwIModel.getClassNameFromId(elem!.ECClassId)); + assert.equal(elem!.Model.Id, drawingModelId); + assert.equal(rwIModel.getClassNameFromId(elem!.Model.RelECClassId), "BisCore:ModelContainsElements"); + assert.isString(elem!.LastMod); + assert.equal(elem!.CodeSpec.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeSpec.RelECClassId), "BisCore:CodeSpecSpecifiesCode"); + + assert.equal(elem!.CodeScope.Id, "0x1"); + assert.equal(rwIModel.getClassNameFromId(elem!.CodeScope.RelECClassId), "BisCore:ElementScopesCode"); + assert.isString(elem!.FederationGuid); + + assert.equal(elem!.Category.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.Category.RelECClassId), "BisCore:GeometricElement2dIsInCategory"); + assert.deepEqual(elem!.Origin, { X: 0, Y: 0 }); + assert.equal(elem!.Rotation, 0); + assert.deepEqual(elem!.BBoxLow, { X: -25, Y: -25 }); + assert.deepEqual(elem!.BBoxHigh, { X: 15, Y: 15 }); + assert.include(String(elem!.GeometryStream), "\"bytes\""); + assert.include(String(elem!.BinProp), "\"bytes\""); + assert.equal(elem!.StrProp, "hello"); + assert.equal(elem!.IntProp, 42); + assert.equal(elem!.LongProp, 9007199254740991); + assert.closeTo(elem!.DblProp as number, 3.14159265358979, 1e-10); + assert.equal(elem!.BoolProp, true); + assert.equal(elem!.DtProp, "2024-01-15T12:00:00.000"); + assert.deepEqual(elem!.Pt2dProp, { X: 1.5, Y: 2.5 }); + assert.deepEqual(elem!.Pt3dProp, { X: 3, Y: 4, Z: 5 }); + assert.deepEqual(elem!.StructProp, { X: 1, Y: 2, Z: 3, Label: "origin", Pt2d: { X: 0.5, Y: 0.5 }, Pt3d: { X: 1, Y: 2, Z: 3 } }); + assert.deepEqual(elem!.IntArrProp, [10, 20, 30]); + assert.deepEqual(elem!.StrArrProp, ["alpha", "beta", "gamma"]); + assert.deepEqual(elem!.StructArrProp, [ + { X: 0, Y: 1, Z: 2, Label: "a", Pt2d: { X: 0, Y: 0 }, Pt3d: { X: 0, Y: 0, Z: 0 } }, + { X: 3, Y: 4, Z: 5, Label: "b", Pt2d: { X: 1, Y: 1 }, Pt3d: { X: 1, Y: 1, Z: 1 } }, + ]); + assert.equal(elem!.RelatedElem.Id, drawingCategoryId); + assert.equal(rwIModel.getClassNameFromId(elem!.RelatedElem.RelECClassId), "TestDomain:Test2dUsesElement"); + // Object.keys + assert.deepEqual(Object.keys(elem!).sort(), [ + "ECInstanceId", "ECClassId", "Model", "LastMod", "CodeSpec", "CodeScope", "FederationGuid", "$meta", + "Category", "Origin", "Rotation", "BBoxLow", "BBoxHigh", "GeometryStream", + "StrProp", "IntProp", "LongProp", "DblProp", "BoolProp", "DtProp", + "Pt2dProp", "Pt3dProp", "StructProp", "IntArrProp", "StrArrProp", "StructArrProp", "RelatedElem", "BinProp" + ].sort()); + // $meta keys + assert.deepEqual(Object.keys(elem!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(elem!.$meta.op, "Inserted"); + assert.equal(elem!.$meta.stage, "New"); + assert.deepEqual([...elem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual([...elem!.$meta.changeIndexes].sort(), [1, 2].sort()); + assert.isString(elem!.$meta.instanceKey); + assert.equal(elem!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(elem!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...elem!.$meta.changeFetchedPropNames].sort(), [ + 'BBoxHigh', 'BBoxLow', 'BinProp', 'BoolProp', 'Category.Id', 'CodeScope.Id', + 'CodeSpec.Id', 'CodeValue', 'DblProp', 'DtProp', 'ECClassId', 'ECInstanceId', + 'FederationGuid', 'GeometryStream', 'IntArrProp', 'IntProp', 'JsonProperties', + 'LastMod', 'LongProp', 'Model.Id', 'Origin', 'Parent', 'Pt2dProp', 'Pt3dProp', + 'RelatedElem', 'Rotation', 'StrArrProp', 'StrProp', 'StructArrProp', 'StructProp.Label', + 'StructProp.Pt2d', 'StructProp.Pt3d', 'StructProp.X', 'StructProp.Y', 'StructProp.Z', + 'TypeDefinition', 'UserLabel' + ].sort()); + assert.deepEqual(elem!.$meta.rowOptions, {}); + assert.equal(elem!.$meta.isIndirectChange, false); + }); - // Sync the second briefcase with the iModel - await secondBriefCase.pullChanges({ accessToken: adminToken }); + it("txn1 insert-full | filter by className", () => { + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createSqliteBackedCache()); + reader.setClassNameFilters(new Set(["BisCore:DrawingModel"])); + while (reader.step()) + pcu.appendFrom(reader); + const instances = Array.from(pcu.instances); + assert.equal(instances.length, 2); + + // --- instances[0]: DrawingModel Updated New --- + const modelNew = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelNew!.ECClassId)); + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + // Object.keys + assert.deepEqual(Object.keys(modelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "GeometryGuid", "$meta"].sort()); + // $meta keys + assert.deepEqual(Object.keys(modelNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelNew!.$meta.op, "Updated"); + assert.equal(modelNew!.$meta.stage, "New"); + assert.deepEqual([...modelNew!.$meta.tables].sort(), ["bis_Model"].sort()); + assert.deepEqual([...modelNew!.$meta.changeIndexes].sort(), [1].sort()); + assert.isString(modelNew!.$meta.instanceKey); + assert.equal(modelNew!.$meta.instanceKey.split(`-`).length, 2); + assert.equal(modelNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, {}); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // --- instances[1]: DrawingModel Updated Old --- + const modelOld = instances.find((i) => i.ECInstanceId === drawingModelId && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.ECInstanceId, drawingModelId); + assert.equal("BisCore:DrawingModel", rwIModel.getClassNameFromId(modelOld!.ECClassId)); + // Object.keys + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + // $meta keys + assert.deepEqual(Object.keys(modelOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "instanceKey", "propFilter", "changeFetchedPropNames", "rowOptions", "isIndirectChange"].sort()); + assert.equal(modelOld!.$meta.op, "Updated"); + assert.equal(modelOld!.$meta.stage, "Old"); + assert.deepEqual([...modelOld!.$meta.tables].sort(), ["bis_Model"].sort()); + assert.equal(modelOld!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual([...modelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelOld!.$meta.rowOptions, {}); + assert.equal(modelOld!.$meta.isIndirectChange, true); + }); +}); - checkClass(firstBriefCase, true, secondBriefCase, true); +describe("ChangesetReader — openFile + openGroup", () => { + let rwIModel: BriefcaseDb; + let rwIModelId: string; + let drawingModelId: Id64String; + let drawingCategoryId: Id64String; + let txn: EditTxn; - // Import the schema - await importSchemaStrings(firstTxn, [` - - + before(async () => { + HubMock.startup("ECChangesetOpenFileGroup", KnownTestLocations.outputDir); + const adminToken = "super manager token"; + const iTwinId = HubMock.iTwinId; + rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "openFileGroup", description: "openFileGroup", accessToken: adminToken }); + rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); - - - - `]); + txn = startTestTxn(rwIModel, "ChangesetReader setup"); - // Push the changeset to the hub - await firstBriefCase.pushChanges({ description: "Delete class major change", accessToken: adminToken }); + // Push 1: import schema + drawing model setup + const schema = ` + + + + bis:GraphicalElement2d + + + + + + + + `; + await importSchemaStrings(txn, [schema]); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - checkClass(firstBriefCase, false, secondBriefCase, true); + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "OpenFileDrawing"; + [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - // Apply the latest changeset to a new briefcase - try { - await secondBriefCase.pullChanges({ accessToken: adminToken }); - } catch (error: any) { - assert.fail(`Should not have failed with the error: ${error.message}`); - } + const foundCat = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "OpenFileCategory"); + drawingCategoryId = foundCat ?? DrawingCategory.insert(txn, IModel.dictionaryId, "OpenFileCategory", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(0,128,255)").toJSON() })); - checkClass(firstBriefCase, false, secondBriefCase, false); + txn.saveChanges("setup"); + await rwIModel.pushChanges({ description: "setup", accessToken: adminToken }); + }); - // Cleanup - firstTxn.end(); - secondBriefCase.close(); - firstBriefCase.close(); + after(() => { + txn.end(); + rwIModel?.close(); + HubMock.shutdown(); }); + it("openFile reads insert and update changesets independently; openGroup reads both as a stream", async () => { + const adminToken = "super manager token"; + const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); - it("Delete class FK constraint violation in cache table through a revert", async () => { - // Helper to check if TestClass exists in schema and cache table for both briefcases - function checkClass(className: string, firstBriefcase: BriefcaseDb, isClassInFirst: boolean, secondBriefcase: BriefcaseDb, isClassInSecond: boolean) { - assert.equal(isClassInFirst, !!firstBriefcase.getSchemaProps("TestSchema").items?.[className]); - assert.equal(isClassInSecond, !!secondBriefcase.getSchemaProps("TestSchema").items?.[className]); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // --- Push 2: insert element — only Y and Z set on Pt3dProp, X omitted (defaults to 0) --- + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + const elementId: Id64String = txn.insertElement({ + classFullName: "TestDomain:SimpleElement", + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + Pt3dProp: { y: 2.5, z: 3.7 }, + BinProp: new Uint8Array([1, 2, 3, 4]), + GuidArrProp: [ + "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "bbbbbbbb-cccc-dddd-eeee-ffffffffffff", + ], + } as any); + txn.saveChanges("insert element"); + await rwIModel.pushChanges({ description: "insert element", accessToken: adminToken }); - const sql = `SELECT ch.classId FROM ec_cache_ClassHierarchy ch JOIN ec_Class c ON ch.classId = c.Id WHERE c.Name = '${className}'`; - const firstStmt = firstBriefcase.prepareSqliteStatement(sql); - assert.equal(firstStmt.step(), isClassInFirst ? DbResult.BE_SQLITE_ROW : DbResult.BE_SQLITE_DONE); - firstStmt[Symbol.dispose](); - const secondStmt = secondBriefcase.prepareSqliteStatement(sql); - assert.equal(secondStmt.step(), isClassInSecond ? DbResult.BE_SQLITE_ROW : DbResult.BE_SQLITE_DONE); - secondStmt[Symbol.dispose](); + let changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); + expect(changesets.length).to.equal(2); + const insertCs = changesets[1]; + // === openFile: insert changeset === + { + using reader = ChangesetReader.openFile({ db: rwIModel, fileName: insertCs.pathname, rowOptions: { abbreviateBlobs: false } }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) + pcu.appendFrom(reader); + const instances = Array.from(pcu.instances); + + const elemNew = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + expect(elemNew).to.exist; + assert.deepEqual(Object.keys(elemNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "propFilter", "rowOptions", "changeFetchedPropNames", "instanceKey", "isIndirectChange"].sort()); + assert.equal(elemNew!.$meta.op, "Inserted"); + assert.equal(elemNew!.$meta.stage, "New"); + assert.deepEqual(elemNew!.$meta.tables.sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual(elemNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elemNew!.$meta.changeIndexes.sort(), [1, 2].sort()); + assert.deepEqual(elemNew!.$meta.rowOptions, { abbreviateBlobs: false }); + assert.equal(elemNew!.$meta.isIndirectChange, false); + + expect(elemNew!.Category).to.exist; + expect(elemNew!.CodeScope).to.exist; + expect(elemNew!.CodeSpec).to.exist; + expect(elemNew!.LastMod).to.exist; + expect(elemNew!.Model).to.exist; + expect(elemNew!.FederationGuid).to.exist; + assert.isUndefined(elemNew!.Pt3dProp); + assert.isNotNull(elemNew!.BinProp); + assert.deepEqual(elemNew!.BinProp, new Uint8Array([1, 2, 3, 4])); + assert.deepEqual(elemNew!.GuidArrProp, [ + "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "bbbbbbbb-cccc-dddd-eeee-ffffffffffff", + ]); + assert.deepEqual(Object.keys(elemNew!).sort(), ["ECInstanceId", "ECClassId", "Model", "CodeSpec", + "CodeScope", "FederationGuid", "$meta", "Category", "LastMod", + "BinProp", "GuidArrProp"].sort()) + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ['BBoxHigh', 'BBoxLow', 'BinProp', + 'Category.Id', 'CodeScope.Id', 'CodeSpec.Id', 'CodeValue', + 'ECClassId', 'ECInstanceId', 'FederationGuid', 'GeometryStream', 'GuidArrProp', + 'JsonProperties', 'LastMod', 'Model.Id', 'Origin', 'Parent', 'Pt3dProp', 'Rotation', + 'TypeDefinition', 'UserLabel'].sort()); } - const adminToken = "super manager token"; - const iModelName = "test"; - const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(rwIModelId); - - // Open two briefcases for the same iModel - const [firstBriefCase, secondBriefCase] = await Promise.all([ - HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }), - HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }) - ]); - const firstTxn = startTestTxn(firstBriefCase, "delete class FK revert"); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the update txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // --- Push 3: update element — change all custom props --- + await rwIModel.locks.acquireLocks({ exclusive: elementId }); + txn.updateElement({ + ...rwIModel.elements.getElementProps(elementId), + Pt3dProp: { x: 1.0, y: 9.9, z: 7.7 }, + BinProp: new Uint8Array([0xaa, 0xbb, 0xcc, 0xdd]), + GuidArrProp: [ + "ffffffff-0000-1111-2222-333344445555", + ], + }); + txn.saveChanges("update element"); + await rwIModel.pushChanges({ description: "update element", accessToken: adminToken }); - // Enable shared channel for both - [firstBriefCase, secondBriefCase].forEach(briefcase => briefcase.channels.addAllowedChannel(ChannelControl.sharedChannelName)); + // Download all changesets: [setup(0), insert(1), update(2)] + changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); + expect(changesets.length).to.equal(3); + const updateCs = changesets[2]; + + // === openFile: update changeset === + { + using reader = ChangesetReader.openFile({ db: rwIModel, fileName: updateCs.pathname, rowOptions: { abbreviateBlobs: false } }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) + pcu.appendFrom(reader); + const instances = Array.from(pcu.instances); + + const elemNew = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + const elemOld = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "Old"); + expect(elemNew).to.exist; + expect(elemOld).to.exist; + assert.equal(elemNew!.$meta.op, "Updated"); + assert.equal(elemNew!.$meta.stage, "New"); + assert.equal(elemNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(Object.keys(elemNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "propFilter", "rowOptions", "changeFetchedPropNames", "instanceKey", "isIndirectChange"].sort()); + assert.equal(elemNew!.$meta.isIndirectChange, false); + assert.deepEqual(Object.keys(elemNew!).sort(), ["ECInstanceId", "ECClassId", "Origin", + "Rotation", "BBoxLow", "BBoxHigh", "Pt3dProp", "BinProp", "GuidArrProp", "$meta", "LastMod"].sort()); + assert.deepEqual(elemNew!.Pt3dProp, { X: 1, Y: 9.9, Z: 7.7 }); + assert.deepEqual(elemNew!.BinProp, new Uint8Array([0xaa, 0xbb, 0xcc, 0xdd])); + assert.deepEqual(elemNew!.GuidArrProp, ["ffffffff-0000-1111-2222-333344445555"]); + assert.deepEqual(elemNew!.Origin, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.BBoxLow, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.BBoxHigh, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.Rotation, 0); + expect(elemNew!.LastMod).to.exist; + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ['BBoxHigh', 'BBoxLow', 'BinProp', 'ECInstanceId', 'GuidArrProp', 'LastMod', 'Origin', 'Pt3dProp', 'Rotation'].sort()); + + assert.equal(elemOld!.$meta.op, "Updated"); + assert.equal(elemOld!.$meta.stage, "Old"); + assert.deepEqual(Object.keys(elemOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "propFilter", "rowOptions", "changeFetchedPropNames", "instanceKey", "isIndirectChange"].sort()); + assert.deepEqual(Object.keys(elemOld!).sort(), ["ECInstanceId", "ECClassId", "BinProp", "GuidArrProp", "$meta", + "LastMod"].sort()); + assert.deepEqual(elemOld!.BinProp, new Uint8Array([1, 2, 3, 4])); + assert.deepEqual(elemOld!.GuidArrProp, [ + "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "bbbbbbbb-cccc-dddd-eeee-ffffffffffff", + ]); + assert.equal(elemOld!.$meta.isIndirectChange, false); + expect(elemOld!.LastMod).to.exist; + assert.deepEqual([...elemOld!.$meta.changeFetchedPropNames].sort(), ['BBoxHigh', 'BBoxLow', 'BinProp', 'ECInstanceId', 'GuidArrProp', 'LastMod', 'Origin', 'Pt3dProp', 'Rotation'].sort()); + } - await importSchemaStrings(firstTxn, [` - - - + // === openGroup: insert + update as a single stream === + // After merging, the elem New key is shared between insert-New and update-New; + // the update-New wins on overlapping props, so the final New reflects the updated state. + // elem Old only comes from the update changeset. + { + using reader = ChangesetReader.openGroup({ db: rwIModel, changesetFiles: [insertCs.pathname, updateCs.pathname], rowOptions: { abbreviateBlobs: false } }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) + pcu.appendFrom(reader); + const instances = Array.from(pcu.instances); + + const elemNew = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + const elemOld = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "Old"); + expect(elemNew).to.exist; + expect(elemOld).to.not.exist; + // New merges insert + update; final values are from the update + assert.deepEqual(elemNew!.Pt3dProp, { X: 1, Y: 9.9, Z: 7.7 }); + assert.deepEqual(elemNew!.BinProp, new Uint8Array([0xaa, 0xbb, 0xcc, 0xdd])); + assert.deepEqual(elemNew!.GuidArrProp, ["ffffffff-0000-1111-2222-333344445555"]); + assert.deepEqual(elemNew!.Origin, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.BBoxLow, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.BBoxHigh, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.Rotation, 0); + expect(elemNew!.LastMod).to.exist; + expect(elemNew!.Category).to.exist; + expect(elemNew!.CodeScope).to.exist; + expect(elemNew!.CodeSpec).to.exist; + expect(elemNew!.LastMod).to.exist; + expect(elemNew!.Model).to.exist; + expect(elemNew!.FederationGuid).to.exist; + assert.deepEqual(Object.keys(elemNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "propFilter", "rowOptions", "changeFetchedPropNames", "instanceKey", "isIndirectChange"].sort()); + assert.equal(elemNew!.$meta.op, "Inserted"); + assert.equal(elemNew!.$meta.stage, "New"); + assert.deepEqual(elemNew!.$meta.tables.sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual(elemNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elemNew!.$meta.changeIndexes.sort(), [1, 2].sort()); + assert.deepEqual(elemNew!.$meta.rowOptions, { abbreviateBlobs: false }); + assert.deepEqual(Object.keys(elemNew!).sort(), ["ECInstanceId", "ECClassId", "Model", "CodeSpec", + "CodeScope", "FederationGuid", "$meta", "Category", "LastMod", + "BinProp", "GuidArrProp", "Origin", "Rotation", "BBoxLow", "BBoxHigh", "Pt3dProp"].sort()); + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ['BBoxHigh', 'BBoxLow', + 'BinProp', 'Category.Id', 'CodeScope.Id', 'CodeSpec.Id', + 'CodeValue', 'ECClassId', 'ECInstanceId', 'FederationGuid', 'GeometryStream', + 'GuidArrProp', 'JsonProperties', 'LastMod', 'Model.Id', 'Origin', 'Parent', 'Pt3dProp', + 'Rotation', 'TypeDefinition', 'UserLabel'].sort()); + assert.equal(elemNew!.$meta.isIndirectChange, false); + } + }); +}); - - - +describe("ChangesetReader — openLocalChanges + openInmemoryChanges", () => { + let rwIModel: BriefcaseDb; + let rwIModelId: string; + let drawingModelId: Id64String; + let drawingCategoryId: Id64String; + let txn: EditTxn; - - bis:PhysicalElement - - `]); + before(async () => { + HubMock.startup("ECChangesetOpenFileGroup", KnownTestLocations.outputDir); + const adminToken = "super manager token"; + const iTwinId = HubMock.iTwinId; + rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "openFileGroup", description: "openFileGroup", accessToken: adminToken }); + rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + txn = startTestTxn(rwIModel, "ChangesetReader setup"); + // Push 1: import schema + drawing model setup + const schema = ` + + + + bis:GraphicalElement2d + + + + + + + + `; + await importSchemaStrings(txn, [schema]); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - // Push the changes to the hub - await firstBriefCase.pushChanges({ description: "push initial schema changeset", accessToken: adminToken }); - // Sync the second briefcase - await secondBriefCase.pullChanges({ accessToken: adminToken }); + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "OpenFileDrawing"; + [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - checkClass("TestClass", firstBriefCase, true, secondBriefCase, true); + const foundCat = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "OpenFileCategory"); + drawingCategoryId = foundCat ?? DrawingCategory.insert(txn, IModel.dictionaryId, "OpenFileCategory", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(0,128,255)").toJSON() })); - // Import the schema - await importSchemaStrings(firstTxn, [` - - - + txn.saveChanges("setup"); + await rwIModel.pushChanges({ description: "setup", accessToken: adminToken }); + }); - - - + after(() => { + txn.end(); + rwIModel?.close(); + HubMock.shutdown(); + }); - - bis:PhysicalElement - + it("opens local and in-memory changes", async () => { + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + const elementId: Id64String = txn.insertElement({ + classFullName: "TestDomain:SimpleElement", + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + Pt3dProp: { y: 2.5, z: 3.7 }, + BinProp: new Uint8Array([1, 2, 3, 4]), + GuidArrProp: [ + "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "bbbbbbbb-cccc-dddd-eeee-ffffffffffff", + ], + } as any); + txn.saveChanges("insert element"); + + + // === openFile: insert changeset === + { + using reader = ChangesetReader.openLocalChanges({ db: rwIModel, rowOptions: { abbreviateBlobs: false } }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createSqliteBackedCache()); + while (reader.step()) + pcu.appendFrom(reader); + const instances = Array.from(pcu.instances); + + const elemNew = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + expect(elemNew).to.exist; + assert.deepEqual(Object.keys(elemNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "propFilter", "rowOptions", "changeFetchedPropNames", "instanceKey", "isIndirectChange"].sort()); + assert.equal(elemNew!.$meta.op, "Inserted"); + assert.equal(elemNew!.$meta.stage, "New"); + assert.deepEqual(elemNew!.$meta.tables.sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual(elemNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elemNew!.$meta.changeIndexes.sort(), [1, 2].sort()); + assert.deepEqual(elemNew!.$meta.rowOptions, { abbreviateBlobs: false }); + assert.equal(elemNew!.$meta.isIndirectChange, false); + + expect(elemNew!.Category).to.exist; + expect(elemNew!.CodeScope).to.exist; + expect(elemNew!.CodeSpec).to.exist; + expect(elemNew!.LastMod).to.exist; + expect(elemNew!.Model).to.exist; + expect(elemNew!.FederationGuid).to.exist; + assert.isUndefined(elemNew!.Pt3dProp); + assert.isNotNull(elemNew!.BinProp); + assert.deepEqual(elemNew!.BinProp, new Uint8Array([1, 2, 3, 4])); + assert.deepEqual(elemNew!.GuidArrProp, [ + "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "bbbbbbbb-cccc-dddd-eeee-ffffffffffff", + ]); + assert.deepEqual(Object.keys(elemNew!).sort(), ["ECInstanceId", "ECClassId", "Model", "CodeSpec", + "CodeScope", "FederationGuid", "$meta", "Category", "LastMod", + "BinProp", "GuidArrProp"].sort()) + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ['BBoxHigh', 'BBoxLow', 'BinProp', + 'Category.Id', 'CodeScope.Id', 'CodeSpec.Id', 'CodeValue', + 'ECClassId', 'ECInstanceId', 'FederationGuid', 'GeometryStream', 'GuidArrProp', + 'JsonProperties', 'LastMod', 'Model.Id', 'Origin', 'Parent', 'Pt3dProp', 'Rotation', + 'TypeDefinition', 'UserLabel'].sort()); + } + // Wait so that LastMod on bis_Model gets a distinct timestamp before the update txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // --- Push 3: update element — change all custom props --- + await rwIModel.locks.acquireLocks({ exclusive: elementId }); + txn.updateElement({ + ...rwIModel.elements.getElementProps(elementId), + Pt3dProp: { x: 1.0, y: 9.9, z: 7.7 }, + BinProp: new Uint8Array([0xaa, 0xbb, 0xcc, 0xdd]), + GuidArrProp: [ + "ffffffff-0000-1111-2222-333344445555", + ], + }); - - bis:PhysicalElement - - `]); + // === openFile: update changeset === + { + using reader = ChangesetReader.openInMemoryChanges({ db: rwIModel, rowOptions: { abbreviateBlobs: false } }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createSqliteBackedCache()); + while (reader.step()) + pcu.appendFrom(reader); + const instances = Array.from(pcu.instances); + + const elemNew = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + const elemOld = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "Old"); + expect(elemNew).to.exist; + expect(elemOld).to.exist; + assert.equal(elemNew!.$meta.op, "Updated"); + assert.equal(elemNew!.$meta.stage, "New"); + assert.equal(elemNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(Object.keys(elemNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "propFilter", "rowOptions", "changeFetchedPropNames", "instanceKey", "isIndirectChange"].sort()); + assert.deepEqual(Object.keys(elemNew!).sort(), ["ECInstanceId", "ECClassId", "Origin", + "Rotation", "BBoxLow", "BBoxHigh", "Pt3dProp", "BinProp", "GuidArrProp", "$meta", "LastMod"].sort()); + assert.deepEqual(elemNew!.Pt3dProp, { X: 1, Y: 9.9, Z: 7.7 }); + assert.deepEqual(elemNew!.BinProp, new Uint8Array([0xaa, 0xbb, 0xcc, 0xdd])); + assert.deepEqual(elemNew!.GuidArrProp, ["ffffffff-0000-1111-2222-333344445555"]); + assert.deepEqual(elemNew!.Origin, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.BBoxLow, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.BBoxHigh, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.Rotation, 0); + expect(elemNew!.LastMod).to.exist; + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ['BBoxHigh', 'BBoxLow', 'BinProp', 'ECInstanceId', 'GuidArrProp', 'LastMod', 'Origin', 'Pt3dProp', 'Rotation'].sort()); + assert.equal(elemNew!.$meta.isIndirectChange, false); + + assert.equal(elemOld!.$meta.op, "Updated"); + assert.equal(elemOld!.$meta.stage, "Old"); + assert.deepEqual(Object.keys(elemOld!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "propFilter", "rowOptions", "changeFetchedPropNames", "instanceKey", "isIndirectChange"].sort()); + assert.deepEqual(Object.keys(elemOld!).sort(), ["ECInstanceId", "ECClassId", "BinProp", "GuidArrProp", "$meta", + "LastMod"].sort()); + assert.deepEqual(elemOld!.BinProp, new Uint8Array([1, 2, 3, 4])); + assert.deepEqual(elemOld!.GuidArrProp, [ + "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "bbbbbbbb-cccc-dddd-eeee-ffffffffffff", + ]); + expect(elemOld!.LastMod).to.exist; + assert.equal(elemOld!.$meta.isIndirectChange, false); + assert.deepEqual([...elemOld!.$meta.changeFetchedPropNames].sort(), ['BBoxHigh', 'BBoxLow', 'BinProp', 'ECInstanceId', 'GuidArrProp', 'LastMod', 'Origin', 'Pt3dProp', 'Rotation'].sort()); + } - // Push the changeset to the hub - await firstBriefCase.pushChanges({ description: "Add another class change", accessToken: adminToken }); - // Sync the second briefcase - await secondBriefCase.pullChanges({ accessToken: adminToken }); + // === openGroup: insert + update as a single stream === + // After merging, the elem New key is shared between insert-New and update-New; + // the update-New wins on overlapping props, so the final New reflects the updated state. + // elem Old only comes from the update changeset. + { + using reader = ChangesetReader.openLocalChanges({ db: rwIModel, includeInMemoryChanges: true, rowOptions: { abbreviateBlobs: false } }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createSqliteBackedCache()); + while (reader.step()) + pcu.appendFrom(reader); + const instances = Array.from(pcu.instances); + + const elemNew = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + const elemOld = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "Old"); + expect(elemNew).to.exist; + expect(elemOld).to.not.exist; + // New merges insert + update; final values are from the update + assert.deepEqual(elemNew!.Pt3dProp, { X: 1, Y: 9.9, Z: 7.7 }); + assert.deepEqual(elemNew!.BinProp, new Uint8Array([0xaa, 0xbb, 0xcc, 0xdd])); + assert.deepEqual(elemNew!.GuidArrProp, ["ffffffff-0000-1111-2222-333344445555"]); + assert.deepEqual(elemNew!.Origin, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.BBoxLow, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.BBoxHigh, { X: 0, Y: 0 }); + assert.deepEqual(elemNew!.Rotation, 0); + expect(elemNew!.LastMod).to.exist; + expect(elemNew!.Category).to.exist; + expect(elemNew!.CodeScope).to.exist; + expect(elemNew!.CodeSpec).to.exist; + expect(elemNew!.LastMod).to.exist; + expect(elemNew!.Model).to.exist; + expect(elemNew!.FederationGuid).to.exist; + assert.deepEqual(Object.keys(elemNew!.$meta).sort(), ["op", "tables", "changeIndexes", "stage", "propFilter", "rowOptions", "changeFetchedPropNames", "instanceKey", "isIndirectChange"].sort()); + assert.equal(elemNew!.$meta.op, "Inserted"); + assert.equal(elemNew!.$meta.stage, "New"); + assert.deepEqual(elemNew!.$meta.tables.sort(), ["bis_Element", "bis_GeometricElement2d"].sort()); + assert.deepEqual(elemNew!.$meta.propFilter, PropertyFilter.All); + assert.deepEqual(elemNew!.$meta.changeIndexes.sort(), [1, 2].sort()); + assert.deepEqual(elemNew!.$meta.rowOptions, { abbreviateBlobs: false }); + assert.deepEqual(Object.keys(elemNew!).sort(), ["ECInstanceId", "ECClassId", "Model", "CodeSpec", + "CodeScope", "FederationGuid", "$meta", "Category", "LastMod", + "BinProp", "GuidArrProp", "Origin", "Rotation", "BBoxLow", "BBoxHigh", "Pt3dProp"].sort()); + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ['BBoxHigh', 'BBoxLow', + 'BinProp', 'Category.Id', 'CodeScope.Id', 'CodeSpec.Id', + 'CodeValue', 'ECClassId', 'ECInstanceId', 'FederationGuid', 'GeometryStream', + 'GuidArrProp', 'JsonProperties', 'LastMod', 'Model.Id', 'Origin', 'Parent', 'Pt3dProp', + 'Rotation', 'TypeDefinition', 'UserLabel'].sort()); + assert.equal(elemNew!.$meta.isIndirectChange, false); + } + }); +}); - checkClass("TestClass", firstBriefCase, true, secondBriefCase, true); - checkClass("AnotherTestClass", firstBriefCase, true, secondBriefCase, true); +describe("ChangesetReader: behaviour in case imodel is not in sync with changeset file being read", async () => { + let rwIModel: BriefcaseDb; + let rwIModelId: string; + let elementId: Id64String; + let drawingModelId: Id64String; + let categoryId1: Id64String; + let categoryId2: Id64String; + let txn: EditTxn; + + before(async () => { + HubMock.startup("ChangesetReaderBugsTest", KnownTestLocations.outputDir); + const adminToken = "super manager token"; + const iTwinId = HubMock.iTwinId; + rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "bugTest", description: "BugTest", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + txn = startTestTxn(rwIModel, "ChangesetReaderBugsTest setup"); + // Push 1: import schema + set up drawing model and two categories + const schema = ` + + + + bis:GraphicalElement2d + + + `; + await importSchemaStrings(txn, [schema]); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - // Revert the latest changeset from the first briefcase - try { - await firstBriefCase.revertAndPushChanges({ toIndex: 2, description: "Revert last changeset" }); - } catch (error: any) { - assert.fail(`Should not have failed with the error: ${error.message}`); - } + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "DrawingModel"; + [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - checkClass("TestClass", firstBriefCase, true, secondBriefCase, true); - checkClass("AnotherTestClass", firstBriefCase, false, secondBriefCase, true); + categoryId1 = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "Category1") + ?? DrawingCategory.insert(txn, IModel.dictionaryId, "Category1", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); - try { - await secondBriefCase.pullChanges({ accessToken: adminToken }); - } catch (error: any) { - assert.fail(`Should not have failed with the error: ${error.message}`); - } + categoryId2 = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "Category2") + ?? DrawingCategory.insert(txn, IModel.dictionaryId, "Category2", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(0,0,255)").toJSON() })); - checkClass("TestClass", firstBriefCase, true, secondBriefCase, true); - checkClass("AnotherTestClass", firstBriefCase, false, secondBriefCase, false); + txn.saveChanges("setup"); + await rwIModel.pushChanges({ description: "setup", accessToken: adminToken }); + }); - // Cleanup - firstTxn.end(); - secondBriefCase.close(); - firstBriefCase.close(); + after(() => { + txn.end(); + rwIModel?.close(); + HubMock.shutdown(); }); - it("Track changeset health stats", async () => { + it("openFile() reads the middle changeset of an insert → update → delete lifecycle", async () => { const adminToken = "super manager token"; - const iModelName = "test"; - const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(rwIModelId); + // Push 2 (insert): insert element with category1 + const geomArray: Arc3d[] = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + Arc3d.createXY(Point3d.create(5, 5), 2), + Arc3d.createXY(Point3d.create(-5, -5), 20), + ]; + const geometryStream: GeometryStreamProps = []; + for (const geom of geomArray) + geometryStream.push(IModelJson.Writer.toIModelJson(geom)); - // Open two briefcases for the same iModel - const [firstBriefcase, secondBriefcase] = await Promise.all([ - HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }), - HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }) - ]); - const firstTxn = startTestTxn(firstBriefcase, "track changeset health first briefcase"); - const secondTxn = startTestTxn(secondBriefcase, "track changeset health second briefcase"); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + elementId = txn.insertElement({ + classFullName: "TestDomain:Test2dElement", + model: drawingModelId, + category: categoryId1, + code: Code.createEmpty(), + geom: geometryStream, + s: { x: 1.5, y: 2.5 }, + } as any); + assert.isTrue(Id64.isValidId64(elementId)); + txn.saveChanges("insert element"); + await rwIModel.pushChanges({ description: "insert element", accessToken: adminToken }); - [firstBriefcase, secondBriefcase].forEach(briefcase => briefcase.channels.addAllowedChannel(ChannelControl.sharedChannelName)); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the update txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Push 3 (update): change category to category2 + await rwIModel.locks.acquireLocks({ exclusive: elementId }); + txn.updateElement({ + ...rwIModel.elements.getElementProps(elementId), + category: categoryId2, + }); + txn.saveChanges("update element"); + await rwIModel.pushChanges({ description: "update element", accessToken: adminToken }); - await importSchemaStrings(firstTxn, [` - - - + // Wait so that LastMod on bis_Model gets a distinct timestamp before the delete txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Push 4 (delete): delete the element + await rwIModel.locks.acquireLocks({ exclusive: elementId }); + txn.deleteElement(elementId); + txn.saveChanges("delete element"); + await rwIModel.pushChanges({ description: "delete element", accessToken: adminToken }); + // changesets: [setup, insert, update, delete] — 4 total; index 2 = update changeset + const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); + const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); + expect(changesets.length).to.equal(4); + const middleChangeset = changesets[Math.floor(changesets.length / 2)]; // index 2 = update + + using reader = ChangesetReader.openFile({ db: rwIModel, fileName: middleChangeset.pathname, propFilter: PropertyFilter.InstanceKey, rowOptions: { classIdsToClassNames: true } }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) + pcu.appendFrom(reader); + + const instances = Array.from(pcu.instances); + + const bisElementOld = instances.filter((i) => i.ECInstanceId === elementId && i.$meta.stage === "Old"); + const bisElementNew = instances.filter((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + expect(bisElementOld).to.exist; + expect(bisElementNew).to.exist; + expect(bisElementOld).to.have.lengthOf(2); + expect(bisElementNew).to.have.lengthOf(2); + // Why this is happening? + // We have deleted the instance from the db when the changeset is opened + // So we are unable to fetch the correct classId for the instance + // and it is defaulting to the base class of the deleted instance which is "BisCore.Element" and "BisCore.GeometricElement2d" instead of "TestDomain.Test2dElement" + // If the instance had not been deleted, we would have been able to fetch the correct classId for the instance and it would have been "TestDomain.Test2dElement" + // And we would have just two entries for the instance with classId "TestDomain.Test2dElement" , one wioth stage "New" and other with "old" + expect(bisElementNew.map((i => i.ECClassId)).sort()).to.deep.equal(["BisCore.GeometricElement2d", "BisCore.Element"].sort()); + expect(bisElementOld.map((i => i.ECClassId)).sort()).to.deep.equal(["BisCore.GeometricElement2d", "BisCore.Element"].sort()); + }); - - - + it("openTxn() reads the middle txn of an insert → update → update lifecycle", async () => { + // Push 2 (insert): insert element with category1 + const geomArray: Arc3d[] = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + Arc3d.createXY(Point3d.create(5, 5), 2), + Arc3d.createXY(Point3d.create(-5, -5), 20), + ]; + const geometryStream: GeometryStreamProps = []; + for (const geom of geomArray) + geometryStream.push(IModelJson.Writer.toIModelJson(geom)); - - bis:PhysicalElement - - `]); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + elementId = txn.insertElement({ + classFullName: "TestDomain:Test2dElement", + model: drawingModelId, + category: categoryId1, + code: Code.createEmpty(), + geom: geometryStream, + s: { x: 1.5, y: 2.5 }, + } as any); + assert.isTrue(Id64.isValidId64(elementId)); + txn.saveChanges("insert element"); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the update txn + await new Promise((resolve) => setTimeout(resolve, 300)); + + await rwIModel.locks.acquireLocks({ exclusive: elementId }); + txn.updateElement({ + ...rwIModel.elements.getElementProps(elementId), + s: { x: 100.0, y: 2.5 }, + }); + txn.saveChanges("update element"); + const txnId = rwIModel.txns.getLastSavedTxnProps()!.id; + assert.isTrue(Id64.isValidId64(txnId)); - // Enable changeset tracking for both briefcases - await Promise.all([firstBriefcase.enableChangesetStatTracking(), secondBriefcase.enableChangesetStatTracking()]); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the update txn + await new Promise((resolve) => setTimeout(resolve, 300)); - await firstBriefcase.pushChanges({ description: "push initial schema changeset", accessToken: adminToken }); - await secondBriefcase.pullChanges({ accessToken: adminToken }); + await rwIModel.locks.acquireLocks({ exclusive: elementId }); + txn.updateElement({ + ...rwIModel.elements.getElementProps(elementId), + s: { x: 100.0, y: 200.0 }, + }); + txn.saveChanges("update element"); + + using reader = ChangesetReader.openTxn({ db: rwIModel, txnId }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) + pcu.appendFrom(reader); + + const instances = Array.from(pcu.instances); + + const elementOld = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "Old"); + const elementNew = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + expect(elementOld).to.exist; + expect(elementNew).to.exist; + // Here as we can see that we correctly captured old value of s.X which is 1.5 and new value of s.X which is 100.0 + // But the Y value, though same is a bit different, it is 200 + // Because in liveDB we have done another tyransaction after the one which we have opened + // In that transaction we have updated the Y value to 200, so when we are fetching the changes for the opened transaction, + // we are getting the latest value of Y which is 200 instead of 2.5 + assert.deepEqual(elementOld!.s, { X: 1.5, Y: 200 }); + assert.deepEqual(elementNew!.s, { X: 100, Y: 200 }); + // But we can find the exact stuff which actually changed and which we fetched from the changeset and not the live DB + // using the changeFetchedPropNames, we can find that only s.X was changed in the changeset and s.Y was not changed in the changeset. + expect(elementNew!.$meta.changeFetchedPropNames).to.include("s.X"); + expect(elementNew!.$meta.changeFetchedPropNames).to.not.include("s.Y"); + expect(elementOld!.$meta.changeFetchedPropNames).to.include("s.X"); + expect(elementOld!.$meta.changeFetchedPropNames).to.not.include("s.Y"); + }); +}); - // Schema upgrade - await importSchemaStrings(secondTxn, [` - - - +describe("ChangesetReader: overflow table graceful recovery when ExclusiveRootClassId is NULL", () => { + let rwIModel: BriefcaseDb; + let rwIModelId: string; + let txn: EditTxn; + let updateChangesetPathname: string; + let id: string; - - - + before(async () => { + HubMock.startup("ECChangesetOverflowNull", KnownTestLocations.outputDir); + const adminToken = "super manager token"; + const iTwinId = HubMock.iTwinId; + const nProps = 36; // enough to spill into the overflow table - - bis:PhysicalElement - - + rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "overflowNull", description: "OverflowNull", accessToken: adminToken }); + rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + txn = startTestTxn(rwIModel, "overflow null test"); - - - - - `]); - - await secondBriefcase.pushChanges({ description: "Added a property to TestClass and an enum", accessToken: adminToken }); - await firstBriefcase.pullChanges({ accessToken: adminToken }); - - // Major schema change - await importSchemaStrings(firstTxn, [` - - - - - - - - - - - - `]); - - await firstBriefcase.pushChanges({ description: "Deleted TestClass", accessToken: adminToken }); - await secondBriefcase.pullChanges({ accessToken: adminToken }); - - const firstBriefcaseChangesets = await firstBriefcase.getAllChangesetHealthData(); - const secondBriefcaseChangesets = await secondBriefcase.getAllChangesetHealthData(); - - assert.equal(firstBriefcaseChangesets.length, 1); - const firstBriefcaseChangeset = firstBriefcaseChangesets[0]; - - expect(firstBriefcaseChangeset.changesetIndex).to.be.eql(2); - expect(firstBriefcaseChangeset.uncompressedSizeBytes).to.be.greaterThan(300); - expect(firstBriefcaseChangeset.insertedRows).to.be.greaterThanOrEqual(4); - expect(firstBriefcaseChangeset.updatedRows).to.be.greaterThanOrEqual(1); - expect(firstBriefcaseChangeset.deletedRows).to.be.eql(0); - expect(firstBriefcaseChangeset.totalFullTableScans).to.be.eql(0); - expect(firstBriefcaseChangeset.perStatementStats.length).to.be.eql(5); - - assert.equal(secondBriefcaseChangesets.length, 2); - const [secondBriefcaseChangeset1, secondBriefcaseChangeset2] = secondBriefcaseChangesets; - - expect(secondBriefcaseChangeset1.changesetIndex).to.be.eql(1); - expect(secondBriefcaseChangeset1.uncompressedSizeBytes).to.be.greaterThan(40000); - expect(secondBriefcaseChangeset1.insertedRows).to.be.eql(52); - expect(secondBriefcaseChangeset1.updatedRows).to.be.greaterThanOrEqual(921); - expect(secondBriefcaseChangeset1.deletedRows).to.be.greaterThanOrEqual(0) - expect(secondBriefcaseChangeset1.totalFullTableScans).to.be.eql(0); - expect(secondBriefcaseChangeset1.perStatementStats.length).to.be.eql(11); - - expect(secondBriefcaseChangeset2.changesetIndex).to.be.eql(3); - expect(secondBriefcaseChangeset2.uncompressedSizeBytes).to.be.greaterThan(40000); - expect(secondBriefcaseChangeset2.insertedRows).to.be.greaterThanOrEqual(0); - expect(secondBriefcaseChangeset2.updatedRows).to.be.greaterThanOrEqual(921); - expect(secondBriefcaseChangeset2.deletedRows).to.be.eql(52); - expect(secondBriefcaseChangeset2.totalFullTableScans).to.be.eql(0); - expect(secondBriefcaseChangeset2.perStatementStats.length).to.be.eql(11); - - // Cleanup - secondTxn.end(); - firstTxn.end(); - secondBriefcase.close(); - firstBriefcase.close(); - }); - it("openInMemory() & step()", async () => { - const adminToken = "super manager token"; - const iModelName = "test"; - const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(rwIModelId); - const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); - const txn = startTestTxn(rwIModel, "openInMemory step"); - // 1. Import schema with class that span overflow table. + // Import schema with class that spans the overflow table (36 properties) const schema = ` bis:GraphicalElement2d - + ${Array(nProps).fill(undefined).map((_, i) => ``).join("\n")} `; await importSchemaStrings(txn, [schema]); rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - // Create drawing model and category await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); const codeProps = Code.createEmpty(); codeProps.value = "DrawingModel"; const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); - let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); + let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "OverflowNullCat"); if (undefined === drawingCategoryId) - drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); - - txn.saveChanges(); - await rwIModel.pushChanges({ description: "setup category", accessToken: adminToken }); + drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "OverflowNullCat", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); - const geomArray: Arc3d[] = [ + const geom: GeometryStreamProps = [ Arc3d.createXY(Point3d.create(0, 0), 5), Arc3d.createXY(Point3d.create(5, 5), 2), Arc3d.createXY(Point3d.create(-5, -5), 20), - ]; + ].map((a) => IModelJson.Writer.toIModelJson(a)); - const geometryStream: GeometryStreamProps = []; - for (const geom of geomArray) { - const arcData = IModelJson.Writer.toIModelJson(geom); - geometryStream.push(arcData); - } + const props = Array(nProps).fill(undefined).map((_, i) => ({ [`p${i}`]: `test_${i}` })).reduce((acc, curr) => ({ ...acc, ...curr }), {}); - const e1 = { - classFullName: `TestDomain:Test2dElement`, + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Push 1: insert element + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + id = txn.insertElement({ + classFullName: "TestDomain:Test2dElement", model: drawingModelId, category: drawingCategoryId, code: Code.createEmpty(), - geom: geometryStream, - ...{ p1: "test1" }, - }; + geom, + ...props, + } as any); + assert.isTrue(Id64.isValidId64(id)); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "insert element", accessToken: adminToken }); - // 2. Insert a element for the class - await rwIModel.locks.acquireLocks({ shared: drawingModelId }); - const e1id = txn.insertElement(e1); - assert.isTrue(Id64.isValidId64(e1id), "insert worked"); - const testElClassId: Id64String = getClassIdByName(rwIModel, "Test2dElement"); - - if (true) { - const reader = SqliteChangesetReader.openInMemory({ db: rwIModel, disableSchemaCheck: true }); - const adaptor = new ChangesetECAdaptor(reader); - const unifier = new PartialECChangeUnifier(rwIModel) - while (adaptor.step()) { - unifier.appendFrom(adaptor); - } - reader.close(); - - // verify the inserted element's properties - const instances = Array.from(unifier.instances); - expect(instances.length).to.equals(1); - const testEl = instances[0]; - expect(testEl.$meta?.op).to.equals("Inserted"); - expect(testEl.$meta?.classFullName).to.equals("TestDomain:Test2dElement"); - expect(testEl.$meta?.stage).to.equals("New"); - expect(testEl.ECClassId).to.equals(testElClassId); - expect(testEl.ECInstanceId).to.equals(e1id); - expect(testEl.Model.Id).to.equals(drawingModelId); - expect(testEl.Category.Id).to.equals(drawingCategoryId); - expect(testEl.Origin.X).to.equals(0); - expect(testEl.Origin.Y).to.equals(0); - expect(testEl.Rotation).to.equals(0); - expect(testEl.BBoxLow.X).to.equals(-25); - expect(testEl.BBoxLow.Y).to.equals(-25); - expect(testEl.BBoxHigh.X).to.equals(15); - expect(testEl.BBoxHigh.Y).to.equals(15); - expect(testEl.p1).to.equals("test1"); - } + // Push 2: update element — this is the changeset we will read against the broken iModel + const updatedProps = Object.assign( + rwIModel.elements.getElementProps(id), + Array(nProps).fill(undefined).map((_, i) => ({ [`p${i}`]: `updated_${i}` })).reduce((acc, curr) => ({ ...acc, ...curr }), {}), + ); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the update txn + await new Promise((resolve) => setTimeout(resolve, 300)); + await rwIModel.locks.acquireLocks({ exclusive: id }); + txn.updateElement(updatedProps); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "update element", accessToken: adminToken }); - // save changes and verify the the txn + // Wait so that LastMod on bis_Model gets a distinct timestamp before the delete txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Push 3: delete element (so the iModel is out of sync when we re-read push 2) + await rwIModel.locks.acquireLocks({ exclusive: id }); + txn.deleteElement(id); txn.saveChanges(); + await rwIModel.pushChanges({ description: "delete element", accessToken: adminToken }); - if (true) { - const txnId = rwIModel.txns.getLastSavedTxnProps()?.id as string; - expect(txnId).to.not.be.undefined; - const reader = SqliteChangesetReader.openTxn({ db: rwIModel, disableSchemaCheck: true, txnId }); - const adaptor = new ChangesetECAdaptor(reader); - const unifier = new PartialECChangeUnifier(rwIModel) - while (adaptor.step()) { - unifier.appendFrom(adaptor); - } - reader.close(); - - // verify the inserted element's properties - const instances = Array.from(unifier.instances); - expect(instances.length).to.equals(3); - const drawingModelClassId: Id64String = getClassIdByName(rwIModel, "DrawingModel"); - - // DrawingModel new instance - const drawingModelElNew = instances[0]; - expect(drawingModelElNew.$meta?.op).to.equals("Updated"); - expect(drawingModelElNew.$meta?.classFullName).to.equals("BisCore:DrawingModel"); - expect(drawingModelElNew.$meta?.stage).to.equals("New"); - expect(drawingModelElNew.ECClassId).to.equals(drawingModelClassId); - expect(drawingModelElNew.ECInstanceId).to.equals(drawingModelId); - expect(drawingModelElNew.LastMod).to.exist; - expect(drawingModelElNew.GeometryGuid).to.exist; - - // DrawingModel old instance - const drawingModelElOld = instances[1]; - expect(drawingModelElOld.$meta?.op).to.equals("Updated"); - expect(drawingModelElOld.$meta?.classFullName).to.equals("BisCore:DrawingModel"); - expect(drawingModelElOld.$meta?.stage).to.equals("Old"); - expect(drawingModelElOld.ECClassId).to.equals(drawingModelClassId); - expect(drawingModelElOld.ECInstanceId).to.equals(drawingModelId); - expect(drawingModelElOld.LastMod).to.null; - expect(drawingModelElOld.GeometryGuid).to.null; - - // Test element instance - const testEl = instances[2]; - expect(testEl.$meta?.op).to.equals("Inserted"); - expect(testEl.$meta?.classFullName).to.equals("TestDomain:Test2dElement"); - expect(testEl.$meta?.stage).to.equals("New"); - expect(testEl.ECClassId).to.equals(testElClassId); - expect(testEl.ECInstanceId).to.equals(e1id); - expect(testEl.Model.Id).to.equals(drawingModelId); - expect(testEl.Category.Id).to.equals(drawingCategoryId); - expect(testEl.Origin.X).to.equals(0); - expect(testEl.Origin.Y).to.equals(0); - expect(testEl.Rotation).to.equals(0); - expect(testEl.BBoxLow.X).to.equals(-25); - expect(testEl.BBoxLow.Y).to.equals(-25); - expect(testEl.BBoxHigh.X).to.equals(15); - expect(testEl.BBoxHigh.Y).to.equals(15); - expect(testEl.p1).to.equals("test1"); - } - await rwIModel.pushChanges({ description: "insert element", accessToken: adminToken }); + const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); + const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); + // changesets: [insert, update, delete] — index 1 = update changeset + updateChangesetPathname = changesets[1].pathname; + }); + + after(() => { txn.end(); - rwIModel.close(); + rwIModel?.close(); + HubMock.shutdown(); + }); + + it("openFile() recovers gracefully when ExclusiveRootClassId is NULL for overflow table", () => { + /** + * Simulate the broken state: clear ExclusiveRootClassId so the reader cannot + * determine which derived class owns the overflow table rows. + * The reader should fall back gracefully rather than crashing, and still + * produce inserted/deleted instances for the overflow table rows. + */ + expect( + rwIModel[_nativeDb].executeSql("UPDATE ec_Table SET ExclusiveRootClassId=NULL WHERE Name='bis_GeometricElement2d_Overflow'"), + ).to.equal(DbResult.BE_SQLITE_OK); + + let assertedOnOverflowTable = false; + + using reader = ChangesetReader.openFile({ db: rwIModel, fileName: updateChangesetPathname }); + while (reader.step()) { + if (reader.tableName !== "bis_GeometricElement2d_Overflow") + continue; + + assert.equal(reader.op, "Updated"); + + // inserted (New stage) + expect(reader.inserted).to.exist; + assert.deepEqual(reader.inserted!.$meta.tables, ["bis_GeometricElement2d_Overflow"]); + assert.equal(reader.inserted!.$meta.op, "Updated"); + assert.equal(reader.inserted!.$meta.stage, "New"); + assert.equal(reader.inserted!.ECInstanceId, id); + assert.equal(rwIModel.getClassNameFromId(reader.inserted!.ECClassId), "BisCore:GeometricElement2d"); + + // deleted (Old stage) + expect(reader.deleted).to.exist; + assert.deepEqual(reader.deleted!.$meta.tables, ["bis_GeometricElement2d_Overflow"]); + assert.equal(reader.deleted!.$meta.op, "Updated"); + assert.equal(reader.deleted!.$meta.stage, "Old"); + assert.equal(reader.deleted!.ECInstanceId, id); + assert.equal(rwIModel.getClassNameFromId(reader.deleted!.ECClassId), "BisCore:GeometricElement2d"); + + assertedOnOverflowTable = true; + } + + assert.isTrue(assertedOnOverflowTable, "Expected at least one row from the overflow table"); }); - it("Instance update to a different class (bug)", async () => { +}); + +describe("ChangesetReader: instance reused with a different class (class change in Updated row)", () => { + it("openFile() correctly identifies ECClassId change from T1 to T2 in a buggy changeset", async () => { /** - * Test scenario: Verifies changeset reader behavior when an instance ID is reused with a different class. + * Same scenario as ChangesetReader.test.ts: "Instance update to a different class (bug)". + * Verifies ChangesetReader behaviour when an instance ID is reused with a different class. * * Steps: * 1. Import schema with two classes (T1 and T2) that inherit from GraphicalElement2d. * - T1 has property 'p' of type string * - T2 has property 'p' of type long - * 2. Insert an element of type T1 with id=elId and property p="wwww" - * 3. Push changeset #1: "insert element" - * 4. Delete the T1 element - * 5. Manipulate the element ID sequence to force reuse of the same ID - * 6. Insert a new element of type T2 with the same id=elId but property p=1111 - * 7. Push changeset #2: "buggy changeset" + * 2. Insert a T1 element, push changeset #1. + * 3. Delete the T1 element, reset the ID sequence to force ID reuse, insert T2 with same ID, push changeset #2. * - * Verification: - * - Changeset #2 should show an "Updated" operation (not Delete+Insert) - * - In bis_Element table: ECClassId changes from T1 to T2 - * - In bis_GeometricElement2d table: ECClassId changes from T1 to T2 - * - Property 'p' changes from string "wwww" to integer 1111 - * - * This tests the changeset reader's ability to handle instance class changes, - * which can occur in edge cases where IDs are reused with different types. + * ChangesetReader should: + * - Report op=Updated for bis_Element and bis_GeometricElement2d rows. + * - inserted.ECClassId resolves to T2, deleted.ECClassId resolves to T1. + * - Property p: inserted carries the T2 integer value (1111), deleted carries the T1 string value ("wwww"). */ + HubMock.startup("ChangesetReaderClassChange", KnownTestLocations.outputDir); const adminToken = "super manager token"; - const iModelName = "test"; - const modelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + const iTwinId = HubMock.iTwinId; + + const modelId = await HubMock.createNewIModel({ iTwinId, iModelName: "classChange", description: "ClassChange", accessToken: adminToken }); assert.isNotEmpty(modelId); + let b1 = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: modelId, accessToken: adminToken }); - let txn = startTestTxn(b1, "instance update to different class"); - // 1. Import schema with classes that span overflow table. + let txn = startTestTxn(b1, "class change test"); + const schema = ` bis:GraphicalElement2d - + bis:GraphicalElement2d - + `; await importSchemaStrings(txn, [schema]); b1.channels.addAllowedChannel(ChannelControl.sharedChannelName); - // Create drawing model and category await b1.locks.acquireLocks({ shared: IModel.dictionaryId }); const codeProps = Code.createEmpty(); codeProps.value = "DrawingModel"; @@ -1507,40 +3942,38 @@ describe("Changeset Reader API", async () => { if (undefined === drawingCategoryId) drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); - - const geomArray: Arc3d[] = [ + const geom: GeometryStreamProps = [ Arc3d.createXY(Point3d.create(0, 0), 5), Arc3d.createXY(Point3d.create(5, 5), 2), Arc3d.createXY(Point3d.create(-5, -5), 20), - ]; - - const geometryStream: GeometryStreamProps = []; - for (const geom of geomArray) { - const arcData = IModelJson.Writer.toIModelJson(geom); - geometryStream.push(arcData); - } - - const geomElementT1 = { - classFullName: `TestDomain:T1`, + ].map((a) => IModelJson.Writer.toIModelJson(a)); + + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Push 1: insert T1 element + await b1.locks.acquireLocks({ shared: drawingModelId }); + const elId = txn.insertElement({ + classFullName: "TestDomain:T1", model: drawingModelId, category: drawingCategoryId, code: Code.createEmpty(), - geom: geometryStream, + geom, p: "wwww", - }; - - const elId = txn.insertElement(geomElementT1); - assert.isTrue(Id64.isValidId64(elId), "insert worked"); + } as any); + assert.isTrue(Id64.isValidId64(elId)); txn.saveChanges(); await b1.pushChanges({ description: "insert element" }); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the delete txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Push 2: delete T1 element, reset ID sequence to force same ID, insert T2 await b1.locks.acquireLocks({ shared: drawingModelId, exclusive: elId }); await b1.locks.acquireLocks({ shared: IModel.dictionaryId }); txn.deleteElement(elId); txn.saveChanges(); - // Force id set to reproduce same instance with different classid - const bid = BigInt(elId) - 1n + // Force ID sequence so the next insert reuses elId + const bid = BigInt(elId) - 1n; b1[_nativeDb].saveLocalValue("bis_elementidsequence", bid.toString()); txn.saveChanges(); const fileName = b1[_nativeDb].getFilePath(); @@ -1549,369 +3982,479 @@ describe("Changeset Reader API", async () => { b1 = await BriefcaseDb.open({ fileName }); b1.channels.addAllowedChannel(ChannelControl.sharedChannelName); - txn = startTestTxn(b1, "instance update to different class reopened briefcase"); - + txn = startTestTxn(b1, "class change test reopened"); - const geomElementT2 = { - classFullName: `TestDomain:T2`, + // Wait so that LastMod on bis_Model gets a distinct timestamp before the INSERT txn + await new Promise((resolve) => setTimeout(resolve, 300)); + const elId2 = txn.insertElement({ + classFullName: "TestDomain:T2", model: drawingModelId, category: drawingCategoryId, code: Code.createEmpty(), - geom: geometryStream, + geom, p: 1111, - }; - - const elId2 = txn.insertElement(geomElementT2); - chai.expect(elId).equals(elId2); - + } as any); + expect(elId2).to.equal(elId); txn.saveChanges(); await b1.pushChanges({ description: "buggy changeset" }); - const getChanges = async () => { - return HubMock.downloadChangesets({ iModelId: modelId, targetDir: path.join(KnownTestLocations.outputDir, modelId, "changesets") }); - }; - + const targetDir = path.join(KnownTestLocations.outputDir, modelId, "changesets"); + const changesets = await HubMock.downloadChangesets({ iModelId: modelId, targetDir }); + expect(changesets.length).to.equal(2); + expect(changesets[0].description).to.equal("insert element"); + expect(changesets[1].description).to.equal("buggy changeset"); - const changesets = await getChanges(); - chai.expect(changesets.length).equals(2); - chai.expect(changesets[0].description).equals("insert element"); - chai.expect(changesets[1].description).equals("buggy changeset"); - - const getClassId = async (name: string) => { - const r = b1.createQueryReader("SELECT FORMAT('0x%x', ec_classid(?))", QueryBinder.from([name])); - if (await r.step()) { - return r.current[0]; - } - } - - const t1ClassId = await getClassId("TestDomain:T1"); - const t2ClassId = await getClassId("TestDomain:T2"); - - const reader = SqliteChangesetReader.openFile({ fileName: changesets[1].pathname, disableSchemaCheck: true, db: b1 }); + // ChangesetReader reads ECClassId from the changeset data for each stage, + // so it correctly sees T2 for the new (inserted) side and T1 for the old (deleted) side. let bisElementAsserted = false; - let bisGeometricElement2dAsserted = false; + let bisGeomElement2dAsserted = false; + + using reader = ChangesetReader.openFile({ db: b1, fileName: changesets[1].pathname }); while (reader.step()) { if (reader.tableName === "bis_Element" && reader.op === "Updated") { bisElementAsserted = true; - chai.expect(reader.getColumnNames(reader.tableName)).deep.equals([ - "Id", - "ECClassId", - "ModelId", - "LastMod", - "CodeSpecId", - "CodeScopeId", - "CodeValue", - "UserLabel", - "ParentId", - "ParentRelECClassId", - "FederationGuid", - "JsonProperties", - ]); - - const oldId = reader.getChangeValueId(0, "Old"); - const newId = reader.getChangeValueId(0, "New"); - chai.expect(oldId).equals(elId); - chai.expect(newId).to.be.undefined; - - const oldClassId = reader.getChangeValueId(1, "Old"); - const newClassId = reader.getChangeValueId(1, "New"); - chai.expect(oldClassId).equals(t1ClassId); - chai.expect(newClassId).equals(t2ClassId); - chai.expect(oldClassId).is.not.equal(newClassId); + expect(reader.inserted).to.exist; + expect(reader.deleted).to.exist; + // ECInstanceId: PK was not changed in the update — only available on the Old side + assert.equal(reader.deleted!.ECInstanceId, elId); + // ECClassId correctly reflects the class change + assert.equal(b1.getClassNameFromId(reader.inserted!.ECClassId), "TestDomain:T2"); + assert.equal(b1.getClassNameFromId(reader.deleted!.ECClassId), "TestDomain:T1"); } if (reader.tableName === "bis_GeometricElement2d" && reader.op === "Updated") { - bisGeometricElement2dAsserted = true; - chai.expect(reader.getColumnNames(reader.tableName)).deep.equals([ - "ElementId", - "ECClassId", - "CategoryId", - "Origin_X", - "Origin_Y", - "Rotation", - "BBoxLow_X", - "BBoxLow_Y", - "BBoxHigh_X", - "BBoxHigh_Y", - "GeometryStream", - "TypeDefinitionId", - "TypeDefinitionRelECClassId", - "js1", - "js2", - ]); - - // ECInstanceId - const oldId = reader.getChangeValueId(0, "Old"); - const newId = reader.getChangeValueId(0, "New"); - chai.expect(oldId).equals(elId); - chai.expect(newId).to.be.undefined; - - // ECClassId (changed) - const oldClassId = reader.getChangeValueId(1, "Old"); - const newClassId = reader.getChangeValueId(1, "New"); - chai.expect(oldClassId).equals(t1ClassId); - chai.expect(newClassId).equals(t2ClassId); - chai.expect(oldClassId).is.not.equal(newClassId); - - // Property 'p' changed type and value. - const oldP = reader.getChangeValueText(13, "Old"); - const newP = reader.getChangeValueInteger(13, "New"); - chai.expect(oldP).equals("wwww"); - chai.expect(newP).equals(1111); + bisGeomElement2dAsserted = true; + expect(reader.inserted).to.exist; + expect(reader.deleted).to.exist; + // ECClassId correctly reflects the class change + assert.equal(b1.getClassNameFromId(reader.inserted!.ECClassId), "TestDomain:T2"); + assert.equal(b1.getClassNameFromId(reader.deleted!.ECClassId), "TestDomain:T1"); + // Property 'p': T2 declares it as long (integer), T1 as string + assert.equal(reader.inserted!.p, 1111); + assert.equal(reader.deleted!.p, "wwww"); } } - chai.expect(bisElementAsserted).to.be.true; - chai.expect(bisGeometricElement2dAsserted).to.be.true; - reader.close(); + assert.isTrue(bisElementAsserted, "Expected an Updated row in bis_Element"); + assert.isTrue(bisGeomElement2dAsserted, "Expected an Updated row in bis_GeometricElement2d"); + txn.end(); + b1.close(); + HubMock.shutdown(); + }); +}); - // ChangesetECAdaptor works incorrectly as it does not expect ECClassId to change in an update. - const adaptor = new ChangesetECAdaptor( - SqliteChangesetReader.openFile({ fileName: changesets[1].pathname, disableSchemaCheck: true, db: b1 }) - ); +describe("ChangesetReader: overflow table insert and update and delete", () => { + let rwIModel: BriefcaseDb; + let elementId: Id64String; + let drawingModelId: Id64String; + let drawingCategoryId: Id64String; + let insertTxnId: string; + let updateTxnId: string; + let txn: EditTxn; + + const nProps = 36; + const propName = (i: number) => `p${i}`; + const insertVal = (i: number) => `insert-${i}`; + const updateVal = (i: number) => `updated-${i}`; + + before(async () => { + HubMock.startup("ECChangesetOverflowInsertUpdate", KnownTestLocations.outputDir); + const adminToken = "super manager token"; + const iTwinId = HubMock.iTwinId; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "overflowInsertUpdate", description: "overflowInsertUpdate", accessToken: adminToken }); + rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + txn = startTestTxn(rwIModel, "ChangesetReader overflow insert/update"); - adaptor.acceptClass(GraphicalElement2d.classFullName) - adaptor.acceptOp("Updated"); + // 36 string properties — enough to spill columns into bis_GeometricElement2d_Overflow + const schema = ` + + + + bis:GraphicalElement2d + ${Array.from({ length: nProps }, (_, i) => ``).join("\n ")} + + `; + await importSchemaStrings(txn, [schema]); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - let ecChangeForElementAsserted = false; - let ecChangeForGeometricElement2dAsserted = false; - while (adaptor.step()) { - if (adaptor.reader.tableName === "bis_Element") { - ecChangeForElementAsserted = true; - chai.expect(adaptor.inserted?.$meta?.classFullName).equals("TestDomain:T1"); // WRONG should be TestDomain:T2 - chai.expect(adaptor.deleted?.$meta?.classFullName).equals("TestDomain:T1"); // WRONG should be TestDomain:T2 - } - if (adaptor.reader.tableName === "bis_GeometricElement2d") { - ecChangeForGeometricElement2dAsserted = true; - chai.expect(adaptor.inserted?.$meta?.classFullName).equals("TestDomain:T1"); // WRONG should be TestDomain:T2 - chai.expect(adaptor.deleted?.$meta?.classFullName).equals("TestDomain:T1"); // WRONG should be TestDomain:T2 - chai.expect(adaptor.inserted?.p).equals("0x457"); // CORRECT p in T2 is integer - chai.expect(adaptor.deleted?.p).equals("wwww"); // CORRECT p in T1 is string - } + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "OverflowDrawing"; + [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); + const foundCat = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "OverflowCat"); + drawingCategoryId = foundCat ?? DrawingCategory.insert(txn, IModel.dictionaryId, "OverflowCat", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(0,128,0)").toJSON() })); + txn.saveChanges("setup"); + + // Txn 2: insert element with every property populated + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + const geom: GeometryStreamProps = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + ].map((a) => IModelJson.Writer.toIModelJson(a)); + const insertProps = Object.fromEntries(Array.from({ length: nProps }, (_, i) => [propName(i), insertVal(i)])); + // Wait so that LastMod on bis_Model gets a distinct timestamp before the insert txn + await new Promise((resolve) => setTimeout(resolve, 300)); + + elementId = txn.insertElement({ + classFullName: "TestDomain:OverflowElement", + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + geom, + ...insertProps, + } as any); + txn.saveChanges("insert overflow element"); + insertTxnId = rwIModel.txns.getLastSavedTxnProps()!.id; + + // Wait so that LastMod on bis_Model gets a distinct timestamp before the UPDATE txn + await new Promise((resolve) => setTimeout(resolve, 300)); + // Txn 3: update only the overflow-table properties (p34, p35 — the last two) + await rwIModel.locks.acquireLocks({ exclusive: elementId }); + txn.updateElement({ + ...rwIModel.elements.getElementProps(elementId), + [propName(34)]: updateVal(34), + [propName(35)]: updateVal(35), + }); + txn.saveChanges("update overflow props"); + updateTxnId = rwIModel.txns.getLastSavedTxnProps()!.id; + }); + + after(() => { + txn.end(); + rwIModel?.close(); + HubMock.shutdown(); + }); + + it("txn | bis_GeometricElement2d_Overflow rows merged; overflow props correct in New and Old", async () => { + // --- update txn --- + const updateInstances = readTxn(rwIModel, updateTxnId, undefined, { classIdsToClassNames: true }); + assert.equal(updateInstances.length, 4); + + // DrawingModel Updated New (indirect side-effect) + const updateModelNew = updateInstances.find((i) => i.ECClassId === "BisCore.DrawingModel" && i.$meta.stage === "New"); + expect(updateModelNew).to.exist; + assert.equal(updateModelNew!.$meta.op, "Updated"); + assert.isString(updateModelNew!.LastMod); + assert.deepEqual(Object.keys(updateModelNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "$meta"].sort()); + assert.deepEqual([...updateModelNew!.$meta.tables], ["bis_Model"]); + assert.deepEqual([...updateModelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod"].sort()); + assert.deepEqual(updateModelNew!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(updateModelNew!.$meta.isIndirectChange, true); + + // DrawingModel Updated Old + const updateModelOld = updateInstances.find((i) => i.ECClassId === "BisCore.DrawingModel" && i.$meta.stage === "Old"); + expect(updateModelOld).to.exist; + assert.equal(updateModelOld!.$meta.op, "Updated"); + assert.isString(updateModelOld!.LastMod); + assert.deepEqual(Object.keys(updateModelOld!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "$meta"].sort()); + assert.deepEqual([...updateModelOld!.$meta.tables], ["bis_Model"]); + assert.deepEqual([...updateModelOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod"].sort()); + assert.equal(updateModelOld!.$meta.isIndirectChange, true); + + // OverflowElement Updated New + const elemNew = updateInstances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + expect(elemNew).to.exist; + assert.equal(elemNew!.ECClassId, "TestDomain.OverflowElement"); + assert.equal(elemNew!.$meta.op, "Updated"); + assert.equal(elemNew!.$meta.isIndirectChange, false); + assert.deepEqual(elemNew!.$meta.rowOptions, { classIdsToClassNames: true }); + // Only bis_Element (LastMod) and bis_GeometricElement2d_Overflow (p34, p35) were touched + assert.deepEqual([...elemNew!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d_Overflow"].sort()); + assert.deepEqual([...elemNew!.$meta.changeIndexes].sort(), [1, 2].sort()); + // changeFetchedPropNames: only the props that changed + assert.deepEqual([...elemNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "p34", "p35"].sort()); + // New stage has updated values for the two overflow props + assert.equal(elemNew![propName(34)], updateVal(34)); + assert.equal(elemNew![propName(35)], updateVal(35)); + // Other props (p0–p33) are not present — not part of the delta + assert.isUndefined(elemNew![propName(0)]); + assert.deepEqual(Object.keys(elemNew!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "p34", "p35", "$meta"].sort()); + + // OverflowElement Updated Old + const elemOld = updateInstances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "Old"); + expect(elemOld).to.exist; + assert.equal(elemOld!.ECClassId, "TestDomain.OverflowElement"); + assert.equal(elemOld!.$meta.op, "Updated"); + assert.equal(elemOld!.$meta.isIndirectChange, false); + assert.deepEqual([...elemOld!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d_Overflow"].sort()); + assert.deepEqual([...elemOld!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "p34", "p35"].sort()); + // Old stage has the pre-update (insert) values for the two overflow props + assert.equal(elemOld![propName(34)], insertVal(34)); + assert.equal(elemOld![propName(35)], insertVal(35)); + assert.isUndefined(elemOld![propName(0)]); + assert.deepEqual(Object.keys(elemOld!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "p34", "p35", "$meta"].sort()); + + // --- insert txn --- + const insertInstances = readTxn(rwIModel, insertTxnId, undefined, { classIdsToClassNames: true }); + assert.equal(insertInstances.length, 3); + + // DrawingModel Updated New (indirect side-effect of inserting into the model) + const modelNew = insertInstances.find((i) => i.ECClassId === "BisCore.DrawingModel" && i.$meta.stage === "New"); + expect(modelNew).to.exist; + assert.equal(modelNew!.$meta.op, "Updated"); + assert.isString(modelNew!.LastMod); + assert.isString(modelNew!.GeometryGuid); + assert.deepEqual([...modelNew!.$meta.tables], ["bis_Model"]); + assert.deepEqual([...modelNew!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(modelNew!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(modelNew!.$meta.isIndirectChange, true); + + // DrawingModel Updated Old + const modelOld = insertInstances.find((i) => i.ECClassId === "BisCore.DrawingModel" && i.$meta.stage === "Old"); + expect(modelOld).to.exist; + assert.equal(modelOld!.$meta.op, "Updated"); + assert.deepEqual(Object.keys(modelOld!).sort(), ["ECInstanceId", "ECClassId", "$meta"].sort()); + assert.deepEqual([...modelOld!.$meta.tables], ["bis_Model"]); + assert.equal(modelOld!.$meta.isIndirectChange, true); + + // OverflowElement Inserted New — key assertions + const insertElem = insertInstances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + expect(insertElem).to.exist; + assert.equal(insertElem!.ECClassId, "TestDomain.OverflowElement"); + assert.equal(insertElem!.$meta.op, "Inserted"); + assert.equal(insertElem!.$meta.stage, "New"); + assert.equal(insertElem!.$meta.isIndirectChange, false); + assert.deepEqual(insertElem!.$meta.rowOptions, { classIdsToClassNames: true }); + + // All three physical tables (including overflow) contributed to this insert + assert.deepEqual([...insertElem!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d", "bis_GeometricElement2d_Overflow"].sort()); + assert.deepEqual([...insertElem!.$meta.changeIndexes].sort(), [1, 2, 3].sort()); + + // changeFetchedPropNames includes all 36 domain props + const fetchedProps = [...insertElem!.$meta.changeFetchedPropNames]; + for (let i = 0; i < nProps; i++) { + assert.include(fetchedProps, propName(i), `changeFetchedPropNames should contain p${i}`); } - chai.expect(ecChangeForElementAsserted).to.be.true; - chai.expect(ecChangeForGeometricElement2dAsserted).to.be.true; - adaptor.close(); - // PartialECChangeUnifier fail to combine changes correctly when ECClassId is updated. - const adaptor2 = new ChangesetECAdaptor( - SqliteChangesetReader.openFile({ fileName: changesets[1].pathname, disableSchemaCheck: true, db: b1 }) - ); - const unifier = new PartialECChangeUnifier(b1); - adaptor2.acceptClass(GraphicalElement2d.classFullName) - adaptor2.acceptOp("Updated"); - while (adaptor2.step()) { - unifier.appendFrom(adaptor2); + // All 36 domain props carry the insert values + for (let i = 0; i < nProps; i++) { + assert.equal(insertElem![propName(i)], insertVal(i), `p${i} should be '${insertVal(i)}'`); } - chai.expect(unifier.getInstanceCount()).to.be.equals(2); // WRONG should be 1 + // Spot-check BIS properties + assert.equal(insertElem!.Model.Id, drawingModelId); + assert.equal(insertElem!.Model.RelECClassId, "BisCore.ModelContainsElements"); + assert.equal(insertElem!.Category.Id, drawingCategoryId); + assert.equal(insertElem!.Category.RelECClassId, "BisCore.GeometricElement2dIsInCategory"); + assert.equal(insertElem!.CodeSpec.Id, "0x1"); + assert.equal(insertElem!.CodeSpec.RelECClassId, "BisCore.CodeSpecSpecifiesCode"); + assert.equal(insertElem!.CodeScope.Id, "0x1"); + assert.equal(insertElem!.CodeScope.RelECClassId, "BisCore.ElementScopesCode"); + assert.isString(insertElem!.FederationGuid); + assert.isString(insertElem!.LastMod); + assert.deepEqual(insertElem!.Origin, { X: 0, Y: 0 }); + assert.equal(insertElem!.Rotation, 0); + assert.deepEqual(insertElem!.BBoxLow, { X: -5, Y: -5 }); + assert.deepEqual(insertElem!.BBoxHigh, { X: 5, Y: 5 }); + assert.include(String(insertElem!.GeometryStream), "\"bytes\""); + + // --- delete txn --- + // Wait so that LastMod on bis_Model gets a distinct timestamp before the delete txn + await new Promise((resolve) => setTimeout(resolve, 300)); + await rwIModel.locks.acquireLocks({ exclusive: elementId }); + txn.deleteElement(elementId); + txn.saveChanges("delete overflow element"); + const deleteTxnId = rwIModel.txns.getLastSavedTxnProps()!.id; + + const deleteInstances = readTxn(rwIModel, deleteTxnId, undefined, { classIdsToClassNames: true }); + // DrawingModel Updated New + Old (indirect) + OverflowElement Deleted Old + assert.equal(deleteInstances.length, 3); + + // DrawingModel Updated New (indirect side-effect) + const updateModelNewForDelete = deleteInstances.find((i) => i.ECClassId === "BisCore.DrawingModel" && i.$meta.stage === "New"); + expect(updateModelNewForDelete).to.exist; + assert.equal(updateModelNewForDelete!.$meta.op, "Updated"); + assert.isString(updateModelNewForDelete!.LastMod); + assert.isString(updateModelNewForDelete!.GeometryGuid); + assert.deepEqual(Object.keys(updateModelNewForDelete!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "$meta", "GeometryGuid"].sort()); + assert.deepEqual([...updateModelNewForDelete!.$meta.tables], ["bis_Model"]); + assert.deepEqual([...updateModelNewForDelete!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.deepEqual(updateModelNewForDelete!.$meta.rowOptions, { classIdsToClassNames: true }); + assert.equal(updateModelNewForDelete!.$meta.isIndirectChange, true); + + // DrawingModel Updated Old + const updateModelOldForDelete = deleteInstances.find((i) => i.ECClassId === "BisCore.DrawingModel" && i.$meta.stage === "Old"); + expect(updateModelOldForDelete).to.exist; + assert.equal(updateModelOldForDelete!.$meta.op, "Updated"); + assert.isString(updateModelOldForDelete!.LastMod); + assert.isString(updateModelNewForDelete!.GeometryGuid); + assert.deepEqual(Object.keys(updateModelOldForDelete!).sort(), ["ECInstanceId", "ECClassId", "LastMod", "$meta", "GeometryGuid"].sort()); + assert.deepEqual([...updateModelOldForDelete!.$meta.tables], ["bis_Model"]); + assert.deepEqual([...updateModelOldForDelete!.$meta.changeFetchedPropNames].sort(), ["ECInstanceId", "LastMod", "GeometryGuid"].sort()); + assert.equal(updateModelOldForDelete!.$meta.isIndirectChange, true); + + // OverflowElement Deleted Old + const deletedElemOld = deleteInstances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "Old"); + expect(deletedElemOld).to.exist; + assert.equal(deletedElemOld!.ECClassId, "TestDomain.OverflowElement"); + assert.equal(deletedElemOld!.$meta.op, "Deleted"); + assert.equal(deletedElemOld!.$meta.stage, "Old"); + assert.equal(deletedElemOld!.$meta.isIndirectChange, false); + assert.deepEqual(deletedElemOld!.$meta.rowOptions, { classIdsToClassNames: true }); + + // All three physical tables contributed to the delete row + assert.deepEqual([...deletedElemOld!.$meta.tables].sort(), ["bis_Element", "bis_GeometricElement2d", "bis_GeometricElement2d_Overflow"].sort()); + + // All 36 domain props are recorded in changeFetchedPropNames + const deleteFetchedProps = [...deletedElemOld!.$meta.changeFetchedPropNames]; + for (let i = 0; i < nProps; i++) { + assert.include(deleteFetchedProps, propName(i), `changeFetchedPropNames should contain p${i}`); + } - txn.end(); - b1.close(); + // p0–p33 still carry the original insert values; p34/p35 carry the updated values + for (let i = 0; i < 34; i++) { + assert.equal(deletedElemOld![propName(i)], insertVal(i), `p${i} should be '${insertVal(i)}'`); + } + assert.equal(deletedElemOld![propName(34)], updateVal(34)); + assert.equal(deletedElemOld![propName(35)], updateVal(35)); + + // Spot-check BIS properties + assert.equal(deletedElemOld!.Model.Id, drawingModelId); + assert.equal(deletedElemOld!.Model.RelECClassId, "BisCore.ModelContainsElements"); + assert.equal(deletedElemOld!.Category.Id, drawingCategoryId); + assert.equal(deletedElemOld!.Category.RelECClassId, "BisCore.GeometricElement2dIsInCategory"); + assert.equal(deletedElemOld!.CodeSpec.Id, "0x1"); + assert.equal(deletedElemOld!.CodeSpec.RelECClassId, "BisCore.CodeSpecSpecifiesCode"); + assert.equal(deletedElemOld!.CodeScope.Id, "0x1"); + assert.equal(deletedElemOld!.CodeScope.RelECClassId, "BisCore.ElementScopesCode"); + assert.isString(deletedElemOld!.FederationGuid); + assert.isString(deletedElemOld!.LastMod); + assert.deepEqual(deletedElemOld!.Origin, { X: 0, Y: 0 }); + assert.equal(deletedElemOld!.Rotation, 0); + assert.deepEqual(deletedElemOld!.BBoxLow, { X: -5, Y: -5 }); + assert.deepEqual(deletedElemOld!.BBoxHigh, { X: 5, Y: 5 }); + assert.include(String(deletedElemOld!.GeometryStream), "\"bytes\""); }); - }); -describe("PRAGMA ECSQL Functions", async () => { - let iTwinId: GuidString; +describe("ChangesetReader: invalid inputs", () => { let iModel: BriefcaseDb; + let txn: EditTxn; + let validChangesetPath: string; + + before(async () => { + HubMock.startup("ChangesetReaderInvalidInputs", KnownTestLocations.outputDir); + const adminToken = "super manager token"; + const iTwinId = HubMock.iTwinId; + const iModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "invalidInputs", accessToken: adminToken }); + iModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId, accessToken: adminToken }); + txn = startTestTxn(iModel, "ChangesetReader invalid inputs setup"); + iModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // Push one changeset so we have a valid .changeset file to reference. + await iModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, Code.createEmpty(), true); + txn.saveChanges("setup"); + await iModel.pushChanges({ description: "setup", accessToken: adminToken }); - before(() => { - HubMock.startup("ChangesetReaderTest", KnownTestLocations.outputDir); - iTwinId = HubMock.iTwinId; + const targetDir = path.join(KnownTestLocations.outputDir, iModelId, "changesets"); + const changesets = await HubMock.downloadChangesets({ iModelId, targetDir }); + assert.isAtLeast(changesets.length, 1); + validChangesetPath = changesets[0].pathname; }); - after(() => HubMock.shutdown()); + after(() => { + txn.end(); + iModel?.close(); + HubMock.shutdown(); + }); - beforeEach(async () => { - // Create new iModel - const adminToken = "super manager token"; - const iModelName = "PRAGMA_test"; - const iModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(iModelId); - iModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId, accessToken: adminToken }); + // --- openFile --- + + it("openFile: non-existent file path throws", () => { + expect(() => + ChangesetReader.openFile({ db: iModel, fileName: "/does/not/exist.changeset" }), + ).to.throw(); }); - afterEach(() => { - // Cleanup - iModel.close(); - }); - - it("should call PRAGMA integrity_check on a new iModel and return no errors", async () => { - // Call PRAGMA integrity_check - const query = "PRAGMA integrity_check ECSQLOPTIONS ENABLE_EXPERIMENTAL_FEATURES"; - const result = iModel.createQueryReader(query, undefined, undefined); - const results = await result.toArray(); - - // Verify no errors - assert(results.length > 0, "Results should be returned from PRAGMA integrity_check"); - assert(results[0][2] === true, "'check_data_columns' check should be true"); - assert(results[1][2] === true, "'check_ec_profile' check should be true") - assert(results[2][2] === true, "'check_nav_class_ids' check should be true") - assert(results[3][2] === true, "'check_nav_ids' check should be true") - assert(results[4][2] === true, "'check_linktable_fk_class_ids' check should be true") - assert(results[5][2] === true, "'check_linktable_fk_ids' check should be true") - assert(results[6][2] === true, "'check_class_ids' check should be true") - assert(results[7][2] === true, "'check_data_schema' check should be true") - assert(results[8][2] === true, "'check_schema_load' check should be true") - }); - - it("should call PRAGMA integrity_check individual checks on a new iModel and return no errors", async () => { - // Call check_ec_profile - let query = "pragma integrity_check(check_ec_profile) options enable_experimental_features"; - let result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); - let resultArray = await result.toArray(); - expect(resultArray.length).to.equal(0); // No errors expected - - // Call check_data_schema - query = "pragma integrity_check(check_data_schema) options enable_experimental_features"; - result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); - resultArray = await result.toArray(); - expect(resultArray.length).to.equal(0); // No errors expected - - // Call check_data_columns - query = "pragma integrity_check(check_data_columns) options enable_experimental_features"; - result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); - resultArray = await result.toArray(); - expect(resultArray.length).to.equal(0); // No errors expected - - // Call check_nav_class_ids - query = "pragma integrity_check(check_nav_class_ids) options enable_experimental_features"; - result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); - resultArray = await result.toArray(); - expect(resultArray.length).to.equal(0); // No errors expected - - // Call check_nav_ids - query = "pragma integrity_check(check_nav_ids) options enable_experimental_features"; - result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); - resultArray = await result.toArray(); - expect(resultArray.length).to.equal(0); // No errors expected - - // Call check_linktable_fk_class_ids - query = "pragma integrity_check(check_linktable_fk_class_ids) options enable_experimental_features"; - result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); - resultArray = await result.toArray(); - expect(resultArray.length).to.equal(0); // No errors expected - - // Call check_linktable_fk_ids - query = "pragma integrity_check(check_linktable_fk_ids) options enable_experimental_features"; - result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); - resultArray = await result.toArray(); - expect(resultArray.length).to.equal(0); // No errors expected - - // Call check_class_ids - query = "pragma integrity_check(check_class_ids) options enable_experimental_features"; - result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); - resultArray = await result.toArray(); - expect(resultArray.length).to.equal(0); // No errors expected - - // Call check_schema_load - query = "pragma integrity_check(check_schema_load) options enable_experimental_features"; - result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); - resultArray = await result.toArray(); - expect(resultArray.length).to.equal(0); // No errors expected - }); - - it("should call PRAGMA integrity_check on a corrupted iModel and return an error", async () => { - // Insert two elements - iModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - await iModel.locks.acquireLocks({ shared: IModel.repositoryModelId }); - const txn = startTestTxn(iModel, "PRAGMA integrity check corrupted iModel"); - - const element1Id = txn.insertElement({ - classFullName: Subject.classFullName, - model: IModel.repositoryModelId, - parent: new SubjectOwnsSubjects(IModel.rootSubjectId), - code: Subject.createCode(iModel, IModel.rootSubjectId, "Subject1"), - }); + it("openFile: empty file name throws", () => { + expect(() => + ChangesetReader.openFile({ db: iModel, fileName: "" }), + ).to.throw(); + }); - const element2Id = txn.insertElement({ - classFullName: Subject.classFullName, - model: IModel.repositoryModelId, - parent: new SubjectOwnsSubjects(IModel.rootSubjectId), - code: Subject.createCode(iModel, IModel.rootSubjectId, "Subject2"), - }); - txn.saveChanges(); + it("openFile: path points to a directory throws", () => { + expect(() => + ChangesetReader.openFile({ db: iModel, fileName: KnownTestLocations.outputDir }), + ).to.throw(); + }); - // Create a relationship between them - await iModel.locks.acquireLocks({ exclusive: Id64.toIdSet([element1Id, element2Id]) }); - const relationship = iModel.relationships.createInstance({ - classFullName: "BisCore:SubjectRefersToSubject", - sourceId: element1Id, - targetId: element2Id, - }); - const relationshipId = txn.insertRelationship(relationship.toJSON()); - assert.isTrue(Id64.isValidId64(relationshipId)); - txn.saveChanges(); + it("openFile: path points to a plain text file (not .changeset) throws", () => { + const txtFile = path.join(KnownTestLocations.outputDir, "not_a_changeset.txt"); + // Write a small non-changeset file and expect the reader to reject it. + fs.writeFileSync(txtFile, "this is not a changeset"); - // Delete one element without deleting the relationship to corrupt the iModel - const deleteResult = iModel[_nativeDb].executeSql(`DELETE FROM bis_Element WHERE Id=${element2Id}`); - expect(deleteResult).to.equal(DbResult.BE_SQLITE_OK); - txn.saveChanges(); + using reader = ChangesetReader.openFile({ db: iModel, fileName: txtFile }); + assert.equal(reader.step(), false, "Expected step() to return false for an invalid changeset file"); + assert.equal(reader.step(), false, "Expected step() to return false for an invalid changeset file"); + assert.equal(reader.step(), false, "Expected step() to return false for an invalid changeset file"); + assert.equal(reader.step(), false, "Expected step() to return false for an invalid changeset file"); + }); - // Call PRAGMA integrity_check - const query = "PRAGMA integrity_check ECSQLOPTIONS ENABLE_EXPERIMENTAL_FEATURES"; - const result = iModel.createQueryReader(query, undefined, undefined); - const results = await result.toArray(); - - // Verify error is reported - assert(results.length > 0, "Results should be returned from PRAGMA integrity_check"); - assert(results[0][2] === true, "'check_data_columns' check should be true"); - assert(results[1][2] === true, "'check_ec_profile' check should be true") - assert(results[2][2] === true, "'check_nav_class_ids' check should be true") - assert(results[3][2] === true, "'check_nav_ids' check should be true") - assert(results[4][2] === true, "'check_linktable_fk_class_ids' check should be true") - assert(results[5][2] === false, "'check_linktable_fk_ids' check should be false") // Expecting error report here - assert(results[6][2] === true, "'check_class_ids' check should be true") - assert(results[7][2] === true, "'check_data_schema' check should be true") - assert(results[8][2] === true, "'check_schema_load' check should be true") - txn.end(); + // --- openGroup --- + + it("openGroup: empty changesetFiles array throws synchronously", () => { + expect(() => + ChangesetReader.openGroup({ db: iModel, changesetFiles: [] }), + ).to.throw(); }); - it("should call PRAGMA integrity_check(check_linktable_fk_class_ids) on a corrupted iModel and return an error", async () => { - // Insert two elements - iModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - await iModel.locks.acquireLocks({ shared: IModel.repositoryModelId }); - const txn = startTestTxn(iModel, "PRAGMA integrity check corrupted linktable fk ids"); - - const element1Id = txn.insertElement({ - classFullName: Subject.classFullName, - model: IModel.repositoryModelId, - parent: new SubjectOwnsSubjects(IModel.rootSubjectId), - code: Subject.createCode(iModel, IModel.rootSubjectId, "Subject1"), - }); + it("openGroup: single non-existent path throws", () => { + expect(() => + ChangesetReader.openGroup({ db: iModel, changesetFiles: ["/does/not/exist.changeset"] }), + ).to.throw(); + }); - const element2Id = txn.insertElement({ - classFullName: Subject.classFullName, - model: IModel.repositoryModelId, - parent: new SubjectOwnsSubjects(IModel.rootSubjectId), - code: Subject.createCode(iModel, IModel.rootSubjectId, "Subject2"), - }); - txn.saveChanges(); + it("openGroup: first file valid, second file non-existent throws", () => { + expect(() => + ChangesetReader.openGroup({ db: iModel, changesetFiles: [validChangesetPath, "/does/not/exist.changeset"] }), + ).to.throw(); + }); - // Create a relationship between them - await iModel.locks.acquireLocks({ exclusive: Id64.toIdSet([element1Id, element2Id]) }); - const relationship = iModel.relationships.createInstance({ - classFullName: "BisCore:SubjectRefersToSubject", - sourceId: element1Id, - targetId: element2Id, - }); - const relationshipId = txn.insertRelationship(relationship.toJSON()); - assert.isTrue(Id64.isValidId64(relationshipId)); - txn.saveChanges(); + it("openGroup: file list containing an empty string throws", () => { + expect(() => + ChangesetReader.openGroup({ db: iModel, changesetFiles: [""] }), + ).to.throw(); + }); - // Delete one element without deleting the relationship to corrupt the iModel - const deleteResult = iModel[_nativeDb].executeSql(`DELETE FROM bis_Element WHERE Id=${element2Id}`); - expect(deleteResult).to.equal(DbResult.BE_SQLITE_OK); - txn.saveChanges(); + it("openGroup: duplicate paths reads the same changeset twice without throwing", () => { + // Passing the same changeset file twice is unusual but not necessarily an error at open time. + // The reader should open and step without throwing; the caller is responsible for any semantic issues. + expect(() => { + using reader = ChangesetReader.openGroup({ db: iModel, changesetFiles: [validChangesetPath, validChangesetPath] }); + while (reader.step()) { /* drain */ } + }).to.not.throw(); + }); - // Call PRAGMA integrity_check - const query = "pragma integrity_check(check_linktable_fk_ids) options enable_experimental_features"; - const result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); - const resultArray = await result.toArray(); - expect(resultArray.length).to.equal(1); // 1 error report expected - expect(resultArray[0].id).to.equal("0x20000000001"); - expect(resultArray[0].key_id).to.equal("0x20000000002"); - txn.end(); + // --- openTxn --- + + it("openTxn: non-existent txn id throws", () => { + expect(() => + ChangesetReader.openTxn({ db: iModel, txnId: "0xdeadbeef" }), + ).to.throw(); + }); + + it("openTxn: empty string txn id throws", () => { + expect(() => + ChangesetReader.openTxn({ db: iModel, txnId: "" }), + ).to.throw(); + }); + + // --- property accessors before step() --- + + it("accessing tableName before step() throws", () => { + using reader = ChangesetReader.openFile({ db: iModel, fileName: validChangesetPath }); + expect(() => reader.tableName).to.throw(); + }); + + it("accessing isECTable before step() throws", () => { + using reader = ChangesetReader.openFile({ db: iModel, fileName: validChangesetPath }); + expect(() => reader.isECTable).to.throw(); + }); + + it("accessing isIndirectChange before step() throws", () => { + using reader = ChangesetReader.openFile({ db: iModel, fileName: validChangesetPath }); + expect(() => reader.isIndirectChange).to.throw(); }); }); + diff --git a/core/backend/src/test/standalone/IntegrityCheck.test.ts b/core/backend/src/test/standalone/IntegrityCheck.test.ts index 74a6d836a434..35a533db4828 100644 --- a/core/backend/src/test/standalone/IntegrityCheck.test.ts +++ b/core/backend/src/test/standalone/IntegrityCheck.test.ts @@ -286,7 +286,7 @@ describe("iModelDb integrityCheck Tests", () => { let iModel: BriefcaseDb; before(() => { - HubMock.startup("ChangesetReaderTest", KnownTestLocations.outputDir); + HubMock.startup("IntegrityCheckTest", KnownTestLocations.outputDir); iTwinId = HubMock.iTwinId; }); @@ -437,7 +437,7 @@ describe("iModelDb integrityCheck Tests", () => { expect(deleteResult).to.equal(DbResult.BE_SQLITE_OK); }); - // Run integrity check specifically for linktable foreign key Ids + // Run integrity check specifically for linktable foreign key Ids const results = await iModel.integrityCheck({ quickCheck: true, specificChecks: { diff --git a/core/backend/src/test/standalone/SQliteChangesetReaderAndChangesetECAdaptor.test.ts b/core/backend/src/test/standalone/SQliteChangesetReaderAndChangesetECAdaptor.test.ts new file mode 100644 index 000000000000..c47e8a84a3f6 --- /dev/null +++ b/core/backend/src/test/standalone/SQliteChangesetReaderAndChangesetECAdaptor.test.ts @@ -0,0 +1,1920 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ +import { DbResult, GuidString, Id64, Id64String } from "@itwin/core-bentley"; +import { Code, ColorDef, GeometryStreamProps, IModel, QueryBinder, QueryRowFormat, SubCategoryAppearance } from "@itwin/core-common"; +import { Arc3d, IModelJson, Point3d } from "@itwin/core-geometry"; +import * as chai from "chai"; +import { assert, expect } from "chai"; +import * as path from "node:path"; +import { DrawingCategory } from "../../Category"; +import { ChangedECInstance, ChangesetECAdaptor, ChangesetECAdaptor as ECChangesetAdaptor, ECChangeUnifierCache, PartialECChangeUnifier } from "../../ChangesetECAdaptor"; +import { _nativeDb, ChannelControl, GraphicalElement2d, Subject, SubjectOwnsSubjects } from "../../core-backend"; +import { BriefcaseDb, SnapshotDb } from "../../IModelDb"; +import { HubMock } from "../../internal/HubMock"; +import { SqliteChangeOp, SqliteChangesetReader } from "../../SqliteChangesetReader"; +import { HubWrappers, IModelTestUtils } from "../IModelTestUtils"; +import { KnownTestLocations } from "../KnownTestLocations"; +import { EditTxn } from "../../EditTxn"; + +/* eslint-disable @typescript-eslint/no-deprecated */ // This test file will be removed subsequently, so we can allow usage of deprecated APIs within it. +// This test file also contains tests outside of the ChangesetECAdaptor, so be cautious while removing it, remove just the tests which are related to ChangesetECAdaptor. + +function startTestTxn(iModel: BriefcaseDb | SnapshotDb, description = "changeset reader"): EditTxn { + const txn = new EditTxn(iModel, description); + txn.start(); + return txn; +} + +async function importSchemaStrings(txn: EditTxn, schemas: string[]): Promise { + if (txn.isActive) + txn.saveChanges(); + await txn.iModel.importSchemaStrings(schemas); +} + +describe("Sqlite Changeset Reader + ChangesetECAdaptor API", async () => { + let iTwinId: GuidString; + + before(() => { + HubMock.startup("SqliteChangesetReaderAndChangesetECAdaptorTest", KnownTestLocations.outputDir); + iTwinId = HubMock.iTwinId; + }); + after(() => HubMock.shutdown()); + it("Able to recover from when ExclusiveRootClassId is NULL for overflow table", async () => { + /** + * 1. Import schema with class that span overflow table. + * 2. Insert a element for the class. + * 3. Push changes to hub. + * 4. Update the element. + * 5. Push changes to hub. + * 6. Delete the element. + * 7. Set ExclusiveRootClassId to NULL for overflow table. (Simulate the issue) + * 8. ECChangesetAdaptor should be able to read the changeset 2 in which element is updated against latest imodel where element is deleted. + */ + const adminToken = "super manager token"; + const iModelName = "test"; + const nProps = 36; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + const txn = startTestTxn(rwIModel, "recover overflow table changeset reader"); + // 1. Import schema with class that span overflow table. + const schema = ` + + + + bis:GraphicalElement2d + ${Array(nProps).fill(undefined).map((_, i) => ``).join("\n")} + + `; + await importSchemaStrings(txn, [schema]); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // Create drawing model and category + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "DrawingModel"; + const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); + let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); + if (undefined === drawingCategoryId) + drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + + // Insert element with 100 properties + const geomArray: Arc3d[] = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + Arc3d.createXY(Point3d.create(5, 5), 2), + Arc3d.createXY(Point3d.create(-5, -5), 20), + ]; + const geometryStream: GeometryStreamProps = []; + for (const geom of geomArray) { + const arcData = IModelJson.Writer.toIModelJson(geom); + geometryStream.push(arcData); + } + const props = Array(nProps).fill(undefined).map((_, i) => { + return { [`p${i}`]: `test_${i}` }; + }).reduce((acc, curr) => { + return { ...acc, ...curr }; + }, {}); + + const geomElement = { + classFullName: `TestDomain:Test2dElement`, + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + geom: geometryStream, + ...props, + }; + + // 2. Insert a element for the class. + const id = txn.insertElement(geomElement); + assert.isTrue(Id64.isValidId64(id), "insert worked"); + txn.saveChanges(); + + // 3. Push changes to hub. + await rwIModel.pushChanges({ description: "insert element", accessToken: adminToken }); + + // 4. Update the element. + const updatedElementProps = Object.assign( + rwIModel.elements.getElementProps(id), + Array(nProps).fill(undefined).map((_, i) => { + return { [`p${i}`]: `updated_${i}` }; + }).reduce((acc, curr) => { + return { ...acc, ...curr }; + }, {})); + + await rwIModel.locks.acquireLocks({ exclusive: id }); + txn.updateElement(updatedElementProps); + txn.saveChanges(); + + // 5. Push changes to hub. + await rwIModel.pushChanges({ description: "update element", accessToken: adminToken }); + + await rwIModel.locks.acquireLocks({ exclusive: id }); + + // 6. Delete the element. + txn.deleteElement(id); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "delete element", accessToken: adminToken }); + + const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); + const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); + const reader = SqliteChangesetReader.openFile({ fileName: changesets[1].pathname, db: rwIModel, disableSchemaCheck: true }); + + // Set ExclusiveRootClassId to NULL for overflow table to simulate the issue + expect(rwIModel[_nativeDb].executeSql("UPDATE ec_Table SET ExclusiveRootClassId=NULL WHERE Name='bis_GeometricElement2d_Overflow'")).to.be.eq(DbResult.BE_SQLITE_OK); + + const adaptor = new ECChangesetAdaptor(reader); + let assertOnOverflowTable = false; + const classId = getClassIdByName(rwIModel, "GeometricElement2d"); + + while (adaptor.step()) { + if (adaptor.op === "Updated" && adaptor.inserted?.$meta?.tables[0] === "bis_GeometricElement2d_Overflow") { + + assert.isUndefined(adaptor.inserted.ECClassId); + assert.equal(adaptor.inserted.ECInstanceId, ""); + assert.deepEqual(adaptor.inserted.$meta?.tables, ["bis_GeometricElement2d_Overflow"]); + assert.equal(adaptor.inserted.$meta?.op, "Updated"); + assert.equal(adaptor.inserted.$meta?.classFullName, "BisCore:GeometricElement2d"); + assert.equal(adaptor.inserted.$meta.fallbackClassId, classId); + assert.deepEqual(adaptor.inserted.$meta?.changeIndexes, [3]); + assert.equal(adaptor.inserted.$meta?.stage, "New"); + + assert.equal(adaptor.deleted!.ECInstanceId, ""); + assert.isUndefined(adaptor.deleted!.ECClassId); + assert.deepEqual(adaptor.deleted!.$meta?.tables, ["bis_GeometricElement2d_Overflow"]); + assert.equal(adaptor.deleted!.$meta?.op, "Updated"); + assert.equal(adaptor.deleted!.$meta?.classFullName, "BisCore:GeometricElement2d"); + assert.equal(adaptor.deleted!.$meta!.fallbackClassId, classId); + assert.deepEqual(adaptor.deleted!.$meta?.changeIndexes, [3]); + assert.equal(adaptor.deleted!.$meta?.stage, "Old"); + + assertOnOverflowTable = true; + } + } + + assert.isTrue(assertOnOverflowTable); + txn.end(); + rwIModel.close(); + }); + + function getClassIdByName(iModel: BriefcaseDb, className: string): Id64String { + // eslint-disable-next-line @typescript-eslint/no-deprecated + return iModel.withPreparedStatement(`SELECT ECInstanceId from meta.ECClassDef where Name=?`, (stmt) => { + stmt.bindString(1, className); + assert.equal(stmt.step(), DbResult.BE_SQLITE_ROW); + return stmt.getValue(0).getId(); + }); + } + + async function getClassNameById(iModel: BriefcaseDb, classId: string): Promise { + const reader = iModel.createQueryReader(`select ec_classname(${classId});`); + + if (await reader.step()) + return reader.current[0] as string; + + return undefined; + } + + it("Changeset reader / EC adaptor", async () => { + const adminToken = "super manager token"; + const iModelName = "test"; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + const txn = startTestTxn(rwIModel, "changeset reader EC adaptor"); + + const schema = ` + + + + bis:GraphicalElement2d + + + `; + await importSchemaStrings(txn, [schema]); + if (true || "push changes") { + // Push the changes to the hub + const prePushChangeSetId = rwIModel.changeset.id; + await rwIModel.pushChanges({ description: "push schema changeset", accessToken: adminToken }); + const postPushChangeSetId = rwIModel.changeset.id; + assert(!!postPushChangeSetId); + expect(prePushChangeSetId !== postPushChangeSetId); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + } + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "DrawingModel"; + let totalEl = 0; + const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); + let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); + if (undefined === drawingCategoryId) + drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + + txn.saveChanges("user 1: create drawing partition"); + if (true || "push changes") { + // Push the changes to the hub + const prePushChangeSetId = rwIModel.changeset.id; + await rwIModel.pushChanges({ description: "user 1: create drawing partition", accessToken: adminToken }); + const postPushChangeSetId = rwIModel.changeset.id; + assert(!!postPushChangeSetId); + expect(prePushChangeSetId !== postPushChangeSetId); + } + + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + const insertElements = (className: string = "Test2dElement", noOfElements: number = 10, userProp: (n: number) => object) => { + for (let m = 0; m < noOfElements; ++m) { + const geomArray: Arc3d[] = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + Arc3d.createXY(Point3d.create(5, 5), 2), + Arc3d.createXY(Point3d.create(-5, -5), 20), + ]; + const geometryStream: GeometryStreamProps = []; + for (const geom of geomArray) { + const arcData = IModelJson.Writer.toIModelJson(geom); + geometryStream.push(arcData); + } + const prop = userProp(++totalEl); + // Create props + const geomElement = { + classFullName: `TestDomain:${className}`, + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + geom: geometryStream, + ...prop, + }; + const id = txn.insertElement(geomElement); + assert.isTrue(Id64.isValidId64(id), "insert worked"); + } + }; + const generatedStr = new Array(10).join("x"); + insertElements("Test2dElement", 1, () => { + return { s: generatedStr }; + }); + + const updatedElements = async () => { + await rwIModel.locks.acquireLocks({ exclusive: "0x20000000004" }); + const updatedElement = rwIModel.elements.getElementProps("0x20000000004"); + (updatedElement as any).s = "updated property"; + txn.updateElement(updatedElement); + txn.saveChanges("user 1: updated data"); + await rwIModel.pushChanges({ description: "user 1: update property id=0x20000000004", accessToken: adminToken }); + }; + + txn.saveChanges("user 1: data"); + + if (true || "test local changes") { + const testChanges = async (changes: ChangedECInstance[]) => { + assert.equal(changes.length, 3); + + assert.equal(changes[0].ECInstanceId, "0x20000000001"); + assert.equal(changes[0].$meta?.classFullName, "BisCore:DrawingModel"); + assert.equal(changes[0].$meta?.op, "Updated"); + assert.equal(changes[0].$meta?.stage, "New"); + assert.isNotNull(changes[0].LastMod); + assert.isNotNull(changes[0].GeometryGuid); + + assert.equal(changes[1].ECInstanceId, "0x20000000001"); + assert.equal(changes[1].$meta?.classFullName, "BisCore:DrawingModel"); + assert.equal(changes[1].$meta?.op, "Updated"); + assert.equal(changes[1].$meta?.stage, "Old"); + assert.isNull(changes[1].LastMod); + assert.isNull(changes[1].GeometryGuid); + + assert.equal(changes[2].ECInstanceId, "0x20000000004"); + assert.equal(changes[2].$meta?.classFullName, "TestDomain:Test2dElement"); + assert.equal(changes[2].$meta?.op, "Inserted"); + assert.equal(changes[2].$meta?.stage, "New"); + + const el = changes.filter((x) => x.ECInstanceId === "0x20000000004")[0]; + assert.equal(el.Rotation, 0); + // eslint-disable-next-line @typescript-eslint/naming-convention + assert.deepEqual(el.Origin, { X: 0, Y: 0 }); + // eslint-disable-next-line @typescript-eslint/naming-convention + assert.deepEqual(el.BBoxLow, { X: -25, Y: -25 }); + // eslint-disable-next-line @typescript-eslint/naming-convention + assert.deepEqual(el.BBoxHigh, { X: 15, Y: 15 }); + + assert.equal(el.Category.Id, "0x20000000002"); + assert.isNotEmpty(el.Category.RelECClassId); + + const categoryRelClass = await getClassNameById(rwIModel, el.Category.RelECClassId); + assert.equal("BisCore:GeometricElement2dIsInCategory", categoryRelClass); + assert.equal(el.s, "xxxxxxxxx"); + assert.isNull(el.CodeValue); + assert.isNull(el.UserLabel); + assert.isNull(el.JsonProperties); + assert.instanceOf(el.GeometryStream, Uint8Array); + assert.typeOf(el.FederationGuid, "string"); + assert.typeOf(el.LastMod, "string"); + // eslint-disable-next-line @typescript-eslint/naming-convention + assert.deepEqual(el.Parent, { Id: null, RelECClassId: null }); + // eslint-disable-next-line @typescript-eslint/naming-convention + assert.deepEqual(el.TypeDefinition, { Id: null, RelECClassId: null }); + + assert.equal(el.CodeSpec.Id, "0x1"); + assert.isNotEmpty(el.CodeSpec.RelECClassId); + + const codeSpecRelClass = await getClassNameById(rwIModel, el.CodeSpec.RelECClassId); + assert.equal("BisCore:CodeSpecSpecifiesCode", codeSpecRelClass); + + assert.equal(el.CodeScope.Id, "0x1"); + assert.isNotEmpty(el.CodeScope.RelECClassId); + + const codeScopeRelClass = await getClassNameById(rwIModel, el.CodeScope.RelECClassId); + assert.equal("BisCore:ElementScopesCode", codeScopeRelClass); + + assert.deepEqual(el.$meta, { + tables: [ + "bis_GeometricElement2d", + "bis_Element", + ], + op: "Inserted", + classFullName: "TestDomain:Test2dElement", + changeIndexes: [ + 2, + 1, + ], + stage: "New", + }); + } + + if (true || "test with InMemoryInstanceCache") { + using reader = SqliteChangesetReader.openLocalChanges({ db: rwIModel, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + await testChanges(Array.from(pcu.instances)); + } + + if (true || "test with SqliteBackedInstanceCache") { + using reader = SqliteChangesetReader.openLocalChanges({ db: rwIModel, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + await testChanges(Array.from(pcu.instances)); + } + } + + const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); + await rwIModel.pushChanges({ description: "schema changeset", accessToken: adminToken }); + + await updatedElements(); + + const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); + if (true || "updated element") { + const testChanges = (changes: ChangedECInstance[]) => { + assert.equal(changes.length, 4); + + const classId: Id64String = getClassIdByName(rwIModel, "Test2dElement"); + + // new value + assert.equal(changes[2].ECInstanceId, "0x20000000004"); + assert.equal(changes[2].ECClassId, classId); + assert.equal(changes[2].s, "updated property"); + assert.equal(changes[2].$meta?.classFullName, "TestDomain:Test2dElement"); + assert.equal(changes[2].$meta?.op, "Updated"); + assert.equal(changes[2].$meta?.stage, "New"); + + // old value + assert.equal(changes[3].ECInstanceId, "0x20000000004"); + assert.equal(changes[3].ECClassId, classId); + assert.equal(changes[3].s, "xxxxxxxxx"); + assert.equal(changes[3].$meta?.classFullName, "TestDomain:Test2dElement"); + assert.equal(changes[3].$meta?.op, "Updated"); + assert.equal(changes[3].$meta?.stage, "Old"); + }; + + if (true || "test with InMemoryInstanceCache") { + using reader = SqliteChangesetReader.openFile({ fileName: changesets[3].pathname, db: rwIModel, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + testChanges(Array.from(pcu.instances)); + } + + if (true || "test with SqliteBackedInstanceCache") { + using reader = SqliteChangesetReader.openFile({ fileName: changesets[3].pathname, db: rwIModel, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + testChanges(Array.from(pcu.instances)); + } + } + + if (true || "updated element when no classId") { + const otherDb = SnapshotDb.openFile(IModelTestUtils.resolveAssetFile("test.bim")); + const testChanges = (changes: ChangedECInstance[]) => { + assert.equal(changes.length, 4); + + // new value + assert.equal(changes[2].ECInstanceId, "0x20000000004"); + assert.isUndefined(changes[2].ECClassId); + assert.isDefined(changes[2].$meta?.fallbackClassId); + assert.equal(changes[2].$meta?.fallbackClassId, "0x3d"); + assert.isUndefined(changes[2].s); + assert.equal(changes[2].$meta?.classFullName, "BisCore:GeometricElement2d"); + assert.equal(changes[2].$meta?.op, "Updated"); + assert.equal(changes[2].$meta?.stage, "New"); + + // old value + assert.equal(changes[3].ECInstanceId, "0x20000000004"); + assert.isUndefined(changes[3].ECClassId); + assert.isDefined(changes[3].$meta?.fallbackClassId); + assert.equal(changes[3].$meta?.fallbackClassId, "0x3d"); + assert.isUndefined(changes[3].s); + assert.equal(changes[3].$meta?.classFullName, "BisCore:GeometricElement2d"); + assert.equal(changes[3].$meta?.op, "Updated"); + assert.equal(changes[3].$meta?.stage, "Old"); + }; + + if (true || "test with InMemoryInstanceCache") { + using reader = SqliteChangesetReader.openFile({ fileName: changesets[3].pathname, db: otherDb, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + testChanges(Array.from(pcu.instances)); + } + + if (true || "test with SqliteBackedInstanceCache") { + using reader = SqliteChangesetReader.openFile({ fileName: changesets[3].pathname, db: otherDb, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + testChanges(Array.from(pcu.instances)); + } + } + + if (true || "test changeset file") { + const testChanges = async (changes: ChangedECInstance[]) => { + assert.equal(changes.length, 3); + + assert.equal(changes[0].ECInstanceId, "0x20000000001"); + assert.equal(changes[0].$meta?.classFullName, "BisCore:DrawingModel"); + assert.equal(changes[0].$meta?.op, "Updated"); + assert.equal(changes[0].$meta?.stage, "New"); + assert.isNotNull(changes[0].LastMod); + assert.isNotNull(changes[0].GeometryGuid); + + assert.equal(changes[1].ECInstanceId, "0x20000000001"); + assert.equal(changes[1].$meta?.classFullName, "BisCore:DrawingModel"); + assert.equal(changes[1].$meta?.op, "Updated"); + assert.equal(changes[1].$meta?.stage, "Old"); + assert.isNull(changes[1].LastMod); + assert.isNull(changes[1].GeometryGuid); + + assert.equal(changes[2].ECInstanceId, "0x20000000004"); + assert.equal(changes[2].$meta?.classFullName, "TestDomain:Test2dElement"); + assert.equal(changes[2].$meta?.op, "Inserted"); + assert.equal(changes[2].$meta?.stage, "New"); + + const el = changes.filter((x) => x.ECInstanceId === "0x20000000004")[0]; + assert.equal(el.Rotation, 0); + // eslint-disable-next-line @typescript-eslint/naming-convention + assert.deepEqual(el.Origin, { X: 0, Y: 0 }); + // eslint-disable-next-line @typescript-eslint/naming-convention + assert.deepEqual(el.BBoxLow, { X: -25, Y: -25 }); + // eslint-disable-next-line @typescript-eslint/naming-convention + assert.deepEqual(el.BBoxHigh, { X: 15, Y: 15 }); + + assert.equal(el.Category.Id, "0x20000000002"); + assert.isNotEmpty(el.Category.RelECClassId); + + const categoryRelClass = await getClassNameById(rwIModel, el.Category.RelECClassId); + assert.equal("BisCore:GeometricElement2dIsInCategory", categoryRelClass); + assert.equal(el.s, "xxxxxxxxx"); + assert.isNull(el.CodeValue); + assert.isNull(el.UserLabel); + assert.isNull(el.JsonProperties); + assert.instanceOf(el.GeometryStream, Uint8Array); + assert.typeOf(el.FederationGuid, "string"); + assert.typeOf(el.LastMod, "string"); + // eslint-disable-next-line @typescript-eslint/naming-convention + assert.deepEqual(el.Parent, { Id: null, RelECClassId: null }); + // eslint-disable-next-line @typescript-eslint/naming-convention + assert.deepEqual(el.TypeDefinition, { Id: null, RelECClassId: null }); + + assert.equal(el.CodeSpec.Id, "0x1"); + assert.isNotEmpty(el.CodeSpec.RelECClassId); + + const codeSpecRelClass = await getClassNameById(rwIModel, el.CodeSpec.RelECClassId); + assert.equal("BisCore:CodeSpecSpecifiesCode", codeSpecRelClass); + + assert.equal(el.CodeScope.Id, "0x1"); + assert.isNotEmpty(el.CodeScope.RelECClassId); + + const codeScopeRelClass = await getClassNameById(rwIModel, el.CodeScope.RelECClassId); + assert.equal("BisCore:ElementScopesCode", codeScopeRelClass); + + assert.deepEqual(el.$meta, { + tables: [ + "bis_GeometricElement2d", + "bis_Element", + ], + op: "Inserted", + classFullName: "TestDomain:Test2dElement", + changeIndexes: [ + 2, + 1, + ], + stage: "New", + }); + } + if (true || "test with InMemoryInstanceCache") { + using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + await testChanges(Array.from(pcu.instances)); + } + + if (true || "test with SqliteBackedInstanceCache") { + using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + await testChanges(Array.from(pcu.instances)); + } + } + if (true || "test ChangesetAdaptor.acceptClass()") { + const testChanges = (changes: ChangedECInstance[]) => { + assert.equal(changes.length, 1); + assert.equal(changes[0].$meta?.classFullName, "TestDomain:Test2dElement"); + }; + if (true || "test with InMemoryInstanceCache") { + using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + adaptor.acceptClass("TestDomain.Test2dElement"); + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + testChanges(Array.from(pcu.instances)); + } + + if (true || "test with SqliteBackedInstanceCache") { + using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + adaptor.acceptClass("TestDomain.Test2dElement"); + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + testChanges(Array.from(pcu.instances)); + } + } + if (true || "test ChangesetAdaptor.adaptor()") { + const testChanges = (changes: ChangedECInstance[]) => { + assert.equal(changes.length, 2); + assert.equal(changes[0].ECInstanceId, "0x20000000001"); + assert.equal(changes[0].$meta?.classFullName, "BisCore:DrawingModel"); + assert.equal(changes[0].$meta?.op, "Updated"); + assert.equal(changes[0].$meta?.stage, "New"); + assert.equal(changes[1].ECInstanceId, "0x20000000001"); + assert.equal(changes[1].$meta?.classFullName, "BisCore:DrawingModel"); + assert.equal(changes[1].$meta?.op, "Updated"); + assert.equal(changes[1].$meta?.stage, "Old"); + }; + if (true || "test with InMemoryInstanceCache") { + using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + adaptor.acceptOp("Updated") + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + testChanges(Array.from(pcu.instances)); + } + + if (true || "test with SqliteBackedInstanceCache") { + using reader = SqliteChangesetReader.openFile({ fileName: changesets[2].pathname, db: rwIModel, disableSchemaCheck: true }); + using adaptor = new ECChangesetAdaptor(reader); + adaptor.acceptOp("Updated") + using pcu = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(rwIModel)); + while (adaptor.step()) { + pcu.appendFrom(adaptor); + } + testChanges(Array.from(pcu.instances)); + } + } + txn.end(); + rwIModel.close(); + }); + it("revert timeline changes", async () => { + const adminToken = "super manager token"; + const iModelName = "test"; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + const txn = startTestTxn(rwIModel, "revert timeline changes"); + let nProps = 0; + // 1. Import schema with class that span overflow table. + const addPropertyAndImportSchema = async () => { + await rwIModel.acquireSchemaLock(); + ++nProps; + const schema = ` + + + + bis:GraphicalElement2d + ${Array(nProps).fill(undefined).map((_, i) => ``).join("\n")} + + `; + await importSchemaStrings(txn, [schema]); + }; + await addPropertyAndImportSchema(); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // Create drawing model and category + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "DrawingModel"; + const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); + let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); + if (undefined === drawingCategoryId) + drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + + txn.saveChanges(); + await rwIModel.pushChanges({ description: "setup category", accessToken: adminToken }); + + const createEl = async (args: { [key: string]: any }) => { + await rwIModel.locks.acquireLocks({ exclusive: drawingModelId }); + const geomArray: Arc3d[] = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + Arc3d.createXY(Point3d.create(5, 5), 2), + Arc3d.createXY(Point3d.create(-5, -5), 20), + ]; + + const geometryStream: GeometryStreamProps = []; + for (const geom of geomArray) { + const arcData = IModelJson.Writer.toIModelJson(geom); + geometryStream.push(arcData); + } + + const e1 = { + classFullName: `TestDomain:Test2dElement`, + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + geom: geometryStream, + ...args, + }; + return txn.insertElement(e1);; + }; + const updateEl = async (id: Id64String, args: { [key: string]: any }) => { + await rwIModel.locks.acquireLocks({ exclusive: id }); + const updatedElementProps = Object.assign(rwIModel.elements.getElementProps(id), args); + txn.updateElement(updatedElementProps); + }; + + const deleteEl = async (id: Id64String) => { + await rwIModel.locks.acquireLocks({ exclusive: id }); + txn.deleteElement(id); + }; + const getChanges = async () => { + return HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir: path.join(KnownTestLocations.outputDir, rwIModelId, "changesets") }); + }; + + const findEl = (id: Id64String) => { + try { + return rwIModel.elements.getElementProps(id); + } catch { + return undefined; + } + }; + // 2. Insert a element for the class + const el1 = await createEl({ p1: "test1" }); + const el2 = await createEl({ p1: "test2" }); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "insert 2 elements" }); + + // 3. Update the element. + await updateEl(el1, { p1: "test3" }); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "update element 1" }); + + // 4. Delete the element. + await deleteEl(el2); + const el3 = await createEl({ p1: "test4" }); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "delete element 2" }); + + // 5. import schema and insert element 4 & update element 3 + await addPropertyAndImportSchema(); + const el4 = await createEl({ p1: "test5", p2: "test6" }); + await updateEl(el3, { p1: "test7", p2: "test8" }); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "import schema, insert element 4 & update element 3" }); + + assert.isDefined(findEl(el1)); + assert.isUndefined(findEl(el2)); + assert.isDefined(findEl(el3)); + assert.isDefined(findEl(el4)); + // eslint-disable-next-line @typescript-eslint/no-deprecated + assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1", "p2"]); + // 6. Revert to timeline 2 + await rwIModel.revertAndPushChanges({ toIndex: 2, description: "revert to timeline 2" }); + assert.equal((await getChanges()).at(-1)!.description, "revert to timeline 2"); + + assert.isUndefined(findEl(el1)); + assert.isUndefined(findEl(el2)); + assert.isUndefined(findEl(el3)); + assert.isUndefined(findEl(el4)); + // eslint-disable-next-line @typescript-eslint/no-deprecated + assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1"]); + + await rwIModel.revertAndPushChanges({ toIndex: 6, description: "reinstate last reverted changeset" }); + assert.equal((await getChanges()).at(-1)!.description, "reinstate last reverted changeset"); + assert.isDefined(findEl(el1)); + assert.isUndefined(findEl(el2)); + assert.isDefined(findEl(el3)); + assert.isDefined(findEl(el4)); + // eslint-disable-next-line @typescript-eslint/no-deprecated + assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1", "p2"]); + + await addPropertyAndImportSchema(); + const el5 = await createEl({ p1: "test9", p2: "test10", p3: "test11" }); + await updateEl(el1, { p1: "test12", p2: "test13", p3: "test114" }); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "import schema, insert element 5 & update element 1" }); + // eslint-disable-next-line @typescript-eslint/no-deprecated + assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1", "p2", "p3"]); + + // skip schema changes & auto generated comment + await rwIModel.revertAndPushChanges({ toIndex: 1, skipSchemaChanges: true }); + assert.equal((await getChanges()).at(-1)!.description, "Reverted changes from 8 to 1 (schema changes skipped)"); + assert.isUndefined(findEl(el1)); + assert.isUndefined(findEl(el2)); + assert.isUndefined(findEl(el3)); + assert.isUndefined(findEl(el4)); + assert.isUndefined(findEl(el5)); + // eslint-disable-next-line @typescript-eslint/no-deprecated + assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1", "p2", "p3"]); + + await rwIModel.revertAndPushChanges({ toIndex: 9 }); + assert.equal((await getChanges()).at(-1)!.description, "Reverted changes from 9 to 9"); + assert.isDefined(findEl(el1)); + assert.isUndefined(findEl(el2)); + assert.isDefined(findEl(el3)); + assert.isDefined(findEl(el4)); + assert.isDefined(findEl(el5)); + // eslint-disable-next-line @typescript-eslint/no-deprecated + assert.deepEqual(Object.getOwnPropertyNames(rwIModel.getMetaData("TestDomain:Test2dElement").properties), ["p1", "p2", "p3"]); + txn.end(); + rwIModel.close(); + }); + it("openGroup() & writeToFile()", async () => { + const adminToken = "super manager token"; + const iModelName = "test"; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + const txn = startTestTxn(rwIModel, "openGroup writeToFile"); + // 1. Import schema with class that span overflow table. + const schema = ` + + + + bis:GraphicalElement2d + + + `; + await importSchemaStrings(txn, [schema]); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // Create drawing model and category + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "DrawingModel"; + const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); + let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); + if (undefined === drawingCategoryId) + drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + + txn.saveChanges(); + await rwIModel.pushChanges({ description: "setup category", accessToken: adminToken }); + const geomArray: Arc3d[] = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + Arc3d.createXY(Point3d.create(5, 5), 2), + Arc3d.createXY(Point3d.create(-5, -5), 20), + ]; + + const geometryStream: GeometryStreamProps = []; + for (const geom of geomArray) { + const arcData = IModelJson.Writer.toIModelJson(geom); + geometryStream.push(arcData); + } + + const e1 = { + classFullName: `TestDomain:Test2dElement`, + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + geom: geometryStream, + ...{ p1: "test1" }, + }; + + // 2. Insert a element for the class + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + const e1id = txn.insertElement(e1); + assert.isTrue(Id64.isValidId64(e1id), "insert worked"); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "insert element", accessToken: adminToken }); + + // 3. Update the element. + const updatedElementProps = Object.assign(rwIModel.elements.getElementProps(e1id), { p1: "test2" }); + await rwIModel.locks.acquireLocks({ exclusive: e1id }); + txn.updateElement(updatedElementProps); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "update element", accessToken: adminToken }); + + // 4. Delete the element. + await rwIModel.locks.acquireLocks({ exclusive: e1id }); + txn.deleteElement(e1id); + txn.saveChanges(); + await rwIModel.pushChanges({ description: "delete element", accessToken: adminToken }); + + const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); + const changesets = (await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir })).slice(1); + + const testElementClassId: Id64String = getClassIdByName(rwIModel, "Test2dElement"); + const drawingModelClassId: Id64String = getClassIdByName(rwIModel, "DrawingModel"); + + if (true || "Grouping changeset [2,3,4] should not contain TestDomain:Test2dElement as insert+update+delete=noop") { + const reader = SqliteChangesetReader.openGroup({ changesetFiles: changesets.map((c) => c.pathname), db: rwIModel, disableSchemaCheck: true }); + const adaptor = new ECChangesetAdaptor(reader); + const instances: ({ id: string, classId?: string, op: SqliteChangeOp, classFullName?: string })[] = []; + while (adaptor.step()) { + if (adaptor.inserted) { + instances.push({ id: adaptor.inserted?.ECInstanceId, classId: adaptor.inserted.ECClassId, op: adaptor.op, classFullName: adaptor.inserted.$meta?.classFullName }); + } else if (adaptor.deleted) { + instances.push({ id: adaptor.deleted?.ECInstanceId, classId: adaptor.deleted.ECClassId, op: adaptor.op, classFullName: adaptor.deleted.$meta?.classFullName }); + } + } + + expect(instances.length).to.eq(1); + expect(instances[0].id).to.eq("0x20000000001"); + expect(instances[0].classId).to.eq(drawingModelClassId); + expect(instances[0].op).to.eq("Updated"); + expect(instances[0].classFullName).to.eq("BisCore:DrawingModel"); + } + + if (true || "Grouping changeset [3,4] should contain update+delete=delete TestDomain:Test2dElement") { + const reader = SqliteChangesetReader.openGroup({ changesetFiles: changesets.slice(1).map((c) => c.pathname), db: rwIModel, disableSchemaCheck: true }); + const adaptor = new ECChangesetAdaptor(reader); + const instances: ({ id: string, classId?: string, op: SqliteChangeOp, classFullName?: string })[] = []; + while (adaptor.step()) { + if (adaptor.inserted) { + instances.push({ id: adaptor.inserted?.ECInstanceId, classId: adaptor.inserted.ECClassId, op: adaptor.op, classFullName: adaptor.inserted.$meta?.classFullName }); + } else if (adaptor.deleted) { + instances.push({ id: adaptor.deleted?.ECInstanceId, classId: adaptor.deleted.ECClassId, op: adaptor.op, classFullName: adaptor.deleted.$meta?.classFullName }); + } + } + expect(instances.length).to.eq(3); + expect(instances[0]).deep.eq({ + id: "0x20000000004", + classId: testElementClassId, + op: "Deleted", + classFullName: "TestDomain:Test2dElement", + }); + expect(instances[1]).deep.eq({ + id: "0x20000000004", + classId: testElementClassId, + op: "Deleted", + classFullName: "TestDomain:Test2dElement", + }); + expect(instances[2]).deep.eq({ + id: "0x20000000001", + classId: drawingModelClassId, + op: "Updated", + classFullName: "BisCore:DrawingModel", + }); + } + + const groupCsFile = path.join(KnownTestLocations.outputDir, "changeset_grouping.ec"); + if (true || "Grouping changeset [2,3] should contain insert+update=insert TestDomain:Test2dElement") { + const reader = SqliteChangesetReader.openGroup({ changesetFiles: changesets.slice(0, 2).map((c) => c.pathname), db: rwIModel, disableSchemaCheck: true }); + const adaptor = new ECChangesetAdaptor(reader); + const instances: ({ id: string, classId?: string, op: SqliteChangeOp, classFullName?: string })[] = []; + while (adaptor.step()) { + if (adaptor.inserted) { + instances.push({ id: adaptor.inserted?.ECInstanceId, classId: adaptor.inserted.ECClassId, op: adaptor.op, classFullName: adaptor.inserted.$meta?.classFullName }); + } else if (adaptor.deleted) { + instances.push({ id: adaptor.deleted?.ECInstanceId, classId: adaptor.deleted.ECClassId, op: adaptor.op, classFullName: adaptor.deleted.$meta?.classFullName }); + } + } + expect(instances.length).to.eq(3); + expect(instances[0]).deep.eq({ + id: "0x20000000004", + classId: testElementClassId, + op: "Inserted", + classFullName: "TestDomain:Test2dElement", + }); + expect(instances[1]).deep.eq({ + id: "0x20000000004", + classId: testElementClassId, + op: "Inserted", + classFullName: "TestDomain:Test2dElement", + }); + expect(instances[2]).deep.eq({ + id: "0x20000000001", + classId: drawingModelClassId, + op: "Updated", + classFullName: "BisCore:DrawingModel", + }); + + reader.writeToFile({ fileName: groupCsFile, containsSchemaChanges: false, overwriteFile: true }); + } + if (true || "writeToFile() test") { + const reader = SqliteChangesetReader.openFile({ fileName: groupCsFile, db: rwIModel, disableSchemaCheck: true }); + const adaptor = new ECChangesetAdaptor(reader); + const instances: ({ id: string, classId?: string, op: SqliteChangeOp, classFullName?: string })[] = []; + while (adaptor.step()) { + if (adaptor.inserted) { + instances.push({ id: adaptor.inserted?.ECInstanceId, classId: adaptor.inserted.ECClassId, op: adaptor.op, classFullName: adaptor.inserted.$meta?.classFullName }); + } else if (adaptor.deleted) { + instances.push({ id: adaptor.deleted?.ECInstanceId, classId: adaptor.deleted.ECClassId, op: adaptor.op, classFullName: adaptor.deleted.$meta?.classFullName }); + } + } + expect(instances.length).to.eq(3); + expect(instances[0]).deep.eq({ + id: "0x20000000004", + classId: testElementClassId, + op: "Inserted", + classFullName: "TestDomain:Test2dElement", + }); + expect(instances[1]).deep.eq({ + id: "0x20000000004", + classId: testElementClassId, + op: "Inserted", + classFullName: "TestDomain:Test2dElement", + }); + expect(instances[2]).deep.eq({ + id: "0x20000000001", + classId: drawingModelClassId, + op: "Updated", + classFullName: "BisCore:DrawingModel", + }); + } + txn.end(); + rwIModel.close(); + }); + + it("Delete class FK constraint violation in cache table", async () => { + // Helper to check if TestClass exists in schema and cache table for both briefcases + function checkClass(firstBriefcase: BriefcaseDb, isClassInFirst: boolean, secondBriefcase: BriefcaseDb, isClassInSecond: boolean) { + const firstItems = firstBriefcase.getSchemaProps("TestSchema").items; + assert.equal(isClassInFirst, !!firstItems?.TestClass); + + const secondItems = secondBriefcase.getSchemaProps("TestSchema").items; + assert.equal(isClassInSecond, !!secondItems?.TestClass); + + const sql = `SELECT ch.classId FROM ec_cache_ClassHierarchy ch JOIN ec_Class c ON ch.classId = c.Id WHERE c.Name = 'TestClass'`; + const firstStmt = firstBriefcase.prepareSqliteStatement(sql); + assert.equal(firstStmt.step(), isClassInFirst ? DbResult.BE_SQLITE_ROW : DbResult.BE_SQLITE_DONE); + firstStmt[Symbol.dispose](); + + const secondStmt = secondBriefcase.prepareSqliteStatement(sql); + assert.equal(secondStmt.step(), isClassInSecond ? DbResult.BE_SQLITE_ROW : DbResult.BE_SQLITE_DONE); + secondStmt[Symbol.dispose](); + } + + const adminToken = "super manager token"; + const iModelName = "test"; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + + // Open two briefcases for the same iModel + const [firstBriefCase, secondBriefCase] = await Promise.all([ + HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }), + HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }) + ]); + const firstTxn = startTestTxn(firstBriefCase, "delete class FK constraint setup"); + + // Enable shared channel for both + [firstBriefCase, secondBriefCase].forEach(briefcase => briefcase.channels.addAllowedChannel(ChannelControl.sharedChannelName)); + + await importSchemaStrings(firstTxn, [` + + + + + + + + + + bis:PhysicalElement + + `]); + + // Push the changes to the hub + await firstBriefCase.pushChanges({ description: "push initial schema changeset", accessToken: adminToken }); + + // Sync the second briefcase with the iModel + await secondBriefCase.pullChanges({ accessToken: adminToken }); + + checkClass(firstBriefCase, true, secondBriefCase, true); + + // Import the schema + await importSchemaStrings(firstTxn, [` + + + + + + + `]); + + // Push the changeset to the hub + await firstBriefCase.pushChanges({ description: "Delete class major change", accessToken: adminToken }); + + checkClass(firstBriefCase, false, secondBriefCase, true); + + // Apply the latest changeset to a new briefcase + try { + await secondBriefCase.pullChanges({ accessToken: adminToken }); + } catch (error: any) { + assert.fail(`Should not have failed with the error: ${error.message}`); + } + + checkClass(firstBriefCase, false, secondBriefCase, false); + + // Cleanup + firstTxn.end(); + secondBriefCase.close(); + firstBriefCase.close(); + }); + + + it("Delete class FK constraint violation in cache table through a revert", async () => { + // Helper to check if TestClass exists in schema and cache table for both briefcases + function checkClass(className: string, firstBriefcase: BriefcaseDb, isClassInFirst: boolean, secondBriefcase: BriefcaseDb, isClassInSecond: boolean) { + assert.equal(isClassInFirst, !!firstBriefcase.getSchemaProps("TestSchema").items?.[className]); + assert.equal(isClassInSecond, !!secondBriefcase.getSchemaProps("TestSchema").items?.[className]); + + const sql = `SELECT ch.classId FROM ec_cache_ClassHierarchy ch JOIN ec_Class c ON ch.classId = c.Id WHERE c.Name = '${className}'`; + const firstStmt = firstBriefcase.prepareSqliteStatement(sql); + assert.equal(firstStmt.step(), isClassInFirst ? DbResult.BE_SQLITE_ROW : DbResult.BE_SQLITE_DONE); + firstStmt[Symbol.dispose](); + + const secondStmt = secondBriefcase.prepareSqliteStatement(sql); + assert.equal(secondStmt.step(), isClassInSecond ? DbResult.BE_SQLITE_ROW : DbResult.BE_SQLITE_DONE); + secondStmt[Symbol.dispose](); + } + + const adminToken = "super manager token"; + const iModelName = "test"; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + + // Open two briefcases for the same iModel + const [firstBriefCase, secondBriefCase] = await Promise.all([ + HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }), + HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }) + ]); + const firstTxn = startTestTxn(firstBriefCase, "delete class FK revert"); + + // Enable shared channel for both + [firstBriefCase, secondBriefCase].forEach(briefcase => briefcase.channels.addAllowedChannel(ChannelControl.sharedChannelName)); + + await importSchemaStrings(firstTxn, [` + + + + + + + + + + bis:PhysicalElement + + `]); + + // Push the changes to the hub + await firstBriefCase.pushChanges({ description: "push initial schema changeset", accessToken: adminToken }); + // Sync the second briefcase + await secondBriefCase.pullChanges({ accessToken: adminToken }); + + checkClass("TestClass", firstBriefCase, true, secondBriefCase, true); + + // Import the schema + await importSchemaStrings(firstTxn, [` + + + + + + + + + + bis:PhysicalElement + + + + bis:PhysicalElement + + `]); + + // Push the changeset to the hub + await firstBriefCase.pushChanges({ description: "Add another class change", accessToken: adminToken }); + // Sync the second briefcase + await secondBriefCase.pullChanges({ accessToken: adminToken }); + + checkClass("TestClass", firstBriefCase, true, secondBriefCase, true); + checkClass("AnotherTestClass", firstBriefCase, true, secondBriefCase, true); + + // Revert the latest changeset from the first briefcase + try { + await firstBriefCase.revertAndPushChanges({ toIndex: 2, description: "Revert last changeset" }); + } catch (error: any) { + assert.fail(`Should not have failed with the error: ${error.message}`); + } + + checkClass("TestClass", firstBriefCase, true, secondBriefCase, true); + checkClass("AnotherTestClass", firstBriefCase, false, secondBriefCase, true); + + try { + await secondBriefCase.pullChanges({ accessToken: adminToken }); + } catch (error: any) { + assert.fail(`Should not have failed with the error: ${error.message}`); + } + + checkClass("TestClass", firstBriefCase, true, secondBriefCase, true); + checkClass("AnotherTestClass", firstBriefCase, false, secondBriefCase, false); + + // Cleanup + firstTxn.end(); + secondBriefCase.close(); + firstBriefCase.close(); + }); + + it("Track changeset health stats", async () => { + const adminToken = "super manager token"; + const iModelName = "test"; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + + // Open two briefcases for the same iModel + const [firstBriefcase, secondBriefcase] = await Promise.all([ + HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }), + HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }) + ]); + const firstTxn = startTestTxn(firstBriefcase, "track changeset health first briefcase"); + const secondTxn = startTestTxn(secondBriefcase, "track changeset health second briefcase"); + + [firstBriefcase, secondBriefcase].forEach(briefcase => briefcase.channels.addAllowedChannel(ChannelControl.sharedChannelName)); + + await importSchemaStrings(firstTxn, [` + + + + + + + + + + bis:PhysicalElement + + `]); + + // Enable changeset tracking for both briefcases + await Promise.all([firstBriefcase.enableChangesetStatTracking(), secondBriefcase.enableChangesetStatTracking()]); + + await firstBriefcase.pushChanges({ description: "push initial schema changeset", accessToken: adminToken }); + await secondBriefcase.pullChanges({ accessToken: adminToken }); + + // Schema upgrade + await importSchemaStrings(secondTxn, [` + + + + + + + + + + bis:PhysicalElement + + + + + + + + `]); + + await secondBriefcase.pushChanges({ description: "Added a property to TestClass and an enum", accessToken: adminToken }); + await firstBriefcase.pullChanges({ accessToken: adminToken }); + + // Major schema change + await importSchemaStrings(firstTxn, [` + + + + + + + + + + + + `]); + + await firstBriefcase.pushChanges({ description: "Deleted TestClass", accessToken: adminToken }); + await secondBriefcase.pullChanges({ accessToken: adminToken }); + + const firstBriefcaseChangesets = await firstBriefcase.getAllChangesetHealthData(); + const secondBriefcaseChangesets = await secondBriefcase.getAllChangesetHealthData(); + + assert.equal(firstBriefcaseChangesets.length, 1); + const firstBriefcaseChangeset = firstBriefcaseChangesets[0]; + + expect(firstBriefcaseChangeset.changesetIndex).to.be.eql(2); + expect(firstBriefcaseChangeset.uncompressedSizeBytes).to.be.greaterThan(300); + expect(firstBriefcaseChangeset.insertedRows).to.be.greaterThanOrEqual(4); + expect(firstBriefcaseChangeset.updatedRows).to.be.greaterThanOrEqual(1); + expect(firstBriefcaseChangeset.deletedRows).to.be.eql(0); + expect(firstBriefcaseChangeset.totalFullTableScans).to.be.eql(0); + expect(firstBriefcaseChangeset.perStatementStats.length).to.be.eql(5); + + assert.equal(secondBriefcaseChangesets.length, 2); + const [secondBriefcaseChangeset1, secondBriefcaseChangeset2] = secondBriefcaseChangesets; + + expect(secondBriefcaseChangeset1.changesetIndex).to.be.eql(1); + expect(secondBriefcaseChangeset1.uncompressedSizeBytes).to.be.greaterThan(40000); + expect(secondBriefcaseChangeset1.insertedRows).to.be.eql(52); + expect(secondBriefcaseChangeset1.updatedRows).to.be.greaterThanOrEqual(921); + expect(secondBriefcaseChangeset1.deletedRows).to.be.greaterThanOrEqual(0) + expect(secondBriefcaseChangeset1.totalFullTableScans).to.be.eql(0); + expect(secondBriefcaseChangeset1.perStatementStats.length).to.be.eql(11); + + expect(secondBriefcaseChangeset2.changesetIndex).to.be.eql(3); + expect(secondBriefcaseChangeset2.uncompressedSizeBytes).to.be.greaterThan(40000); + expect(secondBriefcaseChangeset2.insertedRows).to.be.greaterThanOrEqual(0); + expect(secondBriefcaseChangeset2.updatedRows).to.be.greaterThanOrEqual(921); + expect(secondBriefcaseChangeset2.deletedRows).to.be.eql(52); + expect(secondBriefcaseChangeset2.totalFullTableScans).to.be.eql(0); + expect(secondBriefcaseChangeset2.perStatementStats.length).to.be.eql(11); + + // Cleanup + secondTxn.end(); + firstTxn.end(); + secondBriefcase.close(); + firstBriefcase.close(); + }); + it("openInMemory() & step()", async () => { + const adminToken = "super manager token"; + const iModelName = "test"; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + const txn = startTestTxn(rwIModel, "openInMemory step"); + // 1. Import schema with class that span overflow table. + const schema = ` + + + + bis:GraphicalElement2d + + + `; + await importSchemaStrings(txn, [schema]); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // Create drawing model and category + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "DrawingModel"; + const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); + let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); + if (undefined === drawingCategoryId) + drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + + txn.saveChanges(); + await rwIModel.pushChanges({ description: "setup category", accessToken: adminToken }); + + const geomArray: Arc3d[] = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + Arc3d.createXY(Point3d.create(5, 5), 2), + Arc3d.createXY(Point3d.create(-5, -5), 20), + ]; + + const geometryStream: GeometryStreamProps = []; + for (const geom of geomArray) { + const arcData = IModelJson.Writer.toIModelJson(geom); + geometryStream.push(arcData); + } + + const e1 = { + classFullName: `TestDomain:Test2dElement`, + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + geom: geometryStream, + ...{ p1: "test1" }, + }; + + // 2. Insert a element for the class + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + const e1id = txn.insertElement(e1); + assert.isTrue(Id64.isValidId64(e1id), "insert worked"); + const testElClassId: Id64String = getClassIdByName(rwIModel, "Test2dElement"); + + if (true) { + const reader = SqliteChangesetReader.openInMemory({ db: rwIModel, disableSchemaCheck: true }); + const adaptor = new ChangesetECAdaptor(reader); + const unifier = new PartialECChangeUnifier(rwIModel) + while (adaptor.step()) { + unifier.appendFrom(adaptor); + } + reader.close(); + + // verify the inserted element's properties + const instances = Array.from(unifier.instances); + expect(instances.length).to.equals(1); + const testEl = instances[0]; + expect(testEl.$meta?.op).to.equals("Inserted"); + expect(testEl.$meta?.classFullName).to.equals("TestDomain:Test2dElement"); + expect(testEl.$meta?.stage).to.equals("New"); + expect(testEl.ECClassId).to.equals(testElClassId); + expect(testEl.ECInstanceId).to.equals(e1id); + expect(testEl.Model.Id).to.equals(drawingModelId); + expect(testEl.Category.Id).to.equals(drawingCategoryId); + expect(testEl.Origin.X).to.equals(0); + expect(testEl.Origin.Y).to.equals(0); + expect(testEl.Rotation).to.equals(0); + expect(testEl.BBoxLow.X).to.equals(-25); + expect(testEl.BBoxLow.Y).to.equals(-25); + expect(testEl.BBoxHigh.X).to.equals(15); + expect(testEl.BBoxHigh.Y).to.equals(15); + expect(testEl.p1).to.equals("test1"); + } + + // save changes and verify the the txn + txn.saveChanges(); + + if (true) { + const txnId = rwIModel.txns.getLastSavedTxnProps()?.id as string; + expect(txnId).to.not.be.undefined; + const reader = SqliteChangesetReader.openTxn({ db: rwIModel, disableSchemaCheck: true, txnId }); + const adaptor = new ChangesetECAdaptor(reader); + const unifier = new PartialECChangeUnifier(rwIModel) + while (adaptor.step()) { + unifier.appendFrom(adaptor); + } + reader.close(); + + // verify the inserted element's properties + const instances = Array.from(unifier.instances); + expect(instances.length).to.equals(3); + const drawingModelClassId: Id64String = getClassIdByName(rwIModel, "DrawingModel"); + + // DrawingModel new instance + const drawingModelElNew = instances[0]; + expect(drawingModelElNew.$meta?.op).to.equals("Updated"); + expect(drawingModelElNew.$meta?.classFullName).to.equals("BisCore:DrawingModel"); + expect(drawingModelElNew.$meta?.stage).to.equals("New"); + expect(drawingModelElNew.ECClassId).to.equals(drawingModelClassId); + expect(drawingModelElNew.ECInstanceId).to.equals(drawingModelId); + expect(drawingModelElNew.LastMod).to.exist; + expect(drawingModelElNew.GeometryGuid).to.exist; + + // DrawingModel old instance + const drawingModelElOld = instances[1]; + expect(drawingModelElOld.$meta?.op).to.equals("Updated"); + expect(drawingModelElOld.$meta?.classFullName).to.equals("BisCore:DrawingModel"); + expect(drawingModelElOld.$meta?.stage).to.equals("Old"); + expect(drawingModelElOld.ECClassId).to.equals(drawingModelClassId); + expect(drawingModelElOld.ECInstanceId).to.equals(drawingModelId); + expect(drawingModelElOld.LastMod).to.null; + expect(drawingModelElOld.GeometryGuid).to.null; + + // Test element instance + const testEl = instances[2]; + expect(testEl.$meta?.op).to.equals("Inserted"); + expect(testEl.$meta?.classFullName).to.equals("TestDomain:Test2dElement"); + expect(testEl.$meta?.stage).to.equals("New"); + expect(testEl.ECClassId).to.equals(testElClassId); + expect(testEl.ECInstanceId).to.equals(e1id); + expect(testEl.Model.Id).to.equals(drawingModelId); + expect(testEl.Category.Id).to.equals(drawingCategoryId); + expect(testEl.Origin.X).to.equals(0); + expect(testEl.Origin.Y).to.equals(0); + expect(testEl.Rotation).to.equals(0); + expect(testEl.BBoxLow.X).to.equals(-25); + expect(testEl.BBoxLow.Y).to.equals(-25); + expect(testEl.BBoxHigh.X).to.equals(15); + expect(testEl.BBoxHigh.Y).to.equals(15); + expect(testEl.p1).to.equals("test1"); + } + await rwIModel.pushChanges({ description: "insert element", accessToken: adminToken }); + txn.end(); + rwIModel.close(); + }); + it("Instance update to a different class (bug)", async () => { + /** + * Test scenario: Verifies changeset reader behavior when an instance ID is reused with a different class. + * + * Steps: + * 1. Import schema with two classes (T1 and T2) that inherit from GraphicalElement2d. + * - T1 has property 'p' of type string + * - T2 has property 'p' of type long + * 2. Insert an element of type T1 with id=elId and property p="wwww" + * 3. Push changeset #1: "insert element" + * 4. Delete the T1 element + * 5. Manipulate the element ID sequence to force reuse of the same ID + * 6. Insert a new element of type T2 with the same id=elId but property p=1111 + * 7. Push changeset #2: "buggy changeset" + * + * Verification: + * - Changeset #2 should show an "Updated" operation (not Delete+Insert) + * - In bis_Element table: ECClassId changes from T1 to T2 + * - In bis_GeometricElement2d table: ECClassId changes from T1 to T2 + * - Property 'p' changes from string "wwww" to integer 1111 + * + * This tests the changeset reader's ability to handle instance class changes, + * which can occur in edge cases where IDs are reused with different types. + */ + const adminToken = "super manager token"; + const iModelName = "test"; + const modelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(modelId); + let b1 = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: modelId, accessToken: adminToken }); + let txn = startTestTxn(b1, "instance update to different class"); + // 1. Import schema with classes that span overflow table. + const schema = ` + + + + bis:GraphicalElement2d + + + + bis:GraphicalElement2d + + + `; + + await importSchemaStrings(txn, [schema]); + b1.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // Create drawing model and category + await b1.locks.acquireLocks({ shared: IModel.dictionaryId }); + const codeProps = Code.createEmpty(); + codeProps.value = "DrawingModel"; + const [, drawingModelId] = IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true); + let drawingCategoryId = DrawingCategory.queryCategoryIdByName(b1, IModel.dictionaryId, "MyDrawingCategory"); + if (undefined === drawingCategoryId) + drawingCategoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance({ color: ColorDef.fromString("rgb(255,0,0)").toJSON() })); + + + const geomArray: Arc3d[] = [ + Arc3d.createXY(Point3d.create(0, 0), 5), + Arc3d.createXY(Point3d.create(5, 5), 2), + Arc3d.createXY(Point3d.create(-5, -5), 20), + ]; + + const geometryStream: GeometryStreamProps = []; + for (const geom of geomArray) { + const arcData = IModelJson.Writer.toIModelJson(geom); + geometryStream.push(arcData); + } + + const geomElementT1 = { + classFullName: `TestDomain:T1`, + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + geom: geometryStream, + p: "wwww", + }; + + const elId = txn.insertElement(geomElementT1); + assert.isTrue(Id64.isValidId64(elId), "insert worked"); + txn.saveChanges(); + await b1.pushChanges({ description: "insert element" }); + + await b1.locks.acquireLocks({ shared: drawingModelId, exclusive: elId }); + await b1.locks.acquireLocks({ shared: IModel.dictionaryId }); + txn.deleteElement(elId); + txn.saveChanges(); + + // Force id set to reproduce same instance with different classid + const bid = BigInt(elId) - 1n + b1[_nativeDb].saveLocalValue("bis_elementidsequence", bid.toString()); + txn.saveChanges(); + const fileName = b1[_nativeDb].getFilePath(); + txn.end(); + b1.close(); + + b1 = await BriefcaseDb.open({ fileName }); + b1.channels.addAllowedChannel(ChannelControl.sharedChannelName); + txn = startTestTxn(b1, "instance update to different class reopened briefcase"); + + + const geomElementT2 = { + classFullName: `TestDomain:T2`, + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + geom: geometryStream, + p: 1111, + }; + + const elId2 = txn.insertElement(geomElementT2); + chai.expect(elId).equals(elId2); + + txn.saveChanges(); + await b1.pushChanges({ description: "buggy changeset" }); + + const getChanges = async () => { + return HubMock.downloadChangesets({ iModelId: modelId, targetDir: path.join(KnownTestLocations.outputDir, modelId, "changesets") }); + }; + + + const changesets = await getChanges(); + chai.expect(changesets.length).equals(2); + chai.expect(changesets[0].description).equals("insert element"); + chai.expect(changesets[1].description).equals("buggy changeset"); + + const getClassId = async (name: string) => { + const r = b1.createQueryReader("SELECT FORMAT('0x%x', ec_classid(?))", QueryBinder.from([name])); + if (await r.step()) { + return r.current[0]; + } + } + + const t1ClassId = await getClassId("TestDomain:T1"); + const t2ClassId = await getClassId("TestDomain:T2"); + + const reader = SqliteChangesetReader.openFile({ fileName: changesets[1].pathname, disableSchemaCheck: true, db: b1 }); + let bisElementAsserted = false; + let bisGeometricElement2dAsserted = false; + while (reader.step()) { + if (reader.tableName === "bis_Element" && reader.op === "Updated") { + bisElementAsserted = true; + chai.expect(reader.getColumnNames(reader.tableName)).deep.equals([ + "Id", + "ECClassId", + "ModelId", + "LastMod", + "CodeSpecId", + "CodeScopeId", + "CodeValue", + "UserLabel", + "ParentId", + "ParentRelECClassId", + "FederationGuid", + "JsonProperties", + ]); + + const oldId = reader.getChangeValueId(0, "Old"); + const newId = reader.getChangeValueId(0, "New"); + chai.expect(oldId).equals(elId); + chai.expect(newId).to.be.undefined; + + const oldClassId = reader.getChangeValueId(1, "Old"); + const newClassId = reader.getChangeValueId(1, "New"); + chai.expect(oldClassId).equals(t1ClassId); + chai.expect(newClassId).equals(t2ClassId); + chai.expect(oldClassId).is.not.equal(newClassId); + } + if (reader.tableName === "bis_GeometricElement2d" && reader.op === "Updated") { + bisGeometricElement2dAsserted = true; + chai.expect(reader.getColumnNames(reader.tableName)).deep.equals([ + "ElementId", + "ECClassId", + "CategoryId", + "Origin_X", + "Origin_Y", + "Rotation", + "BBoxLow_X", + "BBoxLow_Y", + "BBoxHigh_X", + "BBoxHigh_Y", + "GeometryStream", + "TypeDefinitionId", + "TypeDefinitionRelECClassId", + "js1", + "js2", + ]); + + // ECInstanceId + const oldId = reader.getChangeValueId(0, "Old"); + const newId = reader.getChangeValueId(0, "New"); + chai.expect(oldId).equals(elId); + chai.expect(newId).to.be.undefined; + + // ECClassId (changed) + const oldClassId = reader.getChangeValueId(1, "Old"); + const newClassId = reader.getChangeValueId(1, "New"); + chai.expect(oldClassId).equals(t1ClassId); + chai.expect(newClassId).equals(t2ClassId); + chai.expect(oldClassId).is.not.equal(newClassId); + + // Property 'p' changed type and value. + const oldP = reader.getChangeValueText(13, "Old"); + const newP = reader.getChangeValueInteger(13, "New"); + chai.expect(oldP).equals("wwww"); + chai.expect(newP).equals(1111); + } + } + + chai.expect(bisElementAsserted).to.be.true; + chai.expect(bisGeometricElement2dAsserted).to.be.true; + reader.close(); + + + // ChangesetECAdaptor works incorrectly as it does not expect ECClassId to change in an update. + const adaptor = new ChangesetECAdaptor( + SqliteChangesetReader.openFile({ fileName: changesets[1].pathname, disableSchemaCheck: true, db: b1 }) + ); + + adaptor.acceptClass(GraphicalElement2d.classFullName) + adaptor.acceptOp("Updated"); + + let ecChangeForElementAsserted = false; + let ecChangeForGeometricElement2dAsserted = false; + while (adaptor.step()) { + if (adaptor.reader.tableName === "bis_Element") { + ecChangeForElementAsserted = true; + chai.expect(adaptor.inserted?.$meta?.classFullName).equals("TestDomain:T1"); // WRONG should be TestDomain:T2 + chai.expect(adaptor.deleted?.$meta?.classFullName).equals("TestDomain:T1"); // WRONG should be TestDomain:T2 + } + if (adaptor.reader.tableName === "bis_GeometricElement2d") { + ecChangeForGeometricElement2dAsserted = true; + chai.expect(adaptor.inserted?.$meta?.classFullName).equals("TestDomain:T1"); // WRONG should be TestDomain:T2 + chai.expect(adaptor.deleted?.$meta?.classFullName).equals("TestDomain:T1"); // WRONG should be TestDomain:T2 + chai.expect(adaptor.inserted?.p).equals("0x457"); // CORRECT p in T2 is integer + chai.expect(adaptor.deleted?.p).equals("wwww"); // CORRECT p in T1 is string + } + } + chai.expect(ecChangeForElementAsserted).to.be.true; + chai.expect(ecChangeForGeometricElement2dAsserted).to.be.true; + adaptor.close(); + + // PartialECChangeUnifier fail to combine changes correctly when ECClassId is updated. + const adaptor2 = new ChangesetECAdaptor( + SqliteChangesetReader.openFile({ fileName: changesets[1].pathname, disableSchemaCheck: true, db: b1 }) + ); + const unifier = new PartialECChangeUnifier(b1); + adaptor2.acceptClass(GraphicalElement2d.classFullName) + adaptor2.acceptOp("Updated"); + while (adaptor2.step()) { + unifier.appendFrom(adaptor2); + } + + chai.expect(unifier.getInstanceCount()).to.be.equals(2); // WRONG should be 1 + + txn.end(); + b1.close(); + }); + +}); + +describe("PRAGMA ECSQL Functions", async () => { + let iTwinId: GuidString; + let iModel: BriefcaseDb; + + before(() => { + HubMock.startup("SqliteChangesetReaderAndChangesetECAdaptorTest", KnownTestLocations.outputDir); + iTwinId = HubMock.iTwinId; + }); + + after(() => HubMock.shutdown()); + + beforeEach(async () => { + // Create new iModel + const adminToken = "super manager token"; + const iModelName = "PRAGMA_test"; + const iModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(iModelId); + iModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId, accessToken: adminToken }); + }); + + afterEach(() => { + // Cleanup + iModel.close(); + }); + + it("should call PRAGMA integrity_check on a new iModel and return no errors", async () => { + // Call PRAGMA integrity_check + const query = "PRAGMA integrity_check ECSQLOPTIONS ENABLE_EXPERIMENTAL_FEATURES"; + const result = iModel.createQueryReader(query, undefined, undefined); + const results = await result.toArray(); + + // Verify no errors + assert(results.length > 0, "Results should be returned from PRAGMA integrity_check"); + assert(results[0][2] === true, "'check_data_columns' check should be true"); + assert(results[1][2] === true, "'check_ec_profile' check should be true") + assert(results[2][2] === true, "'check_nav_class_ids' check should be true") + assert(results[3][2] === true, "'check_nav_ids' check should be true") + assert(results[4][2] === true, "'check_linktable_fk_class_ids' check should be true") + assert(results[5][2] === true, "'check_linktable_fk_ids' check should be true") + assert(results[6][2] === true, "'check_class_ids' check should be true") + assert(results[7][2] === true, "'check_data_schema' check should be true") + assert(results[8][2] === true, "'check_schema_load' check should be true") + }); + + it("should call PRAGMA integrity_check individual checks on a new iModel and return no errors", async () => { + // Call check_ec_profile + let query = "pragma integrity_check(check_ec_profile) options enable_experimental_features"; + let result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); + let resultArray = await result.toArray(); + expect(resultArray.length).to.equal(0); // No errors expected + + // Call check_data_schema + query = "pragma integrity_check(check_data_schema) options enable_experimental_features"; + result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); + resultArray = await result.toArray(); + expect(resultArray.length).to.equal(0); // No errors expected + + // Call check_data_columns + query = "pragma integrity_check(check_data_columns) options enable_experimental_features"; + result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); + resultArray = await result.toArray(); + expect(resultArray.length).to.equal(0); // No errors expected + + // Call check_nav_class_ids + query = "pragma integrity_check(check_nav_class_ids) options enable_experimental_features"; + result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); + resultArray = await result.toArray(); + expect(resultArray.length).to.equal(0); // No errors expected + + // Call check_nav_ids + query = "pragma integrity_check(check_nav_ids) options enable_experimental_features"; + result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); + resultArray = await result.toArray(); + expect(resultArray.length).to.equal(0); // No errors expected + + // Call check_linktable_fk_class_ids + query = "pragma integrity_check(check_linktable_fk_class_ids) options enable_experimental_features"; + result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); + resultArray = await result.toArray(); + expect(resultArray.length).to.equal(0); // No errors expected + + // Call check_linktable_fk_ids + query = "pragma integrity_check(check_linktable_fk_ids) options enable_experimental_features"; + result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); + resultArray = await result.toArray(); + expect(resultArray.length).to.equal(0); // No errors expected + + // Call check_class_ids + query = "pragma integrity_check(check_class_ids) options enable_experimental_features"; + result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); + resultArray = await result.toArray(); + expect(resultArray.length).to.equal(0); // No errors expected + + // Call check_schema_load + query = "pragma integrity_check(check_schema_load) options enable_experimental_features"; + result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); + resultArray = await result.toArray(); + expect(resultArray.length).to.equal(0); // No errors expected + }); + + it("should call PRAGMA integrity_check on a corrupted iModel and return an error", async () => { + // Insert two elements + iModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + await iModel.locks.acquireLocks({ shared: IModel.repositoryModelId }); + const txn = startTestTxn(iModel, "PRAGMA integrity check corrupted iModel"); + + const element1Id = txn.insertElement({ + classFullName: Subject.classFullName, + model: IModel.repositoryModelId, + parent: new SubjectOwnsSubjects(IModel.rootSubjectId), + code: Subject.createCode(iModel, IModel.rootSubjectId, "Subject1"), + }); + + const element2Id = txn.insertElement({ + classFullName: Subject.classFullName, + model: IModel.repositoryModelId, + parent: new SubjectOwnsSubjects(IModel.rootSubjectId), + code: Subject.createCode(iModel, IModel.rootSubjectId, "Subject2"), + }); + txn.saveChanges(); + + // Create a relationship between them + await iModel.locks.acquireLocks({ exclusive: Id64.toIdSet([element1Id, element2Id]) }); + const relationship = iModel.relationships.createInstance({ + classFullName: "BisCore:SubjectRefersToSubject", + sourceId: element1Id, + targetId: element2Id, + }); + const relationshipId = txn.insertRelationship(relationship.toJSON()); + assert.isTrue(Id64.isValidId64(relationshipId)); + txn.saveChanges(); + + // Delete one element without deleting the relationship to corrupt the iModel + const deleteResult = iModel[_nativeDb].executeSql(`DELETE FROM bis_Element WHERE Id=${element2Id}`); + expect(deleteResult).to.equal(DbResult.BE_SQLITE_OK); + txn.saveChanges(); + + // Call PRAGMA integrity_check + const query = "PRAGMA integrity_check ECSQLOPTIONS ENABLE_EXPERIMENTAL_FEATURES"; + const result = iModel.createQueryReader(query, undefined, undefined); + const results = await result.toArray(); + + // Verify error is reported + assert(results.length > 0, "Results should be returned from PRAGMA integrity_check"); + assert(results[0][2] === true, "'check_data_columns' check should be true"); + assert(results[1][2] === true, "'check_ec_profile' check should be true") + assert(results[2][2] === true, "'check_nav_class_ids' check should be true") + assert(results[3][2] === true, "'check_nav_ids' check should be true") + assert(results[4][2] === true, "'check_linktable_fk_class_ids' check should be true") + assert(results[5][2] === false, "'check_linktable_fk_ids' check should be false") // Expecting error report here + assert(results[6][2] === true, "'check_class_ids' check should be true") + assert(results[7][2] === true, "'check_data_schema' check should be true") + assert(results[8][2] === true, "'check_schema_load' check should be true") + txn.end(); + }); + + it("should call PRAGMA integrity_check(check_linktable_fk_class_ids) on a corrupted iModel and return an error", async () => { + // Insert two elements + iModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + await iModel.locks.acquireLocks({ shared: IModel.repositoryModelId }); + const txn = startTestTxn(iModel, "PRAGMA integrity check corrupted linktable fk ids"); + + const element1Id = txn.insertElement({ + classFullName: Subject.classFullName, + model: IModel.repositoryModelId, + parent: new SubjectOwnsSubjects(IModel.rootSubjectId), + code: Subject.createCode(iModel, IModel.rootSubjectId, "Subject1"), + }); + + const element2Id = txn.insertElement({ + classFullName: Subject.classFullName, + model: IModel.repositoryModelId, + parent: new SubjectOwnsSubjects(IModel.rootSubjectId), + code: Subject.createCode(iModel, IModel.rootSubjectId, "Subject2"), + }); + txn.saveChanges(); + + // Create a relationship between them + await iModel.locks.acquireLocks({ exclusive: Id64.toIdSet([element1Id, element2Id]) }); + const relationship = iModel.relationships.createInstance({ + classFullName: "BisCore:SubjectRefersToSubject", + sourceId: element1Id, + targetId: element2Id, + }); + const relationshipId = txn.insertRelationship(relationship.toJSON()); + assert.isTrue(Id64.isValidId64(relationshipId)); + txn.saveChanges(); + + // Delete one element without deleting the relationship to corrupt the iModel + const deleteResult = iModel[_nativeDb].executeSql(`DELETE FROM bis_Element WHERE Id=${element2Id}`); + expect(deleteResult).to.equal(DbResult.BE_SQLITE_OK); + txn.saveChanges(); + + // Call PRAGMA integrity_check + const query = "pragma integrity_check(check_linktable_fk_ids) options enable_experimental_features"; + const result = iModel.createQueryReader(query, undefined, { rowFormat: QueryRowFormat.UseECSqlPropertyNames }); + const resultArray = await result.toArray(); + expect(resultArray.length).to.equal(1); // 1 error report expected + expect(resultArray[0].id).to.equal("0x20000000001"); + expect(resultArray[0].key_id).to.equal("0x20000000002"); + txn.end(); + }); +}); diff --git a/docs/changehistory/NextVersion.md b/docs/changehistory/NextVersion.md index a07a924db288..5e50c31e5619 100644 --- a/docs/changehistory/NextVersion.md +++ b/docs/changehistory/NextVersion.md @@ -19,6 +19,7 @@ publish: false - [@itwin/core-frontend](#itwincore-frontend-1) - [Unified reality model iteration](#unified-reality-model-iteration) - [Backend](#backend) + - [ChangesetReader — native changeset reader](#ChangesetReader--native-changeset-reader) - [Explicit editing transactions with `EditTxn`](#explicit-editing-transactions-with-edittxn) - [What changed](#what-changed) - [Migration guidance](#migration-guidance) @@ -190,6 +191,163 @@ for (const { treeRef, name, description } of view.getRealityModelTreeRefs()) { ## Backend +### [ChangesetReader]($backend) — native changeset reader + +The new [ChangesetReader]($backend) (`@beta`) provides a lower-level, higher-fidelity replacement for the deprecated `ChangesetECAdaptor` / `SqliteChangesetReader` stack. It reads EC-typed change data natively from a changeset file, a group of changeset files, a saved transaction, or local un-pushed changes, and emits one typed [ChangeInstance]($backend) per SQLite table row. + +The companion [PartialChangeUnifier]($backend) merges the per-table partial rows back into complete EC instances that span all tables mapped to a single EC entity. + +#### Reader factory methods + +| Method | Description | +| ------ | ----------- | +| [ChangesetReader.openFile]($backend) | Read a single pushed changeset file | +| [ChangesetReader.openGroup]($backend) | Read several changeset files as one logical stream | +| [ChangesetReader.openLocalChanges]($backend) | Read pending (not yet pushed) local changes | +| [ChangesetReader.openInMemoryChanges]($backend) | Read in-memory (not yet saved) changes | +| [ChangesetReader.openTxn]($backend) | Read a single saved transaction by id | + +#### Example — inspect inserted elements from a changeset file + +```ts +import { ChangesetReader, PartialChangeUnifier, ChangeUnifierCache } from "@itwin/core-backend"; + +using reader = ChangesetReader.openFile({ + db: iModelDb, + fileName: changesetPathname, + rowOptions: { classIdsToClassNames: true }, +}); +using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + +while (reader.step()) { + pcu.appendFrom(reader); +} + +for (const instance of pcu.instances) { + if (instance.$meta.op === "Inserted") { + console.log(instance.ECInstanceId, instance.ECClassId); + } +} +``` + +#### Example — filter to a specific table and inspect raw per-row values + +```ts +using reader = ChangesetReader.openFile({ db: iModelDb, fileName: changesetPathname }); + +reader.setOpCodeFilters(new Set(["Updated"])); + +while (reader.step()) { + if (reader.tableName !== "bis_Element") + continue; + const before = reader.deleted; // pre-change snapshot + const after = reader.inserted; // post-change snapshot + if (before && after) { + console.log(`Element ${after.ECInstanceId}: model changed from ${before.Model?.Id} → ${after.Model?.Id}`); + } +} +``` + +When a row does not match an active filter it is skipped entirely — the reader automatically advances to the next row. + +[ChangesetReader.deleted]($backend) and [ChangesetReader.inserted]($backend) carry a `$meta.changeFetchedPropNames` array that lists exactly which properties were read directly from the changeset binary (vs. resolved from the live iModel), making it straightforward to determine what actually changed. + +For a full explanation of the reader–unifier pipeline, modes, row options, and filtering APIs, see [ChangesetReader](../learning/backend/ChangesetReader.md). + +#### Migrating from `ChangesetECAdaptor` + +The deprecated `ChangesetECAdaptor` / `PartialECChangeUnifier` stack is replaced by [ChangesetReader]($backend) + [PartialChangeUnifier]($backend). + +##### Basic pipeline — read and merge a changeset file + +**Before (deprecated)** + +```ts +// SqliteChangesetReader requires disableSchemaCheck: true for use with ChangesetECAdaptor +const reader = SqliteChangesetReader.openFile({ db: iModelDb, fileName, disableSchemaCheck: true }); +const adaptor = new ChangesetECAdaptor(reader); +const unifier = new PartialECChangeUnifier(iModelDb); + +while (adaptor.step()) + unifier.appendFrom(adaptor); + +for (const instance of unifier.instances) { + if (instance.$meta?.op === "Inserted") + console.log(instance.ECInstanceId); +} + +unifier[Symbol.dispose](); +adaptor[Symbol.dispose](); // also closes the underlying SqliteChangesetReader +``` + +**After (new API)** + +```ts +using reader = ChangesetReader.openFile({ db: iModelDb, fileName }); +using pcu = new PartialChangeUnifier(); + +while (reader.step()) + pcu.appendFrom(reader); + +for (const instance of pcu.instances) { + if (instance.$meta.op === "Inserted") + console.log(instance.ECInstanceId); +} +``` + +##### SQLite-backed cache for large changesets + +**Before (deprecated)** + +```ts +// The old SQLite cache reused the live iModel's connection and wrote to a [temp] table on it +using cache = ECChangeUnifierCache.createSqliteBackedCache(iModelDb); +const unifier = new PartialECChangeUnifier(iModelDb, cache); +``` + +**After (new API)** + +```ts +// The new cache opens its own private temporary SQLite database — no live iModel connection needed +using cache = ChangeUnifierCache.createSqliteBackedCache(); +using pcu = new PartialChangeUnifier(cache); +``` + +##### Filtering + +**Before (deprecated)** — fluent, class-name-based + +```ts +adaptor + .acceptOp("Inserted") + .acceptTable("bis_Element") + .acceptClass("BisCore:Element"); // expands to all derived class ids internally +``` + +**After (new API)** — `Set`-based, class-id-based + +```ts +reader.setOpCodeFilters(new Set(["Inserted"])); +reader.setTableNameFilters(new Set(["bis_Element"])); +reader.setClassNameFilters(new Set(["BisCore:Element"])); // full "SchemaName:ClassName" format, doesnot expand to any derived classes +``` + +##### Key differences + +| Concern | Old (`ChangesetECAdaptor`) | New ([ChangesetReader]($backend)) | +|---|---|---| +| Opening | `SqliteChangesetReader.openFile` + `new ChangesetECAdaptor(reader)` | `ChangesetReader.openFile` directly | +| `disableSchemaCheck` flag | Required (`true`) on `SqliteChangesetReader` | Not needed — schema check is built in | +| Merging multi-table entities | `new PartialECChangeUnifier(db)` + `unifier.appendFrom(adaptor)` | `new PartialChangeUnifier()` + `pcu.appendFrom(reader)` | +| `PartialECChangeUnifier` db arg | Required (uses live iModel connection for temp table) | Not needed (new unifier owns its temp db) | +| SQLite cache db arg | `ECChangeUnifierCache.createSqliteBackedCache(db)` — reuses iModel connection | `ChangeUnifierCache.createSqliteBackedCache()` — self-contained | +| Resource management | Manual `[Symbol.dispose]()` on each object | `using` declaration handles everything | +| Filtering by class | `acceptClass(fullName)` — automatically expands to all derived class ids, so a single class name filters the entire hierarchy | `setClassNameFilters(Set)` — exact match only; does **not** expand to derived classes; pass each class name explicitly in `"SchemaName:ClassName"` format | +| Filtering API style | Fluent (`.acceptOp(...).acceptTable(...)`) | Setter methods with `Set<>` arguments | +| `$meta` on instances | Optional (`disableMetaData` flag could suppress it) | Always present | +| Changed property tracking | Not available | `$meta.changeFetchedPropNames` lists exactly which properties came from the changeset binary | +| Null-valued properties | Null values were included as keys on instance objects (key present, value `null`) | Null values are **not** included as keys — a property absent from the instance object means its stored value was `null`; use `$meta.changeFetchedPropNames` to distinguish "not changed" from "changed to/from null" | + ### Explicit editing transactions with `EditTxn` The backend now provides [EditTxn]($backend) as the preferred way to perform writes to an iModel. This introduces an explicit transaction boundary around a unit of work: start editing, make one or more changes through the transaction, and then either save or abandon that scope. diff --git a/docs/learning/backend/ChangesetReader.md b/docs/learning/backend/ChangesetReader.md new file mode 100644 index 000000000000..7296778176b1 --- /dev/null +++ b/docs/learning/backend/ChangesetReader.md @@ -0,0 +1,336 @@ +# ChangesetReader + +[ChangesetReader]($backend) is a low-level API in `@itwin/core-backend` that reads EC-typed change data from a changeset file, a group of changeset files, an in-memory transaction, or local un-pushed changes. It is designed for use cases where you need to inspect what changed at the EC property level — for example, building audit trails, incremental projections, or custom synchronization logic. + +## Core concepts + +### How change data is stored + +A single EC entity may typically map to multiple tables or a single table. + +[ChangesetReader]($backend) reads these raw table-row changes and emits one [ChangeInstance]($backend) per row. To reconstruct a complete, merged EC instance across all tables, pipe the reader into [PartialChangeUnifier]($backend). + +### The reader–unifier pipeline + +[[include:ChangesetReader.BasicPipeline]] + +After draining the reader, `pcu.instances` yields one entry per (ECInstanceId + stage) pair, with properties merged across all contributing tables. + +### [ChangeInstance]($backend) shape + +Every instance has an `$meta` property plus the EC property bag: + +```ts +interface ChangeMeta { + op: "Inserted" | "Updated" | "Deleted"; + stage: "New" | "Old"; + tables: string[]; // SQLite tables that contributed rows + changeIndexes: number[]; // stream positions of those rows + instanceKey: string; // ECInstanceId-ECClassId key used for merging + propFilter: PropertyFilter; // PropertyFilter.All | PropertyFilter.BisCoreElement | PropertyFilter.InstanceKey + changeFetchedPropNames: string[]; // property names actually read from the change binary + rowOptions?: RowFormatOptions; // the rowOptions passed when opening the reader + isIndirectChange: boolean; // true when the change was applied indirectly +} +``` + +- `stage: "New"` → post-change value (insert or update-after) +- `stage: "Old"` → pre-change value (delete or update-before) +- An **insert** produces only a `"New"` instance. +- A **delete** produces only an `"Old"` instance. +- An **update** produces both a `"New"` and an `"Old"` instance. + +### `changeFetchedPropNames` — what actually changed + +Each [ChangeInstance]($backend) carries a `changeFetchedPropNames` array listing exactly which EC property names were fetched directly from the changeset binary (not from the live iModel). This is the ground truth for "what changed": + +[[include:ChangesetReader.ChangeFetchedPropNames]] + +#### Naming rules + +The names in `changeFetchedPropNames` follow these rules based on the property kind: + +| Property kind | Rule | Example | +|---|---|---| +| **Simple property** | EC property name as declared in the schema | `"LastMod"` | +| **Compound property** (`Point2d`, `Point3d`, navigation) — **all components changed** | Full property name | `"Origin"` | +| **Compound property** — **only some components changed** | Each changed component listed individually, using `.` as separator | `"Origin.X"`, `"Origin.Y"` (when only X and Y changed for a `Point3d` named `"Origin"`) | +| **Struct property member** | Always in `"StructProp.MemberName"` format | `"CustomStruct.Label"` | +| **Compound member inside a struct** — **all components changed** | `"StructProp.MemberName"` | `"CustomStruct.Myp2d"` | +| **Compound member inside a struct** — **only some components changed** | `"StructProp.MemberName.Component"` | `"CustomStruct.Myp2d.X"` (when only X changed for a `Point2d` property `"Myp2d"` inside struct `"CustomStruct"`) | + +> **Note:** `changeFetchedPropNames` always contains the **original EC property names** (e.g. `"LastMod"`, `"Model.Id"`, `"StructProp.X"`) regardless of how `rowOptions` are configured. Even with `useJsName: true`, `changeFetchedPropNames.includes("LastMod")` is correct — **not** `includes("lastMod")`. + +#### Null-valued properties — listed in `changeFetchedPropNames` but absent from the instance object + +A property name can appear in `changeFetchedPropNames` yet be **absent as a key** on the [ChangeInstance]($backend) object. This happens when the stored value of that property in the changeset binary was `null` (e.g. a column whose value was never set, or a compound property such as `Point3d` where all components were `null`). + +`changeFetchedPropNames` records every property that was **read from the binary** — regardless of whether the resulting value was null. The instance object, however, only carries keys for non-null values. The two are therefore complementary: + +- **`changeFetchedPropNames`** tells you which properties were part of the changeset delta (including those that changed to or from `null`). +- **Presence as a key on the instance** tells you whether the resulting value was non-null. + +A concrete example arises with a `Point3d` property. Insert the Point3d property but only partially (e.g. Y and Z without X). A subsequent update that sets all three components will record a `NULL`-to-non-null transition in the changeset binary. Reading that update changeset: + +- The `"New"` instance **has** `Position`(Point3d property) as a key (non-null value). +- The `"Old"` instance does **not** have `Position` as a key (was NULL), but `"Position"` **is** listed in `changeFetchedPropNames` for both stages because the binary recorded the full transition. + +[[include:ChangesetReader.NullValuedPoint3d]] + +**Rule of thumb:** Use `changeFetchedPropNames` to determine *which* properties changed. Use `"propName" in instance` (or optional chaining) to distinguish "changed to/from a non-null value" from "changed to/from null". + +--- + +## Disposal — always close the reader and unifier + +[ChangesetReader]($backend) and [PartialChangeUnifier]($backend) both hold native resources (file handles, SQLite connections, memory allocations) that must be released when you are done. Failing to do so will leak native handles until the garbage collector eventually runs. + +The preferred approach is the `using` declaration (TC39 Explicit Resource Management, available in TypeScript ≥ 5.2). Objects declared with `using` are automatically disposed at the end of the enclosing block — even if an exception is thrown: + +```ts +{ + using reader = ChangesetReader.openFile({ db, fileName: changeset.pathname }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + + while (reader.step()) + pcu.appendFrom(reader); + + for (const instance of pcu.instances) + console.log(instance.$meta.op, instance.ECInstanceId); + // reader and pcu are disposed here automatically +} +``` + +If you cannot use `using` (e.g. the reader must cross async boundaries or live beyond the current block), call `[Symbol.dispose]()` explicitly in a `finally` block: + +```ts +const reader = ChangesetReader.openFile({ db, fileName: changeset.pathname }); +const pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); +try { + while (reader.step()) + pcu.appendFrom(reader); + for (const instance of pcu.instances) + console.log(instance.$meta.op, instance.ECInstanceId); +} finally { + reader[Symbol.dispose](); + pcu[Symbol.dispose](); +} +``` + +> **Important:** The same rule applies to [ChangeCache]($backend) instances created via [ChangeUnifierCache.createSqliteBackedCache]($backend) — they wrap a SQLite connection and must also be disposed. + +--- + +## Opening a reader + +### [ChangesetReader.openFile]($backend) — read a single pushed changeset file + +[[include:ChangesetReader.BasicPipeline]] + +### [ChangesetReader.openGroup]($backend) — read multiple changesets as a single stream + +[ChangesetReader.openGroup]($backend) concatenates multiple changeset files into one logical stream. The unifier merges them across the whole group — an element that was inserted in changeset 1 and updated in changeset 2 surfaces as a single `"Inserted"` `"New"` instance reflecting its final state. + +[[include:ChangesetReader.OpenGroup]] + +### [ChangesetReader.openTxn]($backend) — read a saved (not yet pushed) local transaction + +[[include:ChangesetReader.OpenTxn]] + +### [ChangesetReader.openLocalChanges]($backend) — read all local un-pushed saved changes + +[[include:ChangesetReader.OpenLocalChanges]] + +Pass `includeInMemoryChanges: true` to also include the in-memory (not yet saved) changes on top: + +[[include:ChangesetReader.OpenLocalChangesIncludeInMemory]] + +### [ChangesetReader.openInMemoryChanges]($backend) — read only the in-memory (unsaved) changes + +[[include:ChangesetReader.OpenInMemoryChanges]] + +--- + +## Property filter — controlling which properties are returned + +All `open*` methods accept a `propFilter` argument that controls which properties are included: + +| Filter | Properties returned | +|---|---| +| `All` (default) | All EC properties mapped to changed tables | +| `BisCoreElement` | For classes whose base class is `BisCore:Element` only `BisCore:Element` properties mapped to changed tables are returned. If no `BisCore:Element` class property is changed currently, only `ECInstanceId` and `ECClassId` is returned. For classes whose base class is not `BisCore:Element` all EC Properties mapped to changed tables are returned.| +| `InstanceKey` | Only `ECInstanceId` and `ECClassId` | + +[[include:ChangesetReader.ModeInstanceKey]] + +The active filter is stored as a `PropertyFilter` enum value in `instance.$meta.propFilter`: + +```ts +assert.strictEqual(instance.$meta.propFilter, PropertyFilter.InstanceKey); +``` + +--- + +## Row options — formatting EC property values + +`rowOptions` are passed to the native EC row adaptor and affect how property values are formatted: + +| Option | Effect | +|---|---| +| `abbreviateBlobs: true` (or omitted) | Binary properties summarized as `{ bytes: N }` — this is the default behavior | +| `abbreviateBlobs: false` | Binary properties returned as full `Uint8Array` instead of the default `{ bytes: N }` summary | +| `classIdsToClassNames: true` | `ECClassId` and `RelECClassId` values converted from hex strings to fully-qualified names (e.g. `"BisCore.DrawingModel"`) | +| `useJsName: true` | All property keys and struct sub-keys returned in camelCase (`id`, `className`, `lastMod`, `structProp.x`, etc.). Navigation property sub-keys use `{ id, relClassName }` instead of `{ Id, RelECClassId }`. `ECClassId` and nav-prop class identifiers are automatically resolved to class names. | + +The active `rowOptions` object is stored on every instance's `$meta.rowOptions` for inspection. + +### Example — `classIdsToClassNames` + +[[include:ChangesetReader.RowOptionsClassNames]] + +### Example — `useJsName` + +[[include:ChangesetReader.UseJsName]] + +### Example — reading full binary blobs + +[[include:ChangesetReader.RowOptionsAbbreviateBlobs]] + +--- + +## `changeFetchedPropNames` always uses original EC property names + +`$meta.changeFetchedPropNames` always contains the **original EC property names** regardless of any `rowOptions` in effect. The `useJsName` row option renames the keys on the returned instance object to use JS names, but it does **not** affect the names stored in `changeFetchedPropNames`. + +This means you must always check `changeFetchedPropNames` using the schema-level EC property name, not the JS name: + +[[include:ChangesetReader.UseJsNameAndChangeFetchedPropNames]] + +In short: use `useJsName` names when reading property values off the instance, but always use the original EC schema names when querying `changeFetchedPropNames`. + +--- + +## Filtering — restricting which changes are yielded + +After opening a reader (and before the first [ChangesetReader.step]($backend) call) you can install one or more filters to narrow the change stream. When a row does not match an active filter it is **skipped entirely** — the reader automatically advances to the next row. + +Three independent filter axes are available and can be combined: + +| Priority | Method | Filters on | +|---|---|---| +| 1 | [ChangesetReader.setOpCodeFilters]($backend) | Change operation (`"Inserted"`, `"Updated"`, `"Deleted"`) | +| 2 | [ChangesetReader.setTableNameFilters]($backend) | SQLite table name of the row in proper case | +| 3 | [ChangesetReader.setClassNameFilters]($backend) | EC class name of the instance (format: `"SchemaName:ClassName"`) in proper case | + +Filters are applied in the priority order above. If the op-code filter rejects a row, the table and class-name filters are not evaluated. If the table filter rejects a row, the class-name filter is not evaluated. + +Each setter accepts a `Set<>`. Passing an empty `Set` is equivalent to calling the corresponding `clear*` method. + +### Example — only yield inserts and updates for a specific table + +[[include:ChangesetReader.FilterTable]] + +### Example — only yield changes for a known set of EC class names + +[[include:ChangesetReader.FilterClassNames]] + +### Clearing filters at runtime + +All three filters can be cleared individually without reopening the reader: + +```ts +reader.clearTableNameFilters(); +reader.clearOpCodeFilters(); +reader.clearClassNameFilters(); +``` + +--- + +## Cache strategies + +By default [PartialChangeUnifier]($backend) uses an in-memory cache (`Map`). For very large changesets that would exhaust memory, use the SQLite-backed cache instead: + +[[include:ChangesetReader.CacheStrategies]] + +--- + +## Complete worked example + +The following example imports a custom schema, inserts an element, pushes a second update, and demonstrates reading each changeset independently and then together as a group: + +[[include:ChangesetReader.WorkedExample]] + +--- + +## Out-of-sync iModel behaviour + +> **See also:** the test suite `"ChangesetReader: behaviour in case imodel is not in sync with change file or transaction being read"` in +> `core/backend/src/test/standalone/ChangesetReader.test.ts`. + +[ChangesetReader]($backend) uses the **live iModel** in two ways: to resolve `ECClassId` in case it is not part of the changeset or transaction(very common in cases of `Update` because generally only element props are updated not the class of the instance), and to fill in the non-changed components of compound property values. For compound types — `Point2d`, `Point3d`, and navigation properties - when a changeset records a change to only one component, the reader must fetch the remaining components from the live iModel to reconstruct the full value. For example, if only `X` changes in a `Point2d` property, `Y` is read from the current live database state. This means the reader's output quality depends on the current state of the iModel — specifically whether the entity being read still exists in the database at the time of reading, and whether subsequent transactions have already modified the components that were not part of the recorded changeset delta. + +Two concrete failure modes arise: + +### 1. Deleted instance — class identity lost + +**Scenario:** An element is inserted, updated, then deleted across three changesets. The update changeset (the "middle" one) is read **after** the element has already been deleted from the live iModel. + +**What happens:** As in the update changeset obviously the ECClassId was not updated for the instance, only some properties might have been updated so `ECClassId` was not part of the changeset. So when the reader resolves the `ECClassId` for a row, it performs a lookup in the live iModel's table. Because the element no longer exists, the native layer cannot determine which leaf domain class the row belongs to. It falls back to the per-table base class (`BisCore.Element` for `bis_Element`, `BisCore.GeometricElement2d` for `bis_GeometricElement2d`). The per-table instances are **not merged** into a single `TestDomain.Test2dElement` instance; instead they appear as separate entries under their base-class identities. + +```ts +// After push 2 (insert), push 3 (update), push 4 (delete): +// Reading the update changeset (push 3) AFTER the element has been deleted: + +using reader = ChangesetReader.openFile({ + db: iModelDb, // iModel has already applied the delete + fileName: updateChangeset.pathname, + propFilter: PropertyFilter.InstanceKey, + rowOptions: { classIdsToClassNames: true }, +}); +// ... +// Expected (if in sync): one merged instance with ECClassId "TestDomain.Test2dElement" +// Actual (out of sync): two separate instances: +// { ECClassId: "BisCore.Element", stage: "New" } +// { ECClassId: "BisCore.GeometricElement2d", stage: "New" } +// { ECClassId: "BisCore.Element", stage: "Old" } +// { ECClassId: "BisCore.GeometricElement2d", stage: "Old" } +``` + +**Rule of thumb:** To read a changeset reliably, the iModel's current state should be **at** the change being read or the consumers of the api must be sure that, the instance was not deleted and for the compound properties of the instance like `Point2d`, `Point3d` or `NavProps`, no change was done to them subsequently after. In practice this means: read changesets in order and keep the iModel at the point being inspected. + +### 2. Subsequent unsaved transaction pollutes property values + +**Scenario:** Three transactions occur in sequence: T1 inserts an element with `s = {x:1.5, y:2.5}`, T2 updates `s.X` to `100`, T3 updates `s.Y` to `200`. Reading T2 via [ChangesetReader.openTxn]($backend) after T3 has already been saved. + +**What happens:** Because [ChangesetReader.openTxn]($backend) fetches non-changed properties from the live database (which reflects T3), the `Old` and `New` stage values for properties *not* recorded in the changeset reflect the current live state, not the state at T2. In this example: + +```ts +// After T1 (insert s={1.5,2.5}), T2 (s.X→100), T3 (s.Y→200), reading T2: + +assert.deepEqual(elementOld.s, { X: 1.5, Y: 200 }); // Y is polluted from T3 +assert.deepEqual(elementNew.s, { X: 100, Y: 200 }); // Y is polluted from T3 +``` + +`s.Y` reads `200` in both Old and New because the live iModel already has `200` and the T2 changeset only recorded the delta for `s.X`. + +**Workaround:** Use `changeFetchedPropNames` to identify which properties were sourced directly from the changeset and are therefore trustworthy: + +```ts +// Only s.X was actually changed in T2: +expect(elementNew.$meta.changeFetchedPropNames).to.include("s.X"); +expect(elementNew.$meta.changeFetchedPropNames).not.to.include("s.Y"); + +// So only trust s.X for the before/after comparison: +const oldX = elementOld.s.X; // 1.5 — correct +const newX = elementNew.s.X; // 100 — correct +// Do NOT compare s.Y from Old vs New — it was not part of this changeset. +``` + +### Summary + +It doesnot depend on whether a change group or a changeset or a transaction is opened. It might happen when the iModel's state is not in sync with the change being read. In other words it might happen when the iModel's state is not **at** the change being read. + +| Scenario | Risk | Mitigation | +|---|---|---| +| Reading a changeset after entity is deleted | `ECClassId` resolves to the per-table base class; rows are not merged into the leaf domain class | Read changesets before the entity is deleted from the live iModel | +| Reading a historical transaction after new transactions have been saved | Property values not recorded in that txn's changeset reflect the current live state, not the historical state | Filter trustworthy properties using `$meta.changeFetchedPropNames` | diff --git a/docs/learning/backend/index.md b/docs/learning/backend/index.md index 23308235fba0..c031784565c5 100644 --- a/docs/learning/backend/index.md +++ b/docs/learning/backend/index.md @@ -54,6 +54,7 @@ These packages provide the following functions to support backend operations: - Change Summary - [Change Summary Overview](../ChangeSummaries) + - [ChangesetReader — low-level changeset reading API](./ChangesetReader.md) - Workspaces and Settings - [Workspaces and Settings overview](./WorkspacesAndSettings.md) diff --git a/example-code/snippets/src/backend/ChangesetReader-examples.test.ts b/example-code/snippets/src/backend/ChangesetReader-examples.test.ts new file mode 100644 index 000000000000..d80d3227cb53 --- /dev/null +++ b/example-code/snippets/src/backend/ChangesetReader-examples.test.ts @@ -0,0 +1,643 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ + +import { expect } from "chai"; +import * as path from "path"; +import { + BriefcaseDb, + ChangesetReader, + ChangeUnifierCache, + ChannelControl, + DrawingCategory, + EditTxn, + PartialChangeUnifier, + PropertyFilter, +} from "@itwin/core-backend"; +import { IModelTestUtils as BackendTestUtils, HubWrappers } from "@itwin/core-backend/lib/cjs/test/IModelTestUtils"; +import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; +import { Code, ColorDef, IModel, SubCategoryAppearance } from "@itwin/core-common"; +import { Id64String } from "@itwin/core-bentley"; +import { KnownTestLocations } from "./IModelTestUtils"; + + +function startTestTxn(iModel: BriefcaseDb, description = "changeset reader"): EditTxn { + const txn = new EditTxn(iModel, description); + txn.start(); + return txn; +} + +async function importSchemaStrings(txn: EditTxn, schemas: string[]): Promise { + if (txn.isActive) + txn.saveChanges(); + await txn.iModel.importSchemaStrings(schemas); +} + +describe("ChangesetReader Examples", () => { + let db: BriefcaseDb; + let txn: EditTxn; + let insertChangesetPath: string; + let updateChangesetPath: string; + let elementId: Id64String; + let modelId: Id64String; + let categoryId: Id64String; + + const adminToken = "super manager token"; + + before(async () => { + HubMock.startup("ChangesetReaderExamples", KnownTestLocations.outputDir); + const iTwinId = HubMock.iTwinId; + + const iModelId = await HubMock.createNewIModel({ + iTwinId, + iModelName: "ChangesetReaderExamples", + accessToken: adminToken, + }); + db = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId, accessToken: adminToken }); + txn = startTestTxn(db, "ChangesetReader examples setup"); + // Import a simple schema + const schema = ` + + + + bis:GraphicalElement2d + + + + `; + await importSchemaStrings(txn, [schema]); + db.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // Changeset 1 — setup (model + category) + await db.locks.acquireLocks({ shared: IModel.dictionaryId }); + [, modelId] = BackendTestUtils.createAndInsertDrawingPartitionAndModel(txn, Code.createEmpty(), true); + categoryId = DrawingCategory.insert(txn, IModel.dictionaryId, "WidgetCat", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(0,128,255)").toJSON() })); + txn.saveChanges("setup"); + await db.pushChanges({ description: "setup", accessToken: adminToken }); + + // Changeset 2 — insert widget + await db.locks.acquireLocks({ shared: modelId }); + elementId = txn.insertElement({ + classFullName: "ExSnippets:Widget", + model: modelId, + category: categoryId, + code: Code.createEmpty(), + Label: "first", // eslint-disable-line @typescript-eslint/naming-convention + Tags: ["alpha", "beta"], // eslint-disable-line @typescript-eslint/naming-convention + } as any); + txn.saveChanges("insert widget"); + await db.pushChanges({ description: "insert widget", accessToken: adminToken }); + + // Changeset 3 — update widget + await db.locks.acquireLocks({ exclusive: elementId }); + txn.updateElement({ + ...db.elements.getElementProps(elementId), + Tags: ["alpha", "beta", "gamma"], // eslint-disable-line @typescript-eslint/naming-convention + }); + txn.saveChanges("update widget"); + await db.pushChanges({ description: "update widget", accessToken: adminToken }); + + // Download changesets so we have their paths + const targetDir = path.join(KnownTestLocations.outputDir, iModelId, "changesets"); + const changesets = await HubMock.downloadChangesets({ iModelId, targetDir }); + // changesets[0] = setup, changesets[1] = insert, changesets[2] = update + insertChangesetPath = changesets[1].pathname; + updateChangesetPath = changesets[2].pathname; + }); + + after(() => { + txn.end(); + db.close(); + HubMock.shutdown(); + }); + + it("basic reader–unifier pipeline", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.BasicPipeline + using reader = ChangesetReader.openFile({ db, fileName: insertChangesetPath }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + + while (reader.step()) { + pcu.appendFrom(reader); + } + + for (const instance of pcu.instances) { + expect(instance.ECInstanceId).to.exist; + expect(instance.$meta.op).to.exist; + expect(instance.$meta.stage).to.exist; + + } + // __PUBLISH_EXTRACT_END__ + }); + + it("openGroup — multiple changesets as one stream", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.OpenGroup + // openGroup merges insert + update into a single logical stream. + // An element inserted in the first changeset and updated in the second + // surfaces as a single "Inserted" instance reflecting its final state. + using reader = ChangesetReader.openGroup({ + db, + changesetFiles: [insertChangesetPath, updateChangesetPath], + }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + + while (reader.step()) { + pcu.appendFrom(reader); + } + + for (const instance of pcu.instances) { + if (instance.$meta.stage === "New") { + // op is "Inserted" because the first appearance across the group was an insert + expect(instance.$meta.op).to.exist; + expect(instance.ECInstanceId).to.exist; + } + } + // __PUBLISH_EXTRACT_END__ + }); + + it("filter by table and op-code", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.FilterTable + using reader = ChangesetReader.openFile({ db, fileName: insertChangesetPath }); + + reader.setTableNameFilters(new Set(["bis_Element"])); + reader.setOpCodeFilters(new Set(["Inserted", "Updated"])); + + while (reader.step()) { + if (reader.inserted) { + // Only bis_Element rows with op Inserted or Updated reach here. + // Rows that do not match the active filters are skipped entirely — + // the reader automatically advances to the next row. + expect(reader.inserted.ECInstanceId).to.exist; + expect(reader.inserted.$meta.op).to.exist; + } + // reader.deleted is always undefined because "Deleted" was not included. + } + // __PUBLISH_EXTRACT_END__ + }); + + it("filter by EC class name", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.FilterClassNames + // Restrict the stream to a known set of EC class names (full "SchemaName:ClassName" format). + // Rows for any other class are skipped entirely. + const classNames = new Set(["ExSnippets:Widget"]); + + using reader = ChangesetReader.openFile({ db, fileName: insertChangesetPath }); + reader.setClassNameFilters(classNames); + + while (reader.step()) { + if (reader.inserted) + expect(reader.inserted.ECClassId).to.exist; + } + // __PUBLISH_EXTRACT_END__ + }); + + it("rowOption classIdsToClassNames", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.RowOptionsClassNames + using reader = ChangesetReader.openFile({ + db, + fileName: insertChangesetPath, + rowOptions: { classIdsToClassNames: true }, + }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + for (const instance of pcu.instances) { + // ECClassId is now a fully-qualified name instead of a hex string + expect(instance.ECClassId).to.exist; // e.g. "ExSnippets.Widget" + // Navigation property class identifiers are also resolved: + // instance.Category → { Id: "0x...", RelECClassId: "BisCore.GeometricElement2dIsInCategory" } + } + // __PUBLISH_EXTRACT_END__ + }); + + it("rowOption useJsName and changeFetchedPropNames uses original EC names", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.UseJsNameAndChangeFetchedPropNames + using reader = ChangesetReader.openFile({ + db, + fileName: insertChangesetPath, + rowOptions: { useJsName: true }, // property keys are camelCase + }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + for (const instance of pcu.instances) { + // Property keys on the instance object use JS names (camelCase): + expect(instance.id).to.exist; // ECInstanceId → id + expect(instance.className).to.exist; // ECClassId → className (resolved) + + // changeFetchedPropNames always stores the original EC schema names, + // regardless of useJsName. Always query it with the schema-level name: + const changed = instance.$meta.changeFetchedPropNames; + if (changed.includes("Tags")) // ✅ original EC name — correct + expect(instance.tags).to.exist; + // changed.includes("tags") // ❌ never true — JS name is wrong here + } + // __PUBLISH_EXTRACT_END__ + }); + + it("rowOption abbreviateBlobs false — full binary values", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.RowOptionsAbbreviateBlobs + using reader = ChangesetReader.openFile({ + db, + fileName: insertChangesetPath, + rowOptions: { abbreviateBlobs: false }, // return full Uint8Array instead of { bytes: N } + }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + for (const instance of pcu.instances) { + // Binary properties are now returned as full Uint8Array values + if (instance.GeometryStream instanceof Uint8Array) + expect(instance.GeometryStream.byteLength).to.be.greaterThan(0); + } + // __PUBLISH_EXTRACT_END__ + }); + + it("changeFetchedPropNames — trusting only what the changeset recorded", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.ChangeFetchedPropNames + using reader = ChangesetReader.openFile({ db, fileName: updateChangesetPath }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + for (const instance of pcu.instances) { + if (instance.ECInstanceId !== elementId) continue; + + const changedProps = instance.$meta.changeFetchedPropNames; // string[] + // Only properties listed here were read directly from the changeset binary. + // Other properties on the instance may reflect the current live-iModel state. + if (changedProps.includes("Tags")) + expect(instance.Tags).to.exist; + } + // __PUBLISH_EXTRACT_END__ + + // Basic sanity check + const instances = Array.from(pcu.instances); + const widgetNew = instances.find( + (i) => i.ECInstanceId === elementId && i.$meta.stage === "New", + ); + expect(widgetNew).to.exist; + expect(widgetNew!.$meta.changeFetchedPropNames).to.include("Tags"); + }); + + it("openTxn — read a saved transaction", () => { + const txnProps = db.txns.getLastSavedTxnProps(); + if (!txnProps) return; // no saved txns available in current db state + const txnId = txnProps.id; + + // __PUBLISH_EXTRACT_START__ ChangesetReader.OpenTxn + using reader = ChangesetReader.openTxn({ db, txnId }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + const instances = Array.from(pcu.instances); + const changed = instances.find((i) => i.$meta.stage === "New"); + // __PUBLISH_EXTRACT_END__ + void changed; + }); + + it("useJsName row option", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.UseJsName + using reader = ChangesetReader.openFile({ + db, + fileName: insertChangesetPath, + rowOptions: { useJsName: true }, + }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + for (const instance of pcu.instances) { + // Property keys on the instance use camelCase JS names: + expect(instance.id).to.exist; // ECInstanceId → id + expect(instance.className).to.exist; // ECClassId → className (resolved to full class name) + // Navigation property sub-keys also use camelCase: + // instance.category → { id: "0x...", relClassName: "BisCore.GeometricElement2dIsInCategory" } + // Array property names are also camelCased: + // instance.tags → ["alpha", "beta"] (Tags → tags) + } + // __PUBLISH_EXTRACT_END__ + }); + + it("Instance_Key mode — only ECInstanceId and ECClassId", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.ModeInstanceKey + using reader = ChangesetReader.openFile({ + db, + fileName: insertChangesetPath, + propFilter: PropertyFilter.InstanceKey, + rowOptions: { classIdsToClassNames: true }, + }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + for (const instance of pcu.instances) { + // Only ECInstanceId and ECClassId are populated — all other properties are absent + expect(instance.$meta.op).to.exist; + expect(instance.ECInstanceId).to.exist; + expect(instance.ECClassId).to.exist; + } + // __PUBLISH_EXTRACT_END__ + }); + + it("SQLite-backed cache for large changesets", () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.CacheStrategies + using cache = ChangeUnifierCache.createSqliteBackedCache(); + using pcu = new PartialChangeUnifier(cache); + using reader = ChangesetReader.openFile({ db, fileName: insertChangesetPath }); + while (reader.step()) pcu.appendFrom(reader); + for (const instance of pcu.instances) { + expect(instance.ECInstanceId).to.exist; + expect(instance.$meta.op).to.exist; + } + // __PUBLISH_EXTRACT_END__ + }); + + it("openLocalChanges — read local saved changes", async () => { + await db.locks.acquireLocks({ shared: modelId }); + elementId = txn.insertElement({ + classFullName: "ExSnippets:Widget", + model: modelId, + category: categoryId, + code: Code.createEmpty(), + Label: "second", // eslint-disable-line @typescript-eslint/naming-convention + Tags: ["alpha", "beta"], // eslint-disable-line @typescript-eslint/naming-convention + } as any); + txn.saveChanges("insert second widget"); + // __PUBLISH_EXTRACT_START__ ChangesetReader.OpenLocalChanges + using reader = ChangesetReader.openLocalChanges({ db }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + for (const instance of pcu.instances) { + expect(instance.ECInstanceId).to.exist; + expect(instance.$meta.op).to.exist; + } + // __PUBLISH_EXTRACT_END__ + + // __PUBLISH_EXTRACT_START__ ChangesetReader.OpenLocalChangesIncludeInMemory + // Pass includeInMemoryChanges: true to also include the in-memory (not yet saved) changes: + using reader2 = ChangesetReader.openLocalChanges({ db, includeInMemoryChanges: true }); + // __PUBLISH_EXTRACT_END__ + void reader2; + }); + + it("openInMemoryChanges — read in-memory changes", async () => { + await db.locks.acquireLocks({ shared: modelId }); + elementId = txn.insertElement({ + classFullName: "ExSnippets:Widget", + model: modelId, + category: categoryId, + code: Code.createEmpty(), + Label: "third", // eslint-disable-line @typescript-eslint/naming-convention + Tags: ["alpha", "beta"], // eslint-disable-line @typescript-eslint/naming-convention + } as any); + // __PUBLISH_EXTRACT_START__ ChangesetReader.OpenInMemoryChanges + using reader = ChangesetReader.openInMemoryChanges({ db }); + // __PUBLISH_EXTRACT_END__ + void reader; + }); +}); + +describe("ChangesetReader Examples — complete worked example", () => { + const adminToken = "super manager token"; + + before(() => HubMock.startup("ChangesetReaderWorkedExample", KnownTestLocations.outputDir)); + after(() => HubMock.shutdown()); + + it("complete worked example", async () => { + // __PUBLISH_EXTRACT_START__ ChangesetReader.WorkedExample + const iTwinId = HubMock.iTwinId; + + // 1. Create and open a briefcase + const iModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "demo", accessToken: adminToken }); + const db = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId, accessToken: adminToken }); + const txn = startTestTxn(db, "ChangesetReader worked example setup"); + + // 2. Import a schema with a binary and a string-array property + const schema = ` + + + + bis:GraphicalElement2d + + + + `; + await importSchemaStrings(txn, [schema]); + db.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // 3. Push changeset 1 — model and category setup + await db.locks.acquireLocks({ shared: IModel.dictionaryId }); + const [, modelId] = BackendTestUtils.createAndInsertDrawingPartitionAndModel(txn, Code.createEmpty(), true); + const catId = DrawingCategory.insert(txn, IModel.dictionaryId, "DemoCat", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(0,0,255)").toJSON() })); + txn.saveChanges("setup"); + await db.pushChanges({ description: "setup", accessToken: adminToken }); + + // 4. Push changeset 2 — insert widget + await db.locks.acquireLocks({ shared: modelId }); + const elementId: Id64String = txn.insertElement({ + classFullName: "Demo:Widget", + model: modelId, + category: catId, + code: Code.createEmpty(), + Payload: new Uint8Array([0x01, 0x02, 0x03]), // eslint-disable-line @typescript-eslint/naming-convention + Tags: ["alpha", "beta"], // eslint-disable-line @typescript-eslint/naming-convention + } as any); + txn.saveChanges("insert widget"); + await db.pushChanges({ description: "insert widget", accessToken: adminToken }); + + // 5. Push changeset 3 — update widget + await db.locks.acquireLocks({ exclusive: elementId }); + txn.updateElement({ + ...db.elements.getElementProps(elementId), + Tags: ["alpha", "beta", "gamma"], // eslint-disable-line @typescript-eslint/naming-convention + }); + txn.saveChanges("update widget"); + await db.pushChanges({ description: "update widget", accessToken: adminToken }); + + // 6. Download the pushed changesets + const targetDir = path.join(KnownTestLocations.outputDir, iModelId, "changesets"); + const changesets = await HubMock.downloadChangesets({ iModelId, targetDir }); + const [, insertCs, updateCs] = changesets; // [setup, insert, update] + + // 7. Read the insert changeset individually + { + using reader = ChangesetReader.openFile({ + db, + fileName: insertCs.pathname, + rowOptions: { abbreviateBlobs: false }, + }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + const elem = Array.from(pcu.instances).find( + (i) => i.ECInstanceId === elementId && i.$meta.stage === "New", + ); + // elem.$meta.op === "Inserted" + // elem.Payload instanceof Uint8Array → [1, 2, 3] + // elem.Tags → ["alpha", "beta"] + // elem.$meta.changeFetchedPropNames.includes("Tags") → true + expect(elem?.$meta.op).to.exist; + expect(elem?.Tags).to.exist; + } + + // 8. Read the update changeset individually + { + using reader = ChangesetReader.openFile({ + db, + fileName: updateCs.pathname, + rowOptions: { abbreviateBlobs: false }, + }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + const instances = Array.from(pcu.instances); + const elemNew = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "New"); + const elemOld = instances.find((i) => i.ECInstanceId === elementId && i.$meta.stage === "Old"); + // elemNew.Tags → ["alpha", "beta", "gamma"] + // elemOld.Tags → ["alpha", "beta"] + // elemNew.$meta.changeFetchedPropNames.includes("Tags") → true + expect(elemNew?.Tags).to.exist; + expect(elemOld?.Tags).to.exist; + } + + // 9. Read both changesets as a group + { + using reader = ChangesetReader.openGroup({ + db, + changesetFiles: [insertCs.pathname, updateCs.pathname], + rowOptions: { abbreviateBlobs: false }, + }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + const elem = Array.from(pcu.instances).find( + (i) => i.ECInstanceId === elementId && i.$meta.stage === "New", + ); + // op is "Inserted" because the first appearance across the group was an insert. + // Final Tags value reflects the update (["alpha","beta","gamma"]). + // tables accumulated: ["bis_Element", "bis_GeometricElement2d"] + expect(elem?.$meta.op).to.exist; + expect(elem?.Tags).to.exist; + } + + txn.end(); + db.close(); + // __PUBLISH_EXTRACT_END__ + }); +}); + +describe("ChangesetReader Examples — null-valued Point3d properties", () => { + before(() => HubMock.startup("ChangesetReaderNullProp", KnownTestLocations.outputDir)); + after(() => HubMock.shutdown()); + + it("Point3d stored as NULL when only partial components are given", async () => { + const adminToken2 = "super manager token"; + const iTwinId = HubMock.iTwinId; + + const iModelId = await HubMock.createNewIModel({ iTwinId, iModelName: "nullPropDemo", accessToken: adminToken2 }); + const db = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId, accessToken: adminToken2 }); + const txn = startTestTxn(db, "null-prop example"); + + // __PUBLISH_EXTRACT_START__ ChangesetReader.NullValuedPoint3d + // A Point3d column is stored as NULL whenever any component of the value is not explicitly + // provided. In the example below, X is omitted, so the entire Position column remains NULL + // in the database — the insertion "did not happen" as far as Position is concerned. + const schema = ` + + + + bis:GraphicalElement2d + + + `; + await importSchemaStrings(txn, [schema]); + db.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + await db.locks.acquireLocks({ shared: IModel.dictionaryId }); + const [, modelId] = BackendTestUtils.createAndInsertDrawingPartitionAndModel(txn, Code.createEmpty(), true); + const catId = DrawingCategory.insert(txn, IModel.dictionaryId, "MarkerCat", + new SubCategoryAppearance({ color: ColorDef.fromString("rgb(0,0,255)").toJSON() })); + txn.saveChanges("setup"); + await db.pushChanges({ description: "setup", accessToken: adminToken2 }); + + + await db.locks.acquireLocks({ shared: modelId }); + const markerId: Id64String = txn.insertElement({ + classFullName: "NullPropDemo:Marker", + model: modelId, + category: catId, + code: Code.createEmpty(), + // eslint-disable-next-line @typescript-eslint/naming-convention + Position: { y: 2.5, z: 3.7 }, // X omitted — stored as NULL + } as any); + txn.saveChanges("insert marker"); + await db.pushChanges({ description: "insert marker", accessToken: adminToken2 }); + + const targetDir = path.join(KnownTestLocations.outputDir, iModelId, "changesets"); + let changesets = await HubMock.downloadChangesets({ iModelId, targetDir }); + const insertCs = changesets[1]; // [setup, insert] + + // Reading the insert changeset: + // "Position" appears in changeFetchedPropNames — it was read from the changeset binary. + // But it is NOT a key on the instance because the stored value was NULL. + { + using reader = ChangesetReader.openFile({ db, fileName: insertCs.pathname }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + const markerNew = Array.from(pcu.instances).find( + (i) => i.ECInstanceId === markerId && i.$meta.stage === "New", + ); + + expect(markerNew!.$meta.changeFetchedPropNames.includes("Position")).to.be.true; // true — binary had the column + expect("Position" in markerNew!).to.be.false; // false — value was NULL + expect(markerNew!.Position).to.be.undefined; // undefined + } + + // Update the element with all three components explicitly set. + // Now the column transitions from NULL to a fully-specified Point3d value. + await db.locks.acquireLocks({ exclusive: markerId }); + txn.updateElement({ + ...db.elements.getElementProps(markerId), + // eslint-disable-next-line @typescript-eslint/naming-convention + Position: { x: 1.0, y: 9.9, z: 7.7 }, // all components provided — column becomes non-null + }); + txn.saveChanges("update marker"); + await db.pushChanges({ description: "update marker", accessToken: adminToken2 }); + + changesets = await HubMock.downloadChangesets({ iModelId, targetDir }); + const updateCs = changesets[2]; // [setup, insert, update] + + // Reading the update changeset: + // markerNew — Position IS a key (new value is non-null: { X:1, Y:9.9, Z:7.7 }) + // markerOld — Position is NOT a key (old value was NULL), but IS in changeFetchedPropNames + // because the changeset binary recorded the NULL-to-non-null transition. + { + using reader = ChangesetReader.openFile({ db, fileName: updateCs.pathname }); + using pcu = new PartialChangeUnifier(ChangeUnifierCache.createInMemoryCache()); + while (reader.step()) pcu.appendFrom(reader); + + const instances = Array.from(pcu.instances); + const markerNew = instances.find((i) => i.ECInstanceId === markerId && i.$meta.stage === "New"); + const markerOld = instances.find((i) => i.ECInstanceId === markerId && i.$meta.stage === "Old"); + + // New state: fully specified — Position IS a key. + expect("Position" in markerNew!).to.be.true; // true + // eslint-disable-next-line @typescript-eslint/naming-convention + expect(markerNew!.Position).to.deep.equal({ X: 1, Y: 9.9, Z: 7.7 }); // { X: 1, Y: 9.9, Z: 7.7 } + + // Old state: was NULL — Position is NOT a key. + expect("Position" in markerOld!).to.be.false; // false + expect(markerOld!.Position).to.be.undefined; // undefined + + // Both stages list "Position" in changeFetchedPropNames. + expect(markerNew!.$meta.changeFetchedPropNames.includes("Position")).to.be.true; // true + expect(markerOld!.$meta.changeFetchedPropNames.includes("Position")).to.be.true; // true + // → Position changed from null to {"X":1,"Y":9.9,"Z":7.7} + } + // __PUBLISH_EXTRACT_END__ + + txn.end(); + db.close(); + }); +}); diff --git a/full-stack-tests/backend/package.json b/full-stack-tests/backend/package.json index 1182f0a44595..42965f80041a 100644 --- a/full-stack-tests/backend/package.json +++ b/full-stack-tests/backend/package.json @@ -29,7 +29,8 @@ "perftest:readQueryPerformance": "npm run -s perftest:pre && mocha --timeout=999999999 \"./lib/cjs/perftest/ReadQueryPerf.test.js\"", "perftest:metadataPerformance": "npm run -s perftest:pre && mocha --timeout=999999999 --grep PerformanceElementGetMetadata \"./lib/cjs/perftest/ElementCRUD.test.js\"", "perftest:schemaContextPerformance": "npm run -s perftest:pre && mocha --timeout=999999999 \"./lib/cjs/perftest/SchemaContextIModelDb.test.js\"", - "perftest:changesetPerformance": "npm run -s perftest:pre && mocha --timeout=999999999 --grep ChangesetReaderAPI \"./lib/cjs/perftest/ChangesetReader.test.js\"", + "perftest:sqliteChangesetReaderAndChangesetECAdaptorPerformance": "npm run -s perftest:pre && mocha --timeout=999999999 --grep SqliteChangesetReaderAndChangesetECAdaptorAPI \"./lib/cjs/perftest/SQliteChangesetReaderAndChangesetECAdaptor.test.js\"", + "perftest:changesetReaderPerformance": "npm run -s perftest:pre && mocha --timeout=999999999 --grep ChangesetReaderAPI \"./lib/cjs/perftest/ChangesetReader.test.js\"", "perftest:deleteRelationshipInstances": "npm run -s perftest:pre && mocha --timeout=999999999 \"./lib/cjs/perftest/DeleteRelationshipInstances.test.js\"", "perftest:incrementalLoading": "npm run -s perftest:pre && mocha --timeout=999999999 \"./lib/cjs/perftest/IncrementalLoading.test.js\"", "perftest:specific": "npm run -s perftest:pre && mocha --timeout=999999999" diff --git a/full-stack-tests/backend/src/perftest/ChangesetReader.test.ts b/full-stack-tests/backend/src/perftest/ChangesetReader.test.ts index a35838ffd41f..edccc1942fe8 100644 --- a/full-stack-tests/backend/src/perftest/ChangesetReader.test.ts +++ b/full-stack-tests/backend/src/perftest/ChangesetReader.test.ts @@ -3,14 +3,14 @@ * See LICENSE.md in the project root for license terms and full copyright notice. *--------------------------------------------------------------------------------------------*/ -import { ChangesetECAdaptor, ChannelControl, DrawingCategory, ECChangeUnifierCache, IModelHost, PartialECChangeUnifier, SqliteChangesetReader } from "@itwin/core-backend"; +import { ChangesetReader, ChangeUnifierCache, ChannelControl, DrawingCategory, IModelHost, PartialChangeUnifier } from "@itwin/core-backend"; import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; import { HubWrappers, IModelTestUtils, withEditTxn } from "@itwin/core-backend/lib/cjs/test/index"; import { KnownTestLocations } from "@itwin/core-backend/lib/cjs/test/KnownTestLocations"; import { GuidString, Id64, StopWatch } from "@itwin/core-bentley"; import { Code, IModel, SubCategoryAppearance } from "@itwin/core-common"; import { Reporter } from "@itwin/perf-tools"; -import { assert, expect } from "chai"; +import { assert } from "chai"; import * as path from "node:path"; describe("ChangesetReaderAPI", async () => { @@ -22,17 +22,18 @@ describe("ChangesetReaderAPI", async () => { HubMock.startup("ChangesetReaderTest", KnownTestLocations.outputDir); iTwinId = HubMock.iTwinId; }); + after(async () => { const csvPath = path.join(KnownTestLocations.outputDir, "PerformanceResultsChangesetReader.csv"); // eslint-disable-next-line no-console console.log(`Performance results are stored in ${csvPath}`); reporter.exportCSV(csvPath); - HubMock.shutdown() + HubMock.shutdown(); await IModelHost.shutdown(); }); - it("Large Changeset Performance - InMemoryCache", async () => { + it("Large Changeset Performance", async () => { const adminToken = "super manager token"; const iModelName = "LargeChangesetTest"; const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); @@ -40,7 +41,6 @@ describe("ChangesetReaderAPI", async () => { const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); - // Import schema const schema = ` @@ -52,7 +52,6 @@ describe("ChangesetReaderAPI", async () => { await rwIModel.importSchemaStrings([schema]); rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - // Create drawing model and category const codeProps = Code.createEmpty(); codeProps.value = "DrawingModel"; await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); @@ -63,13 +62,9 @@ describe("ChangesetReaderAPI", async () => { await rwIModel.pushChanges({ description: "Initial Test Data Setup", accessToken: adminToken }); - // Create changesets with different number of inserts const testCases = [ { testCaseNum: 1, numElements: 1000 }, { testCaseNum: 2, numElements: 10000 }, - { testCaseNum: 3, numElements: 100000 }, - // { testCaseNum: 4, numElements: 1000000 }, - // { testCaseNum: 5, numElements: 10000000 }, ]; const elementPropsTemplate = { @@ -83,10 +78,7 @@ describe("ChangesetReaderAPI", async () => { await rwIModel.locks.acquireLocks({ shared: drawingModelId }); withEditTxn(rwIModel, (txn) => { for (let i = 0; i < testCase.numElements; i++) { - const elementProps = { - ...elementPropsTemplate, - name: `Element_${testCase.numElements}_${i}`, - }; + const elementProps = { ...elementPropsTemplate, name: `Element_${testCase.numElements}_${i}` }; assert.isTrue(Id64.isValidId64(txn.insertElement(elementProps)), `Failed to insert element ${elementProps.name}`); } }); @@ -94,187 +86,34 @@ describe("ChangesetReaderAPI", async () => { } const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); - const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); // Get all changesets - - for (const testCase of testCases) { - // Open the changesets one by one and read the changes - const reader = SqliteChangesetReader.openFile({ fileName: changesets[testCase.testCaseNum].pathname, db: rwIModel, disableSchemaCheck: true }); - const adaptor = new ChangesetECAdaptor(reader); - const unifier = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); - adaptor.acceptOp("Inserted"); - assert.equal(unifier.getInstanceCount(), 0, "Unifier should be empty before any changes are applied"); - - const watch = new StopWatch(); - watch.start(); - - while (adaptor.step()) - unifier.appendFrom(adaptor); - - watch.stop(); - - assert.equal(unifier.getInstanceCount(), testCase.numElements, "Number of instances should match the number of inserted elements til now"); - reporter.addEntry("ChangesetReaderAPI", "Unifier-InMemoryCache", "Execution time (seconds)", watch.elapsedSeconds, { iModelId: rwIModelId, changesetId: changesets[testCase.testCaseNum].id, changesetInserts: testCase.numElements }); - } - rwIModel.close(); - }); - - it("Large Changeset Performance - SqliteBackedCache", async () => { - const adminToken = "super manager token"; - const iModelName = "LargeChangesetTest"; - const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(rwIModelId); - - const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); - - // Import schema - const schema = ` - - - - bis:GraphicalElement2d - - - `; - await rwIModel.importSchemaStrings([schema]); - rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); - - // Create drawing model and category - const codeProps = Code.createEmpty(); - codeProps.value = "DrawingModel"; - await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); - const [, drawingModelId] = withEditTxn(rwIModel, (txn) => IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true)); - let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); - if (undefined === drawingCategoryId) - drawingCategoryId = withEditTxn(rwIModel, (txn) => DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance())); + const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); - await rwIModel.pushChanges({ description: "Initial Test Data Setup", accessToken: adminToken }); - - // Create changesets with different number of inserts - const testCases = [ - { testCaseNum: 1, numElements: 1000 }, - { testCaseNum: 2, numElements: 10000 }, - { testCaseNum: 3, numElements: 100000 }, - // { testCaseNum: 4, numElements: 1000000 }, - // { testCaseNum: 5, numElements: 10000000 }, + const cacheConfigs = [ + { label: "Unifier-InMemoryCache", createCache: () => ChangeUnifierCache.createInMemoryCache() }, + { label: "Unifier-SqliteBackedCache", createCache: () => ChangeUnifierCache.createSqliteBackedCache() }, ]; - const elementPropsTemplate = { - classFullName: "TestDomain:TestElement", - model: drawingModelId, - category: drawingCategoryId, - code: Code.createEmpty(), - }; - - for (const testCase of testCases) { - await rwIModel.locks.acquireLocks({ shared: drawingModelId }); - withEditTxn(rwIModel, (txn) => { - for (let i = 0; i < testCase.numElements; i++) { - const elementProps = { - ...elementPropsTemplate, - name: `Element_${testCase.numElements}_${i}`, - }; - assert.isTrue(Id64.isValidId64(txn.insertElement(elementProps)), `Failed to insert element ${elementProps.name}`); - } - }); - await rwIModel.pushChanges({ description: `Changeset with ${testCase.numElements} inserts`, accessToken: adminToken }); - } - - const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); - const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); // Get all changesets - - for (const testCase of testCases) { - // Open the changesets one by one and read the changes - const reader = SqliteChangesetReader.openFile({ fileName: changesets[testCase.testCaseNum].pathname, db: rwIModel, disableSchemaCheck: true }); - const adaptor = new ChangesetECAdaptor(reader); - const unifier = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(reader.db)); - adaptor.acceptOp("Inserted"); - assert.equal(unifier.getInstanceCount(), 0, "Unifier should be empty before any changes are applied"); + for (const cacheConfig of cacheConfigs) { + for (const testCase of testCases) { + using reader = ChangesetReader.openFile({ db: rwIModel, fileName: changesets[testCase.testCaseNum].pathname }); + using cache = cacheConfig.createCache(); + using pcu = new PartialChangeUnifier(cache); + reader.setOpCodeFilters(new Set(["Inserted"])); + assert.equal(pcu.instanceCount, 0, "Unifier should be empty before any changes are applied"); - const watch = new StopWatch(); - watch.start(); + const watch = new StopWatch(); + watch.start(); - while (adaptor.step()) - unifier.appendFrom(adaptor); + while (reader.step()) + pcu.appendFrom(reader); - watch.stop(); + watch.stop(); - assert.equal(unifier.getInstanceCount(), testCase.numElements, "Number of instances should match the number of inserted elements til now"); - reporter.addEntry("ChangesetReaderAPI", "Unifier-SqliteBackedCache", "Execution time (seconds)", watch.elapsedSeconds, { iModelId: rwIModelId, changesetId: changesets[testCase.testCaseNum].id, changesetInserts: testCase.numElements }); - } - rwIModel.close(); - }); - - it("Track changeset health stats", async () => { - const adminToken = "super manager token"; - const iModelName = "LargeChangesetPullTest"; - const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); - assert.isNotEmpty(rwIModelId); - - // Open two briefcases for the same iModel - const [firstBriefcase, secondBriefcase] = await Promise.all([ - HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }), - HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }) - ]); - - // Import schema - await firstBriefcase.importSchemaStrings([` - - - - bis:GraphicalElement2d - - `]); - firstBriefcase.channels.addAllowedChannel(ChannelControl.sharedChannelName); - - // Create drawing model and category - const codeProps = Code.createEmpty(); - codeProps.value = "DrawingModel"; - await firstBriefcase.locks.acquireLocks({ shared: IModel.dictionaryId }); - const [, drawingModelId] = withEditTxn(firstBriefcase, (txn) => IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true)); - let drawingCategoryId = DrawingCategory.queryCategoryIdByName(firstBriefcase, IModel.dictionaryId, "MyDrawingCategory"); - if (undefined === drawingCategoryId) - drawingCategoryId = withEditTxn(firstBriefcase, (txn) => DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance())); - - await Promise.all([firstBriefcase.enableChangesetStatTracking(), secondBriefcase.enableChangesetStatTracking()]); - - await firstBriefcase.pushChanges({ description: "Initial Test Data Setup", accessToken: adminToken }); - - // Insert a large number of elements and push as a single changeset - const numElements = 100000; - const elementPropsTemplate = { - classFullName: "TestSchema:TestElement", - model: drawingModelId, - category: drawingCategoryId, - code: Code.createEmpty(), - }; - await firstBriefcase.locks.acquireLocks({ shared: drawingModelId }); - withEditTxn(firstBriefcase, (txn) => { - for (let i = 0; i < numElements; i++) { - const elementProps = { - ...elementPropsTemplate, - name: `Element_${i}`, - }; - assert.isTrue(Id64.isValidId64(txn.insertElement(elementProps))); + assert.equal(pcu.instanceCount, testCase.numElements, "Number of instances should match the number of inserted elements"); + reporter.addEntry("ChangesetReaderAPI", cacheConfig.label, "Execution time (seconds)", watch.elapsedSeconds, { iModelId: rwIModelId, changesetId: changesets[testCase.testCaseNum].id, changesetInserts: testCase.numElements }); } - }); - await firstBriefcase.pushChanges({ description: `Large changeset with ${numElements} inserts`, accessToken: adminToken }); - await secondBriefcase.pullChanges({ accessToken: adminToken }); - - const firstBriefcaseChangesets = await firstBriefcase.getAllChangesetHealthData(); - assert.equal(firstBriefcaseChangesets.length, 0); // No new changes to be pulled - const secondBriefcaseChangesets = await secondBriefcase.getAllChangesetHealthData(); - assert.equal(secondBriefcaseChangesets.length, 2); // Schema import followed by element insert - - const secondBriefcaseChangeset2 = secondBriefcaseChangesets[1]; - expect(secondBriefcaseChangeset2.insertedRows).to.be.eql(numElements * 2); // 100k in bis_Element + 100k in bis_GeometricElement2d - expect(secondBriefcaseChangeset2.updatedRows).to.be.eql(2); - expect(secondBriefcaseChangeset2.totalElapsedMs).to.be.greaterThan(0); // Ensure it took some time - expect(secondBriefcaseChangeset2.perStatementStats.length).to.be.eql(4); - - reporter.addEntry("ChangesetReaderAPI", "ChangesetHealthStats", "Execution time (ms)", secondBriefcaseChangeset2.totalElapsedMs, { changesetId: secondBriefcaseChangeset2.changesetId, statementsExecuted: secondBriefcaseChangeset2.perStatementStats.length }); + } - // Cleanup - firstBriefcase.close(); - secondBriefcase.close(); + rwIModel.close(); }); }); diff --git a/full-stack-tests/backend/src/perftest/SQliteChangesetReaderAndChangesetECAdaptor.test.ts b/full-stack-tests/backend/src/perftest/SQliteChangesetReaderAndChangesetECAdaptor.test.ts new file mode 100644 index 000000000000..0d6ad4a2f7e8 --- /dev/null +++ b/full-stack-tests/backend/src/perftest/SQliteChangesetReaderAndChangesetECAdaptor.test.ts @@ -0,0 +1,283 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ + +import { ChangesetECAdaptor, ChannelControl, DrawingCategory, ECChangeUnifierCache, IModelHost, PartialECChangeUnifier, SqliteChangesetReader } from "@itwin/core-backend"; +import { HubMock } from "@itwin/core-backend/lib/cjs/internal/HubMock"; +import { HubWrappers, IModelTestUtils, withEditTxn } from "@itwin/core-backend/lib/cjs/test/index"; +import { KnownTestLocations } from "@itwin/core-backend/lib/cjs/test/KnownTestLocations"; +import { GuidString, Id64, StopWatch } from "@itwin/core-bentley"; +import { Code, IModel, SubCategoryAppearance } from "@itwin/core-common"; +import { Reporter } from "@itwin/perf-tools"; +import { assert, expect } from "chai"; +import * as path from "node:path"; + +/* eslint-disable @typescript-eslint/no-deprecated */ // This test file will be removed subsequently, so we can allow usage of deprecated APIs within it. +// This test file also contains tests outside of the ChangesetECAdaptor, so be cautious while removing it, remove just the tests which are related to ChangesetECAdaptor. + +describe("SqliteChangesetReaderAndChangesetECAdaptorAPI", async () => { + let iTwinId: GuidString; + const reporter = new Reporter(); + + before(async () => { + await IModelHost.startup(); + HubMock.startup("SqliteChangesetReaderAndChangesetECAdaptorTest", KnownTestLocations.outputDir); + iTwinId = HubMock.iTwinId; + }); + after(async () => { + const csvPath = path.join(KnownTestLocations.outputDir, "PerformanceResultsSqliteChangesetReaderAndChangesetECAdaptor.csv"); + // eslint-disable-next-line no-console + console.log(`Performance results are stored in ${csvPath}`); + reporter.exportCSV(csvPath); + + HubMock.shutdown() + await IModelHost.shutdown(); + }); + + it("Large Changeset Performance - InMemoryCache", async () => { + const adminToken = "super manager token"; + const iModelName = "LargeChangesetTest"; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + + const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + + // Import schema + const schema = ` + + + + bis:GraphicalElement2d + + + `; + await rwIModel.importSchemaStrings([schema]); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // Create drawing model and category + const codeProps = Code.createEmpty(); + codeProps.value = "DrawingModel"; + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const [, drawingModelId] = withEditTxn(rwIModel, (txn) => IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true)); + let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); + if (undefined === drawingCategoryId) + drawingCategoryId = withEditTxn(rwIModel, (txn) => DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance())); + + await rwIModel.pushChanges({ description: "Initial Test Data Setup", accessToken: adminToken }); + + // Create changesets with different number of inserts + const testCases = [ + { testCaseNum: 1, numElements: 1000 }, + { testCaseNum: 2, numElements: 10000 }, + { testCaseNum: 3, numElements: 100000 }, + // { testCaseNum: 4, numElements: 1000000 }, + // { testCaseNum: 5, numElements: 10000000 }, + ]; + + const elementPropsTemplate = { + classFullName: "TestDomain:TestElement", + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + }; + + for (const testCase of testCases) { + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + withEditTxn(rwIModel, (txn) => { + for (let i = 0; i < testCase.numElements; i++) { + const elementProps = { + ...elementPropsTemplate, + name: `Element_${testCase.numElements}_${i}`, + }; + assert.isTrue(Id64.isValidId64(txn.insertElement(elementProps)), `Failed to insert element ${elementProps.name}`); + } + }); + await rwIModel.pushChanges({ description: `Changeset with ${testCase.numElements} inserts`, accessToken: adminToken }); + } + + const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); + const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); // Get all changesets + + for (const testCase of testCases) { + // Open the changesets one by one and read the changes + const reader = SqliteChangesetReader.openFile({ fileName: changesets[testCase.testCaseNum].pathname, db: rwIModel, disableSchemaCheck: true }); + const adaptor = new ChangesetECAdaptor(reader); + const unifier = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createInMemoryCache()); + adaptor.acceptOp("Inserted"); + assert.equal(unifier.getInstanceCount(), 0, "Unifier should be empty before any changes are applied"); + + const watch = new StopWatch(); + watch.start(); + + while (adaptor.step()) + unifier.appendFrom(adaptor); + + watch.stop(); + + assert.equal(unifier.getInstanceCount(), testCase.numElements, "Number of instances should match the number of inserted elements til now"); + reporter.addEntry("SqliteChangesetReaderAndChangesetECAdaptorAPI", "Unifier-InMemoryCache", "Execution time (seconds)", watch.elapsedSeconds, { iModelId: rwIModelId, changesetId: changesets[testCase.testCaseNum].id, changesetInserts: testCase.numElements }); + } + rwIModel.close(); + }); + + it("Large Changeset Performance - SqliteBackedCache", async () => { + const adminToken = "super manager token"; + const iModelName = "LargeChangesetTest"; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + + const rwIModel = await HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }); + + // Import schema + const schema = ` + + + + bis:GraphicalElement2d + + + `; + await rwIModel.importSchemaStrings([schema]); + rwIModel.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // Create drawing model and category + const codeProps = Code.createEmpty(); + codeProps.value = "DrawingModel"; + await rwIModel.locks.acquireLocks({ shared: IModel.dictionaryId }); + const [, drawingModelId] = withEditTxn(rwIModel, (txn) => IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true)); + let drawingCategoryId = DrawingCategory.queryCategoryIdByName(rwIModel, IModel.dictionaryId, "MyDrawingCategory"); + if (undefined === drawingCategoryId) + drawingCategoryId = withEditTxn(rwIModel, (txn) => DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance())); + + await rwIModel.pushChanges({ description: "Initial Test Data Setup", accessToken: adminToken }); + + // Create changesets with different number of inserts + const testCases = [ + { testCaseNum: 1, numElements: 1000 }, + { testCaseNum: 2, numElements: 10000 }, + { testCaseNum: 3, numElements: 100000 }, + // { testCaseNum: 4, numElements: 1000000 }, + // { testCaseNum: 5, numElements: 10000000 }, + ]; + + const elementPropsTemplate = { + classFullName: "TestDomain:TestElement", + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + }; + + for (const testCase of testCases) { + await rwIModel.locks.acquireLocks({ shared: drawingModelId }); + withEditTxn(rwIModel, (txn) => { + for (let i = 0; i < testCase.numElements; i++) { + const elementProps = { + ...elementPropsTemplate, + name: `Element_${testCase.numElements}_${i}`, + }; + assert.isTrue(Id64.isValidId64(txn.insertElement(elementProps)), `Failed to insert element ${elementProps.name}`); + } + }); + await rwIModel.pushChanges({ description: `Changeset with ${testCase.numElements} inserts`, accessToken: adminToken }); + } + + const targetDir = path.join(KnownTestLocations.outputDir, rwIModelId, "changesets"); + const changesets = await HubMock.downloadChangesets({ iModelId: rwIModelId, targetDir }); // Get all changesets + + for (const testCase of testCases) { + // Open the changesets one by one and read the changes + const reader = SqliteChangesetReader.openFile({ fileName: changesets[testCase.testCaseNum].pathname, db: rwIModel, disableSchemaCheck: true }); + const adaptor = new ChangesetECAdaptor(reader); + const unifier = new PartialECChangeUnifier(reader.db, ECChangeUnifierCache.createSqliteBackedCache(reader.db)); + adaptor.acceptOp("Inserted"); + assert.equal(unifier.getInstanceCount(), 0, "Unifier should be empty before any changes are applied"); + + const watch = new StopWatch(); + watch.start(); + + while (adaptor.step()) + unifier.appendFrom(adaptor); + + watch.stop(); + + assert.equal(unifier.getInstanceCount(), testCase.numElements, "Number of instances should match the number of inserted elements til now"); + reporter.addEntry("SqliteChangesetReaderAndChangesetECAdaptorAPI", "Unifier-SqliteBackedCache", "Execution time (seconds)", watch.elapsedSeconds, { iModelId: rwIModelId, changesetId: changesets[testCase.testCaseNum].id, changesetInserts: testCase.numElements }); + } + rwIModel.close(); + }); + + it("Track changeset health stats", async () => { + const adminToken = "super manager token"; + const iModelName = "LargeChangesetPullTest"; + const rwIModelId = await HubMock.createNewIModel({ iTwinId, iModelName, description: "TestSubject", accessToken: adminToken }); + assert.isNotEmpty(rwIModelId); + + // Open two briefcases for the same iModel + const [firstBriefcase, secondBriefcase] = await Promise.all([ + HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }), + HubWrappers.downloadAndOpenBriefcase({ iTwinId, iModelId: rwIModelId, accessToken: adminToken }) + ]); + + // Import schema + await firstBriefcase.importSchemaStrings([` + + + + bis:GraphicalElement2d + + `]); + firstBriefcase.channels.addAllowedChannel(ChannelControl.sharedChannelName); + + // Create drawing model and category + const codeProps = Code.createEmpty(); + codeProps.value = "DrawingModel"; + await firstBriefcase.locks.acquireLocks({ shared: IModel.dictionaryId }); + const [, drawingModelId] = withEditTxn(firstBriefcase, (txn) => IModelTestUtils.createAndInsertDrawingPartitionAndModel(txn, codeProps, true)); + let drawingCategoryId = DrawingCategory.queryCategoryIdByName(firstBriefcase, IModel.dictionaryId, "MyDrawingCategory"); + if (undefined === drawingCategoryId) + drawingCategoryId = withEditTxn(firstBriefcase, (txn) => DrawingCategory.insert(txn, IModel.dictionaryId, "MyDrawingCategory", new SubCategoryAppearance())); + + await Promise.all([firstBriefcase.enableChangesetStatTracking(), secondBriefcase.enableChangesetStatTracking()]); + + await firstBriefcase.pushChanges({ description: "Initial Test Data Setup", accessToken: adminToken }); + + // Insert a large number of elements and push as a single changeset + const numElements = 100000; + const elementPropsTemplate = { + classFullName: "TestSchema:TestElement", + model: drawingModelId, + category: drawingCategoryId, + code: Code.createEmpty(), + }; + await firstBriefcase.locks.acquireLocks({ shared: drawingModelId }); + withEditTxn(firstBriefcase, (txn) => { + for (let i = 0; i < numElements; i++) { + const elementProps = { + ...elementPropsTemplate, + name: `Element_${i}`, + }; + assert.isTrue(Id64.isValidId64(txn.insertElement(elementProps))); + } + }); + await firstBriefcase.pushChanges({ description: `Large changeset with ${numElements} inserts`, accessToken: adminToken }); + await secondBriefcase.pullChanges({ accessToken: adminToken }); + + const firstBriefcaseChangesets = await firstBriefcase.getAllChangesetHealthData(); + assert.equal(firstBriefcaseChangesets.length, 0); // No new changes to be pulled + const secondBriefcaseChangesets = await secondBriefcase.getAllChangesetHealthData(); + assert.equal(secondBriefcaseChangesets.length, 2); // Schema import followed by element insert + + const secondBriefcaseChangeset2 = secondBriefcaseChangesets[1]; + expect(secondBriefcaseChangeset2.insertedRows).to.be.eql(numElements * 2); // 100k in bis_Element + 100k in bis_GeometricElement2d + expect(secondBriefcaseChangeset2.updatedRows).to.be.eql(2); + expect(secondBriefcaseChangeset2.totalElapsedMs).to.be.greaterThan(0); // Ensure it took some time + expect(secondBriefcaseChangeset2.perStatementStats.length).to.be.eql(4); + + reporter.addEntry("SqliteChangesetReaderAndChangesetECAdaptorAPI", "ChangesetHealthStats", "Execution time (ms)", secondBriefcaseChangeset2.totalElapsedMs, { changesetId: secondBriefcaseChangeset2.changesetId, statementsExecuted: secondBriefcaseChangeset2.perStatementStats.length }); + + // Cleanup + firstBriefcase.close(); + secondBriefcase.close(); + }); +}); diff --git a/test-apps/display-test-app/android/imodeljs-test-app/app/build.gradle b/test-apps/display-test-app/android/imodeljs-test-app/app/build.gradle index 7d836c8da12c..d61e6ba8df7c 100644 --- a/test-apps/display-test-app/android/imodeljs-test-app/app/build.gradle +++ b/test-apps/display-test-app/android/imodeljs-test-app/app/build.gradle @@ -42,7 +42,7 @@ dependencies { implementation 'com.google.android.material:material:1.7.0' implementation 'androidx.constraintlayout:constraintlayout:2.1.4' implementation 'androidx.navigation:navigation-ui:2.5.3' - implementation 'com.github.itwin:mobile-native-android:5.9.11' + implementation 'com.github.itwin:mobile-native-android:5.9.14' implementation 'androidx.webkit:webkit:1.5.0' } diff --git a/test-apps/display-test-app/ios/imodeljs-test-app/imodeljs-test-app.xcodeproj/project.pbxproj b/test-apps/display-test-app/ios/imodeljs-test-app/imodeljs-test-app.xcodeproj/project.pbxproj index 3c6a1dc9c095..da51c55b8225 100644 --- a/test-apps/display-test-app/ios/imodeljs-test-app/imodeljs-test-app.xcodeproj/project.pbxproj +++ b/test-apps/display-test-app/ios/imodeljs-test-app/imodeljs-test-app.xcodeproj/project.pbxproj @@ -455,7 +455,7 @@ repositoryURL = "https://github.com/iTwin/mobile-native-ios"; requirement = { kind = exactVersion; - version = 5.9.11; + version = 5.9.14; }; }; /* End XCRemoteSwiftPackageReference section */ diff --git a/tools/internal/ios/core-test-runner/core-test-runner.xcodeproj/project.pbxproj b/tools/internal/ios/core-test-runner/core-test-runner.xcodeproj/project.pbxproj index 597b3e374c55..041bf9190b6f 100644 --- a/tools/internal/ios/core-test-runner/core-test-runner.xcodeproj/project.pbxproj +++ b/tools/internal/ios/core-test-runner/core-test-runner.xcodeproj/project.pbxproj @@ -554,7 +554,7 @@ repositoryURL = "https://github.com/iTwin/mobile-native-ios"; requirement = { kind = exactVersion; - version = 5.9.11; + version = 5.9.14; }; }; /* End XCRemoteSwiftPackageReference section */