chore: make entities inherit agent vault based events, integrate update tracking to api and integrity checks

Author: kuco23

README.md

@@ -64,7 +64,7 @@ To run the API, run `yarn run-api`.
 
 ## Docker Deployment
 
-The docker image can be either built from this project, or one from github can be used. The repository also features the `compose.yaml` file that features a local postgres deployment and requires the following `.env` config:
+A local docker image can be either built from this repository, or one from github can be utilised. The repository also includes the `compose.yaml` file that features a local postgres deployment and requires the following `.env` config:
 
 - **COMPOSE_PROJECT_NAME**: needs to be set to a unique name of your docker deployment;
 - **COMPOSE_PROFILES**: needs to be a list of options `core`, `api`, `ripple`, `watchdog`;

packages/fasset-indexer-api/src/analytics/metadata.ts

@@ -1,9 +1,6 @@
 import { getVar, type ORM } from "fasset-indexer-core/orm"
-import {
-  FIRST_UNHANDLED_EVENT_BLOCK_DB_KEY,
-  backUpdateLastBlockName,
-  backUpdateFirstUnhandledBlockName
-} from "fasset-indexer-core/config"
+import { updateDistance } from "fasset-indexer-core/utils"
+import { FIRST_UNHANDLED_EVENT_BLOCK_DB_KEY } from "fasset-indexer-core/config"
 
 
 export class MetadataAnalytics {
@@ -14,16 +11,14 @@ export class MetadataAnalytics {
     return (v && v.value) ? parseInt(v.value) : null
   }
 
-  async blocksToBackSync(): Promise<number | null> {
-    const currentUpdate = await getVar(this.orm.em.fork(), 'current_update')
-    if (currentUpdate === null) {
-      return null
+  async blocksToBackSync(): Promise<[string | null, number | null]> {
+    const em = this.orm.em.fork()
+    const currentUpdate = await getVar(em, 'current_update')
+    if (currentUpdate == null) {
+      return [null, null]
     }
     const currentUpdateName = currentUpdate.value!
-    const start = await getVar(this.orm.em.fork(), backUpdateFirstUnhandledBlockName(currentUpdateName))
-    if (start === null || start.value === undefined) return null
-    const end = await getVar(this.orm.em.fork(), backUpdateLastBlockName(currentUpdateName))
-    if (end === null || end.value === undefined) return null
-    return parseInt(end.value) - parseInt(start.value) + 1
+    const currentUpdateDist = await updateDistance(em, currentUpdateName)
+    return [currentUpdateName, currentUpdateDist]
   }
 }

packages/fasset-indexer-api/src/controllers/metadata.controller.ts

@@ -18,8 +18,8 @@ export class MetadataController {
     return apiResponse(this.service.currentBlock(), 200)
   }
 
-  @Get('/blocks-to-back-sync')
-  getBlocksToBackSync(): Promise<ApiResponse<number | null>> {
+  @Get('/blocks-for-current-update-sync')
+  getBlocksToBackSync(): Promise<ApiResponse<[string | null, number | null]>> {
     return apiResponse(this.service.blocksToBackSync(), 200)
   }
 

packages/fasset-indexer-core/src/config/constants.ts

@@ -16,14 +16,6 @@ export const MAX_DATABASE_ENTRIES_FETCH = 200
 export const FIRST_UNHANDLED_EVENT_BLOCK_DB_KEY = "firstUnhandledEventBlock"
 export const MIN_EVM_BLOCK_NUMBER_DB_KEY = "minEvmBlockNumber"
 
-// db back indexer names
-export function backUpdateFirstUnhandledBlockName(updateName: string): string {
-  return `firstUnhandledEventBlock_${updateName}`
-}
-export function backUpdateLastBlockName(updateName: string): string {
-  return `lastEventBlock_${updateName}`
-}
-
 // event names
 // agent
 export const EVENTS = {

packages/fasset-indexer-core/src/config/index.ts

@@ -1,7 +1,5 @@
 export {
     MIN_EVM_BLOCK_NUMBER_DB_KEY, EVENTS,
     MIN_DATABASE_POOL_CONNECTIONS, MAX_DATABASE_POOL_CONNECTIONS,
-    FIRST_UNHANDLED_EVENT_BLOCK_DB_KEY,
-    backUpdateLastBlockName,
-    backUpdateFirstUnhandledBlockName
+    FIRST_UNHANDLED_EVENT_BLOCK_DB_KEY
  } from "./constants"

packages/fasset-indexer-core/src/indexer/reindexing/indexer-back-population.ts

@@ -1,5 +1,5 @@
 import { getVar, setVar } from "../../orm/utils"
-import { backUpdateLastBlockName, backUpdateFirstUnhandledBlockName } from "../../config/constants"
+import { backUpdateLastBlockName, backUpdateFirstUnhandledBlockName } from "../../utils"
 import { EventIndexer } from "../indexer"
 import type { Context } from "../../context/context"
 
@@ -34,4 +34,3 @@ export class EventIndexerBackPopulation extends EventIndexer {
     await setVar(this.context.orm.em.fork(), this.firstUnhandledEventBlockForCurrentUpdateKey, blockNumber.toString())
   }
 }
-

packages/fasset-indexer-core/src/indexer/reindexing/indexer-parallel-race-population.ts

@@ -1,4 +1,4 @@
-import { join } from "../../utils"
+import { join, raceIndexerSyncedFlagName, raceUpdateFirstUnhandledBlockName } from "../../utils"
 import { getVar, setVar } from "../../orm/utils"
 import { EventIndexer } from "../indexer"
 import type { Context } from "../../context/context"
@@ -20,9 +20,10 @@ export class EventIndexerParallelRacePopulation {
     frontInsertionEvents: string[]
   ) {
     this.frontIndexer = new EventIndexer(context, frontInsertionEvents)
-    this.backIndexer = new EventIndexer(context, backInsertionEvents, updateName)
+    const firstUnhandledEventBlockName = raceUpdateFirstUnhandledBlockName(updateName)
+    this.backIndexer = new EventIndexer(context, backInsertionEvents, firstUnhandledEventBlockName)
     this.indexer = new EventIndexer(context, join(frontInsertionEvents, backInsertionEvents))
-    this.updateSynced = `front_indexing_synced_${updateName}`
+    this.updateSynced = raceIndexerSyncedFlagName(updateName)
   }
 
   async run(): Promise<EventIndexer | void> {

packages/fasset-indexer-core/src/orm/entities/events/_bound.ts

@@ -1,6 +1,7 @@
-import { OneToOne, Enum } from "@mikro-orm/core"
+import { OneToOne, Enum, ManyToOne } from "@mikro-orm/core"
 import { FAssetType } from "../../../shared"
 import { EvmLog } from "../evm/log"
+import { AgentVault } from "../agent"
 
 export class EventBound {
 
@@ -12,4 +13,10 @@ export class FAssetEventBound extends EventBound {
 
   @Enum(() => FAssetType)
   fasset!: FAssetType
+}
+
+export class AgentEventBound extends FAssetEventBound {
+
+  @ManyToOne({ entity: () => AgentVault })
+  agentVault!: AgentVault
 }

packages/fasset-indexer-core/src/orm/entities/events/agent.ts

@@ -1,28 +1,16 @@
-import { Entity, ManyToOne, OneToOne, Property } from "@mikro-orm/core"
+import { Entity, OneToOne, Property } from "@mikro-orm/core"
 import { uint256, uint64 } from "../../custom/uint"
-import { FAssetEventBound } from "./_bound"
-import { AgentVault } from "../agent"
+import { AgentEventBound, FAssetEventBound } from "./_bound"
 
 
 @Entity()
-export class AgentVaultCreated extends FAssetEventBound {
-
-  @OneToOne({ entity: () => AgentVault, owner: true })
-  agentVault!: AgentVault
-}
+export class AgentVaultCreated extends AgentEventBound {}
 
 @Entity()
-export class AgentVaultDestroyed extends FAssetEventBound {
-
-  @OneToOne({ entity: () => AgentVault, owner: true })
-  agentVault!: AgentVault
-}
+export class AgentVaultDestroyed extends AgentEventBound {}
 
 @Entity()
-export class AgentSettingChanged extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class AgentSettingChanged extends AgentEventBound {
 
   @Property({ type: 'text' })
   name!: string
@@ -32,20 +20,14 @@ export class AgentSettingChanged extends FAssetEventBound {
 }
 
 @Entity()
-export class SelfClose extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class SelfClose extends AgentEventBound {
 
   @Property({ type: new uint256() })
   valueUBA!: bigint
 }
 
 @Entity()
-export class VaultCollateralWithdrawalAnnounced extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class VaultCollateralWithdrawalAnnounced extends AgentEventBound {
 
   @Property({ type: new uint256() })
   amountWei!: bigint
@@ -55,10 +37,7 @@ export class VaultCollateralWithdrawalAnnounced extends FAssetEventBound {
 }
 
 @Entity()
-export class PoolTokenRedemptionAnnounced extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class PoolTokenRedemptionAnnounced extends AgentEventBound {
 
   @Property({ type: new uint256() })
   amountWei!: bigint
@@ -68,10 +47,7 @@ export class PoolTokenRedemptionAnnounced extends FAssetEventBound {
 }
 
 @Entity()
-export class UnderlyingWithdrawalAnnounced extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class UnderlyingWithdrawalAnnounced extends AgentEventBound {
 
   @Property({ type: new uint64() })
   announcementId!: bigint
@@ -101,10 +77,7 @@ export class UnderlyingWithdrawalCancelled extends FAssetEventBound {
 }
 
 @Entity()
-export class UnderlyingBalanceToppedUp extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class UnderlyingBalanceToppedUp extends AgentEventBound {
 
   @Property({ type: 'text' })
   transactionHash!: string
@@ -114,20 +87,14 @@ export class UnderlyingBalanceToppedUp extends FAssetEventBound {
 }
 
 @Entity()
-export class UnderlyingBalanceChanged extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class UnderlyingBalanceChanged extends AgentEventBound {
 
   @Property({ type: new uint256() })
   balanceUBA!: bigint
 }
 
 @Entity()
-export class DustChanged extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class DustChanged extends AgentEventBound {
 
   @Property({ type: new uint256() })
   dustUBA!: bigint

packages/fasset-indexer-core/src/orm/entities/events/challenge.ts

@@ -1,25 +1,18 @@
-import { Entity, ManyToOne, Property } from '@mikro-orm/core'
+import { Entity, Property } from '@mikro-orm/core'
 import { uint256 } from '../../custom/uint'
-import { FAssetEventBound } from './_bound'
-import { AgentVault } from '../agent'
+import { AgentEventBound } from './_bound'
 import { BYTES32_LENGTH } from '../../../config/constants'
 
 
 @Entity()
-export class IllegalPaymentConfirmed extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class IllegalPaymentConfirmed extends AgentEventBound {
 
   @Property({ type: "text", length: BYTES32_LENGTH, unique: true })
   transactionHash!: string
 }
 
 @Entity()
-export class DuplicatePaymentConfirmed extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class DuplicatePaymentConfirmed extends AgentEventBound {
 
   @Property({ type: "text", length: BYTES32_LENGTH, unique: true })
   transactionHash1!: string
@@ -29,10 +22,7 @@ export class DuplicatePaymentConfirmed extends FAssetEventBound {
 }
 
 @Entity()
-export class UnderlyingBalanceTooLow extends FAssetEventBound {
-
-  @ManyToOne({ entity: () => AgentVault })
-  agentVault!: AgentVault
+export class UnderlyingBalanceTooLow extends AgentEventBound {
 
   @Property({ type: new uint256() })
   balance!: bigint