fix: do not queue multiple jobs when same SIWF payload is posted multiple times (#752)

# Problem
If the same authorizationCode was posted to the `/v2/accounts/siwf`
endpoint, multiple queue tasks would be created for (essentially) the
same payload (at least, the same sequence of extrinsics).

# Root Cause
The work queue in BullMQ is intended to be unique on `jobId`. The
`jobId` was being created from a hash of the payload retrieved from SIWF
(Frequency Access). However, when a payload is retrieved from SIWF using
an authorization code, that payload is *SIGNED ON DEMAND*--meaning the
signed extrinsics within the payload differ; different contents =>
different hash, different job ID, hence multiple jobs.

# Solution
If an authorization code is being used to retrieve a payload, use the
authorization code as the job ID instead of the retrieved payload hash

Note, it's still possible that the same operations could be submitted by
an external agent who handles their own signing/payload creation, or
handles fetching the payload from SIWF themselves and submits the actual
payload to Gateway instead of the authorization code. This solution does
not prevent that scenario from causing multiple jobs to be queued for
the same sequence of user creation transactions. (Tracking as #753)

Closes #674

---------

Co-authored-by: Aramik <[email protected]>

Author: JoeCap08055

apps/account-api/src/controllers/v2/accounts-v2.controller.ts

@@ -116,9 +116,13 @@ export class AccountsControllerV2 {
     }
 
     // Trigger chain submissions, if any
-    await this.siwfV2Service.queueChainActions(payload);
+    const jobStatus = await this.siwfV2Service.queueChainActions(payload, callbackRequest);
 
-    const response = this.siwfV2Service.getSiwfV2LoginResponse(payload);
+    const response = await this.siwfV2Service.getSiwfV2LoginResponse(payload);
+    if (jobStatus) {
+      response.signUpReferenceId = jobStatus.referenceId;
+      response.signUpStatus = jobStatus.state;
+    }
 
     return response;
   }

apps/account-api/src/services/siwfV2.service.spec.ts

@@ -349,7 +349,7 @@ describe('SiwfV2Service', () => {
 
     it('should do nothing if there are no chain submissions', async () => {
       const enqueueSpy = jest.spyOn(enqueueService, 'enqueueRequest');
-      const result = await siwfV2Service.queueChainActions(validSiwfLoginResponsePayload);
+      const result = await siwfV2Service.queueChainActions(validSiwfLoginResponsePayload, {});
 
       expect(result).toBeNull();
       expect(enqueueSpy).not.toHaveBeenCalled();
@@ -367,7 +367,7 @@ describe('SiwfV2Service', () => {
           ]) as ApiPromise,
         );
 
-      await expect(siwfV2Service.queueChainActions(validSiwfAddDelegationResponsePayload)).resolves.not.toThrow();
+      await expect(siwfV2Service.queueChainActions(validSiwfAddDelegationResponsePayload, {})).resolves.not.toThrow();
       expect(enqueueSpy).toHaveBeenCalledWith({
         calls: [
           {
@@ -397,7 +397,7 @@ describe('SiwfV2Service', () => {
       );
 
       try {
-        await siwfV2Service.queueChainActions(validSiwfNewUserResponse);
+        await siwfV2Service.queueChainActions(validSiwfNewUserResponse, {});
       } catch (err: any) {
         console.error(err);
         throw err;

apps/account-api/src/services/siwfV2.service.ts

@@ -30,6 +30,10 @@ import { isNotNull } from '#utils/common/common.utils';
 import { chainSignature, statefulStoragePayload } from '#utils/common/signature.util';
 import { ApiPromise } from '@polkadot/api';
 
+interface IAuthCode {
+  authorizationCode?: string;
+}
+
 @Injectable()
 export class SiwfV2Service {
   private readonly logger: Logger;
@@ -177,12 +181,9 @@ export class SiwfV2Service {
 
     response.controlKey = payload.userPublicKey.encodedValue;
 
-    // Try to look up the MSA id, if there is no createSponsoredAccountWithDelegation request
-    if (payload.payloads.every((x) => x.endpoint?.extrinsic !== 'createSponsoredAccountWithDelegation')) {
-      // Get the MSA Id from the chain
-      const msaId = await this.blockchainService.publicKeyToMsaId(response.controlKey);
-      if (msaId) response.msaId = msaId;
-    }
+    // Get the MSA Id from the chain, if it exists
+    const msaId = await this.blockchainService.publicKeyToMsaId(response.controlKey);
+    if (msaId) response.msaId = msaId;
 
     // Parse out the email, phone, and graph
     const email = payload.credentials.find(isCredentialEmail)?.credentialSubject.emailAddress;
@@ -198,7 +199,10 @@ export class SiwfV2Service {
     return response;
   }
 
-  async queueChainActions(response: SiwfResponse): Promise<TransactionResponse | null> {
+  async queueChainActions(
+    response: SiwfResponse,
+    { authorizationCode }: IAuthCode,
+  ): Promise<TransactionResponse | null> {
     // Don't do anything if there is nothing to do
     if (!hasChainSubmissions(response)) return null;
 
@@ -212,6 +216,7 @@ export class SiwfV2Service {
       return this.enqueueService.enqueueRequest<PublishSIWFSignupRequestDto>({
         calls,
         type: TransactionType.SIWF_SIGNUP,
+        authorizationCode,
       });
     } catch (e) {
       this.logger.warn('Error during SIWF V2 Chain Action Queuing', { error: e.toString() });

docs/account/index.html

@@ -1,4 +1,3 @@
-/* eslint-disable class-methods-use-this */
 import { InjectQueue } from '@nestjs/bullmq';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { Queue } from 'bullmq';
@@ -19,30 +18,49 @@ export class EnqueueService {
     this.logger = new Logger(this.constructor.name);
   }
 
-  private calculateJobId<RequestType>(jobWithoutId: RequestType): string {
+  private static calculateJobId<RequestType>(jobWithoutId: RequestType): string {
     const stringVal = JSON.stringify(jobWithoutId);
     return createHash('sha1').update(stringVal).digest('base64url');
   }
 
-  async enqueueRequest<RequestType>(request: RequestType): Promise<TransactionResponse> {
+  async enqueueRequest<RequestType extends Object>(request: RequestType): Promise<TransactionResponse> {
     const { providerId } = this.config;
+    // Best-effort attempt at de-duping payloads that are submitted multiple times.
+    // For payloads submitted via authorization code, we can de-dupe on that.
+    // For payloads submitted directly, we can de-dupe IFF the exact same payload
+    // is submitted multiple times; however, it is possible that the same set of extrinsic
+    // calls could be submitted multiple times, but with different signatures. In that case
+    // the only way to de-dupe would be to decode the extrinsics in the payload and de-dupe
+    // based on examination of the public key and other parameters of the extrinsics. While
+    // technically possible, deemed overkill for now.
+    const authorizationCode =
+      'authorizationCode' in request && typeof request.authorizationCode === 'string'
+        ? request.authorizationCode
+        : undefined;
+    const referenceId = authorizationCode || EnqueueService.calculateJobId(request);
+
     const data: TransactionData<RequestType> = {
       ...request,
       providerId: providerId.toString(),
-      referenceId: this.calculateJobId(request),
+      referenceId,
     };
 
     // add() will not fail if Redis is down, it will keep waiting for a reconnection to occur
     // and then add the job to the queue.
     // Configuring enableOfflineQueue: false will not queue the job if the connection is lost.
     // The REST API will return a 500 error if the connection is lost.
+    //
+    // Also: if this job already exists in the queue (whether waiting, completed, failed, etc),
+    // then '.add()' will simply return the existing job, not add a new one.
     const job = await this.transactionPublishQueue.add(`Transaction Job - ${data.referenceId}`, data, {
       jobId: data.referenceId,
     });
+    const jobState = await job.getState();
 
-    this.logger.log('Job enqueued: ', JSON.stringify(job));
+    this.logger.log('Job enqueued or retrieved: ', JSON.stringify(job));
     return {
       referenceId: data.referenceId,
+      state: jobState,
     };
   }
 }

libs/account-lib/src/services/enqueue-request.service.ts

@@ -72,3 +72,28 @@ export type TxWebhookRsp =
   | PublishGraphKeysWebhookRsp
   | RetireMsaWebhookRsp
   | RevokeDelegationWebhookRsp;
+
+export type PostTransactionNotify_Data = {
+  body: TxWebhookRsp;
+  path?: never;
+  query?: never;
+  url: '/transaction-notify';
+};
+
+export type PostTransactionNotify_Errors = {
+  /**
+   * Bad request
+   */
+  400: unknown;
+};
+
+export type PostTransactionNotify_Responses = {
+  /**
+   * Successful notification
+   */
+  200: unknown;
+};
+
+export type ClientOptions = {
+  baseUrl: `${string}://openapi-specs` | (string & {});
+};

libs/types/src/account-webhook/types.gen.ts

@@ -16,6 +16,7 @@ export interface SIWFEncodedExtrinsic {
 export type PublishSIWFSignupRequestDto = {
   calls: SIWFEncodedExtrinsic[];
   type: TransactionType.SIWF_SIGNUP;
+  authorizationCode?: string;
 };
 
 export type TransactionData<

libs/types/src/dtos/account/transaction.request.dto.ts

@@ -1,6 +1,12 @@
-import { IsNotEmpty } from 'class-validator';
+import { ApiProperty } from '@nestjs/swagger';
+import { JobState } from 'bullmq';
 
 export class TransactionResponse {
-  @IsNotEmpty()
   referenceId: string;
+
+  // JobState is a string union type; it's not possible to extract the specific values here
+  // without hard-coding them, and since it's a response only, we don't need to validate, so it
+  // can just be 'String'
+  @ApiProperty({ description: 'Job state', name: 'state', type: String, example: 'waiting' })
+  state?: JobState | 'unknown';
 }

libs/types/src/dtos/account/transaction.response.dto.ts

@@ -1,62 +1,54 @@
 /* eslint-disable max-classes-per-file */
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
-import { IsArray, IsOptional, IsString } from 'class-validator';
 
 export class GraphKeySubject {
   @ApiProperty({
     description: 'The id type of the VerifiedGraphKeyCredential.',
     required: true,
     example: 'did:key:z6QNucQV4AF1XMQV4kngbmnBHwYa6mVswPEGrkFrUayhttT1',
   })
-  @IsString()
   id: string;
 
   @ApiProperty({
     description: 'The encoded public key.',
     required: true,
     example: '0xb5032900293f1c9e5822fd9c120b253cb4a4dfe94c214e688e01f32db9eedf17',
   })
-  @IsString()
   encodedPublicKeyValue: string;
 
   @ApiProperty({
     description: 'The encoded private key. WARNING: This is sensitive user information!',
     required: true,
     example: '0xd0910c853563723253c4ed105c08614fc8aaaf1b0871375520d72251496e8d87',
   })
-  @IsString()
   encodedPrivateKeyValue: string;
 
   @ApiProperty({
     description: 'How the encoded keys are encoded. Only "base16" (aka hex) currently.',
     required: true,
     example: 'base16',
   })
-  @IsString()
   encoding: string;
 
   @ApiProperty({
     description: 'Any addition formatting options. Only: "bare" currently.',
     required: true,
     example: 'bare',
   })
-  @IsString()
   format: string;
 
   @ApiProperty({
     description: 'The encryption key algorithm.',
     required: true,
     example: 'X25519',
   })
-  @IsString()
   type: string;
 
   @ApiProperty({
     description: 'The DSNP key type.',
     required: true,
     example: 'dsnp.public-key-key-agreement',
   })
-  @IsString()
   keyType: string;
 }
 
@@ -67,42 +59,50 @@ export class WalletV2LoginResponseDto {
     type: String,
     example: 'f6cL4wq1HUNx11TcvdABNf9UNXXoyH47mVUwT59tzSFRW8yDH',
   })
-  @IsString()
   controlKey: string;
 
+  @ApiPropertyOptional({
+    description: 'ReferenceId of an associated sign-up request queued task, if applicable',
+    required: false,
+    type: String,
+    example: 'MjY3MjI3NWZlMGM0NTZmYjY3MWU0ZjQxN2ZiMmY5ODkyYzc1NzNiYQo',
+  })
+  signUpReferenceId?: string;
+
+  @ApiPropertyOptional({
+    description: 'Status of associated sign-up request queued task, if applicable',
+    required: false,
+    type: String,
+    example: 'waiting',
+  })
+  signUpStatus?: string;
+
   @ApiPropertyOptional({
     description: "The user's MSA Id, if one is already created. Will be empty if it is still being processed.",
     type: String,
     example: '314159265358979323846264338',
   })
-  @IsString()
-  @IsOptional()
   msaId?: string;
 
   @ApiPropertyOptional({
     description: "The users's validated email",
     type: String,
     example: '[email protected]',
   })
-  @IsString()
-  @IsOptional()
   email?: string;
 
   @ApiPropertyOptional({
     description: "The users's validated SMS/Phone Number",
     type: String,
     example: '555-867-5309',
   })
-  @IsString()
-  @IsOptional()
   phoneNumber?: string;
 
   @ApiPropertyOptional({
     description: "The users's Private Graph encryption key.",
     type: GraphKeySubject,
     example: '555-867-5309',
   })
-  @IsOptional()
   graphKey?: GraphKeySubject;
 
   @ApiPropertyOptional({
@@ -159,6 +159,5 @@ export class WalletV2LoginResponseDto {
       },
     ],
   })
-  @IsArray()
   rawCredentials?: Object[];
 }

libs/types/src/dtos/account/wallet.v2.login.response.dto.ts

@@ -902,6 +902,16 @@
             "description": "The ss58 encoded MSA Control Key of the login.",
             "example": "f6cL4wq1HUNx11TcvdABNf9UNXXoyH47mVUwT59tzSFRW8yDH"
           },
+          "signUpReferenceId": {
+            "type": "string",
+            "description": "ReferenceId of an associated sign-up request queued task, if applicable",
+            "example": "MjY3MjI3NWZlMGM0NTZmYjY3MWU0ZjQxN2ZiMmY5ODkyYzc1NzNiYQo"
+          },
+          "signUpStatus": {
+            "type": "string",
+            "description": "Status of associated sign-up request queued task, if applicable",
+            "example": "waiting"
+          },
           "msaId": {
             "type": "string",
             "description": "The user's MSA Id, if one is already created. Will be empty if it is still being processed.",
@@ -1231,6 +1241,11 @@
       "TransactionResponse": {
         "type": "object",
         "properties": {
+          "state": {
+            "type": "string",
+            "description": "Job state",
+            "example": "waiting"
+          },
           "referenceId": {
             "type": "string"
           }

openapi-specs/account.openapi.json

@@ -2,16 +2,25 @@
 import { defineConfig, UserConfig } from '@hey-api/openapi-ts';
 
 export default defineConfig({
-  client: 'axios',
+  input: '',
   exportCore: false,
   output: {
     format: 'prettier',
     lint: 'eslint',
     path: 'types',
   },
+  plugins: [
+    {
+      dates: true,
+      name: '@hey-api/transformers',
+    },
+    {
+      identifierCase: 'preserve',
+      enums: 'typescript',
+      enumsCase: 'preserve',
+      name: '@hey-api/typescript',
+    },
+  ],
   schemas: false,
   services: false,
-  types: {
-    enums: 'typescript',
-  },
 } as UserConfig);