feat: split modes dynamic modules (#87)

Author: dohaki

Diff for: .env.sample

@@ -1,3 +1,6 @@
+########################
+# SHARED ENV VARIABLES #
+########################
 PORT=3001
 DB_HOST=postgres
 DB_PORT=5432
@@ -7,33 +10,46 @@ DB_DATABASE_NAME=scraperapi
 REDIS_HOST=redis
 REDIS_PORT=6379
 REDIS_PASSWORD=password
+JWT_SECRET=secret
+# MerkleDistributor overrides
+MERKLE_DISTRIBUTOR_CHAIN_ID=
+MERKLE_DISTRIBUTOR_ADDRESS=
+# set the list of operation modes ("normal", "test", "scraper")
+RUN_MODES=normal
+
+########################
+# NORMAL ENV VARIABLES #
+########################
+DISTRIBUTOR_PROOFS_CACHE_SECONDS_DURATION=
+REFERRALS_SUMMARY_CACHE_SECONDS_DURATION=
+# the Discord app's credentials obtained from https://discord.com/developers/applications.
+DISCORD_CLIENT_ID=clientId
+DISCORD_CLIENT_SECRET=clientSecret
+# the url accessed after the Discord authorization processed is fulfilled
+DISCORD_REDIRECT_URI=http://localhost
+AMPLITUDE_API_KEY=
+
+#########################
+# SCRAPER ENV VARIABLES #
+#########################
+
 WEB3_NODE_URL_1=https://mainnet.infura.io/v3/
 WEB3_NODE_URL_10=https://optimism-mainnet.infura.io/v3/
 WEB3_NODE_URL_288=https://boba-mainnet.gateway.pokt.network/v1/lb/
 WEB3_NODE_URL_42161=https://arbitrum-mainnet.infura.io/v3/
 WEB3_NODE_URL_137=https://polygon-mainnet.infura.io/v3/
 WEB3_NODE_URL_5=https://goerli.infura.io/v3/
+# the timestamp after which the referral address is extracted using the delimiter
 REFERRAL_DELIMITER_START_TIMESTAMP=1657290720
+# enable fetching SpokePool events from the contracts
 ENABLE_SPOKE_POOLS_EVENTS_PROCESSING=false
+# enable fetching MerkleDistributor events from the contracts
+ENABLE_MERKLE_DISTRIBUTOR_EVENTS_PROCESSING=false
+# enable the refresh of the referrals materialized view. 
 ENABLE_REFERRALS_MATERIALIZED_VIEW_REFRESH=false
 # specify the strategy used for updating sticky referrals. Valid options: queue | cron | disable
 STICKY_REFERRAL_ADDRESSES_MECHANISM=queue
-DISABLE_CRONS=true
 # Following distances in blocks to guarantee finality on each chain. Format should be a map chainId -> blocks.
 # E.g. { "137": 100 }
 FOLLOWING_DISTANCES=
-JWT_SECRET=secret
-# DISCORD_CLIENT_ID and DISCORD_CLIENT_SECRET are the credentials of the Discord app and they are obtained from https://discord.com/developers/applications.
-DISCORD_CLIENT_ID=clientId
-DISCORD_CLIENT_SECRET=clientSecret
-# the url accessed after the Discord authorization processed is fulfilled
-DISCORD_REDIRECT_URI=http://localhost
-
-# MerkleDistributor overrides
-MERKLE_DISTRIBUTOR_CHAIN_ID=
-MERKLE_DISTRIBUTOR_ADDRESS=
-ENABLE_MERKLE_DISTRIBUTOR_EVENTS_PROCESSING=false
-
-DISTRIBUTOR_PROOFS_CACHE_SECONDS_DURATION=0
-REFERRALS_SUMMARY_CACHE_SECONDS_DURATION=0
-AMPLITUDE_API_KEY=
+DISABLE_CRONS=true

Diff for: .github/workflows/lint-build-test.yaml

@@ -43,6 +43,7 @@ jobs:
       DISCORD_REDIRECT_URI: http://localhost
       DISTRIBUTOR_PROOFS_CACHE_SECONDS_DURATION: 0
       REFERRALS_SUMMARY_CACHE_SECONDS_DURATION: 0
+      RUN_MODES: normal,test,scraper
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup

Diff for: README.md

@@ -36,3 +36,54 @@ git rebase master
   - `feature-branch` -> `master`: The branch will be deployed on the production environment for testing purposes. Most of the time, all feature branches will be merged into `stage` branch before the `master` branch.
 
 3. Update the `stage` branch with the latest master whenever is needed 
+
+## Run modes
+
+The application has the ability the load modules and dependencies depending on predefined running modes that can be one or multiple of: scraper, normal, test. Use the RUN_MODES env variable to configure the behaviour of the application:
+
+```
+RUN_MODES=normal,test
+```
+
+In order to configure a module to behave differently depending on the running mode, use the static `forRoot` method like in the example below:
+
+```ts
+@Module({})
+export class ExampleModule {
+  static forRoot(moduleOptions: ModuleOptions): DynamicModule {
+    let module: DynamicModule = { module: ExampleModule, providers: [], controllers: [], imports: [], exports: [] };
+
+    if (moduleOptions.runModes.includes(RunMode.Normal)) {
+      module = {
+        ...module,
+        controllers: [...module.controllers, ...],
+        providers: [...module.providers, ...],
+        imports: [...module.imports, ...],
+        exports: [...module.exports, ...],
+      };
+    }
+
+    if (moduleOptions.runModes.includes(RunMode.Scraper)) {
+      module = {
+        ...module,
+        controllers: [...module.controllers, ...],
+        providers: [...module.providers, ...],
+        imports: [...module.imports, ...],
+        exports: [...module.exports, ...],
+      };
+    }
+
+    if (moduleOptions.runModes.includes(RunMode.Test)) {
+      module = {
+        ...module,
+        controllers: [...module.controllers, ...],
+        providers: [...module.providers, ...],
+        imports: [...module.imports, ...],
+        exports: [...module.exports, ...],
+      };
+    }
+
+    return module;
+  }
+}
+```

Diff for: src/app.module.ts

@@ -1,10 +1,10 @@
 import { BullModule } from "@nestjs/bull";
-import { CacheModule, MiddlewareConsumer, Module } from "@nestjs/common";
+import { CacheModule, MiddlewareConsumer, Module, DynamicModule } from "@nestjs/common";
 import { ConfigModule } from "@nestjs/config";
 import { ScheduleModule } from "@nestjs/schedule";
 import { TypeOrmModule } from "@nestjs/typeorm";
 import { LogsMiddleware } from "./logging.interceptor";
-import configuration from "./modules/configuration";
+import configuration, { configValues } from "./modules/configuration";
 import { DatabaseModule } from "./modules/database/database.module";
 import { TypeOrmDefaultConfigService } from "./modules/database/database.providers";
 import { HealthModule } from "./modules/health/health.module";
@@ -18,38 +18,51 @@ import { DepositModule } from "./modules/deposit/module";
 import { AuthModule } from "./modules/auth/auth.module";
 import { UserModule } from "./modules/user/module";
 import { AirdropModule } from "./modules/airdrop/module";
+import { ModuleOptions, RunMode } from "./dynamic-module";
 
-@Module({
-  imports: [
-    ConfigModule.forRoot({
-      ignoreEnvFile: false,
-      ignoreEnvVars: false,
-      isGlobal: true,
-      expandVariables: true,
-      load: [configuration],
-    }),
-    TypeOrmModule.forRootAsync({
-      imports: [DatabaseModule],
-      useExisting: TypeOrmDefaultConfigService,
-    }),
-    HealthModule,
-    Web3Module,
-    ScraperModule,
-    BullModule.forRootAsync({
-      imports: [MessagingModule],
-      useExisting: BullConfigService,
-    }),
-    ReferralModule,
-    MarketPriceModule,
-    ScheduleModule.forRoot(),
-    DepositModule,
-    AuthModule,
-    UserModule,
-    AirdropModule,
-    CacheModule.register({ isGlobal: true }),
-  ],
-})
+@Module({})
 export class AppModule {
+  static forRoot(moduleOptions: ModuleOptions): DynamicModule {
+    const imports = [
+      ConfigModule.forRoot({
+        ignoreEnvFile: false,
+        ignoreEnvVars: false,
+        isGlobal: true,
+        expandVariables: true,
+        load: [configuration],
+      }),
+      TypeOrmModule.forRootAsync({
+        imports: [DatabaseModule],
+        useExisting: TypeOrmDefaultConfigService,
+      }),
+      HealthModule,
+      Web3Module,
+      ReferralModule.forRoot(moduleOptions),
+      MarketPriceModule.forRoot(moduleOptions),
+      ScheduleModule.forRoot(),
+      DepositModule.forRoot(moduleOptions),
+      AuthModule.forRoot(moduleOptions),
+      UserModule.forRoot(moduleOptions),
+      AirdropModule.forRoot(moduleOptions),
+      CacheModule.register({ isGlobal: true }),
+    ];
+
+    if (moduleOptions.runModes.includes(RunMode.Scraper)) {
+      imports.push(
+        ScraperModule.forRoot(moduleOptions),
+        BullModule.forRootAsync({
+          imports: [MessagingModule],
+          useExisting: BullConfigService,
+        }),
+      );
+    }
+
+    return {
+      module: AppModule,
+      imports,
+    };
+  }
+
   configure(consumer: MiddlewareConsumer) {
     consumer.apply(LogsMiddleware).forRoutes("*");
   }

Diff for: src/dynamic-module.ts

@@ -0,0 +1,9 @@
+export enum RunMode {
+  Scraper = "scraper",
+  Normal = "normal",
+  Test = "test",
+}
+
+export interface ModuleOptions {
+  runModes: RunMode[];
+}

Diff for: src/main.ts

@@ -6,9 +6,12 @@ import { AppModule } from "./app.module";
 import { AppConfig } from "./modules/configuration/configuration.service";
 import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger";
 import { ValidationPipe } from "./validation.pipe";
+import { configValues } from "./modules/configuration";
 
 async function bootstrap() {
-  const app = await NestFactory.create<NestExpressApplication>(AppModule);
+  const app = await NestFactory.create<NestExpressApplication>(
+    AppModule.forRoot({ runModes: configValues().app.runModes }),
+  );
   const config = app.get(AppConfig);
   const port = config.values.app.port;
 

Diff for: src/modules/scraper/adapter/db/claim-fixture.ts renamed to src/modules/airdrop/adapter/db/claim-fixture.ts

@@ -77,6 +77,7 @@ export class AirdropController {
     ),
   )
   @ApiTags("airdrop")
+  @ApiBearerAuth()
   async feedWalletRewards(
     @UploadedFiles() files: { walletRewards?: Express.Multer.File[]; communityRewards?: Express.Multer.File[] },
   ) {
@@ -96,6 +97,7 @@ export class AirdropController {
     }),
   )
   @ApiTags("airdrop")
+  @ApiBearerAuth()
   uploadMerkleDistributorRecipients(@UploadedFile() file: Express.Multer.File) {
     return this.airdropService.processMerkleDistributorRecipientsFile(file);
   }

Diff for: src/modules/airdrop/entry-points/http/controller.ts

@@ -8,7 +8,7 @@ import {
   UpdateDateColumn,
   ManyToOne,
 } from "typeorm";
-import { MerkleDistributorWindow } from "../../airdrop/model/merkle-distributor-window.entity";
+import { MerkleDistributorWindow } from "./merkle-distributor-window.entity";
 
 @Entity()
 @Unique("UK_claim_windowIndex_accountIndex", ["windowIndex", "accountIndex"])

Diff for: src/modules/scraper/model/claim.entity.ts renamed to src/modules/airdrop/model/claim.entity.ts

@@ -1,6 +1,6 @@
 import { Column, CreateDateColumn, Entity, OneToMany, PrimaryGeneratedColumn, Unique } from "typeorm";
 import { MerkleDistributorRecipient } from "./merkle-distributor-recipient.entity";
-import { Claim } from "../../scraper/model/claim.entity";
+import { Claim } from "./claim.entity";
 
 @Entity()
 // Don't allow duplicates of the window index

Diff for: src/modules/airdrop/model/merkle-distributor-window.entity.ts

@@ -1,4 +1,4 @@
-import { Module } from "@nestjs/common";
+import { Module, Provider, DynamicModule } from "@nestjs/common";
 import { TypeOrmModule } from "@nestjs/typeorm";
 
 import { UserModule } from "../user/module";
@@ -11,32 +11,55 @@ import { AirdropController } from "./entry-points/http/controller";
 
 import { CommunityRewards } from "./model/community-rewards.entity";
 import { WalletRewards } from "./model/wallet-rewards.entity";
-import { Deposit } from "../scraper/model/deposit.entity";
+import { Deposit } from "../deposit/model/deposit.entity";
 import { MerkleDistributorWindowFixture } from "./adapter/db/merkle-distributor-window-fixture";
 import { MerkleDistributorWindow } from "./model/merkle-distributor-window.entity";
 import { MerkleDistributorRecipient } from "./model/merkle-distributor-recipient.entity";
 import { MerkleDistributorRecipientFixture } from "./adapter/db/merkle-distributor-recipient";
+import { ModuleOptions, RunMode } from "../../dynamic-module";
+import { ClaimFixture } from "./adapter/db/claim-fixture";
+import { Claim } from "./model/claim.entity";
 
-@Module({
-  providers: [
-    AirdropService,
-    WalletRewardsFixture,
-    CommunityRewardsFixture,
-    MerkleDistributorWindowFixture,
-    MerkleDistributorRecipientFixture,
-  ],
-  controllers: [AirdropController],
-  imports: [
-    TypeOrmModule.forFeature([
-      CommunityRewards,
-      WalletRewards,
-      Deposit,
-      MerkleDistributorWindow,
-      MerkleDistributorRecipient,
-    ]),
-    UserModule,
-    AppConfigModule,
-  ],
-  exports: [],
-})
-export class AirdropModule {}
+@Module({})
+export class AirdropModule {
+  static forRoot(moduleOptions: ModuleOptions): DynamicModule {
+    let module: DynamicModule = { module: AirdropModule, providers: [], controllers: [], imports: [], exports: [] };
+
+    if (moduleOptions.runModes.includes(RunMode.Normal) || moduleOptions.runModes.includes(RunMode.Test)) {
+      module = {
+        ...module,
+        providers: [...module.providers, AirdropService, ClaimFixture],
+        controllers: [...module.controllers, AirdropController],
+        imports: [
+          ...module.imports,
+          TypeOrmModule.forFeature([
+            CommunityRewards,
+            WalletRewards,
+            Deposit,
+            MerkleDistributorWindow,
+            MerkleDistributorRecipient,
+            Claim,
+          ]),
+          UserModule.forRoot(moduleOptions),
+          AppConfigModule,
+        ],
+      };
+    }
+
+    if (moduleOptions.runModes.includes(RunMode.Test)) {
+      module = {
+        ...module,
+        providers: [
+          ...module.providers,
+          ClaimFixture,
+          WalletRewardsFixture,
+          CommunityRewardsFixture,
+          MerkleDistributorWindowFixture,
+          MerkleDistributorRecipientFixture,
+        ],
+      };
+    }
+
+    return module;
+  }
+}

Diff for: src/modules/airdrop/module.ts

@@ -6,7 +6,7 @@ import BigNumber from "bignumber.js";
 import { DataSource, IsNull, Not, QueryFailedError, Repository } from "typeorm";
 import { Cache } from "cache-manager";
 
-import { Deposit } from "../../scraper/model/deposit.entity";
+import { Deposit } from "../../deposit/model/deposit.entity";
 import { CommunityRewards } from "../model/community-rewards.entity";
 import { WalletRewards } from "../model/wallet-rewards.entity";
 import { Token } from "../../web3/model/token.entity";
@@ -36,8 +36,6 @@ export class AirdropService {
     @InjectRepository(CommunityRewards) private communityRewardsRepository: Repository<CommunityRewards>,
     @InjectRepository(WalletRewards) private walletRewardsRepository: Repository<WalletRewards>,
     @InjectRepository(Deposit) private depositRepository: Repository<Deposit>,
-    @InjectRepository(Deposit) private merkleDistributorWindowRepository: Repository<MerkleDistributorWindow>,
-    @InjectRepository(Deposit) private merkleDistributorRecipientRepository: Repository<MerkleDistributorRecipient>,
     private userService: UserService,
     private appConfig: AppConfig,
     private dataSource: DataSource,