Merge pull request #59 from CinCoders/main

upmerge

Author: dcruzb

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "research-back",
-  "version": "1.6.3",
+  "version": "1.6.4",
   "description": "Research Dashboard API - System to display information extracted from professors' Lattes curriculum.",
   "license": "MIT",
   "author": "CInCoders <cincoders@cin.ufpe.br> (https://cincoders.cin.ufpe.br)",
@@ -46,7 +46,6 @@
     "iconv-lite": "^0.7.0",
     "nest-keycloak-connect": "^1.9.1",
     "nestjs-soap": "^3.0.4",
-    "node-cron": "^4.2.1",
     "pg": "^8.7.1",
     "rimraf": "^3.0.2",
     "string-similarity": "^4.0.4",

package.json

@@ -1,9 +1,10 @@
-import { Controller, Get, Param, Post, Query, Res, UploadedFiles, UseInterceptors } from '@nestjs/common';
+import { Controller, Get, HttpException, HttpStatus, Param, Post, Query, Res, UploadedFiles, UseInterceptors } from '@nestjs/common';
 import { AnyFilesInterceptor } from '@nestjs/platform-express';
 import { ApiOAuth2, ApiResponse, ApiTags } from '@nestjs/swagger';
 import { Response } from 'express';
 import { AuthenticatedUser, Roles } from 'nest-keycloak-connect';
 import { extname } from 'path';
+import { ProfessorService } from 'src/professor/professor.service';
 import { SystemRoles } from 'src/types/enums';
 import { Page } from '../types/page.dto';
 import { ImportXmlDto } from './dto/import-xml.dto';
@@ -14,7 +15,7 @@ import { ImportXmlService } from './import-xml.service';
 @Controller('import-xml')
 @ApiOAuth2([])
 export class ImportXmlController {
-  constructor(private readonly importXmlService: ImportXmlService) {}
+  constructor(private readonly importXmlService: ImportXmlService, private readonly professorService: ProfessorService) {}
 
   @Post()
   @UseInterceptors(
@@ -79,8 +80,15 @@ export class ImportXmlController {
   @Post('professors/lattes/import')
   async importAllProfessors(@AuthenticatedUser() user: any, @Res() res: Response) {
     const username = `${user.name} (${user.email})`;
-    await this.importXmlService.importAllProfessors(username);
-    return res.sendStatus(200);
+    const professorsCount = await this.professorService.count();
+
+    if (professorsCount === 0) {
+      throw new HttpException('Nenhum professor encontrado', HttpStatus.NOT_FOUND);
+    }
+
+    this.importXmlService.executeBackgroundProfessorsUpdate(username);
+
+    return res.status(200).json({ professorsCount });
   }
 
   @ApiResponse({

src/import-xml/import-xml.controller.ts

@@ -1,5 +1,4 @@
 import { Inject, Injectable } from '@nestjs/common';
-import { Cron, CronExpression } from '@nestjs/schedule';
 import AdmZip from 'adm-zip';
 import { Buffer } from 'buffer';
 import extract from 'extract-zip';
@@ -1506,7 +1505,11 @@ export class ImportXmlService {
     return !lastImport || latestUpdate > new Date(lastImport.includedAt);
   }
 
-  async processProfessorData(identifier: string, username?: string): Promise<void> {
+  async processProfessorData(identifier: string, username?: string): Promise<Express.Multer.File> {
+    if (!identifier) {
+      throw new Error('Professor identifier is undefined');
+    }
+
     const args: WsCurriculoGetCurriculoCompactado = { id: identifier };
     const [result] = await this.lattesSoapClient.getCurriculoCompactadoAsync(args);
     const base64Zip = result.return;
@@ -1524,10 +1527,6 @@ export class ImportXmlService {
       xmlCustomEncoding && iconv.encodingExists(xmlCustomEncoding)
         ? iconv.decode(xmlData, xmlCustomEncoding)
         : xmlContentTemp;
-
-    if (!identifier) {
-      throw new Error('Professor identifier is undefined');
-    }
     const filePath = this.generateFilePath(identifier);
 
     await fs.promises.mkdir(this.XML_PATH, { recursive: true });
@@ -1547,58 +1546,46 @@ export class ImportXmlService {
       stream: new Readable(),
     };
 
-    await this.enqueueFiles([tempFile], username || 'atualizado automaticamente');
+    return tempFile;
   }
 
-  @Cron(CronExpression.EVERY_WEEK)
-  async getCurriculum(): Promise<void> {
-    try {
-      const professors = await this.professorService.findAll();
+  // @Cron(CronExpression.EVERY_WEEK)
+  // async getCurriculum(): Promise<void> {
+  //   try {
+  //     const professors = await this.professorService.findAll();
 
-      for (const professorTableDto of professors) {
-        const professor = await this.professorService.findOne(undefined, professorTableDto.identifier);
+  //     for (const professorTableDto of professors) {
+  //       const professor = await this.professorService.findOne(undefined, professorTableDto.identifier);
 
-        if (!professor || !professor.identifier) continue;
+  //       if (!professor || !professor.identifier) continue;
 
-        const hasUpdates = await this.hasProfessorUpdates(professor);
+  //       const hasUpdates = await this.hasProfessorUpdates(professor);
 
-        if (hasUpdates) {
-          await this.processProfessorData(professor.identifier);
-        }
-      }
-    } catch (error) {
-      await logErrorToDatabase(error, EntityType.IMPORT);
-    }
-  }
+  //       if (hasUpdates) {
+  //         await this.processProfessorData(professor.identifier);
+  //       }
+  //     }
+  //   } catch (error) {
+  //     await logErrorToDatabase(error, EntityType.IMPORT);
+  //   }
+  // }
 
-  async importAllProfessors(username: string): Promise<void> {
+  async executeBackgroundProfessorsUpdate(username: string): Promise<void> {
     try {
       const professors = await this.professorService.findAll();
-      let currentIndex = 0;
-
-      const worker = async () => {
-        while (true) {
-          const index = currentIndex++;
+      const tempFiles: Express.Multer.File[] = []; 
 
-          if (index >= professors.length) break;
-
-          const professor = professors[index];
-          const prof = await this.professorService.findOne(undefined, professor.identifier);
-
-          if (prof) {
-            try {
-              if (!prof.identifier) throw new Error('Professor identifier is undefined');
+      for (const { identifier, professorId } of professors) {
+        try {
+          if (!identifier) throw new Error('Professor identifier is undefined');
 
-              await this.processProfessorData(prof.identifier, username);
-            } catch (err) {
-              await logErrorToDatabase(err, EntityType.IMPORT, professor.identifier);
-            }
-          }
+          tempFiles.push(await this.processProfessorData(identifier, username));
+        } catch (err) {
+          await logErrorToDatabase(err, EntityType.IMPORT, String(professorId));
         }
-      };
+      }
 
-      const workers = Array.from({ length: 10 }, () => worker());
-      await Promise.all(workers);
+      await this.enqueueFiles(tempFiles, username || 'atualizado automaticamente')
     } catch (error) {
       await logErrorToDatabase(error, EntityType.IMPORT);
       throw error;
@@ -1607,7 +1594,8 @@ export class ImportXmlService {
 
   async importProfessorById(identifier: string, username: string): Promise<void> {
     try {
-      await this.processProfessorData(identifier, username);
+      const tempFile = await this.processProfessorData(identifier, username);
+      await this.enqueueFiles([tempFile], username || 'atualizado automaticamente');
     } catch (error) {
       await logErrorToDatabase(error, EntityType.IMPORT);
       throw error;

src/import-xml/import-xml.service.ts

@@ -58,7 +58,7 @@ async function bootstrap() {
 
   await AppDataSource.initialize()
     .then(() => console.log('LOG [Typeorm] Success connection'))
-    .catch((error) => console.log(error));
+    .catch(error => console.log(error));
 
   const document = SwaggerModule.createDocument(app, config);
 

src/main.ts

@@ -33,6 +33,10 @@ export class ProfessorService {
     return professor;
   }
 
+  async count(): Promise<number> {
+    return AppDataSource.createQueryBuilder().select('p').from(Professor, 'p').getCount();
+  }
+
   async findAll(): Promise<ProfessorTableDto[]> {
     const professors = await AppDataSource.createQueryBuilder()
       .select(['p.id as id', 'p.identifier as identifier', 'p.name as name'])