From e91c8637d38bc079ea33e5c1e4c54c38f7b5ea7f Mon Sep 17 00:00:00 2001 From: Eliseu Cordeiro Date: Wed, 19 Nov 2025 11:46:10 -0300 Subject: [PATCH 1/6] fix: remove cron job --- src/import-xml/import-xml.service.ts | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/import-xml/import-xml.service.ts b/src/import-xml/import-xml.service.ts index 97b1eb0..65e3817 100644 --- a/src/import-xml/import-xml.service.ts +++ b/src/import-xml/import-xml.service.ts @@ -1,5 +1,4 @@ import { Inject, Injectable } from '@nestjs/common'; -import { Cron, CronExpression } from '@nestjs/schedule'; import AdmZip from 'adm-zip'; import { Buffer } from 'buffer'; import extract from 'extract-zip'; @@ -1550,26 +1549,26 @@ export class ImportXmlService { await this.enqueueFiles([tempFile], username || 'atualizado automaticamente'); } - @Cron(CronExpression.EVERY_WEEK) - async getCurriculum(): Promise { - try { - const professors = await this.professorService.findAll(); + // @Cron(CronExpression.EVERY_WEEK) + // async getCurriculum(): Promise { + // try { + // const professors = await this.professorService.findAll(); - for (const professorTableDto of professors) { - const professor = await this.professorService.findOne(undefined, professorTableDto.identifier); + // for (const professorTableDto of professors) { + // const professor = await this.professorService.findOne(undefined, professorTableDto.identifier); - if (!professor || !professor.identifier) continue; + // if (!professor || !professor.identifier) continue; - const hasUpdates = await this.hasProfessorUpdates(professor); + // const hasUpdates = await this.hasProfessorUpdates(professor); - if (hasUpdates) { - await this.processProfessorData(professor.identifier); - } - } - } catch (error) { - await logErrorToDatabase(error, EntityType.IMPORT); - } - } + // if (hasUpdates) { + // await this.processProfessorData(professor.identifier); + // } + // } + // } catch (error) { + // await logErrorToDatabase(error, EntityType.IMPORT); + // } + // } async importAllProfessors(username: string): Promise { try { From 877e8fd89fc7393d70323963198b6defc4ca2a61 Mon Sep 17 00:00:00 2001 From: Eliseu Cordeiro Date: Wed, 19 Nov 2025 13:54:05 -0300 Subject: [PATCH 2/6] chore: remove node-cron deps --- package-lock.json | 15 --------------- package.json | 1 - 2 files changed, 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9c48e12..56f8e09 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,6 @@ "iconv-lite": "^0.7.0", "nest-keycloak-connect": "^1.9.1", "nestjs-soap": "^3.0.4", - "node-cron": "^4.2.1", "pg": "^8.7.1", "rimraf": "^3.0.2", "string-similarity": "^4.0.4", @@ -9833,15 +9832,6 @@ "dev": true, "license": "MIT" }, - "node_modules/node-cron": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-4.2.1.tgz", - "integrity": "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==", - "license": "ISC", - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/node-emoji": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz", @@ -20150,11 +20140,6 @@ "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", "dev": true }, - "node-cron": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-4.2.1.tgz", - "integrity": "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==" - }, "node-emoji": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz", diff --git a/package.json b/package.json index 9729d22..241707e 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "iconv-lite": "^0.7.0", "nest-keycloak-connect": "^1.9.1", "nestjs-soap": "^3.0.4", - "node-cron": "^4.2.1", "pg": "^8.7.1", "rimraf": "^3.0.2", "string-similarity": "^4.0.4", From d0b4f9999d554ce28afbf8a55db3fe7fe5646951 Mon Sep 17 00:00:00 2001 From: Eliseu Cordeiro Date: Mon, 24 Nov 2025 12:46:10 -0300 Subject: [PATCH 3/6] refactor: change professors CV update to execute in background --- src/import-xml/import-xml.controller.ts | 9 ++++--- src/import-xml/import-xml.service.ts | 31 +++++++------------------ src/professor/professor.service.ts | 4 ++++ 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/import-xml/import-xml.controller.ts b/src/import-xml/import-xml.controller.ts index 10db9ba..89d44d1 100644 --- a/src/import-xml/import-xml.controller.ts +++ b/src/import-xml/import-xml.controller.ts @@ -4,6 +4,7 @@ import { ApiOAuth2, ApiResponse, ApiTags } from '@nestjs/swagger'; import { Response } from 'express'; import { AuthenticatedUser, Roles } from 'nest-keycloak-connect'; import { extname } from 'path'; +import { ProfessorService } from 'src/professor/professor.service'; import { SystemRoles } from 'src/types/enums'; import { Page } from '../types/page.dto'; import { ImportXmlDto } from './dto/import-xml.dto'; @@ -14,7 +15,7 @@ import { ImportXmlService } from './import-xml.service'; @Controller('import-xml') @ApiOAuth2([]) export class ImportXmlController { - constructor(private readonly importXmlService: ImportXmlService) {} + constructor(private readonly importXmlService: ImportXmlService, private readonly professorService: ProfessorService) {} @Post() @UseInterceptors( @@ -79,8 +80,10 @@ export class ImportXmlController { @Post('professors/lattes/import') async importAllProfessors(@AuthenticatedUser() user: any, @Res() res: Response) { const username = `${user.name} (${user.email})`; - await this.importXmlService.importAllProfessors(username); - return res.sendStatus(200); + const professorsCount = await this.professorService.count(); + this.importXmlService.executeBackgroundProfessorsUpdate(username); + + return res.status(200).json({ professorsCount }); } @ApiResponse({ diff --git a/src/import-xml/import-xml.service.ts b/src/import-xml/import-xml.service.ts index 65e3817..d5317e4 100644 --- a/src/import-xml/import-xml.service.ts +++ b/src/import-xml/import-xml.service.ts @@ -1570,34 +1570,19 @@ export class ImportXmlService { // } // } - async importAllProfessors(username: string): Promise { + async executeBackgroundProfessorsUpdate(username: string): Promise { try { const professors = await this.professorService.findAll(); - let currentIndex = 0; - const worker = async () => { - while (true) { - const index = currentIndex++; - - if (index >= professors.length) break; - - const professor = professors[index]; - const prof = await this.professorService.findOne(undefined, professor.identifier); - - if (prof) { - try { - if (!prof.identifier) throw new Error('Professor identifier is undefined'); + for (const { identifier, professorId } of professors) { + try { + if (!identifier) throw new Error('Professor identifier is undefined'); - await this.processProfessorData(prof.identifier, username); - } catch (err) { - await logErrorToDatabase(err, EntityType.IMPORT, professor.identifier); - } - } + await this.processProfessorData(identifier, username); + } catch (err) { + await logErrorToDatabase(err, EntityType.IMPORT, String(professorId)); } - }; - - const workers = Array.from({ length: 10 }, () => worker()); - await Promise.all(workers); + } } catch (error) { await logErrorToDatabase(error, EntityType.IMPORT); throw error; diff --git a/src/professor/professor.service.ts b/src/professor/professor.service.ts index fda3030..3558a44 100644 --- a/src/professor/professor.service.ts +++ b/src/professor/professor.service.ts @@ -33,6 +33,10 @@ export class ProfessorService { return professor; } + async count(): Promise { + return AppDataSource.createQueryBuilder().select('p').from(Professor, 'p').getCount(); + } + async findAll(): Promise { const professors = await AppDataSource.createQueryBuilder() .select(['p.id as id', 'p.identifier as identifier', 'p.name as name']) From 54dcb1fa75d177b907bcf3d7ed82500bf828df3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Brand=C3=A3o?= Date: Mon, 24 Nov 2025 13:56:51 -0300 Subject: [PATCH 4/6] Validates professor identifier earlier Ensures professor identifier is validated before attempting to retrieve data. This prevents potential errors and improves the robustness of the professor data processing flow. Fixes issue introduced in hotfix/461. --- src/import-xml/import-xml.service.ts | 8 ++++---- src/main.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/import-xml/import-xml.service.ts b/src/import-xml/import-xml.service.ts index d5317e4..933661d 100644 --- a/src/import-xml/import-xml.service.ts +++ b/src/import-xml/import-xml.service.ts @@ -1506,6 +1506,10 @@ export class ImportXmlService { } async processProfessorData(identifier: string, username?: string): Promise { + if (!identifier) { + throw new Error('Professor identifier is undefined'); + } + const args: WsCurriculoGetCurriculoCompactado = { id: identifier }; const [result] = await this.lattesSoapClient.getCurriculoCompactadoAsync(args); const base64Zip = result.return; @@ -1523,10 +1527,6 @@ export class ImportXmlService { xmlCustomEncoding && iconv.encodingExists(xmlCustomEncoding) ? iconv.decode(xmlData, xmlCustomEncoding) : xmlContentTemp; - - if (!identifier) { - throw new Error('Professor identifier is undefined'); - } const filePath = this.generateFilePath(identifier); await fs.promises.mkdir(this.XML_PATH, { recursive: true }); diff --git a/src/main.ts b/src/main.ts index c948f25..7f97407 100644 --- a/src/main.ts +++ b/src/main.ts @@ -58,7 +58,7 @@ async function bootstrap() { await AppDataSource.initialize() .then(() => console.log('LOG [Typeorm] Success connection')) - .catch((error) => console.log(error)); + .catch(error => console.log(error)); const document = SwaggerModule.createDocument(app, config); From bf3b6147ad098273f398b1d52de01b02bfac63d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Brand=C3=A3o?= Date: Mon, 24 Nov 2025 13:57:36 -0300 Subject: [PATCH 5/6] Version 1.6.4 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 56f8e09..37753ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "research-back", - "version": "1.6.3", + "version": "1.6.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "research-back", - "version": "1.6.3", + "version": "1.6.4", "license": "MIT", "dependencies": { "@nestjs/common": "^10.4.17", diff --git a/package.json b/package.json index 241707e..a05ec73 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "research-back", - "version": "1.6.3", + "version": "1.6.4", "description": "Research Dashboard API - System to display information extracted from professors' Lattes curriculum.", "license": "MIT", "author": "CInCoders (https://cincoders.cin.ufpe.br)", From 2a0f779f99a338165e67ed0381c64cc354ec11e1 Mon Sep 17 00:00:00 2001 From: Eliseu Cordeiro Date: Mon, 24 Nov 2025 16:12:34 -0300 Subject: [PATCH 6/6] fix: sending all files to the queue at once to avoid overloading the pool. --- src/import-xml/import-xml.controller.ts | 7 ++++++- src/import-xml/import-xml.service.ts | 12 ++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/import-xml/import-xml.controller.ts b/src/import-xml/import-xml.controller.ts index 89d44d1..054d7ce 100644 --- a/src/import-xml/import-xml.controller.ts +++ b/src/import-xml/import-xml.controller.ts @@ -1,4 +1,4 @@ -import { Controller, Get, Param, Post, Query, Res, UploadedFiles, UseInterceptors } from '@nestjs/common'; +import { Controller, Get, HttpException, HttpStatus, Param, Post, Query, Res, UploadedFiles, UseInterceptors } from '@nestjs/common'; import { AnyFilesInterceptor } from '@nestjs/platform-express'; import { ApiOAuth2, ApiResponse, ApiTags } from '@nestjs/swagger'; import { Response } from 'express'; @@ -81,6 +81,11 @@ export class ImportXmlController { async importAllProfessors(@AuthenticatedUser() user: any, @Res() res: Response) { const username = `${user.name} (${user.email})`; const professorsCount = await this.professorService.count(); + + if (professorsCount === 0) { + throw new HttpException('Nenhum professor encontrado', HttpStatus.NOT_FOUND); + } + this.importXmlService.executeBackgroundProfessorsUpdate(username); return res.status(200).json({ professorsCount }); diff --git a/src/import-xml/import-xml.service.ts b/src/import-xml/import-xml.service.ts index d5317e4..df4e631 100644 --- a/src/import-xml/import-xml.service.ts +++ b/src/import-xml/import-xml.service.ts @@ -1505,7 +1505,7 @@ export class ImportXmlService { return !lastImport || latestUpdate > new Date(lastImport.includedAt); } - async processProfessorData(identifier: string, username?: string): Promise { + async processProfessorData(identifier: string, username?: string): Promise { const args: WsCurriculoGetCurriculoCompactado = { id: identifier }; const [result] = await this.lattesSoapClient.getCurriculoCompactadoAsync(args); const base64Zip = result.return; @@ -1546,7 +1546,7 @@ export class ImportXmlService { stream: new Readable(), }; - await this.enqueueFiles([tempFile], username || 'atualizado automaticamente'); + return tempFile; } // @Cron(CronExpression.EVERY_WEEK) @@ -1573,16 +1573,19 @@ export class ImportXmlService { async executeBackgroundProfessorsUpdate(username: string): Promise { try { const professors = await this.professorService.findAll(); + const tempFiles: Express.Multer.File[] = []; for (const { identifier, professorId } of professors) { try { if (!identifier) throw new Error('Professor identifier is undefined'); - await this.processProfessorData(identifier, username); + tempFiles.push(await this.processProfessorData(identifier, username)); } catch (err) { await logErrorToDatabase(err, EntityType.IMPORT, String(professorId)); } } + + await this.enqueueFiles(tempFiles, username || 'atualizado automaticamente') } catch (error) { await logErrorToDatabase(error, EntityType.IMPORT); throw error; @@ -1591,7 +1594,8 @@ export class ImportXmlService { async importProfessorById(identifier: string, username: string): Promise { try { - await this.processProfessorData(identifier, username); + const tempFile = await this.processProfessorData(identifier, username); + await this.enqueueFiles([tempFile], username || 'atualizado automaticamente'); } catch (error) { await logErrorToDatabase(error, EntityType.IMPORT); throw error;