From 8d73c7af2feeb4c591f1a3d8b532ba64c2b86091 Mon Sep 17 00:00:00 2001 From: Benevanio Santos Date: Thu, 9 Apr 2026 11:44:56 -0300 Subject: [PATCH 1/9] feat: secure redis env config for VPS --- README.md | 23 ++++ backend/.env.example | 11 ++ backend/package.json | 1 + backend/src/cache/cache.js | 138 +++++++++++++++++++- backend/src/config.js | 51 ++++++-- backend/src/jobsApiApp.js | 46 ++++++- backend/src/pipeline/searchJobsWithCache.js | 6 +- backend/tests/integration/jobsApi.test.js | 27 +++- backend/tests/unit/utils/config.test.js | 15 +++ docker-compose.yml | 3 + package-lock.json | 96 +++++++++++++- 11 files changed, 395 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index d4a68115..5f2d672a 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,29 @@ npm run dev:frontend npm run dev:backend ``` +## Configuracao segura na VPS (Redis / Postgres) + +Para ambiente de VPS e projeto open source, **nao coloque `DATABASE_URL` nem `REDIS_URL` em `environment.json`**. +Use somente `backend/.env` no servidor (arquivo ignorado pelo Git) e mantenha `KEYWORDS_STORAGE_MODE=env`. + +Exemplo seguro: + +```env +KEYWORDS_STORAGE_MODE=env +SEARCH_KEYWORDS=Java,Spring,RabbitMQ,Docker +DATABASE_URL=postgresql://app_user:senha_forte@127.0.0.1:5432/jobsglobalscraper +REDIS_URL=redis://:senha_forte@127.0.0.1:6379/0 +REDIS_KEY_PREFIX=vagas-full +CACHE_TTL_MS=600000 +``` + +Recomendacoes: + +- use `127.0.0.1` ou a rede interna do Docker na VPS, em vez de expor Redis publicamente +- prefira `rediss://` se o Redis estiver fora da maquina local/VPS +- mantenha a porta `6379` fechada para acesso externo no firewall +- com `KEYWORDS_STORAGE_MODE=env`, o backend deixa de depender de `backend/src/db/environment.json` em producao + ## Scripts principais ### Raiz diff --git a/backend/.env.example b/backend/.env.example index 1ba237e8..85566535 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -12,8 +12,19 @@ SEARCH_LANGUAGE=pt REMOTE_ONLY=true JOB_TYPES=C,F TIME_FILTER=r604800 + +# Keywords storage: +# - env => recomendado para VPS / Docker / projeto open source +# - file => mantém compatibilidade com backend/src/db/environment.json +KEYWORDS_STORAGE_MODE=env SEARCH_KEYWORDS=UX Designer,UI Designer,Product Manager,Product Owner +# External services (nunca commitar valores reais) +DATABASE_URL=postgresql://app_user:change_me@127.0.0.1:5432/jobsglobalscraper +REDIS_URL=redis://:change_me@127.0.0.1:6379/0 +REDIS_KEY_PREFIX=vagas-full +CACHE_TTL_MS=600000 + # Scraping behavior WAIT_BETWEEN_SEARCHES_MS=5000 PAGE_TIMEOUT_MS=10000 diff --git a/backend/package.json b/backend/package.json index 5713a234..4279d0d0 100644 --- a/backend/package.json +++ b/backend/package.json @@ -37,6 +37,7 @@ "dotenv": "^17.3.1", "express": "^5.2.1", "pdfkit": "^0.18.0", + "redis": "^4.7.1", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.1", "xlsx": "^0.18.5" diff --git a/backend/src/cache/cache.js b/backend/src/cache/cache.js index 45310b24..c483ef36 100644 --- a/backend/src/cache/cache.js +++ b/backend/src/cache/cache.js @@ -1,3 +1,54 @@ +let redisClientPromise = null; +let redisWarningShown = false; + +function warnRedisFallback(message, error) { + if (redisWarningShown) { + return; + } + + redisWarningShown = true; + + const errorMessage = error instanceof Error ? error.message : ""; + // eslint-disable-next-line no-console + console.warn(`${message}${errorMessage ? ` (${errorMessage})` : ""}`); +} + +async function getRedisClient() { + const redisUrl = process.env.REDIS_URL?.trim(); + + if (!redisUrl) { + return null; + } + + if (!redisClientPromise) { + redisClientPromise = import("redis") + .then(async ({ createClient }) => { + const client = createClient({ + url: redisUrl, + socket: { + reconnectStrategy(retries) { + return Math.min(retries * 50, 1_000); + }, + }, + }); + + client.on("error", (error) => { + warnRedisFallback("Redis indisponivel, usando cache em memoria.", error); + }); + + await client.connect(); + return client; + }) + .catch((error) => { + warnRedisFallback("Falha ao conectar no Redis, usando cache em memoria.", error); + redisClientPromise = null; + return null; + }); + } + + return redisClientPromise; +} + export class MemoryCache { constructor() { this.store = new Map(); @@ -38,4 +89,89 @@ export class MemoryCache { } } -export const cache = new MemoryCache(); +export class RedisCache { + constructor(options = {}) { + this.prefix = options.prefix || process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full"; + this.memoryFallback = new MemoryCache(); + } + + buildKey(key) { + return `${this.prefix}:${key}`; + } + + async get(key) { + const client = await getRedisClient(); + + if (!client) { + return this.memoryFallback.get(key); + } + + try { + const raw = await client.get(this.buildKey(key)); + return raw ? JSON.parse(raw) : null; + } catch (error) { + warnRedisFallback("Erro ao ler do Redis, usando cache em memoria.", error); + return this.memoryFallback.get(key); + } + } + + async set(key, value, ttlMs) { + this.memoryFallback.set(key, value, ttlMs); + + const client = await getRedisClient(); + if (!client) { + return; + } + + try { + await client.set(this.buildKey(key), JSON.stringify(value), { + PX: Math.max(1, Number(ttlMs) || 1), + }); + } catch (error) { + warnRedisFallback("Erro ao salvar no Redis, usando cache em memoria.", error); + } + } + + async delete(key) { + this.memoryFallback.delete(key); + + const client = await getRedisClient(); + if (!client) { + return; + } + + try { + await client.del(this.buildKey(key)); + } catch (error) { + warnRedisFallback("Erro ao remover chave do Redis.", error); + } + } + + async clear() { + this.memoryFallback.clear(); + + const client = await getRedisClient(); + if (!client) { + return; + } + + try { + const keys = []; + for await (const key of client.scanIterator({ MATCH: `${this.prefix}:*` })) { + keys.push(key); + } + + if (keys.length > 0) { + await client.del(keys); + } + } catch (error) { + warnRedisFallback("Erro ao limpar o Redis.", error); + } + } + + async has(key) { + return (await this.get(key)) !== null; + } +} + +export const cache = process.env.REDIS_URL?.trim() ? new RedisCache() : new MemoryCache(); diff --git a/backend/src/config.js b/backend/src/config.js index bdb13bd2..5216dbdb 100644 --- a/backend/src/config.js +++ b/backend/src/config.js @@ -38,6 +38,14 @@ function parseNumber(value, fallback) { return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } +function getKeywordsStorageMode() { + const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "file") + .trim() + .toLowerCase(); + + return configuredMode === "env" ? "env" : "file"; +} + function getKeywordsFilePath() { const configuredPath = process.env.KEYWORDS_FILE_PATH?.trim(); return configuredPath @@ -53,8 +61,29 @@ function normalizeKeywords(keywords) { return [...new Set(keywords.map((item) => String(item ?? "").trim()).filter(Boolean))]; } +function parseKeywordsFromEnv(value) { + if (!value) { + return null; + } + + const keywords = String(value) + .split(",") + .map((item) => item.trim()) + .filter(Boolean); + + return keywords.length > 0 ? keywords : null; +} + function parseKeywords(value) { - // Tenta pegar do arquivo environment.json + const keywordsStorageMode = getKeywordsStorageMode(); + + if (keywordsStorageMode === "env") { + const keywordsFromEnv = parseKeywordsFromEnv(value); + if (keywordsFromEnv) { + return keywordsFromEnv; + } + } + try { const envPath = getKeywordsFilePath(); if (existsSync(envPath)) { @@ -63,20 +92,15 @@ function parseKeywords(value) { return normalizeKeywords(data.KEYWORDS) ?? []; } } - } catch (err) { + } catch { // Se falhar, fallback } - // Tenta pegar da variavel de ambiente - if (value) { - const keywords = value - .split(",") - .map((item) => item.trim()) - .filter(Boolean); - if (keywords.length > 0) return keywords; + const keywordsFromEnv = parseKeywordsFromEnv(value); + if (keywordsFromEnv) { + return keywordsFromEnv; } - // Fallback para as keywords padrao return DEFAULT_KEYWORDS; } @@ -108,6 +132,11 @@ export function getConfig() { jobTypes: process.env.JOB_TYPES || "C,F", // f_TPR examples: r86400 (24h), r604800 (7 dias), r2592000 (30 dias) timeFilter: parseTimeFilter(process.env.TIME_FILTER, "r604800"), - keywords: parseKeywords(process.env.SEARCH_KEYWORDS) + keywords: parseKeywords(process.env.SEARCH_KEYWORDS), + keywordsStorageMode: getKeywordsStorageMode(), + cacheTtlMs: parseNumber(process.env.CACHE_TTL_MS, 10 * 60 * 1000), + databaseUrl: process.env.DATABASE_URL?.trim() || "", + redisUrl: process.env.REDIS_URL?.trim() || "", + redisKeyPrefix: process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full" }; } diff --git a/backend/src/jobsApiApp.js b/backend/src/jobsApiApp.js index 9da94af9..c08f29c6 100644 --- a/backend/src/jobsApiApp.js +++ b/backend/src/jobsApiApp.js @@ -17,6 +17,14 @@ const DEFAULT_ALLOWED_ORIGINS = [ "http://localhost:5174", ]; +function getKeywordsStorageMode() { + const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "file") + .trim() + .toLowerCase(); + + return configuredMode === "env" ? "env" : "file"; +} + function getKeywordsFilePath() { const configuredPath = process.env.KEYWORDS_FILE_PATH?.trim(); return configuredPath @@ -32,7 +40,20 @@ function normalizeKeywords(keywords) { return [...new Set(keywords.map((item) => String(item ?? "").trim()).filter(Boolean))]; } +function parseKeywordsFromEnv(value) { + return normalizeKeywords( + String(value ?? "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + ) ?? []; +} + function readEnvironmentData() { + if (getKeywordsStorageMode() === "env") { + return { KEYWORDS: parseKeywordsFromEnv(process.env.SEARCH_KEYWORDS) }; + } + const envPath = getKeywordsFilePath(); if (!existsSync(envPath)) { @@ -48,9 +69,26 @@ function readEnvironmentData() { } function writeEnvironmentData(data) { + const normalizedKeywords = normalizeKeywords(data?.KEYWORDS) ?? []; + + if (getKeywordsStorageMode() === "env") { + process.env.SEARCH_KEYWORDS = normalizedKeywords.join(","); + return { + ...(data && typeof data === "object" ? data : {}), + KEYWORDS: normalizedKeywords, + }; + } + const envPath = getKeywordsFilePath(); mkdirSync(path.dirname(envPath), { recursive: true }); - writeFileSync(envPath, JSON.stringify(data, null, 2), "utf-8"); + + const nextData = { + ...(data && typeof data === "object" ? data : {}), + KEYWORDS: normalizedKeywords, + }; + + writeFileSync(envPath, JSON.stringify(nextData, null, 2), "utf-8"); + return nextData; } function parseAllowedOrigins(value) { @@ -305,12 +343,10 @@ export function createJobsApiApp(options = {}) { }); } - const envData = { + const envData = writeEnvironmentData({ ...readEnvironmentData(), KEYWORDS: normalizedKeywords, - }; - - writeEnvironmentData(envData); + }); return res.json({ ok: true, diff --git a/backend/src/pipeline/searchJobsWithCache.js b/backend/src/pipeline/searchJobsWithCache.js index 006332b3..3f58a964 100644 --- a/backend/src/pipeline/searchJobsWithCache.js +++ b/backend/src/pipeline/searchJobsWithCache.js @@ -34,7 +34,7 @@ export async function searchJobsWithCache( ) { const cacheKey = buildCacheKey(config); - const cached = cache.get(cacheKey); + const cached = await cache.get(cacheKey); if (cached) { return { ...cached, @@ -43,7 +43,7 @@ export async function searchJobsWithCache( } return withRequestDedup(cacheKey, async () => { - const cachedInsideDedup = cache.get(cacheKey); + const cachedInsideDedup = await cache.get(cacheKey); if (cachedInsideDedup) { return { ...cachedInsideDedup, @@ -60,7 +60,7 @@ export async function searchJobsWithCache( fromCache: false, }; - cache.set(cacheKey, result, ttlMs); + await cache.set(cacheKey, result, ttlMs); return result; }); diff --git a/backend/tests/integration/jobsApi.test.js b/backend/tests/integration/jobsApi.test.js index e37867c6..486b9b85 100644 --- a/backend/tests/integration/jobsApi.test.js +++ b/backend/tests/integration/jobsApi.test.js @@ -1,4 +1,4 @@ -import { mkdtempSync } from "fs"; +import { existsSync, mkdtempSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; import request from "supertest"; @@ -22,10 +22,14 @@ describe("jobs API", () => { vi.clearAllMocks(); mocks.run.mockResolvedValue(undefined); delete process.env.KEYWORDS_FILE_PATH; + delete process.env.KEYWORDS_STORAGE_MODE; + delete process.env.SEARCH_KEYWORDS; }); afterEach(() => { delete process.env.KEYWORDS_FILE_PATH; + delete process.env.KEYWORDS_STORAGE_MODE; + delete process.env.SEARCH_KEYWORDS; tmpDir = undefined; }); @@ -198,6 +202,27 @@ describe("jobs API", () => { }); }); + it("POST /api/keywords em modo env atualiza sem depender de arquivo json", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + process.env.KEYWORDS_STORAGE_MODE = "env"; + process.env.KEYWORDS_FILE_PATH = join(tmpDir, "nested", "environment.json"); + process.env.SEARCH_KEYWORDS = "Java,Node"; + + const app = createJobsApiApp({ outputDir: tmpDir }); + const res = await request(app) + .post("/api/keywords") + .send({ keywords: ["Redis", "PostgreSQL"] }) + .expect(200); + + expect(res.body).toEqual({ + ok: true, + message: "Keywords atualizadas com sucesso.", + keywords: ["Redis", "PostgreSQL"], + }); + expect(process.env.SEARCH_KEYWORDS).toBe("Redis,PostgreSQL"); + expect(existsSync(process.env.KEYWORDS_FILE_PATH)).toBe(false); + }); + it("GET /api/keywords retorna 200 com as keywords", async () => { const app = createJobsApiApp({ outputDir: tmpDir }); const res = await request(app) diff --git a/backend/tests/unit/utils/config.test.js b/backend/tests/unit/utils/config.test.js index 3f0f9a79..c6ac9af0 100644 --- a/backend/tests/unit/utils/config.test.js +++ b/backend/tests/unit/utils/config.test.js @@ -21,6 +21,7 @@ const CONFIG_ENV_KEYS = [ "TIME_FILTER", "SEARCH_KEYWORDS", "KEYWORDS_FILE_PATH", + "KEYWORDS_STORAGE_MODE", ]; describe("getConfig", () => { @@ -57,6 +58,20 @@ describe("getConfig", () => { expect(config.keywords).toEqual(["Java", "Spring", "RabbitMQ", "Docker"]); }); + it("prioriza SEARCH_KEYWORDS sobre o arquivo quando ambos existem", () => { + const tempDir = mkdtempSync(path.join(tmpdir(), "jobs-config-")); + const keywordsFile = path.join(tempDir, "environment.json"); + + writeFileSync(keywordsFile, JSON.stringify({ KEYWORDS: ["Legado", "Arquivo"] }), "utf-8"); + + vi.stubEnv("KEYWORDS_FILE_PATH", keywordsFile); + vi.stubEnv("SEARCH_KEYWORDS", "Java,Spring,RabbitMQ,Docker"); + vi.stubEnv("KEYWORDS_STORAGE_MODE", "env"); + + const config = getConfig(); + expect(config.keywords).toEqual(["Java", "Spring", "RabbitMQ", "Docker"]); + }); + it("rejeita TIME_FILTER invalido e usa fallback", () => { vi.stubEnv("TIME_FILTER", "invalid"); const config = getConfig(); diff --git a/docker-compose.yml b/docker-compose.yml index 9d64018b..5573f978 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,9 @@ services: - ./backend/.env environment: - PORT=3001 + - KEYWORDS_STORAGE_MODE=${KEYWORDS_STORAGE_MODE:-env} + - CACHE_TTL_MS=${CACHE_TTL_MS:-600000} + - REDIS_KEY_PREFIX=${REDIS_KEY_PREFIX:-vagas-full} volumes: - ./backend/output:/app/output ports: diff --git a/package-lock.json b/package-lock.json index e6cf47d9..f4ad1eae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,6 +38,7 @@ "dotenv": "^17.3.1", "express": "^5.2.1", "pdfkit": "^0.18.0", + "redis": "^4.7.1", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.1", "xlsx": "^0.18.5" @@ -2110,6 +2111,65 @@ "node": ">=14" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz", + "integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==", + "license": "MIT", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.0.0-rc.12", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.12.tgz", @@ -4820,6 +4880,15 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/codepage": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", @@ -6952,6 +7021,15 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -10221,6 +10299,23 @@ "node": ">=8" } }, + "node_modules/redis": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.1.tgz", + "integrity": "sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ==", + "license": "MIT", + "workspaces": [ + "./packages/*" + ], + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.1", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -13081,7 +13176,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, "license": "ISC" }, "node_modules/yargs": { From 13513027816a24fcd75542640354fb9e79938c0e Mon Sep 17 00:00:00 2001 From: Benevanio Date: Thu, 9 Apr 2026 12:02:11 -0300 Subject: [PATCH 2/9] Update backend/src/jobsApiApp.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Benevanio --- backend/src/jobsApiApp.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/src/jobsApiApp.js b/backend/src/jobsApiApp.js index c08f29c6..b63df8dd 100644 --- a/backend/src/jobsApiApp.js +++ b/backend/src/jobsApiApp.js @@ -51,7 +51,11 @@ function parseKeywordsFromEnv(value) { function readEnvironmentData() { if (getKeywordsStorageMode() === "env") { - return { KEYWORDS: parseKeywordsFromEnv(process.env.SEARCH_KEYWORDS) }; + const envKeywords = parseKeywordsFromEnv(process.env.SEARCH_KEYWORDS); + + if (envKeywords.length > 0) { + return { KEYWORDS: envKeywords }; + } } const envPath = getKeywordsFilePath(); From 777528729f78cf3014fac671b25245a5d746e1ee Mon Sep 17 00:00:00 2001 From: Benevanio Date: Thu, 9 Apr 2026 12:02:34 -0300 Subject: [PATCH 3/9] Update docker-compose.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Benevanio --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5573f978..9d64018b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,9 +9,6 @@ services: - ./backend/.env environment: - PORT=3001 - - KEYWORDS_STORAGE_MODE=${KEYWORDS_STORAGE_MODE:-env} - - CACHE_TTL_MS=${CACHE_TTL_MS:-600000} - - REDIS_KEY_PREFIX=${REDIS_KEY_PREFIX:-vagas-full} volumes: - ./backend/output:/app/output ports: From 7b8b471bbc7ff7479cb9a672d79e5bbd638b5332 Mon Sep 17 00:00:00 2001 From: Benevanio Date: Thu, 9 Apr 2026 12:02:52 -0300 Subject: [PATCH 4/9] Update backend/src/config.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Benevanio --- backend/src/config.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/backend/src/config.js b/backend/src/config.js index 5216dbdb..ab07b50c 100644 --- a/backend/src/config.js +++ b/backend/src/config.js @@ -133,10 +133,6 @@ export function getConfig() { // f_TPR examples: r86400 (24h), r604800 (7 dias), r2592000 (30 dias) timeFilter: parseTimeFilter(process.env.TIME_FILTER, "r604800"), keywords: parseKeywords(process.env.SEARCH_KEYWORDS), - keywordsStorageMode: getKeywordsStorageMode(), - cacheTtlMs: parseNumber(process.env.CACHE_TTL_MS, 10 * 60 * 1000), - databaseUrl: process.env.DATABASE_URL?.trim() || "", - redisUrl: process.env.REDIS_URL?.trim() || "", - redisKeyPrefix: process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full" + keywordsStorageMode: getKeywordsStorageMode() }; } From 2dfef49f068291a0363505e2e3a11c5a6a33b713 Mon Sep 17 00:00:00 2001 From: Benevanio Santos Date: Thu, 9 Apr 2026 16:49:46 -0300 Subject: [PATCH 5/9] fix: persist keywords in redis --- backend/src/app.js | 7 +- backend/src/cache/cache.js | 83 +++++++++++++- backend/src/db/keywordsStore.js | 126 ++++++++++++++++++++++ backend/src/jobsApiApp.js | 110 ++++--------------- backend/src/server.js | 9 ++ backend/tests/integration/jobsApi.test.js | 14 ++- 6 files changed, 252 insertions(+), 97 deletions(-) create mode 100644 backend/src/db/keywordsStore.js diff --git a/backend/src/app.js b/backend/src/app.js index 106666e6..593f86ea 100644 --- a/backend/src/app.js +++ b/backend/src/app.js @@ -1,11 +1,16 @@ import { getConfig } from "./config.js"; +import { loadKeywords } from "./db/keywordsStore.js"; import { exportToExcel, exportToPDF } from "./exporter.js"; import { logInfo } from "./logger.js"; import { searchJobsWithCache } from "./pipeline/searchJobsWithCache.js"; import { sources } from "./sources/index.js"; export async function run() { - const config = getConfig(); + const baseConfig = getConfig(); + const config = { + ...baseConfig, + keywords: await loadKeywords(baseConfig.keywords), + }; logInfo(`Localização da busca: ${config.searchLocation}`); logInfo(`Total de palavras-chave: ${config.keywords.length}`); diff --git a/backend/src/cache/cache.js b/backend/src/cache/cache.js index c483ef36..3b7c34a6 100644 --- a/backend/src/cache/cache.js +++ b/backend/src/cache/cache.js @@ -1,7 +1,33 @@ let redisClientPromise = null; +let redisClientUrl = ""; let redisWarningShown = false; +const cacheStatus = { + provider: "memory", + configured: false, + connected: false, + lastError: null, +}; + +function readRedisUrl() { + return process.env.REDIS_URL?.trim() || ""; +} + +function syncCacheStatus() { + const redisUrl = readRedisUrl(); + cacheStatus.configured = Boolean(redisUrl); + cacheStatus.provider = redisUrl ? "redis" : "memory"; + + if (!redisUrl) { + cacheStatus.connected = false; + cacheStatus.lastError = null; + } +} + function warnRedisFallback(message, error) { + cacheStatus.connected = false; + cacheStatus.lastError = error instanceof Error ? error.message : null; + if (redisWarningShown) { return; } @@ -13,25 +39,45 @@ function warnRedisFallback(message, error) { console.warn(`${message}${errorMessage ? ` (${errorMessage})` : ""}`); } -async function getRedisClient() { - const redisUrl = process.env.REDIS_URL?.trim(); +export async function getRedisClient() { + const redisUrl = readRedisUrl(); + syncCacheStatus(); if (!redisUrl) { return null; } + if (redisClientUrl !== redisUrl) { + redisClientPromise = null; + redisClientUrl = redisUrl; + } + if (!redisClientPromise) { redisClientPromise = import("redis") .then(async ({ createClient }) => { const client = createClient({ url: redisUrl, socket: { + connectTimeout: 3_000, reconnectStrategy(retries) { - return Math.min(retries * 50, 1_000); + if (retries >= 2) { + return false; + } + + return Math.min((retries + 1) * 100, 500); }, }, }); + client.on("ready", () => { + cacheStatus.connected = true; + cacheStatus.lastError = null; + }); + + client.on("end", () => { + cacheStatus.connected = false; + }); + client.on("error", (error) => { warnRedisFallback("Redis indisponivel, usando cache em memoria.", error); }); @@ -49,6 +95,37 @@ async function getRedisClient() { return redisClientPromise; } +export function getCacheStatus() { + syncCacheStatus(); + return { + ...cacheStatus, + type: cache.constructor.name, + }; +} + +export async function warmupCache() { + syncCacheStatus(); + + if (!cacheStatus.configured) { + return getCacheStatus(); + } + + const client = await getRedisClient(); + if (!client) { + return getCacheStatus(); + } + + try { + const pong = await client.ping(); + cacheStatus.connected = pong === "PONG"; + cacheStatus.lastError = cacheStatus.connected ? null : "PING sem resposta esperada"; + } catch (error) { + warnRedisFallback("Falha ao validar conexao com Redis.", error); + } + + return getCacheStatus(); +} + export class MemoryCache { constructor() { this.store = new Map(); diff --git a/backend/src/db/keywordsStore.js b/backend/src/db/keywordsStore.js new file mode 100644 index 00000000..d13a94fb --- /dev/null +++ b/backend/src/db/keywordsStore.js @@ -0,0 +1,126 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; +import path from "path"; +import { fileURLToPath } from "url"; +import { getRedisClient } from "../cache/cache.js"; + +const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); + +function getKeywordsStorageMode() { + const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "file") + .trim() + .toLowerCase(); + + return configuredMode === "env" ? "env" : "file"; +} + +function getKeywordsFilePath() { + const configuredPath = process.env.KEYWORDS_FILE_PATH?.trim(); + return configuredPath + ? path.resolve(configuredPath) + : path.resolve(MODULE_DIR, "environment.json"); +} + +function getKeywordsRedisKey() { + const configuredKey = process.env.KEYWORDS_REDIS_KEY?.trim(); + if (configuredKey) { + return configuredKey; + } + + const prefix = process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full"; + return `${prefix}:keywords`; +} + +export function normalizeKeywords(keywords) { + if (!Array.isArray(keywords)) { + return null; + } + + return [...new Set(keywords.map((item) => String(item ?? "").trim()).filter(Boolean))]; +} + +function parseKeywordsFromEnv(value) { + return normalizeKeywords( + String(value ?? "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + ) ?? []; +} + +function readKeywordsFromFile() { + const envPath = getKeywordsFilePath(); + + if (!existsSync(envPath)) { + return null; + } + + try { + const data = JSON.parse(readFileSync(envPath, "utf-8")); + return Array.isArray(data?.KEYWORDS) ? normalizeKeywords(data.KEYWORDS) ?? [] : []; + } catch { + return []; + } +} + +function writeKeywordsToFile(keywords) { + const envPath = getKeywordsFilePath(); + mkdirSync(path.dirname(envPath), { recursive: true }); + writeFileSync(envPath, JSON.stringify({ KEYWORDS: keywords }, null, 2), "utf-8"); +} + +export async function loadKeywords(fallback = []) { + const client = await getRedisClient(); + + if (client) { + try { + const raw = await client.get(getKeywordsRedisKey()); + if (raw) { + const parsed = JSON.parse(raw); + + if (Array.isArray(parsed)) { + return normalizeKeywords(parsed) ?? []; + } + + if (Array.isArray(parsed?.KEYWORDS)) { + return normalizeKeywords(parsed.KEYWORDS) ?? []; + } + } + } catch { + // fallback below + } + } + + const envKeywords = parseKeywordsFromEnv(process.env.SEARCH_KEYWORDS); + if (envKeywords.length > 0) { + return envKeywords; + } + + const fileKeywords = readKeywordsFromFile(); + if (fileKeywords !== null) { + return fileKeywords; + } + + return normalizeKeywords(fallback) ?? []; +} + +export async function saveKeywords(keywords) { + const normalizedKeywords = normalizeKeywords(keywords); + + if (normalizedKeywords === null) { + return null; + } + + const client = await getRedisClient(); + if (client) { + await client.set(getKeywordsRedisKey(), JSON.stringify(normalizedKeywords)); + return normalizedKeywords; + } + + if (getKeywordsStorageMode() === "env") { + process.env.SEARCH_KEYWORDS = normalizedKeywords.join(","); + return normalizedKeywords; + } + + writeKeywordsToFile(normalizedKeywords); + return normalizedKeywords; +} diff --git a/backend/src/jobsApiApp.js b/backend/src/jobsApiApp.js index c08f29c6..cb6eb5d7 100644 --- a/backend/src/jobsApiApp.js +++ b/backend/src/jobsApiApp.js @@ -1,96 +1,22 @@ import cors from "cors"; import express from "express"; -import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "fs"; +import { existsSync, mkdirSync, readdirSync, statSync } from "fs"; import path from "path"; -import { fileURLToPath } from "url"; import XLSX from "xlsx"; import { run as runScraper } from "./app.js"; +import { getCacheStatus } from "./cache/cache.js"; import { getConfig } from "./config.js"; +import { loadKeywords, normalizeKeywords, saveKeywords } from "./db/keywordsStore.js"; import { searchJobsWithCache } from "./pipeline/searchJobsWithCache.js"; import { sources } from "./sources/index.js"; -const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); - const DEFAULT_ALLOWED_ORIGINS = [ "https://painel-vagas-lake.vercel.app", + "https://painel-vagas-m6hbzlqeh-bene-teslas-projects.vercel.app", "http://localhost:5173", "http://localhost:5174", ]; -function getKeywordsStorageMode() { - const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "file") - .trim() - .toLowerCase(); - - return configuredMode === "env" ? "env" : "file"; -} - -function getKeywordsFilePath() { - const configuredPath = process.env.KEYWORDS_FILE_PATH?.trim(); - return configuredPath - ? path.resolve(configuredPath) - : path.resolve(MODULE_DIR, "db", "environment.json"); -} - -function normalizeKeywords(keywords) { - if (!Array.isArray(keywords)) { - return null; - } - - return [...new Set(keywords.map((item) => String(item ?? "").trim()).filter(Boolean))]; -} - -function parseKeywordsFromEnv(value) { - return normalizeKeywords( - String(value ?? "") - .split(",") - .map((item) => item.trim()) - .filter(Boolean), - ) ?? []; -} - -function readEnvironmentData() { - if (getKeywordsStorageMode() === "env") { - return { KEYWORDS: parseKeywordsFromEnv(process.env.SEARCH_KEYWORDS) }; - } - - const envPath = getKeywordsFilePath(); - - if (!existsSync(envPath)) { - return { KEYWORDS: [] }; - } - - try { - const data = JSON.parse(readFileSync(envPath, "utf-8")); - return data && typeof data === "object" ? data : { KEYWORDS: [] }; - } catch { - return { KEYWORDS: [] }; - } -} - -function writeEnvironmentData(data) { - const normalizedKeywords = normalizeKeywords(data?.KEYWORDS) ?? []; - - if (getKeywordsStorageMode() === "env") { - process.env.SEARCH_KEYWORDS = normalizedKeywords.join(","); - return { - ...(data && typeof data === "object" ? data : {}), - KEYWORDS: normalizedKeywords, - }; - } - - const envPath = getKeywordsFilePath(); - mkdirSync(path.dirname(envPath), { recursive: true }); - - const nextData = { - ...(data && typeof data === "object" ? data : {}), - KEYWORDS: normalizedKeywords, - }; - - writeFileSync(envPath, JSON.stringify(nextData, null, 2), "utf-8"); - return nextData; -} - function parseAllowedOrigins(value) { const configuredOrigins = String(value ?? "") .split(",") @@ -187,7 +113,10 @@ export function createJobsApiApp(options = {}) { * description: API funcionando corretamente */ app.get("/api/health", (_req, res) => { - res.json({ ok: true }); + res.json({ + ok: true, + cache: getCacheStatus(), + }); }); /** @@ -237,16 +166,18 @@ export function createJobsApiApp(options = {}) { // Novo endpoint para busca de vagas com cache app.get("/api/jobs/search", async (req, res) => { try { + const baseConfig = getConfig(); const config = { - ...getConfig(), + ...baseConfig, keywords: req.query.keywords ? String(req.query.keywords) .split(",") .map((k) => k.trim()) - : getConfig().keywords, + .filter(Boolean) + : await loadKeywords(baseConfig.keywords), }; - const ttlMs = 10 * 60 * 1000; // 10 minutos + const ttlMs = baseConfig.cacheTtlMs; const result = await searchJobsWithCache(sources, config, ttlMs); return res.json(result); @@ -333,7 +264,7 @@ export function createJobsApiApp(options = {}) { * 400: * description: Dados inválidos */ - app.post("/api/keywords", (req, res) => { + app.post("/api/keywords", async (req, res) => { try { const normalizedKeywords = normalizeKeywords(req.body?.keywords); @@ -343,15 +274,12 @@ export function createJobsApiApp(options = {}) { }); } - const envData = writeEnvironmentData({ - ...readEnvironmentData(), - KEYWORDS: normalizedKeywords, - }); + const savedKeywords = await saveKeywords(normalizedKeywords); return res.json({ ok: true, message: "Keywords atualizadas com sucesso.", - keywords: envData.KEYWORDS, + keywords: savedKeywords, }); } catch (error) { return res.status(500).json({ @@ -371,13 +299,13 @@ export function createJobsApiApp(options = {}) { * 200: * description: Lista de keywords */ - app.get("/api/keywords", (_req, res) => { + app.get("/api/keywords", async (_req, res) => { try { - const envData = readEnvironmentData(); + const keywords = await loadKeywords(getConfig().keywords); return res.json({ ok: true, - keywords: normalizeKeywords(envData.KEYWORDS) ?? [], + keywords, }); } catch (error) { return res.status(500).json({ diff --git a/backend/src/server.js b/backend/src/server.js index fed890d8..c63217e8 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -2,6 +2,7 @@ import "dotenv/config"; import express from "express"; import { existsSync } from "fs"; import path from "path"; +import { warmupCache } from "./cache/cache.js"; import { createJobsApiApp } from "./jobsApiApp.js"; const PORT = Number(process.env.PORT || 3001); @@ -32,6 +33,14 @@ async function registerSwaggerDocs() { async function startServer() { await registerSwaggerDocs(); + const cacheStatus = await warmupCache(); + // eslint-disable-next-line no-console + console.log( + cacheStatus.configured + ? `Cache provider: ${cacheStatus.provider} (${cacheStatus.connected ? "conectado" : "fallback em memoria"})` + : "Cache provider: memory (REDIS_URL nao configurada)", + ); + // ── Electron static-file serving ───────────────────────────────────────── // When ELECTRON_STATIC_DIR is set, serve the React production build so that // the Electron window can load everything from http://localhost:. diff --git a/backend/tests/integration/jobsApi.test.js b/backend/tests/integration/jobsApi.test.js index 486b9b85..6765a82b 100644 --- a/backend/tests/integration/jobsApi.test.js +++ b/backend/tests/integration/jobsApi.test.js @@ -24,20 +24,30 @@ describe("jobs API", () => { delete process.env.KEYWORDS_FILE_PATH; delete process.env.KEYWORDS_STORAGE_MODE; delete process.env.SEARCH_KEYWORDS; + delete process.env.REDIS_URL; + delete process.env.KEYWORDS_REDIS_KEY; }); afterEach(() => { delete process.env.KEYWORDS_FILE_PATH; delete process.env.KEYWORDS_STORAGE_MODE; delete process.env.SEARCH_KEYWORDS; + delete process.env.REDIS_URL; + delete process.env.KEYWORDS_REDIS_KEY; tmpDir = undefined; }); - it("GET /api/health retorna ok", async () => { + it("GET /api/health retorna ok com status do cache", async () => { tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); const app = createJobsApiApp({ outputDir: tmpDir }); const res = await request(app).get("/api/health").expect(200); - expect(res.body).toEqual({ ok: true }); + + expect(res.body.ok).toBe(true); + expect(res.body.cache).toMatchObject({ + configured: false, + connected: false, + provider: "memory", + }); }); it("GET /api/jobs/files retorna lista vazia sem xlsx", async () => { From 1a201243328cfd9942663999f7d718cfb9b4f19f Mon Sep 17 00:00:00 2001 From: Benevanio Santos Date: Thu, 9 Apr 2026 18:05:15 -0300 Subject: [PATCH 6/9] chore: allow VPS Redis runtime config --- backend/.env.example | 3 ++- backend/src/jobsApiApp.js | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/.env.example b/backend/.env.example index 85566535..220f7037 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -3,7 +3,7 @@ NODE_ENV=production PORT=3001 # CORS / frontend access -CORS_ALLOWED_ORIGINS=https://painel-vagas-lake.vercel.app,http://localhost:5173,http://localhost:5174 +CORS_ALLOWED_ORIGINS=https://painel-vagas-lake.vercel.app,https://painel-vagas-m6hbzlqeh-bene-teslas-projects.vercel.app,http://localhost:5173,http://localhost:5174,http://jobsglobalscraper.ddns.net,https://jobsglobalscraper.ddns.net # LinkedIn search filters SEARCH_LOCATION=Brasil @@ -23,6 +23,7 @@ SEARCH_KEYWORDS=UX Designer,UI Designer,Product Manager,Product Owner DATABASE_URL=postgresql://app_user:change_me@127.0.0.1:5432/jobsglobalscraper REDIS_URL=redis://:change_me@127.0.0.1:6379/0 REDIS_KEY_PREFIX=vagas-full +KEYWORDS_REDIS_KEY=vagas-full:keywords CACHE_TTL_MS=600000 # Scraping behavior diff --git a/backend/src/jobsApiApp.js b/backend/src/jobsApiApp.js index cb6eb5d7..4c6a9ab1 100644 --- a/backend/src/jobsApiApp.js +++ b/backend/src/jobsApiApp.js @@ -13,6 +13,8 @@ import { sources } from "./sources/index.js"; const DEFAULT_ALLOWED_ORIGINS = [ "https://painel-vagas-lake.vercel.app", "https://painel-vagas-m6hbzlqeh-bene-teslas-projects.vercel.app", + "https://jobsglobalscraper.ddns.net", + "http://jobsglobalscraper.ddns.net", "http://localhost:5173", "http://localhost:5174", ]; From eeb87e0b4c33d0a7b153fa60d2bf753357ce9e3b Mon Sep 17 00:00:00 2001 From: Benevanio Santos Date: Thu, 9 Apr 2026 18:21:56 -0300 Subject: [PATCH 7/9] test: add mocks for keywords and cache in unit tests --- backend/tests/unit/services/app.test.js | 3 +++ backend/tests/unit/services/server.test.js | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/tests/unit/services/app.test.js b/backend/tests/unit/services/app.test.js index e3a82ecd..45dcfce6 100644 --- a/backend/tests/unit/services/app.test.js +++ b/backend/tests/unit/services/app.test.js @@ -19,6 +19,9 @@ const mocks = vi.hoisted(() => ({ })); vi.mock("../../../src/config.js", () => ({ getConfig: mocks.getConfigMock })); +vi.mock("../../../src/db/keywordsStore.js", () => ({ + loadKeywords: vi.fn(async (fallback = []) => fallback), +})); vi.mock("../../../src/pipeline/searchJobsWithCache.js", () => ({ searchJobsWithCache: mocks.searchJobsWithCacheMock, })); diff --git a/backend/tests/unit/services/server.test.js b/backend/tests/unit/services/server.test.js index 2e270163..8f8a48e9 100644 --- a/backend/tests/unit/services/server.test.js +++ b/backend/tests/unit/services/server.test.js @@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({ expressStatic: vi.fn(), swaggerServe: vi.fn(), swaggerSetup: vi.fn(() => vi.fn()), + warmupCache: vi.fn(), })); mocks.createJobsApiApp.mockReturnValue({ @@ -21,9 +22,13 @@ mocks.createJobsApiApp.mockReturnValue({ }); mocks.expressStatic.mockReturnValue(mocks.staticMiddleware); - +mocks.warmupCache.mockResolvedValue({ provider: "memory", configured: false, connected: false }); vi.mock("dotenv/config", () => ({})); +vi.mock("../../../src/cache/cache.js", () => ({ + warmupCache: mocks.warmupCache, +})); + vi.mock("../../../src/jobsApiApp.js", () => ({ createJobsApiApp: mocks.createJobsApiApp, })); @@ -53,6 +58,8 @@ async function importServerEntry() { await import("../../../src/server.js"); await vi.dynamicImportSettled(); await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); } describe("server entry", () => { From 80838b55d147efc8e0472fe57615d9b8f3b44628 Mon Sep 17 00:00:00 2001 From: Benevanio Santos Date: Thu, 9 Apr 2026 18:44:57 -0300 Subject: [PATCH 8/9] test: enhance integration and unit tests for Redis and keywords functionality --- backend/tests/integration/jobsApi.test.js | 148 ++++++++++++- backend/tests/unit/services/app.test.js | 19 +- .../tests/unit/services/cache/cache.test.js | 208 +++++++++++++++++- .../unit/services/db/keywordsStore.test.js | 141 ++++++++++++ 4 files changed, 507 insertions(+), 9 deletions(-) create mode 100644 backend/tests/unit/services/db/keywordsStore.test.js diff --git a/backend/tests/integration/jobsApi.test.js b/backend/tests/integration/jobsApi.test.js index 6765a82b..75cf43d1 100644 --- a/backend/tests/integration/jobsApi.test.js +++ b/backend/tests/integration/jobsApi.test.js @@ -7,12 +7,36 @@ import XLSX from "xlsx"; const mocks = vi.hoisted(() => ({ run: vi.fn(), + searchJobsWithCache: vi.fn(), + loadKeywords: vi.fn(), + saveKeywords: vi.fn(), + getCacheStatus: vi.fn(), })); vi.mock("../../src/app.js", () => ({ run: mocks.run, })); +vi.mock("../../src/pipeline/searchJobsWithCache.js", () => ({ + searchJobsWithCache: mocks.searchJobsWithCache, +})); + +vi.mock("../../src/db/keywordsStore.js", () => ({ + loadKeywords: mocks.loadKeywords, + normalizeKeywords: (keywords) => { + if (!Array.isArray(keywords)) { + return null; + } + + return [...new Set(keywords.map((item) => String(item ?? "").trim()).filter(Boolean))]; + }, + saveKeywords: mocks.saveKeywords, +})); + +vi.mock("../../src/cache/cache.js", () => ({ + getCacheStatus: mocks.getCacheStatus, +})); + import { createJobsApiApp } from "../../src/jobsApiApp.js"; describe("jobs API", () => { @@ -21,6 +45,26 @@ describe("jobs API", () => { beforeEach(() => { vi.clearAllMocks(); mocks.run.mockResolvedValue(undefined); + mocks.searchJobsWithCache.mockResolvedValue({ + jobs: [{ titulo: "Dev", empresa: "ACME" }], + total: 1, + fromCache: false, + cachedAt: "2026-04-09T00:00:00.000Z", + }); + mocks.loadKeywords.mockResolvedValue(["Java", "Spring", "RabbitMQ", "Docker"]); + mocks.saveKeywords.mockImplementation(async (keywords) => { + if (process.env.KEYWORDS_STORAGE_MODE === "env") { + process.env.SEARCH_KEYWORDS = keywords.join(","); + } + + return keywords; + }); + mocks.getCacheStatus.mockReturnValue({ + configured: false, + connected: false, + provider: "memory", + type: "MemoryCache", + }); delete process.env.KEYWORDS_FILE_PATH; delete process.env.KEYWORDS_STORAGE_MODE; delete process.env.SEARCH_KEYWORDS; @@ -242,5 +286,107 @@ describe("jobs API", () => { keywords: ["Java","Spring","RabbitMQ","Docker"] }); }); - + + it("GET /api/jobs/search usa keywords da query quando fornecidas", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app) + .get("/api/jobs/search") + .query({ keywords: " Node, React ,, " }) + .expect(200); + + expect(res.body.total).toBe(1); + expect(mocks.loadKeywords).not.toHaveBeenCalled(); + expect(mocks.searchJobsWithCache).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ keywords: ["Node", "React"] }), + expect.anything(), + ); + }); + + it("GET /api/jobs/search usa keywords salvas quando a query não é enviada", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + mocks.loadKeywords.mockResolvedValueOnce(["Redis", "PostgreSQL"]); + const app = createJobsApiApp({ outputDir: tmpDir }); + + await request(app).get("/api/jobs/search").expect(200); + + expect(mocks.loadKeywords).toHaveBeenCalledTimes(1); + expect(mocks.searchJobsWithCache).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ keywords: ["Redis", "PostgreSQL"] }), + expect.anything(), + ); + }); + + it("GET /api/jobs/search retorna 500 quando a busca falha", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + mocks.searchJobsWithCache.mockRejectedValueOnce(new Error("falha na busca")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app) + .get("/api/jobs/search") + .query({ keywords: "Java" }) + .expect(500); + + expect(res.body).toEqual({ + message: "Erro ao buscar vagas.", + error: "falha na busca", + }); + }); + + it("POST /api/keywords retorna 400 quando keywords não é array", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app) + .post("/api/keywords") + .send({ keywords: "Java" }) + .expect(400); + + expect(res.body.message).toBe("O campo 'keywords' deve ser um array de strings."); + }); + + it("POST /api/keywords retorna 500 quando saveKeywords falha", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + mocks.saveKeywords.mockRejectedValueOnce(new Error("redis down")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app) + .post("/api/keywords") + .send({ keywords: ["Java"] }) + .expect(500); + + expect(res.body).toEqual({ + message: "Erro ao salvar keywords.", + error: "redis down", + }); + }); + + it("GET /api/keywords retorna 500 quando loadKeywords falha", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + mocks.loadKeywords.mockRejectedValueOnce(new Error("read failed")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app).get("/api/keywords").expect(500); + + expect(res.body).toEqual({ + message: "Erro ao buscar keywords.", + error: "read failed", + }); + }); + + it("adiciona HSTS quando a requisição chega por HTTPS via proxy", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app) + .get("/api/health") + .set("x-forwarded-proto", "https") + .expect(200); + + expect(res.headers["strict-transport-security"]).toContain("max-age=31536000"); + }); + }); diff --git a/backend/tests/unit/services/app.test.js b/backend/tests/unit/services/app.test.js index 45dcfce6..a5fc89ce 100644 --- a/backend/tests/unit/services/app.test.js +++ b/backend/tests/unit/services/app.test.js @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ getConfigMock: vi.fn(() => ({ @@ -37,6 +37,10 @@ vi.mock("../../../src/logger.js", () => ({ logInfo: mocks.logInfoMock })); import { run } from "../../../src/app.js"; describe("run", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + it("orquestra coleta e exportacao", async () => { await run(); @@ -51,5 +55,18 @@ describe("run", () => { "output/a.pdf", ); expect(mocks.logInfoMock).toHaveBeenCalled(); + expect(mocks.logInfoMock).toHaveBeenCalledWith("Resultado do cache: MISS"); + }); + + it("registra HIT quando o resultado vem do cache", async () => { + mocks.searchJobsWithCacheMock.mockResolvedValueOnce({ + jobs: [{ titulo: "Dev", palavra: "React" }], + total: 1, + fromCache: true, + }); + + await run(); + + expect(mocks.logInfoMock).toHaveBeenCalledWith("Resultado do cache: HIT"); }); }); diff --git a/backend/tests/unit/services/cache/cache.test.js b/backend/tests/unit/services/cache/cache.test.js index 73b95184..6057b209 100644 --- a/backend/tests/unit/services/cache/cache.test.js +++ b/backend/tests/unit/services/cache/cache.test.js @@ -1,12 +1,64 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { MemoryCache } from "../../../../src/cache/cache.js"; + +const mocks = vi.hoisted(() => ({ + createClient: vi.fn(), + consoleWarn: vi.fn(), +})); + +vi.mock("redis", () => ({ + createClient: mocks.createClient, +})); + +function buildRedisClientMock(options = {}) { + const handlers = {}; + const client = { + on: vi.fn((event, handler) => { + handlers[event] = handler; + return client; + }), + connect: vi.fn(async () => { + if (options.connectError) { + throw options.connectError; + } + + if (options.triggerReady !== false) { + handlers.ready?.(); + } + }), + ping: vi.fn(async () => options.ping ?? "PONG"), + get: vi.fn(async () => options.get ?? null), + set: vi.fn(async () => "OK"), + del: vi.fn(async () => 1), + scanIterator: vi.fn(async function* () { + if (options.scanError) { + throw options.scanError; + } + + for (const key of options.scanKeys ?? []) { + yield key; + } + }), + }; + + return { client, handlers }; +} + +async function importCacheModule() { + return import("../../../../src/cache/cache.js"); +} describe("MemoryCache", () => { let memoryCache; - beforeEach(() => { - memoryCache = new MemoryCache(); + beforeEach(async () => { + vi.resetModules(); + vi.clearAllMocks(); vi.restoreAllMocks(); + delete process.env.REDIS_URL; + delete process.env.REDIS_KEY_PREFIX; + + const { MemoryCache } = await importCacheModule(); + memoryCache = new MemoryCache(); }); it("retorna null quando a chave não existe", () => { @@ -24,10 +76,10 @@ describe("MemoryCache", () => { it("remove e retorna null quando o item expirou", () => { const nowSpy = vi.spyOn(Date, "now"); - nowSpy.mockReturnValueOnce(1000); // set + nowSpy.mockReturnValueOnce(1000); memoryCache.set("jobs:node", [{ titulo: "Dev Node" }], 500); - nowSpy.mockReturnValueOnce(1601); // get + nowSpy.mockReturnValueOnce(1601); expect(memoryCache.get("jobs:node")).toBeNull(); expect(memoryCache.store.has("jobs:node")).toBe(false); @@ -66,10 +118,152 @@ describe("MemoryCache", () => { it("has retorna false quando a chave expirou", () => { const nowSpy = vi.spyOn(Date, "now"); - nowSpy.mockReturnValueOnce(2000); // set + nowSpy.mockReturnValueOnce(2000); memoryCache.set("jobs", ["vaga 1"], 100); - nowSpy.mockReturnValueOnce(2201); // has -> get + nowSpy.mockReturnValueOnce(2201); expect(memoryCache.has("jobs")).toBe(false); }); +}); + +describe("Redis cache helpers", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + vi.restoreAllMocks(); + delete process.env.REDIS_URL; + delete process.env.REDIS_KEY_PREFIX; + vi.spyOn(console, "warn").mockImplementation(mocks.consoleWarn); + }); + + it("retorna status de memoria quando REDIS_URL não está configurada", async () => { + const { MemoryCache, cache, getCacheStatus, getRedisClient, warmupCache } = await importCacheModule(); + + expect(await getRedisClient()).toBeNull(); + expect(cache).toBeInstanceOf(MemoryCache); + expect(getCacheStatus()).toMatchObject({ + provider: "memory", + configured: false, + connected: false, + type: "MemoryCache", + }); + expect(await warmupCache()).toMatchObject({ + provider: "memory", + configured: false, + connected: false, + }); + expect(mocks.createClient).not.toHaveBeenCalled(); + }); + + it("conecta ao Redis, reaproveita o cliente e executa operações no RedisCache", async () => { + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + process.env.REDIS_KEY_PREFIX = "jobs-test"; + + const { client, handlers } = buildRedisClientMock({ + scanKeys: ["jobs-test:a", "jobs-test:b"], + }); + mocks.createClient.mockReturnValue(client); + + const { RedisCache, getCacheStatus, getRedisClient, warmupCache } = await importCacheModule(); + + const firstClient = await getRedisClient(); + const secondClient = await getRedisClient(); + expect(firstClient).toBe(client); + expect(secondClient).toBe(client); + expect(mocks.createClient).toHaveBeenCalledTimes(1); + + const socketConfig = mocks.createClient.mock.calls[0][0].socket; + expect(socketConfig.reconnectStrategy(0)).toBe(100); + expect(socketConfig.reconnectStrategy(3)).toBe(false); + + const redisCache = new RedisCache({ prefix: "jobs-test" }); + await redisCache.set("react", { ok: true }, 5000); + client.get.mockResolvedValueOnce(JSON.stringify({ ok: true })); + expect(await redisCache.get("react")).toEqual({ ok: true }); + + await redisCache.delete("react"); + await redisCache.clear(); + + expect(client.set).toHaveBeenCalledWith("jobs-test:react", JSON.stringify({ ok: true }), { + PX: 5000, + }); + expect(client.del).toHaveBeenCalledWith("jobs-test:react"); + expect(client.del).toHaveBeenCalledWith(["jobs-test:a", "jobs-test:b"]); + + expect(await warmupCache()).toMatchObject({ + provider: "redis", + configured: true, + connected: true, + type: "RedisCache", + }); + + handlers.end?.(); + expect(getCacheStatus().connected).toBe(false); + + process.env.REDIS_URL = "redis://127.0.0.1:6380"; + await getRedisClient(); + expect(mocks.createClient).toHaveBeenCalledTimes(2); + }); + + it("faz fallback quando a conexão com Redis falha", async () => { + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + + const { client } = buildRedisClientMock({ + connectError: new Error("connect failed"), + triggerReady: false, + }); + mocks.createClient.mockReturnValue(client); + + const { getCacheStatus, getRedisClient } = await importCacheModule(); + + await expect(getRedisClient()).resolves.toBeNull(); + expect(mocks.consoleWarn).toHaveBeenCalledTimes(1); + expect(mocks.consoleWarn.mock.calls[0][0]).toContain("Falha ao conectar no Redis"); + expect(getCacheStatus()).toMatchObject({ + provider: "redis", + configured: true, + connected: false, + lastError: "connect failed", + }); + }); + + it("usa fallback em memória quando operações do Redis lançam erro", async () => { + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + + const { client } = buildRedisClientMock(); + client.get.mockResolvedValueOnce("{invalid-json"); + client.set.mockRejectedValueOnce(new Error("set failed")); + client.del.mockRejectedValueOnce(new Error("del failed")); + client.scanIterator.mockImplementationOnce(async function* () { + throw new Error("scan failed"); + }); + mocks.createClient.mockReturnValue(client); + + const { RedisCache } = await importCacheModule(); + + const redisCache = new RedisCache({ prefix: "jobs-test" }); + redisCache.memoryFallback.set("node", { cached: true }, 1000); + + await expect(redisCache.get("node")).resolves.toEqual({ cached: true }); + await expect(redisCache.set("node", { cached: false }, 1000)).resolves.toBeUndefined(); + await expect(redisCache.delete("node")).resolves.toBeUndefined(); + await expect(redisCache.clear()).resolves.toBeUndefined(); + + expect(mocks.consoleWarn).toHaveBeenCalledTimes(1); + }); + + it("marca o status como desconectado quando o ping não retorna PONG", async () => { + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + + const { client } = buildRedisClientMock({ ping: "NOPE" }); + mocks.createClient.mockReturnValue(client); + + const { warmupCache } = await importCacheModule(); + + await expect(warmupCache()).resolves.toMatchObject({ + configured: true, + connected: false, + lastError: "PING sem resposta esperada", + }); + }); }); \ No newline at end of file diff --git a/backend/tests/unit/services/db/keywordsStore.test.js b/backend/tests/unit/services/db/keywordsStore.test.js new file mode 100644 index 00000000..ae124d20 --- /dev/null +++ b/backend/tests/unit/services/db/keywordsStore.test.js @@ -0,0 +1,141 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + existsSync: vi.fn(), + mkdirSync: vi.fn(), + readFileSync: vi.fn(), + writeFileSync: vi.fn(), + getRedisClient: vi.fn(), +})); + +vi.mock("fs", () => ({ + existsSync: mocks.existsSync, + mkdirSync: mocks.mkdirSync, + readFileSync: mocks.readFileSync, + writeFileSync: mocks.writeFileSync, +})); + +vi.mock("../../../../src/cache/cache.js", () => ({ + getRedisClient: mocks.getRedisClient, +})); + +async function importKeywordsStore() { + return import("../../../../src/db/keywordsStore.js"); +} + +describe("keywordsStore", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + delete process.env.SEARCH_KEYWORDS; + delete process.env.KEYWORDS_STORAGE_MODE; + delete process.env.KEYWORDS_REDIS_KEY; + delete process.env.REDIS_KEY_PREFIX; + delete process.env.KEYWORDS_FILE_PATH; + mocks.getRedisClient.mockResolvedValue(null); + mocks.existsSync.mockReturnValue(false); + }); + + it("normaliza keywords removendo vazios e duplicadas", async () => { + const { normalizeKeywords } = await importKeywordsStore(); + + expect(normalizeKeywords([" Node ", "React", "Node", "", null])).toEqual([ + "Node", + "React", + ]); + expect(normalizeKeywords("Node")).toBeNull(); + }); + + it("carrega keywords do Redis quando existe um array salvo", async () => { + process.env.KEYWORDS_REDIS_KEY = "custom:keywords"; + const client = { + get: vi.fn().mockResolvedValue(JSON.stringify([" Node ", "React", "Node"])), + }; + mocks.getRedisClient.mockResolvedValue(client); + + const { loadKeywords } = await importKeywordsStore(); + + await expect(loadKeywords(["fallback"])) .resolves.toEqual(["Node", "React"]); + expect(client.get).toHaveBeenCalledWith("custom:keywords"); + }); + + it("carrega keywords do formato legado KEYWORDS vindo do Redis", async () => { + const client = { + get: vi.fn().mockResolvedValue(JSON.stringify({ KEYWORDS: ["Java", "Spring", "Java"] })), + }; + mocks.getRedisClient.mockResolvedValue(client); + + const { loadKeywords } = await importKeywordsStore(); + + await expect(loadKeywords(["fallback"])) .resolves.toEqual(["Java", "Spring"]); + }); + + it("faz fallback para SEARCH_KEYWORDS quando Redis não retorna valor", async () => { + process.env.SEARCH_KEYWORDS = " Suporte, Node, Suporte "; + const client = { get: vi.fn().mockResolvedValue(null) }; + mocks.getRedisClient.mockResolvedValue(client); + + const { loadKeywords } = await importKeywordsStore(); + + await expect(loadKeywords(["fallback"])) .resolves.toEqual(["Suporte", "Node"]); + }); + + it("lê do arquivo quando env está vazio e retorna [] para JSON inválido", async () => { + process.env.KEYWORDS_FILE_PATH = "./tmp/keywords.json"; + mocks.existsSync.mockReturnValue(true); + mocks.readFileSync.mockReturnValueOnce(JSON.stringify({ KEYWORDS: ["QA", "QA", "Node"] })); + + const { loadKeywords } = await importKeywordsStore(); + + await expect(loadKeywords(["fallback"])) .resolves.toEqual(["QA", "Node"]); + expect(mocks.readFileSync).toHaveBeenCalledWith(expect.stringContaining("keywords.json"), "utf-8"); + + mocks.readFileSync.mockReturnValueOnce("{invalid-json"); + await expect(loadKeywords(["fallback"])) .resolves.toEqual([]); + }); + + it("usa o fallback recebido quando não encontra Redis, env nem arquivo", async () => { + const { loadKeywords } = await importKeywordsStore(); + + await expect(loadKeywords([" Produto ", "Produto", "Dados "])) .resolves.toEqual([ + "Produto", + "Dados", + ]); + }); + + it("salva no Redis usando a chave configurada", async () => { + process.env.REDIS_KEY_PREFIX = "vagas-full"; + const client = { set: vi.fn().mockResolvedValue("OK") }; + mocks.getRedisClient.mockResolvedValue(client); + + const { saveKeywords } = await importKeywordsStore(); + + await expect(saveKeywords([" Node ", "React", "Node"])) .resolves.toEqual(["Node", "React"]); + expect(client.set).toHaveBeenCalledWith("vagas-full:keywords", JSON.stringify(["Node", "React"])); + }); + + it("salva em SEARCH_KEYWORDS quando está em modo env e não há Redis", async () => { + process.env.KEYWORDS_STORAGE_MODE = "env"; + + const { saveKeywords } = await importKeywordsStore(); + + await expect(saveKeywords(["Backend", "Java", "Backend"])) .resolves.toEqual(["Backend", "Java"]); + expect(process.env.SEARCH_KEYWORDS).toBe("Backend,Java"); + await expect(saveKeywords("invalido")).resolves.toBeNull(); + }); + + it("salva em arquivo quando está em modo file e não há Redis", async () => { + process.env.KEYWORDS_STORAGE_MODE = "file"; + process.env.KEYWORDS_FILE_PATH = "./tmp/file-keywords.json"; + + const { saveKeywords } = await importKeywordsStore(); + + await expect(saveKeywords(["SRE", "Platform"])) .resolves.toEqual(["SRE", "Platform"]); + expect(mocks.mkdirSync).toHaveBeenCalledWith(expect.stringContaining("tmp"), { recursive: true }); + expect(mocks.writeFileSync).toHaveBeenCalledWith( + expect.stringContaining("file-keywords.json"), + JSON.stringify({ KEYWORDS: ["SRE", "Platform"] }, null, 2), + "utf-8", + ); + }); +}); \ No newline at end of file From 41cdc808c1d658846cd91ebac92047771edb43d8 Mon Sep 17 00:00:00 2001 From: Benevanio Santos Date: Thu, 9 Apr 2026 18:59:37 -0300 Subject: [PATCH 9/9] feat: update keywords storage to use Redis as primary source and adjust related configurations --- backend/.env.example | 6 +- backend/src/config.js | 69 +++---------- backend/src/db/keywordsStore.js | 99 ++++++------------- .../unit/services/db/keywordsStore.test.js | 55 ++--------- backend/tests/unit/utils/config.test.js | 32 ++---- 5 files changed, 62 insertions(+), 199 deletions(-) diff --git a/backend/.env.example b/backend/.env.example index 220f7037..b897a2d5 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -14,9 +14,9 @@ JOB_TYPES=C,F TIME_FILTER=r604800 # Keywords storage: -# - env => recomendado para VPS / Docker / projeto open source -# - file => mantém compatibilidade com backend/src/db/environment.json -KEYWORDS_STORAGE_MODE=env +# - redis => fonte de verdade para salvar e ler keywords +# SEARCH_KEYWORDS fica apenas como fallback inicial se a chave ainda estiver vazia no Redis +KEYWORDS_STORAGE_MODE=redis SEARCH_KEYWORDS=UX Designer,UI Designer,Product Manager,Product Owner # External services (nunca commitar valores reais) diff --git a/backend/src/config.js b/backend/src/config.js index 5216dbdb..08da5240 100644 --- a/backend/src/config.js +++ b/backend/src/config.js @@ -1,10 +1,4 @@ -import { existsSync, readFileSync } from "fs"; -import path from "path"; -import { fileURLToPath } from "url"; - -const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); - -const DEFAULT_KEYWORDS = [ +const DEFAULT_KEYWORDS = [ "Java", "JavaScript", "React", @@ -39,18 +33,11 @@ function parseNumber(value, fallback) { } function getKeywordsStorageMode() { - const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "file") + const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "redis") .trim() .toLowerCase(); - return configuredMode === "env" ? "env" : "file"; -} - -function getKeywordsFilePath() { - const configuredPath = process.env.KEYWORDS_FILE_PATH?.trim(); - return configuredPath - ? path.resolve(configuredPath) - : path.resolve(MODULE_DIR, "db", "environment.json"); + return configuredMode === "env" ? "env" : "redis"; } function normalizeKeywords(keywords) { @@ -62,46 +49,18 @@ function normalizeKeywords(keywords) { } function parseKeywordsFromEnv(value) { - if (!value) { - return null; - } - - const keywords = String(value) - .split(",") - .map((item) => item.trim()) - .filter(Boolean); - - return keywords.length > 0 ? keywords : null; + const keywords = normalizeKeywords( + String(value ?? "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + ); + + return keywords?.length ? keywords : null; } function parseKeywords(value) { - const keywordsStorageMode = getKeywordsStorageMode(); - - if (keywordsStorageMode === "env") { - const keywordsFromEnv = parseKeywordsFromEnv(value); - if (keywordsFromEnv) { - return keywordsFromEnv; - } - } - - try { - const envPath = getKeywordsFilePath(); - if (existsSync(envPath)) { - const data = JSON.parse(readFileSync(envPath, "utf-8")); - if (Array.isArray(data.KEYWORDS)) { - return normalizeKeywords(data.KEYWORDS) ?? []; - } - } - } catch { - // Se falhar, fallback - } - - const keywordsFromEnv = parseKeywordsFromEnv(value); - if (keywordsFromEnv) { - return keywordsFromEnv; - } - - return DEFAULT_KEYWORDS; + return parseKeywordsFromEnv(value) ?? DEFAULT_KEYWORDS; } function parseTimeFilter(value, fallback) { @@ -121,7 +80,7 @@ export function getConfig() { maxPagesPerKeyword: parseNumber(process.env.MAX_PAGES_PER_KEYWORD, 5), viewport: { width: parseNumber(process.env.VIEWPORT_WIDTH, 1280), - height: parseNumber(process.env.VIEWPORT_HEIGHT, 800) + height: parseNumber(process.env.VIEWPORT_HEIGHT, 800), }, outputFile: process.env.OUTPUT_FILE || "output/vagas_remoto.xlsx", pdfFile: process.env.PDF_FILE || "output/vagas_remoto.pdf", @@ -137,6 +96,6 @@ export function getConfig() { cacheTtlMs: parseNumber(process.env.CACHE_TTL_MS, 10 * 60 * 1000), databaseUrl: process.env.DATABASE_URL?.trim() || "", redisUrl: process.env.REDIS_URL?.trim() || "", - redisKeyPrefix: process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full" + redisKeyPrefix: process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full", }; } diff --git a/backend/src/db/keywordsStore.js b/backend/src/db/keywordsStore.js index d13a94fb..fa5e0764 100644 --- a/backend/src/db/keywordsStore.js +++ b/backend/src/db/keywordsStore.js @@ -1,25 +1,5 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; -import path from "path"; -import { fileURLToPath } from "url"; import { getRedisClient } from "../cache/cache.js"; -const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); - -function getKeywordsStorageMode() { - const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "file") - .trim() - .toLowerCase(); - - return configuredMode === "env" ? "env" : "file"; -} - -function getKeywordsFilePath() { - const configuredPath = process.env.KEYWORDS_FILE_PATH?.trim(); - return configuredPath - ? path.resolve(configuredPath) - : path.resolve(MODULE_DIR, "environment.json"); -} - function getKeywordsRedisKey() { const configuredKey = process.env.KEYWORDS_REDIS_KEY?.trim(); if (configuredKey) { @@ -47,60 +27,43 @@ function parseKeywordsFromEnv(value) { ) ?? []; } -function readKeywordsFromFile() { - const envPath = getKeywordsFilePath(); - - if (!existsSync(envPath)) { - return null; - } - - try { - const data = JSON.parse(readFileSync(envPath, "utf-8")); - return Array.isArray(data?.KEYWORDS) ? normalizeKeywords(data.KEYWORDS) ?? [] : []; - } catch { - return []; +function getFallbackKeywords(fallback = []) { + const envKeywords = parseKeywordsFromEnv(process.env.SEARCH_KEYWORDS); + if (envKeywords.length > 0) { + return envKeywords; } -} -function writeKeywordsToFile(keywords) { - const envPath = getKeywordsFilePath(); - mkdirSync(path.dirname(envPath), { recursive: true }); - writeFileSync(envPath, JSON.stringify({ KEYWORDS: keywords }, null, 2), "utf-8"); + return normalizeKeywords(fallback) ?? []; } export async function loadKeywords(fallback = []) { + const fallbackKeywords = getFallbackKeywords(fallback); const client = await getRedisClient(); - if (client) { - try { - const raw = await client.get(getKeywordsRedisKey()); - if (raw) { - const parsed = JSON.parse(raw); - - if (Array.isArray(parsed)) { - return normalizeKeywords(parsed) ?? []; - } - - if (Array.isArray(parsed?.KEYWORDS)) { - return normalizeKeywords(parsed.KEYWORDS) ?? []; - } - } - } catch { - // fallback below - } + if (!client) { + return fallbackKeywords; } - const envKeywords = parseKeywordsFromEnv(process.env.SEARCH_KEYWORDS); - if (envKeywords.length > 0) { - return envKeywords; - } + try { + const raw = await client.get(getKeywordsRedisKey()); + if (!raw) { + return fallbackKeywords; + } - const fileKeywords = readKeywordsFromFile(); - if (fileKeywords !== null) { - return fileKeywords; + const parsed = JSON.parse(raw); + + if (Array.isArray(parsed)) { + return normalizeKeywords(parsed) ?? []; + } + + if (Array.isArray(parsed?.KEYWORDS)) { + return normalizeKeywords(parsed.KEYWORDS) ?? []; + } + } catch { + // fallback below } - return normalizeKeywords(fallback) ?? []; + return fallbackKeywords; } export async function saveKeywords(keywords) { @@ -111,16 +74,10 @@ export async function saveKeywords(keywords) { } const client = await getRedisClient(); - if (client) { - await client.set(getKeywordsRedisKey(), JSON.stringify(normalizedKeywords)); - return normalizedKeywords; - } - - if (getKeywordsStorageMode() === "env") { - process.env.SEARCH_KEYWORDS = normalizedKeywords.join(","); - return normalizedKeywords; + if (!client) { + throw new Error("Redis indisponivel para salvar keywords."); } - writeKeywordsToFile(normalizedKeywords); + await client.set(getKeywordsRedisKey(), JSON.stringify(normalizedKeywords)); return normalizedKeywords; } diff --git a/backend/tests/unit/services/db/keywordsStore.test.js b/backend/tests/unit/services/db/keywordsStore.test.js index ae124d20..95577e03 100644 --- a/backend/tests/unit/services/db/keywordsStore.test.js +++ b/backend/tests/unit/services/db/keywordsStore.test.js @@ -1,20 +1,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ - existsSync: vi.fn(), - mkdirSync: vi.fn(), - readFileSync: vi.fn(), - writeFileSync: vi.fn(), getRedisClient: vi.fn(), })); -vi.mock("fs", () => ({ - existsSync: mocks.existsSync, - mkdirSync: mocks.mkdirSync, - readFileSync: mocks.readFileSync, - writeFileSync: mocks.writeFileSync, -})); - vi.mock("../../../../src/cache/cache.js", () => ({ getRedisClient: mocks.getRedisClient, })); @@ -33,7 +22,6 @@ describe("keywordsStore", () => { delete process.env.REDIS_KEY_PREFIX; delete process.env.KEYWORDS_FILE_PATH; mocks.getRedisClient.mockResolvedValue(null); - mocks.existsSync.mockReturnValue(false); }); it("normaliza keywords removendo vazios e duplicadas", async () => { @@ -80,27 +68,18 @@ describe("keywordsStore", () => { await expect(loadKeywords(["fallback"])) .resolves.toEqual(["Suporte", "Node"]); }); - it("lê do arquivo quando env está vazio e retorna [] para JSON inválido", async () => { - process.env.KEYWORDS_FILE_PATH = "./tmp/keywords.json"; - mocks.existsSync.mockReturnValue(true); - mocks.readFileSync.mockReturnValueOnce(JSON.stringify({ KEYWORDS: ["QA", "QA", "Node"] })); - - const { loadKeywords } = await importKeywordsStore(); - - await expect(loadKeywords(["fallback"])) .resolves.toEqual(["QA", "Node"]); - expect(mocks.readFileSync).toHaveBeenCalledWith(expect.stringContaining("keywords.json"), "utf-8"); - - mocks.readFileSync.mockReturnValueOnce("{invalid-json"); - await expect(loadKeywords(["fallback"])) .resolves.toEqual([]); - }); - - it("usa o fallback recebido quando não encontra Redis, env nem arquivo", async () => { + it("usa o fallback recebido quando Redis está indisponível ou retorna JSON inválido", async () => { const { loadKeywords } = await importKeywordsStore(); await expect(loadKeywords([" Produto ", "Produto", "Dados "])) .resolves.toEqual([ "Produto", "Dados", ]); + + const client = { get: vi.fn().mockResolvedValue("{invalid-json") }; + mocks.getRedisClient.mockResolvedValue(client); + + await expect(loadKeywords(["QA", "Dados"])) .resolves.toEqual(["QA", "Dados"]); }); it("salva no Redis usando a chave configurada", async () => { @@ -114,28 +93,12 @@ describe("keywordsStore", () => { expect(client.set).toHaveBeenCalledWith("vagas-full:keywords", JSON.stringify(["Node", "React"])); }); - it("salva em SEARCH_KEYWORDS quando está em modo env e não há Redis", async () => { - process.env.KEYWORDS_STORAGE_MODE = "env"; - + it("retorna null para payload inválido e falha quando Redis está indisponível", async () => { const { saveKeywords } = await importKeywordsStore(); - await expect(saveKeywords(["Backend", "Java", "Backend"])) .resolves.toEqual(["Backend", "Java"]); - expect(process.env.SEARCH_KEYWORDS).toBe("Backend,Java"); await expect(saveKeywords("invalido")).resolves.toBeNull(); - }); - - it("salva em arquivo quando está em modo file e não há Redis", async () => { - process.env.KEYWORDS_STORAGE_MODE = "file"; - process.env.KEYWORDS_FILE_PATH = "./tmp/file-keywords.json"; - - const { saveKeywords } = await importKeywordsStore(); - - await expect(saveKeywords(["SRE", "Platform"])) .resolves.toEqual(["SRE", "Platform"]); - expect(mocks.mkdirSync).toHaveBeenCalledWith(expect.stringContaining("tmp"), { recursive: true }); - expect(mocks.writeFileSync).toHaveBeenCalledWith( - expect.stringContaining("file-keywords.json"), - JSON.stringify({ KEYWORDS: ["SRE", "Platform"] }, null, 2), - "utf-8", + await expect(saveKeywords(["Backend", "Java"])) .rejects.toThrow( + "Redis indisponivel para salvar keywords.", ); }); }); \ No newline at end of file diff --git a/backend/tests/unit/utils/config.test.js b/backend/tests/unit/utils/config.test.js index c6ac9af0..6f0dd67f 100644 --- a/backend/tests/unit/utils/config.test.js +++ b/backend/tests/unit/utils/config.test.js @@ -1,6 +1,3 @@ -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { getConfig } from "../../../src/config.js"; @@ -50,26 +47,20 @@ describe("getConfig", () => { }); it("parseia SEARCH_KEYWORDS em lista", () => { - const tempDir = mkdtempSync(path.join(tmpdir(), "jobs-config-")); - vi.stubEnv("KEYWORDS_FILE_PATH", path.join(tempDir, "missing-environment.json")); vi.stubEnv("SEARCH_KEYWORDS", "Java,Spring,RabbitMQ,Docker"); const config = getConfig(); expect(config.keywords).toEqual(["Java", "Spring", "RabbitMQ", "Docker"]); }); - it("prioriza SEARCH_KEYWORDS sobre o arquivo quando ambos existem", () => { - const tempDir = mkdtempSync(path.join(tmpdir(), "jobs-config-")); - const keywordsFile = path.join(tempDir, "environment.json"); - - writeFileSync(keywordsFile, JSON.stringify({ KEYWORDS: ["Legado", "Arquivo"] }), "utf-8"); - - vi.stubEnv("KEYWORDS_FILE_PATH", keywordsFile); + it("ignora o modo legado de arquivo e usa SEARCH_KEYWORDS como fallback local", () => { + vi.stubEnv("KEYWORDS_FILE_PATH", "/tmp/legacy-environment.json"); + vi.stubEnv("KEYWORDS_STORAGE_MODE", "file"); vi.stubEnv("SEARCH_KEYWORDS", "Java,Spring,RabbitMQ,Docker"); - vi.stubEnv("KEYWORDS_STORAGE_MODE", "env"); const config = getConfig(); expect(config.keywords).toEqual(["Java", "Spring", "RabbitMQ", "Docker"]); + expect(config.keywordsStorageMode).toBe("redis"); }); it("rejeita TIME_FILTER invalido e usa fallback", () => { @@ -120,8 +111,6 @@ describe("getConfig", () => { }); it("retorna keywords padrao quando lista informada e vazia", () => { - const tempDir = mkdtempSync(path.join(tmpdir(), "jobs-config-")); - vi.stubEnv("KEYWORDS_FILE_PATH", path.join(tempDir, "missing-environment.json")); vi.stubEnv("SEARCH_KEYWORDS", " , , "); const config = getConfig(); @@ -129,15 +118,10 @@ describe("getConfig", () => { expect(config.keywords).toContain("Java"); }); - it("preserva lista vazia quando o arquivo de keywords ja existe", () => { - const tempDir = mkdtempSync(path.join(tmpdir(), "jobs-config-")); - const keywordsFile = path.join(tempDir, "environment.json"); - writeFileSync(keywordsFile, JSON.stringify({ KEYWORDS: [] }), "utf-8"); - - vi.stubEnv("KEYWORDS_FILE_PATH", keywordsFile); - vi.stubEnv("SEARCH_KEYWORDS", "Java,Node"); - + it("retorna keywords padrao quando SEARCH_KEYWORDS não existe", () => { const config = getConfig(); - expect(config.keywords).toEqual([]); + + expect(config.keywords).toContain("Java"); + expect(config.keywordsStorageMode).toBe("redis"); }); });