diff --git a/README.md b/README.md index d4a68115..5f2d672a 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,29 @@ npm run dev:frontend npm run dev:backend ``` +## Configuracao segura na VPS (Redis / Postgres) + +Para ambiente de VPS e projeto open source, **nao coloque `DATABASE_URL` nem `REDIS_URL` em `environment.json`**. +Use somente `backend/.env` no servidor (arquivo ignorado pelo Git) e mantenha `KEYWORDS_STORAGE_MODE=env`. + +Exemplo seguro: + +```env +KEYWORDS_STORAGE_MODE=env +SEARCH_KEYWORDS=Java,Spring,RabbitMQ,Docker +DATABASE_URL=postgresql://app_user:senha_forte@127.0.0.1:5432/jobsglobalscraper +REDIS_URL=redis://:senha_forte@127.0.0.1:6379/0 +REDIS_KEY_PREFIX=vagas-full +CACHE_TTL_MS=600000 +``` + +Recomendacoes: + +- use `127.0.0.1` ou a rede interna do Docker na VPS, em vez de expor Redis publicamente +- prefira `rediss://` se o Redis estiver fora da maquina local/VPS +- mantenha a porta `6379` fechada para acesso externo no firewall +- com `KEYWORDS_STORAGE_MODE=env`, o backend deixa de depender de `backend/src/db/environment.json` em producao + ## Scripts principais ### Raiz diff --git a/backend/.env.example b/backend/.env.example index 1ba237e8..b897a2d5 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -3,7 +3,7 @@ NODE_ENV=production PORT=3001 # CORS / frontend access -CORS_ALLOWED_ORIGINS=https://painel-vagas-lake.vercel.app,http://localhost:5173,http://localhost:5174 +CORS_ALLOWED_ORIGINS=https://painel-vagas-lake.vercel.app,https://painel-vagas-m6hbzlqeh-bene-teslas-projects.vercel.app,http://localhost:5173,http://localhost:5174,http://jobsglobalscraper.ddns.net,https://jobsglobalscraper.ddns.net # LinkedIn search filters SEARCH_LOCATION=Brasil @@ -12,8 +12,20 @@ SEARCH_LANGUAGE=pt REMOTE_ONLY=true JOB_TYPES=C,F TIME_FILTER=r604800 + +# Keywords storage: +# - redis => fonte de verdade para salvar e ler keywords +# SEARCH_KEYWORDS fica apenas como fallback inicial se a chave ainda estiver vazia no Redis +KEYWORDS_STORAGE_MODE=redis SEARCH_KEYWORDS=UX Designer,UI Designer,Product Manager,Product Owner +# External services (nunca commitar valores reais) +DATABASE_URL=postgresql://app_user:change_me@127.0.0.1:5432/jobsglobalscraper +REDIS_URL=redis://:change_me@127.0.0.1:6379/0 +REDIS_KEY_PREFIX=vagas-full +KEYWORDS_REDIS_KEY=vagas-full:keywords +CACHE_TTL_MS=600000 + # Scraping behavior WAIT_BETWEEN_SEARCHES_MS=5000 PAGE_TIMEOUT_MS=10000 diff --git a/backend/package.json b/backend/package.json index 5713a234..4279d0d0 100644 --- a/backend/package.json +++ b/backend/package.json @@ -37,6 +37,7 @@ "dotenv": "^17.3.1", "express": "^5.2.1", "pdfkit": "^0.18.0", + "redis": "^4.7.1", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.1", "xlsx": "^0.18.5" diff --git a/backend/src/app.js b/backend/src/app.js index 106666e6..593f86ea 100644 --- a/backend/src/app.js +++ b/backend/src/app.js @@ -1,11 +1,16 @@ import { getConfig } from "./config.js"; +import { loadKeywords } from "./db/keywordsStore.js"; import { exportToExcel, exportToPDF } from "./exporter.js"; import { logInfo } from "./logger.js"; import { searchJobsWithCache } from "./pipeline/searchJobsWithCache.js"; import { sources } from "./sources/index.js"; export async function run() { - const config = getConfig(); + const baseConfig = getConfig(); + const config = { + ...baseConfig, + keywords: await loadKeywords(baseConfig.keywords), + }; logInfo(`Localização da busca: ${config.searchLocation}`); logInfo(`Total de palavras-chave: ${config.keywords.length}`); diff --git a/backend/src/cache/cache.js b/backend/src/cache/cache.js index 45310b24..3b7c34a6 100644 --- a/backend/src/cache/cache.js +++ b/backend/src/cache/cache.js @@ -1,3 +1,131 @@ +let redisClientPromise = null; +let redisClientUrl = ""; +let redisWarningShown = false; + +const cacheStatus = { + provider: "memory", + configured: false, + connected: false, + lastError: null, +}; + +function readRedisUrl() { + return process.env.REDIS_URL?.trim() || ""; +} + +function syncCacheStatus() { + const redisUrl = readRedisUrl(); + cacheStatus.configured = Boolean(redisUrl); + cacheStatus.provider = redisUrl ? "redis" : "memory"; + + if (!redisUrl) { + cacheStatus.connected = false; + cacheStatus.lastError = null; + } +} + +function warnRedisFallback(message, error) { + cacheStatus.connected = false; + cacheStatus.lastError = error instanceof Error ? error.message : null; + + if (redisWarningShown) { + return; + } + + redisWarningShown = true; + + const errorMessage = error instanceof Error ? error.message : ""; + // eslint-disable-next-line no-console + console.warn(`${message}${errorMessage ? ` (${errorMessage})` : ""}`); +} + +export async function getRedisClient() { + const redisUrl = readRedisUrl(); + syncCacheStatus(); + + if (!redisUrl) { + return null; + } + + if (redisClientUrl !== redisUrl) { + redisClientPromise = null; + redisClientUrl = redisUrl; + } + + if (!redisClientPromise) { + redisClientPromise = import("redis") + .then(async ({ createClient }) => { + const client = createClient({ + url: redisUrl, + socket: { + connectTimeout: 3_000, + reconnectStrategy(retries) { + if (retries >= 2) { + return false; + } + + return Math.min((retries + 1) * 100, 500); + }, + }, + }); + + client.on("ready", () => { + cacheStatus.connected = true; + cacheStatus.lastError = null; + }); + + client.on("end", () => { + cacheStatus.connected = false; + }); + + client.on("error", (error) => { + warnRedisFallback("Redis indisponivel, usando cache em memoria.", error); + }); + + await client.connect(); + return client; + }) + .catch((error) => { + warnRedisFallback("Falha ao conectar no Redis, usando cache em memoria.", error); + redisClientPromise = null; + return null; + }); + } + + return redisClientPromise; +} + +export function getCacheStatus() { + syncCacheStatus(); + return { + ...cacheStatus, + type: cache.constructor.name, + }; +} + +export async function warmupCache() { + syncCacheStatus(); + + if (!cacheStatus.configured) { + return getCacheStatus(); + } + + const client = await getRedisClient(); + if (!client) { + return getCacheStatus(); + } + + try { + const pong = await client.ping(); + cacheStatus.connected = pong === "PONG"; + cacheStatus.lastError = cacheStatus.connected ? null : "PING sem resposta esperada"; + } catch (error) { + warnRedisFallback("Falha ao validar conexao com Redis.", error); + } + + return getCacheStatus(); +} + export class MemoryCache { constructor() { this.store = new Map(); @@ -38,4 +166,89 @@ export class MemoryCache { } } -export const cache = new MemoryCache(); +export class RedisCache { + constructor(options = {}) { + this.prefix = options.prefix || process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full"; + this.memoryFallback = new MemoryCache(); + } + + buildKey(key) { + return `${this.prefix}:${key}`; + } + + async get(key) { + const client = await getRedisClient(); + + if (!client) { + return this.memoryFallback.get(key); + } + + try { + const raw = await client.get(this.buildKey(key)); + return raw ? JSON.parse(raw) : null; + } catch (error) { + warnRedisFallback("Erro ao ler do Redis, usando cache em memoria.", error); + return this.memoryFallback.get(key); + } + } + + async set(key, value, ttlMs) { + this.memoryFallback.set(key, value, ttlMs); + + const client = await getRedisClient(); + if (!client) { + return; + } + + try { + await client.set(this.buildKey(key), JSON.stringify(value), { + PX: Math.max(1, Number(ttlMs) || 1), + }); + } catch (error) { + warnRedisFallback("Erro ao salvar no Redis, usando cache em memoria.", error); + } + } + + async delete(key) { + this.memoryFallback.delete(key); + + const client = await getRedisClient(); + if (!client) { + return; + } + + try { + await client.del(this.buildKey(key)); + } catch (error) { + warnRedisFallback("Erro ao remover chave do Redis.", error); + } + } + + async clear() { + this.memoryFallback.clear(); + + const client = await getRedisClient(); + if (!client) { + return; + } + + try { + const keys = []; + for await (const key of client.scanIterator({ MATCH: `${this.prefix}:*` })) { + keys.push(key); + } + + if (keys.length > 0) { + await client.del(keys); + } + } catch (error) { + warnRedisFallback("Erro ao limpar o Redis.", error); + } + } + + async has(key) { + return (await this.get(key)) !== null; + } +} + +export const cache = process.env.REDIS_URL?.trim() ? new RedisCache() : new MemoryCache(); diff --git a/backend/src/config.js b/backend/src/config.js index bdb13bd2..08da5240 100644 --- a/backend/src/config.js +++ b/backend/src/config.js @@ -1,10 +1,4 @@ -import { existsSync, readFileSync } from "fs"; -import path from "path"; -import { fileURLToPath } from "url"; - -const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); - -const DEFAULT_KEYWORDS = [ +const DEFAULT_KEYWORDS = [ "Java", "JavaScript", "React", @@ -38,11 +32,12 @@ function parseNumber(value, fallback) { return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } -function getKeywordsFilePath() { - const configuredPath = process.env.KEYWORDS_FILE_PATH?.trim(); - return configuredPath - ? path.resolve(configuredPath) - : path.resolve(MODULE_DIR, "db", "environment.json"); +function getKeywordsStorageMode() { + const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "redis") + .trim() + .toLowerCase(); + + return configuredMode === "env" ? "env" : "redis"; } function normalizeKeywords(keywords) { @@ -53,31 +48,19 @@ function normalizeKeywords(keywords) { return [...new Set(keywords.map((item) => String(item ?? "").trim()).filter(Boolean))]; } -function parseKeywords(value) { - // Tenta pegar do arquivo environment.json - try { - const envPath = getKeywordsFilePath(); - if (existsSync(envPath)) { - const data = JSON.parse(readFileSync(envPath, "utf-8")); - if (Array.isArray(data.KEYWORDS)) { - return normalizeKeywords(data.KEYWORDS) ?? []; - } - } - } catch (err) { - // Se falhar, fallback - } - - // Tenta pegar da variavel de ambiente - if (value) { - const keywords = value +function parseKeywordsFromEnv(value) { + const keywords = normalizeKeywords( + String(value ?? "") .split(",") .map((item) => item.trim()) - .filter(Boolean); - if (keywords.length > 0) return keywords; - } + .filter(Boolean), + ); - // Fallback para as keywords padrao - return DEFAULT_KEYWORDS; + return keywords?.length ? keywords : null; +} + +function parseKeywords(value) { + return parseKeywordsFromEnv(value) ?? DEFAULT_KEYWORDS; } function parseTimeFilter(value, fallback) { @@ -97,7 +80,7 @@ export function getConfig() { maxPagesPerKeyword: parseNumber(process.env.MAX_PAGES_PER_KEYWORD, 5), viewport: { width: parseNumber(process.env.VIEWPORT_WIDTH, 1280), - height: parseNumber(process.env.VIEWPORT_HEIGHT, 800) + height: parseNumber(process.env.VIEWPORT_HEIGHT, 800), }, outputFile: process.env.OUTPUT_FILE || "output/vagas_remoto.xlsx", pdfFile: process.env.PDF_FILE || "output/vagas_remoto.pdf", @@ -108,6 +91,11 @@ export function getConfig() { jobTypes: process.env.JOB_TYPES || "C,F", // f_TPR examples: r86400 (24h), r604800 (7 dias), r2592000 (30 dias) timeFilter: parseTimeFilter(process.env.TIME_FILTER, "r604800"), - keywords: parseKeywords(process.env.SEARCH_KEYWORDS) + keywords: parseKeywords(process.env.SEARCH_KEYWORDS), + keywordsStorageMode: getKeywordsStorageMode(), + cacheTtlMs: parseNumber(process.env.CACHE_TTL_MS, 10 * 60 * 1000), + databaseUrl: process.env.DATABASE_URL?.trim() || "", + redisUrl: process.env.REDIS_URL?.trim() || "", + redisKeyPrefix: process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full", }; } diff --git a/backend/src/db/keywordsStore.js b/backend/src/db/keywordsStore.js new file mode 100644 index 00000000..fa5e0764 --- /dev/null +++ b/backend/src/db/keywordsStore.js @@ -0,0 +1,83 @@ +import { getRedisClient } from "../cache/cache.js"; + +function getKeywordsRedisKey() { + const configuredKey = process.env.KEYWORDS_REDIS_KEY?.trim(); + if (configuredKey) { + return configuredKey; + } + + const prefix = process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full"; + return `${prefix}:keywords`; +} + +export function normalizeKeywords(keywords) { + if (!Array.isArray(keywords)) { + return null; + } + + return [...new Set(keywords.map((item) => String(item ?? "").trim()).filter(Boolean))]; +} + +function parseKeywordsFromEnv(value) { + return normalizeKeywords( + String(value ?? "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + ) ?? []; +} + +function getFallbackKeywords(fallback = []) { + const envKeywords = parseKeywordsFromEnv(process.env.SEARCH_KEYWORDS); + if (envKeywords.length > 0) { + return envKeywords; + } + + return normalizeKeywords(fallback) ?? []; +} + +export async function loadKeywords(fallback = []) { + const fallbackKeywords = getFallbackKeywords(fallback); + const client = await getRedisClient(); + + if (!client) { + return fallbackKeywords; + } + + try { + const raw = await client.get(getKeywordsRedisKey()); + if (!raw) { + return fallbackKeywords; + } + + const parsed = JSON.parse(raw); + + if (Array.isArray(parsed)) { + return normalizeKeywords(parsed) ?? []; + } + + if (Array.isArray(parsed?.KEYWORDS)) { + return normalizeKeywords(parsed.KEYWORDS) ?? []; + } + } catch { + // fallback below + } + + return fallbackKeywords; +} + +export async function saveKeywords(keywords) { + const normalizedKeywords = normalizeKeywords(keywords); + + if (normalizedKeywords === null) { + return null; + } + + const client = await getRedisClient(); + if (!client) { + throw new Error("Redis indisponivel para salvar keywords."); + } + + await client.set(getKeywordsRedisKey(), JSON.stringify(normalizedKeywords)); + return normalizedKeywords; +} diff --git a/backend/src/jobsApiApp.js b/backend/src/jobsApiApp.js index 9da94af9..4c6a9ab1 100644 --- a/backend/src/jobsApiApp.js +++ b/backend/src/jobsApiApp.js @@ -1,58 +1,24 @@ import cors from "cors"; import express from "express"; -import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "fs"; +import { existsSync, mkdirSync, readdirSync, statSync } from "fs"; import path from "path"; -import { fileURLToPath } from "url"; import XLSX from "xlsx"; import { run as runScraper } from "./app.js"; +import { getCacheStatus } from "./cache/cache.js"; import { getConfig } from "./config.js"; +import { loadKeywords, normalizeKeywords, saveKeywords } from "./db/keywordsStore.js"; import { searchJobsWithCache } from "./pipeline/searchJobsWithCache.js"; import { sources } from "./sources/index.js"; -const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); - const DEFAULT_ALLOWED_ORIGINS = [ "https://painel-vagas-lake.vercel.app", + "https://painel-vagas-m6hbzlqeh-bene-teslas-projects.vercel.app", + "https://jobsglobalscraper.ddns.net", + "http://jobsglobalscraper.ddns.net", "http://localhost:5173", "http://localhost:5174", ]; -function getKeywordsFilePath() { - const configuredPath = process.env.KEYWORDS_FILE_PATH?.trim(); - return configuredPath - ? path.resolve(configuredPath) - : path.resolve(MODULE_DIR, "db", "environment.json"); -} - -function normalizeKeywords(keywords) { - if (!Array.isArray(keywords)) { - return null; - } - - return [...new Set(keywords.map((item) => String(item ?? "").trim()).filter(Boolean))]; -} - -function readEnvironmentData() { - const envPath = getKeywordsFilePath(); - - if (!existsSync(envPath)) { - return { KEYWORDS: [] }; - } - - try { - const data = JSON.parse(readFileSync(envPath, "utf-8")); - return data && typeof data === "object" ? data : { KEYWORDS: [] }; - } catch { - return { KEYWORDS: [] }; - } -} - -function writeEnvironmentData(data) { - const envPath = getKeywordsFilePath(); - mkdirSync(path.dirname(envPath), { recursive: true }); - writeFileSync(envPath, JSON.stringify(data, null, 2), "utf-8"); -} - function parseAllowedOrigins(value) { const configuredOrigins = String(value ?? "") .split(",") @@ -149,7 +115,10 @@ export function createJobsApiApp(options = {}) { * description: API funcionando corretamente */ app.get("/api/health", (_req, res) => { - res.json({ ok: true }); + res.json({ + ok: true, + cache: getCacheStatus(), + }); }); /** @@ -199,16 +168,18 @@ export function createJobsApiApp(options = {}) { // Novo endpoint para busca de vagas com cache app.get("/api/jobs/search", async (req, res) => { try { + const baseConfig = getConfig(); const config = { - ...getConfig(), + ...baseConfig, keywords: req.query.keywords ? String(req.query.keywords) .split(",") .map((k) => k.trim()) - : getConfig().keywords, + .filter(Boolean) + : await loadKeywords(baseConfig.keywords), }; - const ttlMs = 10 * 60 * 1000; // 10 minutos + const ttlMs = baseConfig.cacheTtlMs; const result = await searchJobsWithCache(sources, config, ttlMs); return res.json(result); @@ -295,7 +266,7 @@ export function createJobsApiApp(options = {}) { * 400: * description: Dados inválidos */ - app.post("/api/keywords", (req, res) => { + app.post("/api/keywords", async (req, res) => { try { const normalizedKeywords = normalizeKeywords(req.body?.keywords); @@ -305,17 +276,12 @@ export function createJobsApiApp(options = {}) { }); } - const envData = { - ...readEnvironmentData(), - KEYWORDS: normalizedKeywords, - }; - - writeEnvironmentData(envData); + const savedKeywords = await saveKeywords(normalizedKeywords); return res.json({ ok: true, message: "Keywords atualizadas com sucesso.", - keywords: envData.KEYWORDS, + keywords: savedKeywords, }); } catch (error) { return res.status(500).json({ @@ -335,13 +301,13 @@ export function createJobsApiApp(options = {}) { * 200: * description: Lista de keywords */ - app.get("/api/keywords", (_req, res) => { + app.get("/api/keywords", async (_req, res) => { try { - const envData = readEnvironmentData(); + const keywords = await loadKeywords(getConfig().keywords); return res.json({ ok: true, - keywords: normalizeKeywords(envData.KEYWORDS) ?? [], + keywords, }); } catch (error) { return res.status(500).json({ diff --git a/backend/src/pipeline/searchJobsWithCache.js b/backend/src/pipeline/searchJobsWithCache.js index 006332b3..3f58a964 100644 --- a/backend/src/pipeline/searchJobsWithCache.js +++ b/backend/src/pipeline/searchJobsWithCache.js @@ -34,7 +34,7 @@ export async function searchJobsWithCache( ) { const cacheKey = buildCacheKey(config); - const cached = cache.get(cacheKey); + const cached = await cache.get(cacheKey); if (cached) { return { ...cached, @@ -43,7 +43,7 @@ export async function searchJobsWithCache( } return withRequestDedup(cacheKey, async () => { - const cachedInsideDedup = cache.get(cacheKey); + const cachedInsideDedup = await cache.get(cacheKey); if (cachedInsideDedup) { return { ...cachedInsideDedup, @@ -60,7 +60,7 @@ export async function searchJobsWithCache( fromCache: false, }; - cache.set(cacheKey, result, ttlMs); + await cache.set(cacheKey, result, ttlMs); return result; }); diff --git a/backend/src/server.js b/backend/src/server.js index fed890d8..c63217e8 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -2,6 +2,7 @@ import "dotenv/config"; import express from "express"; import { existsSync } from "fs"; import path from "path"; +import { warmupCache } from "./cache/cache.js"; import { createJobsApiApp } from "./jobsApiApp.js"; const PORT = Number(process.env.PORT || 3001); @@ -32,6 +33,14 @@ async function registerSwaggerDocs() { async function startServer() { await registerSwaggerDocs(); + const cacheStatus = await warmupCache(); + // eslint-disable-next-line no-console + console.log( + cacheStatus.configured + ? `Cache provider: ${cacheStatus.provider} (${cacheStatus.connected ? "conectado" : "fallback em memoria"})` + : "Cache provider: memory (REDIS_URL nao configurada)", + ); + // ── Electron static-file serving ───────────────────────────────────────── // When ELECTRON_STATIC_DIR is set, serve the React production build so that // the Electron window can load everything from http://localhost:. diff --git a/backend/tests/integration/jobsApi.test.js b/backend/tests/integration/jobsApi.test.js index e37867c6..75cf43d1 100644 --- a/backend/tests/integration/jobsApi.test.js +++ b/backend/tests/integration/jobsApi.test.js @@ -1,4 +1,4 @@ -import { mkdtempSync } from "fs"; +import { existsSync, mkdtempSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; import request from "supertest"; @@ -7,12 +7,36 @@ import XLSX from "xlsx"; const mocks = vi.hoisted(() => ({ run: vi.fn(), + searchJobsWithCache: vi.fn(), + loadKeywords: vi.fn(), + saveKeywords: vi.fn(), + getCacheStatus: vi.fn(), })); vi.mock("../../src/app.js", () => ({ run: mocks.run, })); +vi.mock("../../src/pipeline/searchJobsWithCache.js", () => ({ + searchJobsWithCache: mocks.searchJobsWithCache, +})); + +vi.mock("../../src/db/keywordsStore.js", () => ({ + loadKeywords: mocks.loadKeywords, + normalizeKeywords: (keywords) => { + if (!Array.isArray(keywords)) { + return null; + } + + return [...new Set(keywords.map((item) => String(item ?? "").trim()).filter(Boolean))]; + }, + saveKeywords: mocks.saveKeywords, +})); + +vi.mock("../../src/cache/cache.js", () => ({ + getCacheStatus: mocks.getCacheStatus, +})); + import { createJobsApiApp } from "../../src/jobsApiApp.js"; describe("jobs API", () => { @@ -21,19 +45,53 @@ describe("jobs API", () => { beforeEach(() => { vi.clearAllMocks(); mocks.run.mockResolvedValue(undefined); + mocks.searchJobsWithCache.mockResolvedValue({ + jobs: [{ titulo: "Dev", empresa: "ACME" }], + total: 1, + fromCache: false, + cachedAt: "2026-04-09T00:00:00.000Z", + }); + mocks.loadKeywords.mockResolvedValue(["Java", "Spring", "RabbitMQ", "Docker"]); + mocks.saveKeywords.mockImplementation(async (keywords) => { + if (process.env.KEYWORDS_STORAGE_MODE === "env") { + process.env.SEARCH_KEYWORDS = keywords.join(","); + } + + return keywords; + }); + mocks.getCacheStatus.mockReturnValue({ + configured: false, + connected: false, + provider: "memory", + type: "MemoryCache", + }); delete process.env.KEYWORDS_FILE_PATH; + delete process.env.KEYWORDS_STORAGE_MODE; + delete process.env.SEARCH_KEYWORDS; + delete process.env.REDIS_URL; + delete process.env.KEYWORDS_REDIS_KEY; }); afterEach(() => { delete process.env.KEYWORDS_FILE_PATH; + delete process.env.KEYWORDS_STORAGE_MODE; + delete process.env.SEARCH_KEYWORDS; + delete process.env.REDIS_URL; + delete process.env.KEYWORDS_REDIS_KEY; tmpDir = undefined; }); - it("GET /api/health retorna ok", async () => { + it("GET /api/health retorna ok com status do cache", async () => { tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); const app = createJobsApiApp({ outputDir: tmpDir }); const res = await request(app).get("/api/health").expect(200); - expect(res.body).toEqual({ ok: true }); + + expect(res.body.ok).toBe(true); + expect(res.body.cache).toMatchObject({ + configured: false, + connected: false, + provider: "memory", + }); }); it("GET /api/jobs/files retorna lista vazia sem xlsx", async () => { @@ -198,6 +256,27 @@ describe("jobs API", () => { }); }); + it("POST /api/keywords em modo env atualiza sem depender de arquivo json", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + process.env.KEYWORDS_STORAGE_MODE = "env"; + process.env.KEYWORDS_FILE_PATH = join(tmpDir, "nested", "environment.json"); + process.env.SEARCH_KEYWORDS = "Java,Node"; + + const app = createJobsApiApp({ outputDir: tmpDir }); + const res = await request(app) + .post("/api/keywords") + .send({ keywords: ["Redis", "PostgreSQL"] }) + .expect(200); + + expect(res.body).toEqual({ + ok: true, + message: "Keywords atualizadas com sucesso.", + keywords: ["Redis", "PostgreSQL"], + }); + expect(process.env.SEARCH_KEYWORDS).toBe("Redis,PostgreSQL"); + expect(existsSync(process.env.KEYWORDS_FILE_PATH)).toBe(false); + }); + it("GET /api/keywords retorna 200 com as keywords", async () => { const app = createJobsApiApp({ outputDir: tmpDir }); const res = await request(app) @@ -207,5 +286,107 @@ describe("jobs API", () => { keywords: ["Java","Spring","RabbitMQ","Docker"] }); }); - + + it("GET /api/jobs/search usa keywords da query quando fornecidas", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app) + .get("/api/jobs/search") + .query({ keywords: " Node, React ,, " }) + .expect(200); + + expect(res.body.total).toBe(1); + expect(mocks.loadKeywords).not.toHaveBeenCalled(); + expect(mocks.searchJobsWithCache).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ keywords: ["Node", "React"] }), + expect.anything(), + ); + }); + + it("GET /api/jobs/search usa keywords salvas quando a query não é enviada", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + mocks.loadKeywords.mockResolvedValueOnce(["Redis", "PostgreSQL"]); + const app = createJobsApiApp({ outputDir: tmpDir }); + + await request(app).get("/api/jobs/search").expect(200); + + expect(mocks.loadKeywords).toHaveBeenCalledTimes(1); + expect(mocks.searchJobsWithCache).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ keywords: ["Redis", "PostgreSQL"] }), + expect.anything(), + ); + }); + + it("GET /api/jobs/search retorna 500 quando a busca falha", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + mocks.searchJobsWithCache.mockRejectedValueOnce(new Error("falha na busca")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app) + .get("/api/jobs/search") + .query({ keywords: "Java" }) + .expect(500); + + expect(res.body).toEqual({ + message: "Erro ao buscar vagas.", + error: "falha na busca", + }); + }); + + it("POST /api/keywords retorna 400 quando keywords não é array", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app) + .post("/api/keywords") + .send({ keywords: "Java" }) + .expect(400); + + expect(res.body.message).toBe("O campo 'keywords' deve ser um array de strings."); + }); + + it("POST /api/keywords retorna 500 quando saveKeywords falha", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + mocks.saveKeywords.mockRejectedValueOnce(new Error("redis down")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app) + .post("/api/keywords") + .send({ keywords: ["Java"] }) + .expect(500); + + expect(res.body).toEqual({ + message: "Erro ao salvar keywords.", + error: "redis down", + }); + }); + + it("GET /api/keywords retorna 500 quando loadKeywords falha", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + mocks.loadKeywords.mockRejectedValueOnce(new Error("read failed")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app).get("/api/keywords").expect(500); + + expect(res.body).toEqual({ + message: "Erro ao buscar keywords.", + error: "read failed", + }); + }); + + it("adiciona HSTS quando a requisição chega por HTTPS via proxy", async () => { + tmpDir = mkdtempSync(join(tmpdir(), "jobs-api-")); + const app = createJobsApiApp({ outputDir: tmpDir }); + + const res = await request(app) + .get("/api/health") + .set("x-forwarded-proto", "https") + .expect(200); + + expect(res.headers["strict-transport-security"]).toContain("max-age=31536000"); + }); + }); diff --git a/backend/tests/unit/services/app.test.js b/backend/tests/unit/services/app.test.js index e3a82ecd..a5fc89ce 100644 --- a/backend/tests/unit/services/app.test.js +++ b/backend/tests/unit/services/app.test.js @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ getConfigMock: vi.fn(() => ({ @@ -19,6 +19,9 @@ const mocks = vi.hoisted(() => ({ })); vi.mock("../../../src/config.js", () => ({ getConfig: mocks.getConfigMock })); +vi.mock("../../../src/db/keywordsStore.js", () => ({ + loadKeywords: vi.fn(async (fallback = []) => fallback), +})); vi.mock("../../../src/pipeline/searchJobsWithCache.js", () => ({ searchJobsWithCache: mocks.searchJobsWithCacheMock, })); @@ -34,6 +37,10 @@ vi.mock("../../../src/logger.js", () => ({ logInfo: mocks.logInfoMock })); import { run } from "../../../src/app.js"; describe("run", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + it("orquestra coleta e exportacao", async () => { await run(); @@ -48,5 +55,18 @@ describe("run", () => { "output/a.pdf", ); expect(mocks.logInfoMock).toHaveBeenCalled(); + expect(mocks.logInfoMock).toHaveBeenCalledWith("Resultado do cache: MISS"); + }); + + it("registra HIT quando o resultado vem do cache", async () => { + mocks.searchJobsWithCacheMock.mockResolvedValueOnce({ + jobs: [{ titulo: "Dev", palavra: "React" }], + total: 1, + fromCache: true, + }); + + await run(); + + expect(mocks.logInfoMock).toHaveBeenCalledWith("Resultado do cache: HIT"); }); }); diff --git a/backend/tests/unit/services/cache/cache.test.js b/backend/tests/unit/services/cache/cache.test.js index 73b95184..6057b209 100644 --- a/backend/tests/unit/services/cache/cache.test.js +++ b/backend/tests/unit/services/cache/cache.test.js @@ -1,12 +1,64 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { MemoryCache } from "../../../../src/cache/cache.js"; + +const mocks = vi.hoisted(() => ({ + createClient: vi.fn(), + consoleWarn: vi.fn(), +})); + +vi.mock("redis", () => ({ + createClient: mocks.createClient, +})); + +function buildRedisClientMock(options = {}) { + const handlers = {}; + const client = { + on: vi.fn((event, handler) => { + handlers[event] = handler; + return client; + }), + connect: vi.fn(async () => { + if (options.connectError) { + throw options.connectError; + } + + if (options.triggerReady !== false) { + handlers.ready?.(); + } + }), + ping: vi.fn(async () => options.ping ?? "PONG"), + get: vi.fn(async () => options.get ?? null), + set: vi.fn(async () => "OK"), + del: vi.fn(async () => 1), + scanIterator: vi.fn(async function* () { + if (options.scanError) { + throw options.scanError; + } + + for (const key of options.scanKeys ?? []) { + yield key; + } + }), + }; + + return { client, handlers }; +} + +async function importCacheModule() { + return import("../../../../src/cache/cache.js"); +} describe("MemoryCache", () => { let memoryCache; - beforeEach(() => { - memoryCache = new MemoryCache(); + beforeEach(async () => { + vi.resetModules(); + vi.clearAllMocks(); vi.restoreAllMocks(); + delete process.env.REDIS_URL; + delete process.env.REDIS_KEY_PREFIX; + + const { MemoryCache } = await importCacheModule(); + memoryCache = new MemoryCache(); }); it("retorna null quando a chave não existe", () => { @@ -24,10 +76,10 @@ describe("MemoryCache", () => { it("remove e retorna null quando o item expirou", () => { const nowSpy = vi.spyOn(Date, "now"); - nowSpy.mockReturnValueOnce(1000); // set + nowSpy.mockReturnValueOnce(1000); memoryCache.set("jobs:node", [{ titulo: "Dev Node" }], 500); - nowSpy.mockReturnValueOnce(1601); // get + nowSpy.mockReturnValueOnce(1601); expect(memoryCache.get("jobs:node")).toBeNull(); expect(memoryCache.store.has("jobs:node")).toBe(false); @@ -66,10 +118,152 @@ describe("MemoryCache", () => { it("has retorna false quando a chave expirou", () => { const nowSpy = vi.spyOn(Date, "now"); - nowSpy.mockReturnValueOnce(2000); // set + nowSpy.mockReturnValueOnce(2000); memoryCache.set("jobs", ["vaga 1"], 100); - nowSpy.mockReturnValueOnce(2201); // has -> get + nowSpy.mockReturnValueOnce(2201); expect(memoryCache.has("jobs")).toBe(false); }); +}); + +describe("Redis cache helpers", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + vi.restoreAllMocks(); + delete process.env.REDIS_URL; + delete process.env.REDIS_KEY_PREFIX; + vi.spyOn(console, "warn").mockImplementation(mocks.consoleWarn); + }); + + it("retorna status de memoria quando REDIS_URL não está configurada", async () => { + const { MemoryCache, cache, getCacheStatus, getRedisClient, warmupCache } = await importCacheModule(); + + expect(await getRedisClient()).toBeNull(); + expect(cache).toBeInstanceOf(MemoryCache); + expect(getCacheStatus()).toMatchObject({ + provider: "memory", + configured: false, + connected: false, + type: "MemoryCache", + }); + expect(await warmupCache()).toMatchObject({ + provider: "memory", + configured: false, + connected: false, + }); + expect(mocks.createClient).not.toHaveBeenCalled(); + }); + + it("conecta ao Redis, reaproveita o cliente e executa operações no RedisCache", async () => { + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + process.env.REDIS_KEY_PREFIX = "jobs-test"; + + const { client, handlers } = buildRedisClientMock({ + scanKeys: ["jobs-test:a", "jobs-test:b"], + }); + mocks.createClient.mockReturnValue(client); + + const { RedisCache, getCacheStatus, getRedisClient, warmupCache } = await importCacheModule(); + + const firstClient = await getRedisClient(); + const secondClient = await getRedisClient(); + expect(firstClient).toBe(client); + expect(secondClient).toBe(client); + expect(mocks.createClient).toHaveBeenCalledTimes(1); + + const socketConfig = mocks.createClient.mock.calls[0][0].socket; + expect(socketConfig.reconnectStrategy(0)).toBe(100); + expect(socketConfig.reconnectStrategy(3)).toBe(false); + + const redisCache = new RedisCache({ prefix: "jobs-test" }); + await redisCache.set("react", { ok: true }, 5000); + client.get.mockResolvedValueOnce(JSON.stringify({ ok: true })); + expect(await redisCache.get("react")).toEqual({ ok: true }); + + await redisCache.delete("react"); + await redisCache.clear(); + + expect(client.set).toHaveBeenCalledWith("jobs-test:react", JSON.stringify({ ok: true }), { + PX: 5000, + }); + expect(client.del).toHaveBeenCalledWith("jobs-test:react"); + expect(client.del).toHaveBeenCalledWith(["jobs-test:a", "jobs-test:b"]); + + expect(await warmupCache()).toMatchObject({ + provider: "redis", + configured: true, + connected: true, + type: "RedisCache", + }); + + handlers.end?.(); + expect(getCacheStatus().connected).toBe(false); + + process.env.REDIS_URL = "redis://127.0.0.1:6380"; + await getRedisClient(); + expect(mocks.createClient).toHaveBeenCalledTimes(2); + }); + + it("faz fallback quando a conexão com Redis falha", async () => { + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + + const { client } = buildRedisClientMock({ + connectError: new Error("connect failed"), + triggerReady: false, + }); + mocks.createClient.mockReturnValue(client); + + const { getCacheStatus, getRedisClient } = await importCacheModule(); + + await expect(getRedisClient()).resolves.toBeNull(); + expect(mocks.consoleWarn).toHaveBeenCalledTimes(1); + expect(mocks.consoleWarn.mock.calls[0][0]).toContain("Falha ao conectar no Redis"); + expect(getCacheStatus()).toMatchObject({ + provider: "redis", + configured: true, + connected: false, + lastError: "connect failed", + }); + }); + + it("usa fallback em memória quando operações do Redis lançam erro", async () => { + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + + const { client } = buildRedisClientMock(); + client.get.mockResolvedValueOnce("{invalid-json"); + client.set.mockRejectedValueOnce(new Error("set failed")); + client.del.mockRejectedValueOnce(new Error("del failed")); + client.scanIterator.mockImplementationOnce(async function* () { + throw new Error("scan failed"); + }); + mocks.createClient.mockReturnValue(client); + + const { RedisCache } = await importCacheModule(); + + const redisCache = new RedisCache({ prefix: "jobs-test" }); + redisCache.memoryFallback.set("node", { cached: true }, 1000); + + await expect(redisCache.get("node")).resolves.toEqual({ cached: true }); + await expect(redisCache.set("node", { cached: false }, 1000)).resolves.toBeUndefined(); + await expect(redisCache.delete("node")).resolves.toBeUndefined(); + await expect(redisCache.clear()).resolves.toBeUndefined(); + + expect(mocks.consoleWarn).toHaveBeenCalledTimes(1); + }); + + it("marca o status como desconectado quando o ping não retorna PONG", async () => { + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + + const { client } = buildRedisClientMock({ ping: "NOPE" }); + mocks.createClient.mockReturnValue(client); + + const { warmupCache } = await importCacheModule(); + + await expect(warmupCache()).resolves.toMatchObject({ + configured: true, + connected: false, + lastError: "PING sem resposta esperada", + }); + }); }); \ No newline at end of file diff --git a/backend/tests/unit/services/db/keywordsStore.test.js b/backend/tests/unit/services/db/keywordsStore.test.js new file mode 100644 index 00000000..95577e03 --- /dev/null +++ b/backend/tests/unit/services/db/keywordsStore.test.js @@ -0,0 +1,104 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + getRedisClient: vi.fn(), +})); + +vi.mock("../../../../src/cache/cache.js", () => ({ + getRedisClient: mocks.getRedisClient, +})); + +async function importKeywordsStore() { + return import("../../../../src/db/keywordsStore.js"); +} + +describe("keywordsStore", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + delete process.env.SEARCH_KEYWORDS; + delete process.env.KEYWORDS_STORAGE_MODE; + delete process.env.KEYWORDS_REDIS_KEY; + delete process.env.REDIS_KEY_PREFIX; + delete process.env.KEYWORDS_FILE_PATH; + mocks.getRedisClient.mockResolvedValue(null); + }); + + it("normaliza keywords removendo vazios e duplicadas", async () => { + const { normalizeKeywords } = await importKeywordsStore(); + + expect(normalizeKeywords([" Node ", "React", "Node", "", null])).toEqual([ + "Node", + "React", + ]); + expect(normalizeKeywords("Node")).toBeNull(); + }); + + it("carrega keywords do Redis quando existe um array salvo", async () => { + process.env.KEYWORDS_REDIS_KEY = "custom:keywords"; + const client = { + get: vi.fn().mockResolvedValue(JSON.stringify([" Node ", "React", "Node"])), + }; + mocks.getRedisClient.mockResolvedValue(client); + + const { loadKeywords } = await importKeywordsStore(); + + await expect(loadKeywords(["fallback"])) .resolves.toEqual(["Node", "React"]); + expect(client.get).toHaveBeenCalledWith("custom:keywords"); + }); + + it("carrega keywords do formato legado KEYWORDS vindo do Redis", async () => { + const client = { + get: vi.fn().mockResolvedValue(JSON.stringify({ KEYWORDS: ["Java", "Spring", "Java"] })), + }; + mocks.getRedisClient.mockResolvedValue(client); + + const { loadKeywords } = await importKeywordsStore(); + + await expect(loadKeywords(["fallback"])) .resolves.toEqual(["Java", "Spring"]); + }); + + it("faz fallback para SEARCH_KEYWORDS quando Redis não retorna valor", async () => { + process.env.SEARCH_KEYWORDS = " Suporte, Node, Suporte "; + const client = { get: vi.fn().mockResolvedValue(null) }; + mocks.getRedisClient.mockResolvedValue(client); + + const { loadKeywords } = await importKeywordsStore(); + + await expect(loadKeywords(["fallback"])) .resolves.toEqual(["Suporte", "Node"]); + }); + + it("usa o fallback recebido quando Redis está indisponível ou retorna JSON inválido", async () => { + const { loadKeywords } = await importKeywordsStore(); + + await expect(loadKeywords([" Produto ", "Produto", "Dados "])) .resolves.toEqual([ + "Produto", + "Dados", + ]); + + const client = { get: vi.fn().mockResolvedValue("{invalid-json") }; + mocks.getRedisClient.mockResolvedValue(client); + + await expect(loadKeywords(["QA", "Dados"])) .resolves.toEqual(["QA", "Dados"]); + }); + + it("salva no Redis usando a chave configurada", async () => { + process.env.REDIS_KEY_PREFIX = "vagas-full"; + const client = { set: vi.fn().mockResolvedValue("OK") }; + mocks.getRedisClient.mockResolvedValue(client); + + const { saveKeywords } = await importKeywordsStore(); + + await expect(saveKeywords([" Node ", "React", "Node"])) .resolves.toEqual(["Node", "React"]); + expect(client.set).toHaveBeenCalledWith("vagas-full:keywords", JSON.stringify(["Node", "React"])); + }); + + it("retorna null para payload inválido e falha quando Redis está indisponível", async () => { + const { saveKeywords } = await importKeywordsStore(); + + await expect(saveKeywords("invalido")).resolves.toBeNull(); + await expect(saveKeywords(["Backend", "Java"])) .rejects.toThrow( + "Redis indisponivel para salvar keywords.", + ); + }); +}); \ No newline at end of file diff --git a/backend/tests/unit/services/server.test.js b/backend/tests/unit/services/server.test.js index 2e270163..8f8a48e9 100644 --- a/backend/tests/unit/services/server.test.js +++ b/backend/tests/unit/services/server.test.js @@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({ expressStatic: vi.fn(), swaggerServe: vi.fn(), swaggerSetup: vi.fn(() => vi.fn()), + warmupCache: vi.fn(), })); mocks.createJobsApiApp.mockReturnValue({ @@ -21,9 +22,13 @@ mocks.createJobsApiApp.mockReturnValue({ }); mocks.expressStatic.mockReturnValue(mocks.staticMiddleware); - +mocks.warmupCache.mockResolvedValue({ provider: "memory", configured: false, connected: false }); vi.mock("dotenv/config", () => ({})); +vi.mock("../../../src/cache/cache.js", () => ({ + warmupCache: mocks.warmupCache, +})); + vi.mock("../../../src/jobsApiApp.js", () => ({ createJobsApiApp: mocks.createJobsApiApp, })); @@ -53,6 +58,8 @@ async function importServerEntry() { await import("../../../src/server.js"); await vi.dynamicImportSettled(); await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); } describe("server entry", () => { diff --git a/backend/tests/unit/utils/config.test.js b/backend/tests/unit/utils/config.test.js index 3f0f9a79..6f0dd67f 100644 --- a/backend/tests/unit/utils/config.test.js +++ b/backend/tests/unit/utils/config.test.js @@ -1,6 +1,3 @@ -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { getConfig } from "../../../src/config.js"; @@ -21,6 +18,7 @@ const CONFIG_ENV_KEYS = [ "TIME_FILTER", "SEARCH_KEYWORDS", "KEYWORDS_FILE_PATH", + "KEYWORDS_STORAGE_MODE", ]; describe("getConfig", () => { @@ -49,14 +47,22 @@ describe("getConfig", () => { }); it("parseia SEARCH_KEYWORDS em lista", () => { - const tempDir = mkdtempSync(path.join(tmpdir(), "jobs-config-")); - vi.stubEnv("KEYWORDS_FILE_PATH", path.join(tempDir, "missing-environment.json")); vi.stubEnv("SEARCH_KEYWORDS", "Java,Spring,RabbitMQ,Docker"); const config = getConfig(); expect(config.keywords).toEqual(["Java", "Spring", "RabbitMQ", "Docker"]); }); + it("ignora o modo legado de arquivo e usa SEARCH_KEYWORDS como fallback local", () => { + vi.stubEnv("KEYWORDS_FILE_PATH", "/tmp/legacy-environment.json"); + vi.stubEnv("KEYWORDS_STORAGE_MODE", "file"); + vi.stubEnv("SEARCH_KEYWORDS", "Java,Spring,RabbitMQ,Docker"); + + const config = getConfig(); + expect(config.keywords).toEqual(["Java", "Spring", "RabbitMQ", "Docker"]); + expect(config.keywordsStorageMode).toBe("redis"); + }); + it("rejeita TIME_FILTER invalido e usa fallback", () => { vi.stubEnv("TIME_FILTER", "invalid"); const config = getConfig(); @@ -105,8 +111,6 @@ describe("getConfig", () => { }); it("retorna keywords padrao quando lista informada e vazia", () => { - const tempDir = mkdtempSync(path.join(tmpdir(), "jobs-config-")); - vi.stubEnv("KEYWORDS_FILE_PATH", path.join(tempDir, "missing-environment.json")); vi.stubEnv("SEARCH_KEYWORDS", " , , "); const config = getConfig(); @@ -114,15 +118,10 @@ describe("getConfig", () => { expect(config.keywords).toContain("Java"); }); - it("preserva lista vazia quando o arquivo de keywords ja existe", () => { - const tempDir = mkdtempSync(path.join(tmpdir(), "jobs-config-")); - const keywordsFile = path.join(tempDir, "environment.json"); - writeFileSync(keywordsFile, JSON.stringify({ KEYWORDS: [] }), "utf-8"); - - vi.stubEnv("KEYWORDS_FILE_PATH", keywordsFile); - vi.stubEnv("SEARCH_KEYWORDS", "Java,Node"); - + it("retorna keywords padrao quando SEARCH_KEYWORDS não existe", () => { const config = getConfig(); - expect(config.keywords).toEqual([]); + + expect(config.keywords).toContain("Java"); + expect(config.keywordsStorageMode).toBe("redis"); }); }); diff --git a/docker-compose.yml b/docker-compose.yml index 9d64018b..5573f978 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,9 @@ services: - ./backend/.env environment: - PORT=3001 + - KEYWORDS_STORAGE_MODE=${KEYWORDS_STORAGE_MODE:-env} + - CACHE_TTL_MS=${CACHE_TTL_MS:-600000} + - REDIS_KEY_PREFIX=${REDIS_KEY_PREFIX:-vagas-full} volumes: - ./backend/output:/app/output ports: diff --git a/package-lock.json b/package-lock.json index e6cf47d9..f4ad1eae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,6 +38,7 @@ "dotenv": "^17.3.1", "express": "^5.2.1", "pdfkit": "^0.18.0", + "redis": "^4.7.1", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.1", "xlsx": "^0.18.5" @@ -2110,6 +2111,65 @@ "node": ">=14" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz", + "integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==", + "license": "MIT", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "license": "MIT", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.0.0-rc.12", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.12.tgz", @@ -4820,6 +4880,15 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/codepage": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", @@ -6952,6 +7021,15 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -10221,6 +10299,23 @@ "node": ">=8" } }, + "node_modules/redis": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.1.tgz", + "integrity": "sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ==", + "license": "MIT", + "workspaces": [ + "./packages/*" + ], + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.1", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -13081,7 +13176,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, "license": "ISC" }, "node_modules/yargs": {