diff --git a/apps/api/src/scrape/scrape-ui.controller.spec.ts b/apps/api/src/scrape/scrape-ui.controller.spec.ts index bba2941..e1b05fe 100644 --- a/apps/api/src/scrape/scrape-ui.controller.spec.ts +++ b/apps/api/src/scrape/scrape-ui.controller.spec.ts @@ -8,6 +8,7 @@ import { SseTicketService } from './sse-ticket.service'; import { SchedulerService } from './scheduler.service'; import { DatabaseService } from '../database/database.service'; import { AuthorResolverService } from './author-resolver.service'; +import { SecretRedactionService } from '../_logger/secret-redaction.service'; import { of } from 'rxjs'; describe('ScrapeUIController', () => { @@ -18,6 +19,7 @@ describe('ScrapeUIController', () => { let mockSchedulerService: any; let mockDatabaseService: any; let mockAuthorResolverService: any; + let mockSecretRedaction: any; const createMockResponse = () => { const res: any = { @@ -96,6 +98,16 @@ describe('ScrapeUIController', () => { .mockImplementation((meta) => Promise.resolve(meta || {})), }; + // Gibt das Objekt unveraendert zurueck, ersetzt aber einen bekannten + // Geheimwert — so laesst sich pruefen, DASS geschwaerzt wird. + mockSecretRedaction = { + redactObject: vi.fn((o) => + JSON.parse( + JSON.stringify(o).replaceAll('<>', '***'), + ), + ), + }; + const module: TestingModule = await Test.createTestingModule({ controllers: [ScrapeUIController], providers: [ @@ -105,6 +117,10 @@ describe('ScrapeUIController', () => { { provide: SchedulerService, useValue: mockSchedulerService }, { provide: DatabaseService, useValue: mockDatabaseService }, { provide: AuthorResolverService, useValue: mockAuthorResolverService }, + { + provide: SecretRedactionService, + useValue: mockSecretRedaction, + }, ], }).compile(); @@ -803,6 +819,33 @@ describe('ScrapeUIController', () => { // ============ GET /runs/:runId/debug ============ describe('getRunDebugData', () => { + it('does not hand out credentials stored in the debug data', async () => { + mockDatabaseService.getRun = vi + .fn() + .mockResolvedValue({ id: 'run-1', scrapeId: 'amazon' }); + mockDatabaseService.dataSource = { + getRepository: vi.fn().mockReturnValue({ + findOne: vi.fn().mockResolvedValue({ + value: JSON.stringify({ + var_email: 'user@example.com', + var_password: '<>', + }), + }), + }), + }; + + const res: any = { + status: vi.fn().mockReturnThis(), + json: vi.fn().mockReturnThis(), + }; + + await controller.getRunDebugData('run-1', res); + + expect(mockSecretRedaction.redactObject).toHaveBeenCalled(); + const ausgeliefert = JSON.stringify(res.json.mock.calls[0][0]); + expect(ausgeliefert).not.toContain('<>'); + expect(ausgeliefert).toContain('***'); + }); it('should return 404 if run not found', async () => { const res = createMockResponse(); mockDatabaseService.getRun.mockResolvedValue(null); diff --git a/apps/api/src/scrape/scrape-ui.controller.ts b/apps/api/src/scrape/scrape-ui.controller.ts index cb86e5f..be6771b 100644 --- a/apps/api/src/scrape/scrape-ui.controller.ts +++ b/apps/api/src/scrape/scrape-ui.controller.ts @@ -20,6 +20,7 @@ import { SseTicketService } from './sse-ticket.service'; import { SchedulerService } from './scheduler.service'; import { DatabaseService } from '../database/database.service'; import { AuthorResolverService } from './author-resolver.service'; +import { SecretRedactionService } from '../_logger/secret-redaction.service'; import { Public } from '../auth/decorators/public.decorator'; import { Observable, map } from 'rxjs'; import * as jsonata from 'jsonata'; @@ -56,6 +57,7 @@ export class ScrapeUIController { private schedulerService: SchedulerService, private databaseService: DatabaseService, private authorResolverService: AuthorResolverService, + private secretRedaction: SecretRedactionService, ) {} @Get('scrapes') @@ -453,7 +455,12 @@ export class ScrapeUIController { return; } - const debugData = JSON.parse(debugDataEntry.value); + // Auch beim Ausliefern geschwaerzt — fuer Zeilen, die vor dem Fix + // an der Quelle geschrieben wurden. Wirkt nur, solange die Secrets + // in diesem Prozess registriert sind: Sicherheitsnetz, kein Ersatz. + const debugData = this.secretRedaction.redactObject( + JSON.parse(debugDataEntry.value), + ); res.status(HttpStatus.OK).json(debugData); } catch (error) { res.status(HttpStatus.INTERNAL_SERVER_ERROR).json({ @@ -494,7 +501,12 @@ export class ScrapeUIController { return; } - const debugData = JSON.parse(debugDataEntry.value); + // Auch beim Ausliefern geschwaerzt — fuer Zeilen, die vor dem Fix + // an der Quelle geschrieben wurden. Wirkt nur, solange die Secrets + // in diesem Prozess registriert sind: Sicherheitsnetz, kein Ersatz. + const debugData = this.secretRedaction.redactObject( + JSON.parse(debugDataEntry.value), + ); // Extract artifacts from debug data // Strategy: Only extract artifacts from the deepest loop level to avoid duplicates diff --git a/apps/api/src/scrape/services/scrape-execution.service.spec.ts b/apps/api/src/scrape/services/scrape-execution.service.spec.ts index 01a86f4..62bfb72 100644 --- a/apps/api/src/scrape/services/scrape-execution.service.spec.ts +++ b/apps/api/src/scrape/services/scrape-execution.service.spec.ts @@ -19,6 +19,7 @@ vi.mock('../../_logger/scrape-logger.service', () => { describe('ScrapeExecutionService', () => { let service: ScrapeExecutionService; + let mockSecretRedaction: any; let mockPuppeteerService: any; let mockActionHandlerService: any; let mockScrapeEventsService: any; @@ -76,7 +77,7 @@ describe('ScrapeExecutionService', () => { publish: vi.fn().mockResolvedValue(undefined), }; - const mockSecretRedaction = { + mockSecretRedaction = { registerSecret: vi.fn(), redact: vi.fn((msg: string) => msg), redactObject: vi.fn((obj: any) => obj), @@ -577,6 +578,33 @@ describe('ScrapeExecutionService', () => { ); }); + it('redacts the debug data before it reaches the database', async () => { + // Kern der Sache: der Klartext darf gar nicht erst gespeichert werden. + // Nur beim Ausliefern zu schwaerzen liesse ihn in der Datenbank, in + // jedem Backup und in jedem Dump stehen. + mockSecretRedaction.redactObject.mockImplementation((o: any) => + JSON.parse( + JSON.stringify(o).replaceAll('<>', '***'), + ), + ); + + const previousData = new Map(); + // Bewusst ein offensichtlicher Platzhalter: ein realistisch aussehender + // Wert neben dem Feldnamen var_password laesst Secret-Scanner anschlagen + // (GitGuardian hat genau das an PR #153 gemeldet). + previousData.set('var_password', '<>'); + + mockActionHandlerService.handleAction.mockResolvedValue(undefined); + await service.executeScrape(createScrape(), 'run-1', previousData, {}); + + expect(mockSecretRedaction.redactObject).toHaveBeenCalled(); + const gespeichert = mockDatabaseService.storeData.mock.calls.find( + (c: any[]) => c[1] === '__debugData', + ); + expect(gespeichert[2]).not.toContain('<>'); + expect(gespeichert[2]).toContain('***'); + }); + it('should build debug data summarizing loop results', async () => { const previousData = new Map(); previousData.set('loopResult', { iterations: [1, 2], total: 5 }); diff --git a/apps/api/src/scrape/services/scrape-execution.service.ts b/apps/api/src/scrape/services/scrape-execution.service.ts index 8f6799c..7e2de92 100644 --- a/apps/api/src/scrape/services/scrape-execution.service.ts +++ b/apps/api/src/scrape/services/scrape-execution.service.ts @@ -84,7 +84,14 @@ export class ScrapeExecutionService { ); // Build and save debug data - const debugData = this.buildDebugData(previousData); + // Redacted BEFORE persisting, not on the way out: the run's variables + // include resolved secrets, so an unredacted debugData writes login + // credentials to the database in plain text. Anything that later reads + // that row — the debug endpoint, a backup, a database dump — hands them + // out. Redacting only on read would leave the plaintext at rest. + const debugData = this.secretRedaction.redactObject( + this.buildDebugData(previousData), + ); await this.databaseService.storeData( scrape.id, '__debugData',