diff --git a/README.md b/README.md index 68a0324..ad17df6 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ See: - `docs/api.md` (message reaction HTTP and WebSocket contracts) - `docs/user-profiles.md` (profile settings, avatars, themes, and WebSocket presence) - `docs/channels.md` (public channels, private DMs, APIs, WebSocket routing, and migration operations) -- `docs/operations.md` (message archiving schedules, batching, and export storage) +- `docs/operations.md` (data retention, message archiving, batching, backups, and export storage) ## Troubleshooting diff --git a/app/Commands/RetentionApply.php b/app/Commands/RetentionApply.php new file mode 100644 index 0000000..f39841e --- /dev/null +++ b/app/Commands/RetentionApply.php @@ -0,0 +1,168 @@ + 'Report eligible rows without changing the database.', + '--policy' => 'Run one named retention policy instead of all policies.', + '--batch-size' => 'Rows per delete/archive transaction, from 1 to 10000 (default: 5000).', + ]; + + public function run(array $params): int + { + $policyName = $params['policy'] ?? CLI::getOption('policy'); + $batchSize = self::parseBatchSize($params['batch-size'] ?? CLI::getOption('batch-size') ?? (string) self::DEFAULT_BATCH_SIZE); + $dryRun = array_key_exists('dry-run', $params) || CLI::getOption('dry-run') === true; + + if ($policyName !== null && (! is_string($policyName) || $policyName === '')) { + CLI::error('The --policy option requires a policy name.'); + + return 1; + } + if ($batchSize === null) { + CLI::error('The --batch-size option must be an integer between 1 and 10000.'); + + return 1; + } + + try { + $policies = config(Retention::class)->selectedPolicies($policyName); + $db = db_connect(); + if (! $db instanceof BaseConnection) { + throw new InvalidArgumentException('Retention requires a database connection.'); + } + + $total = 0; + foreach ($policies as $table => $policy) { + $this->assertSchema($db, $table, $policy['column']); + $count = $this->applyPolicy($db, $table, $policy, $batchSize, $dryRun); + $total += $count; + $verb = $dryRun ? 'Would process' : ($policy['action'] === 'archive' ? 'Archived' : 'Deleted'); + CLI::write(sprintf( + '%s %s row%s for %s.', + $verb, + number_format($count), + $count === 1 ? '' : 's', + $table, + ), $dryRun ? 'yellow' : 'green'); + } + + CLI::write(sprintf( + '%s %s row%s across %s polic%s.', + $dryRun ? 'Dry run found' : 'Retention processed', + number_format($total), + $total === 1 ? '' : 's', + number_format(count($policies)), + count($policies) === 1 ? 'y' : 'ies', + )); + } catch (Throwable $exception) { + CLI::error('Unable to apply retention: ' . $exception->getMessage()); + + return 1; + } + + return 0; + } + + public static function parseBatchSize(mixed $value): ?int + { + if (! is_string($value) || preg_match('/^[1-9]\d*$/', $value) !== 1) { + return null; + } + $batchSize = (int) $value; + + return $batchSize <= 10_000 ? $batchSize : null; + } + + /** @param array{column: string, max_age_days: int, action: 'archive'|'delete'} $policy */ + private function applyPolicy( + BaseConnection $db, + string $table, + array $policy, + int $batchSize, + bool $dryRun, + ): int { + $cutoffTimestamp = time() - ($policy['max_age_days'] * 86_400); + $cutoff = $table === 'messages' ? $cutoffTimestamp : date('Y-m-d H:i:s', $cutoffTimestamp); + $eligible = $this->countEligible($db, $table, $policy['column'], $cutoff); + + if ($dryRun || $eligible === 0) { + return $eligible; + } + if ($policy['action'] === 'archive') { + $exitCode = $this->call('messages:archive', [ + 'older-than' => $policy['max_age_days'] . 'd', + 'channel' => 'all', + 'batch-size' => (string) $batchSize, + ]); + if ($exitCode !== 0) { + throw new InvalidArgumentException('The messages archive command failed.'); + } + + return $eligible; + } + + return $this->deleteInBatches($db, $table, $policy['column'], $cutoff, $batchSize); + } + + private function countEligible(BaseConnection $db, string $table, string $column, int|string $cutoff): int + { + return $db->table($table)->where("{$column} <", $cutoff)->countAllResults(); + } + + private function deleteInBatches( + BaseConnection $db, + string $table, + string $column, + int|string $cutoff, + int $batchSize, + ): int { + $deleted = 0; + + do { + $rows = $db->table($table) + ->select('id') + ->where("{$column} <", $cutoff) + ->orderBy('id') + ->limit($batchSize) + ->get() + ->getResultArray(); + if ($rows === []) { + break; + } + + $ids = array_map(static fn (array $row): int => (int) $row['id'], $rows); + $db->transException(true)->transStart(); + $db->table($table)->whereIn('id', $ids)->delete(); + $db->transComplete(); + $deleted += count($ids); + } while (count($rows) === $batchSize); + + return $deleted; + } + + private function assertSchema(BaseConnection $db, string $table, string $column): void + { + if (! $db->tableExists($table)) { + throw new InvalidArgumentException("Retention table does not exist: {$table}"); + } + if (! $db->fieldExists('id', $table) || ! $db->fieldExists($column, $table)) { + throw new InvalidArgumentException("Retention table {$table} is missing id or {$column}."); + } + } +} diff --git a/app/Config/Retention.php b/app/Config/Retention.php new file mode 100644 index 0000000..fda34a2 --- /dev/null +++ b/app/Config/Retention.php @@ -0,0 +1,83 @@ + */ + public array $policies = [ + 'messages' => [ + 'column' => 'time', + 'max_age_days' => 90, + 'action' => 'archive', + ], + 'archived_messages' => [ + 'column' => 'archived_at', + 'max_age_days' => 730, + 'action' => 'delete', + ], + 'csp_reports' => [ + 'column' => 'reported_at', + 'max_age_days' => 30, + 'action' => 'delete', + ], + 'audit_log' => [ + 'column' => 'created_at', + 'max_age_days' => 365, + 'action' => 'delete', + ], + ]; + + /** + * @return array + */ + public function selectedPolicies(?string $name = null): array + { + $this->validatePolicies(); + + if ($name === null) { + return $this->policies; + } + if (! isset($this->policies[$name])) { + throw new InvalidArgumentException("Unknown retention policy: {$name}"); + } + + return [$name => $this->policies[$name]]; + } + + private function validatePolicies(): void + { + foreach ($this->policies as $table => $policy) { + if (preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $table) !== 1) { + throw new InvalidArgumentException("Invalid retention table name: {$table}"); + } + if (! is_array($policy)) { + throw new InvalidArgumentException("Retention policy {$table} must be an array."); + } + + $column = $policy['column'] ?? null; + if (! is_string($column) || preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $column) !== 1) { + throw new InvalidArgumentException("Retention policy {$table} has an invalid age column."); + } + + $maxAgeDays = $policy['max_age_days'] ?? null; + if (! is_int($maxAgeDays) || $maxAgeDays < 1) { + throw new InvalidArgumentException("Retention policy {$table} must have a positive max_age_days value."); + } + + $action = $policy['action'] ?? null; + if (! in_array($action, ['archive', 'delete'], true)) { + throw new InvalidArgumentException("Retention policy {$table} has an unsupported action."); + } + if ($action === 'archive' && $table !== 'messages') { + throw new InvalidArgumentException('Only the messages policy currently supports archiving.'); + } + } + } +} diff --git a/docs/operations.md b/docs/operations.md index 7829c78..474e4c4 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,5 +1,64 @@ # Operations +## Data retention + +The `retention:apply` Spark command is the normal entry point for scheduled +cleanup. It reads the policies in `app/Config/Retention.php` and applies every +configured lifecycle rule: + +| Policy | Age | Action | +| --- | ---: | --- | +| `messages` | 90 days | Move live rows to `archived_messages` | +| `archived_messages` | 730 days | Permanently delete archive rows | +| `csp_reports` | 30 days | Permanently delete CSP reports | +| `audit_log` | 365 days | Permanently delete audit events | + +Start with a dry run. It counts eligible rows without changing the database: + +```bash +php spark retention:apply --dry-run +``` + +Run every policy, or select one policy for a one-off cleanup: + +```bash +php spark retention:apply +php spark retention:apply --policy=csp_reports +``` + +Delete actions select primary keys and remove at most 5,000 rows per +transaction. Archive actions delegate to `messages:archive`, which performs the +live-to-archive move transactionally. Use `--batch-size` to reduce lock and +transaction-log pressure on busy MySQL installations; valid values are 1 to +10,000. Completed batches no longer match their policy, so interrupted and +completed runs are both safe to repeat. + +`archived_messages` is a hot-cold tier, not a permanent record. Channel history +and exports can read it, but rows older than two years are deleted because the +database backup is the recovery layer. Take and verify a backup before enabling +hard-delete policies, and make sure the backup retention window matches the +organization's recovery expectations. + +### Scheduling retention + +Run from the application directory so CodeIgniter loads the intended `.env`. +For example, apply all policies every day at 03:00: + +```cron +0 3 * * * cd /srv/codeigniter-chat && /usr/bin/php spark retention:apply >> writable/logs/retention.log 2>&1 +``` + +Daily execution is appropriate for messages and CSP reports; daily or weekly +execution is sufficient for the longer audit and archive policies. Deploy the +cron entry with `--dry-run` first, inspect several runs, then remove that flag +to enable writes. Do not overlap invocations; schedule the next run after the +largest observed cleanup window. + +Hard deletes do not necessarily return filesystem space immediately. After a +large first cleanup, use the database vendor's manual maintenance procedure +(`OPTIMIZE TABLE` for MySQL or `VACUUM` for SQLite) during a maintenance window. +The application command deliberately does not run storage maintenance. + ## Archiving old messages The `messages:archive` Spark command moves old rows from `messages` to @@ -29,10 +88,12 @@ Archived messages are read-only. Because reactions belong to live messages, their rows are removed by the existing foreign-key cascade when a message is archived. Search also remains live-message-only. -### Scheduling +### Scheduling archive-only runs Run from the application directory so CodeIgniter loads the intended `.env`. -For example, archive messages older than 90 days every day at 02:15: +The unified retention command above is recommended for routine scheduling. If +an archive-only schedule is needed, archive messages older than 90 days every +day at 02:15: ```cron 15 2 * * * cd /srv/codeigniter-chat && /usr/bin/php spark messages:archive --older-than 90d --channel all >> writable/logs/archive.log 2>&1 diff --git a/docs/security.md b/docs/security.md index 992a58c..d748d6a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -8,4 +8,4 @@ CSP reports are retained for 30 days; the report endpoint prunes older rows when ## Audit event taxonomy -Authentication events use `auth.register`, `auth.login.success`, `auth.login.failure`, and `auth.logout`. Audit rows retain IP and user agent for 90 days by default; run `audit:prune --older-than=90d` to enforce retention. +Authentication events use `auth.register`, `auth.login.success`, `auth.login.failure`, and `auth.logout`. The scheduled `retention:apply` policy keeps audit rows for 365 days by default. For an ad hoc, shorter cleanup, `audit:prune --older-than=90d` removes rows older than 90 days. diff --git a/tests/Tests/Integration/RetentionIntegrationTest.php b/tests/Tests/Integration/RetentionIntegrationTest.php new file mode 100644 index 0000000..8f8566f --- /dev/null +++ b/tests/Tests/Integration/RetentionIntegrationTest.php @@ -0,0 +1,94 @@ +insertCspReport('old', date('Y-m-d H:i:s', strtotime('-31 days'))); + $this->insertCspReport('new', date('Y-m-d H:i:s', strtotime('-29 days'))); + + $command = new RetentionApply(service('logger'), service('commands')); + $this->assertSame(0, $command->run([ + 'policy' => 'csp_reports', + 'dry-run' => null, + 'batch-size' => '1', + ])); + + $this->seeInDatabase('csp_reports', ['violated_directive' => 'old']); + $this->seeInDatabase('csp_reports', ['violated_directive' => 'new']); + } + + public function testDeletePolicyUsesBoundedBatchesAndIsIdempotent(): void + { + $old = date('Y-m-d H:i:s', strtotime('-31 days')); + foreach (['old-one', 'old-two', 'old-three'] as $directive) { + $this->insertCspReport($directive, $old); + } + $this->insertCspReport('new', date('Y-m-d H:i:s', strtotime('-29 days'))); + + $command = new RetentionApply(service('logger'), service('commands')); + $params = ['policy' => 'csp_reports', 'batch-size' => '2']; + $this->assertSame(0, $command->run($params)); + $this->assertSame(0, $this->db->table('csp_reports')->like('violated_directive', 'old-', 'after')->countAllResults()); + $this->seeInDatabase('csp_reports', ['violated_directive' => 'new']); + + $this->assertSame(0, $command->run($params)); + $this->assertSame(1, $this->db->table('csp_reports')->countAllResults()); + } + + public function testArchivePolicyDelegatesToMessageArchivingAndIsIdempotent(): void + { + $user = new UserModel(); + $userId = $user->createUser('retention-user', 'retention@example.com', 'Password123!'); + $this->assertIsInt($userId); + $channelId = (new ChannelModel())->createPublic('Retention', 'retention', null, $userId); + $this->assertIsInt($channelId); + + $messages = new ChatModel(); + $messages->insertMsg('retention-user', 'old message', strtotime('-91 days'), $channelId); + $messages->insertMsg('retention-user', 'new message', strtotime('-89 days'), $channelId); + + $command = new RetentionApply(service('logger'), service('commands')); + $params = ['policy' => 'messages', 'batch-size' => '1']; + $this->assertSame(0, $command->run($params)); + $this->seeInDatabase('archived_messages', ['channel_id' => $channelId, 'msg' => 'old message']); + $this->seeInDatabase('messages', ['channel_id' => $channelId, 'msg' => 'new message']); + + $this->assertSame(0, $command->run($params)); + $this->assertSame(1, $this->db->table('archived_messages')->where('channel_id', $channelId)->countAllResults()); + } + + public function testInvalidOptionsReturnFailureWithoutMutation(): void + { + $command = new RetentionApply(service('logger'), service('commands')); + + $this->assertSame(1, $command->run(['policy' => 'missing'])); + $this->assertSame(1, $command->run(['batch-size' => '0'])); + $this->assertNull(RetentionApply::parseBatchSize('0')); + $this->assertNull(RetentionApply::parseBatchSize('10001')); + $this->assertSame(5_000, RetentionApply::parseBatchSize('5000')); + } + + private function insertCspReport(string $directive, string $reportedAt): void + { + $this->db->table('csp_reports')->insert([ + 'document_uri' => 'https://example.com/chat', + 'violated_directive' => $directive, + 'blocked_uri' => null, + 'source_file' => null, + 'line_number' => null, + 'user_agent' => 'PHPUnit', + 'reported_at' => $reportedAt, + ]); + } +} diff --git a/tests/Tests/Unit/RetentionConfigTest.php b/tests/Tests/Unit/RetentionConfigTest.php new file mode 100644 index 0000000..a9978e5 --- /dev/null +++ b/tests/Tests/Unit/RetentionConfigTest.php @@ -0,0 +1,69 @@ +selectedPolicies(); + + $this->assertSame(['messages', 'archived_messages', 'csp_reports', 'audit_log'], array_keys($policies)); + $this->assertSame('archive', $policies['messages']['action']); + $this->assertSame('delete', $policies['archived_messages']['action']); + } + + public function testOnePolicyCanBeSelected(): void + { + $policies = (new Retention())->selectedPolicies('csp_reports'); + + $this->assertSame(['csp_reports'], array_keys($policies)); + $this->assertSame(30, $policies['csp_reports']['max_age_days']); + } + + public function testUnknownPolicyIsRejected(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Unknown retention policy: missing'); + + (new Retention())->selectedPolicies('missing'); + } + + public function testUnsafeIdentifiersAndUnsupportedActionsAreRejected(): void + { + $config = new Retention(); + $config->policies = [ + 'audit_log; DROP TABLE users' => [ + 'column' => 'created_at', + 'max_age_days' => 1, + 'action' => 'delete', + ], + ]; + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Invalid retention table name'); + + $config->selectedPolicies(); + } + + public function testOnlyMessagesCanUseArchiveAction(): void + { + $config = new Retention(); + $config->policies = [ + 'audit_log' => [ + 'column' => 'created_at', + 'max_age_days' => 1, + 'action' => 'archive', + ], + ]; + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Only the messages policy currently supports archiving.'); + + $config->selectedPolicies(); + } +}