Add configurable data retention workflow

README.md

@@ -162,7 +162,7 @@ See:
 - `docs/api.md` (message reaction HTTP and WebSocket contracts)
 - `docs/user-profiles.md` (profile settings, avatars, themes, and WebSocket presence)
 - `docs/channels.md` (public channels, private DMs, APIs, WebSocket routing, and migration operations)
-- `docs/operations.md` (message archiving schedules, batching, and export storage)
+- `docs/operations.md` (data retention, message archiving, batching, backups, and export storage)
 
 ## Troubleshooting
 

app/Commands/RetentionApply.php

@@ -0,0 +1,168 @@
+<?php
+
+namespace App\Commands;
+
+use CodeIgniter\CLI\BaseCommand;
+use CodeIgniter\CLI\CLI;
+use CodeIgniter\Database\BaseConnection;
+use Config\Retention;
+use InvalidArgumentException;
+use Throwable;
+
+final class RetentionApply extends BaseCommand
+{
+    private const DEFAULT_BATCH_SIZE = 5_000;
+
+    protected $group = 'Maintenance';
+    protected $name = 'retention:apply';
+    protected $description = 'Applies configured archive and deletion retention policies.';
+    protected $usage = 'retention:apply [--dry-run] [--policy=messages] [--batch-size=5000]';
+    protected $options = [
+        '--dry-run' => 'Report eligible rows without changing the database.',
+        '--policy' => 'Run one named retention policy instead of all policies.',
+        '--batch-size' => 'Rows per delete/archive transaction, from 1 to 10000 (default: 5000).',
+    ];
+
+    public function run(array $params): int
+    {
+        $policyName = $params['policy'] ?? CLI::getOption('policy');
+        $batchSize = self::parseBatchSize($params['batch-size'] ?? CLI::getOption('batch-size') ?? (string) self::DEFAULT_BATCH_SIZE);
+        $dryRun = array_key_exists('dry-run', $params) || CLI::getOption('dry-run') === true;
+
+        if ($policyName !== null && (! is_string($policyName) || $policyName === '')) {
+            CLI::error('The --policy option requires a policy name.');
+
+            return 1;
+        }
+        if ($batchSize === null) {
+            CLI::error('The --batch-size option must be an integer between 1 and 10000.');
+
+            return 1;
+        }
+
+        try {
+            $policies = config(Retention::class)->selectedPolicies($policyName);
+            $db = db_connect();
+            if (! $db instanceof BaseConnection) {
+                throw new InvalidArgumentException('Retention requires a database connection.');
+            }
+
+            $total = 0;
+            foreach ($policies as $table => $policy) {
+                $this->assertSchema($db, $table, $policy['column']);
+                $count = $this->applyPolicy($db, $table, $policy, $batchSize, $dryRun);
+                $total += $count;
+                $verb = $dryRun ? 'Would process' : ($policy['action'] === 'archive' ? 'Archived' : 'Deleted');
+                CLI::write(sprintf(
+                    '%s %s row%s for %s.',
+                    $verb,
+                    number_format($count),
+                    $count === 1 ? '' : 's',
+                    $table,
+                ), $dryRun ? 'yellow' : 'green');
+            }
+
+            CLI::write(sprintf(
+                '%s %s row%s across %s polic%s.',
+                $dryRun ? 'Dry run found' : 'Retention processed',
+                number_format($total),
+                $total === 1 ? '' : 's',
+                number_format(count($policies)),
+                count($policies) === 1 ? 'y' : 'ies',
+            ));
+        } catch (Throwable $exception) {
+            CLI::error('Unable to apply retention: ' . $exception->getMessage());
+
+            return 1;
+        }
+
+        return 0;
+    }
+
+    public static function parseBatchSize(mixed $value): ?int
+    {
+        if (! is_string($value) || preg_match('/^[1-9]\d*$/', $value) !== 1) {
+            return null;
+        }
+        $batchSize = (int) $value;
+
+        return $batchSize <= 10_000 ? $batchSize : null;
+    }
+
+    /** @param array{column: string, max_age_days: int, action: 'archive'|'delete'} $policy */
+    private function applyPolicy(
+        BaseConnection $db,
+        string $table,
+        array $policy,
+        int $batchSize,
+        bool $dryRun,
+    ): int {
+        $cutoffTimestamp = time() - ($policy['max_age_days'] * 86_400);
+        $cutoff = $table === 'messages' ? $cutoffTimestamp : date('Y-m-d H:i:s', $cutoffTimestamp);
+        $eligible = $this->countEligible($db, $table, $policy['column'], $cutoff);
+
+        if ($dryRun || $eligible === 0) {
+            return $eligible;
+        }
+        if ($policy['action'] === 'archive') {
+            $exitCode = $this->call('messages:archive', [
+                'older-than' => $policy['max_age_days'] . 'd',
+                'channel' => 'all',
+                'batch-size' => (string) $batchSize,
+            ]);
+            if ($exitCode !== 0) {
+                throw new InvalidArgumentException('The messages archive command failed.');
+            }
+
+            return $eligible;
+        }
+
+        return $this->deleteInBatches($db, $table, $policy['column'], $cutoff, $batchSize);
+    }
+
+    private function countEligible(BaseConnection $db, string $table, string $column, int|string $cutoff): int
+    {
+        return $db->table($table)->where("{$column} <", $cutoff)->countAllResults();
+    }
+
+    private function deleteInBatches(
+        BaseConnection $db,
+        string $table,
+        string $column,
+        int|string $cutoff,
+        int $batchSize,
+    ): int {
+        $deleted = 0;
+
+        do {
+            $rows = $db->table($table)
+                ->select('id')
+                ->where("{$column} <", $cutoff)
+                ->orderBy('id')
+                ->limit($batchSize)
+                ->get()
+                ->getResultArray();
+            if ($rows === []) {
+                break;
+            }
+
+            $ids = array_map(static fn (array $row): int => (int) $row['id'], $rows);
+            $db->transException(true)->transStart();
+            $db->table($table)->whereIn('id', $ids)->delete();
+            $db->transComplete();
+            $deleted += count($ids);
+        } while (count($rows) === $batchSize);
+
+        return $deleted;
+    }
+
+    private function assertSchema(BaseConnection $db, string $table, string $column): void
+    {
+        if (! $db->tableExists($table)) {
+            throw new InvalidArgumentException("Retention table does not exist: {$table}");
+        }
+        if (! $db->fieldExists('id', $table) || ! $db->fieldExists($column, $table)) {
+            throw new InvalidArgumentException("Retention table {$table} is missing id or {$column}.");
+        }
+    }
+}

app/Config/Retention.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace Config;
+
+use CodeIgniter\Config\BaseConfig;
+use InvalidArgumentException;
+
+/**
+ * @phpstan-type RetentionPolicy array{column: non-empty-string, max_age_days: positive-int, action: 'archive'|'delete'}
+ */
+final class Retention extends BaseConfig
+{
+    /** @var array<string, RetentionPolicy> */
+    public array $policies = [
+        'messages' => [
+            'column' => 'time',
+            'max_age_days' => 90,
+            'action' => 'archive',
+        ],
+        'archived_messages' => [
+            'column' => 'archived_at',
+            'max_age_days' => 730,
+            'action' => 'delete',
+        ],
+        'csp_reports' => [
+            'column' => 'reported_at',
+            'max_age_days' => 30,
+            'action' => 'delete',
+        ],
+        'audit_log' => [
+            'column' => 'created_at',
+            'max_age_days' => 365,
+            'action' => 'delete',
+        ],
+    ];
+
+    /**
+     * @return array<string, RetentionPolicy>
+     */
+    public function selectedPolicies(?string $name = null): array
+    {
+        $this->validatePolicies();
+
+        if ($name === null) {
+            return $this->policies;
+        }
+        if (! isset($this->policies[$name])) {
+            throw new InvalidArgumentException("Unknown retention policy: {$name}");
+        }
+
+        return [$name => $this->policies[$name]];
+    }
+
+    private function validatePolicies(): void
+    {
+        foreach ($this->policies as $table => $policy) {
+            if (preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $table) !== 1) {
+                throw new InvalidArgumentException("Invalid retention table name: {$table}");
+            }
+            if (! is_array($policy)) {
+                throw new InvalidArgumentException("Retention policy {$table} must be an array.");
+            }
+
+            $column = $policy['column'] ?? null;
+            if (! is_string($column) || preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $column) !== 1) {
+                throw new InvalidArgumentException("Retention policy {$table} has an invalid age column.");
+            }
+
+            $maxAgeDays = $policy['max_age_days'] ?? null;
+            if (! is_int($maxAgeDays) || $maxAgeDays < 1) {
+                throw new InvalidArgumentException("Retention policy {$table} must have a positive max_age_days value.");
+            }
+
+            $action = $policy['action'] ?? null;
+            if (! in_array($action, ['archive', 'delete'], true)) {
+                throw new InvalidArgumentException("Retention policy {$table} has an unsupported action.");
+            }
+            if ($action === 'archive' && $table !== 'messages') {
+                throw new InvalidArgumentException('Only the messages policy currently supports archiving.');
+            }
+        }
+    }
+}

docs/operations.md

@@ -1,5 +1,64 @@
 # Operations
 
+## Data retention
+
+The `retention:apply` Spark command is the normal entry point for scheduled
+cleanup. It reads the policies in `app/Config/Retention.php` and applies every
+configured lifecycle rule:
+
+| Policy | Age | Action |
+| --- | ---: | --- |
+| `messages` | 90 days | Move live rows to `archived_messages` |
+| `archived_messages` | 730 days | Permanently delete archive rows |
+| `csp_reports` | 30 days | Permanently delete CSP reports |
+| `audit_log` | 365 days | Permanently delete audit events |
+
+Start with a dry run. It counts eligible rows without changing the database:
+
+```bash
+php spark retention:apply --dry-run
+```
+
+Run every policy, or select one policy for a one-off cleanup:
+
+```bash
+php spark retention:apply
+php spark retention:apply --policy=csp_reports
+```
+
+Delete actions select primary keys and remove at most 5,000 rows per
+transaction. Archive actions delegate to `messages:archive`, which performs the
+live-to-archive move transactionally. Use `--batch-size` to reduce lock and
+transaction-log pressure on busy MySQL installations; valid values are 1 to
+10,000. Completed batches no longer match their policy, so interrupted and
+completed runs are both safe to repeat.
+
+`archived_messages` is a hot-cold tier, not a permanent record. Channel history
+and exports can read it, but rows older than two years are deleted because the
+database backup is the recovery layer. Take and verify a backup before enabling
+hard-delete policies, and make sure the backup retention window matches the
+organization's recovery expectations.
+
+### Scheduling retention
+
+Run from the application directory so CodeIgniter loads the intended `.env`.
+For example, apply all policies every day at 03:00:
+
+```cron
+0 3 * * * cd /srv/codeigniter-chat && /usr/bin/php spark retention:apply >> writable/logs/retention.log 2>&1
+```
+
+Daily execution is appropriate for messages and CSP reports; daily or weekly
+execution is sufficient for the longer audit and archive policies. Deploy the
+cron entry with `--dry-run` first, inspect several runs, then remove that flag
+to enable writes. Do not overlap invocations; schedule the next run after the
+largest observed cleanup window.
+
+Hard deletes do not necessarily return filesystem space immediately. After a
+large first cleanup, use the database vendor's manual maintenance procedure
+(`OPTIMIZE TABLE` for MySQL or `VACUUM` for SQLite) during a maintenance window.
+The application command deliberately does not run storage maintenance.
+
 ## Archiving old messages
 
 The `messages:archive` Spark command moves old rows from `messages` to
@@ -29,10 +88,12 @@ Archived messages are read-only. Because reactions belong to live messages,
 their rows are removed by the existing foreign-key cascade when a message is
 archived. Search also remains live-message-only.
 
-### Scheduling
+### Scheduling archive-only runs
 
 Run from the application directory so CodeIgniter loads the intended `.env`.
-For example, archive messages older than 90 days every day at 02:15:
+The unified retention command above is recommended for routine scheduling. If
+an archive-only schedule is needed, archive messages older than 90 days every
+day at 02:15:
 
 ```cron
 15 2 * * * cd /srv/codeigniter-chat && /usr/bin/php spark messages:archive --older-than 90d --channel all >> writable/logs/archive.log 2>&1

docs/security.md

@@ -8,4 +8,4 @@ CSP reports are retained for 30 days; the report endpoint prunes older rows when
 
 ## Audit event taxonomy
 
-Authentication events use `auth.register`, `auth.login.success`, `auth.login.failure`, and `auth.logout`. Audit rows retain IP and user agent for 90 days by default; run `audit:prune --older-than=90d` to enforce retention.
+Authentication events use `auth.register`, `auth.login.success`, `auth.login.failure`, and `auth.logout`. The scheduled `retention:apply` policy keeps audit rows for 365 days by default. For an ad hoc, shorter cleanup, `audit:prune --older-than=90d` removes rows older than 90 days.