Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ See:
- `docs/api.md` (message reaction HTTP and WebSocket contracts)
- `docs/user-profiles.md` (profile settings, avatars, themes, and WebSocket presence)
- `docs/channels.md` (public channels, private DMs, APIs, WebSocket routing, and migration operations)
- `docs/operations.md` (message archiving schedules, batching, and export storage)

## Troubleshooting

Expand Down
98 changes: 98 additions & 0 deletions app/Commands/ArchiveMessages.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
<?php

namespace App\Commands;

use CodeIgniter\CLI\BaseCommand;
use CodeIgniter\CLI\CLI;
use Config\Services;
use Throwable;

final class ArchiveMessages extends BaseCommand
{
protected $group = 'Chat';
protected $name = 'messages:archive';
protected $description = 'Moves old live messages into the archive in bounded transactions.';
protected $usage = 'messages:archive --older-than=90d --channel=all';
protected $options = [
'--older-than' => 'Required age such as 90d, 24h, or 30m.',
'--channel' => 'Channel ID or all (default: all).',
'--batch-size' => 'Rows per transaction, from 1 to 10000 (default: 1000).',
];

public function run(array $params): int
{
$seconds = self::parseAge($params['older-than'] ?? CLI::getOption('older-than'));
$channelId = self::parseChannel($params['channel'] ?? CLI::getOption('channel') ?? 'all');
$batchSize = self::parseBatchSize($params['batch-size'] ?? CLI::getOption('batch-size') ?? '1000');

if ($seconds === null) {
CLI::error('The --older-than option is required and must look like 90d, 24h, or 30m.');

return 1;
}
if ($channelId === false) {
CLI::error('The --channel option must be a positive channel ID or all.');

return 1;
}
if ($batchSize === null) {
CLI::error('The --batch-size option must be an integer between 1 and 10000.');

return 1;
}

try {
$count = Services::chatRepository()->archiveMessagesBefore(
time() - $seconds,
$channelId,
$batchSize,
);
} catch (Throwable $exception) {
CLI::error('Unable to archive messages: ' . $exception->getMessage());

return 1;
}

CLI::write(sprintf('Archived %s message%s.', number_format($count), $count === 1 ? '' : 's'), 'green');

return 0;
}

public static function parseAge(mixed $value): ?int
{
if (! is_string($value) || preg_match('/^([1-9]\d*)([mhd])$/', strtolower($value), $matches) !== 1) {
return null;
}

$multiplier = match ($matches[2]) {
'm' => 60,
'h' => 3_600,
'd' => 86_400,
};
$seconds = (int) $matches[1] * $multiplier;

return $seconds > 0 ? $seconds : null;
}

public static function parseChannel(mixed $value): int|false|null
{
if ($value === 'all') {
return null;
}
if (! is_string($value) || preg_match('/^[1-9]\d*$/', $value) !== 1 || (float) $value > PHP_INT_MAX) {
return false;
}

return (int) $value;
}

public static function parseBatchSize(mixed $value): ?int
{
if (! is_string($value) || preg_match('/^[1-9]\d*$/', $value) !== 1) {
return null;
}
$batchSize = (int) $value;

return $batchSize <= 10_000 ? $batchSize : null;
}
}
4 changes: 4 additions & 0 deletions app/Config/RateLimit.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ class RateLimit extends BaseConfig
'anonymous' => [10, 60],
'authenticated' => [120, 60],
],
'export' => [
'anonymous' => [1, 3_600],
'authenticated' => [5, 3_600],
],
];

public function __construct()
Expand Down
2 changes: 2 additions & 0 deletions app/Config/Routes.php
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@
$routes->get('channels/(:num)/messages', 'ChannelsController::messages/$1', ['filter' => ['apiFormat', 'auth']]);
$routes->get('channels/(:num)/messages/search', 'ChannelsController::searchMessages/$1', ['filter' => ['apiFormat', 'auth']]);
$routes->post('channels/(:num)/messages', 'ChannelsController::createMessage/$1', ['filter' => ['apiFormat', 'auth', 'rate:write']]);
$routes->get('channels/(:num)/export', 'ChannelsController::exportChannel/$1', ['filter' => ['auth', 'rate:export']]);
$routes->get('users/me/export', 'ChannelsController::exportMine', ['filter' => ['auth', 'rate:export']]);
$routes->post('dms', 'ChannelsController::createDm', ['filter' => ['apiFormat', 'auth', 'rate:write']]);
$routes->get('messages/(:num)/reactions', 'ReactionsController::list/$1', ['filter' => ['apiFormat', 'auth']]);
$routes->post('messages/(:num)/reactions', 'ReactionsController::create/$1', ['filter' => ['apiFormat', 'auth', 'rate:react']]);
Expand Down
13 changes: 13 additions & 0 deletions app/Contracts/ChatRepository.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,14 @@ interface ChatRepository
*/
public function getMsgPaginated(int $page = 1, int $perPage = 10, ?int $channelId = null): array;

/**
* @return array{
* messages: list<array<string, mixed>>,
* pagination: array{limit: int, nextBefore: ?int, hasMore: bool}
* }
*/
public function getMessageHistory(int $channelId, ?int $before = null, int $limit = 25): array;

/**
* @return array{
* messages: list<array<string, mixed>>,
Expand All @@ -29,4 +37,9 @@ public function searchMessages(
): array;

public function insertMsg(string $name, string $message, int $current, ?int $channelId = null): int|bool;

public function archiveMessagesBefore(int $timestamp, ?int $channelId = null, int $batchSize = 1_000): int;

/** @return iterable<array{id: int|string, channel_id: int|string, user: string, msg: string, time: int|string, archived: int|string}> */
public function exportMessages(?int $channelId = null, ?string $username = null, int $batchSize = 500): iterable;
}
116 changes: 110 additions & 6 deletions app/Controllers/Api/V1/ChannelsController.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use App\Contracts\ChatRepository;
use App\Contracts\UserRepository;
use App\Controllers\BaseController;
use CodeIgniter\HTTP\DownloadResponse;
use CodeIgniter\HTTP\ResponseInterface;
use Throwable;

Expand Down Expand Up @@ -174,13 +175,21 @@ public function messages(int $channelId): ResponseInterface
return $membershipError;
}

$page = $this->positiveIntegerQuery('page', 1, PHP_INT_MAX);
$perPage = $this->positiveIntegerQuery('per_page', 10, 100);
if (is_string($page) || is_string($perPage)) {
return $this->error('validation', is_string($page) ? $page : $perPage, 400);
if ($this->request->getGet('before') !== null || $this->request->getGet('limit') !== null) {
$before = $this->optionalPositiveIntegerQuery('before');
$limit = $this->positiveIntegerQuery('limit', 25, 100);
if (is_string($before) || is_string($limit)) {
return $this->error('validation', is_string($before) ? $before : $limit, 400);
}
$result = $this->messages->getMessageHistory($channelId, $before, $limit);
} else {
$page = $this->positiveIntegerQuery('page', 1, PHP_INT_MAX);
$perPage = $this->positiveIntegerQuery('per_page', 10, 100);
if (is_string($page) || is_string($perPage)) {
return $this->error('validation', is_string($page) ? $page : $perPage, 400);
}
$result = $this->messages->getMsgPaginated($page, $perPage, $channelId);
}

$result = $this->messages->getMsgPaginated($page, $perPage, $channelId);
$this->channels->markRead($channelId, $this->userId());

return $this->channelJson($result);
Expand Down Expand Up @@ -254,6 +263,27 @@ public function searchMessages(int $channelId): ResponseInterface
]);
}

public function exportChannel(int $channelId): ResponseInterface
{
$membershipError = $this->memberChannel($channelId);
if ($membershipError instanceof ResponseInterface) {
return $membershipError;
}

return $this->exportResponse(
$this->messages->exportMessages(channelId: $channelId),
'channel-' . $channelId . '-messages',
);
}

public function exportMine(): ResponseInterface
{
return $this->exportResponse(
$this->messages->exportMessages(username: (string) $this->getCurrentUsername()),
'my-messages',
);
}

private function memberChannel(int $channelId): ?ResponseInterface
{
$channel = $this->channels->findChannel($channelId);
Expand Down Expand Up @@ -322,6 +352,19 @@ private function positiveIntegerQuery(string $name, int $default, int $maximum):
return (int) $value;
}

private function optionalPositiveIntegerQuery(string $name): int|string|null
{
$value = $this->request->getGet($name);
if ($value === null) {
return null;
}
if (! is_string($value) || preg_match('/^[1-9]\d*$/', $value) !== 1 || (float) $value > PHP_INT_MAX) {
return "The {$name} parameter must be a positive integer.";
}

return (int) $value;
}

/** @return array{value: ?string, error: ?string} */
private function optionalQueryString(string $name, int $maximum): array
{
Expand Down Expand Up @@ -364,4 +407,65 @@ private function channelJson(mixed $data, int $status = 200): ResponseInterface
{
return $this->respondWithJson($data, $status)->setHeader('X-CSRF-TOKEN', csrf_hash());
}

/** @param iterable<array{id: int|string, channel_id: int|string, user: string, msg: string, time: int|string, archived: int|string}> $messages */
private function exportResponse(iterable $messages, string $basename): ResponseInterface
{
$format = strtolower((string) ($this->request->getGet('format') ?? 'json'));
if (! in_array($format, ['json', 'csv'], true)) {
return $this->error('validation', 'The format parameter must be json or csv.', 400);
}

$path = tempnam(WRITEPATH . 'cache', 'chat-export-');
if ($path === false) {
return $this->error('export', 'The export could not be prepared.', 500);
}
$handle = fopen($path, 'wb');
if ($handle === false) {
@unlink($path);

return $this->error('export', 'The export could not be prepared.', 500);
}

if ($format === 'csv') {
fputcsv($handle, ['id', 'channel_id', 'user', 'msg', 'time', 'archived'], escape: '');
foreach ($messages as $message) {
fputcsv($handle, [
$message['id'],
$message['channel_id'],
$message['user'],
$message['msg'],
$message['time'],
$message['archived'],
], escape: '');
}
} else {
fwrite($handle, '[');
$first = true;
foreach ($messages as $message) {
if (! $first) {
fwrite($handle, ',');
}
fwrite($handle, json_encode($message, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_SUBSTITUTE));
$first = false;
}
fwrite($handle, ']');
}
fclose($handle);
register_shutdown_function(static function () use ($path): void {
@unlink($path);
});

$response = $this->response->download($path, null, true);
if (! $response instanceof DownloadResponse) {
@unlink($path);

return $this->error('export', 'The export could not be prepared.', 500);
}

return $response
->setFileName($basename . '.' . $format)
->setContentType($format === 'csv' ? 'text/csv' : 'application/json')
->setHeader('X-CSRF-TOKEN', csrf_hash());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
<?php

namespace App\Database\Migrations;

use CodeIgniter\Database\BaseConnection;
use CodeIgniter\Database\Migration;

final class CreateArchivedMessages extends Migration
{
public function up(): void
{
$this->forge->addField([
'id' => ['type' => 'INT', 'constraint' => 7],
'channel_id' => ['type' => 'INT', 'constraint' => 11, 'unsigned' => true],
'user' => ['type' => 'VARCHAR', 'constraint' => 255],
'msg' => ['type' => 'TEXT'],
'time' => ['type' => 'INT', 'constraint' => 11, 'default' => 0],
'archived_at' => ['type' => 'DATETIME'],
]);
$this->forge->addKey('id', true);
$this->forge->addKey(['channel_id', 'time', 'id']);
$this->forge->addKey(['user', 'id']);
$this->forge->addForeignKey('channel_id', 'channels', 'id', 'CASCADE', 'CASCADE');
$this->forge->createTable('archived_messages');

if ($this->database()->getPlatform() === 'MySQLi') {
$table = $this->database()->prefixTable('archived_messages');
$this->database()->query("ALTER TABLE {$table} MODIFY user VARCHAR(255) CHARACTER SET latin1 NOT NULL");
$this->database()->query("ALTER TABLE {$table} MODIFY msg TEXT CHARACTER SET latin1 NOT NULL");
}
}

public function down(): void
{
$this->forge->dropTable('archived_messages');
}

private function database(): BaseConnection
{
if (! $this->db instanceof BaseConnection) {
throw new \LogicException('Archive migrations require a CodeIgniter base database connection.');
}

return $this->db;
}
}
Loading
Loading