From 70cc5631629a50e974a6de2219d757276575b617 Mon Sep 17 00:00:00 2001 From: Adhik Joshi Date: Thu, 20 Aug 2026 05:03:03 +0530 Subject: [PATCH 1/4] Close leaked server-side prepared statements from pooled connections Statements a request leaves in cycle garbage were destroyed after the connection re-entered the pool. mysqlnd skips COM_STMT_CLOSE while a connection is busy in another coroutine and never retries, so every such statement leaked server-side until MySQL's max_prepared_stmt_count tripped (error 1461). - Worker now detaches pooled connections, tears down request state, runs gc_collect_cycles() at that safe point, and only then releases the connections back to their pools. - DatabasePool recycles connections past a configurable max_lifetime (default 300s) on release and in the pruner, bounding any residual leak from paths the worker cannot see. --- src/Swoole/Database/DatabaseManager.php | 1 + src/Swoole/Database/DatabasePool.php | 61 ++++++- src/Worker.php | 101 ++++++++---- ...WorkerPooledConnectionReleaseOrderTest.php | 94 +++++++++++ tests/Unit/DatabasePoolTest.php | 150 ++++++++++++++++++ 5 files changed, 370 insertions(+), 37 deletions(-) create mode 100644 tests/Feature/WorkerPooledConnectionReleaseOrderTest.php diff --git a/src/Swoole/Database/DatabaseManager.php b/src/Swoole/Database/DatabaseManager.php index 39f36ef..730b6d2 100644 --- a/src/Swoole/Database/DatabaseManager.php +++ b/src/Swoole/Database/DatabaseManager.php @@ -58,6 +58,7 @@ protected function getPool($name) 'wait_timeout' => 3.0, 'heartbeat' => -1, 'max_idle_time' => 60.0, + 'max_lifetime' => 300.0, ]; $this->pools[$name] = new DatabasePool( diff --git a/src/Swoole/Database/DatabasePool.php b/src/Swoole/Database/DatabasePool.php index deb2e9f..73b10d9 100644 --- a/src/Swoole/Database/DatabasePool.php +++ b/src/Swoole/Database/DatabasePool.php @@ -21,6 +21,7 @@ class DatabasePool protected ConnectionFactory $factory; protected array $connectionConfig; protected array $idleSince = []; + protected array $createdAt = []; protected ?int $idlePruneTimerId = null; /** @@ -154,6 +155,21 @@ public function release($connection): void return; } + // Recycle connections past max_lifetime instead of re-pooling them. + // PDO statements destroyed while a connection is busy in another + // coroutine leak server-side (mysqlnd skips COM_STMT_CLOSE on a busy + // connection and never retries), so any long-lived pooled connection + // accumulates leaked prepared statements. Closing it here frees them + // all; disconnecting also discards any dirty server-side state. + if ($this->hasOutlivedMaxLifetime($connection)) { + $this->markBorrowed($connection); + $this->closeConnection($connection); + $this->currentConnections--; + $this->pruneIdleConnections(); + + return; + } + try { $this->resetConnection($connection); $this->markIdle($connection); @@ -184,13 +200,19 @@ public function release($connection): void public function pruneIdleConnections(?float $now = null): int { $maxIdleTime = (float) ($this->config['max_idle_time'] ?? 60.0); + $minConnections = (int) ($this->config['min_connections'] ?? 1); + + // Idle pruning respects min_connections; lifetime pruning does not, + // since an over-age connection carries leaked server-side prepared + // statements that only closing it can free. The pool refills on demand. + $idlePruningActive = $maxIdleTime > 0 && $this->currentConnections > $minConnections; + $lifetimePruningActive = ((float) ($this->config['max_lifetime'] ?? 300.0)) > 0; - if ($maxIdleTime <= 0 || $this->currentConnections <= ($this->config['min_connections'] ?? 1)) { + if (! $idlePruningActive && ! $lifetimePruningActive) { return 0; } $now ??= microtime(true); - $minConnections = (int) ($this->config['min_connections'] ?? 1); $available = $this->channel->length(); $kept = []; $closed = 0; @@ -206,7 +228,11 @@ public function pruneIdleConnections(?float $now = null): int $idleSince = $this->idleSince[$connectionId] ?? $now; $idleFor = $now - $idleSince; - if ($this->currentConnections > $minConnections && $idleFor >= $maxIdleTime) { + $idleExpired = $maxIdleTime > 0 + && $this->currentConnections > $minConnections + && $idleFor >= $maxIdleTime; + + if ($idleExpired || $this->hasOutlivedMaxLifetime($connection, $now)) { unset($this->idleSince[$connectionId]); $this->closeConnection($connection); $this->currentConnections--; @@ -302,6 +328,10 @@ protected function resetConnection($connection): void */ protected function closeConnection($connection): void { + if (is_object($connection)) { + unset($this->createdAt[spl_object_id($connection)]); + } + try { if ($connection instanceof Connection) { $connection->disconnect(); @@ -311,6 +341,26 @@ protected function closeConnection($connection): void } } + /** + * Whether a connection is older than the pool's max_lifetime. + * + * A value of 0 or less disables lifetime recycling. Connections without a + * recorded creation time are treated as brand new. + */ + protected function hasOutlivedMaxLifetime($connection, ?float $now = null): bool + { + $maxLifetime = (float) ($this->config['max_lifetime'] ?? 300.0); + + if ($maxLifetime <= 0 || ! is_object($connection)) { + return false; + } + + $now ??= microtime(true); + $createdAt = $this->createdAt[spl_object_id($connection)] ?? $now; + + return ($now - $createdAt) >= $maxLifetime; + } + /** * Create a new database connection */ @@ -321,6 +371,10 @@ protected function createConnection() try { $connection = $this->factory->make($this->connectionConfig, $this->name); + if (is_object($connection)) { + $this->createdAt[spl_object_id($connection)] = microtime(true); + } + if ($connection instanceof Connection) { // Without a reconnector, Connection::reconnect() throws // LostConnectionException and the pool silently discards and @@ -419,6 +473,7 @@ public function getStats(): array 'max_connections' => $this->config['max_connections'] ?? 10, 'min_connections' => $this->config['min_connections'] ?? 1, 'max_idle_time' => $this->config['max_idle_time'] ?? 60.0, + 'max_lifetime' => $this->config['max_lifetime'] ?? 300.0, ]; } diff --git a/src/Worker.php b/src/Worker.php index ab0597b..88993f7 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -135,46 +135,50 @@ public function handle(Request $request, RequestContext $context): void } catch (Throwable $e) { $this->handleWorkerError($e, $sandbox, $request, $context, $responded); } finally { - if ($inCoroutine) { - if ($sandbox->bound(LivewireCoroutineMutex::class)) { - $sandbox->make(LivewireCoroutineMutex::class)->releaseAllForCurrentCoroutine(); - } + // Detach this coroutine's pooled connections but do NOT return + // them to the pool yet. Statements the request still references + // must be destroyed while these connections are idle: mysqlnd + // silently skips COM_STMT_CLOSE when a connection is busy in + // another coroutine, permanently leaking the statement server-side. + $pooledConnections = []; + + try { + if ($inCoroutine) { + if ($sandbox->bound(LivewireCoroutineMutex::class)) { + $sandbox->make(LivewireCoroutineMutex::class)->releaseAllForCurrentCoroutine(); + } - // Release coroutine-local database connections before flushing - // request scope. Flushing first can drop the context references - // needed by the pool, leaving its counters exhausted while PDOs - // are already gone. - try { - if ($sandbox->bound('db')) { - $db = $sandbox->make('db'); - if (method_exists($db, 'releaseConnections')) { - $db->releaseConnections(); - } + try { + $pooledConnections = $this->detachDatabaseConnectionsFromContext(); + } catch (Throwable $e) { + error_log('⚠️ Failed to detach coroutine DB connections: '.$e->getMessage()); } - $this->releaseDatabaseConnectionsFromContext(); - } catch (Throwable $e) { - error_log('⚠️ Failed to release coroutine DB connections: '.$e->getMessage()); + $this->releaseCoroutineRedisConnections(); } - $this->releaseCoroutineRedisConnections(); - } + $sandbox->flush(); - $sandbox->flush(); + $this->app->make('view.engine.resolver')->forget('blade'); + $this->app->make('view.engine.resolver')->forget('php'); - $this->app->make('view.engine.resolver')->forget('blade'); - $this->app->make('view.engine.resolver')->forget('php'); + if ($inCoroutine) { + Context::clear(); + } else { + CurrentApplication::set($this->app); + } - if ($inCoroutine) { - Context::clear(); - } else { - CurrentApplication::set($this->app); - } + // After the request handling process has completed we will unset some variables + // plus reset the current application state back to its original state before + // it was cloned. Then we will be ready for the next worker iteration loop. + unset($gateway, $sandbox, $scope, $context, $request, $response, $octaneResponse, $output); - // After the request handling process has completed we will unset some variables - // plus reset the current application state back to its original state before - // it was cloned. Then we will be ready for the next worker iteration loop. - unset($gateway, $sandbox, $scope, $context, $request, $response, $octaneResponse, $output); + if ($inCoroutine && $pooledConnections !== []) { + gc_collect_cycles(); + } + } finally { + $this->releaseDetachedDatabaseConnections($pooledConnections); + } } } @@ -203,10 +207,21 @@ protected function releaseCoroutineRedisConnections(): void } /** - * Release DB pools directly from coroutine context as a final guard. + * Remove this coroutine's pooled DB connections from context without + * releasing them yet. + * + * The caller must destroy the request's remaining object graph (and run + * cycle collection) BEFORE handing these back via + * releaseDetachedDatabaseConnections(). Once a connection re-enters the + * pool another coroutine can be mid-query on it, and any PDOStatement + * destroyed at that moment leaks its server-side prepared statement. + * + * @return array */ - protected function releaseDatabaseConnectionsFromContext(): void + protected function detachDatabaseConnectionsFromContext(): array { + $detached = []; + foreach (Context::all() as $key => $value) { if (! is_string($key) || ! str_ends_with($key, '.pool')) { continue; @@ -216,12 +231,30 @@ protected function releaseDatabaseConnectionsFromContext(): void $connection = Context::get($connectionKey); if ($connection && $value instanceof \Laravel\Octane\Swoole\Database\DatabasePool) { - $value->release($connection); + $detached[] = [$value, $connection]; } Context::delete($key); Context::delete($connectionKey); } + + return $detached; + } + + /** + * Return detached pooled connections to their pools. + * + * @param array $pooledConnections + */ + protected function releaseDetachedDatabaseConnections(array $pooledConnections): void + { + foreach ($pooledConnections as [$pool, $connection]) { + try { + $pool->release($connection); + } catch (Throwable $e) { + error_log('⚠️ Failed to release coroutine DB connection: '.$e->getMessage()); + } + } } /** diff --git a/tests/Feature/WorkerPooledConnectionReleaseOrderTest.php b/tests/Feature/WorkerPooledConnectionReleaseOrderTest.php new file mode 100644 index 0000000..5d3977d --- /dev/null +++ b/tests/Feature/WorkerPooledConnectionReleaseOrderTest.php @@ -0,0 +1,94 @@ +markTestSkipped('Swoole coroutine support is required.'); + } + + $events = new ArrayObject(); + + $this->app['router']->get('/pool-release-order', function () use ($events) { + Context::set('db.connection.probe', new stdClass()); + Context::set('db.connection.probe.pool', new ReleaseOrderRecordingPool($events)); + + $probe = new ReleaseOrderDestructProbe($events); + $probe->self = $probe; + unset($probe); + + return response()->json(['ok' => true]); + }); + + $client = new FakeClient([]); + $worker = $this->createWorker($client); + $worker->boot(); + + \Swoole\Coroutine\run(function () use ($worker) { + Coroutine::create(function () use ($worker) { + $request = Request::create('/pool-release-order', 'GET'); + $worker->handle($request, new RequestContext(['request' => $request])); + }); + }); + + $this->assertSame( + ['destruct', 'release'], + iterator_to_array($events), + 'Request garbage must be destroyed (destruct) before the pooled connection is released (release).' + ); + } +} + +class ReleaseOrderRecordingPool extends DatabasePool +{ + public function __construct(protected ArrayObject $events) + { + } + + public function release($connection): void + { + $this->events[] = 'release'; + } + + public function pruneIdleConnections(?float $now = null): int + { + return 0; + } +} + +class ReleaseOrderDestructProbe +{ + public ?self $self = null; + + public function __construct(protected ArrayObject $events) + { + } + + public function __destruct() + { + $this->events[] = 'destruct'; + } +} diff --git a/tests/Unit/DatabasePoolTest.php b/tests/Unit/DatabasePoolTest.php index 9559d7f..e24a722 100644 --- a/tests/Unit/DatabasePoolTest.php +++ b/tests/Unit/DatabasePoolTest.php @@ -414,6 +414,156 @@ public function test_prune_idle_connections_closes_available_connections_above_m $this->assertSame(1, $stats['idle_tracked_connections']); } + public function test_release_recycles_connection_past_max_lifetime(): void + { + $this->skipIfNoSwooleCoroutine(); + + if (! extension_loaded('pdo_sqlite')) { + $this->markTestSkipped('PDO SQLite is required.'); + } + + $factory = Mockery::mock(ConnectionFactory::class); + $factory->shouldReceive('make') + ->twice() + ->withAnyArgs() + ->andReturnUsing(fn () => new Connection(new PDO('sqlite::memory:'), 'database', '', [])); + + $first = null; + $second = null; + $statsAfterRelease = null; + + \Swoole\Coroutine\run(function () use ($factory, &$first, &$second, &$statsAfterRelease) { + $pool = new DatabasePool([ + 'min_connections' => 0, + 'max_connections' => 2, + 'wait_timeout' => 0.1, + 'max_lifetime' => 0.05, + ], [], 'sqlite', $factory); + + $first = $pool->get(); + \Swoole\Coroutine::sleep(0.08); + $pool->release($first); + + $statsAfterRelease = $pool->getStats(); + + $second = $pool->get(); + }); + + $this->assertSame(0, $statsAfterRelease['current_connections'], 'Expired connection must be closed, not re-pooled.'); + $this->assertSame(0, $statsAfterRelease['available_connections']); + $this->assertNotSame($first, $second, 'A fresh connection must replace the expired one.'); + } + + public function test_release_repools_connection_within_max_lifetime(): void + { + $this->skipIfNoSwooleCoroutine(); + + if (! extension_loaded('pdo_sqlite')) { + $this->markTestSkipped('PDO SQLite is required.'); + } + + $factory = Mockery::mock(ConnectionFactory::class); + $factory->shouldReceive('make') + ->once() + ->withAnyArgs() + ->andReturnUsing(fn () => new Connection(new PDO('sqlite::memory:'), 'database', '', [])); + + $first = null; + $second = null; + $statsAfterRelease = null; + + \Swoole\Coroutine\run(function () use ($factory, &$first, &$second, &$statsAfterRelease) { + $pool = new DatabasePool([ + 'min_connections' => 0, + 'max_connections' => 2, + 'wait_timeout' => 0.1, + 'max_lifetime' => 60.0, + ], [], 'sqlite', $factory); + + $first = $pool->get(); + $pool->release($first); + + $statsAfterRelease = $pool->getStats(); + + $second = $pool->get(); + }); + + $this->assertSame(1, $statsAfterRelease['current_connections']); + $this->assertSame(1, $statsAfterRelease['available_connections']); + $this->assertSame($first, $second, 'A young connection must be re-pooled and reused.'); + } + + public function test_zero_max_lifetime_disables_recycling(): void + { + $this->skipIfNoSwooleCoroutine(); + + if (! extension_loaded('pdo_sqlite')) { + $this->markTestSkipped('PDO SQLite is required.'); + } + + $factory = Mockery::mock(ConnectionFactory::class); + $factory->shouldReceive('make') + ->once() + ->withAnyArgs() + ->andReturnUsing(fn () => new Connection(new PDO('sqlite::memory:'), 'database', '', [])); + + $first = null; + $second = null; + + \Swoole\Coroutine\run(function () use ($factory, &$first, &$second) { + $pool = new DatabasePool([ + 'min_connections' => 0, + 'max_connections' => 2, + 'wait_timeout' => 0.1, + 'max_lifetime' => 0, + ], [], 'sqlite', $factory); + + $first = $pool->get(); + \Swoole\Coroutine::sleep(0.05); + $pool->release($first); + $second = $pool->get(); + }); + + $this->assertSame($first, $second, 'max_lifetime of 0 must never expire connections.'); + } + + public function test_prune_closes_over_age_connections_even_at_min_connections(): void + { + $this->skipIfNoSwooleCoroutine(); + + if (! extension_loaded('pdo_sqlite')) { + $this->markTestSkipped('PDO SQLite is required.'); + } + + $factory = Mockery::mock(ConnectionFactory::class); + $factory->shouldReceive('make') + ->once() + ->withAnyArgs() + ->andReturnUsing(fn () => new Connection(new PDO('sqlite::memory:'), 'database', '', [])); + + $closed = null; + $stats = null; + + \Swoole\Coroutine\run(function () use ($factory, &$closed, &$stats) { + $pool = new DatabasePool([ + 'min_connections' => 1, + 'max_connections' => 1, + 'wait_timeout' => 0.1, + 'heartbeat' => -1, + 'max_idle_time' => 60.0, + 'max_lifetime' => 0.01, + ], [], 'sqlite', $factory); + + \Swoole\Coroutine::sleep(0.03); + + $closed = $pool->pruneIdleConnections(); + $stats = $pool->getStats(); + }); + + $this->assertSame(1, $closed, 'Over-age connections must be pruned even when the pool is at min_connections.'); + $this->assertSame(0, $stats['current_connections']); + } + protected function newPoolWithoutConstructor(): DatabasePool { $reflection = new \ReflectionClass(DatabasePool::class); From 96e6a61534d1d1fcc912dc0247c886f472f11b91 Mon Sep 17 00:00:00 2001 From: Adhik Joshi Date: Thu, 20 Aug 2026 05:05:12 +0530 Subject: [PATCH 2/4] Probe the release fast-path so the pruner cannot mask it in tests --- tests/Unit/DatabasePoolTest.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/Unit/DatabasePoolTest.php b/tests/Unit/DatabasePoolTest.php index e24a722..66fd086 100644 --- a/tests/Unit/DatabasePoolTest.php +++ b/tests/Unit/DatabasePoolTest.php @@ -442,6 +442,13 @@ public function test_release_recycles_connection_past_max_lifetime(): void $first = $pool->get(); \Swoole\Coroutine::sleep(0.08); + + // Probe: the expired fast-path must close the connection without + // running the full session reset first (resetConnection flushes + // the query log, so a surviving entry proves it was skipped). + $first->enableQueryLog(); + $first->logQuery('probe', [], 0); + $pool->release($first); $statsAfterRelease = $pool->getStats(); @@ -452,6 +459,7 @@ public function test_release_recycles_connection_past_max_lifetime(): void $this->assertSame(0, $statsAfterRelease['current_connections'], 'Expired connection must be closed, not re-pooled.'); $this->assertSame(0, $statsAfterRelease['available_connections']); $this->assertNotSame($first, $second, 'A fresh connection must replace the expired one.'); + $this->assertCount(1, $first->getQueryLog(), 'Expired connections must be closed directly, without a pointless session reset.'); } public function test_release_repools_connection_within_max_lifetime(): void From 6fe923e9e57ab17709618a77f2e31894a76948f6 Mon Sep 17 00:00:00 2001 From: Adhik Joshi Date: Thu, 20 Aug 2026 05:06:40 +0530 Subject: [PATCH 3/4] Document pooled-connection max_lifetime recycling --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 679aa90..9f88427 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,12 @@ DB_PERSISTENT=false DB_POOL_HEARTBEAT=10 DB_POOL_MAX_IDLE_TIME=60 +# Pooled connections are recycled after this many seconds (default 300, +# 0 disables). PDO statements destroyed while a pooled connection is busy in +# another coroutine leak server-side (mysqlnd skips COM_STMT_CLOSE and never +# retries), so recycling bounds what any one connection can accumulate. +# Set via the 'max_lifetime' key of the connection's 'pool' config array. + # Redis persistent sockets are unsafe for request-scoped coroutine managers. REDIS_PERSISTENT=false REDIS_SESSION_PERSISTENT=false From 598f3ff2443300e31bf316beb7341eb1a8c2d208 Mon Sep 17 00:00:00 2001 From: Adhik Joshi Date: Thu, 20 Aug 2026 05:23:52 +0530 Subject: [PATCH 4/4] Harden the leak fix per adversarial review - Detach pooled connections first in the finally block so a throw from the Livewire mutex or Redis release can never orphan a borrowed slot. - Sweep connections that destructors borrow into context during teardown or gc, and fold flush-time borrows in before Context::clear(). - Unset the caught worker exception before releasing: its trace args reference the request graph, including PDO statements. - Keep per-request pruning across ALL pools (previously done by DatabaseManager::releaseConnections, which the Worker no longer calls). - Gate the teardown gc on gc_status()['roots']. - Roll back abandoned transactions before recycling an expired connection: disconnect() only drops the PDO reference, so a leaked statement could keep the session and its row locks alive until a future gc. - Restart the lifetime clock when the reconnector swaps in a fresh PDO. - Close (instead of silently dropping) connections the pruner fails to re-pool, and trim the '.pool' suffix instead of str_replace. --- README.md | 6 ++ src/Swoole/Database/DatabaseManager.php | 14 +++- src/Swoole/Database/DatabasePool.php | 60 ++++++++++++++-- src/Worker.php | 70 +++++++++++++++---- ...WorkerPooledConnectionReleaseOrderTest.php | 60 ++++++++++++++++ tests/Unit/DatabasePoolTest.php | 36 ++++++++++ 6 files changed, 224 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9f88427..f5aec3c 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,12 @@ DB_POOL_MAX_IDLE_TIME=60 # another coroutine leak server-side (mysqlnd skips COM_STMT_CLOSE and never # retries), so recycling bounds what any one connection can accumulate. # Set via the 'max_lifetime' key of the connection's 'pool' config array. +# +# The Worker also collects each request's cycle garbage BEFORE its pooled +# connections re-enter the pool, which prevents the leak for statements the +# request left behind. Garbage abandoned by one coroutine can still be +# collected while another coroutine's connection is busy — rare, and +# max_lifetime recycling bounds that residue; it cannot be eliminated. # Redis persistent sockets are unsafe for request-scoped coroutine managers. REDIS_PERSISTENT=false diff --git a/src/Swoole/Database/DatabaseManager.php b/src/Swoole/Database/DatabaseManager.php index 730b6d2..ca1de9f 100644 --- a/src/Swoole/Database/DatabaseManager.php +++ b/src/Swoole/Database/DatabaseManager.php @@ -58,7 +58,7 @@ protected function getPool($name) 'wait_timeout' => 3.0, 'heartbeat' => -1, 'max_idle_time' => 60.0, - 'max_lifetime' => 300.0, + 'max_lifetime' => DatabasePool::DEFAULT_MAX_LIFETIME, ]; $this->pools[$name] = new DatabasePool( @@ -120,6 +120,16 @@ protected function syncApplication(): void } } + /** + * Release this coroutine's pooled connections immediately. + * + * The Worker no longer uses this: it detaches connections, destroys the + * request's object graph, and only then releases, so statements held in + * cycle garbage cannot be destroyed while the connection is busy in + * another coroutine (which leaks the server-side prepared statement). + * Calling this mid-request re-pools connections without that ordering — + * only use it when no statement from this coroutine can still be alive. + */ public function releaseConnections() { if (!Context::inCoroutine()) { @@ -131,7 +141,7 @@ public function releaseConnections() foreach ($allContext as $key => $value) { if (str_ends_with($key, '.pool')) { // Get the connection - $connectionKey = str_replace('.pool', '', $key); + $connectionKey = substr($key, 0, -strlen('.pool')); $connection = Context::get($connectionKey); if ($connection && $value instanceof DatabasePool) { diff --git a/src/Swoole/Database/DatabasePool.php b/src/Swoole/Database/DatabasePool.php index 73b10d9..57a421b 100644 --- a/src/Swoole/Database/DatabasePool.php +++ b/src/Swoole/Database/DatabasePool.php @@ -14,6 +14,13 @@ */ class DatabasePool { + /** + * Default number of seconds a pooled connection lives before it is + * recycled. Recycling frees any server-side prepared statements the + * connection has leaked (see release()). + */ + public const DEFAULT_MAX_LIFETIME = 300.0; + protected Channel $channel; protected int $currentConnections = 0; protected array $config; @@ -159,10 +166,13 @@ public function release($connection): void // PDO statements destroyed while a connection is busy in another // coroutine leak server-side (mysqlnd skips COM_STMT_CLOSE on a busy // connection and never retries), so any long-lived pooled connection - // accumulates leaked prepared statements. Closing it here frees them - // all; disconnecting also discards any dirty server-side state. + // accumulates leaked prepared statements. Closing it frees them all. + // Transactions are rolled back explicitly first: disconnect() only + // drops the Connection's PDO reference, and a leaked statement can + // keep the PDO (and its row locks) alive until a future gc run. if ($this->hasOutlivedMaxLifetime($connection)) { $this->markBorrowed($connection); + $this->rollBackAbandonedTransactions($connection); $this->closeConnection($connection); $this->currentConnections--; $this->pruneIdleConnections(); @@ -206,7 +216,7 @@ public function pruneIdleConnections(?float $now = null): int // since an over-age connection carries leaked server-side prepared // statements that only closing it can free. The pool refills on demand. $idlePruningActive = $maxIdleTime > 0 && $this->currentConnections > $minConnections; - $lifetimePruningActive = ((float) ($this->config['max_lifetime'] ?? 300.0)) > 0; + $lifetimePruningActive = ((float) ($this->config['max_lifetime'] ?? self::DEFAULT_MAX_LIFETIME)) > 0; if (! $idlePruningActive && ! $lifetimePruningActive) { return 0; @@ -245,7 +255,13 @@ public function pruneIdleConnections(?float $now = null): int } foreach ($kept as $connection) { - $this->channel->push($connection, 0.001); + if (! $this->channel->push($connection, 0.001)) { + error_log('⚠️ Could not re-pool a kept connection - closing it instead'); + $this->markBorrowed($connection); + $this->closeConnection($connection); + $this->currentConnections--; + $closed++; + } } return $closed; @@ -341,6 +357,34 @@ protected function closeConnection($connection): void } } + /** + * Roll back any transaction left open on a connection about to be closed. + * + * @see release() for why closing alone is not enough. + */ + protected function rollBackAbandonedTransactions($connection): void + { + if (! $connection instanceof Connection) { + return; + } + + try { + if ($connection->transactionLevel() > 0) { + $connection->rollBack(0); + } + + $pdo = $connection->getPdo(); + + if ($pdo && $pdo->inTransaction()) { + $pdo->rollBack(); + } + + $connection->unsetTransactionManager(); + } catch (Throwable $e) { + error_log('⚠️ Could not roll back before recycling expired connection: '.$e->getMessage()); + } + } + /** * Whether a connection is older than the pool's max_lifetime. * @@ -349,7 +393,7 @@ protected function closeConnection($connection): void */ protected function hasOutlivedMaxLifetime($connection, ?float $now = null): bool { - $maxLifetime = (float) ($this->config['max_lifetime'] ?? 300.0); + $maxLifetime = (float) ($this->config['max_lifetime'] ?? self::DEFAULT_MAX_LIFETIME); if ($maxLifetime <= 0 || ! is_object($connection)) { return false; @@ -385,6 +429,10 @@ protected function createConnection() $connection->setPdo($fresh->getRawPdo()) ->setReadPdo($fresh->getRawReadPdo()); + + // The server session is brand new, so restart the + // max_lifetime clock along with it. + $this->createdAt[spl_object_id($connection)] = microtime(true); }); } @@ -473,7 +521,7 @@ public function getStats(): array 'max_connections' => $this->config['max_connections'] ?? 10, 'min_connections' => $this->config['min_connections'] ?? 1, 'max_idle_time' => $this->config['max_idle_time'] ?? 60.0, - 'max_lifetime' => $this->config['max_lifetime'] ?? 300.0, + 'max_lifetime' => $this->config['max_lifetime'] ?? self::DEFAULT_MAX_LIFETIME, ]; } diff --git a/src/Worker.php b/src/Worker.php index 88993f7..3f081e9 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -135,23 +135,29 @@ public function handle(Request $request, RequestContext $context): void } catch (Throwable $e) { $this->handleWorkerError($e, $sandbox, $request, $context, $responded); } finally { - // Detach this coroutine's pooled connections but do NOT return - // them to the pool yet. Statements the request still references - // must be destroyed while these connections are idle: mysqlnd - // silently skips COM_STMT_CLOSE when a connection is busy in - // another coroutine, permanently leaking the statement server-side. + // Detach this coroutine's pooled connections FIRST — before any + // call that could throw — but do NOT return them to the pool yet. + // Statements the request still references must be destroyed while + // these connections are idle: mysqlnd silently skips + // COM_STMT_CLOSE when a connection is busy in another coroutine, + // permanently leaking the statement server-side. (On the error + // paths below the connections are still released, just without + // the garbage-first ordering — a bounded, logged degradation.) $pooledConnections = []; + $dbManager = null; try { if ($inCoroutine) { - if ($sandbox->bound(LivewireCoroutineMutex::class)) { - $sandbox->make(LivewireCoroutineMutex::class)->releaseAllForCurrentCoroutine(); - } - try { $pooledConnections = $this->detachDatabaseConnectionsFromContext(); - } catch (Throwable $e) { - error_log('⚠️ Failed to detach coroutine DB connections: '.$e->getMessage()); + } catch (Throwable $detachException) { + error_log('⚠️ Failed to detach coroutine DB connections: '.$detachException->getMessage()); + } + + $dbManager = $sandbox->bound('db') ? $sandbox->make('db') : null; + + if ($sandbox->bound(LivewireCoroutineMutex::class)) { + $sandbox->make(LivewireCoroutineMutex::class)->releaseAllForCurrentCoroutine(); } $this->releaseCoroutineRedisConnections(); @@ -163,6 +169,15 @@ public function handle(Request $request, RequestContext $context): void $this->app->make('view.engine.resolver')->forget('php'); if ($inCoroutine) { + // Destructors triggered by flush() may have borrowed fresh + // pooled connections into context. Fold them into the + // deferred set before the context is cleared, or their + // pool slots would leak. + $pooledConnections = array_merge( + $pooledConnections, + $this->detachDatabaseConnectionsFromContext() + ); + Context::clear(); } else { CurrentApplication::set($this->app); @@ -171,13 +186,40 @@ public function handle(Request $request, RequestContext $context): void // After the request handling process has completed we will unset some variables // plus reset the current application state back to its original state before // it was cloned. Then we will be ready for the next worker iteration loop. - unset($gateway, $sandbox, $scope, $context, $request, $response, $octaneResponse, $output); + // $e is included: its trace args reference the request's object + // graph (potentially PDO statements), which must die before the + // connections below are released. + unset($gateway, $sandbox, $scope, $context, $request, $response, $octaneResponse, $output, $e); - if ($inCoroutine && $pooledConnections !== []) { + if ($inCoroutine && $pooledConnections !== [] && gc_status()['roots'] > 0) { gc_collect_cycles(); } } finally { $this->releaseDetachedDatabaseConnections($pooledConnections); + + if ($inCoroutine) { + // A destructor run during teardown or gc above may have + // borrowed another pooled connection into context after + // the detach passes. Sweep again so it cannot orphan a + // pool slot. + try { + $this->releaseDetachedDatabaseConnections($this->detachDatabaseConnectionsFromContext()); + } catch (Throwable $sweepException) { + error_log('⚠️ Failed to sweep late coroutine DB connections: '.$sweepException->getMessage()); + } + + // Keep pruning ALL pools once per request, as the removed + // DatabaseManager::releaseConnections() call used to, so + // pools for connections this request never touched still + // shed idle and over-age entries. + if ($dbManager && method_exists($dbManager, 'pruneIdleConnections')) { + try { + $dbManager->pruneIdleConnections(); + } catch (Throwable $pruneException) { + error_log('⚠️ Failed to prune idle DB connections: '.$pruneException->getMessage()); + } + } + } } } } @@ -227,7 +269,7 @@ protected function detachDatabaseConnectionsFromContext(): array continue; } - $connectionKey = str_replace('.pool', '', $key); + $connectionKey = substr($key, 0, -strlen('.pool')); $connection = Context::get($connectionKey); if ($connection && $value instanceof \Laravel\Octane\Swoole\Database\DatabasePool) { diff --git a/tests/Feature/WorkerPooledConnectionReleaseOrderTest.php b/tests/Feature/WorkerPooledConnectionReleaseOrderTest.php index 5d3977d..f33cf0c 100644 --- a/tests/Feature/WorkerPooledConnectionReleaseOrderTest.php +++ b/tests/Feature/WorkerPooledConnectionReleaseOrderTest.php @@ -60,6 +60,66 @@ public function test_cycle_garbage_is_collected_before_pooled_connections_are_re 'Request garbage must be destroyed (destruct) before the pooled connection is released (release).' ); } + /** + * A destructor that runs during teardown gc can borrow a fresh pooled + * connection into the already-detached context. The worker must sweep + * and release it, or the pool slot is orphaned and the pool eventually + * reports exhaustion. + * + * @requires extension swoole + */ + public function test_connections_borrowed_by_destructors_during_teardown_are_released(): void + { + if (! class_exists(Coroutine::class) || ! function_exists('Swoole\\Coroutine\\run')) { + $this->markTestSkipped('Swoole coroutine support is required.'); + } + + $events = new ArrayObject(); + $latePool = new ReleaseOrderRecordingPool($events); + + $this->app['router']->get('/pool-late-borrow', function () use ($events, $latePool) { + Context::set('db.connection.main', new stdClass()); + Context::set('db.connection.main.pool', new ReleaseOrderRecordingPool(new ArrayObject())); + + $probe = new LateBorrowProbe($latePool); + $probe->self = $probe; + unset($probe); + + return response()->json(['ok' => true]); + }); + + $client = new FakeClient([]); + $worker = $this->createWorker($client); + $worker->boot(); + + \Swoole\Coroutine\run(function () use ($worker) { + Coroutine::create(function () use ($worker) { + $request = Request::create('/pool-late-borrow', 'GET'); + $worker->handle($request, new RequestContext(['request' => $request])); + }); + }); + + $this->assertContains( + 'release', + iterator_to_array($events), + 'A connection borrowed into context by a destructor during teardown must still be released.' + ); + } +} + +class LateBorrowProbe +{ + public ?self $self = null; + + public function __construct(protected DatabasePool $pool) + { + } + + public function __destruct() + { + Context::set('db.connection.late', new stdClass()); + Context::set('db.connection.late.pool', $this->pool); + } } class ReleaseOrderRecordingPool extends DatabasePool diff --git a/tests/Unit/DatabasePoolTest.php b/tests/Unit/DatabasePoolTest.php index 66fd086..099996d 100644 --- a/tests/Unit/DatabasePoolTest.php +++ b/tests/Unit/DatabasePoolTest.php @@ -501,6 +501,42 @@ public function test_release_repools_connection_within_max_lifetime(): void $this->assertSame($first, $second, 'A young connection must be re-pooled and reused.'); } + public function test_expired_connection_rolls_back_open_transaction_before_close(): void + { + $this->skipIfNoSwooleCoroutine(); + + if (! extension_loaded('pdo_sqlite')) { + $this->markTestSkipped('PDO SQLite is required.'); + } + + $pdo = new PDO('sqlite::memory:'); + + $factory = Mockery::mock(ConnectionFactory::class); + $factory->shouldReceive('make') + ->once() + ->withAnyArgs() + ->andReturn(new Connection($pdo, 'database', '', [])); + + \Swoole\Coroutine\run(function () use ($factory) { + $pool = new DatabasePool([ + 'min_connections' => 0, + 'max_connections' => 1, + 'wait_timeout' => 0.1, + 'max_lifetime' => 0.01, + ], [], 'sqlite', $factory); + + $connection = $pool->get(); + $connection->beginTransaction(); + \Swoole\Coroutine::sleep(0.03); + $pool->release($connection); + }); + + // Connection::disconnect() only drops the PDO reference; if another + // reference keeps the PDO alive (we do here, as a leaked statement + // would), an un-rolled-back transaction would keep holding its locks. + $this->assertFalse($pdo->inTransaction(), 'The expired fast-path must roll back abandoned transactions before closing.'); + } + public function test_zero_max_lifetime_disables_recycling(): void { $this->skipIfNoSwooleCoroutine();