diff --git a/app/Http/Controllers/WikiController.php b/app/Http/Controllers/WikiController.php index 79bfa0dbe..6cbcb4fbc 100644 --- a/app/Http/Controllers/WikiController.php +++ b/app/Http/Controllers/WikiController.php @@ -5,7 +5,6 @@ use App\Helper\DomainHelper; use App\Helper\DomainValidator; use App\Helper\ProfileValidator; -use App\Jobs\CirrusSearch\ElasticSearchIndexInit; use App\Jobs\ElasticSearchAliasInit; use App\Jobs\KubernetesIngressCreate; use App\Jobs\MediawikiInit; @@ -34,14 +33,12 @@ public function __construct(DomainValidator $domainValidator, ProfileValidator $ } public function create(Request $request): \Illuminate\Http\Response { - $clusterWithoutSharedIndex = Config::get('wbstack.elasticsearch_cluster_without_shared_index'); - $sharedIndexHost = Config::get('wbstack.elasticsearch_shared_index_host'); $sharedIndexPrefix = Config::get('wbstack.elasticsearch_shared_index_prefix'); + $esHosts = Config::get('wbstack.elasticsearch_hosts'); + $isSearchConfigValid = $esHosts && $sharedIndexPrefix; - if (Config::get('wbstack.elasticsearch_enabled_by_default')) { - if (!$clusterWithoutSharedIndex && !($sharedIndexHost && $sharedIndexPrefix)) { - abort(503, 'Search enabled, but its configuration is invalid'); - } + if (Config::get('wbstack.elasticsearch_enabled_by_default') && !$isSearchConfigValid) { + abort(503, 'Search enabled, but its configuration is invalid'); } $user = $request->user(); @@ -171,12 +168,9 @@ public function create(Request $request): \Illuminate\Http\Response { } // dispatch elasticsearch init job to enable the feature - if (Config::get('wbstack.elasticsearch_enabled_by_default')) { - if ($clusterWithoutSharedIndex) { - dispatch(new ElasticSearchIndexInit($wiki->id, $clusterWithoutSharedIndex)); - } - if ($sharedIndexHost && $sharedIndexPrefix) { - dispatch(new ElasticSearchAliasInit($wiki->id)); + if (Config::get('wbstack.elasticsearch_enabled_by_default') && $isSearchConfigValid) { + foreach ($esHosts as $esHost) { + dispatch(new ElasticSearchAliasInit($wiki->id, $esHost)); } } diff --git a/app/Jobs/CirrusSearch/ElasticSearchIndexInit.php b/app/Jobs/CirrusSearch/ElasticSearchIndexInit.php deleted file mode 100644 index dfa2aaf46..000000000 --- a/app/Jobs/CirrusSearch/ElasticSearchIndexInit.php +++ /dev/null @@ -1,64 +0,0 @@ -cluster = $cluster; - parent::__construct($wikiId); - } - - public function cluster(): string { - return $this->cluster; - } - - public function apiModule(): string { - return 'wbstackElasticSearchInit'; - } - - private function logFailure(): void { - Log::error(__METHOD__ . ': Failed initializing elasticsearch.'); - } - - public function handleResponse(string $rawResponse, $error): void { - $response = json_decode($rawResponse, true); - - if (!$this->validateOrFailRequest($response, $rawResponse, $error)) { - $this->logFailure(); - - return; - } - - if (!$this->validateSuccess($response, $rawResponse, $error)) { - $this->logFailure(); - - return; - } - - $output = $response[$this->apiModule()]['output']; - - // if newly created index failed, script ran and update was unsuccessful, then log error. - if (!( - in_array("\tCreating index...ok", $output) || - in_array("\t\tValidating {$this->wikiDB->name}_general alias...ok", $output) - )) { - - Log::error(__METHOD__ . ": Job finished but didn't create or update, something is weird"); - $this->logFailure(); - $this->fail(new \RuntimeException($this->apiModule() . ' call for ' . $this->wikiId . ' was not successful:' . $rawResponse)); - } - } - - protected function getRequestTimeout(): int { - return getenv('CURLOPT_TIMEOUT_ELASTICSEARCH_INIT') !== false - ? intval(getenv('CURLOPT_TIMEOUT_ELASTICSEARCH_INIT')) : parent::getRequestTimeout(); - } - - protected function getQueryParams() { - return parent::getQueryParams() . '&cluster=' . $this->cluster; - } -} diff --git a/app/Jobs/ElasticSearchAliasInit.php b/app/Jobs/ElasticSearchAliasInit.php index 9eb266309..ff06c8e34 100644 --- a/app/Jobs/ElasticSearchAliasInit.php +++ b/app/Jobs/ElasticSearchAliasInit.php @@ -9,6 +9,8 @@ class ElasticSearchAliasInit extends Job { private $wikiId; + private $esHost; + private $dbName; private $sharedPrefix; @@ -16,8 +18,9 @@ class ElasticSearchAliasInit extends Job { /** * @param string $dbName */ - public function __construct(int $wikiId, ?string $sharedPrefix = null) { + public function __construct(int $wikiId, string $esHost, ?string $sharedPrefix = null) { $this->wikiId = $wikiId; + $this->esHost = $esHost; $this->sharedPrefix = $sharedPrefix ?? getenv('ELASTICSEARCH_SHARED_INDEX_PREFIX'); } @@ -68,7 +71,7 @@ public function handle(HttpRequest $request) { } $request->setOptions([ - CURLOPT_URL => getenv('ELASTICSEARCH_SHARED_INDEX_HOST') . '/_aliases', + CURLOPT_URL => $this->esHost . '/_aliases', CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 60 * 15, CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1, diff --git a/config/wbstack.php b/config/wbstack.php index 25d4bcb2b..76f4a2bb9 100644 --- a/config/wbstack.php +++ b/config/wbstack.php @@ -21,8 +21,6 @@ 'elasticsearch_hosts' => array_filter(explode(',', env('ELASTICSEARCH_HOST', ''))), 'elasticsearch_enabled_by_default' => env('WBSTACK_ELASTICSEARCH_ENABLED_BY_DEFAULT', false), - 'elasticsearch_cluster_without_shared_index' => env('ELASTICSEARCH_CLUSTER_WITHOUT_SHARED_INDEX', null), - 'elasticsearch_shared_index_host' => env('ELASTICSEARCH_SHARED_INDEX_HOST', null), 'elasticsearch_shared_index_prefix' => env('ELASTICSEARCH_SHARED_INDEX_PREFIX', null), 'signup_throttling_limit' => env('WBSTACK_SIGNUP_THROTTLING_LIMIT', ''), diff --git a/tests/Jobs/CirrusSearch/ElasticSearchAliasInitTest.php b/tests/Jobs/CirrusSearch/ElasticSearchAliasInitTest.php index 418bc84b1..fe6c8b52d 100644 --- a/tests/Jobs/CirrusSearch/ElasticSearchAliasInitTest.php +++ b/tests/Jobs/CirrusSearch/ElasticSearchAliasInitTest.php @@ -12,6 +12,8 @@ class ElasticSearchAliasInitTest extends TestCase { private $wikiId; + private $esHost; + private $prefix; private $dbName; @@ -19,6 +21,7 @@ class ElasticSearchAliasInitTest extends TestCase { protected function setUp(): void { parent::setUp(); $this->wikiId = Wiki::factory()->create()->id; + $this->esHost = 'elasticsearch-1.localhost'; $this->dbName = WikiDb::factory()->create(['wiki_id' => $this->wikiId])->name; $this->prefix = 'testing_1'; putenv('ELASTICSEARCH_SHARED_INDEX_PREFIX'); @@ -48,7 +51,7 @@ private function getMockRequest() { ->method('setOptions') ->with( [ - CURLOPT_URL => getenv('ELASTICSEARCH_SHARED_INDEX_HOST') . '/_aliases', + CURLOPT_URL => $this->esHost . '/_aliases', CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 60 * 15, CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1, @@ -80,7 +83,7 @@ public function testSuccess() { ->method('fail') ->withAnyParameters(); - $job = new ElasticSearchAliasInit($this->wikiId, $this->prefix); + $job = new ElasticSearchAliasInit($this->wikiId, $this->esHost, $this->prefix); $job->setJob($mockJob); $job->handle($request); } @@ -95,7 +98,7 @@ public function testFailure() { ->method('fail') ->with(new \RuntimeException("Updating Elasticsearch aliases failed for $this->wikiId with {\"acknowledged\":false}")); - $job = new ElasticSearchAliasInit($this->wikiId, $this->prefix); + $job = new ElasticSearchAliasInit($this->wikiId, $this->esHost, $this->prefix); $job->setJob($mockJob); $job->handle($request); } @@ -109,7 +112,7 @@ public function testMissingDatabaseFailure() { ->method('fail') ->with(new \RuntimeException("Failed to get database name for $this->wikiId")); - $job = new ElasticSearchAliasInit($this->wikiId, $this->prefix); + $job = new ElasticSearchAliasInit($this->wikiId, $this->esHost, $this->prefix); $job->setJob($mockJob); $job->handle($request); } @@ -127,7 +130,7 @@ public function testSuccessWithPrefixEnv() { ->method('fail') ->withAnyParameters(); - $job = new ElasticSearchAliasInit($this->wikiId); + $job = new ElasticSearchAliasInit($this->wikiId, $this->esHost); $job->setJob($mockJob); $job->handle($request); } @@ -140,7 +143,7 @@ public function testMissingPrefixFailure() { ->method('fail') ->with(new \RuntimeException("Missing shared index prefix for $this->wikiId")); - $job = new ElasticSearchAliasInit($this->wikiId); + $job = new ElasticSearchAliasInit($this->wikiId, $this->esHost); $job->setJob($mockJob); $job->handle($request); } diff --git a/tests/Jobs/CirrusSearch/ElasticSearchIndexInitTest.php b/tests/Jobs/CirrusSearch/ElasticSearchIndexInitTest.php deleted file mode 100644 index c87d02b19..000000000 --- a/tests/Jobs/CirrusSearch/ElasticSearchIndexInitTest.php +++ /dev/null @@ -1,224 +0,0 @@ -user = User::factory()->create(['verified' => true]); - $this->wiki = Wiki::factory()->create(); - WikiManager::factory()->create(['wiki_id' => $this->wiki->id, 'user_id' => $this->user->id]); - WikiSetting::factory()->create( - [ - 'wiki_id' => $this->wiki->id, - 'name' => WikiSetting::wwExtEnableElasticSearch, - 'value' => true, - ] - ); - - $this->wikiDb = WikiDb::factory()->create([ - 'wiki_id' => $this->wiki->id, - ]); - - $this->mwBackendHost = 'mediawiki.localhost'; - - $this->mockMwHostResolver = $this->createMock(MediaWikiHostResolver::class); - $this->mockMwHostResolver->method('getBackendHostForDomain')->willReturn( - $this->mwBackendHost - ); - } - - public function testDispatching() { - $mockJob = $this->createMock(Job::class); - $job = new ElasticSearchIndexInit($this->wiki->id); - $job->setJob($mockJob); - $mockJob->expects($this->once()) - ->method('fail'); - $job->handle(new CurlRequest, $this->mockMwHostResolver); - } - - public function testSuccess() { - $mockResponse = [ - 'warnings' => [], - 'wbstackElasticSearchInit' => [ - 'return' => 0, - 'output' => [ - "\tCreating index...ok", // successfully created some index - ], - ], - ]; - $request = $this->createMock(HttpRequest::class); - $request->method('execute')->willReturn(json_encode($mockResponse)); - - putenv('CURLOPT_TIMEOUT_ELASTICSEARCH_INIT=1234'); - - $request->expects($this->once()) - ->method('setOptions') - ->with([ - CURLOPT_URL => $this->mwBackendHost . '/w/api.php?action=wbstackElasticSearchInit&format=json&cluster=all', - CURLOPT_RETURNTRANSFER => true, - CURLOPT_ENCODING => '', - CURLOPT_TIMEOUT => 1234, - CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1, - CURLOPT_CUSTOMREQUEST => 'POST', - CURLOPT_HTTPHEADER => [ - 'content-type: application/x-www-form-urlencoded', - 'host: ' . $this->wiki->domain, - ], - ]); - - $mockJob = $this->createMock(Job::class); - $mockJob->expects($this->never()) - ->method('fail') - ->withAnyParameters(); - - $job = new ElasticSearchIndexInit($this->wiki->id); - $job->setJob($mockJob); - $job->handle($request, $this->mockMwHostResolver); - - // feature should get enabled - $this->assertSame( - 1, - WikiSetting::where(['wiki_id' => $this->wiki->id, 'name' => WikiSetting::wwExtEnableElasticSearch, 'value' => true])->count() - ); - } - - public function testUpdate() { - $mockResponse = [ - 'warnings' => [], - 'wbstackElasticSearchInit' => [ - 'return' => 0, - 'output' => [ - "\t\tValidating {$this->wikiDb->name}_general alias...ok", - ], - ], - ]; - $request = $this->createMock(HttpRequest::class); - $request->method('execute')->willReturn(json_encode($mockResponse)); - - $mockJob = $this->createMock(Job::class); - $mockJob->expects($this->never()) - ->method('fail') - ->withAnyParameters(); - - $job = new ElasticSearchIndexInit($this->wiki->id); - $job->setJob($mockJob); - $job->handle($request, $this->mockMwHostResolver); - - // feature should get enabled - $this->assertSame( - 1, - WikiSetting::where(['wiki_id' => $this->wiki->id, 'name' => WikiSetting::wwExtEnableElasticSearch, 'value' => true])->count() - ); - } - - public function testJobTriggeredButNoSetting() { - WikiSetting::where(['wiki_id' => $this->wiki->id, 'name' => WikiSetting::wwExtEnableElasticSearch])->first()->delete(); - $request = $this->createMock(HttpRequest::class); - $request->expects($this->never())->method('execute'); - - $job = new ElasticSearchIndexInit($this->wiki->id); - $job->handle($request, $this->mockMwHostResolver); - } - - /** - * @dataProvider failureProvider - * - * @expectedException RuntimeException - */ - public function testFailure(string $expectedFailure, $mockResponse) { - $request = $this->createMock(HttpRequest::class); - $mockJob = $this->createMock(Job::class); - $mockJob->expects($this->once()) - ->method('fail'); - - $request->method('execute')->willReturn(json_encode($mockResponse)); - - $job = new ElasticSearchIndexInit($this->wiki->id); - $job->setJob($mockJob); - $job->handle($request, $this->mockMwHostResolver); - - $this->assertSame( - 1, - WikiSetting::where(['wiki_id' => $this->wiki->id, 'name' => WikiSetting::wwExtEnableElasticSearch, 'value' => true])->count() - ); - } - - public function testCurlFailure() { - $expectedFailure = 'wbstackElasticSearchInit curl error for : Scary Error!'; - $mockResponse = []; - - $request = $this->createMock(HttpRequest::class); - $request->method('error')->willReturn('Scary Error!'); - $mockJob = $this->createMock(Job::class); - $mockJob->expects($this->once()) - ->method('fail'); - - $request->method('execute')->willReturn(json_encode($mockResponse)); - - $job = new ElasticSearchIndexInit($this->wiki->id); - $job->setJob($mockJob); - $job->handle($request, $this->mockMwHostResolver); - - $this->assertSame( - 1, - WikiSetting::where(['wiki_id' => $this->wiki->id, 'name' => WikiSetting::wwExtEnableElasticSearch, 'value' => true])->count() - ); - } - - public static function failureProvider() { - - $mockResponse = []; - yield [ - 'wbstackElasticSearchInit call for . No wbstackElasticSearchInit key in response: []', - $mockResponse, - ]; - - $mockResponse = [ - 'warnings' => [], - 'wbstackElasticSearchInit' => [ - 'return' => 0, - 'output' => [], - ], - ]; - - yield [ - 'wbstackElasticSearchInit call for was not successful:{"warnings":[],"wbstackElasticSearchInit":{"return":0,"output":[]}}', - $mockResponse, - ]; - - $mockResponse['wbstackElasticSearchInit']['return'] = 1; - yield [ - 'wbstackElasticSearchInit call for was not successful:{"warnings":[],"wbstackElasticSearchInit":{"return":1,"output":[]}}', - $mockResponse, - ]; - } -} diff --git a/tests/Routes/Wiki/CreateTest.php b/tests/Routes/Wiki/CreateTest.php index a5e482e9b..6d66842ad 100644 --- a/tests/Routes/Wiki/CreateTest.php +++ b/tests/Routes/Wiki/CreateTest.php @@ -2,7 +2,6 @@ namespace Tests\Routes\Wiki\Managers; -use App\Jobs\CirrusSearch\ElasticSearchIndexInit; use App\Jobs\ElasticSearchAliasInit; use App\Jobs\MediawikiInit; use App\Jobs\ProvisionWikiDbJob; @@ -41,14 +40,12 @@ class CreateTest extends TestCase { */ public function testWikiCreateDispatchesSomeJobs($elasticSearchConfig) { $enabledForNewWikis = $elasticSearchConfig['enabledForNewWikis']; - $clusterWithoutSharedIndex = $elasticSearchConfig['clusterWithoutSharedIndex'] ?? null; - $sharedIndexHost = $elasticSearchConfig['sharedIndexHost'] ?? null; $sharedIndexPrefix = $elasticSearchConfig['sharedIndexPrefix'] ?? null; + $esHosts = $elasticSearchConfig['esHosts'] ?? null; Config::set('wbstack.elasticsearch_enabled_by_default', $enabledForNewWikis); - Config::set('wbstack.elasticsearch_cluster_without_shared_index', $clusterWithoutSharedIndex); - Config::set('wbstack.elasticsearch_shared_index_host', $sharedIndexHost); Config::set('wbstack.elasticsearch_shared_index_prefix', $sharedIndexPrefix); + Config::set('wbstack.elasticsearch_hosts', $esHosts); $this->createSQLandQSDBs(); @@ -73,21 +70,13 @@ public function testWikiCreateDispatchesSomeJobs($elasticSearchConfig) { ] ); - if ($enabledForNewWikis && $clusterWithoutSharedIndex) { - Queue::assertPushed(function (ElasticSearchIndexInit $job) use ($clusterWithoutSharedIndex) { - return $job->cluster() === $clusterWithoutSharedIndex; - }); - } else { - Queue::assertNotPushed(ElasticSearchIndexInit::class); - } - - if ($enabledForNewWikis && $sharedIndexHost && $sharedIndexPrefix) { - Queue::assertPushed(ElasticSearchAliasInit::class, 1); + if ($enabledForNewWikis && $esHosts && $sharedIndexPrefix) { + Queue::assertPushed(ElasticSearchAliasInit::class, count($esHosts)); } else { Queue::assertNotPushed(ElasticSearchAliasInit::class); } - if ($enabledForNewWikis && !$clusterWithoutSharedIndex && !($sharedIndexHost && $sharedIndexPrefix)) { + if ($enabledForNewWikis && !($esHosts && $sharedIndexPrefix)) { $response->assertStatus(503) ->assertJsonPath('message', 'Search enabled, but its configuration is invalid'); @@ -119,20 +108,26 @@ public function testWikiCreateDispatchesSomeJobs($elasticSearchConfig) { public static function createDispatchesSomeJobsProvider() { yield [[ 'enabledForNewWikis' => true, - 'clusterWithoutSharedIndex' => 'all', - 'sharedIndexHost' => 'somehost', 'sharedIndexPrefix' => 'testing_1', + 'esHosts' => ['elasticsearch-1.localhost'], ]]; yield [[ 'enabledForNewWikis' => true, - 'clusterWithoutSharedIndex' => 'default', + 'sharedIndexPrefix' => 'testing_1', + 'esHosts' => ['elasticsearch-1.localhost', 'elasticsearch-2.localhost'], + ]]; + + yield [[ + 'enabledForNewWikis' => false, + 'sharedIndexPrefix' => 'testing_1', + 'esHosts' => ['elasticsearch-1.localhost'], ]]; yield [[ 'enabledForNewWikis' => true, - 'sharedIndexHost' => 'somehost', 'sharedIndexPrefix' => 'testing_1', + 'esHosts' => [], ]]; yield [[