diff --git a/packages/mediawiki/0008-nginx-stream-access-log.diff b/packages/mediawiki/0008-nginx-stream-access-log.diff new file mode 100644 index 000000000..54e522c61 --- /dev/null +++ b/packages/mediawiki/0008-nginx-stream-access-log.diff @@ -0,0 +1,38 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +diff --git a/base/NginxDaemon.php b/base/NginxDaemon.php +index aa48307..608e8f1 100644 +--- a/base/NginxDaemon.php ++++ b/base/NginxDaemon.php +@@ -47,13 +47,26 @@ final class NginxDaemon extends Process { + $page_results = Map {}; + + // Custom format: '$status $body_bytes_sent $request_time "$request"' +- $log = file_get_contents($this->options->tempDir.'/access.log'); +- $entries = explode("\n", trim($log)); ++ // Stream the access log one line at a time rather than slurping the whole ++ // file into a single string. On high-throughput hosts (many-core ARM, long ++ // wrk durations) the log can exceed HHVM's ~2GB single-string limit, and ++ // file_get_contents() then aborts the run with ++ // "StringBuffer exceeded 2147483623 bytes of memory". ++ $access_log = $this->options->tempDir.'/access.log'; + $entries_by_request = array(); +- foreach ($entries as $entry) { ++ $fh = fopen($access_log, 'r'); ++ invariant($fh !== false, 'Failed to open nginx access log %s', $access_log); ++ while (($line = fgets($fh)) !== false) { ++ // Strip null bytes that can appear due to nginx log buffer races on ++ // high-throughput ARM systems, then drop the trailing newline. ++ $entry = rtrim(str_replace("\x00", "", $line), "\r\n"); ++ if ($entry === '') { ++ continue; ++ } + $request = explode('"', $entry)[1]; + $entries_by_request[$request][] = $entry; + } ++ fclose($fh); + $combined_times = Vector {}; + + foreach ($entries_by_request as $request => $entries) { diff --git a/packages/mediawiki/0008-nginx-strip-null-bytes.diff b/packages/mediawiki/0008-nginx-strip-null-bytes.diff deleted file mode 100644 index 3b34d899e..000000000 --- a/packages/mediawiki/0008-nginx-strip-null-bytes.diff +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -diff --git a/base/NginxDaemon.php b/base/NginxDaemon.php -index aa48307..3e9bb2a 100644 ---- a/base/NginxDaemon.php -+++ b/base/NginxDaemon.php -@@ -48,6 +48,8 @@ final class NginxDaemon extends Process { - - // Custom format: '$status $body_bytes_sent $request_time "$request"' - $log = file_get_contents($this->options->tempDir.'/access.log'); -+ // Strip null bytes that can appear due to nginx log buffer races on high-throughput ARM systems -+ $log = str_replace("\x00", "", $log); - $entries = explode("\n", trim($log)); - $entries_by_request = array(); - foreach ($entries as $entry) { diff --git a/packages/mediawiki/install_oss_performance_mediawiki.sh b/packages/mediawiki/install_oss_performance_mediawiki.sh index 9f03c0b94..a75664a33 100755 --- a/packages/mediawiki/install_oss_performance_mediawiki.sh +++ b/packages/mediawiki/install_oss_performance_mediawiki.sh @@ -243,9 +243,11 @@ git apply --check "${TEMPLATES_DIR}/0001-oss-performance-scalable-hhvm.diff" \ # templates/oss-performance-mediawiki/0006-oss-performance-reuse-mediawiki-hhvm.diff git apply --check "${TEMPLATES_DIR}/0006-oss-performance-reuse-mediawiki-hhvm.diff" \ && git apply "${TEMPLATES_DIR}/0006-oss-performance-reuse-mediawiki-hhvm.diff" -# Strip null bytes from nginx access log to fix UTF-8 errors in metrics on ARM -git apply --check "${TEMPLATES_DIR}/0008-nginx-strip-null-bytes.diff" \ - && git apply "${TEMPLATES_DIR}/0008-nginx-strip-null-bytes.diff" +# Stream the nginx access log line-by-line (and strip null bytes) so metrics +# collection does not overflow HHVM's ~2GB single-string limit on +# high-throughput ARM hosts +git apply --check "${TEMPLATES_DIR}/0008-nginx-stream-access-log.diff" \ + && git apply "${TEMPLATES_DIR}/0008-nginx-stream-access-log.diff" # apply options for mediawiki mini patch git apply --check "${TEMPLATES_DIR}/0007-oss-performance-more-warmup-options.diff" \