diff --git a/source/Stream/Filters/EnsureEndOfLinesFilter.php b/source/Stream/Filters/EnsureEndOfLinesFilter.php new file mode 100644 index 0000000..30203f0 --- /dev/null +++ b/source/Stream/Filters/EnsureEndOfLinesFilter.php @@ -0,0 +1,39 @@ +incompleteLine; + while ($bucket = stream_bucket_make_writeable($in)) { + $buffer .= $bucket->data; + } + $consumed += mb_strlen($buffer); + + $this->incompleteLine = ''; + if (!$closing) { + // Есть как минимум один перенос строки + if (preg_match("/((?s)^.*[\n\r])(.*)$/mui", $buffer, $matches)) { + $buffer = $matches[1]; // все строки с EOL + $this->incompleteLine = $matches[2]; // последняя строка без EOL + } else { + // Всего одна строка без EOL, ждём следующей порции + $this->incompleteLine = $buffer; + return \PSFS_FEED_ME; + } + } + $bucket = stream_bucket_new($this->stream, $buffer); + stream_bucket_append($out, $bucket); + return \PSFS_PASS_ON; + } +} diff --git a/source/Stream/GeneratorBasedReader.php b/source/Stream/GeneratorBasedReader.php index 6c92202..f6db0cd 100644 --- a/source/Stream/GeneratorBasedReader.php +++ b/source/Stream/GeneratorBasedReader.php @@ -5,6 +5,7 @@ use Psr\Log\LogLevel; use t1gor\RobotsTxtParser\LogsIfAvailableTrait; use t1gor\RobotsTxtParser\RobotsTxtParser; +use t1gor\RobotsTxtParser\Stream\Filters\EnsureEndOfLinesFilter; use t1gor\RobotsTxtParser\Stream\Filters\SkipDirectivesWithInvalidValuesFilter; use t1gor\RobotsTxtParser\Stream\Filters\SkipEndOfCommentedLineFilter; use t1gor\RobotsTxtParser\Stream\Filters\SkipCommentedLinesFilter; @@ -27,6 +28,7 @@ class GeneratorBasedReader implements ReaderInterface { protected function __construct() { /** @note order matters */ $this->filters = [ + EnsureEndOfLinesFilter::class => false, SkipCommentedLinesFilter::class => false, SkipEndOfCommentedLineFilter::class => false, TrimSpacesLeftFilter::class => false,