Skip to content

Adds a filter that guarantees to return batches ending with a line br… #119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
edim24 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Adds a filter that guarantees to return batches ending with a line br… #119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions source/Stream/Filters/EnsureEndOfLinesFilter.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
<?php declare(strict_types=1);

namespace t1gor\RobotsTxtParser\Stream\Filters;

use t1gor\RobotsTxtParser\Stream\CustomFilterInterface;

class EnsureEndOfLinesFilter extends \php_user_filter implements CustomFilterInterface
{
public const NAME = 'RTP_ensure_end_of_lines';

public $filtername = self::NAME;

protected string $incompleteLine = '';

public function filter($in, $out, &$consumed, $closing)
{
$buffer = $this->incompleteLine;
while ($bucket = stream_bucket_make_writeable($in)) {
$buffer .= $bucket->data;
}
$consumed += mb_strlen($buffer);

$this->incompleteLine = '';
if (!$closing) {
// Есть как минимум один перенос строки
if (preg_match("/((?s)^.*[\n\r])(.*)$/mui", $buffer, $matches)) {
$buffer = $matches[1]; // все строки с EOL
$this->incompleteLine = $matches[2]; // последняя строка без EOL
} else {
// Всего одна строка без EOL, ждём следующей порции
$this->incompleteLine = $buffer;
return \PSFS_FEED_ME;
}
}
$bucket = stream_bucket_new($this->stream, $buffer);
stream_bucket_append($out, $bucket);
return \PSFS_PASS_ON;
}
}
2 changes: 2 additions & 0 deletions source/Stream/GeneratorBasedReader.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
use Psr\Log\LogLevel;
use t1gor\RobotsTxtParser\LogsIfAvailableTrait;
use t1gor\RobotsTxtParser\RobotsTxtParser;
use t1gor\RobotsTxtParser\Stream\Filters\EnsureEndOfLinesFilter;
use t1gor\RobotsTxtParser\Stream\Filters\SkipDirectivesWithInvalidValuesFilter;
use t1gor\RobotsTxtParser\Stream\Filters\SkipEndOfCommentedLineFilter;
use t1gor\RobotsTxtParser\Stream\Filters\SkipCommentedLinesFilter;
Expand All @@ -27,6 +28,7 @@ class GeneratorBasedReader implements ReaderInterface {
protected function __construct() {
/** @note order matters */
$this->filters = [
EnsureEndOfLinesFilter::class => false,
SkipCommentedLinesFilter::class => false,
SkipEndOfCommentedLineFilter::class => false,
TrimSpacesLeftFilter::class => false,
Expand Down