From 2fbafe8062d2c2c792b17336f798190b345399bd Mon Sep 17 00:00:00 2001 From: Chris Jenkinson Date: Fri, 25 Sep 2026 05:22:36 +0000 Subject: [PATCH 1/5] Make SimpleTextMatcher match all remaining text, including newlines --- spec/Matcher/SimpleTextMatcherSpec.php | 14 ++++++++++++++ src/Matcher/SimpleTextMatcher.php | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/spec/Matcher/SimpleTextMatcherSpec.php b/spec/Matcher/SimpleTextMatcherSpec.php index 776aa85..77de977 100644 --- a/spec/Matcher/SimpleTextMatcherSpec.php +++ b/spec/Matcher/SimpleTextMatcherSpec.php @@ -21,6 +21,20 @@ public function it_matches_any_text(): void $this->match('1234')->shouldBeLike($matchedText); } + public function it_matches_text_across_newlines(): void + { + $matchedText = new MatchedText(['all' => "12\n34\n"]); + + $this->match("12\n34\n")->shouldBeLike($matchedText); + } + + public function it_matches_text_starting_with_a_newline(): void + { + $matchedText = new MatchedText(['all' => "\n1234"]); + + $this->match("\n1234")->shouldBeLike($matchedText); + } + public function it_does_not_match_an_empty_string(): void { $this->match('')->shouldReturn(null); diff --git a/src/Matcher/SimpleTextMatcher.php b/src/Matcher/SimpleTextMatcher.php index e2ac2de..1ef73a1 100644 --- a/src/Matcher/SimpleTextMatcher.php +++ b/src/Matcher/SimpleTextMatcher.php @@ -15,7 +15,7 @@ class SimpleTextMatcher extends AbstractMatcher public function __construct() { - $pattern = '/(?.+)/Ax'; + $pattern = '/(?.+)/As'; $this->finder = new RegexFinder($pattern); } From a763ef831af8854d18719a9de909dc90d6c6473b Mon Sep 17 00:00:00 2001 From: Chris Jenkinson Date: Fri, 25 Sep 2026 05:26:43 +0000 Subject: [PATCH 2/5] Throw RegexFailedException when preg_match() fails instead of treating it as no match --- spec/Finder/RegexFinderSpec.php | 8 ++++++++ src/Finder/RegexFailedException.php | 25 +++++++++++++++++++++++++ src/Finder/RegexFinder.php | 8 +++++++- 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 src/Finder/RegexFailedException.php diff --git a/spec/Finder/RegexFinderSpec.php b/spec/Finder/RegexFinderSpec.php index 8e8e95f..760bff0 100644 --- a/spec/Finder/RegexFinderSpec.php +++ b/spec/Finder/RegexFinderSpec.php @@ -4,6 +4,7 @@ namespace spec\chrisjenkinson\StructuredDocumentParser\Finder; +use chrisjenkinson\StructuredDocumentParser\Finder\RegexFailedException; use PhpSpec\ObjectBehavior; class RegexFinderSpec extends ObjectBehavior @@ -22,6 +23,13 @@ public function it_returns_false_if_nothing_is_found(): void $this->find('def')->shouldReturn(false); } + public function it_throws_if_the_regex_fails(): void + { + $this->beConstructedWith('/abc/u'); + + $this->shouldThrow(new RegexFailedException('/abc/u', 'Malformed UTF-8 characters, possibly incorrectly encoded'))->during('find', ["\xff"]); + } + public function it_stores_matches(): void { $this->beConstructedWith('/(?.*)/Ax'); diff --git a/src/Finder/RegexFailedException.php b/src/Finder/RegexFailedException.php new file mode 100644 index 0000000..863a540 --- /dev/null +++ b/src/Finder/RegexFailedException.php @@ -0,0 +1,25 @@ +pattern; + } +} diff --git a/src/Finder/RegexFinder.php b/src/Finder/RegexFinder.php index 82858de..8adfe66 100644 --- a/src/Finder/RegexFinder.php +++ b/src/Finder/RegexFinder.php @@ -23,7 +23,13 @@ public function __construct(string $pattern) public function find(string $text): bool { - if (preg_match($this->pattern, $text, $matches)) { + $result = preg_match($this->pattern, $text, $matches); + + if (false === $result) { + throw new RegexFailedException($this->pattern, preg_last_error_msg()); + } + + if (1 === $result) { $this->matches = $matches; return true; From 6b0507b844ad10fb38fa2d63638b8bec96e34e21 Mon Sep 17 00:00:00 2001 From: Chris Jenkinson Date: Fri, 25 Sep 2026 05:28:51 +0000 Subject: [PATCH 3/5] Document the 'all' key contract on MatcherInterface and reject matched text that breaks it --- spec/State/InitialStateSpec.php | 31 +++++++++++++++++++++++ src/Matcher/MatcherInterface.php | 7 +++++ src/State/AbstractState.php | 15 +++++++++++ src/State/InvalidMatchedTextException.php | 25 ++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 src/State/InvalidMatchedTextException.php diff --git a/spec/State/InitialStateSpec.php b/spec/State/InitialStateSpec.php index fefcaa6..381c98a 100644 --- a/spec/State/InitialStateSpec.php +++ b/spec/State/InitialStateSpec.php @@ -9,6 +9,7 @@ use chrisjenkinson\StructuredDocumentParser\Matcher\MatchedText; use chrisjenkinson\StructuredDocumentParser\Matcher\MatcherInterface; use chrisjenkinson\StructuredDocumentParser\State\AmbiguousTokenFoundException; +use chrisjenkinson\StructuredDocumentParser\State\InvalidMatchedTextException; use chrisjenkinson\StructuredDocumentParser\State\NoTokenFoundException; use chrisjenkinson\StructuredDocumentParser\Token\TokenInterface; use PhpSpec\ObjectBehavior; @@ -96,6 +97,36 @@ public function it_uses_the_whole_matcher_name_as_the_token_type_without_a_match $this->findMatchingToken($lexer, $cursor)->getType()->shouldReturn('Heading'); } + public function it_throws_if_the_matched_text_has_no_all_key(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void + { + $cursor->getRemainingText()->willReturn('remainingText'); + $cursor->getCurrentPosition()->willReturn(0); + + $this->registerMatcher($matcher); + + $matcher->match('remainingText')->willReturn($matchedText); + $matcher->getName()->willReturn('HeadingMatcher'); + + $matchedText->getAll()->willReturn(['heading' => 'remainingText']); + + $this->shouldThrow(new InvalidMatchedTextException('HeadingMatcher', 'it has no "all" key'))->duringFindMatchingToken($lexer, $cursor); + } + + public function it_throws_if_the_all_key_is_not_a_string(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void + { + $cursor->getRemainingText()->willReturn('remainingText'); + $cursor->getCurrentPosition()->willReturn(0); + + $this->registerMatcher($matcher); + + $matcher->match('remainingText')->willReturn($matchedText); + $matcher->getName()->willReturn('HeadingMatcher'); + + $matchedText->getAll()->willReturn(['all' => 13]); + + $this->shouldThrow(new InvalidMatchedTextException('HeadingMatcher', 'its "all" value is not a string'))->duringFindMatchingToken($lexer, $cursor); + } + public function it_calls_a_callback(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void { $cursor->getRemainingText()->willReturn('remainingText'); diff --git a/src/Matcher/MatcherInterface.php b/src/Matcher/MatcherInterface.php index ff2ed91..8f7e600 100644 --- a/src/Matcher/MatcherInterface.php +++ b/src/Matcher/MatcherInterface.php @@ -6,6 +6,13 @@ interface MatcherInterface { + /** + * Matches against the start of $text, returning null if it does not match. + * + * The returned MatchedText must have an "all" key holding the exact text consumed + * from the start of $text, which the lexer uses to advance. Any other keys are + * made available on the resulting token. + */ public function match(string $text): ?MatchedText; /** diff --git a/src/State/AbstractState.php b/src/State/AbstractState.php index 8470d7f..fec7529 100644 --- a/src/State/AbstractState.php +++ b/src/State/AbstractState.php @@ -37,6 +37,8 @@ public function findMatchingToken(Lexer $lexer, Cursor $cursor): TokenInterface $matchedText = $matchedText[0]; $callback = $callbacks[0]; + $this->guardAgainstInvalidMatchedText($matcher, $matchedText); + if (is_callable($callback)) { $callback($lexer); } @@ -53,6 +55,19 @@ private function getTokenType(string $matcherName): string return $matcherName; } + private function guardAgainstInvalidMatchedText(string $matcherName, MatchedText $matchedText): void + { + $matches = $matchedText->getAll(); + + if (!array_key_exists('all', $matches)) { + throw new InvalidMatchedTextException($matcherName, 'it has no "all" key'); + } + + if (!is_string($matches['all'])) { + throw new InvalidMatchedTextException($matcherName, 'its "all" value is not a string'); + } + } + public function guardAgainstWrongNumberOfMatches(array $matchedText, string $remainingText, array $calledMatchers, int $currentPosition): void { if (1 < count($matchedText)) { diff --git a/src/State/InvalidMatchedTextException.php b/src/State/InvalidMatchedTextException.php new file mode 100644 index 0000000..d5c8ad2 --- /dev/null +++ b/src/State/InvalidMatchedTextException.php @@ -0,0 +1,25 @@ +matcherName; + } +} From 17a74c26254dde147812cdd6150b770ea3419367 Mon Sep 17 00:00:00 2001 From: Chris Jenkinson Date: Fri, 25 Sep 2026 05:30:59 +0000 Subject: [PATCH 4/5] Reject matches whose 'all' value is not at the start of the remaining text --- spec/State/InitialStateSpec.php | 36 ++++++++++++++++++++++++++++++++- src/State/AbstractState.php | 10 ++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/spec/State/InitialStateSpec.php b/spec/State/InitialStateSpec.php index 381c98a..3f26bd1 100644 --- a/spec/State/InitialStateSpec.php +++ b/spec/State/InitialStateSpec.php @@ -32,7 +32,7 @@ public function it_throws_an_exception_if_there_is_an_ambiguous_token(Lexer $lex $matcher1->match('remainingText')->willReturn($matchedText); $matcher2->match('remainingText')->willReturn($matchedText); - $matchedText->getAll()->willReturn([]); + $matchedText->getAll()->willReturn(['all' => 'remaining']); $this->registerMatcher($matcher1); $this->registerMatcher($matcher2); @@ -127,6 +127,40 @@ public function it_throws_if_the_all_key_is_not_a_string(Lexer $lexer, Cursor $c $this->shouldThrow(new InvalidMatchedTextException('HeadingMatcher', 'its "all" value is not a string'))->duringFindMatchingToken($lexer, $cursor); } + public function it_throws_if_the_all_value_is_not_at_the_start_of_the_text(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void + { + $cursor->getRemainingText()->willReturn('remainingText'); + $cursor->getCurrentPosition()->willReturn(0); + + $this->registerMatcher($matcher); + + $matcher->match('remainingText')->willReturn($matchedText); + $matcher->getName()->willReturn('HeadingMatcher'); + + $matchedText->getAll()->willReturn(['all' => 'Text']); + + $this->shouldThrow(new InvalidMatchedTextException('HeadingMatcher', 'its "all" value is not at the start of the text'))->duringFindMatchingToken($lexer, $cursor); + } + + public function it_reports_a_misplaced_match_rather_than_an_ambiguous_token(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher1, MatcherInterface $matcher2, MatchedText $matchedText1, MatchedText $matchedText2): void + { + $cursor->getRemainingText()->willReturn('remainingText'); + $cursor->getCurrentPosition()->willReturn(0); + + $this->registerMatcher($matcher1); + $this->registerMatcher($matcher2); + + $matcher1->match('remainingText')->willReturn($matchedText1); + $matcher1->getName()->willReturn('WordMatcher'); + $matcher2->match('remainingText')->willReturn($matchedText2); + $matcher2->getName()->willReturn('HeadingMatcher'); + + $matchedText1->getAll()->willReturn(['all' => 'remaining']); + $matchedText2->getAll()->willReturn(['all' => 'Text']); + + $this->shouldThrow(new InvalidMatchedTextException('HeadingMatcher', 'its "all" value is not at the start of the text'))->duringFindMatchingToken($lexer, $cursor); + } + public function it_calls_a_callback(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void { $cursor->getRemainingText()->willReturn('remainingText'); diff --git a/src/State/AbstractState.php b/src/State/AbstractState.php index fec7529..4b289e7 100644 --- a/src/State/AbstractState.php +++ b/src/State/AbstractState.php @@ -37,8 +37,6 @@ public function findMatchingToken(Lexer $lexer, Cursor $cursor): TokenInterface $matchedText = $matchedText[0]; $callback = $callbacks[0]; - $this->guardAgainstInvalidMatchedText($matcher, $matchedText); - if (is_callable($callback)) { $callback($lexer); } @@ -55,7 +53,7 @@ private function getTokenType(string $matcherName): string return $matcherName; } - private function guardAgainstInvalidMatchedText(string $matcherName, MatchedText $matchedText): void + private function guardAgainstInvalidMatchedText(string $matcherName, MatchedText $matchedText, string $text): void { $matches = $matchedText->getAll(); @@ -66,6 +64,10 @@ private function guardAgainstInvalidMatchedText(string $matcherName, MatchedText if (!is_string($matches['all'])) { throw new InvalidMatchedTextException($matcherName, 'its "all" value is not a string'); } + + if (!str_starts_with($text, $matches['all'])) { + throw new InvalidMatchedTextException($matcherName, 'its "all" value is not at the start of the text'); + } } public function guardAgainstWrongNumberOfMatches(array $matchedText, string $remainingText, array $calledMatchers, int $currentPosition): void @@ -90,6 +92,8 @@ public function runMatchers(string $text): array $callback = $matcherAndCallback['callback']; if ($matches = $matcher->match($text)) { + $this->guardAgainstInvalidMatchedText($matcher->getName(), $matches, $text); + $matchedTokens[] = $matches; $calledMatchers[] = $matcher->getName(); $callbacks[] = $callback; From fe86448c1cee514c30047155a47b9a074625946f Mon Sep 17 00:00:00 2001 From: Chris Jenkinson Date: Fri, 25 Sep 2026 09:24:56 +0000 Subject: [PATCH 5/5] Track a byte offset in Cursor and a current index in TokenStream to avoid quadratic lexing and consuming --- spec/Lexer/CursorSpec.php | 14 ++++++++++++++ spec/Token/TokenStreamSpec.php | 18 ++++++++++++++++++ src/Lexer/Cursor.php | 11 ++++++++++- src/Token/TokenStream.php | 27 ++++++++++++--------------- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/spec/Lexer/CursorSpec.php b/spec/Lexer/CursorSpec.php index 614ad42..b7389ce 100644 --- a/spec/Lexer/CursorSpec.php +++ b/spec/Lexer/CursorSpec.php @@ -47,4 +47,18 @@ public function it_returns_an_empty_string_when_returning_remaining_text_at_end( $this->getRemainingText()->shouldReturn(''); } + + public function it_returns_the_remaining_text_after_advancing_over_multibyte_characters(): void + { + $this->beConstructedWith("é€\nxé"); + + $this->advance(3); + + $this->getRemainingText()->shouldReturn('xé'); + $this->getCurrentPosition()->shouldReturn(3); + + $this->advance(2); + + $this->isEndOfText()->shouldReturn(true); + } } diff --git a/spec/Token/TokenStreamSpec.php b/spec/Token/TokenStreamSpec.php index 9887b2c..1868ae8 100644 --- a/spec/Token/TokenStreamSpec.php +++ b/spec/Token/TokenStreamSpec.php @@ -121,4 +121,22 @@ public function it_casts_to_a_string(TokenInterface $token1, TokenInterface $tok $this->__toString()->shouldReturn("SomethingToken (1)\nSomethingToken (2)"); } + + public function it_only_includes_unconsumed_tokens_after_consuming(TokenInterface $token1, TokenInterface $token2, TokenInterface $token3): void + { + $token2->__toString()->willReturn('token2'); + $token3->__toString()->willReturn('token3'); + + $this->add($token1); + $this->add($token2); + $this->add($token3); + + $this->consumeToken()->shouldReturn($token1); + + $this->shouldHaveCount(2); + $this->getCurrentToken()->shouldReturn($token2); + $this->lookAhead()->shouldReturn($token3); + $this->lookAhead(2)->shouldReturn(null); + $this->__toString()->shouldReturn("token2\ntoken3"); + } } diff --git a/src/Lexer/Cursor.php b/src/Lexer/Cursor.php index 9bab2ea..4a990fe 100644 --- a/src/Lexer/Cursor.php +++ b/src/Lexer/Cursor.php @@ -21,6 +21,11 @@ class Cursor */ private $text; + /** + * @var int + */ + private $byteOffset = 0; + public function __construct(string $text) { $this->text = $text; @@ -34,7 +39,7 @@ public function getRemainingText(): string return ''; } - return mb_substr($this->text, $this->currentPosition); + return substr($this->text, $this->byteOffset); } public function isEndOfText(): bool @@ -44,7 +49,11 @@ public function isEndOfText(): bool public function advance(int $length): void { + // A UTF-8 character is at most 4 bytes, so this slice always holds $length characters. + $consumed = mb_substr(substr($this->text, $this->byteOffset, $length * 4), 0, $length); + $this->currentPosition += $length; + $this->byteOffset += strlen($consumed); } public function getCurrentPosition(): int diff --git a/src/Token/TokenStream.php b/src/Token/TokenStream.php index ca512a0..e6d42c6 100644 --- a/src/Token/TokenStream.php +++ b/src/Token/TokenStream.php @@ -14,9 +14,14 @@ class TokenStream implements Countable */ private $tokens = []; + /** + * @var int + */ + private $current = 0; + public function __toString(): string { - return implode("\n", $this->tokens); + return implode("\n", array_slice($this->tokens, $this->current)); } public function add(TokenInterface $token): void @@ -26,16 +31,12 @@ public function add(TokenInterface $token): void public function count(): int { - return count($this->tokens); + return count($this->tokens) - $this->current; } public function lookAhead(int $distance = 1): ?TokenInterface { - if (!array_key_exists($distance, $this->tokens)) { - return null; - } - - return $this->tokens[$distance]; + return $this->tokens[$this->current + $distance] ?? null; } public function consumeToken(): TokenInterface @@ -44,16 +45,12 @@ public function consumeToken(): TokenInterface throw new RuntimeException('End of token stream'); } - return array_shift($this->tokens); + return $this->tokens[$this->current++]; } public function getCurrentToken(): ?TokenInterface { - if (0 === count($this->tokens)) { - return null; - } - - return $this->tokens[0]; + return $this->tokens[$this->current] ?? null; } /** @@ -63,7 +60,7 @@ public function getCurrentToken(): ?TokenInterface */ public function expectTokenType(string $expectedType): bool { - if (0 === count($this->tokens)) { + if (0 === count($this)) { throw new RuntimeException(sprintf('No more tokens; expected %s', $expectedType)); } @@ -85,7 +82,7 @@ public function expectTokenType(string $expectedType): bool */ public function expectTokenTypes(array $expectedTypes): bool { - if (0 === count($this->tokens)) { + if (0 === count($this)) { throw new RuntimeException(sprintf('No more tokens; expected any of %s', implode(', ', $expectedTypes))); }