Fix matching and lexer performance

spec/Finder/RegexFinderSpec.php

@@ -4,6 +4,7 @@
 
 namespace spec\chrisjenkinson\StructuredDocumentParser\Finder;
 
+use chrisjenkinson\StructuredDocumentParser\Finder\RegexFailedException;
 use PhpSpec\ObjectBehavior;
 
 class RegexFinderSpec extends ObjectBehavior
@@ -22,6 +23,13 @@ public function it_returns_false_if_nothing_is_found(): void
         $this->find('def')->shouldReturn(false);
     }
 
+    public function it_throws_if_the_regex_fails(): void
+    {
+        $this->beConstructedWith('/abc/u');
+
+        $this->shouldThrow(new RegexFailedException('/abc/u', 'Malformed UTF-8 characters, possibly incorrectly encoded'))->during('find', ["\xff"]);
+    }
+
     public function it_stores_matches(): void
     {
         $this->beConstructedWith('/(?<match1>.*)/Ax');

spec/Lexer/CursorSpec.php

@@ -47,4 +47,18 @@ public function it_returns_an_empty_string_when_returning_remaining_text_at_end(
 
         $this->getRemainingText()->shouldReturn('');
     }
+
+    public function it_returns_the_remaining_text_after_advancing_over_multibyte_characters(): void
+    {
+        $this->beConstructedWith("é€\nxé");
+
+        $this->advance(3);
+
+        $this->getRemainingText()->shouldReturn('xé');
+        $this->getCurrentPosition()->shouldReturn(3);
+
+        $this->advance(2);
+
+        $this->isEndOfText()->shouldReturn(true);
+    }
 }

spec/Matcher/SimpleTextMatcherSpec.php

@@ -21,6 +21,20 @@ public function it_matches_any_text(): void
         $this->match('1234')->shouldBeLike($matchedText);
     }
 
+    public function it_matches_text_across_newlines(): void
+    {
+        $matchedText = new MatchedText(['all' => "12\n34\n"]);
+
+        $this->match("12\n34\n")->shouldBeLike($matchedText);
+    }
+
+    public function it_matches_text_starting_with_a_newline(): void
+    {
+        $matchedText = new MatchedText(['all' => "\n1234"]);
+
+        $this->match("\n1234")->shouldBeLike($matchedText);
+    }
+
     public function it_does_not_match_an_empty_string(): void
     {
         $this->match('')->shouldReturn(null);

spec/State/InitialStateSpec.php

@@ -9,6 +9,7 @@
 use chrisjenkinson\StructuredDocumentParser\Matcher\MatchedText;
 use chrisjenkinson\StructuredDocumentParser\Matcher\MatcherInterface;
 use chrisjenkinson\StructuredDocumentParser\State\AmbiguousTokenFoundException;
+use chrisjenkinson\StructuredDocumentParser\State\InvalidMatchedTextException;
 use chrisjenkinson\StructuredDocumentParser\State\NoTokenFoundException;
 use chrisjenkinson\StructuredDocumentParser\Token\TokenInterface;
 use PhpSpec\ObjectBehavior;
@@ -31,7 +32,7 @@ public function it_throws_an_exception_if_there_is_an_ambiguous_token(Lexer $lex
         $matcher1->match('remainingText')->willReturn($matchedText);
         $matcher2->match('remainingText')->willReturn($matchedText);
 
-        $matchedText->getAll()->willReturn([]);
+        $matchedText->getAll()->willReturn(['all' => 'remaining']);
 
         $this->registerMatcher($matcher1);
         $this->registerMatcher($matcher2);
@@ -96,6 +97,70 @@ public function it_uses_the_whole_matcher_name_as_the_token_type_without_a_match
         $this->findMatchingToken($lexer, $cursor)->getType()->shouldReturn('Heading');
     }
 
+    public function it_throws_if_the_matched_text_has_no_all_key(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void
+    {
+        $cursor->getRemainingText()->willReturn('remainingText');
+        $cursor->getCurrentPosition()->willReturn(0);
+
+        $this->registerMatcher($matcher);
+
+        $matcher->match('remainingText')->willReturn($matchedText);
+        $matcher->getName()->willReturn('HeadingMatcher');
+
+        $matchedText->getAll()->willReturn(['heading' => 'remainingText']);
+
+        $this->shouldThrow(new InvalidMatchedTextException('HeadingMatcher', 'it has no "all" key'))->duringFindMatchingToken($lexer, $cursor);
+    }
+
+    public function it_throws_if_the_all_key_is_not_a_string(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void
+    {
+        $cursor->getRemainingText()->willReturn('remainingText');
+        $cursor->getCurrentPosition()->willReturn(0);
+
+        $this->registerMatcher($matcher);
+
+        $matcher->match('remainingText')->willReturn($matchedText);
+        $matcher->getName()->willReturn('HeadingMatcher');
+
+        $matchedText->getAll()->willReturn(['all' => 13]);
+
+        $this->shouldThrow(new InvalidMatchedTextException('HeadingMatcher', 'its "all" value is not a string'))->duringFindMatchingToken($lexer, $cursor);
+    }
+
+    public function it_throws_if_the_all_value_is_not_at_the_start_of_the_text(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void
+    {
+        $cursor->getRemainingText()->willReturn('remainingText');
+        $cursor->getCurrentPosition()->willReturn(0);
+
+        $this->registerMatcher($matcher);
+
+        $matcher->match('remainingText')->willReturn($matchedText);
+        $matcher->getName()->willReturn('HeadingMatcher');
+
+        $matchedText->getAll()->willReturn(['all' => 'Text']);
+
+        $this->shouldThrow(new InvalidMatchedTextException('HeadingMatcher', 'its "all" value is not at the start of the text'))->duringFindMatchingToken($lexer, $cursor);
+    }
+
+    public function it_reports_a_misplaced_match_rather_than_an_ambiguous_token(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher1, MatcherInterface $matcher2, MatchedText $matchedText1, MatchedText $matchedText2): void
+    {
+        $cursor->getRemainingText()->willReturn('remainingText');
+        $cursor->getCurrentPosition()->willReturn(0);
+
+        $this->registerMatcher($matcher1);
+        $this->registerMatcher($matcher2);
+
+        $matcher1->match('remainingText')->willReturn($matchedText1);
+        $matcher1->getName()->willReturn('WordMatcher');
+        $matcher2->match('remainingText')->willReturn($matchedText2);
+        $matcher2->getName()->willReturn('HeadingMatcher');
+
+        $matchedText1->getAll()->willReturn(['all' => 'remaining']);
+        $matchedText2->getAll()->willReturn(['all' => 'Text']);
+
+        $this->shouldThrow(new InvalidMatchedTextException('HeadingMatcher', 'its "all" value is not at the start of the text'))->duringFindMatchingToken($lexer, $cursor);
+    }
+
     public function it_calls_a_callback(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void
     {
         $cursor->getRemainingText()->willReturn('remainingText');

spec/Token/TokenStreamSpec.php

@@ -121,4 +121,22 @@ public function it_casts_to_a_string(TokenInterface $token1, TokenInterface $tok
 
         $this->__toString()->shouldReturn("SomethingToken (1)\nSomethingToken (2)");
     }
+
+    public function it_only_includes_unconsumed_tokens_after_consuming(TokenInterface $token1, TokenInterface $token2, TokenInterface $token3): void
+    {
+        $token2->__toString()->willReturn('token2');
+        $token3->__toString()->willReturn('token3');
+
+        $this->add($token1);
+        $this->add($token2);
+        $this->add($token3);
+
+        $this->consumeToken()->shouldReturn($token1);
+
+        $this->shouldHaveCount(2);
+        $this->getCurrentToken()->shouldReturn($token2);
+        $this->lookAhead()->shouldReturn($token3);
+        $this->lookAhead(2)->shouldReturn(null);
+        $this->__toString()->shouldReturn("token2\ntoken3");
+    }
 }

src/Finder/RegexFailedException.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+namespace chrisjenkinson\StructuredDocumentParser\Finder;
+
+use RuntimeException;
+use Throwable;
+
+class RegexFailedException extends RuntimeException
+{
+    public function __construct(
+        private readonly string $pattern,
+        string $error,
+        int $code = 0,
+        ?Throwable $previous = null
+    ) {
+        parent::__construct(sprintf('Regex %s failed: %s', $pattern, $error), $code, $previous);
+    }
+
+    public function getPattern(): string
+    {
+        return $this->pattern;
+    }
+}

src/Finder/RegexFinder.php

@@ -23,7 +23,13 @@ public function __construct(string $pattern)
 
     public function find(string $text): bool
     {
-        if (preg_match($this->pattern, $text, $matches)) {
+        $result = preg_match($this->pattern, $text, $matches);
+
+        if (false === $result) {
+            throw new RegexFailedException($this->pattern, preg_last_error_msg());
+        }
+
+        if (1 === $result) {
             $this->matches = $matches;
 
             return true;

src/Lexer/Cursor.php

@@ -21,6 +21,11 @@ class Cursor
      */
     private $text;
 
+    /**
+     * @var int
+     */
+    private $byteOffset = 0;
+
     public function __construct(string $text)
     {
         $this->text            = $text;
@@ -34,7 +39,7 @@ public function getRemainingText(): string
             return '';
         }
 
-        return mb_substr($this->text, $this->currentPosition);
+        return substr($this->text, $this->byteOffset);
     }
 
     public function isEndOfText(): bool
@@ -44,7 +49,11 @@ public function isEndOfText(): bool
 
     public function advance(int $length): void
     {
+        // A UTF-8 character is at most 4 bytes, so this slice always holds $length characters.
+        $consumed = mb_substr(substr($this->text, $this->byteOffset, $length * 4), 0, $length);
+
         $this->currentPosition += $length;
+        $this->byteOffset += strlen($consumed);
     }
 
     public function getCurrentPosition(): int

src/Matcher/MatcherInterface.php

@@ -6,6 +6,13 @@
 
 interface MatcherInterface
 {
+    /**
+     * Matches against the start of $text, returning null if it does not match.
+     *
+     * The returned MatchedText must have an "all" key holding the exact text consumed
+     * from the start of $text, which the lexer uses to advance. Any other keys are
+     * made available on the resulting token.
+     */
     public function match(string $text): ?MatchedText;
 
     /**

src/Matcher/SimpleTextMatcher.php

@@ -15,7 +15,7 @@ class SimpleTextMatcher extends AbstractMatcher
 
     public function __construct()
     {
-        $pattern = '/(?<all>.+)/Ax';
+        $pattern = '/(?<all>.+)/As';
 
         $this->finder = new RegexFinder($pattern);
     }

src/State/AbstractState.php

@@ -53,6 +53,23 @@ private function getTokenType(string $matcherName): string
         return $matcherName;
     }
 
+    private function guardAgainstInvalidMatchedText(string $matcherName, MatchedText $matchedText, string $text): void
+    {
+        $matches = $matchedText->getAll();
+
+        if (!array_key_exists('all', $matches)) {
+            throw new InvalidMatchedTextException($matcherName, 'it has no "all" key');
+        }
+
+        if (!is_string($matches['all'])) {
+            throw new InvalidMatchedTextException($matcherName, 'its "all" value is not a string');
+        }
+
+        if (!str_starts_with($text, $matches['all'])) {
+            throw new InvalidMatchedTextException($matcherName, 'its "all" value is not at the start of the text');
+        }
+    }
+
     public function guardAgainstWrongNumberOfMatches(array $matchedText, string $remainingText, array $calledMatchers, int $currentPosition): void
     {
         if (1 < count($matchedText)) {
@@ -75,6 +92,8 @@ public function runMatchers(string $text): array
             $callback = $matcherAndCallback['callback'];
 
             if ($matches = $matcher->match($text)) {
+                $this->guardAgainstInvalidMatchedText($matcher->getName(), $matches, $text);
+
                 $matchedTokens[]  = $matches;
                 $calledMatchers[] = $matcher->getName();
                 $callbacks[]      = $callback;

src/State/InvalidMatchedTextException.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+namespace chrisjenkinson\StructuredDocumentParser\State;
+
+use LogicException;
+use Throwable;
+
+class InvalidMatchedTextException extends LogicException
+{
+    public function __construct(
+        private readonly string $matcherName,
+        string $reason,
+        int $code = 0,
+        ?Throwable $previous = null
+    ) {
+        parent::__construct(sprintf('Matcher %s returned invalid matched text: %s', $matcherName, $reason), $code, $previous);
+    }
+
+    public function getMatcherName(): string
+    {
+        return $this->matcherName;
+    }
+}