Give tokens and token exceptions a line and column

spec/Lexer/CursorSpec.php

@@ -48,6 +48,44 @@ public function it_returns_an_empty_string_when_returning_remaining_text_at_end(
         $this->getRemainingText()->shouldReturn('');
     }
 
+    public function it_starts_at_line_one_column_one(): void
+    {
+        $this->getLine()->shouldReturn(1);
+        $this->getColumn()->shouldReturn(1);
+    }
+
+    public function it_tracks_the_column_within_a_line(): void
+    {
+        $this->advance(5);
+
+        $this->getLine()->shouldReturn(1);
+        $this->getColumn()->shouldReturn(6);
+    }
+
+    public function it_tracks_the_line_and_column_across_newlines(): void
+    {
+        $this->beConstructedWith("ab\ncd\nefg");
+
+        $this->advance(4);
+
+        $this->getLine()->shouldReturn(2);
+        $this->getColumn()->shouldReturn(2);
+
+        $this->advance(4);
+
+        $this->getLine()->shouldReturn(3);
+        $this->getColumn()->shouldReturn(3);
+    }
+
+    public function it_counts_columns_in_characters(): void
+    {
+        $this->beConstructedWith('éé€x');
+
+        $this->advance(3);
+
+        $this->getColumn()->shouldReturn(4);
+    }
+
     public function it_returns_the_remaining_text_after_advancing_over_multibyte_characters(): void
     {
         $this->beConstructedWith("é€\nxé");

spec/Lexer/LexerSpec.php

@@ -14,6 +14,7 @@
 use chrisjenkinson\StructuredDocumentParser\State\InitialState;
 use chrisjenkinson\StructuredDocumentParser\State\NoTokenFoundException;
 use chrisjenkinson\StructuredDocumentParser\State\StateInterface;
+use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition;
 use chrisjenkinson\StructuredDocumentParser\Token\TokenStream;
 use PhpSpec\ObjectBehavior;
 
@@ -136,6 +137,19 @@ public function it_restores_the_initial_state_after_tokenising(): void
         $this->shouldThrow(NoPreviousStateException::class)->during('getLastState');
     }
 
+    public function it_gives_each_token_its_position(): void
+    {
+        $state = new InitialState();
+        $state->registerMatcher(new LexerSpecRegexMatcher('Line', '/(?<all>[^\n]*\n|[^\n]+)/A'));
+
+        $this->beConstructedWith($state);
+
+        $tokens = $this->tokenise("ab\ncd");
+
+        $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(1, 1));
+        $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(2, 1));
+    }
+
     public function it_switches_state_on_a_zero_length_lookahead_match(): void
     {
         $origState = new InitialState();

spec/State/AmbiguousTokenFoundExceptionSpec.php

@@ -5,6 +5,7 @@
 namespace spec\chrisjenkinson\StructuredDocumentParser\State;
 
 use chrisjenkinson\StructuredDocumentParser\Matcher\MatchedText;
+use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition;
 use PhpSpec\ObjectBehavior;
 use RuntimeException;
 
@@ -16,11 +17,12 @@ public function let(MatchedText $matchedText1, MatchedText $matchedText2): void
             'stateName',
             'text',
             ['calledMatcher1', 'calledMatcher2'],
-            [$matchedText1, $matchedText2]
+            [$matchedText1, $matchedText2],
+            new TokenPosition(12, 5)
         );
 
-        $matchedText1->getAll()->willReturn([]);
-        $matchedText2->getAll()->willReturn([]);
+        $matchedText1->getAll()->willReturn(['all' => "## Heading\n"]);
+        $matchedText2->getAll()->willReturn(['all' => '##']);
     }
 
     public function it_is_an_exception(): void
@@ -47,4 +49,14 @@ public function it_has_the_matched_tokens(MatchedText $matchedText1, MatchedText
     {
         $this->getMatchedTokens()->shouldReturn([$matchedText1, $matchedText2]);
     }
+
+    public function it_has_the_position(): void
+    {
+        $this->getPosition()->shouldBeLike(new TokenPosition(12, 5));
+    }
+
+    public function it_shows_the_position_and_each_match_in_the_message(): void
+    {
+        $this->getMessage()->shouldReturn('Ambiguous token found with state stateName at line 12, column 5: matchers calledMatcher1 ("## Heading"), calledMatcher2 ("##")');
+    }
 }

spec/State/InitialStateSpec.php

@@ -12,6 +12,7 @@
 use chrisjenkinson\StructuredDocumentParser\State\InvalidMatchedTextException;
 use chrisjenkinson\StructuredDocumentParser\State\NoTokenFoundException;
 use chrisjenkinson\StructuredDocumentParser\Token\TokenInterface;
+use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition;
 use PhpSpec\ObjectBehavior;
 
 class InitialStateSpec extends ObjectBehavior
@@ -25,6 +26,8 @@ public function it_throws_an_exception_if_there_is_an_ambiguous_token(Lexer $lex
     {
         $cursor->getRemainingText()->willReturn('remainingText');
         $cursor->getCurrentPosition()->willReturn(0);
+        $cursor->getLine()->willReturn(4);
+        $cursor->getColumn()->willReturn(2);
 
         $matcher1->getName()->willReturn('matcher1');
         $matcher2->getName()->willReturn('matcher2');
@@ -44,6 +47,8 @@ public function it_throws_an_exception_if_there_is_no_token(Lexer $lexer, Cursor
     {
         $cursor->getRemainingText()->willReturn('remainingText');
         $cursor->getCurrentPosition()->willReturn(0);
+        $cursor->getLine()->willReturn(4);
+        $cursor->getColumn()->willReturn(2);
 
         $this->registerMatcher($matcher);
 
@@ -56,6 +61,8 @@ public function it_returns_a_token(Lexer $lexer, Cursor $cursor, MatcherInterfac
     {
         $cursor->getRemainingText()->willReturn('remainingText');
         $cursor->getCurrentPosition()->willReturn(0);
+        $cursor->getLine()->willReturn(4);
+        $cursor->getColumn()->willReturn(2);
 
         $this->registerMatcher($matcher);
 
@@ -64,13 +71,18 @@ public function it_returns_a_token(Lexer $lexer, Cursor $cursor, MatcherInterfac
 
         $matchedText->getAll()->willReturn(['all' => 'remainingText']);
 
-        $this->findMatchingToken($lexer, $cursor)->shouldReturnAnInstanceOf(TokenInterface::class);
+        $token = $this->findMatchingToken($lexer, $cursor);
+
+        $token->shouldReturnAnInstanceOf(TokenInterface::class);
+        $token->getPosition()->shouldBeLike(new TokenPosition(4, 2));
     }
 
     public function it_strips_the_matcher_suffix_from_the_token_type(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void
     {
         $cursor->getRemainingText()->willReturn('remainingText');
         $cursor->getCurrentPosition()->willReturn(0);
+        $cursor->getLine()->willReturn(4);
+        $cursor->getColumn()->willReturn(2);
 
         $this->registerMatcher($matcher);
 
@@ -86,6 +98,8 @@ public function it_uses_the_whole_matcher_name_as_the_token_type_without_a_match
     {
         $cursor->getRemainingText()->willReturn('remainingText');
         $cursor->getCurrentPosition()->willReturn(0);
+        $cursor->getLine()->willReturn(4);
+        $cursor->getColumn()->willReturn(2);
 
         $this->registerMatcher($matcher);
 
@@ -101,6 +115,8 @@ public function it_throws_if_the_matched_text_has_no_all_key(Lexer $lexer, Curso
     {
         $cursor->getRemainingText()->willReturn('remainingText');
         $cursor->getCurrentPosition()->willReturn(0);
+        $cursor->getLine()->willReturn(4);
+        $cursor->getColumn()->willReturn(2);
 
         $this->registerMatcher($matcher);
 
@@ -116,6 +132,8 @@ public function it_throws_if_the_all_key_is_not_a_string(Lexer $lexer, Cursor $c
     {
         $cursor->getRemainingText()->willReturn('remainingText');
         $cursor->getCurrentPosition()->willReturn(0);
+        $cursor->getLine()->willReturn(4);
+        $cursor->getColumn()->willReturn(2);
 
         $this->registerMatcher($matcher);
 
@@ -131,6 +149,8 @@ public function it_throws_if_the_all_value_is_not_at_the_start_of_the_text(Lexer
     {
         $cursor->getRemainingText()->willReturn('remainingText');
         $cursor->getCurrentPosition()->willReturn(0);
+        $cursor->getLine()->willReturn(4);
+        $cursor->getColumn()->willReturn(2);
 
         $this->registerMatcher($matcher);
 
@@ -146,6 +166,8 @@ public function it_reports_a_misplaced_match_rather_than_an_ambiguous_token(Lexe
     {
         $cursor->getRemainingText()->willReturn('remainingText');
         $cursor->getCurrentPosition()->willReturn(0);
+        $cursor->getLine()->willReturn(4);
+        $cursor->getColumn()->willReturn(2);
 
         $this->registerMatcher($matcher1);
         $this->registerMatcher($matcher2);
@@ -165,6 +187,8 @@ public function it_calls_a_callback(Lexer $lexer, Cursor $cursor, MatcherInterfa
     {
         $cursor->getRemainingText()->willReturn('remainingText');
         $cursor->getCurrentPosition()->willReturn(0);
+        $cursor->getLine()->willReturn(4);
+        $cursor->getColumn()->willReturn(2);
 
         $lexer->getState()->shouldBeCalled();
 

spec/State/NoTokenFoundExceptionSpec.php

@@ -4,14 +4,15 @@
 
 namespace spec\chrisjenkinson\StructuredDocumentParser\State;
 
+use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition;
 use PhpSpec\ObjectBehavior;
 use RuntimeException;
 
 class NoTokenFoundExceptionSpec extends ObjectBehavior
 {
     public function let(): void
     {
-        $this->beConstructedWith('stateName', 50, 'remainingText');
+        $this->beConstructedWith('stateName', 50, "## Heading\nmore text", new TokenPosition(12, 5));
     }
 
     public function it_is_an_exception(): void
@@ -31,6 +32,30 @@ public function it_has_the_current_position(): void
 
     public function it_has_the_remaining_text(): void
     {
-        $this->getRemainingText()->shouldReturn('remainingText');
+        $this->getRemainingText()->shouldReturn("## Heading\nmore text");
+    }
+
+    public function it_has_the_position(): void
+    {
+        $this->getPosition()->shouldBeLike(new TokenPosition(12, 5));
+    }
+
+    public function it_shows_the_position_and_the_rest_of_the_line_in_the_message(): void
+    {
+        $this->getMessage()->shouldReturn('No token found with state stateName at line 12, column 5: "## Heading"');
+    }
+
+    public function it_caps_the_excerpt_at_100_characters(): void
+    {
+        $this->beConstructedWith('stateName', 50, str_repeat('é', 150), new TokenPosition(12, 5));
+
+        $this->getMessage()->shouldReturn('No token found with state stateName at line 12, column 5: "'.str_repeat('é', 100).'…"');
+    }
+
+    public function it_shows_a_newline_when_the_text_starts_with_one(): void
+    {
+        $this->beConstructedWith('stateName', 50, "\nmore text", new TokenPosition(12, 5));
+
+        $this->getMessage()->shouldReturn('No token found with state stateName at line 12, column 5: "\\n"');
     }
 }

spec/Token/TokenSpec.php

@@ -11,7 +11,7 @@ class TokenSpec extends ObjectBehavior
 {
     public function let(): void
     {
-        $this->beConstructedWith('Something', ['all' => 'value']);
+        $this->beConstructedWith('Something', ['all' => 'value'], new TokenPosition(3, 7));
     }
 
     public function it_has_a_type(): void
@@ -39,11 +39,9 @@ public function it_can_tell_if_it_has_a_key(): void
         $this->hasKey('all')->shouldReturn(true);
     }
 
-    public function it_stores_position_information(TokenPosition $position): void
+    public function it_has_a_position(): void
     {
-        $this->setPosition($position);
-
-        $this->getPosition()->shouldBeAnInstanceOf(TokenPosition::class);
+        $this->getPosition()->shouldBeLike(new TokenPosition(3, 7));
     }
 
     public function it_casts_to_a_string(): void

src/Lexer/Cursor.php

@@ -21,6 +21,16 @@ class Cursor
      */
     private $text;
 
+    /**
+     * @var int
+     */
+    private $line = 1;
+
+    /**
+     * @var int
+     */
+    private $column = 1;
+
     /**
      * @var int
      */
@@ -51,11 +61,29 @@ public function advance(int $length): void
     {
         // A UTF-8 character is at most 4 bytes, so this slice always holds $length characters.
         $consumed = mb_substr(substr($this->text, $this->byteOffset, $length * 4), 0, $length);
+        $newlines = mb_substr_count($consumed, "\n");
+
+        if (0 < $newlines) {
+            $this->line  += $newlines;
+            $this->column = mb_strlen($consumed) - mb_strrpos($consumed, "\n");
+        } else {
+            $this->column += mb_strlen($consumed);
+        }
 
         $this->currentPosition += $length;
         $this->byteOffset += strlen($consumed);
     }
 
+    public function getLine(): int
+    {
+        return $this->line;
+    }
+
+    public function getColumn(): int
+    {
+        return $this->column;
+    }
+
     public function getCurrentPosition(): int
     {
         return $this->currentPosition;

src/State/AbstractState.php

@@ -10,6 +10,7 @@
 use chrisjenkinson\StructuredDocumentParser\Matcher\MatcherInterface;
 use chrisjenkinson\StructuredDocumentParser\Token\Token;
 use chrisjenkinson\StructuredDocumentParser\Token\TokenInterface;
+use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition;
 use ReflectionClass;
 
 abstract class AbstractState implements StateInterface
@@ -26,11 +27,12 @@ public function registerMatcher(MatcherInterface $matcher, ?callable $callback =
 
     public function findMatchingToken(Lexer $lexer, Cursor $cursor): TokenInterface
     {
-        $text = $cursor->getRemainingText();
+        $text     = $cursor->getRemainingText();
+        $position = new TokenPosition($cursor->getLine(), $cursor->getColumn());
 
         list($matchedText, $calledMatchers, $callbacks) = $this->runMatchers($text);
 
-        $this->guardAgainstWrongNumberOfMatches($matchedText, $text, $calledMatchers, $cursor->getCurrentPosition());
+        $this->guardAgainstWrongNumberOfMatches($matchedText, $text, $calledMatchers, $cursor->getCurrentPosition(), $position);
 
         $matcher = $calledMatchers[0];
         /** @var MatchedText $matchedText */
@@ -41,7 +43,11 @@ public function findMatchingToken(Lexer $lexer, Cursor $cursor): TokenInterface
             $callback($lexer);
         }
 
-        return new Token($this->getTokenType($matcher), $matchedText->getAll());
+        return new Token(
+            $this->getTokenType($matcher),
+            $matchedText->getAll(),
+            $position
+        );
     }
 
     private function getTokenType(string $matcherName): string
@@ -70,14 +76,14 @@ private function guardAgainstInvalidMatchedText(string $matcherName, MatchedText
         }
     }
 
-    public function guardAgainstWrongNumberOfMatches(array $matchedText, string $remainingText, array $calledMatchers, int $currentPosition): void
+    public function guardAgainstWrongNumberOfMatches(array $matchedText, string $remainingText, array $calledMatchers, int $currentPosition, TokenPosition $position): void
     {
         if (1 < count($matchedText)) {
-            throw new AmbiguousTokenFoundException($this->getName(), $remainingText, $calledMatchers, $matchedText);
+            throw new AmbiguousTokenFoundException($this->getName(), $remainingText, $calledMatchers, $matchedText, $position);
         }
 
         if (1 > count($matchedText)) {
-            throw new NoTokenFoundException($this->getName(), $currentPosition, $remainingText);
+            throw new NoTokenFoundException($this->getName(), $currentPosition, $remainingText, $position);
         }
     }