From e9b50c29d2b973a82c8909d53f3fb3dc62c1fb52 Mon Sep 17 00:00:00 2001 From: Chris Jenkinson Date: Fri, 25 Sep 2026 05:36:25 +0000 Subject: [PATCH 1/2] Give every token its line and column position from the cursor --- spec/Lexer/CursorSpec.php | 38 +++++++++++++++++++++++++++++++++ spec/Lexer/LexerSpec.php | 14 ++++++++++++ spec/State/InitialStateSpec.php | 26 +++++++++++++++++++++- spec/Token/TokenSpec.php | 8 +++---- src/Lexer/Cursor.php | 28 ++++++++++++++++++++++++ src/State/AbstractState.php | 7 +++++- src/Token/Token.php | 12 ++++------- 7 files changed, 118 insertions(+), 15 deletions(-) diff --git a/spec/Lexer/CursorSpec.php b/spec/Lexer/CursorSpec.php index b7389ce..4f85dd7 100644 --- a/spec/Lexer/CursorSpec.php +++ b/spec/Lexer/CursorSpec.php @@ -48,6 +48,44 @@ public function it_returns_an_empty_string_when_returning_remaining_text_at_end( $this->getRemainingText()->shouldReturn(''); } + public function it_starts_at_line_one_column_one(): void + { + $this->getLine()->shouldReturn(1); + $this->getColumn()->shouldReturn(1); + } + + public function it_tracks_the_column_within_a_line(): void + { + $this->advance(5); + + $this->getLine()->shouldReturn(1); + $this->getColumn()->shouldReturn(6); + } + + public function it_tracks_the_line_and_column_across_newlines(): void + { + $this->beConstructedWith("ab\ncd\nefg"); + + $this->advance(4); + + $this->getLine()->shouldReturn(2); + $this->getColumn()->shouldReturn(2); + + $this->advance(4); + + $this->getLine()->shouldReturn(3); + $this->getColumn()->shouldReturn(3); + } + + public function it_counts_columns_in_characters(): void + { + $this->beConstructedWith('éé€x'); + + $this->advance(3); + + $this->getColumn()->shouldReturn(4); + } + public function it_returns_the_remaining_text_after_advancing_over_multibyte_characters(): void { $this->beConstructedWith("é€\nxé"); diff --git a/spec/Lexer/LexerSpec.php b/spec/Lexer/LexerSpec.php index da26798..2919c7d 100644 --- a/spec/Lexer/LexerSpec.php +++ b/spec/Lexer/LexerSpec.php @@ -14,6 +14,7 @@ use chrisjenkinson\StructuredDocumentParser\State\InitialState; use chrisjenkinson\StructuredDocumentParser\State\NoTokenFoundException; use chrisjenkinson\StructuredDocumentParser\State\StateInterface; +use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition; use chrisjenkinson\StructuredDocumentParser\Token\TokenStream; use PhpSpec\ObjectBehavior; @@ -136,6 +137,19 @@ public function it_restores_the_initial_state_after_tokenising(): void $this->shouldThrow(NoPreviousStateException::class)->during('getLastState'); } + public function it_gives_each_token_its_position(): void + { + $state = new InitialState(); + $state->registerMatcher(new LexerSpecRegexMatcher('Line', '/(?[^\n]*\n|[^\n]+)/A')); + + $this->beConstructedWith($state); + + $tokens = $this->tokenise("ab\ncd"); + + $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(1, 1)); + $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(2, 1)); + } + public function it_switches_state_on_a_zero_length_lookahead_match(): void { $origState = new InitialState(); diff --git a/spec/State/InitialStateSpec.php b/spec/State/InitialStateSpec.php index 3f26bd1..d6e7d7e 100644 --- a/spec/State/InitialStateSpec.php +++ b/spec/State/InitialStateSpec.php @@ -12,6 +12,7 @@ use chrisjenkinson\StructuredDocumentParser\State\InvalidMatchedTextException; use chrisjenkinson\StructuredDocumentParser\State\NoTokenFoundException; use chrisjenkinson\StructuredDocumentParser\Token\TokenInterface; +use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition; use PhpSpec\ObjectBehavior; class InitialStateSpec extends ObjectBehavior @@ -25,6 +26,8 @@ public function it_throws_an_exception_if_there_is_an_ambiguous_token(Lexer $lex { $cursor->getRemainingText()->willReturn('remainingText'); $cursor->getCurrentPosition()->willReturn(0); + $cursor->getLine()->willReturn(4); + $cursor->getColumn()->willReturn(2); $matcher1->getName()->willReturn('matcher1'); $matcher2->getName()->willReturn('matcher2'); @@ -44,6 +47,8 @@ public function it_throws_an_exception_if_there_is_no_token(Lexer $lexer, Cursor { $cursor->getRemainingText()->willReturn('remainingText'); $cursor->getCurrentPosition()->willReturn(0); + $cursor->getLine()->willReturn(4); + $cursor->getColumn()->willReturn(2); $this->registerMatcher($matcher); @@ -56,6 +61,8 @@ public function it_returns_a_token(Lexer $lexer, Cursor $cursor, MatcherInterfac { $cursor->getRemainingText()->willReturn('remainingText'); $cursor->getCurrentPosition()->willReturn(0); + $cursor->getLine()->willReturn(4); + $cursor->getColumn()->willReturn(2); $this->registerMatcher($matcher); @@ -64,13 +71,18 @@ public function it_returns_a_token(Lexer $lexer, Cursor $cursor, MatcherInterfac $matchedText->getAll()->willReturn(['all' => 'remainingText']); - $this->findMatchingToken($lexer, $cursor)->shouldReturnAnInstanceOf(TokenInterface::class); + $token = $this->findMatchingToken($lexer, $cursor); + + $token->shouldReturnAnInstanceOf(TokenInterface::class); + $token->getPosition()->shouldBeLike(new TokenPosition(4, 2)); } public function it_strips_the_matcher_suffix_from_the_token_type(Lexer $lexer, Cursor $cursor, MatcherInterface $matcher, MatchedText $matchedText): void { $cursor->getRemainingText()->willReturn('remainingText'); $cursor->getCurrentPosition()->willReturn(0); + $cursor->getLine()->willReturn(4); + $cursor->getColumn()->willReturn(2); $this->registerMatcher($matcher); @@ -86,6 +98,8 @@ public function it_uses_the_whole_matcher_name_as_the_token_type_without_a_match { $cursor->getRemainingText()->willReturn('remainingText'); $cursor->getCurrentPosition()->willReturn(0); + $cursor->getLine()->willReturn(4); + $cursor->getColumn()->willReturn(2); $this->registerMatcher($matcher); @@ -101,6 +115,8 @@ public function it_throws_if_the_matched_text_has_no_all_key(Lexer $lexer, Curso { $cursor->getRemainingText()->willReturn('remainingText'); $cursor->getCurrentPosition()->willReturn(0); + $cursor->getLine()->willReturn(4); + $cursor->getColumn()->willReturn(2); $this->registerMatcher($matcher); @@ -116,6 +132,8 @@ public function it_throws_if_the_all_key_is_not_a_string(Lexer $lexer, Cursor $c { $cursor->getRemainingText()->willReturn('remainingText'); $cursor->getCurrentPosition()->willReturn(0); + $cursor->getLine()->willReturn(4); + $cursor->getColumn()->willReturn(2); $this->registerMatcher($matcher); @@ -131,6 +149,8 @@ public function it_throws_if_the_all_value_is_not_at_the_start_of_the_text(Lexer { $cursor->getRemainingText()->willReturn('remainingText'); $cursor->getCurrentPosition()->willReturn(0); + $cursor->getLine()->willReturn(4); + $cursor->getColumn()->willReturn(2); $this->registerMatcher($matcher); @@ -146,6 +166,8 @@ public function it_reports_a_misplaced_match_rather_than_an_ambiguous_token(Lexe { $cursor->getRemainingText()->willReturn('remainingText'); $cursor->getCurrentPosition()->willReturn(0); + $cursor->getLine()->willReturn(4); + $cursor->getColumn()->willReturn(2); $this->registerMatcher($matcher1); $this->registerMatcher($matcher2); @@ -165,6 +187,8 @@ public function it_calls_a_callback(Lexer $lexer, Cursor $cursor, MatcherInterfa { $cursor->getRemainingText()->willReturn('remainingText'); $cursor->getCurrentPosition()->willReturn(0); + $cursor->getLine()->willReturn(4); + $cursor->getColumn()->willReturn(2); $lexer->getState()->shouldBeCalled(); diff --git a/spec/Token/TokenSpec.php b/spec/Token/TokenSpec.php index 7411f2b..edcb99c 100644 --- a/spec/Token/TokenSpec.php +++ b/spec/Token/TokenSpec.php @@ -11,7 +11,7 @@ class TokenSpec extends ObjectBehavior { public function let(): void { - $this->beConstructedWith('Something', ['all' => 'value']); + $this->beConstructedWith('Something', ['all' => 'value'], new TokenPosition(3, 7)); } public function it_has_a_type(): void @@ -39,11 +39,9 @@ public function it_can_tell_if_it_has_a_key(): void $this->hasKey('all')->shouldReturn(true); } - public function it_stores_position_information(TokenPosition $position): void + public function it_has_a_position(): void { - $this->setPosition($position); - - $this->getPosition()->shouldBeAnInstanceOf(TokenPosition::class); + $this->getPosition()->shouldBeLike(new TokenPosition(3, 7)); } public function it_casts_to_a_string(): void diff --git a/src/Lexer/Cursor.php b/src/Lexer/Cursor.php index 4a990fe..2fabb6f 100644 --- a/src/Lexer/Cursor.php +++ b/src/Lexer/Cursor.php @@ -21,6 +21,16 @@ class Cursor */ private $text; + /** + * @var int + */ + private $line = 1; + + /** + * @var int + */ + private $column = 1; + /** * @var int */ @@ -51,11 +61,29 @@ public function advance(int $length): void { // A UTF-8 character is at most 4 bytes, so this slice always holds $length characters. $consumed = mb_substr(substr($this->text, $this->byteOffset, $length * 4), 0, $length); + $newlines = mb_substr_count($consumed, "\n"); + + if (0 < $newlines) { + $this->line += $newlines; + $this->column = mb_strlen($consumed) - mb_strrpos($consumed, "\n"); + } else { + $this->column += mb_strlen($consumed); + } $this->currentPosition += $length; $this->byteOffset += strlen($consumed); } + public function getLine(): int + { + return $this->line; + } + + public function getColumn(): int + { + return $this->column; + } + public function getCurrentPosition(): int { return $this->currentPosition; diff --git a/src/State/AbstractState.php b/src/State/AbstractState.php index 4b289e7..1bf964e 100644 --- a/src/State/AbstractState.php +++ b/src/State/AbstractState.php @@ -10,6 +10,7 @@ use chrisjenkinson\StructuredDocumentParser\Matcher\MatcherInterface; use chrisjenkinson\StructuredDocumentParser\Token\Token; use chrisjenkinson\StructuredDocumentParser\Token\TokenInterface; +use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition; use ReflectionClass; abstract class AbstractState implements StateInterface @@ -41,7 +42,11 @@ public function findMatchingToken(Lexer $lexer, Cursor $cursor): TokenInterface $callback($lexer); } - return new Token($this->getTokenType($matcher), $matchedText->getAll()); + return new Token( + $this->getTokenType($matcher), + $matchedText->getAll(), + new TokenPosition($cursor->getLine(), $cursor->getColumn()) + ); } private function getTokenType(string $matcherName): string diff --git a/src/Token/Token.php b/src/Token/Token.php index 0002909..a3015d7 100644 --- a/src/Token/Token.php +++ b/src/Token/Token.php @@ -29,10 +29,11 @@ class Token implements TokenInterface * @param string $type * @param mixed $value */ - public function __construct(string $type, $value) + public function __construct(string $type, $value, TokenPosition $position) { - $this->type = $type; - $this->value = $value; + $this->type = $type; + $this->value = $value; + $this->position = $position; } public function __toString(): string @@ -53,11 +54,6 @@ public function getPosition(): TokenPosition return $this->position; } - public function setPosition(TokenPosition $position): void - { - $this->position = $position; - } - public function hasKey(string $key): bool { return array_key_exists($key, $this->value); From a15b6e6c0dcf80bbeb6d32d403232bfe103a9fd8 Mon Sep 17 00:00:00 2001 From: Chris Jenkinson Date: Fri, 25 Sep 2026 05:41:44 +0000 Subject: [PATCH 2/2] Show the line, column and a one-line excerpt in token exceptions instead of the whole remaining text --- .../AmbiguousTokenFoundExceptionSpec.php | 18 +++++++++-- spec/State/NoTokenFoundExceptionSpec.php | 29 +++++++++++++++-- src/State/AbstractState.php | 13 ++++---- src/State/AmbiguousTokenFoundException.php | 27 ++++++++++++---- src/State/NoTokenFoundException.php | 21 ++++++++++--- src/State/TextExcerpt.php | 31 +++++++++++++++++++ 6 files changed, 118 insertions(+), 21 deletions(-) create mode 100644 src/State/TextExcerpt.php diff --git a/spec/State/AmbiguousTokenFoundExceptionSpec.php b/spec/State/AmbiguousTokenFoundExceptionSpec.php index 0a9504f..0f6a1ed 100644 --- a/spec/State/AmbiguousTokenFoundExceptionSpec.php +++ b/spec/State/AmbiguousTokenFoundExceptionSpec.php @@ -5,6 +5,7 @@ namespace spec\chrisjenkinson\StructuredDocumentParser\State; use chrisjenkinson\StructuredDocumentParser\Matcher\MatchedText; +use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition; use PhpSpec\ObjectBehavior; use RuntimeException; @@ -16,11 +17,12 @@ public function let(MatchedText $matchedText1, MatchedText $matchedText2): void 'stateName', 'text', ['calledMatcher1', 'calledMatcher2'], - [$matchedText1, $matchedText2] + [$matchedText1, $matchedText2], + new TokenPosition(12, 5) ); - $matchedText1->getAll()->willReturn([]); - $matchedText2->getAll()->willReturn([]); + $matchedText1->getAll()->willReturn(['all' => "## Heading\n"]); + $matchedText2->getAll()->willReturn(['all' => '##']); } public function it_is_an_exception(): void @@ -47,4 +49,14 @@ public function it_has_the_matched_tokens(MatchedText $matchedText1, MatchedText { $this->getMatchedTokens()->shouldReturn([$matchedText1, $matchedText2]); } + + public function it_has_the_position(): void + { + $this->getPosition()->shouldBeLike(new TokenPosition(12, 5)); + } + + public function it_shows_the_position_and_each_match_in_the_message(): void + { + $this->getMessage()->shouldReturn('Ambiguous token found with state stateName at line 12, column 5: matchers calledMatcher1 ("## Heading"), calledMatcher2 ("##")'); + } } diff --git a/spec/State/NoTokenFoundExceptionSpec.php b/spec/State/NoTokenFoundExceptionSpec.php index 0c1ac9d..3ac6236 100644 --- a/spec/State/NoTokenFoundExceptionSpec.php +++ b/spec/State/NoTokenFoundExceptionSpec.php @@ -4,6 +4,7 @@ namespace spec\chrisjenkinson\StructuredDocumentParser\State; +use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition; use PhpSpec\ObjectBehavior; use RuntimeException; @@ -11,7 +12,7 @@ class NoTokenFoundExceptionSpec extends ObjectBehavior { public function let(): void { - $this->beConstructedWith('stateName', 50, 'remainingText'); + $this->beConstructedWith('stateName', 50, "## Heading\nmore text", new TokenPosition(12, 5)); } public function it_is_an_exception(): void @@ -31,6 +32,30 @@ public function it_has_the_current_position(): void public function it_has_the_remaining_text(): void { - $this->getRemainingText()->shouldReturn('remainingText'); + $this->getRemainingText()->shouldReturn("## Heading\nmore text"); + } + + public function it_has_the_position(): void + { + $this->getPosition()->shouldBeLike(new TokenPosition(12, 5)); + } + + public function it_shows_the_position_and_the_rest_of_the_line_in_the_message(): void + { + $this->getMessage()->shouldReturn('No token found with state stateName at line 12, column 5: "## Heading"'); + } + + public function it_caps_the_excerpt_at_100_characters(): void + { + $this->beConstructedWith('stateName', 50, str_repeat('é', 150), new TokenPosition(12, 5)); + + $this->getMessage()->shouldReturn('No token found with state stateName at line 12, column 5: "'.str_repeat('é', 100).'…"'); + } + + public function it_shows_a_newline_when_the_text_starts_with_one(): void + { + $this->beConstructedWith('stateName', 50, "\nmore text", new TokenPosition(12, 5)); + + $this->getMessage()->shouldReturn('No token found with state stateName at line 12, column 5: "\\n"'); } } diff --git a/src/State/AbstractState.php b/src/State/AbstractState.php index 1bf964e..36eec78 100644 --- a/src/State/AbstractState.php +++ b/src/State/AbstractState.php @@ -27,11 +27,12 @@ public function registerMatcher(MatcherInterface $matcher, ?callable $callback = public function findMatchingToken(Lexer $lexer, Cursor $cursor): TokenInterface { - $text = $cursor->getRemainingText(); + $text = $cursor->getRemainingText(); + $position = new TokenPosition($cursor->getLine(), $cursor->getColumn()); list($matchedText, $calledMatchers, $callbacks) = $this->runMatchers($text); - $this->guardAgainstWrongNumberOfMatches($matchedText, $text, $calledMatchers, $cursor->getCurrentPosition()); + $this->guardAgainstWrongNumberOfMatches($matchedText, $text, $calledMatchers, $cursor->getCurrentPosition(), $position); $matcher = $calledMatchers[0]; /** @var MatchedText $matchedText */ @@ -45,7 +46,7 @@ public function findMatchingToken(Lexer $lexer, Cursor $cursor): TokenInterface return new Token( $this->getTokenType($matcher), $matchedText->getAll(), - new TokenPosition($cursor->getLine(), $cursor->getColumn()) + $position ); } @@ -75,14 +76,14 @@ private function guardAgainstInvalidMatchedText(string $matcherName, MatchedText } } - public function guardAgainstWrongNumberOfMatches(array $matchedText, string $remainingText, array $calledMatchers, int $currentPosition): void + public function guardAgainstWrongNumberOfMatches(array $matchedText, string $remainingText, array $calledMatchers, int $currentPosition, TokenPosition $position): void { if (1 < count($matchedText)) { - throw new AmbiguousTokenFoundException($this->getName(), $remainingText, $calledMatchers, $matchedText); + throw new AmbiguousTokenFoundException($this->getName(), $remainingText, $calledMatchers, $matchedText, $position); } if (1 > count($matchedText)) { - throw new NoTokenFoundException($this->getName(), $currentPosition, $remainingText); + throw new NoTokenFoundException($this->getName(), $currentPosition, $remainingText, $position); } } diff --git a/src/State/AmbiguousTokenFoundException.php b/src/State/AmbiguousTokenFoundException.php index 065bed0..41efab6 100644 --- a/src/State/AmbiguousTokenFoundException.php +++ b/src/State/AmbiguousTokenFoundException.php @@ -5,6 +5,7 @@ namespace chrisjenkinson\StructuredDocumentParser\State; use chrisjenkinson\StructuredDocumentParser\Matcher\MatchedText; +use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition; use RuntimeException; use Throwable; @@ -30,11 +31,17 @@ class AmbiguousTokenFoundException extends RuntimeException */ private $matchedTokens; + /** + * @var TokenPosition + */ + private $position; + public function __construct( string $stateName, string $text, array $calledMatchers, array $matchedTokens, + TokenPosition $position, int $code = 0, ?Throwable $previous = null ) { @@ -42,15 +49,18 @@ public function __construct( $this->text = $text; $this->calledMatchers = $calledMatchers; $this->matchedTokens = $matchedTokens; + $this->position = $position; + + $matches = array_map(function (string $matcherName, MatchedText $matchedText): string { + return sprintf('%s (%s)', $matcherName, TextExcerpt::of($matchedText->getAll()['all'])); + }, $calledMatchers, $matchedTokens); $message = sprintf( - 'Ambiguous token found with state %s in text %s with matchers %s, matches: %s', + 'Ambiguous token found with state %s at line %d, column %d: matchers %s', $stateName, - $text, - implode(', ', $calledMatchers), - var_export(array_map(function (MatchedText $matchedText): array { - return $matchedText->getAll(); - }, $matchedTokens), true) + $position->getLine(), + $position->getColumn(), + implode(', ', $matches) ); parent::__construct($message, $code, $previous); @@ -75,4 +85,9 @@ public function getText(): string { return $this->text; } + + public function getPosition(): TokenPosition + { + return $this->position; + } } diff --git a/src/State/NoTokenFoundException.php b/src/State/NoTokenFoundException.php index 153aab7..46c60f2 100644 --- a/src/State/NoTokenFoundException.php +++ b/src/State/NoTokenFoundException.php @@ -4,6 +4,7 @@ namespace chrisjenkinson\StructuredDocumentParser\State; +use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition; use RuntimeException; use Throwable; @@ -24,17 +25,24 @@ class NoTokenFoundException extends RuntimeException */ private $remainingText; - public function __construct(string $stateName, int $currentPosition, string $remainingText, int $code = 0, ?Throwable $previous = null) + /** + * @var TokenPosition + */ + private $position; + + public function __construct(string $stateName, int $currentPosition, string $remainingText, TokenPosition $position, int $code = 0, ?Throwable $previous = null) { $this->stateName = $stateName; $this->currentPosition = $currentPosition; $this->remainingText = $remainingText; + $this->position = $position; $message = sprintf( - 'No token found with state %s, current position: %d, remaining text: %s', + 'No token found with state %s at line %d, column %d: %s', $stateName, - $currentPosition, - $remainingText + $position->getLine(), + $position->getColumn(), + TextExcerpt::of($remainingText) ); parent::__construct($message, $code, $previous); @@ -54,4 +62,9 @@ public function getRemainingText(): string { return $this->remainingText; } + + public function getPosition(): TokenPosition + { + return $this->position; + } } diff --git a/src/State/TextExcerpt.php b/src/State/TextExcerpt.php new file mode 100644 index 0000000..c70c5ee --- /dev/null +++ b/src/State/TextExcerpt.php @@ -0,0 +1,31 @@ +