From 8a84334b7208c4f62fa13195d41b6ffd5612346d Mon Sep 17 00:00:00 2001 From: Chris Jenkinson Date: Fri, 25 Sep 2026 10:39:41 +0000 Subject: [PATCH 1/3] Add Lexer integration tests covering end-to-end tokenising scenarios --- tests/Lexer/LexerTest.php | 137 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 tests/Lexer/LexerTest.php diff --git a/tests/Lexer/LexerTest.php b/tests/Lexer/LexerTest.php new file mode 100644 index 0000000..0806060 --- /dev/null +++ b/tests/Lexer/LexerTest.php @@ -0,0 +1,137 @@ +registerMatcher(new RegexMatcher('Letter', '/(?a)/A')); + $origState->registerMatcher(new RegexMatcher('Lookahead', '/(?)(?=b)/A'), static function (Lexer $lexer) use ($newState): void { + $lexer->pushState($newState); + }); + $newState->registerMatcher(new RegexMatcher('Rest', '/(?b+)/A')); + + Assert::assertSame("Letter (a)\nLookahead ()\nRest (bb)", (string) (new Lexer($origState))->tokenise('abb')); + } + + public function testItThrowsIfAZeroLengthMatchDoesNotSwitchState(): void + { + $state = new InitialState(); + $state->registerMatcher(new RegexMatcher('Empty', '/(?)/A')); + + $this->expectException(ZeroLengthTokenLoopException::class); + + (new Lexer($state))->tokenise('a'); + } + + public function testItThrowsIfZeroLengthMatchesReturnToAStateWithoutAdvancing(): void + { + $firstState = new InitialState(); + $secondState = new InitialState(); + + $firstState->registerMatcher(new RegexMatcher('ToSecond', '/(?)/A'), static function (Lexer $lexer) use ($secondState): void { + $lexer->pushState($secondState); + }); + $secondState->registerMatcher(new RegexMatcher('ToFirst', '/(?)/A'), static function (Lexer $lexer) use ($firstState): void { + $lexer->pushState($firstState); + }); + + $this->expectException(ZeroLengthTokenLoopException::class); + + (new Lexer($firstState))->tokenise('a'); + } + + public function testItGivesTheSameTokensWhenTokenisingTheSameTextTwice(): void + { + $origState = new InitialState(); + $newState = new InitialState(); + + $origState->registerMatcher(new RegexMatcher('Letter', '/(?a)/A'), static function (Lexer $lexer) use ($newState): void { + $lexer->pushState($newState); + }); + $newState->registerMatcher(new RegexMatcher('Other', '/(?a)/A')); + + $lexer = new Lexer($origState); + + Assert::assertSame('Letter (a)', (string) $lexer->tokenise('a')); + Assert::assertSame('Letter (a)', (string) $lexer->tokenise('a')); + } + + public function testItGivesEachTokenItsPosition(): void + { + $tokens = (new Lexer($this->lineState()))->tokenise("ab\ncd"); + + Assert::assertEquals(new TokenPosition(1, 1), $tokens->consumeToken()->getPosition()); + Assert::assertEquals(new TokenPosition(2, 1), $tokens->consumeToken()->getPosition()); + } + + public function testItGivesPositionsRelativeToAGivenStartPosition(): void + { + $tokens = (new Lexer($this->lineState()))->tokenise("ab\ncd", new TokenPosition(10, 5)); + + Assert::assertEquals(new TokenPosition(10, 5), $tokens->consumeToken()->getPosition()); + Assert::assertEquals(new TokenPosition(11, 1), $tokens->consumeToken()->getPosition()); + } + + public function testItReportsErrorsRelativeToAGivenStartPosition(): void + { + try { + (new Lexer(new InitialState()))->tokenise('ab', new TokenPosition(10, 5)); + } catch (NoTokenFoundException $exception) { + Assert::assertEquals(new TokenPosition(10, 5), $exception->getPosition()); + + return; + } + + Assert::fail('Expected a NoTokenFoundException.'); + } + + private function lineState(): InitialState + { + $state = new InitialState(); + $state->registerMatcher(new RegexMatcher('Line', '/(?[^\n]*\n|[^\n]+)/A')); + + return $state; + } +} + +class RegexMatcher extends AbstractMatcher +{ + private readonly RegexFinder $finder; + + public function __construct(private readonly string $type, string $pattern) + { + $this->finder = new RegexFinder($pattern); + } + + public function match(string $text): ?MatchedText + { + if ($this->finder->find($text)) { + return new MatchedText($this->finder->getMatches(['all'])); + } + + return null; + } + + public function getName(): string + { + return $this->type . 'Matcher'; + } +} From b3d58653cfe9c5a100fd376bba75007fcd2f7711 Mon Sep 17 00:00:00 2001 From: Chris Jenkinson Date: Fri, 25 Sep 2026 10:40:49 +0000 Subject: [PATCH 2/3] Rewrite LexerSpec with state and token doubles instead of real states and matchers --- spec/Lexer/LexerSpec.php | 238 +++++++++++++-------------------------- 1 file changed, 80 insertions(+), 158 deletions(-) diff --git a/spec/Lexer/LexerSpec.php b/spec/Lexer/LexerSpec.php index 3bf3b9d..3e26a1d 100644 --- a/spec/Lexer/LexerSpec.php +++ b/spec/Lexer/LexerSpec.php @@ -4,86 +4,44 @@ namespace spec\chrisjenkinson\StructuredDocumentParser\Lexer; -use chrisjenkinson\StructuredDocumentParser\Finder\RegexFinder; -use chrisjenkinson\StructuredDocumentParser\Lexer\Lexer; +use chrisjenkinson\StructuredDocumentParser\Lexer\Cursor; use chrisjenkinson\StructuredDocumentParser\Lexer\NoPreviousStateException; use chrisjenkinson\StructuredDocumentParser\Lexer\ZeroLengthTokenLoopException; -use chrisjenkinson\StructuredDocumentParser\Matcher\AbstractMatcher; -use chrisjenkinson\StructuredDocumentParser\Matcher\MatchedText; -use chrisjenkinson\StructuredDocumentParser\Matcher\SimpleTextMatcher; -use chrisjenkinson\StructuredDocumentParser\State\InitialState; -use chrisjenkinson\StructuredDocumentParser\State\NoTokenFoundException; use chrisjenkinson\StructuredDocumentParser\State\StateInterface; +use chrisjenkinson\StructuredDocumentParser\Token\TokenInterface; use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition; -use chrisjenkinson\StructuredDocumentParser\Token\TokenStream; -use PhpSpec\Exception\Example\FailureException; use PhpSpec\ObjectBehavior; +use Prophecy\Argument; +use RuntimeException; class LexerSpec extends ObjectBehavior { public function let(StateInterface $state): void { - $this->beConstructedWith($state); - } - - public function it_has_a_state(): void - { - $this->getState()->shouldBeAnInstanceOf(StateInterface::class); - } - - public function it_can_tokenise(): void - { - $state = new InitialState(); - $state->registerMatcher(new SimpleTextMatcher()); + $state->getName()->willReturn('FirstState'); $this->beConstructedWith($state); - - $this->tokenise('1234')->shouldReturnAnInstanceOf(TokenStream::class); } - public function it_can_switch_state(): void + public function it_has_a_state(StateInterface $state): void { - $origState = new InitialState(); - $origState->registerMatcher(new SimpleTextMatcher()); - - $newState = new InitialState(); - $newState->registerMatcher(new SimpleTextMatcher()); - - $this->beConstructedWith($origState); - - $this->getState()->shouldReturn($origState); - - $this->pushState($newState); - - $this->getState()->shouldReturn($newState); + $this->getState()->shouldReturn($state); } - public function it_records_previous_states(): void + public function it_pushes_a_state_and_records_the_previous_one(StateInterface $state, StateInterface $newState): void { - $origState = new InitialState(); - $origState->registerMatcher(new SimpleTextMatcher()); - - $newState = new InitialState(); - $newState->registerMatcher(new SimpleTextMatcher()); - - $this->beConstructedWith($origState); - $this->pushState($newState); - $this->getLastState()->shouldReturn($origState); + $this->getState()->shouldReturn($newState); + $this->getLastState()->shouldReturn($state); } - public function it_keeps_set_state_as_an_alias_for_push_state(): void + public function it_keeps_set_state_as_an_alias_for_push_state(StateInterface $state, StateInterface $newState): void { - $origState = new InitialState(); - $newState = new InitialState(); - - $this->beConstructedWith($origState); - $this->setState($newState); $this->getState()->shouldReturn($newState); - $this->getLastState()->shouldReturn($origState); + $this->getLastState()->shouldReturn($state); } public function it_throws_if_there_is_no_previous_state(): void @@ -91,25 +49,19 @@ public function it_throws_if_there_is_no_previous_state(): void $this->shouldThrow(NoPreviousStateException::class)->during('getLastState'); } - public function it_pops_back_to_the_previous_state(): void + public function it_pops_back_to_the_previous_state(StateInterface $state, StateInterface $secondState, StateInterface $thirdState): void { - $firstState = new InitialState(); - $secondState = new InitialState(); - $thirdState = new InitialState(); - - $this->beConstructedWith($firstState); - $this->pushState($secondState); $this->pushState($thirdState); $this->popState(); $this->getState()->shouldReturn($secondState); - $this->getLastState()->shouldReturn($firstState); + $this->getLastState()->shouldReturn($state); $this->popState(); - $this->getState()->shouldReturn($firstState); + $this->getState()->shouldReturn($state); $this->shouldThrow(NoPreviousStateException::class)->during('getLastState'); } @@ -118,148 +70,118 @@ public function it_throws_when_popping_without_a_previous_state(): void $this->shouldThrow(NoPreviousStateException::class)->during('popState'); } - public function it_gives_the_same_tokens_when_tokenising_the_same_text_twice(): void + public function it_asks_the_state_for_tokens_until_the_text_is_consumed(StateInterface $state, TokenInterface $first, TokenInterface $second): void { - $origState = new InitialState(); - $newState = new InitialState(); + $first->getText()->willReturn('ab'); + $second->getText()->willReturn('c'); - $origState->registerMatcher(new LexerSpecRegexMatcher('Letter', '/(?a)/A'), static function (Lexer $lexer) use ($newState): void { - $lexer->pushState($newState); - }); - $newState->registerMatcher(new LexerSpecRegexMatcher('Other', '/(?a)/A')); + $state->findMatchingToken($this, $this->cursorAt('abc'))->willReturn($first); + $state->findMatchingToken($this, $this->cursorAt('c'))->willReturn($second); - $this->beConstructedWith($origState); + $tokens = $this->tokenise('abc'); - $this->tokenise('a')->__toString()->shouldReturn('Letter (a)'); - $this->tokenise('a')->__toString()->shouldReturn('Letter (a)'); + $tokens->consumeToken()->shouldReturn($first); + $tokens->consumeToken()->shouldReturn($second); + $tokens->shouldHaveCount(0); } - public function it_restores_the_initial_state_after_tokenising(): void + public function it_starts_the_cursor_at_line_one_column_one_by_default(StateInterface $state, TokenInterface $token): void { - $origState = new InitialState(); - $newState = new InitialState(); - - $origState->registerMatcher(new LexerSpecRegexMatcher('Letter', '/(?a)/A'), static function (Lexer $lexer) use ($newState): void { - $lexer->pushState($newState); - }); + $token->getText()->willReturn('a'); - $this->beConstructedWith($origState); + $state->findMatchingToken($this, Argument::that(static fn (Cursor $cursor): bool => 1 === $cursor->getLine() && 1 === $cursor->getColumn())) + ->shouldBeCalled() + ->willReturn($token); - $this->shouldThrow(NoTokenFoundException::class)->during('tokenise', ['ab']); - - $this->getState()->shouldReturn($origState); - $this->shouldThrow(NoPreviousStateException::class)->during('getLastState'); + $this->tokenise('a'); } - public function it_gives_each_token_its_position(): void + public function it_starts_the_cursor_at_a_given_position(StateInterface $state, TokenInterface $token): void { - $state = new InitialState(); - $state->registerMatcher(new LexerSpecRegexMatcher('Line', '/(?[^\n]*\n|[^\n]+)/A')); - - $this->beConstructedWith($state); + $token->getText()->willReturn('a'); - $tokens = $this->tokenise("ab\ncd"); + $state->findMatchingToken($this, Argument::that(static fn (Cursor $cursor): bool => 10 === $cursor->getLine() && 5 === $cursor->getColumn())) + ->shouldBeCalled() + ->willReturn($token); - $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(1, 1)); - $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(2, 1)); + $this->tokenise('a', new TokenPosition(10, 5)); } - public function it_can_start_from_a_given_position(): void + public function it_allows_a_zero_length_token_that_switches_state(StateInterface $state, StateInterface $newState, TokenInterface $lookahead, TokenInterface $letter): void { - $state = new InitialState(); - $state->registerMatcher(new LexerSpecRegexMatcher('Line', '/(?[^\n]*\n|[^\n]+)/A')); + $lookahead->getText()->willReturn(''); + $letter->getText()->willReturn('a'); - $this->beConstructedWith($state); + $state->findMatchingToken($this, Argument::type(Cursor::class))->will(static function (array $arguments) use ($newState, $lookahead): TokenInterface { + $arguments[0]->pushState($newState->getWrappedObject()); - $tokens = $this->tokenise("ab\ncd", new TokenPosition(10, 5)); + return $lookahead->getWrappedObject(); + }); + $newState->findMatchingToken($this, Argument::type(Cursor::class))->willReturn($letter); - $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(10, 5)); - $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(11, 1)); + $this->tokenise('a')->shouldHaveCount(2); } - public function it_reports_errors_relative_to_the_given_position(): void + public function it_throws_if_a_zero_length_token_does_not_switch_state(StateInterface $state, TokenInterface $token): void { - $state = new InitialState(); + $token->getText()->willReturn(''); - $this->beConstructedWith($state); + $state->findMatchingToken($this, Argument::type(Cursor::class))->willReturn($token); - try { - $this->getWrappedObject()->tokenise('ab', new TokenPosition(10, 5)); - } catch (NoTokenFoundException $exception) { - if (10 !== $exception->getPosition()->getLine() || 5 !== $exception->getPosition()->getColumn()) { - throw new FailureException('Expected the error at line 10, column 5.'); - } - - return; - } - - throw new FailureException('Expected a NoTokenFoundException.'); + $this->shouldThrow(new ZeroLengthTokenLoopException('FirstState', 0))->during('tokenise', ['a']); } - public function it_switches_state_on_a_zero_length_lookahead_match(): void + public function it_throws_if_zero_length_tokens_return_to_a_state_without_advancing(StateInterface $state, StateInterface $secondState, TokenInterface $token): void { - $origState = new InitialState(); - $newState = new InitialState(); + $token->getText()->willReturn(''); + + $state->findMatchingToken($this, Argument::type(Cursor::class))->will(static function (array $arguments) use ($secondState, $token): TokenInterface { + $arguments[0]->pushState($secondState->getWrappedObject()); - $origState->registerMatcher(new LexerSpecRegexMatcher('Letter', '/(?a)/A')); - $origState->registerMatcher(new LexerSpecRegexMatcher('Lookahead', '/(?)(?=b)/A'), static function (Lexer $lexer) use ($newState): void { - $lexer->pushState($newState); + return $token->getWrappedObject(); }); - $newState->registerMatcher(new LexerSpecRegexMatcher('Rest', '/(?b+)/A')); + $secondState->findMatchingToken($this, Argument::type(Cursor::class))->will(static function (array $arguments) use ($state, $token): TokenInterface { + $arguments[0]->pushState($state->getWrappedObject()); - $this->beConstructedWith($origState); + return $token->getWrappedObject(); + }); - $this->tokenise('abb')->__toString()->shouldReturn("Letter (a)\nLookahead ()\nRest (bb)"); + $this->shouldThrow(new ZeroLengthTokenLoopException('FirstState', 0))->during('tokenise', ['a']); } - public function it_throws_if_a_zero_length_match_does_not_switch_state(): void + public function it_starts_each_call_from_the_initial_state(StateInterface $state, StateInterface $newState, TokenInterface $token): void { - $state = new InitialState(); - $state->registerMatcher(new LexerSpecRegexMatcher('Empty', '/(?)/A')); + $token->getText()->willReturn('a'); - $this->beConstructedWith($state); + $state->findMatchingToken($this, Argument::type(Cursor::class))->will(static function (array $arguments) use ($newState, $token): TokenInterface { + $arguments[0]->pushState($newState->getWrappedObject()); - $this->shouldThrow(new ZeroLengthTokenLoopException('InitialState', 0))->during('tokenise', ['a']); - } - - public function it_throws_if_zero_length_matches_return_to_a_state_without_advancing(): void - { - $firstState = new InitialState(); - $secondState = new InitialState(); - - $firstState->registerMatcher(new LexerSpecRegexMatcher('ToSecond', '/(?)/A'), static function (Lexer $lexer) use ($secondState): void { - $lexer->pushState($secondState); - }); - $secondState->registerMatcher(new LexerSpecRegexMatcher('ToFirst', '/(?)/A'), static function (Lexer $lexer) use ($firstState): void { - $lexer->pushState($firstState); + return $token->getWrappedObject(); }); - $this->beConstructedWith($firstState); + $this->tokenise('a'); + $this->tokenise('a'); - $this->shouldThrow(new ZeroLengthTokenLoopException('InitialState', 0))->during('tokenise', ['a']); + $state->findMatchingToken($this, Argument::type(Cursor::class))->shouldHaveBeenCalledTimes(2); + $newState->findMatchingToken(Argument::cetera())->shouldNotHaveBeenCalled(); } -} - -class LexerSpecRegexMatcher extends AbstractMatcher -{ - private RegexFinder $finder; - public function __construct(private string $type, string $pattern) + public function it_restores_the_initial_state_when_tokenising_fails(StateInterface $state, StateInterface $newState): void { - $this->finder = new RegexFinder($pattern); - } + $state->findMatchingToken($this, Argument::type(Cursor::class))->will(static function (array $arguments) use ($newState): never { + $arguments[0]->pushState($newState->getWrappedObject()); - public function match(string $text): ?MatchedText - { - if ($this->finder->find($text)) { - return new MatchedText($this->finder->getMatches(['all'])); - } + throw new RuntimeException('No token'); + }); + + $this->shouldThrow(RuntimeException::class)->during('tokenise', ['a']); - return null; + $this->getState()->shouldReturn($state); + $this->shouldThrow(NoPreviousStateException::class)->during('getLastState'); } - public function getName(): string + private function cursorAt(string $remainingText): Argument\Token\CallbackToken { - return $this->type . 'Matcher'; + return Argument::that(static fn (Cursor $cursor): bool => $remainingText === $cursor->getRemainingText()); } } From 6a10cee98d8e35f39b8fac0b454de4ef29f745a8 Mon Sep 17 00:00:00 2001 From: Chris Jenkinson Date: Fri, 25 Sep 2026 10:41:35 +0000 Subject: [PATCH 3/3] Use node doubles in SimpleNodeSpec's string export example --- spec/Node/SimpleNodeSpec.php | 43 +++++++++++------------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/spec/Node/SimpleNodeSpec.php b/spec/Node/SimpleNodeSpec.php index 28dcd67..cc61e19 100644 --- a/spec/Node/SimpleNodeSpec.php +++ b/spec/Node/SimpleNodeSpec.php @@ -7,7 +7,6 @@ use chrisjenkinson\StructuredDocumentParser\Node\DuplicateNodeException; use chrisjenkinson\StructuredDocumentParser\Node\NodeInterface; use chrisjenkinson\StructuredDocumentParser\Node\NodeNotFoundException; -use chrisjenkinson\StructuredDocumentParser\Node\SimpleNode; use PhpSpec\ObjectBehavior; use RuntimeException; @@ -75,44 +74,28 @@ public function it_throws_when_replacing_a_node_that_does_not_exist(NodeInterfac $this->shouldThrow(NodeNotFoundException::class)->duringReplaceNode($node); } - public function it_exports_a_tree_as_a_string(): void + public function it_exports_a_tree_as_a_string(NodeInterface $child, NodeInterface $listItem): void { - $child = new SimpleNode(); - $grandchild = new SimpleNode(); - $listItem = new SimpleNode(); - - $grandchild->setAttribute('depth', 2); - $listItem->setAttribute('item', 'first'); - $child->addNode($grandchild); - $child->setAttribute('items', [$listItem]); + $child->getName()->willReturn('ChildNode'); + $child->jsonSerialize()->willReturn(['exported' => 'child']); + $listItem->jsonSerialize()->willReturn(['exported' => 'list item']); $this->setAttribute('something', 'result'); + $this->setAttribute('items', [$listItem]); $this->addNode($child); $this->__toString()->shouldReturn('{ "attributes": { - "something": "result" + "something": "result", + "items": [ + { + "exported": "list item" + } + ] }, "nodes": { - "SimpleNode": { - "attributes": { - "items": [ - { - "attributes": { - "item": "first" - }, - "nodes": [] - } - ] - }, - "nodes": { - "SimpleNode": { - "attributes": { - "depth": 2 - }, - "nodes": [] - } - } + "ChildNode": { + "exported": "child" } } }');