Skip to content

Keep specs to unit tests and move lexer scenarios to PHPUnit #161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chrisjenkinson merged 3 commits into from
Sep 25, 2026
Merged

Keep specs to unit tests and move lexer scenarios to PHPUnit #161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8a84334
Add Lexer integration tests covering end-to-end tokenising scenarios
chrisjenkinson Sep 25, 2026
b3d5865
Rewrite LexerSpec with state and token doubles instead of real states…
chrisjenkinson Sep 25, 2026
6a10cee
Use node doubles in SimpleNodeSpec's string export example
chrisjenkinson Sep 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
238 changes: 80 additions & 158 deletions spec/Lexer/LexerSpec.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,112 +4,64 @@

namespace spec\chrisjenkinson\StructuredDocumentParser\Lexer;

use chrisjenkinson\StructuredDocumentParser\Finder\RegexFinder;
use chrisjenkinson\StructuredDocumentParser\Lexer\Lexer;
use chrisjenkinson\StructuredDocumentParser\Lexer\Cursor;
use chrisjenkinson\StructuredDocumentParser\Lexer\NoPreviousStateException;
use chrisjenkinson\StructuredDocumentParser\Lexer\ZeroLengthTokenLoopException;
use chrisjenkinson\StructuredDocumentParser\Matcher\AbstractMatcher;
use chrisjenkinson\StructuredDocumentParser\Matcher\MatchedText;
use chrisjenkinson\StructuredDocumentParser\Matcher\SimpleTextMatcher;
use chrisjenkinson\StructuredDocumentParser\State\InitialState;
use chrisjenkinson\StructuredDocumentParser\State\NoTokenFoundException;
use chrisjenkinson\StructuredDocumentParser\State\StateInterface;
use chrisjenkinson\StructuredDocumentParser\Token\TokenInterface;
use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition;
use chrisjenkinson\StructuredDocumentParser\Token\TokenStream;
use PhpSpec\Exception\Example\FailureException;
use PhpSpec\ObjectBehavior;
use Prophecy\Argument;
use RuntimeException;

class LexerSpec extends ObjectBehavior
{
public function let(StateInterface $state): void
{
$this->beConstructedWith($state);
}

public function it_has_a_state(): void
{
$this->getState()->shouldBeAnInstanceOf(StateInterface::class);
}

public function it_can_tokenise(): void
{
$state = new InitialState();
$state->registerMatcher(new SimpleTextMatcher());
$state->getName()->willReturn('FirstState');

$this->beConstructedWith($state);

$this->tokenise('1234')->shouldReturnAnInstanceOf(TokenStream::class);
}

public function it_can_switch_state(): void
public function it_has_a_state(StateInterface $state): void
{
$origState = new InitialState();
$origState->registerMatcher(new SimpleTextMatcher());

$newState = new InitialState();
$newState->registerMatcher(new SimpleTextMatcher());

$this->beConstructedWith($origState);

$this->getState()->shouldReturn($origState);

$this->pushState($newState);

$this->getState()->shouldReturn($newState);
$this->getState()->shouldReturn($state);
}

public function it_records_previous_states(): void
public function it_pushes_a_state_and_records_the_previous_one(StateInterface $state, StateInterface $newState): void
{
$origState = new InitialState();
$origState->registerMatcher(new SimpleTextMatcher());

$newState = new InitialState();
$newState->registerMatcher(new SimpleTextMatcher());

$this->beConstructedWith($origState);

$this->pushState($newState);

$this->getLastState()->shouldReturn($origState);
$this->getState()->shouldReturn($newState);
$this->getLastState()->shouldReturn($state);
}

public function it_keeps_set_state_as_an_alias_for_push_state(): void
public function it_keeps_set_state_as_an_alias_for_push_state(StateInterface $state, StateInterface $newState): void
{
$origState = new InitialState();
$newState = new InitialState();

$this->beConstructedWith($origState);

$this->setState($newState);

$this->getState()->shouldReturn($newState);
$this->getLastState()->shouldReturn($origState);
$this->getLastState()->shouldReturn($state);
}

public function it_throws_if_there_is_no_previous_state(): void
{
$this->shouldThrow(NoPreviousStateException::class)->during('getLastState');
}

public function it_pops_back_to_the_previous_state(): void
public function it_pops_back_to_the_previous_state(StateInterface $state, StateInterface $secondState, StateInterface $thirdState): void
{
$firstState = new InitialState();
$secondState = new InitialState();
$thirdState = new InitialState();

$this->beConstructedWith($firstState);

$this->pushState($secondState);
$this->pushState($thirdState);

$this->popState();

$this->getState()->shouldReturn($secondState);
$this->getLastState()->shouldReturn($firstState);
$this->getLastState()->shouldReturn($state);

$this->popState();

$this->getState()->shouldReturn($firstState);
$this->getState()->shouldReturn($state);
$this->shouldThrow(NoPreviousStateException::class)->during('getLastState');
}

Expand All @@ -118,148 +70,118 @@ public function it_throws_when_popping_without_a_previous_state(): void
$this->shouldThrow(NoPreviousStateException::class)->during('popState');
}

public function it_gives_the_same_tokens_when_tokenising_the_same_text_twice(): void
public function it_asks_the_state_for_tokens_until_the_text_is_consumed(StateInterface $state, TokenInterface $first, TokenInterface $second): void
{
$origState = new InitialState();
$newState = new InitialState();
$first->getText()->willReturn('ab');
$second->getText()->willReturn('c');

$origState->registerMatcher(new LexerSpecRegexMatcher('Letter', '/(?<all>a)/A'), static function (Lexer $lexer) use ($newState): void {
$lexer->pushState($newState);
});
$newState->registerMatcher(new LexerSpecRegexMatcher('Other', '/(?<all>a)/A'));
$state->findMatchingToken($this, $this->cursorAt('abc'))->willReturn($first);
$state->findMatchingToken($this, $this->cursorAt('c'))->willReturn($second);

$this->beConstructedWith($origState);
$tokens = $this->tokenise('abc');

$this->tokenise('a')->__toString()->shouldReturn('Letter (a)');
$this->tokenise('a')->__toString()->shouldReturn('Letter (a)');
$tokens->consumeToken()->shouldReturn($first);
$tokens->consumeToken()->shouldReturn($second);
$tokens->shouldHaveCount(0);
}

public function it_restores_the_initial_state_after_tokenising(): void
public function it_starts_the_cursor_at_line_one_column_one_by_default(StateInterface $state, TokenInterface $token): void
{
$origState = new InitialState();
$newState = new InitialState();

$origState->registerMatcher(new LexerSpecRegexMatcher('Letter', '/(?<all>a)/A'), static function (Lexer $lexer) use ($newState): void {
$lexer->pushState($newState);
});
$token->getText()->willReturn('a');

$this->beConstructedWith($origState);
$state->findMatchingToken($this, Argument::that(static fn (Cursor $cursor): bool => 1 === $cursor->getLine() && 1 === $cursor->getColumn()))
->shouldBeCalled()
->willReturn($token);

$this->shouldThrow(NoTokenFoundException::class)->during('tokenise', ['ab']);

$this->getState()->shouldReturn($origState);
$this->shouldThrow(NoPreviousStateException::class)->during('getLastState');
$this->tokenise('a');
}

public function it_gives_each_token_its_position(): void
public function it_starts_the_cursor_at_a_given_position(StateInterface $state, TokenInterface $token): void
{
$state = new InitialState();
$state->registerMatcher(new LexerSpecRegexMatcher('Line', '/(?<all>[^\n]*\n|[^\n]+)/A'));

$this->beConstructedWith($state);
$token->getText()->willReturn('a');

$tokens = $this->tokenise("ab\ncd");
$state->findMatchingToken($this, Argument::that(static fn (Cursor $cursor): bool => 10 === $cursor->getLine() && 5 === $cursor->getColumn()))
->shouldBeCalled()
->willReturn($token);

$tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(1, 1));
$tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(2, 1));
$this->tokenise('a', new TokenPosition(10, 5));
}

public function it_can_start_from_a_given_position(): void
public function it_allows_a_zero_length_token_that_switches_state(StateInterface $state, StateInterface $newState, TokenInterface $lookahead, TokenInterface $letter): void
{
$state = new InitialState();
$state->registerMatcher(new LexerSpecRegexMatcher('Line', '/(?<all>[^\n]*\n|[^\n]+)/A'));
$lookahead->getText()->willReturn('');
$letter->getText()->willReturn('a');

$this->beConstructedWith($state);
$state->findMatchingToken($this, Argument::type(Cursor::class))->will(static function (array $arguments) use ($newState, $lookahead): TokenInterface {
$arguments[0]->pushState($newState->getWrappedObject());

$tokens = $this->tokenise("ab\ncd", new TokenPosition(10, 5));
return $lookahead->getWrappedObject();
});
$newState->findMatchingToken($this, Argument::type(Cursor::class))->willReturn($letter);

$tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(10, 5));
$tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(11, 1));
$this->tokenise('a')->shouldHaveCount(2);
}

public function it_reports_errors_relative_to_the_given_position(): void
public function it_throws_if_a_zero_length_token_does_not_switch_state(StateInterface $state, TokenInterface $token): void
{
$state = new InitialState();
$token->getText()->willReturn('');

$this->beConstructedWith($state);
$state->findMatchingToken($this, Argument::type(Cursor::class))->willReturn($token);

try {
$this->getWrappedObject()->tokenise('ab', new TokenPosition(10, 5));
} catch (NoTokenFoundException $exception) {
if (10 !== $exception->getPosition()->getLine() || 5 !== $exception->getPosition()->getColumn()) {
throw new FailureException('Expected the error at line 10, column 5.');
}

return;
}

throw new FailureException('Expected a NoTokenFoundException.');
$this->shouldThrow(new ZeroLengthTokenLoopException('FirstState', 0))->during('tokenise', ['a']);
}

public function it_switches_state_on_a_zero_length_lookahead_match(): void
public function it_throws_if_zero_length_tokens_return_to_a_state_without_advancing(StateInterface $state, StateInterface $secondState, TokenInterface $token): void
{
$origState = new InitialState();
$newState = new InitialState();
$token->getText()->willReturn('');

$state->findMatchingToken($this, Argument::type(Cursor::class))->will(static function (array $arguments) use ($secondState, $token): TokenInterface {
$arguments[0]->pushState($secondState->getWrappedObject());

$origState->registerMatcher(new LexerSpecRegexMatcher('Letter', '/(?<all>a)/A'));
$origState->registerMatcher(new LexerSpecRegexMatcher('Lookahead', '/(?<all>)(?=b)/A'), static function (Lexer $lexer) use ($newState): void {
$lexer->pushState($newState);
return $token->getWrappedObject();
});
$newState->registerMatcher(new LexerSpecRegexMatcher('Rest', '/(?<all>b+)/A'));
$secondState->findMatchingToken($this, Argument::type(Cursor::class))->will(static function (array $arguments) use ($state, $token): TokenInterface {
$arguments[0]->pushState($state->getWrappedObject());

$this->beConstructedWith($origState);
return $token->getWrappedObject();
});

$this->tokenise('abb')->__toString()->shouldReturn("Letter (a)\nLookahead ()\nRest (bb)");
$this->shouldThrow(new ZeroLengthTokenLoopException('FirstState', 0))->during('tokenise', ['a']);
}

public function it_throws_if_a_zero_length_match_does_not_switch_state(): void
public function it_starts_each_call_from_the_initial_state(StateInterface $state, StateInterface $newState, TokenInterface $token): void
{
$state = new InitialState();
$state->registerMatcher(new LexerSpecRegexMatcher('Empty', '/(?<all>)/A'));
$token->getText()->willReturn('a');

$this->beConstructedWith($state);
$state->findMatchingToken($this, Argument::type(Cursor::class))->will(static function (array $arguments) use ($newState, $token): TokenInterface {
$arguments[0]->pushState($newState->getWrappedObject());

$this->shouldThrow(new ZeroLengthTokenLoopException('InitialState', 0))->during('tokenise', ['a']);
}

public function it_throws_if_zero_length_matches_return_to_a_state_without_advancing(): void
{
$firstState = new InitialState();
$secondState = new InitialState();

$firstState->registerMatcher(new LexerSpecRegexMatcher('ToSecond', '/(?<all>)/A'), static function (Lexer $lexer) use ($secondState): void {
$lexer->pushState($secondState);
});
$secondState->registerMatcher(new LexerSpecRegexMatcher('ToFirst', '/(?<all>)/A'), static function (Lexer $lexer) use ($firstState): void {
$lexer->pushState($firstState);
return $token->getWrappedObject();
});

$this->beConstructedWith($firstState);
$this->tokenise('a');
$this->tokenise('a');

$this->shouldThrow(new ZeroLengthTokenLoopException('InitialState', 0))->during('tokenise', ['a']);
$state->findMatchingToken($this, Argument::type(Cursor::class))->shouldHaveBeenCalledTimes(2);
$newState->findMatchingToken(Argument::cetera())->shouldNotHaveBeenCalled();
}
}

class LexerSpecRegexMatcher extends AbstractMatcher
{
private RegexFinder $finder;

public function __construct(private string $type, string $pattern)
public function it_restores_the_initial_state_when_tokenising_fails(StateInterface $state, StateInterface $newState): void
{
$this->finder = new RegexFinder($pattern);
}
$state->findMatchingToken($this, Argument::type(Cursor::class))->will(static function (array $arguments) use ($newState): never {
$arguments[0]->pushState($newState->getWrappedObject());

public function match(string $text): ?MatchedText
{
if ($this->finder->find($text)) {
return new MatchedText($this->finder->getMatches(['all']));
}
throw new RuntimeException('No token');
});

$this->shouldThrow(RuntimeException::class)->during('tokenise', ['a']);

return null;
$this->getState()->shouldReturn($state);
$this->shouldThrow(NoPreviousStateException::class)->during('getLastState');
}

public function getName(): string
private function cursorAt(string $remainingText): Argument\Token\CallbackToken
{
return $this->type . 'Matcher';
return Argument::that(static fn (Cursor $cursor): bool => $remainingText === $cursor->getRemainingText());
}
}
Loading
Loading