diff --git a/spec/Lexer/CursorSpec.php b/spec/Lexer/CursorSpec.php
index 4f85dd7..8d67e8b 100644
--- a/spec/Lexer/CursorSpec.php
+++ b/spec/Lexer/CursorSpec.php
@@ -99,4 +99,21 @@ public function it_returns_the_remaining_text_after_advancing_over_multibyte_cha
 
         $this->isEndOfText()->shouldReturn(true);
     }
+
+    public function it_can_start_at_a_given_line_and_column(): void
+    {
+        $this->beConstructedWith("ab\ncd", 10, 5);
+
+        $this->getLine()->shouldReturn(10);
+        $this->getColumn()->shouldReturn(5);
+
+        $this->advance(1);
+
+        $this->getColumn()->shouldReturn(6);
+
+        $this->advance(3);
+
+        $this->getLine()->shouldReturn(11);
+        $this->getColumn()->shouldReturn(2);
+    }
 }
diff --git a/spec/Lexer/LexerSpec.php b/spec/Lexer/LexerSpec.php
index e573a9d..3bf3b9d 100644
--- a/spec/Lexer/LexerSpec.php
+++ b/spec/Lexer/LexerSpec.php
@@ -16,6 +16,7 @@
 use chrisjenkinson\StructuredDocumentParser\State\StateInterface;
 use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition;
 use chrisjenkinson\StructuredDocumentParser\Token\TokenStream;
+use PhpSpec\Exception\Example\FailureException;
 use PhpSpec\ObjectBehavior;
 
 class LexerSpec extends ObjectBehavior
@@ -163,6 +164,38 @@ public function it_gives_each_token_its_position(): void
         $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(2, 1));
     }
 
+    public function it_can_start_from_a_given_position(): void
+    {
+        $state = new InitialState();
+        $state->registerMatcher(new LexerSpecRegexMatcher('Line', '/(?<all>[^\n]*\n|[^\n]+)/A'));
+
+        $this->beConstructedWith($state);
+
+        $tokens = $this->tokenise("ab\ncd", new TokenPosition(10, 5));
+
+        $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(10, 5));
+        $tokens->consumeToken()->getPosition()->shouldBeLike(new TokenPosition(11, 1));
+    }
+
+    public function it_reports_errors_relative_to_the_given_position(): void
+    {
+        $state = new InitialState();
+
+        $this->beConstructedWith($state);
+
+        try {
+            $this->getWrappedObject()->tokenise('ab', new TokenPosition(10, 5));
+        } catch (NoTokenFoundException $exception) {
+            if (10 !== $exception->getPosition()->getLine() || 5 !== $exception->getPosition()->getColumn()) {
+                throw new FailureException('Expected the error at line 10, column 5.');
+            }
+
+            return;
+        }
+
+        throw new FailureException('Expected a NoTokenFoundException.');
+    }
+
     public function it_switches_state_on_a_zero_length_lookahead_match(): void
     {
         $origState = new InitialState();
diff --git a/src/Lexer/Cursor.php b/src/Lexer/Cursor.php
index 3fcd038..4352501 100644
--- a/src/Lexer/Cursor.php
+++ b/src/Lexer/Cursor.php
@@ -10,14 +10,16 @@ class Cursor
 
     private readonly int $textLength;
 
-    private int $line = 1;
-
-    private int $column = 1;
-
     private int $byteOffset = 0;
 
-    public function __construct(private readonly string $text)
-    {
+    /**
+     * $line and $column give the position of the start of $text, for text taken from a larger document.
+     */
+    public function __construct(
+        private readonly string $text,
+        private int $line = 1,
+        private int $column = 1,
+    ) {
         $this->textLength = mb_strlen($text);
     }
 
diff --git a/src/Lexer/Lexer.php b/src/Lexer/Lexer.php
index 27fa4aa..ab6ca7d 100644
--- a/src/Lexer/Lexer.php
+++ b/src/Lexer/Lexer.php
@@ -5,6 +5,7 @@
 namespace chrisjenkinson\StructuredDocumentParser\Lexer;
 
 use chrisjenkinson\StructuredDocumentParser\State\StateInterface;
+use chrisjenkinson\StructuredDocumentParser\Token\TokenPosition;
 use chrisjenkinson\StructuredDocumentParser\Token\TokenStream;
 
 class Lexer
@@ -21,12 +22,16 @@ public function __construct(private readonly StateInterface $initialState)
         $this->state = $initialState;
     }
 
-    public function tokenise(string $text): TokenStream
+    /**
+     * Pass $start when $text comes from a larger document, such as a paragraph lexed
+     * again for inline syntax, so token positions refer to that document.
+     */
+    public function tokenise(string $text, ?TokenPosition $start = null): TokenStream
     {
         $this->reset();
 
         try {
-            return $this->tokeniseFromInitialState($text);
+            return $this->tokeniseFromInitialState($text, $start ?? new TokenPosition(1, 1));
         } finally {
             $this->reset();
         }
@@ -73,10 +78,10 @@ public function popState(): void
         array_pop($this->previousStates);
     }
 
-    private function tokeniseFromInitialState(string $text): TokenStream
+    private function tokeniseFromInitialState(string $text, TokenPosition $start): TokenStream
     {
         $tokens = new TokenStream();
-        $cursor = new Cursor($text);
+        $cursor = new Cursor($text, $start->getLine(), $start->getColumn());
 
         $statesAtPosition = [];