chore: simplify Comment token emission

Drops the per-comment `src.Slice(...).ToString()` allocation in
`ConsumeLineComment` so the Comment token uses `Value = ""` to match
the existing Whitespace / Continuation pattern — callers that need
the literal text can slice the original input via SourceStart /
SourceLength. Also tightens the dispatch comment in BashLexer's
outer scan loop and removes redundant `Assert.DoesNotContain` checks
in two lexer tests where the existing token-count assertion already
proves no Comment token is present.

Author: Aaronontheweb

SPEC.md

@@ -463,8 +463,10 @@ must handle this.
 - The terminating newline is **not** consumed by the Comment token.
   It survives as a Whitespace token, preserving statement-boundary
   semantics for the parser (see §4).
-- A Comment token's `Value` includes the leading `#` for source
-  fidelity. `SourceStart` / `SourceLength` cover the same range.
+- A Comment token's `Value` is empty (matching `Whitespace` /
+  `Continuation`); `SourceStart` / `SourceLength` identify the slice
+  including the leading `#` so callers that need the literal text can
+  recover it from the original input span.
 - **Effect on parsing**: comment-only input parses to
   `Clauses = []`, `IsUnparseable = false` — mirroring empty-input
   behavior. A comment leading, trailing, or interleaved with a clause

src/ShellSyntaxTree/Internal/Bash/Lexing/BashLexer.cs

@@ -184,15 +184,11 @@ internal static IReadOnlyList<BashToken> Tokenize(string input)
             }
 
             // ---- line comment ----
-            // Reaching this branch means `c` is at a fresh dispatch position
-            // — whitespace, newlines, operators, quotes, and opaque regions
-            // are already handled above — so `#` here is at a "word
-            // boundary" in bash terms and starts a comment to EOL. A `#`
-            // in the interior of a word never reaches this branch because
-            // ReadWord consumes the whole word before the outer loop
-            // resumes. Bash semantics: backslash-escaped `\#` is consumed
-            // by ReadWord's escape handling and never reaches here either.
-            // SPEC §5.
+            // Reaching this branch implies a word boundary (quotes,
+            // operators, and opaque regions are dispatched above), so `#`
+            // here starts a comment to EOL. Mid-word `#` is consumed by
+            // ReadWord, and `\#` by its escape handling — neither reaches
+            // this point. SPEC §5.
             if (c == '#')
             {
                 i = ConsumeLineComment(src, i, tokens);
@@ -411,7 +407,9 @@ private static int ConsumeBacktickSubstitution(
     // Consume `#` through (but not including) the next newline. The
     // terminating newline stays in the stream so the outer loop emits it
     // as a Whitespace token, preserving SPEC §4 clause-boundary
-    // semantics. The Value retains the leading `#` for source fidelity.
+    // semantics. Value is "" to match Whitespace/Continuation — callers
+    // that need the literal text can slice the source via
+    // SourceStart/SourceLength.
     private static int ConsumeLineComment(
         ReadOnlySpan<char> src, int start, List<BashToken> tokens)
     {
@@ -421,14 +419,8 @@ private static int ConsumeLineComment(
             i++;
         }
 
-        var length = i - start;
         tokens.Add(new BashToken(
-            BashTokenKind.Comment,
-            src.Slice(start, length).ToString(),
-            null,
-            start,
-            length,
-            null));
+            BashTokenKind.Comment, "", null, start, i - start, null));
         return i;
     }
 

src/ShellSyntaxTree/Internal/Bash/Lexing/BashTokenKind.cs

@@ -41,8 +41,10 @@ internal enum BashTokenKind
     /// <summary>A bash line comment — <c>#</c> at a word boundary
     /// through end-of-line (the terminating newline is preserved as
     /// a separate <see cref="Whitespace"/> token so statement
-    /// boundaries are unaffected). Emitted for source fidelity; the
-    /// parser drops these in <c>FilterSignificant</c> alongside
+    /// boundaries are unaffected). Emitted for source fidelity with
+    /// empty <see cref="BashToken.Value"/>; <see cref="BashToken.SourceStart"/>
+    /// and <see cref="BashToken.SourceLength"/> identify the slice.
+    /// The parser drops these in <c>FilterSignificant</c> alongside
     /// <see cref="Whitespace"/> and <see cref="Continuation"/>.
     /// SPEC §5.</summary>
     Comment,

tests/ShellSyntaxTree.Tests/Lexing/BashLexerTests.cs

@@ -575,7 +575,6 @@ public void Comment_only_input_lexes_as_single_comment_token()
         var tokens = BashLexer.Tokenize("# just a note");
         var t = Assert.Single(tokens);
         Assert.Equal(BashTokenKind.Comment, t.Kind);
-        Assert.Equal("# just a note", t.Value);
         Assert.Equal(0, t.SourceStart);
         Assert.Equal(13, t.SourceLength);
     }
@@ -586,7 +585,7 @@ public void Leading_comment_followed_by_newline_and_command_emits_three_signific
         // `# fetch\ngit pull` → Comment, Whitespace(\n), Word(git), Whitespace, Word(pull)
         var all = BashLexer.Tokenize("# fetch\ngit pull");
         Assert.Equal(BashTokenKind.Comment, all[0].Kind);
-        Assert.Equal("# fetch", all[0].Value);
+        Assert.Equal(7, all[0].SourceLength);
         Assert.Equal(BashTokenKind.Whitespace, all[1].Kind);
         Assert.Equal(BashTokenKind.Word, all[2].Kind);
         Assert.Equal("git", all[2].Value);
@@ -605,7 +604,8 @@ public void Inline_trailing_comment_does_not_swallow_preceding_word()
         Assert.Equal(BashTokenKind.Word, nonWs[1].Kind);
         Assert.Equal("pull", nonWs[1].Value);
         Assert.Equal(BashTokenKind.Comment, nonWs[2].Kind);
-        Assert.Equal("# update local", nonWs[2].Value);
+        Assert.Equal(11, nonWs[2].SourceStart);
+        Assert.Equal(14, nonWs[2].SourceLength);
     }
 
     [Fact]
@@ -625,15 +625,13 @@ public void Hash_in_middle_of_unquoted_word_is_literal_not_comment()
     [Fact]
     public void Hash_inside_double_quotes_is_literal_not_comment()
     {
+        // The count-equality below would fail if a stray Comment token
+        // appeared, so a separate DoesNotContain isn't needed.
         var tokens = LexNonWs("echo \"hash is #1234\"");
         Assert.Equal(2, tokens.Length);
         Assert.Equal(BashTokenKind.Word, tokens[0].Kind);
         Assert.Equal(BashTokenKind.QuotedString, tokens[1].Kind);
         Assert.Equal("hash is #1234", tokens[1].Value);
-        // No Comment token anywhere.
-        Assert.DoesNotContain(
-            BashLexer.Tokenize("echo \"hash is #1234\""),
-            t => t.Kind == BashTokenKind.Comment);
     }
 
     [Fact]
@@ -645,9 +643,6 @@ public void Hash_inside_single_quotes_is_literal_not_comment()
         Assert.Equal(BashTokenKind.QuotedString, tokens[1].Kind);
         Assert.Equal("use #foo", tokens[1].Value);
         Assert.True(tokens[1].IsSingleQuoted);
-        Assert.DoesNotContain(
-            BashLexer.Tokenize("echo 'use #foo'"),
-            t => t.Kind == BashTokenKind.Comment);
     }
 
     [Fact]
@@ -676,7 +671,8 @@ public void Comment_starts_immediately_after_operator_without_whitespace()
         Assert.Equal(BashTokenKind.Operator, all[2].Kind);
         Assert.Equal("&&", all[2].OperatorText);
         Assert.Equal(BashTokenKind.Comment, all[3].Kind);
-        Assert.Equal("# foo", all[3].Value);
+        Assert.Equal(6, all[3].SourceStart);
+        Assert.Equal(5, all[3].SourceLength);
     }
 
     [Fact]
@@ -686,7 +682,6 @@ public void Comment_at_EOF_without_trailing_newline_terminates_naturally()
         // Word(echo), Whitespace, Word(hi), Whitespace, Comment(# done)
         Assert.Equal(5, tokens.Count);
         Assert.Equal(BashTokenKind.Comment, tokens[4].Kind);
-        Assert.Equal("# done", tokens[4].Value);
         Assert.Equal(14, tokens[4].SourceStart + tokens[4].SourceLength);
     }
 
@@ -697,7 +692,8 @@ public void Comment_does_not_consume_terminating_newline()
         // a statement boundary between `# a` and `cmd`.
         var tokens = BashLexer.Tokenize("# a\ncmd");
         Assert.Equal(BashTokenKind.Comment, tokens[0].Kind);
-        Assert.Equal("# a", tokens[0].Value);
+        Assert.Equal(0, tokens[0].SourceStart);
+        Assert.Equal(3, tokens[0].SourceLength);
         Assert.Equal(BashTokenKind.Whitespace, tokens[1].Kind);
         // The Whitespace token covers the newline.
         Assert.Equal(3, tokens[1].SourceStart);