buildAliasMap(Grammar grammar,
return aliases;
}
+ /**
+ * 0.6.2 — detect LEXER rules eligible for inline expansion. A rule is
+ * eligible iff (after unwrapping acceptable wrappers and a token boundary)
+ * its body is a {@link Expression.Sequence} of single-character
+ * {@link Expression.Literal}s optionally terminated by a single
+ * {@code !literal} negative lookahead, AND the rule is not already aliased,
+ * not a skip-prefix rule, and is currently kind-assigned as LEXER. For each
+ * eligible rule the constituent inline-literal kinds are allocated (so the
+ * lexer actually produces the single-char tokens) and the rule's own kind is
+ * removed from {@code ruleNameToKind} so the parser never matches it as a
+ * fused single token.
+ *
+ * The shape match deliberately requires the trailing {@code !literal}: a
+ * bare {@code literal+} sequence (e.g. {@code RShiftAssign <- '>' '>' '='})
+ * is DFA-compilable as a single token and is left untouched to preserve the
+ * existing compound-assignment behavior.
+ */
+ private static Map detectInlineExpansions(
+ Grammar grammar,
+ RuleClassifier.Classification classification,
+ Map ruleNameToKind,
+ Map ruleNameToAliasKinds,
+ Map inlineLiteralToKind,
+ List kindNames,
+ Set usedNames,
+ int[] nextKindRef,
+ List aliasLiteralsOut) {
+ var result = new LinkedHashMap();
+ for ( var rule : grammar.rules()) {
+ if ( classification.kinds().get(rule.name()) != RuleKind.LEXER) {
+ continue;}
+ if ( !ruleNameToKind.containsKey(rule.name())) {
+ continue;}
+ if ( ruleNameToAliasKinds.containsKey(rule.name())) {
+ continue;}
+ if ( classification.keywordSkip().containsKey(rule.name())) {
+ continue;}
+ var shape = matchInlineExpansionShape(rule.expression());
+ if ( shape.isEmpty()) {
+ continue;}
+ var s = shape.unwrap();
+ var literalKinds = new int[s.literals().size()];
+ for ( int i = 0; i < s.literals().size(); i++) {
+ literalKinds[i] = ensureInlineKind(s.literals().get(i), inlineLiteralToKind, kindNames, usedNames, nextKindRef, aliasLiteralsOut);}
+ var negKinds = new int[s.negLookaheads().size()];
+ for ( int i = 0; i < s.negLookaheads().size(); i++) {
+ negKinds[i] = ensureInlineKind(s.negLookaheads().get(i), inlineLiteralToKind, kindNames, usedNames, nextKindRef, aliasLiteralsOut);}
+ result.put(rule.name(), new InlineExpansion(literalKinds, negKinds));
+ ruleNameToKind.remove(rule.name());
+ }
+ return result;
+ }
+
+ private record ExpansionShape(List literals, List negLookaheads){}
+
+ /**
+ * Match the {@code literal+ (!literal)*} inline-expansion shape. Returns the
+ * positive single-char literals and the trailing negative-lookahead literals
+ * (e.g. {@code RShift <- '>' '>' !'>' !'='} yields positives {@code [>,>]}
+ * and negatives {@code [>,=]}), or {@code Option.none()} when the body
+ * doesn't fit. Every positive element must be a non-empty, case-sensitive
+ * single-character literal; every trailing {@code !literal} must likewise be
+ * a single-char case-sensitive literal, and all negatives must follow all
+ * positives. At least one trailing negative lookahead is required: a bare
+ * literal sequence is DFA-compilable as a single token and left untouched.
+ */
+ private static Option matchInlineExpansionShape(Expression expr) {
+ var unwrapped = unwrapAcceptableWrappers(expr);
+ if ( ! (unwrapped instanceof Expression.Sequence seq)) {
+ return Option.none();}
+ var elements = seq.elements();
+ if ( elements.isEmpty()) {
+ return Option.none();}
+ var positives = new ArrayList();
+ var negLookaheads = new ArrayList();
+ for ( var element : elements) {
+ var el = unwrapAcceptableWrappers(element);
+ if ( el instanceof Expression.Not not) {
+ var notInner = unwrapAcceptableWrappers(not.expression());
+ if ( ! (notInner instanceof Expression.Literal notLit) || !isSingleCharLiteral(notLit)) {
+ return Option.none();}
+ negLookaheads.add(notLit);
+ } else
+ if ( el instanceof Expression.Literal lit && isSingleCharLiteral(lit)) {
+ // A positive literal after a negative lookahead is not the
+ // supported shape (negatives must be strictly trailing).
+ if ( !negLookaheads.isEmpty()) {
+ return Option.none();}
+ positives.add(lit);
+ } else
+ {
+ return Option.none();}
+ }
+ if ( positives.isEmpty() || negLookaheads.isEmpty()) {
+ return Option.none();}
+ return Option.some(new ExpansionShape(positives, negLookaheads));
+ }
+
+ private static boolean isSingleCharLiteral(Expression.Literal lit) {
+ return lit.text().length() == 1 && !lit.caseInsensitive();
+ }
+
/**
* Strip outer wrappers from {@code expr} and, if the result is
* {@code Sequence(actualBody, Not(CharClass))}, return {@code actualBody}.
@@ -636,13 +779,11 @@ private static Nfa buildNfaWithSkips(Grammar grammar,
absorbLiteralFragment(nfa, lit, kind, priorityRef, globalStart);
}
if ( grammar.whitespace().isPresent()) {
- tryAbsorb(nfa,
- "%whitespace",
- grammar.whitespace().unwrap(),
- KIND_WHITESPACE,
- priorityRef,
- globalStart,
- skipped);}
+ absorbWhitespace(nfa,
+ grammar.whitespace().unwrap(),
+ priorityRef,
+ globalStart,
+ skipped);}
for ( var rule : grammar.rules()) {
if ( classification.kinds().get(rule.name()) != RuleKind.LEXER) {
continue;}
@@ -652,6 +793,11 @@ private static Nfa buildNfaWithSkips(Grammar grammar,
// importantly) the DFA build would fail on the !CharClass guard.
if ( assignment.ruleNameToAliasKinds().containsKey(rule.name())) {
continue;}
+ // 0.6.2 — inline-expanded rules (e.g. LShift) have no own token kind;
+ // the parser matches their constituent inline-literal kinds directly.
+ // They must not be compiled as standalone DFA accepts.
+ if ( assignment.inlineExpansions().containsKey(rule.name())) {
+ continue;}
int kind = assignment.ruleNameToKind().get(rule.name());
// Phase B.0 — for skip-prefix rules, compile the body expression only.
// The Not(Reference) head is replaced by post-match keyword resolution
@@ -667,6 +813,141 @@ private static Nfa buildNfaWithSkips(Grammar grammar,
return nfa;
}
+ /**
+ * 0.6.2 — absorb the {@code %whitespace} body with per-alternative trivia
+ * kinds, so that a single coalescing {@code (...)*} wrapper around a Choice
+ * of trivia kinds (whitespace / line comment / block comment / doc variants)
+ * does NOT collapse an entire mixed-trivia run into one {@code KIND_WHITESPACE}
+ * token. Each Choice alternative becomes its own NFA accept fragment whose
+ * kind is decided structurally (by the alternative's leading literal), not by
+ * a post-match prefix sniff of the matched text.
+ *
+ * Recognised body shapes (after unwrapping Group/TokenBoundary/etc.):
+ *
+ * - {@code ZeroOrMore(Choice)} / {@code OneOrMore(Choice)} — the canonical
+ * "fold the whole trivia run" shape. The outer repetition is dropped and
+ * each alternative is absorbed individually; the lexer's own maximal-munch
+ * loop performs the iteration, emitting one token per alternative match
+ * with the correct kind.
+ * - {@code ZeroOrMore(expr)} / {@code OneOrMore(expr)} where {@code expr} is
+ * not a Choice — treated as a 1-alternative choice.
+ * - a bare {@code Choice} or single expression — treated directly.
+ *
+ *
+ * Each absorbed alternative is forced to match at least one character
+ * (a bare whitespace char-class alternative is wrapped as one-or-more) so the
+ * DFA start state never accepts the empty string — the unsplit form therefore
+ * does NOT trigger the empty-match warning of the folded-{@code *} shape.
+ *
+ *
Falls back to the previous whole-body {@link #tryAbsorb} (with kind
+ * {@code KIND_WHITESPACE} and the post-match prefix sniff) only when the body
+ * does not fit any recognised shape — preserving behaviour for exotic
+ * {@code %whitespace} bodies.
+ */
+ private static void absorbWhitespace(Nfa nfa,
+ Expression whitespaceBody,
+ int[] priorityRef,
+ int globalStart,
+ List skipped) {
+ var alternatives = whitespaceAlternatives(whitespaceBody);
+ if ( alternatives.isEmpty()) {
+ tryAbsorb(nfa, "%whitespace", whitespaceBody, KIND_WHITESPACE, priorityRef, globalStart, skipped);
+ return;
+ }
+ int absorbed = 0;
+ for ( var alt : alternatives) {
+ int kind = classifyWhitespaceAlternativeKind(alt);
+ var oneOrMore = ensureNonEmptyWhitespaceAlternative(alt);
+ var result = compileExpression(nfa, oneOrMore, "%whitespace");
+ if ( !result.isSuccess()) {
+ continue;}
+ var fragment = result.unwrap();
+ nfa.addEpsilon(globalStart, fragment.start);
+ nfa.markAccept(fragment.accept, kind, priorityRef[0]);
+ priorityRef[0]++;
+ absorbed++;
+ }
+ // If no alternative compiled (e.g. all used unsupported constructs), fall
+ // back to the legacy whole-body path so whitespace lexing still works.
+ if ( absorbed == 0) {
+ tryAbsorb(nfa, "%whitespace", whitespaceBody, KIND_WHITESPACE, priorityRef, globalStart, skipped);}
+ }
+
+ /**
+ * Return the top-level alternatives of a {@code %whitespace} body for
+ * per-kind absorption, or an empty list when the body does not fit a
+ * recognised shape (signalling the caller to use the legacy whole-body path).
+ * The outer {@code ZeroOrMore}/{@code OneOrMore} wrapper is stripped; the
+ * lexer's maximal-munch loop supplies the repetition.
+ */
+ private static List whitespaceAlternatives(Expression body) {
+ var unwrapped = unwrapAcceptableWrappers(body);
+ Expression inner;
+ if ( unwrapped instanceof Expression.ZeroOrMore zom) {
+ inner = unwrapAcceptableWrappers(zom.expression());} else
+ if ( unwrapped instanceof Expression.OneOrMore oom) {
+ inner = unwrapAcceptableWrappers(oom.expression());} else
+ {
+ inner = unwrapped;}
+ if ( inner instanceof Expression.Choice choice) {
+ return List.copyOf(choice.alternatives());}
+ return List.of(inner);
+ }
+
+ /**
+ * Decide the {@link TokenArray}-equivalent trivia kind for a single
+ * {@code %whitespace} alternative by inspecting its leading literal. Mirrors
+ * the longest-prefix precedence used by the split-rule + post-match-sniff
+ * path: {@code /**} (doc block) before {@code /*} (block); {@code ///} (doc
+ * line) before {@code //} (line). Anything else — including a leading
+ * whitespace char-class — maps to {@link #KIND_WHITESPACE}.
+ */
+ private static int classifyWhitespaceAlternativeKind(Expression alternative) {
+ var prefix = leadingLiteralText(alternative);
+ if ( prefix.startsWith("/**")) {
+ return KIND_DOC_BLOCK_COMMENT;}
+ if ( prefix.startsWith("/*")) {
+ return KIND_BLOCK_COMMENT;}
+ if ( prefix.startsWith("///")) {
+ return KIND_DOC_LINE_COMMENT;}
+ if ( prefix.startsWith("//")) {
+ return KIND_LINE_COMMENT;}
+ return KIND_WHITESPACE;
+ }
+
+ /**
+ * Extract the leading literal text of an alternative: the text of its first
+ * {@link Expression.Literal} element (when the alternative is a Sequence) or
+ * the alternative itself (when it is a bare Literal). Returns the empty
+ * string for any other shape (e.g. a leading char class).
+ */
+ private static String leadingLiteralText(Expression alternative) {
+ var unwrapped = unwrapAcceptableWrappers(alternative);
+ if ( unwrapped instanceof Expression.Literal lit) {
+ return lit.text();}
+ if ( unwrapped instanceof Expression.Sequence seq && !seq.elements().isEmpty()) {
+ var first = unwrapAcceptableWrappers(seq.elements().get(0));
+ if ( first instanceof Expression.Literal lit) {
+ return lit.text();}
+ }
+ return "";
+ }
+
+ /**
+ * Ensure a whitespace alternative matches at least one character so that no
+ * absorbed fragment makes the DFA start state accept the empty string. A
+ * bare char-class alternative (e.g. {@code [ \t\r\n]}) is wrapped as
+ * one-or-more so a maximal whitespace run is still consumed as one token;
+ * all other alternatives (literal-prefixed comment patterns) already match
+ * a non-empty opening delimiter and are returned unchanged.
+ */
+ private static Expression ensureNonEmptyWhitespaceAlternative(Expression alternative) {
+ var unwrapped = unwrapAcceptableWrappers(alternative);
+ if ( unwrapped instanceof Expression.CharClass cc) {
+ return new Expression.OneOrMore(cc.span(), cc);}
+ return alternative;
+ }
+
private static void absorbLiteralFragment(Nfa nfa,
InlineLiteral lit,
int kind,
diff --git a/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/LexerEngine.java b/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/LexerEngine.java
index 4cf6964..258f7a1 100644
--- a/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/LexerEngine.java
+++ b/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/LexerEngine.java
@@ -128,15 +128,26 @@ public TokenArray lex(String input) {
if ( override != null) {
lastAcceptKind = override;}
}
- // Phase A.6 / 0.6.1 — content-based trivia classification. WHITESPACE tokens
- // whose text begins with a comment prefix are reclassified into the matching
- // (regular or doc) line/block-comment kind. Pure whitespace runs never start
- // with '/', so this is a sound prefix check.
- // // → LINE_COMMENT (also /+ that isn't doc; see below)
+ // Phase A.6 / 0.6.1 / 0.6.2 — content-based trivia refinement. Two
+ // entry conditions:
+ // 1. legacy fallback: a coalesced WHITESPACE token whose text begins
+ // with a comment prefix (folded-%whitespace path that did not go
+ // through structural per-alternative absorption);
+ // 2. structural base kind: DfaBuilder.absorbWhitespace already
+ // assigned LINE_COMMENT / BLOCK_COMMENT structurally, but the
+ // grammar's single alternative cannot distinguish the doc variant
+ // (/// vs //, /** vs /*) — only the matched text can.
+ // In both cases the matched span text is inspected to pick the (regular
+ // or doc) line/block-comment kind. Pure whitespace runs never start
+ // with '/', so the WHITESPACE branch is a sound prefix check.
+ // // → LINE_COMMENT
// /// → DOC_LINE_COMMENT (3 or more slashes)
// /* ... */ → BLOCK_COMMENT
// /** ... */ → DOC_BLOCK_COMMENT (NOT the smallest empty block /**/)
- if ( lastAcceptKind == TokenArray.KIND_WHITESPACE && lastAcceptEnd > pos + 1) {
+ if ( (lastAcceptKind == TokenArray.KIND_WHITESPACE
+ || lastAcceptKind == TokenArray.KIND_LINE_COMMENT
+ || lastAcceptKind == TokenArray.KIND_BLOCK_COMMENT)
+ && lastAcceptEnd > pos + 1) {
char c0 = input.charAt(pos);
char c1 = input.charAt(pos + 1);
if ( c0 == '/') {
diff --git a/peglib-core/src/test/java/org/pragmatica/peg/v6/diagnostic/Java25SelfHostDiagTest.java b/peglib-core/src/test/java/org/pragmatica/peg/v6/diagnostic/Java25SelfHostDiagTest.java
index 0146738..605e5b6 100644
--- a/peglib-core/src/test/java/org/pragmatica/peg/v6/diagnostic/Java25SelfHostDiagTest.java
+++ b/peglib-core/src/test/java/org/pragmatica/peg/v6/diagnostic/Java25SelfHostDiagTest.java
@@ -1,6 +1,5 @@
package org.pragmatica.peg.v6.diagnostic;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.pragmatica.peg.v6.PegParser;
import org.pragmatica.peg.v6.cst.CstArray;
@@ -85,7 +84,6 @@ public void dumpDiagnostics() throws Exception {
// cleanly via parseRuleFrom(Shift) / parseRuleFrom(Expr) in isolation.
// Hypothesis: Type / Relational / TypeArgs interaction with '<' literals
// corrupts parser state on backtrack. Deferred to 0.6.2.
- @Disabled("TODO 0.6.2 — shift-in-FieldDecl bug; see comment")
@Test
public void selfHostFixtureParsesCleanly() throws Exception {
String grammar = Files.readString(GRAMMAR);
@@ -122,7 +120,6 @@ public void selfHostFixtureParsesCleanly() throws Exception {
}
}
- @Disabled("TODO 0.6.2 — depends on shift-in-FieldDecl fix")
@Test
public void selfHostFixtureProducesShallowCST() throws Exception {
String grammar = Files.readString(GRAMMAR);
diff --git a/peglib-core/src/test/java/org/pragmatica/peg/v6/generator/ShiftOperatorInlineExpansionTest.java b/peglib-core/src/test/java/org/pragmatica/peg/v6/generator/ShiftOperatorInlineExpansionTest.java
new file mode 100644
index 0000000..7a7570f
--- /dev/null
+++ b/peglib-core/src/test/java/org/pragmatica/peg/v6/generator/ShiftOperatorInlineExpansionTest.java
@@ -0,0 +1,214 @@
+package org.pragmatica.peg.v6.generator;
+
+import org.pragmatica.peg.v6.PegParser;
+import org.pragmatica.peg.v6.cst.CstArray;
+import org.pragmatica.peg.v6.cst.ParseResult;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * 0.6.2 — regression coverage for the shift-operator inline-expansion fix.
+ *
+ * The Java25 grammar splits shift operators into single-character literals so
+ * that nested generics ({@code List>}) still parse:
+ * {@code LShift <- '<' '<' !'='}, {@code RShift <- '>' '>' !'>' !'='},
+ * {@code URShift <- '>' '>' '>' !'='}. These rules classify as LEXER but the DFA
+ * builder cannot compile their negative lookahead, so they are expanded inline by
+ * the parser generator as adjacent multi-token matches against the constituent
+ * {@code <}/{@code >}/{@code =} inline-literal kinds.
+ *
+ * Before the fix every {@code <<}/{@code >>}/{@code >>>} in field- or
+ * local-variable-initializer context produced a cascade of "trailing input not
+ * consumed" diagnostics; nested generics and relational comparisons must keep
+ * parsing cleanly after the fix.
+ */
+class ShiftOperatorInlineExpansionTest {
+ private static final Path GRAMMAR_PATH = Paths.get("src/test/resources/java25.peg");
+
+ private static PegParserWrapper parser;
+
+ /** Thin holder so the heavy generate+compile happens exactly once. */
+ private record PegParserWrapper(org.pragmatica.peg.v6.Parser delegate) {
+ ParseResult parse(String input) {
+ return delegate.parse(input, 200);
+ }
+ }
+
+ @BeforeAll
+ static void setupOnce() throws IOException {
+ var grammarText = Files.readString(GRAMMAR_PATH, StandardCharsets.UTF_8);
+ parser = new PegParserWrapper(PegParser.fromGrammar(grammarText).unwrap());
+ }
+
+ private static int countShiftNodesWithOperator(CstArray cst) {
+ // A Shift node that actually applied a shift operator has more than one
+ // Additive child (Shift <- Additive ((URShift / LShift / RShift) Additive)*).
+ int withOp = 0;
+ for (int i = 0; i < cst.nodeCount(); i++) {
+ if (!"Shift".equals(cst.kindNameAt(i))) {
+ continue;
+ }
+ int additives = 0;
+ for (int c = cst.firstChildAt(i); c >= 0; c = cst.nextSiblingAt(c)) {
+ if ("Additive".equals(cst.kindNameAt(c))) {
+ additives++;
+ }
+ }
+ if (additives > 1) {
+ withOp++;
+ }
+ }
+ return withOp;
+ }
+
+ private static void assertParsesCleanly(String source, int minNodes) {
+ var result = parser.parse(source);
+ assertThat(result.diagnostics())
+ .as("diagnostics for: %s", source)
+ .isEmpty();
+ assertThat(result.cst().nodeCount())
+ .as("CST node count for: %s (guards against empty-match bailout)", source)
+ .isGreaterThanOrEqualTo(minNodes);
+ }
+
+ @Nested
+ class ShiftInExpressions {
+ @Test
+ void leftShift_parsesCleanly_inFieldInitializer() {
+ assertParsesCleanly("class A { int x = 1 << 2; }", 20);
+ assertThat(countShiftNodesWithOperator(parser.parse("class A { int x = 1 << 2; }").cst()))
+ .as("<< should be recognised as a shift operator")
+ .isEqualTo(1);
+ }
+
+ @Test
+ void rightShift_parsesCleanly_inFieldInitializer() {
+ assertParsesCleanly("class A { int x = 1 >> 2; }", 20);
+ assertThat(countShiftNodesWithOperator(parser.parse("class A { int x = 1 >> 2; }").cst()))
+ .isEqualTo(1);
+ }
+
+ @Test
+ void unsignedRightShift_parsesCleanly_inFieldInitializer() {
+ assertParsesCleanly("class A { int x = 1 >>> 2; }", 20);
+ assertThat(countShiftNodesWithOperator(parser.parse("class A { int x = 1 >>> 2; }").cst()))
+ .isEqualTo(1);
+ }
+
+ @Test
+ void leftShift_parsesCleanly_inLocalVariableInitializer() {
+ assertParsesCleanly("class A { void m() { int x = 1 << 2; } }", 25);
+ }
+
+ @Test
+ void rightShift_parsesCleanly_inLocalVariableInitializer() {
+ assertParsesCleanly("class A { void m() { int x = 1 >> 2; } }", 25);
+ }
+
+ @Test
+ void unsignedRightShift_parsesCleanly_inLocalVariableInitializer() {
+ assertParsesCleanly("class A { void m() { int x = 1 >>> 2; } }", 25);
+ }
+
+ @Test
+ void leftShift_parsesCleanly_whenParenthesized() {
+ assertParsesCleanly("class A { int x = (1 << 2); }", 25);
+ }
+
+ @Test
+ void chainedShift_parsesCleanly() {
+ assertParsesCleanly("class A { int x = 1 << 2 >> 3; }", 25);
+ assertThat(countShiftNodesWithOperator(parser.parse("class A { int x = 1 << 2 >> 3; }").cst()))
+ .isEqualTo(1);
+ }
+ }
+
+ @Nested
+ class ShiftAssignment {
+ @Test
+ void leftShiftAssign_parsesCleanly() {
+ assertParsesCleanly("class A { void m() { x <<= 2; } }", 25);
+ }
+
+ @Test
+ void rightShiftAssign_parsesCleanly() {
+ assertParsesCleanly("class A { void m() { x >>= 2; } }", 25);
+ }
+
+ @Test
+ void unsignedRightShiftAssign_parsesCleanly() {
+ assertParsesCleanly("class A { void m() { x >>>= 2; } }", 25);
+ }
+ }
+
+ @Nested
+ class GenericsStillWork {
+ @Test
+ void nestedGenerics_parseWithZeroDiagnostics() {
+ assertParsesCleanly("class A { List> x; }", 15);
+ assertThat(countShiftNodesWithOperator(parser.parse("class A { List> x; }").cst()))
+ .as(">> closing nested generics must NOT be parsed as a shift operator")
+ .isZero();
+ }
+
+ @Test
+ void parameterizedMap_parsesWithZeroDiagnostics() {
+ assertParsesCleanly("class A { java.util.Map> x; }", 20);
+ assertThat(countShiftNodesWithOperator(parser.parse("class A { java.util.Map> x; }").cst()))
+ .isZero();
+ }
+
+ @Test
+ void relationalComparison_parsesWithZeroDiagnostics() {
+ assertParsesCleanly("class A { boolean b = a < b; }", 15);
+ assertThat(countShiftNodesWithOperator(parser.parse("class A { boolean b = a < b; }").cst()))
+ .as("a single '<' is relational, not a shift")
+ .isZero();
+ }
+ }
+
+ @Nested
+ class Adjacency {
+ @Test
+ void spacedLessThans_doNotFuseIntoLeftShift() {
+ // `1 < < 2` has whitespace between the two '<' tokens, so the inline
+ // expansion's adjacency check must refuse to treat it as `<<`. Java
+ // itself rejects this, so the parser should NOT see a shift operator.
+ var result = parser.parse("class A { int x = 1 < < 2; }");
+ assertThat(countShiftNodesWithOperator(result.cst()))
+ .as("`< <` (spaced) must not parse as a `<<` shift")
+ .isZero();
+ }
+ }
+
+ @Nested
+ class LoudGuard {
+ @Test
+ void fromGrammar_fails_whenParserRuleReferencesSkippedLexerRuleWithoutExpansion() {
+ // Tok is LEXER (literals only) with a multi-char negative lookahead,
+ // which the DFA cannot compile and which does not fit the single-char
+ // `literal+ !literal` inline-expansion shape. Start (PARSER) references
+ // it, so the generator must reject rather than emit a dead-kind match.
+ var grammar = """
+ Start <- Tok ';'
+ Tok <- 'ab' !'cd'
+ """;
+ org.pragmatica.peg.v6.PegParser.fromGrammar(grammar)
+ .onSuccess(__ -> org.junit.jupiter.api.Assertions.fail("expected SkippedRuleReferenced failure"))
+ .onFailure(cause -> assertThat(cause.message())
+ .contains("Tok")
+ .contains("skipped"));
+ }
+ }
+}
+
diff --git a/peglib-core/src/test/java/org/pragmatica/peg/v6/lexer/TriviaClassificationTest.java b/peglib-core/src/test/java/org/pragmatica/peg/v6/lexer/TriviaClassificationTest.java
index ec189f6..01b8441 100644
--- a/peglib-core/src/test/java/org/pragmatica/peg/v6/lexer/TriviaClassificationTest.java
+++ b/peglib-core/src/test/java/org/pragmatica/peg/v6/lexer/TriviaClassificationTest.java
@@ -1,9 +1,9 @@
package org.pragmatica.peg.v6.lexer;
import org.pragmatica.peg.grammar.GrammarParser;
+import org.pragmatica.peg.v6.generator.LexerGenerator;
import org.pragmatica.peg.v6.token.TokenArray;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@@ -109,7 +109,6 @@ void lineComment_isClassifiedAsLineComment() {
.isEqualTo(input);
}
- @Disabled("Block-comment alternative inside Choice doesn't route through compileDelimitedBlock; lexer cannot lex `/*...*/` from this grammar shape. Fix in a future task.")
@Test
void blockComment_isClassifiedAsBlockComment() {
var engine = engineFor(GRAMMAR_WITH_COMMENTS);
@@ -127,7 +126,6 @@ void blockComment_isClassifiedAsBlockComment() {
.isEqualTo(input);
}
- @Disabled("Block-comment alternative inside Choice doesn't lex; defer until lexer driver supports per-iteration trivia tokens.")
@Test
void mixedContent_classifiesEachTriviaTokenByItsPrefix() {
var engine = engineFor(GRAMMAR_WITH_COMMENTS);
@@ -326,4 +324,145 @@ void docKinds_areTrivia() {
.as("DOC_BLOCK_COMMENT is trivia")
.isTrue();
}
+
+ // ---- 0.6.2 — unsplit (folded) %whitespace per-kind absorption -------------
+ //
+ // The unsplit form `([ \t\r\n] / '//' [^\n]* / '/*' (!'*/' .)* '*/')*` used to
+ // coalesce an entire mixed-trivia run into ONE KIND_WHITESPACE token. 0.6.2's
+ // DfaBuilder.absorbWhitespace absorbs each Choice alternative at its own trivia
+ // kind so the lexer emits one correctly-classified token per chunk — producing
+ // a token stream byte-identical to the split form.
+
+ /** Unsplit folded form — the shape this task makes work. */
+ private static final String GRAMMAR_UNSPLIT = """
+ Word <- [a-zA-Z]+
+ %whitespace <- ([ \\t\\r\\n] / '//' [^\\n]* / '/*' (!'*/' .)* '*/')*
+ """;
+
+ /** Split per-kind form — the c4169b6 workaround shape, used as the parity oracle. */
+ private static final String GRAMMAR_SPLIT = """
+ Word <- [a-zA-Z]+
+ %whitespace <- [ \\t\\r\\n]+ / '//' [^\\n]* / '/*' (!'*/' .)* '*/'
+ """;
+
+ private static void assertTokenArraysIdentical(TokenArray a, TokenArray b, String input) {
+ assertThat(a.count())
+ .as("token count for input <%s>", input)
+ .isEqualTo(b.count());
+ for (int i = 0; i < a.count(); i++ ) {
+ assertThat(a.kindAt(i))
+ .as("kind of token %d for input <%s>", i, input)
+ .isEqualTo(b.kindAt(i));
+ assertThat(a.startAt(i))
+ .as("start of token %d for input <%s>", i, input)
+ .isEqualTo(b.startAt(i));
+ assertThat(a.endAt(i))
+ .as("end of token %d for input <%s>", i, input)
+ .isEqualTo(b.endAt(i));
+ }
+ }
+
+ private static void assertSplitUnsplitParity(String input) {
+ var unsplit = engineFor(GRAMMAR_UNSPLIT)
+ .lex(input);
+ var split = engineFor(GRAMMAR_SPLIT)
+ .lex(input);
+ assertTokenArraysIdentical(unsplit, split, input);
+ assertThat(reconstruct(unsplit))
+ .isEqualTo(input);
+ }
+
+ @Test
+ void unsplitWhitespace_producesIdenticalTokenArrayToSplit_mixedTrivia() {
+ assertSplitUnsplitParity(" // hi\n/* blk */ hello");
+ }
+
+ @Test
+ void unsplitWhitespace_producesIdenticalTokenArrayToSplit_docVariants() {
+ assertSplitUnsplitParity("/// doc\n/** doc */ x");
+ }
+
+ @Test
+ void unsplitWhitespace_producesIdenticalTokenArrayToSplit_consecutiveBlockComments() {
+ assertSplitUnsplitParity("/*a*//*b*/");
+ }
+
+ @Test
+ void unsplitWhitespace_producesIdenticalTokenArrayToSplit_consecutiveLineComments() {
+ assertSplitUnsplitParity("//x\n//y\n");
+ }
+
+ @Test
+ void unsplitWhitespace_classifiesEachTriviaTokenByKind() {
+ var tokens = engineFor(GRAMMAR_UNSPLIT)
+ .lex(" // hi\n/* blk */ x");
+ // Expected: WS, LINE_COMMENT, WS(\n), BLOCK_COMMENT, WS, Word(x).
+ assertThat(countByKind(tokens, TokenArray.KIND_LINE_COMMENT))
+ .as("one LINE_COMMENT")
+ .isEqualTo(1);
+ assertThat(countByKind(tokens, TokenArray.KIND_BLOCK_COMMENT))
+ .as("one BLOCK_COMMENT")
+ .isEqualTo(1);
+ int nonTrivia = 0;
+ for (int i = 0; i < tokens.count(); i++ ) {
+ if (!tokens.isTrivia(i)) {
+ nonTrivia++ ;
+ }
+ }
+ assertThat(nonTrivia)
+ .as("only the Word token is non-trivia")
+ .isEqualTo(1);
+ assertThat(reconstruct(tokens))
+ .isEqualTo(" // hi\n/* blk */ x");
+ }
+
+ @Test
+ void unsplitWhitespace_zeroTrivia_noEmptyTokensNoInfiniteLoop() {
+ var tokens = engineFor(GRAMMAR_UNSPLIT)
+ .lex("hello");
+ assertThat(tokens.count())
+ .isEqualTo(1);
+ assertThat(tokens.isTrivia(0))
+ .isFalse();
+ for (int i = 0; i < tokens.count(); i++ ) {
+ assertThat(tokens.endAt(i))
+ .as("token %d is non-empty", i)
+ .isGreaterThan(tokens.startAt(i));
+ }
+ assertThat(reconstruct(tokens))
+ .isEqualTo("hello");
+ }
+
+ @Test
+ void unsplitWhitespace_roundTripsViaReconstruct_mixedTrivia() {
+ var input = " // hi\n/* blk */\n/// doc\n/** db */ end";
+ var tokens = engineFor(GRAMMAR_UNSPLIT)
+ .lex(input);
+ assertThat(reconstruct(tokens))
+ .isEqualTo(input);
+ }
+
+ @Test
+ void unsplitWhitespace_doesNotEmitEmptyMatchWarning() {
+ assertThat(generationWarnings(GRAMMAR_UNSPLIT))
+ .as("unsplit %whitespace must not trip the empty-match warning")
+ .isEmpty();
+ }
+
+ private static java.util.List generationWarnings(String grammarText) {
+ var grammar = GrammarParser.parse(grammarText)
+ .unwrap();
+ var classification = RuleClassifier.classify(grammar)
+ .unwrap();
+ var built = DfaBuilder.build(grammar, classification)
+ .unwrap();
+ return LexerGenerator.generate(grammar,
+ classification,
+ built.dfa(),
+ built.kinds(),
+ "test.pkg",
+ "GLexer")
+ .unwrap()
+ .warnings();
+ }
}
diff --git a/peglib-core/src/test/resources/java25.peg b/peglib-core/src/test/resources/java25.peg
index 226d466..d479136 100644
--- a/peglib-core/src/test/resources/java25.peg
+++ b/peglib-core/src/test/resources/java25.peg
@@ -197,12 +197,13 @@ NumLit <- < '0' [xX] [0-9a-fA-F_]+ [lL]? > / < '0' [bB] [01_]+ [lL]? > / < [0-9]
# Hard keywords only - contextual keywords (var, yield, record, sealed, non-sealed, permits, when, module) are handled by their specific rules
Keyword <- ('abstract' / 'assert' / 'boolean' / 'break' / 'byte' / 'case' / 'catch' / 'char' / 'class' / 'const' / 'continue' / 'default' / 'double' / 'do' / 'else' / 'enum' / 'extends' / 'false' / 'finally' / 'final' / 'float' / 'for' / 'goto' / 'implements' / 'import' / 'instanceof' / 'interface' / 'int' / 'if' / 'long' / 'native' / 'new' / 'null' / 'package' / 'private' / 'protected' / 'public' / 'return' / 'short' / 'static' / 'strictfp' / 'super' / 'switch' / 'synchronized' / 'this' / 'throws' / 'throw' / 'transient' / 'true' / 'try' / 'void' / 'volatile' / 'while') ![a-zA-Z0-9_$]
-# NB: each alternative consumes exactly one chunk; the lexer's outer
-# maximal-munch loop iterates naturally, so v6 per-kind trivia classification
-# (LINE_COMMENT, DOC_LINE_COMMENT via /// prefix, etc.) fires correctly.
-# Wrapping this in an outer * would fold the whole run into a single
-# WHITESPACE-kind token and defeat classification — see GRAMMAR-DSL.md.
-%whitespace <- [ \t\r\n]+ / '//' [^\n]* / '/*' (!'*/' .)* '*/'
+# 0.6.2 — the unsplit folded form is supported again: DfaBuilder.absorbWhitespace
+# absorbs each Choice alternative at its own trivia kind (WHITESPACE / LINE_COMMENT
+# / DOC_LINE_COMMENT / BLOCK_COMMENT / DOC_BLOCK_COMMENT), so the lexer's maximal-
+# munch loop emits one correctly-classified token per trivia chunk. The whitespace
+# alternative is absorbed as one-or-more, so the DFA start state never accepts the
+# empty string (no empty-match warning).
+%whitespace <- ([ \t\r\n] / '//' [^\n]* / '/*' (!'*/' .)* '*/')*
# 0.6.1 — incremental-reparse boundaries consumed by IncrementalParser.
# Each %checkpoint names a rule whose CST subtree may be re-parsed in
diff --git a/peglib-formatter/pom.xml b/peglib-formatter/pom.xml
index 89369df..8f7c6b4 100644
--- a/peglib-formatter/pom.xml
+++ b/peglib-formatter/pom.xml
@@ -7,7 +7,7 @@
org.pragmatica-lite
peglib-parent
- 0.6.1
+ 0.6.2
../pom.xml
diff --git a/peglib-incremental/pom.xml b/peglib-incremental/pom.xml
index 2a735a6..3fd59b9 100644
--- a/peglib-incremental/pom.xml
+++ b/peglib-incremental/pom.xml
@@ -7,7 +7,7 @@
org.pragmatica-lite
peglib-parent
- 0.6.1
+ 0.6.2
../pom.xml
diff --git a/peglib-maven-plugin/pom.xml b/peglib-maven-plugin/pom.xml
index a279de7..f80cd19 100644
--- a/peglib-maven-plugin/pom.xml
+++ b/peglib-maven-plugin/pom.xml
@@ -7,7 +7,7 @@
org.pragmatica-lite
peglib-parent
- 0.6.1
+ 0.6.2
../pom.xml
diff --git a/peglib-playground/pom.xml b/peglib-playground/pom.xml
index c7b5b8a..d41ebb3 100644
--- a/peglib-playground/pom.xml
+++ b/peglib-playground/pom.xml
@@ -7,7 +7,7 @@
org.pragmatica-lite
peglib-parent
- 0.6.1
+ 0.6.2
../pom.xml
diff --git a/peglib-runtime/pom.xml b/peglib-runtime/pom.xml
index cf06657..77895c2 100644
--- a/peglib-runtime/pom.xml
+++ b/peglib-runtime/pom.xml
@@ -7,7 +7,7 @@
org.pragmatica-lite
peglib-parent
- 0.6.1
+ 0.6.2
../pom.xml
diff --git a/pom.xml b/pom.xml
index 902cb16..e7df621 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
org.pragmatica-lite
peglib-parent
- 0.6.1
+ 0.6.2
pom
Peglib Parent