From b607567e80f634252d823576c1d7a368671de0f9 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Sat, 6 Jun 2026 16:00:15 +0200 Subject: [PATCH 1/4] chore: prepare release 0.6.2 --- CHANGELOG.md | 6 ++++++ README.md | 6 +++--- docs/VISITOR-TUTORIAL.md | 2 +- peglib-core/pom.xml | 2 +- peglib-formatter/pom.xml | 2 +- peglib-incremental/pom.xml | 2 +- peglib-maven-plugin/pom.xml | 2 +- peglib-playground/pom.xml | 2 +- peglib-runtime/pom.xml | 2 +- pom.xml | 2 +- 10 files changed, 17 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e04a57..b2bbcf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.6.2] - 2026-06-06 + +### Fixed + +- (placeholder — release in progress) + ## [0.6.1] - 2026-05-12 Patch release closing gaps from the 0.6.0 ship. Adds doc-comment trivia, completes diff --git a/README.md b/README.md index 5d31ae1..3ff7d7a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A PEG (Parsing Expression Grammar) parser library for Java. Tokens-first lex-then-parse architecture, flat int[] CST, visitor pattern, true incremental reparse. -Maven Central: `org.pragmatica-lite:peglib:0.6.1` +Maven Central: `org.pragmatica-lite:peglib:0.6.2` Migrating from 0.5.x? See [`docs/MIGRATION-0.5-TO-0.6.md`](docs/MIGRATION-0.5-TO-0.6.md). Design rationale: [`docs/ARCHITECTURE-0.6.0.md`](docs/ARCHITECTURE-0.6.0.md). @@ -29,7 +29,7 @@ Design rationale: [`docs/ARCHITECTURE-0.6.0.md`](docs/ARCHITECTURE-0.6.0.md). org.pragmatica-lite peglib - 0.6.1 + 0.6.2 ``` @@ -284,7 +284,7 @@ pre-compiled classes — no `fromGrammar` cost at runtime: org.pragmatica-lite peglib-maven-plugin - 0.6.1 + 0.6.2 generate-v6 diff --git a/docs/VISITOR-TUTORIAL.md b/docs/VISITOR-TUTORIAL.md index 42ece40..df5bde9 100644 --- a/docs/VISITOR-TUTORIAL.md +++ b/docs/VISITOR-TUTORIAL.md @@ -93,7 +93,7 @@ Configure `peglib-maven-plugin` in your `pom.xml`: org.pragmatica-lite peglib-maven-plugin - 0.6.1 + 0.6.2 generate-v6 diff --git a/peglib-core/pom.xml b/peglib-core/pom.xml index 38dc2b5..7c3a4e0 100644 --- a/peglib-core/pom.xml +++ b/peglib-core/pom.xml @@ -7,7 +7,7 @@ org.pragmatica-lite peglib-parent - 0.6.1 + 0.6.2 ../pom.xml diff --git a/peglib-formatter/pom.xml b/peglib-formatter/pom.xml index 89369df..8f7c6b4 100644 --- a/peglib-formatter/pom.xml +++ b/peglib-formatter/pom.xml @@ -7,7 +7,7 @@ org.pragmatica-lite peglib-parent - 0.6.1 + 0.6.2 ../pom.xml diff --git a/peglib-incremental/pom.xml b/peglib-incremental/pom.xml index 2a735a6..3fd59b9 100644 --- a/peglib-incremental/pom.xml +++ b/peglib-incremental/pom.xml @@ -7,7 +7,7 @@ org.pragmatica-lite peglib-parent - 0.6.1 + 0.6.2 ../pom.xml diff --git a/peglib-maven-plugin/pom.xml b/peglib-maven-plugin/pom.xml index a279de7..f80cd19 100644 --- a/peglib-maven-plugin/pom.xml +++ b/peglib-maven-plugin/pom.xml @@ -7,7 +7,7 @@ org.pragmatica-lite peglib-parent - 0.6.1 + 0.6.2 ../pom.xml diff --git a/peglib-playground/pom.xml b/peglib-playground/pom.xml index c7b5b8a..d41ebb3 100644 --- a/peglib-playground/pom.xml +++ b/peglib-playground/pom.xml @@ -7,7 +7,7 @@ org.pragmatica-lite peglib-parent - 0.6.1 + 0.6.2 ../pom.xml diff --git a/peglib-runtime/pom.xml b/peglib-runtime/pom.xml index cf06657..77895c2 100644 --- a/peglib-runtime/pom.xml +++ b/peglib-runtime/pom.xml @@ -7,7 +7,7 @@ org.pragmatica-lite peglib-parent - 0.6.1 + 0.6.2 ../pom.xml diff --git a/pom.xml b/pom.xml index 902cb16..e7df621 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.pragmatica-lite peglib-parent - 0.6.1 + 0.6.2 pom Peglib Parent From 21725a4160b6a9bf45b1571e1db1d6e9156fcd85 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Sat, 6 Jun 2026 16:37:15 +0200 Subject: [PATCH 2/4] fix: shift operators in initializer context via inline expansion of DFA-skipped lexer rules --- CHANGELOG.md | 20 +- .../java/org/pragmatica/peg/v6/PegParser.java | 7 +- .../peg/v6/generator/ParserGenerator.java | 123 +++++++++- .../pragmatica/peg/v6/lexer/DfaBuilder.java | 148 ++++++++++++ .../v6/diagnostic/Java25SelfHostDiagTest.java | 3 - .../ShiftOperatorInlineExpansionTest.java | 214 ++++++++++++++++++ 6 files changed, 508 insertions(+), 7 deletions(-) create mode 100644 peglib-core/src/test/java/org/pragmatica/peg/v6/generator/ShiftOperatorInlineExpansionTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index b2bbcf6..8effed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- (placeholder — release in progress) +- Shift operators (`<<`, `>>`, `>>>`) in field and local-variable initializer context + failed at `CompilationUnit` level. Root cause: `LShift`/`RShift`/`URShift` lexer rules + contain negative lookaheads (`!'='`), which the DFA builder cannot compile — the rules + were silently skipped while their assigned token kinds stayed live in the generated + parser, so the shift alternatives could never match. Fixed by inline expansion: + DFA-skipped LEXER rules of shape `literal+ (!literal)*` now expand in the generated + parser as adjacent constituent-token matches (byte-contiguity checked, so `1 < < 2` + does not fuse) with trailing negative lookaheads. Selfhost fixture (40K LOC) now + parses with 0 diagnostics; both previously `@Disabled` `Java25SelfHostDiagTest` + assertions re-enabled. + +### Added + +- Loud `fromGrammar` failure (`SkippedRuleReferenced`) when a parser rule references a + lexer rule whose token kind was assigned but whose DFA compilation was skipped without + an inline-expansion path — broken parsers are no longer generated silently. +- `ShiftOperatorInlineExpansionTest` — 16 regression tests covering shift operators in + field/local-var/parenthesized/chained contexts, compound shift assignments, adjacency + non-fusing, nested generics, and the loud guard. ## [0.6.1] - 2026-05-12 diff --git a/peglib-core/src/main/java/org/pragmatica/peg/v6/PegParser.java b/peglib-core/src/main/java/org/pragmatica/peg/v6/PegParser.java index d312b57..65d7d40 100644 --- a/peglib-core/src/main/java/org/pragmatica/peg/v6/PegParser.java +++ b/peglib-core/src/main/java/org/pragmatica/peg/v6/PegParser.java @@ -105,7 +105,12 @@ private static Result compileParser(Grammar grammar, RuleClassifier.Classification classification, DfaBuilder.Built built, String className) { - return ParserGenerator.generate(grammar, classification, built.kinds(), GENERATED_PACKAGE, className) + var skippedReasons = built.skipped() + .stream() + .collect(java.util.stream.Collectors.toMap(DfaBuilder.SkippedRule::ruleName, + DfaBuilder.SkippedRule::reason, + (a, __) -> a)); + return ParserGenerator.generate(grammar, classification, built.kinds(), skippedReasons, GENERATED_PACKAGE, className) .flatMap(ParserCompiler::compile); } } diff --git a/peglib-core/src/main/java/org/pragmatica/peg/v6/generator/ParserGenerator.java b/peglib-core/src/main/java/org/pragmatica/peg/v6/generator/ParserGenerator.java index 538bf6d..2f1f366 100644 --- a/peglib-core/src/main/java/org/pragmatica/peg/v6/generator/ParserGenerator.java +++ b/peglib-core/src/main/java/org/pragmatica/peg/v6/generator/ParserGenerator.java @@ -59,7 +59,8 @@ public sealed interface ParserGenerationError extends Cause permits ParserGenera ParserGenerationError.StartRuleNotParser, ParserGenerationError.UnsupportedExpression, ParserGenerationError.UnknownLiteral, - ParserGenerationError.UnknownReference { + ParserGenerationError.UnknownReference, + ParserGenerationError.SkippedRuleReferenced { record InvalidIdentifier(String component, String value) implements ParserGenerationError { @Override public String message() { return "Invalid Java identifier for " + component + ": '" + value + "'"; @@ -96,6 +97,25 @@ record UnknownReference(String ruleName, String referencedRule) implements Parse return "Rule '" + ruleName + "' references undefined rule '" + referencedRule + "'"; } } + + /** + * 0.6.2 — loud guard. A PARSER/MIXED rule references a LEXER rule that + * was assigned a token kind but skipped by the DFA builder (unsupported + * lookahead / non-regular body) and has neither an alias set nor an + * inline-expansion path. The lexer can never produce that kind, so the + * generated reference {@code if (peek() != KIND_X)} would always fail — + * silently breaking every use of the rule. Fail generation naming both + * rules instead of emitting a broken parser. + */ + record SkippedRuleReferenced(String ruleName, String referencedRule, String reason) + implements ParserGenerationError { + @Override public String message() { + return "Rule '" + ruleName + "' references LEXER rule '" + referencedRule + + "' which was assigned a token kind but the lexer cannot produce it (skipped by the " + + "DFA builder) and it has no alias or inline-expansion path. The generated reference would " + + "never match. Original skip reason: " + reason; + } + } } public record GeneratedParser(String packageName, String className, String source) { @@ -111,6 +131,24 @@ public static Result generate(Grammar grammar, DfaBuilder.TokenKindAssignment kinds, String packageName, String className) { + return generate(grammar, classification, kinds, Map.of(), packageName, className); + } + + /** + * 0.6.2 — generation with the DFA builder's skipped-rule reasons (rule name → + * skip reason). References to a skipped LEXER rule that has a live token kind + * but no alias / inline-expansion path are rejected with + * {@link ParserGenerationError.SkippedRuleReferenced} rather than silently + * emitting a dead-kind match. The {@link #generate(Grammar, RuleClassifier.Classification, + * DfaBuilder.TokenKindAssignment, String, String) 5-arg overload} passes an empty + * map (no guard) for callers that only inspect generated source. + */ + public static Result generate(Grammar grammar, + RuleClassifier.Classification classification, + DfaBuilder.TokenKindAssignment kinds, + Map skippedReasons, + String packageName, + String className) { // Internal entry: callers are PegParser/tests with validated inputs. if ( !isValidQualifiedPackage(packageName)) { return new ParserGenerationError.InvalidIdentifier("packageName", String.valueOf(packageName)).result();} @@ -123,7 +161,7 @@ public static Result generate(Grammar grammar, var startKind = classification.kinds().get(startRule.name()); if ( startKind != RuleKind.PARSER && startKind != RuleKind.MIXED) { return new ParserGenerationError.StartRuleNotParser(startRule.name(), startKind).result();} - return new Renderer(grammar, classification, kinds, packageName, className).render(); + return new Renderer(grammar, classification, kinds, skippedReasons, packageName, className).render(); } /** Render the full parser source. Stateful to keep helper methods readable. */ @@ -149,14 +187,20 @@ private static final class Renderer { // int[] of acceptable kinds (idKind + all fallback kinds). private final Map idFallbackArrays; + // 0.6.2 — DFA-builder skipped-rule reasons (rule name → reason). Used to + // reject references to a skipped LEXER rule with a dead token kind. + private final Map skippedReasons; + Renderer(Grammar grammar, RuleClassifier.Classification classification, DfaBuilder.TokenKindAssignment kinds, + Map skippedReasons, String packageName, String className) { this.grammar = grammar; this.classification = classification; this.kinds = kinds; + this.skippedReasons = skippedReasons; this.packageName = packageName; this.className = className; this.ruleMap = grammar.ruleMap(); @@ -1041,12 +1085,29 @@ private Result emitReference(Expression.Reference ref, EmitContext ctx) { emitAliasMatch(ref.ruleName(), aliasKinds, ctx); return Result.unitResult(); } + // 0.6.2 — LEXER rule expanded inline as a multi-token match. The rule + // (e.g. {@code LShift <- '<' '<' !'='}) has no single DFA token kind; + // instead match each constituent single-char inline-literal kind as + // an adjacent token (byte-contiguous, so {@code < <} with trivia in + // between does NOT match) and verify the trailing negative lookahead. + var expansion = kinds.inlineExpansions().get(ref.ruleName()); + if ( expansion != null) { + emitInlineExpansion(ref.ruleName(), expansion, ctx); + return Result.unitResult(); + } // LEXER reference — consume one token of the matching kind. var kind = kinds.ruleNameToKind().get(ref.ruleName()); if ( kind == null) { // Rule was demoted to LEXER but DFA didn't pick it up (e.g. skipped). // Fall back to ANY_CHAR or report. return new ParserGenerationError.UnknownReference(ctx.ruleName, ref.ruleName()).result();} + // 0.6.2 — loud guard. The rule has a live kind but no alias and no + // inline expansion, and the DFA builder skipped it: the lexer cannot + // produce this kind, so a {@code peek() != KIND_X} match here would + // always fail. Reject generation naming both rules. + var skipReason = skippedReasons.get(ref.ruleName()); + if ( skipReason != null) { + return new ParserGenerationError.SkippedRuleReferenced(ctx.ruleName, ref.ruleName(), skipReason).result();} usedTokenKinds.add(kind); // Phase 0.6.0 — identifier fallback. If the referenced rule is a // skip-prefix rule (e.g. {@code Identifier <- !Keyword [a-zA-Z_$]...}), @@ -1074,6 +1135,64 @@ private Result emitReference(Expression.Reference ref, EmitContext ctx) { return Result.unitResult(); } + /** + * 0.6.2 — emit a multi-token inline expansion for a LEXER rule of shape + * {@code literal+ !literal?} (e.g. {@code LShift <- '<' '<' !'='}). The + * rule has no fused DFA token; instead each constituent single-char + * inline-literal kind is matched as a separate token. Consecutive + * positive literals must be byte-adjacent (the previous token's end + * offset equals the next token's start offset) so that {@code < <} with + * intervening trivia does NOT match as {@code <<}. After the positive + * literals, an optional negative-lookahead literal kind is verified: the + * current (non-trivia) token must NOT be that kind. On any mismatch the + * generated code calls {@code fail(...)} and dispatches {@code ctx.failAction}. + */ + private void emitInlineExpansion(String ruleName, DfaBuilder.InlineExpansion expansion, EmitContext ctx) { + var indent = indent(ctx.depth); + var literalKinds = expansion.literalKinds(); + for ( var k : literalKinds) { + usedTokenKinds.add(k);} + for ( var k : expansion.negLookaheadKinds()) { + usedTokenKinds.add(k);} + var failTag = escapeJavaString(ruleName); + var ruleKind = ruleKindConst(ctx); + var prevVar = "__ie_prev_" + ctx.nextLabelId(); + ctx.sb.append(indent).append("int ").append(prevVar).append(" = -1;\n"); + for ( int i = 0; i < literalKinds.length; i++) { + var kindConst = "KIND_" + sanitize(kinds.kindNameTable() [literalKinds[i]]); + if ( i == 0) { + ctx.sb.append(indent).append("if (peek() != ").append(kindConst) + .append(") { fail(\"").append(failTag).append("\", ").append(ruleKind) + .append("); ").append(ctx.failAction).append(" }\n"); + } else + { + // Adjacency: the previously consumed (non-trivia) token must be + // byte-contiguous with the current one — no trivia, no gap — to + // count as a fused multi-char operator. {@code < <} (spaced) has + // an intervening whitespace token and therefore does NOT match. + ctx.sb.append(indent).append("if (peek() != ").append(kindConst) + .append(" || ").append(prevVar).append(" < 0 || tokens.endAt(").append(prevVar) + .append(") != tokens.startAt(pos)") + .append(") { fail(\"").append(failTag).append("\", ").append(ruleKind) + .append("); ").append(ctx.failAction).append(" }\n"); + } + ctx.sb.append(indent).append(prevVar).append(" = pos;\n"); + ctx.sb.append(indent).append("advance();\n"); + } + for ( var negKind : expansion.negLookaheadKinds()) { + var negConst = "KIND_" + sanitize(kinds.kindNameTable() [negKind]); + // Negative lookahead: the next non-trivia token must not be the + // guarded kind while byte-adjacent to the last consumed token. A + // non-adjacent token of that kind is a separate operator and does + // not violate the lookahead. + ctx.sb.append(indent).append("if (peek() == ").append(negConst) + .append(" && ").append(prevVar).append(" >= 0 && tokens.endAt(").append(prevVar) + .append(") == tokens.startAt(pos)") + .append(") { fail(\"").append(failTag).append("\", ").append(ruleKind) + .append("); ").append(ctx.failAction).append(" }\n"); + } + } + /** * 0.6.1 — Item B — the RULE__KIND constant string, used as * the rule-kind argument to emitted {@code fail(..., kind)} calls so the diff --git a/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/DfaBuilder.java b/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/DfaBuilder.java index 41711fd..faf26f3 100644 --- a/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/DfaBuilder.java +++ b/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/DfaBuilder.java @@ -97,11 +97,34 @@ public record KeywordResolution(int identKind, Map textToKind){ * "contextual keywords fall through to Identifier elsewhere" behavior * that the 0.5.x interpreter got for free via PEG ordered choice. */ + /** + * 0.6.2 — inline-expansion plan for a LEXER rule whose body is a sequence of + * single-character literals optionally followed by a negative-lookahead + * literal (shape {@code literal+ !literal}). Such rules (e.g. + * {@code LShift <- '<' '<' !'='}) cannot be compiled to a single DFA accept + * state because the {@code Not} lookahead is not regular, AND fusing the + * constituent characters into one token would break the deliberate + * two-token model for {@code <}/{@code >} that nested generics + * ({@code List>}) depend on. Instead the generated parser + * matches each constituent inline-literal kind as a separate adjacent token + * and verifies the negative lookahead, expanding the rule body inline at the + * reference site. + * + * @param literalKinds inline-literal token kinds for each positive + * single-char literal, in body order (length ≥ 1) + * @param negLookaheadKinds inline-literal token kinds for each trailing + * {@code !literal} guard, in body order (possibly + * empty). A match requires that the next non-trivia + * token is none of these kinds (when byte-adjacent). + */ + public record InlineExpansion(int[] literalKinds, int[] negLookaheadKinds){} + public record TokenKindAssignment(Map ruleNameToKind, Map inlineLiteralToKind, Map keywordResolutions, Map ruleNameToAliasKinds, Map identifierFallbackKinds, + Map inlineExpansions, int anyCharKind, String[] kindNameTable){} @@ -244,6 +267,23 @@ private static Result assignKinds(Grammar grammar, usedNames, nextKindRef, aliasLiteralsOut); + // 0.6.2 — inline-expansion detection. For each LEXER rule whose body is a + // sequence of single-char literals with a trailing negative-lookahead + // literal (shape {@code literal+ !literal}, e.g. {@code LShift <- '<' '<' + // !'='}), the DFA cannot build it (the Not is not regular) AND fusing the + // characters into one token would break the two-token model for that + // nested generics depend on. Allocate inline-literal kinds for each + // constituent character, record an InlineExpansion, and REMOVE the rule's + // own kind so the parser never references a dead single-token kind. + var inlineExpansions = detectInlineExpansions(grammar, + classification, + ruleNameToKind, + ruleNameToAliasKinds, + inlineLiteralToKind, + kindNames, + usedNames, + nextKindRef, + aliasLiteralsOut); // Phase B.0 — keyword resolution. For each skip-prefix rule, walk the // referenced literal-set rule and map every keyword text to a token // kind. Reuse existing kinds where possible (inline literals or @@ -286,6 +326,7 @@ private static Result assignKinds(Grammar grammar, Map.copyOf(keywordResolutions), Map.copyOf(ruleNameToAliasKinds), Map.copyOf(identifierFallbackKinds), + Map.copyOf(inlineExpansions), anyCharKind, kindNames.toArray(new String[0]))); } @@ -438,6 +479,108 @@ private static Map buildAliasMap(Grammar grammar, return aliases; } + /** + * 0.6.2 — detect LEXER rules eligible for inline expansion. A rule is + * eligible iff (after unwrapping acceptable wrappers and a token boundary) + * its body is a {@link Expression.Sequence} of single-character + * {@link Expression.Literal}s optionally terminated by a single + * {@code !literal} negative lookahead, AND the rule is not already aliased, + * not a skip-prefix rule, and is currently kind-assigned as LEXER. For each + * eligible rule the constituent inline-literal kinds are allocated (so the + * lexer actually produces the single-char tokens) and the rule's own kind is + * removed from {@code ruleNameToKind} so the parser never matches it as a + * fused single token. + * + *

The shape match deliberately requires the trailing {@code !literal}: a + * bare {@code literal+} sequence (e.g. {@code RShiftAssign <- '>' '>' '='}) + * is DFA-compilable as a single token and is left untouched to preserve the + * existing compound-assignment behavior. + */ + private static Map detectInlineExpansions( + Grammar grammar, + RuleClassifier.Classification classification, + Map ruleNameToKind, + Map ruleNameToAliasKinds, + Map inlineLiteralToKind, + List kindNames, + Set usedNames, + int[] nextKindRef, + List aliasLiteralsOut) { + var result = new LinkedHashMap(); + for ( var rule : grammar.rules()) { + if ( classification.kinds().get(rule.name()) != RuleKind.LEXER) { + continue;} + if ( !ruleNameToKind.containsKey(rule.name())) { + continue;} + if ( ruleNameToAliasKinds.containsKey(rule.name())) { + continue;} + if ( classification.keywordSkip().containsKey(rule.name())) { + continue;} + var shape = matchInlineExpansionShape(rule.expression()); + if ( shape.isEmpty()) { + continue;} + var s = shape.unwrap(); + var literalKinds = new int[s.literals().size()]; + for ( int i = 0; i < s.literals().size(); i++) { + literalKinds[i] = ensureInlineKind(s.literals().get(i), inlineLiteralToKind, kindNames, usedNames, nextKindRef, aliasLiteralsOut);} + var negKinds = new int[s.negLookaheads().size()]; + for ( int i = 0; i < s.negLookaheads().size(); i++) { + negKinds[i] = ensureInlineKind(s.negLookaheads().get(i), inlineLiteralToKind, kindNames, usedNames, nextKindRef, aliasLiteralsOut);} + result.put(rule.name(), new InlineExpansion(literalKinds, negKinds)); + ruleNameToKind.remove(rule.name()); + } + return result; + } + + private record ExpansionShape(List literals, List negLookaheads){} + + /** + * Match the {@code literal+ (!literal)*} inline-expansion shape. Returns the + * positive single-char literals and the trailing negative-lookahead literals + * (e.g. {@code RShift <- '>' '>' !'>' !'='} yields positives {@code [>,>]} + * and negatives {@code [>,=]}), or {@code Option.none()} when the body + * doesn't fit. Every positive element must be a non-empty, case-sensitive + * single-character literal; every trailing {@code !literal} must likewise be + * a single-char case-sensitive literal, and all negatives must follow all + * positives. At least one trailing negative lookahead is required: a bare + * literal sequence is DFA-compilable as a single token and left untouched. + */ + private static Option matchInlineExpansionShape(Expression expr) { + var unwrapped = unwrapAcceptableWrappers(expr); + if ( ! (unwrapped instanceof Expression.Sequence seq)) { + return Option.none();} + var elements = seq.elements(); + if ( elements.isEmpty()) { + return Option.none();} + var positives = new ArrayList(); + var negLookaheads = new ArrayList(); + for ( var element : elements) { + var el = unwrapAcceptableWrappers(element); + if ( el instanceof Expression.Not not) { + var notInner = unwrapAcceptableWrappers(not.expression()); + if ( ! (notInner instanceof Expression.Literal notLit) || !isSingleCharLiteral(notLit)) { + return Option.none();} + negLookaheads.add(notLit); + } else + if ( el instanceof Expression.Literal lit && isSingleCharLiteral(lit)) { + // A positive literal after a negative lookahead is not the + // supported shape (negatives must be strictly trailing). + if ( !negLookaheads.isEmpty()) { + return Option.none();} + positives.add(lit); + } else + { + return Option.none();} + } + if ( positives.isEmpty() || negLookaheads.isEmpty()) { + return Option.none();} + return Option.some(new ExpansionShape(positives, negLookaheads)); + } + + private static boolean isSingleCharLiteral(Expression.Literal lit) { + return lit.text().length() == 1 && !lit.caseInsensitive(); + } + /** * Strip outer wrappers from {@code expr} and, if the result is * {@code Sequence(actualBody, Not(CharClass))}, return {@code actualBody}. @@ -652,6 +795,11 @@ private static Nfa buildNfaWithSkips(Grammar grammar, // importantly) the DFA build would fail on the !CharClass guard. if ( assignment.ruleNameToAliasKinds().containsKey(rule.name())) { continue;} + // 0.6.2 — inline-expanded rules (e.g. LShift) have no own token kind; + // the parser matches their constituent inline-literal kinds directly. + // They must not be compiled as standalone DFA accepts. + if ( assignment.inlineExpansions().containsKey(rule.name())) { + continue;} int kind = assignment.ruleNameToKind().get(rule.name()); // Phase B.0 — for skip-prefix rules, compile the body expression only. // The Not(Reference) head is replaced by post-match keyword resolution diff --git a/peglib-core/src/test/java/org/pragmatica/peg/v6/diagnostic/Java25SelfHostDiagTest.java b/peglib-core/src/test/java/org/pragmatica/peg/v6/diagnostic/Java25SelfHostDiagTest.java index 0146738..605e5b6 100644 --- a/peglib-core/src/test/java/org/pragmatica/peg/v6/diagnostic/Java25SelfHostDiagTest.java +++ b/peglib-core/src/test/java/org/pragmatica/peg/v6/diagnostic/Java25SelfHostDiagTest.java @@ -1,6 +1,5 @@ package org.pragmatica.peg.v6.diagnostic; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.pragmatica.peg.v6.PegParser; import org.pragmatica.peg.v6.cst.CstArray; @@ -85,7 +84,6 @@ public void dumpDiagnostics() throws Exception { // cleanly via parseRuleFrom(Shift) / parseRuleFrom(Expr) in isolation. // Hypothesis: Type / Relational / TypeArgs interaction with '<' literals // corrupts parser state on backtrack. Deferred to 0.6.2. - @Disabled("TODO 0.6.2 — shift-in-FieldDecl bug; see comment") @Test public void selfHostFixtureParsesCleanly() throws Exception { String grammar = Files.readString(GRAMMAR); @@ -122,7 +120,6 @@ public void selfHostFixtureParsesCleanly() throws Exception { } } - @Disabled("TODO 0.6.2 — depends on shift-in-FieldDecl fix") @Test public void selfHostFixtureProducesShallowCST() throws Exception { String grammar = Files.readString(GRAMMAR); diff --git a/peglib-core/src/test/java/org/pragmatica/peg/v6/generator/ShiftOperatorInlineExpansionTest.java b/peglib-core/src/test/java/org/pragmatica/peg/v6/generator/ShiftOperatorInlineExpansionTest.java new file mode 100644 index 0000000..7a7570f --- /dev/null +++ b/peglib-core/src/test/java/org/pragmatica/peg/v6/generator/ShiftOperatorInlineExpansionTest.java @@ -0,0 +1,214 @@ +package org.pragmatica.peg.v6.generator; + +import org.pragmatica.peg.v6.PegParser; +import org.pragmatica.peg.v6.cst.CstArray; +import org.pragmatica.peg.v6.cst.ParseResult; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 0.6.2 — regression coverage for the shift-operator inline-expansion fix. + * + *

The Java25 grammar splits shift operators into single-character literals so + * that nested generics ({@code List>}) still parse: + * {@code LShift <- '<' '<' !'='}, {@code RShift <- '>' '>' !'>' !'='}, + * {@code URShift <- '>' '>' '>' !'='}. These rules classify as LEXER but the DFA + * builder cannot compile their negative lookahead, so they are expanded inline by + * the parser generator as adjacent multi-token matches against the constituent + * {@code <}/{@code >}/{@code =} inline-literal kinds. + * + *

Before the fix every {@code <<}/{@code >>}/{@code >>>} in field- or + * local-variable-initializer context produced a cascade of "trailing input not + * consumed" diagnostics; nested generics and relational comparisons must keep + * parsing cleanly after the fix. + */ +class ShiftOperatorInlineExpansionTest { + private static final Path GRAMMAR_PATH = Paths.get("src/test/resources/java25.peg"); + + private static PegParserWrapper parser; + + /** Thin holder so the heavy generate+compile happens exactly once. */ + private record PegParserWrapper(org.pragmatica.peg.v6.Parser delegate) { + ParseResult parse(String input) { + return delegate.parse(input, 200); + } + } + + @BeforeAll + static void setupOnce() throws IOException { + var grammarText = Files.readString(GRAMMAR_PATH, StandardCharsets.UTF_8); + parser = new PegParserWrapper(PegParser.fromGrammar(grammarText).unwrap()); + } + + private static int countShiftNodesWithOperator(CstArray cst) { + // A Shift node that actually applied a shift operator has more than one + // Additive child (Shift <- Additive ((URShift / LShift / RShift) Additive)*). + int withOp = 0; + for (int i = 0; i < cst.nodeCount(); i++) { + if (!"Shift".equals(cst.kindNameAt(i))) { + continue; + } + int additives = 0; + for (int c = cst.firstChildAt(i); c >= 0; c = cst.nextSiblingAt(c)) { + if ("Additive".equals(cst.kindNameAt(c))) { + additives++; + } + } + if (additives > 1) { + withOp++; + } + } + return withOp; + } + + private static void assertParsesCleanly(String source, int minNodes) { + var result = parser.parse(source); + assertThat(result.diagnostics()) + .as("diagnostics for: %s", source) + .isEmpty(); + assertThat(result.cst().nodeCount()) + .as("CST node count for: %s (guards against empty-match bailout)", source) + .isGreaterThanOrEqualTo(minNodes); + } + + @Nested + class ShiftInExpressions { + @Test + void leftShift_parsesCleanly_inFieldInitializer() { + assertParsesCleanly("class A { int x = 1 << 2; }", 20); + assertThat(countShiftNodesWithOperator(parser.parse("class A { int x = 1 << 2; }").cst())) + .as("<< should be recognised as a shift operator") + .isEqualTo(1); + } + + @Test + void rightShift_parsesCleanly_inFieldInitializer() { + assertParsesCleanly("class A { int x = 1 >> 2; }", 20); + assertThat(countShiftNodesWithOperator(parser.parse("class A { int x = 1 >> 2; }").cst())) + .isEqualTo(1); + } + + @Test + void unsignedRightShift_parsesCleanly_inFieldInitializer() { + assertParsesCleanly("class A { int x = 1 >>> 2; }", 20); + assertThat(countShiftNodesWithOperator(parser.parse("class A { int x = 1 >>> 2; }").cst())) + .isEqualTo(1); + } + + @Test + void leftShift_parsesCleanly_inLocalVariableInitializer() { + assertParsesCleanly("class A { void m() { int x = 1 << 2; } }", 25); + } + + @Test + void rightShift_parsesCleanly_inLocalVariableInitializer() { + assertParsesCleanly("class A { void m() { int x = 1 >> 2; } }", 25); + } + + @Test + void unsignedRightShift_parsesCleanly_inLocalVariableInitializer() { + assertParsesCleanly("class A { void m() { int x = 1 >>> 2; } }", 25); + } + + @Test + void leftShift_parsesCleanly_whenParenthesized() { + assertParsesCleanly("class A { int x = (1 << 2); }", 25); + } + + @Test + void chainedShift_parsesCleanly() { + assertParsesCleanly("class A { int x = 1 << 2 >> 3; }", 25); + assertThat(countShiftNodesWithOperator(parser.parse("class A { int x = 1 << 2 >> 3; }").cst())) + .isEqualTo(1); + } + } + + @Nested + class ShiftAssignment { + @Test + void leftShiftAssign_parsesCleanly() { + assertParsesCleanly("class A { void m() { x <<= 2; } }", 25); + } + + @Test + void rightShiftAssign_parsesCleanly() { + assertParsesCleanly("class A { void m() { x >>= 2; } }", 25); + } + + @Test + void unsignedRightShiftAssign_parsesCleanly() { + assertParsesCleanly("class A { void m() { x >>>= 2; } }", 25); + } + } + + @Nested + class GenericsStillWork { + @Test + void nestedGenerics_parseWithZeroDiagnostics() { + assertParsesCleanly("class A { List> x; }", 15); + assertThat(countShiftNodesWithOperator(parser.parse("class A { List> x; }").cst())) + .as(">> closing nested generics must NOT be parsed as a shift operator") + .isZero(); + } + + @Test + void parameterizedMap_parsesWithZeroDiagnostics() { + assertParsesCleanly("class A { java.util.Map> x; }", 20); + assertThat(countShiftNodesWithOperator(parser.parse("class A { java.util.Map> x; }").cst())) + .isZero(); + } + + @Test + void relationalComparison_parsesWithZeroDiagnostics() { + assertParsesCleanly("class A { boolean b = a < b; }", 15); + assertThat(countShiftNodesWithOperator(parser.parse("class A { boolean b = a < b; }").cst())) + .as("a single '<' is relational, not a shift") + .isZero(); + } + } + + @Nested + class Adjacency { + @Test + void spacedLessThans_doNotFuseIntoLeftShift() { + // `1 < < 2` has whitespace between the two '<' tokens, so the inline + // expansion's adjacency check must refuse to treat it as `<<`. Java + // itself rejects this, so the parser should NOT see a shift operator. + var result = parser.parse("class A { int x = 1 < < 2; }"); + assertThat(countShiftNodesWithOperator(result.cst())) + .as("`< <` (spaced) must not parse as a `<<` shift") + .isZero(); + } + } + + @Nested + class LoudGuard { + @Test + void fromGrammar_fails_whenParserRuleReferencesSkippedLexerRuleWithoutExpansion() { + // Tok is LEXER (literals only) with a multi-char negative lookahead, + // which the DFA cannot compile and which does not fit the single-char + // `literal+ !literal` inline-expansion shape. Start (PARSER) references + // it, so the generator must reject rather than emit a dead-kind match. + var grammar = """ + Start <- Tok ';' + Tok <- 'ab' !'cd' + """; + org.pragmatica.peg.v6.PegParser.fromGrammar(grammar) + .onSuccess(__ -> org.junit.jupiter.api.Assertions.fail("expected SkippedRuleReferenced failure")) + .onFailure(cause -> assertThat(cause.message()) + .contains("Tok") + .contains("skipped")); + } + } +} + From 1afae1d399321a05f3b729cc44cbca6d269132af Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Sat, 6 Jun 2026 18:50:48 +0200 Subject: [PATCH 3/4] fix: emit per-kind trivia tokens from folded %whitespace via per-alternative DFA absorption --- CHANGELOG.md | 6 + docs/GRAMMAR-DSL.md | 46 +++--- .../peg/v6/generator/LexerGenerator.java | 10 +- .../pragmatica/peg/v6/lexer/DfaBuilder.java | 147 +++++++++++++++++- .../pragmatica/peg/v6/lexer/LexerEngine.java | 23 ++- .../v6/lexer/TriviaClassificationTest.java | 145 ++++++++++++++++- peglib-core/src/test/resources/java25.peg | 13 +- 7 files changed, 340 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8effed1..e2b15d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 does not fuse) with trailing negative lookaheads. Selfhost fixture (40K LOC) now parses with 0 diagnostics; both previously `@Disabled` `Java25SelfHostDiagTest` assertions re-enabled. +- Per-iteration `%whitespace` tokenization: an unsplit folded `%whitespace <- (ws / comment / ...)*` + rule coalesced an entire trivia run into a single token classified only by prefix-sniffing. + The DFA builder now absorbs each closure alternative as its own structurally-classified trivia + kind, so the lexer emits one correctly-kinded token per iteration. The canonical `java25.peg` + `%whitespace` grammar-split workaround is reverted; the empty-match lexer warning no longer + fires for the folded form. Two previously `@Disabled` block-comment trivia tests re-enabled. ### Added diff --git a/docs/GRAMMAR-DSL.md b/docs/GRAMMAR-DSL.md index ef72465..9be87da 100644 --- a/docs/GRAMMAR-DSL.md +++ b/docs/GRAMMAR-DSL.md @@ -198,39 +198,33 @@ These appear at the top level of the grammar, alongside `%whitespace`. ### `%whitespace` shape and v6 per-kind trivia classification -In the v6 (0.6.x) tokens-first lexer, trivia is classified by inspecting the -matched span's leading prefix: `//` → `LINE_COMMENT`, `///` → `DOC_LINE_COMMENT`, -`/*` → `BLOCK_COMMENT`, `/**` (followed by non-`/`) → `DOC_BLOCK_COMMENT`, -otherwise `WHITESPACE`. The check only fires when the entire matched token IS -the comment. - -This means **the shape of the `%whitespace` rule matters**. Two equivalent-looking -forms produce very different token streams: +In the v6 (0.6.x) tokens-first lexer, each `%whitespace` Choice alternative is +absorbed into the DFA at its own trivia kind, decided **structurally** by the +alternative's leading literal: a whitespace char-class alternative → `WHITESPACE`, +an alternative beginning with `'//'` → `LINE_COMMENT`, with `'/*'` → +`BLOCK_COMMENT`. The doc variants (`///` → `DOC_LINE_COMMENT`, `/**` followed by +non-`/` → `DOC_BLOCK_COMMENT`) cannot be told apart from their regular form by +the grammar alone (both share the same `'//' …` / `'/*' …` alternative), so they +are refined from the matched span text after the structural base kind is set. + +As of 0.6.2 **both shapes below produce the same per-kind token stream** — the +folded `(...)*` form no longer collapses a mixed-trivia run into a single +`WHITESPACE` token: ```peg -# Folds runs into ONE WHITESPACE-kind token — comment classification never fires: +# Folded form — DfaBuilder.absorbWhitespace absorbs each alternative at its kind: %whitespace <- ([ \t\r\n] / '//' [^\n]* / '/*' (!'*/' .)* '*/')* -# Emits one token per chunk — classification works: +# Split form — equivalent token stream: %whitespace <- [ \t\r\n]+ / '//' [^\n]* / '/*' (!'*/' .)* '*/' ``` -The outer `*` in the first form causes the lexer's longest-match scan to consume -multi-chunk runs (whitespace + comment + whitespace) as a single token whose -first char is whitespace — the prefix check fails and the entire run stays -`KIND_WHITESPACE`. Dropping the outer `*` and making the inner whitespace -alternative `[ \t\r\n]+` lets each invocation match exactly one chunk; the -lexer's outer maximal-munch loop iterates and emits separate tokens, each -classified correctly. - -A side benefit: the split form does NOT match the empty string, so the -"`LEXER rule '%whitespace' matches the empty string`" warning emitted by -`peglib:generate-v6` goes away. - -A 0.6.2 follow-up will add per-iteration trivia tokenization so the folded -form also produces per-kind tokens, but until then the split form is the -canonical shape. The example grammar at -`peglib-core/src/test/resources/java25.peg` uses the split form. +The folded form's outer `*` is dropped during absorption (the lexer's own +maximal-munch loop supplies the repetition), and the whitespace char-class +alternative is absorbed as one-or-more, so the DFA start state never accepts the +empty string — the `"LEXER rule '%whitespace' matches the empty string"` warning +does **not** fire for either shape. The canonical example grammar at +`peglib-core/src/test/resources/java25.peg` uses the folded form. ### `%suggest` diff --git a/peglib-core/src/main/java/org/pragmatica/peg/v6/generator/LexerGenerator.java b/peglib-core/src/main/java/org/pragmatica/peg/v6/generator/LexerGenerator.java index 4ef9b1d..552f366 100644 --- a/peglib-core/src/main/java/org/pragmatica/peg/v6/generator/LexerGenerator.java +++ b/peglib-core/src/main/java/org/pragmatica/peg/v6/generator/LexerGenerator.java @@ -297,12 +297,18 @@ private static void renderLexMethod(StringBuilder sb, int alphabet, boolean hasR sb.append(" }\n"); sb.append(" }\n"); } - // Phase A.6 / 0.6.1 — content-based trivia classification (mirrors LexerEngine). + // Phase A.6 / 0.6.1 / 0.6.2 — content-based trivia refinement (mirrors LexerEngine). + // Fires for the legacy coalesced WHITESPACE token AND for structurally-assigned + // LINE_COMMENT / BLOCK_COMMENT base kinds whose doc variant can only be told + // apart by the matched text. // // → LINE_COMMENT (also // without third '/') // /// → DOC_LINE_COMMENT (3+ slashes) // /* ... */ → BLOCK_COMMENT // /** ... */ → DOC_BLOCK_COMMENT (NOT '/**/' — smallest empty block) - sb.append(" if (lastAcceptKind == TokenArray.KIND_WHITESPACE && lastAcceptEnd > pos + 1) {\n"); + sb.append(" if ((lastAcceptKind == TokenArray.KIND_WHITESPACE\n"); + sb.append(" || lastAcceptKind == TokenArray.KIND_LINE_COMMENT\n"); + sb.append(" || lastAcceptKind == TokenArray.KIND_BLOCK_COMMENT)\n"); + sb.append(" && lastAcceptEnd > pos + 1) {\n"); sb.append(" char c0 = input.charAt(pos);\n"); sb.append(" char c1 = input.charAt(pos + 1);\n"); sb.append(" if (c0 == '/') {\n"); diff --git a/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/DfaBuilder.java b/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/DfaBuilder.java index faf26f3..17aee68 100644 --- a/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/DfaBuilder.java +++ b/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/DfaBuilder.java @@ -779,13 +779,11 @@ private static Nfa buildNfaWithSkips(Grammar grammar, absorbLiteralFragment(nfa, lit, kind, priorityRef, globalStart); } if ( grammar.whitespace().isPresent()) { - tryAbsorb(nfa, - "%whitespace", - grammar.whitespace().unwrap(), - KIND_WHITESPACE, - priorityRef, - globalStart, - skipped);} + absorbWhitespace(nfa, + grammar.whitespace().unwrap(), + priorityRef, + globalStart, + skipped);} for ( var rule : grammar.rules()) { if ( classification.kinds().get(rule.name()) != RuleKind.LEXER) { continue;} @@ -815,6 +813,141 @@ private static Nfa buildNfaWithSkips(Grammar grammar, return nfa; } + /** + * 0.6.2 — absorb the {@code %whitespace} body with per-alternative trivia + * kinds, so that a single coalescing {@code (...)*} wrapper around a Choice + * of trivia kinds (whitespace / line comment / block comment / doc variants) + * does NOT collapse an entire mixed-trivia run into one {@code KIND_WHITESPACE} + * token. Each Choice alternative becomes its own NFA accept fragment whose + * kind is decided structurally (by the alternative's leading literal), not by + * a post-match prefix sniff of the matched text. + * + *

Recognised body shapes (after unwrapping Group/TokenBoundary/etc.): + *

    + *
  • {@code ZeroOrMore(Choice)} / {@code OneOrMore(Choice)} — the canonical + * "fold the whole trivia run" shape. The outer repetition is dropped and + * each alternative is absorbed individually; the lexer's own maximal-munch + * loop performs the iteration, emitting one token per alternative match + * with the correct kind.
    • + *
  • {@code ZeroOrMore(expr)} / {@code OneOrMore(expr)} where {@code expr} is + * not a Choice — treated as a 1-alternative choice.
    • + *
  • a bare {@code Choice} or single expression — treated directly.
    • + *
+ * + *

Each absorbed alternative is forced to match at least one character + * (a bare whitespace char-class alternative is wrapped as one-or-more) so the + * DFA start state never accepts the empty string — the unsplit form therefore + * does NOT trigger the empty-match warning of the folded-{@code *} shape. + * + *

Falls back to the previous whole-body {@link #tryAbsorb} (with kind + * {@code KIND_WHITESPACE} and the post-match prefix sniff) only when the body + * does not fit any recognised shape — preserving behaviour for exotic + * {@code %whitespace} bodies. + */ + private static void absorbWhitespace(Nfa nfa, + Expression whitespaceBody, + int[] priorityRef, + int globalStart, + List skipped) { + var alternatives = whitespaceAlternatives(whitespaceBody); + if ( alternatives.isEmpty()) { + tryAbsorb(nfa, "%whitespace", whitespaceBody, KIND_WHITESPACE, priorityRef, globalStart, skipped); + return; + } + int absorbed = 0; + for ( var alt : alternatives) { + int kind = classifyWhitespaceAlternativeKind(alt); + var oneOrMore = ensureNonEmptyWhitespaceAlternative(alt); + var result = compileExpression(nfa, oneOrMore, "%whitespace"); + if ( !result.isSuccess()) { + continue;} + var fragment = result.unwrap(); + nfa.addEpsilon(globalStart, fragment.start); + nfa.markAccept(fragment.accept, kind, priorityRef[0]); + priorityRef[0]++; + absorbed++; + } + // If no alternative compiled (e.g. all used unsupported constructs), fall + // back to the legacy whole-body path so whitespace lexing still works. + if ( absorbed == 0) { + tryAbsorb(nfa, "%whitespace", whitespaceBody, KIND_WHITESPACE, priorityRef, globalStart, skipped);} + } + + /** + * Return the top-level alternatives of a {@code %whitespace} body for + * per-kind absorption, or an empty list when the body does not fit a + * recognised shape (signalling the caller to use the legacy whole-body path). + * The outer {@code ZeroOrMore}/{@code OneOrMore} wrapper is stripped; the + * lexer's maximal-munch loop supplies the repetition. + */ + private static List whitespaceAlternatives(Expression body) { + var unwrapped = unwrapAcceptableWrappers(body); + Expression inner; + if ( unwrapped instanceof Expression.ZeroOrMore zom) { + inner = unwrapAcceptableWrappers(zom.expression());} else + if ( unwrapped instanceof Expression.OneOrMore oom) { + inner = unwrapAcceptableWrappers(oom.expression());} else + { + inner = unwrapped;} + if ( inner instanceof Expression.Choice choice) { + return List.copyOf(choice.alternatives());} + return List.of(inner); + } + + /** + * Decide the {@link TokenArray}-equivalent trivia kind for a single + * {@code %whitespace} alternative by inspecting its leading literal. Mirrors + * the longest-prefix precedence used by the split-rule + post-match-sniff + * path: {@code /**} (doc block) before {@code /*} (block); {@code ///} (doc + * line) before {@code //} (line). Anything else — including a leading + * whitespace char-class — maps to {@link #KIND_WHITESPACE}. + */ + private static int classifyWhitespaceAlternativeKind(Expression alternative) { + var prefix = leadingLiteralText(alternative); + if ( prefix.startsWith("/**")) { + return KIND_DOC_BLOCK_COMMENT;} + if ( prefix.startsWith("/*")) { + return KIND_BLOCK_COMMENT;} + if ( prefix.startsWith("///")) { + return KIND_DOC_LINE_COMMENT;} + if ( prefix.startsWith("//")) { + return KIND_LINE_COMMENT;} + return KIND_WHITESPACE; + } + + /** + * Extract the leading literal text of an alternative: the text of its first + * {@link Expression.Literal} element (when the alternative is a Sequence) or + * the alternative itself (when it is a bare Literal). Returns the empty + * string for any other shape (e.g. a leading char class). + */ + private static String leadingLiteralText(Expression alternative) { + var unwrapped = unwrapAcceptableWrappers(alternative); + if ( unwrapped instanceof Expression.Literal lit) { + return lit.text();} + if ( unwrapped instanceof Expression.Sequence seq && !seq.elements().isEmpty()) { + var first = unwrapAcceptableWrappers(seq.elements().get(0)); + if ( first instanceof Expression.Literal lit) { + return lit.text();} + } + return ""; + } + + /** + * Ensure a whitespace alternative matches at least one character so that no + * absorbed fragment makes the DFA start state accept the empty string. A + * bare char-class alternative (e.g. {@code [ \t\r\n]}) is wrapped as + * one-or-more so a maximal whitespace run is still consumed as one token; + * all other alternatives (literal-prefixed comment patterns) already match + * a non-empty opening delimiter and are returned unchanged. + */ + private static Expression ensureNonEmptyWhitespaceAlternative(Expression alternative) { + var unwrapped = unwrapAcceptableWrappers(alternative); + if ( unwrapped instanceof Expression.CharClass cc) { + return new Expression.OneOrMore(cc.span(), cc);} + return alternative; + } + private static void absorbLiteralFragment(Nfa nfa, InlineLiteral lit, int kind, diff --git a/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/LexerEngine.java b/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/LexerEngine.java index 4cf6964..258f7a1 100644 --- a/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/LexerEngine.java +++ b/peglib-core/src/main/java/org/pragmatica/peg/v6/lexer/LexerEngine.java @@ -128,15 +128,26 @@ public TokenArray lex(String input) { if ( override != null) { lastAcceptKind = override;} } - // Phase A.6 / 0.6.1 — content-based trivia classification. WHITESPACE tokens - // whose text begins with a comment prefix are reclassified into the matching - // (regular or doc) line/block-comment kind. Pure whitespace runs never start - // with '/', so this is a sound prefix check. - // // → LINE_COMMENT (also /+ that isn't doc; see below) + // Phase A.6 / 0.6.1 / 0.6.2 — content-based trivia refinement. Two + // entry conditions: + // 1. legacy fallback: a coalesced WHITESPACE token whose text begins + // with a comment prefix (folded-%whitespace path that did not go + // through structural per-alternative absorption); + // 2. structural base kind: DfaBuilder.absorbWhitespace already + // assigned LINE_COMMENT / BLOCK_COMMENT structurally, but the + // grammar's single alternative cannot distinguish the doc variant + // (/// vs //, /** vs /*) — only the matched text can. + // In both cases the matched span text is inspected to pick the (regular + // or doc) line/block-comment kind. Pure whitespace runs never start + // with '/', so the WHITESPACE branch is a sound prefix check. + // // → LINE_COMMENT // /// → DOC_LINE_COMMENT (3 or more slashes) // /* ... */ → BLOCK_COMMENT // /** ... */ → DOC_BLOCK_COMMENT (NOT the smallest empty block /**/) - if ( lastAcceptKind == TokenArray.KIND_WHITESPACE && lastAcceptEnd > pos + 1) { + if ( (lastAcceptKind == TokenArray.KIND_WHITESPACE + || lastAcceptKind == TokenArray.KIND_LINE_COMMENT + || lastAcceptKind == TokenArray.KIND_BLOCK_COMMENT) + && lastAcceptEnd > pos + 1) { char c0 = input.charAt(pos); char c1 = input.charAt(pos + 1); if ( c0 == '/') { diff --git a/peglib-core/src/test/java/org/pragmatica/peg/v6/lexer/TriviaClassificationTest.java b/peglib-core/src/test/java/org/pragmatica/peg/v6/lexer/TriviaClassificationTest.java index ec189f6..01b8441 100644 --- a/peglib-core/src/test/java/org/pragmatica/peg/v6/lexer/TriviaClassificationTest.java +++ b/peglib-core/src/test/java/org/pragmatica/peg/v6/lexer/TriviaClassificationTest.java @@ -1,9 +1,9 @@ package org.pragmatica.peg.v6.lexer; import org.pragmatica.peg.grammar.GrammarParser; +import org.pragmatica.peg.v6.generator.LexerGenerator; import org.pragmatica.peg.v6.token.TokenArray; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -109,7 +109,6 @@ void lineComment_isClassifiedAsLineComment() { .isEqualTo(input); } - @Disabled("Block-comment alternative inside Choice doesn't route through compileDelimitedBlock; lexer cannot lex `/*...*/` from this grammar shape. Fix in a future task.") @Test void blockComment_isClassifiedAsBlockComment() { var engine = engineFor(GRAMMAR_WITH_COMMENTS); @@ -127,7 +126,6 @@ void blockComment_isClassifiedAsBlockComment() { .isEqualTo(input); } - @Disabled("Block-comment alternative inside Choice doesn't lex; defer until lexer driver supports per-iteration trivia tokens.") @Test void mixedContent_classifiesEachTriviaTokenByItsPrefix() { var engine = engineFor(GRAMMAR_WITH_COMMENTS); @@ -326,4 +324,145 @@ void docKinds_areTrivia() { .as("DOC_BLOCK_COMMENT is trivia") .isTrue(); } + + // ---- 0.6.2 — unsplit (folded) %whitespace per-kind absorption ------------- + // + // The unsplit form `([ \t\r\n] / '//' [^\n]* / '/*' (!'*/' .)* '*/')*` used to + // coalesce an entire mixed-trivia run into ONE KIND_WHITESPACE token. 0.6.2's + // DfaBuilder.absorbWhitespace absorbs each Choice alternative at its own trivia + // kind so the lexer emits one correctly-classified token per chunk — producing + // a token stream byte-identical to the split form. + + /** Unsplit folded form — the shape this task makes work. */ + private static final String GRAMMAR_UNSPLIT = """ + Word <- [a-zA-Z]+ + %whitespace <- ([ \\t\\r\\n] / '//' [^\\n]* / '/*' (!'*/' .)* '*/')* + """; + + /** Split per-kind form — the c4169b6 workaround shape, used as the parity oracle. */ + private static final String GRAMMAR_SPLIT = """ + Word <- [a-zA-Z]+ + %whitespace <- [ \\t\\r\\n]+ / '//' [^\\n]* / '/*' (!'*/' .)* '*/' + """; + + private static void assertTokenArraysIdentical(TokenArray a, TokenArray b, String input) { + assertThat(a.count()) + .as("token count for input <%s>", input) + .isEqualTo(b.count()); + for (int i = 0; i < a.count(); i++ ) { + assertThat(a.kindAt(i)) + .as("kind of token %d for input <%s>", i, input) + .isEqualTo(b.kindAt(i)); + assertThat(a.startAt(i)) + .as("start of token %d for input <%s>", i, input) + .isEqualTo(b.startAt(i)); + assertThat(a.endAt(i)) + .as("end of token %d for input <%s>", i, input) + .isEqualTo(b.endAt(i)); + } + } + + private static void assertSplitUnsplitParity(String input) { + var unsplit = engineFor(GRAMMAR_UNSPLIT) + .lex(input); + var split = engineFor(GRAMMAR_SPLIT) + .lex(input); + assertTokenArraysIdentical(unsplit, split, input); + assertThat(reconstruct(unsplit)) + .isEqualTo(input); + } + + @Test + void unsplitWhitespace_producesIdenticalTokenArrayToSplit_mixedTrivia() { + assertSplitUnsplitParity(" // hi\n/* blk */ hello"); + } + + @Test + void unsplitWhitespace_producesIdenticalTokenArrayToSplit_docVariants() { + assertSplitUnsplitParity("/// doc\n/** doc */ x"); + } + + @Test + void unsplitWhitespace_producesIdenticalTokenArrayToSplit_consecutiveBlockComments() { + assertSplitUnsplitParity("/*a*//*b*/"); + } + + @Test + void unsplitWhitespace_producesIdenticalTokenArrayToSplit_consecutiveLineComments() { + assertSplitUnsplitParity("//x\n//y\n"); + } + + @Test + void unsplitWhitespace_classifiesEachTriviaTokenByKind() { + var tokens = engineFor(GRAMMAR_UNSPLIT) + .lex(" // hi\n/* blk */ x"); + // Expected: WS, LINE_COMMENT, WS(\n), BLOCK_COMMENT, WS, Word(x). + assertThat(countByKind(tokens, TokenArray.KIND_LINE_COMMENT)) + .as("one LINE_COMMENT") + .isEqualTo(1); + assertThat(countByKind(tokens, TokenArray.KIND_BLOCK_COMMENT)) + .as("one BLOCK_COMMENT") + .isEqualTo(1); + int nonTrivia = 0; + for (int i = 0; i < tokens.count(); i++ ) { + if (!tokens.isTrivia(i)) { + nonTrivia++ ; + } + } + assertThat(nonTrivia) + .as("only the Word token is non-trivia") + .isEqualTo(1); + assertThat(reconstruct(tokens)) + .isEqualTo(" // hi\n/* blk */ x"); + } + + @Test + void unsplitWhitespace_zeroTrivia_noEmptyTokensNoInfiniteLoop() { + var tokens = engineFor(GRAMMAR_UNSPLIT) + .lex("hello"); + assertThat(tokens.count()) + .isEqualTo(1); + assertThat(tokens.isTrivia(0)) + .isFalse(); + for (int i = 0; i < tokens.count(); i++ ) { + assertThat(tokens.endAt(i)) + .as("token %d is non-empty", i) + .isGreaterThan(tokens.startAt(i)); + } + assertThat(reconstruct(tokens)) + .isEqualTo("hello"); + } + + @Test + void unsplitWhitespace_roundTripsViaReconstruct_mixedTrivia() { + var input = " // hi\n/* blk */\n/// doc\n/** db */ end"; + var tokens = engineFor(GRAMMAR_UNSPLIT) + .lex(input); + assertThat(reconstruct(tokens)) + .isEqualTo(input); + } + + @Test + void unsplitWhitespace_doesNotEmitEmptyMatchWarning() { + assertThat(generationWarnings(GRAMMAR_UNSPLIT)) + .as("unsplit %whitespace must not trip the empty-match warning") + .isEmpty(); + } + + private static java.util.List generationWarnings(String grammarText) { + var grammar = GrammarParser.parse(grammarText) + .unwrap(); + var classification = RuleClassifier.classify(grammar) + .unwrap(); + var built = DfaBuilder.build(grammar, classification) + .unwrap(); + return LexerGenerator.generate(grammar, + classification, + built.dfa(), + built.kinds(), + "test.pkg", + "GLexer") + .unwrap() + .warnings(); + } } diff --git a/peglib-core/src/test/resources/java25.peg b/peglib-core/src/test/resources/java25.peg index 226d466..d479136 100644 --- a/peglib-core/src/test/resources/java25.peg +++ b/peglib-core/src/test/resources/java25.peg @@ -197,12 +197,13 @@ NumLit <- < '0' [xX] [0-9a-fA-F_]+ [lL]? > / < '0' [bB] [01_]+ [lL]? > / < [0-9] # Hard keywords only - contextual keywords (var, yield, record, sealed, non-sealed, permits, when, module) are handled by their specific rules Keyword <- ('abstract' / 'assert' / 'boolean' / 'break' / 'byte' / 'case' / 'catch' / 'char' / 'class' / 'const' / 'continue' / 'default' / 'double' / 'do' / 'else' / 'enum' / 'extends' / 'false' / 'finally' / 'final' / 'float' / 'for' / 'goto' / 'implements' / 'import' / 'instanceof' / 'interface' / 'int' / 'if' / 'long' / 'native' / 'new' / 'null' / 'package' / 'private' / 'protected' / 'public' / 'return' / 'short' / 'static' / 'strictfp' / 'super' / 'switch' / 'synchronized' / 'this' / 'throws' / 'throw' / 'transient' / 'true' / 'try' / 'void' / 'volatile' / 'while') ![a-zA-Z0-9_$] -# NB: each alternative consumes exactly one chunk; the lexer's outer -# maximal-munch loop iterates naturally, so v6 per-kind trivia classification -# (LINE_COMMENT, DOC_LINE_COMMENT via /// prefix, etc.) fires correctly. -# Wrapping this in an outer * would fold the whole run into a single -# WHITESPACE-kind token and defeat classification — see GRAMMAR-DSL.md. -%whitespace <- [ \t\r\n]+ / '//' [^\n]* / '/*' (!'*/' .)* '*/' +# 0.6.2 — the unsplit folded form is supported again: DfaBuilder.absorbWhitespace +# absorbs each Choice alternative at its own trivia kind (WHITESPACE / LINE_COMMENT +# / DOC_LINE_COMMENT / BLOCK_COMMENT / DOC_BLOCK_COMMENT), so the lexer's maximal- +# munch loop emits one correctly-classified token per trivia chunk. The whitespace +# alternative is absorbed as one-or-more, so the DFA start state never accepts the +# empty string (no empty-match warning). +%whitespace <- ([ \t\r\n] / '//' [^\n]* / '/*' (!'*/' .)* '*/')* # 0.6.1 — incremental-reparse boundaries consumed by IncrementalParser. # Each %checkpoint names a rule whose CST subtree may be re-parsed in From f3c627675bb9f201dcf91b19b8c518ff8bd78f03 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Sat, 6 Jun 2026 18:57:12 +0200 Subject: [PATCH 4/4] docs: update HANDOVER, README history, and stale figures for 0.6.2 --- README.md | 1 + docs/ARCHITECTURE-0.6.0.md | 2 +- docs/HANDOVER.md | 38 ++++++++++++++++++++++++++++++-------- docs/VISITOR-TUTORIAL.md | 2 +- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3ff7d7a..486acc2 100644 --- a/README.md +++ b/README.md @@ -343,6 +343,7 @@ Full history in [`CHANGELOG.md`](CHANGELOG.md). | Version | Date | What | |---|---|---| +| **0.6.2** | 2026-06-06 | Patch release. Shift operators (`<<`/`>>`/`>>>`) in field/local-var initializer context fixed via inline expansion of DFA-skipped lexer rules; loud `SkippedRuleReferenced` guard. Per-iteration `%whitespace` tokenization (folded form emits per-kind trivia; grammar-split workaround reverted). Selfhost fixture now parses with 0 diagnostics. | | **0.6.1** | 2026-05-12 | Patch release. Doc-comment trivia kinds (`KIND_DOC_LINE_COMMENT`, `KIND_DOC_BLOCK_COMMENT`), per-rule `%recover` runtime, `%checkpoint` directive parsing, named captures and back-references restored, `MIXED`-rule char-level fallback, diagnostic cap honored. | | **0.6.0** | 2026-05-11 | Clean-slate redesign. Tokens-first lex-then-parse, flat `int[]` CST, visitor pattern, always-on recovery, true partial reparse. ~12x faster than 0.5.x; parity-class with `javac`. **BREAKING** — see [migration guide](docs/MIGRATION-0.5-TO-0.6.md). | | **0.5.1** | 2026-05-08 | Final 0.5.x — selfhost stability and minor fixes. | diff --git a/docs/ARCHITECTURE-0.6.0.md b/docs/ARCHITECTURE-0.6.0.md index 8ef4045..27ef2a6 100644 --- a/docs/ARCHITECTURE-0.6.0.md +++ b/docs/ARCHITECTURE-0.6.0.md @@ -807,7 +807,7 @@ Reference machine: same Apple Silicon used for 0.5.x bench session. | First-call (cold compile) | n/a | **≤ 600 ms** (one-time) | — | | Subsequent parse (warm cache) | n/a | reference parse target | — | -For comparison: javac parses 1900-LOC Java in ~9 ms (parse-only mode). 0.6.0 target of ~10 ms for reference fixture brings peglib to **parity-with-or-faster-than javac** on Java parsing — while emitting strictly more output (CST + trivia tokens, vs javac's parse-only AST). +For comparison: measured javac parse-only for the 1900-LOC reference is ≈2.2 ms; peglib's warm reference parse (≈2.68 ms as of 0.6.2) is ≈1.2× javac — **parity-class with javac** on Java parsing while emitting strictly more output (CST + trivia tokens, vs javac's parse-only AST). --- diff --git a/docs/HANDOVER.md b/docs/HANDOVER.md index b86d8a7..d5e3ab1 100644 --- a/docs/HANDOVER.md +++ b/docs/HANDOVER.md @@ -1,6 +1,28 @@ # peglib — Handover -**Last updated:** 2026-05-14 — **0.6.1 SHIPPING to Maven Central.** Closing checkpoint after Session 6. +**Last updated:** 2026-06-06 — 0.6.2 release branch (release-0.6.2) + +--- + +## Session 7 — 0.6.2 (2026-06-06) + +### State at a glance + +| | | +|---|---| +| **Release** | **v0.6.2** on `release-0.6.2` (patch) | +| **Tests** | **1420 passing** across 7 modules, 0 failures, 2 pre-existing skips | +| **Java25 corpus** | 20/20 clean parse | +| **Selfhost (40K LOC)** | **0 diagnostics**, 1.26M CST nodes | +| **Real-world Java** | FactoryClassGenerator (1900 LOC): 0 diagnostics | +| **JMH** | reference 2.68 ms, selfhost 71.9 ms — flat vs 0.6.1 | + +### What landed in session 7 (0.6.2 patch items) + +1. **Shift operators (`<<`/`>>`/`>>>`) in field/local-var initializer context** — fixed. Root cause was NOT the 0.6.1 rollback hypothesis (`Type`/`Relational`/`TypeArgs` `<` rollback); it was a **dead token kind from a lexer rule silently skipped by the DFA**. Fix: inline expansion of DFA-skipped lexer rules of the `literal+ (!literal)*` shape, plus a loud `SkippedRuleReferenced` guard at `fromGrammar`. Both `Java25SelfHostDiagTest` assertions re-enabled; selfhost fixture (40K LOC) now parses with **0 diagnostics**, 1,261,302 CST nodes. +2. **Per-iteration `%whitespace` tokenization** — folded `%whitespace <- (...)*` now emits per-kind trivia tokens via per-alternative DFA absorption. The c4169b6 canonical-grammar split workaround is **REVERTED** in `java25.peg`; the empty-match warning is gone for the folded form; 2 previously-disabled trivia tests re-enabled. + +Skips moved 4 → 2 (remaining: `LexerGeneratorTest` parity 1, `TriviaAdversarialTest$OptionalCutFailurePending` 1). --- @@ -43,9 +65,9 @@ - BASIC/ADVANCED recovery split; inline `{...}` action blocks; `AstNode` type; packrat memoization **Deferred to 0.6.2 or 0.7**: -- **Shift-in-FieldDecl bug** (0.6.2 target): shift operators in field/local-var init context fail at `CompilationUnit` level. Hypothesis: `Type`/`Relational`/`TypeArgs` `<` literal rollback corrupts state. 2 `Java25SelfHostDiagTest` assertions `@Disabled` pending fix. -- **Per-iteration `%whitespace` tokenization** (Item A harder part): ZeroOrMore loop itself should drive per-iteration token emission rather than relying on grammar split. Deferred. -- **jbct v6 API migration** (0.6.2): jbct `46ac5e993` applied the `%whitespace` grammar split fix; full peglib v6 API migration remains. +- **Shift-in-FieldDecl bug** (0.6.2 target): shift operators in field/local-var init context fail at `CompilationUnit` level. Hypothesis: `Type`/`Relational`/`TypeArgs` `<` literal rollback corrupts state. 2 `Java25SelfHostDiagTest` assertions `@Disabled` pending fix. ✅ RESOLVED in 0.6.2 +- **Per-iteration `%whitespace` tokenization** (Item A harder part): ZeroOrMore loop itself should drive per-iteration token emission rather than relying on grammar split. Deferred. ✅ RESOLVED in 0.6.2 +- **jbct v6 API migration** (0.6.2): jbct `46ac5e993` applied the `%whitespace` grammar split fix; full peglib v6 API migration remains. (split workaround obsolete since 0.6.2 — folded form now emits per-kind trivia; full jbct v6 API migration still pending) - **JBCT `true`** in `peglib-core/pom.xml`: formatter convergence bug on 5 v6 files. Lint passes cleanly; tracking upstream. - **Items H (token pool), I (lexer modes), K (JBCT plugin bump)**: 0.7+ backlog. @@ -53,8 +75,8 @@ 1. **Read this file**, then `docs/ARCHITECTURE-0.6.0.md` if going architectural. 2. **Read `CLAUDE.md`** — banked lessons (parser-domain + collaboration notes). -3. **Verify**: `mvn install -Djbct.skip=true` → 1440 tests pass. -4. **0.6.2 target**: fix shift-in-FieldDecl bug (see `Java25SelfHostDiagTest` disabled assertions), then re-enable the gate. +3. **Verify**: `mvn install -Djbct.skip=true` → 1420 tests pass. +4. **0.6.2 target**: fix shift-in-FieldDecl bug (see `Java25SelfHostDiagTest` disabled assertions), then re-enable the gate. (done in 0.6.2) 5. **Agents**: `jbct-coder` for ALL coding, `build-runner` for `mvn`, `chore-runner` for git/changelog. --- @@ -108,7 +130,7 @@ - Per-iteration trivia tokens for `%whitespace` ZeroOrMore (currently coalesces into single token) - Named captures + back-references runtime (rejects at `fromGrammar` with helpful error) - JBCT `true` in `peglib-core/pom.xml` (lint passes cleanly; only the upstream formatter has a convergence bug on 5 v6 files) -- True partial parse on `selfhost` (40K LOC) fixture (parses with some diagnostics — grammar gaps in specific Java patterns) +- True partial parse on `selfhost` (40K LOC) fixture (parses with some diagnostics — grammar gaps in specific Java patterns) (resolved: 0 diagnostics as of 0.6.2) ### Possible next-session targets @@ -662,7 +684,7 @@ Reference machine: same Apple Silicon used for 0.5.x bench session. Numbers from | Incremental edit median (Regime B) | 5.0 ms | **≤ 3 ms** | ≤ 1 ms | | First-call (cold compile) | n/a | **≤ 600 ms** (one-time) | — | -For comparison: javac parses 1900-LOC Java in ~9 ms (parse-only). 0.6.0 reference target of ~10 ms = parity with javac while emitting strictly more output. +For comparison: measured javac parse-only for the 1900-LOC reference is ≈2.2 ms; peglib's warm reference parse (≈2.68 ms as of 0.6.2) is ≈1.2× javac while emitting strictly more output. --- diff --git a/docs/VISITOR-TUTORIAL.md b/docs/VISITOR-TUTORIAL.md index df5bde9..94be2f6 100644 --- a/docs/VISITOR-TUTORIAL.md +++ b/docs/VISITOR-TUTORIAL.md @@ -483,7 +483,7 @@ emitting either the original token text or your transformed version per node. for dropping actions in favor of `GVisitor`. - [`peglib-core/src/main/java/org/pragmatica/peg/v6/generator/VisitorGenerator.java`](../peglib-core/src/main/java/org/pragmatica/peg/v6/generator/VisitorGenerator.java) — the codegen source. Read it once; it's short. -- [`peglib-core/src/main/java/org/pragmatica/peg/v6/cst/CstArray.java`](../peglib-core/src/main/java/org/pragmatica/peg/v6/cst/CstArray.java) +- [`peglib-core/src/main/java/org/pragmatica/peg/v6/cst/CstArray.java`](../peglib-runtime/src/main/java/org/pragmatica/peg/v6/cst/CstArray.java) — full CST API surface: `children`, `descendants`, `viewAt`, `kindAt`, `kindNameAt`, `firstChildAt`, `nextSiblingAt`, `textAt`, `spanStart`, `spanEnd`, `leadingTriviaTokens`, `trailingTriviaTokens`.