Specify resolver fragment provenance for v0.3 (#75)

Author: Aaronontheweb

IMPLEMENTATION_PLAN.md

@@ -151,8 +151,20 @@ priorities.
       synchronize the accepted shared and PowerShell contracts into
       `SPEC.md` / `SPEC.POWERSHELL.md` together with source and snapshot tests
       so the repository authority never intentionally drifts from the assembly.
+- [ ] Correct the lexer-to-resolver provenance boundary before issue #69.
+      Paired Bash and PowerShell shell-oracle cases must distinguish escaped
+      literal resolver syntax from expandable syntax even when both decode to
+      the same string, including standalone, adjacent-token, all-static
+      mixed-quote, within-token escape, and literal-plus-expandable cases.
+      Preserve ordered literal / expandable / opaque fragments internally;
+      require exact composition when every fragment and resolver fact is
+      exact, do not change the v0.2 public API, and never infer expansion from
+      decoded text.
 - [ ] Implement [issue #69](https://github.com/Aaronontheweb/ShellSyntaxTree/issues/69)
-      as the first behavior-preserving preparation after contract lock.
+      against the corrected fragment contract. Preserve raw, decoded, and span
+      facts plus unaffected classifications; explicitly document only
+      shell-oracle-proved compatibility corrections to false path claims and
+      avoidable `DynamicSkip` results.
 - [ ] Add the structural and command-occurrence projections for the existing
       grammar before enabling any control-flow construct.
 - [ ] Deliver Bash `for ... in` and PowerShell `foreach` as the first two

openspec/changes/v0-3-structured-shell-analysis/design.md

@@ -109,6 +109,64 @@ consumption, error recovery, quoting, and expression boundaries that already
 differ between the two shells. Shared components are composed as explicit
 classifiers and analysis passes instead.
 
+### Preserve resolver-relevant fragments through decoding
+
+The v0.2 lexer-to-resolver contract is too weak for a security parser. A
+decoded `string` plus `IsSingleQuoted` cannot distinguish Bash `\$HOME` from
+`$HOME`, or PowerShell `` `$HOME `` from `$HOME`. Both pairs decode to the same
+text, but the first value in each pair is literal and the second expands. The
+current resolver consequently reports a different filesystem path from the
+one the shell passes to the executable.
+
+Both shell front ends SHALL retain resolver-relevant value fragments through
+decoding. The internal representation is not public API, but it must preserve
+the ordered decoded text, exact source range when available, and one of these
+shell-owned dispositions for every fragment:
+
+- `Literal`: quoting or escaping suppresses resolver transformation;
+- `Expandable`: the selected shell permits the relevant transformation;
+- `Opaque`: the parser cannot prove the produced value.
+
+A representative internal shape is:
+
+```csharp
+internal enum ShellValueFragmentKind
+{
+    Literal,
+    Expandable,
+    Opaque,
+}
+
+internal readonly record struct ShellValueFragment(
+    string Value,
+    ShellValueFragmentKind Kind,
+    int? SourceStart,
+    int? SourceLength);
+
+internal readonly record struct ShellValue(
+    string Decoded,
+    IReadOnlyList<ShellValueFragment> Fragments);
+```
+
+This mock is non-normative: an equivalent boundary map or compact segment
+representation is acceptable. A single aggregate `IsLiteral` flag is not
+acceptable because one authored word can contain both expandable and escaped
+regions. The resolver transforms only eligible `Expandable` regions and
+preserves `Literal` regions byte-for-byte. When every fragment, supported
+transformation, and the required cwd or home fact is exact, it must compose and
+resolve the exact result even when literal and expandable fragments are mixed;
+it cannot choose `DynamicSkip` merely because retaining provenance requires
+more work. `DynamicSkip` is reserved for an `Opaque` fragment, an incomplete
+boundary, an unknown required fact, or an unsupported transformation that
+prevents an exact path claim. The resolver never infers expansion solely from
+the decoded string.
+
+Issue #69's shared native argument classifier consumes these proved fragments
+through explicit Bash and PowerShell adapters. It may own adjacency, decoded
+concatenation, and source-span aggregation, but the adapters retain shell
+quoting and escaping semantics. This correction is internal and leaves the
+v0.2 public leaf records and the locked additive v0.3 public API unchanged.
+
 ### Represent simple commands with existing Clause leaves
 
 A simple-command syntax node wraps the same `Clause` value exposed through the
@@ -227,14 +285,16 @@ declared completely analyzable.
 The implementation order is:
 
 1. Lock public types, compatibility behavior, completeness, and fixed bounds.
-2. Extract issue #69 and other behavior-preserving shared helpers.
-3. Produce `Syntax`, `Commands`, and unchanged `Clauses` for existing grammar.
-4. Validate the new consumer path on existing Netclaw cases.
-5. Add Bash `for ... in` with literal values first.
-6. Add PowerShell `foreach` with literal arrays next.
-7. Extract shared occurrence/value/state machinery proven by both slices.
-8. Add bounded patterns and iterator/substitution command discovery.
-9. Add condition loops and branches in separately testable shell-specific
+2. Correct resolver-fragment provenance with paired real-shell oracles.
+3. Extract issue #69 and other shared helpers against the corrected behavior.
+4. Produce `Syntax`, `Commands`, and retained compatibility `Clauses` for
+   existing grammar, including the explicit oracle-proved resolver corrections.
+5. Validate the new consumer path on existing Netclaw cases.
+6. Add Bash `for ... in` with literal values first.
+7. Add PowerShell `foreach` with literal arrays next.
+8. Extract shared occurrence/value/state machinery proven by both slices.
+9. Add bounded patterns and iterator/substitution command discovery.
+10. Add condition loops and branches in separately testable shell-specific
    slices.
 
 This order prevents a complete Bash implementation from hardening a
@@ -267,6 +327,9 @@ corpus remains sanitized under the existing PII audit.
 - **[Shared abstractions erase language semantics]** -> Keep lexers and
   structural parsers separate; extract only duplication demonstrated by both
   working slices.
+- **[Decoded values erase expansion provenance]** -> Retain ordered internal
+  literal, expandable, and opaque fragments; never let a resolver reconstruct
+  those facts from decoded text alone.
 - **[Partial trees invite partial authorization]** -> Keep
   `IsUnparseable=true`, return empty `Commands` and `Clauses`, and keep any
   partial syntax diagnostic-only.
@@ -283,12 +346,17 @@ corpus remains sanitized under the existing PII audit.
 2. Accept the OpenSpec and synchronize the locked API and grammar into
    `SPEC.md`, `SPEC.POWERSHELL.md`, `PROJECT_CONTEXT.md`, and
    `IMPLEMENTATION_PLAN.md`.
-3. Ship the new structural and occurrence API in a 0.3.0 alpha while all
-   existing grammar produces byte-for-byte equivalent compatibility facts.
-4. Migrate Netclaw to `Commands` and explicit redirect facts before enabling
+3. Correct the internal resolver-fragment contract and promote each paired
+   design case into the executable corpus before extracting issue #69.
+4. Ship the new structural and occurrence API in a 0.3.0 alpha while existing
+   grammar preserves raw spelling, decoded logical values, source spans, and
+   unaffected compatibility facts. Paired shell-oracle corrections to false
+   path or `DynamicSkip` claims are explicit compatibility notes, not hidden
+   behavior-preserving changes.
+5. Migrate Netclaw to `Commands` and explicit redirect facts before enabling
    supported control flow for authorization reuse.
-5. Add Bash and PowerShell vertical slices behind corpus and integration gates.
-6. Promote 0.3.0 only after both shells, old-consumer fail-closed behavior, and
+6. Add Bash and PowerShell vertical slices behind corpus and integration gates.
+7. Promote 0.3.0 only after both shells, old-consumer fail-closed behavior, and
    the new Netclaw consumer path pass their acceptance matrices.
 
 Before stable 0.3.0, a flawed new surface can be revised with prerelease

openspec/changes/v0-3-structured-shell-analysis/proposal.md

@@ -22,6 +22,9 @@ fail-closed behavior for incomplete analysis.
 - Add conservative value and shell-state analysis that distinguishes exact,
   finite, bounded-symbolic, and unknown facts without executing commands or
   enumerating the filesystem.
+- Preserve resolver-relevant lexical fragments through decoding so escaped or
+  quoted literal syntax cannot be mistaken for expandable syntax with the same
+  decoded text.
 - Add explicit redirect operation and target facts so consumers do not infer
   descriptor duplication, close, move, combined output, or dynamic targets
   from raw strings and `DynamicSkip` alone.
@@ -73,6 +76,12 @@ PowerShell specifications, and `docs/CONSUMER_GUIDE.md`. Adding properties to
 public records also changes generated equality, hashing, `ToString()`, and
 default serialization and therefore requires explicit migration notes.
 
+The implementation also corrects a v0.2 security defect at an internal
+boundary: decoded token text currently loses whether resolver-sensitive bytes
+were literal or expandable. The correction fixes false path claims and
+avoidable `DynamicSkip` results while retaining the shipped public API, raw
+spelling, decoded logical values, and source spans.
+
 Netclaw is the validating consumer. Its 0.25.4 redirect workaround remains the
 short-term containment; a v0.3 integration must switch authorization traversal
 to the command-occurrence projection and retain strict or prompt behavior for

openspec/changes/v0-3-structured-shell-analysis/specs/bounded-shell-analysis/spec.md

@@ -1,5 +1,66 @@
 ## ADDED Requirements
 
+### Requirement: Decoded values retain resolver provenance
+The parser SHALL retain enough shell-specific lexical provenance to distinguish
+resolver-sensitive literal, expandable, and opaque fragments after escape and
+quote decoding. It SHALL NOT infer whether text expands solely from the
+decoded string. Literal fragments SHALL remain literal and eligible expandable
+fragments SHALL be transformed under the existing bounded rules. When every
+fragment, supported transformation, and required cwd or home fact is exact,
+the parser SHALL compose the exact shell value and SHALL return the exact
+compatibility path result for a path position. It SHALL NOT return `DynamicSkip`
+solely because one value contains both literal and expandable fragments. An
+opaque or incompletely mapped fragment, unknown required fact, or unsupported
+transformation SHALL produce an unknown or `DynamicSkip` fact rather than a
+false path.
+
+The provenance representation is internal and SHALL NOT change the v0.2 public
+leaf API. Raw spelling, decoded logical values, and exact-or-null source spans
+retain their existing meanings.
+
+#### Scenario: Bash escaped variable is a literal path component
+- **WHEN** Bash parses `cat \$HOME` with an exact working directory
+- **THEN** the shell value is the literal `$HOME`
+- **THEN** the compatibility argument is a literal path resolved as `<cwd>/$HOME`
+- **THEN** it is not `DynamicSkip` and is not resolved as the configured home directory
+
+#### Scenario: PowerShell escaped variable is a literal path component
+- **WHEN** PowerShell parses ``Get-Content `$HOME`` with an exact working directory
+- **THEN** the shell value is the literal `$HOME`
+- **THEN** the compatibility argument is a literal path resolved as `<cwd>/$HOME`
+- **THEN** it is not `DynamicSkip` and is not resolved as the configured home directory
+
+#### Scenario: Escaped prefix composes with an adjacent quoted suffix
+- **WHEN** either shell parses its escaped-dollar spelling of `curl --data=@$HOME".json" URL`
+- **THEN** the complete native argument retains exact raw and decoded provenance
+- **THEN** curl's compatibility file operand is the literal path `<cwd>/$HOME.json`
+- **THEN** it is not `DynamicSkip` and is not a path under the configured home directory
+
+#### Scenario: All-static mixed quoting remains exact
+- **WHEN** either shell parses `curl --data='@$HOME'".json" URL`
+- **THEN** both adjacent fragments retain literal provenance
+- **THEN** the compatibility file operand is the literal path `<cwd>/$HOME.json`
+- **THEN** it is not `DynamicSkip`
+
+#### Scenario: Escape provenance survives inside one quoted token
+- **WHEN** either shell parses its escaped-dollar spelling of a quoted `$HOME.txt` path
+- **THEN** the literal dollar and the rest of the token compose exactly
+- **THEN** the compatibility argument resolves as `<cwd>/$HOME.txt`, not under the configured home directory
+
+#### Scenario: Literal and expandable regions compose inside one token
+- **WHEN** either shell parses its spelling of a quoted literal `${HOME}` followed by an expandable `$HOME`
+- **THEN** the first region remains literal and the second uses the configured home fact
+- **THEN** the exact composed shell value is retained rather than becoming `DynamicSkip`
+
+#### Scenario: Expandable variable remains expandable
+- **WHEN** either shell parses an unescaped expandable `$HOME` in a supported path position
+- **THEN** the resolver may use the configured home-directory fact
+- **THEN** the escaped and expandable spellings do not collapse to the same provenance
+
+#### Scenario: Opaque fragment remains fail closed
+- **WHEN** an adjacent native argument contains a command substitution, subexpression, splat, or another opaque fragment
+- **THEN** the parser does not synthesize an exact path from the remaining decoded text
+
 ### Requirement: Shell values use explicit proof domains
 The analysis SHALL classify a policy-relevant shell value as exact, finite,
 bounded symbolic pattern, or unknown, and SHALL NOT present a weaker proof as a

openspec/changes/v0-3-structured-shell-analysis/tasks.md

@@ -11,12 +11,14 @@
 - [x] 1.9 Add a paired Bash and PowerShell design corpus that records current behavior, desired structure, command occurrences, bounded values, redirect facts, compatibility projections, and security invariants.
 - [ ] 1.10 Promote each design case into the executable corpus as its production parser slice lands.
 
-## 2. Behavior-Preserving Shared Preparation
-
-- [ ] 2.1 Implement issue #69's shell-neutral native argument-fragment classifier with explicit Bash and PowerShell adapters.
-- [ ] 2.2 Prove all existing raw, decoded, span, path, and `DynamicSkip` results remain unchanged in both corpora.
-- [ ] 2.3 Audit duplicated Bash and PowerShell path-normalization helpers and extract only rules with identical shell semantics.
-- [ ] 2.4 Run Release build, full tests, and header verification for the behavior-preserving refactor.
+## 2. Resolver Provenance Correction and Shared Preparation
+
+- [ ] 2.1 Implement shell-specific lexical fragment provenance that distinguishes literal, expandable, and opaque resolver input without changing the public API.
+- [ ] 2.2 Add paired Bash and PowerShell shell-oracle regressions for standalone escapes, adjacent escaped values, all-static mixed quoting, within-token escapes, and genuine literal-plus-expandable values; require exact compatibility path results when every fragment and required resolver fact is exact, otherwise fail closed.
+- [ ] 2.3 Implement issue #69's shell-neutral native argument-fragment classifier with explicit Bash and PowerShell adapters that preserve the new provenance.
+- [ ] 2.4 Prove raw spelling, decoded logical values, source spans, and unaffected classifications remain unchanged; document each oracle-proved path or `DynamicSkip` compatibility correction.
+- [ ] 2.5 Audit duplicated Bash and PowerShell path-normalization helpers and extract only rules with identical shell semantics.
+- [ ] 2.6 Run Release build, full tests, header verification, and the adversarial security corpus for the completed preparation.
 
 ## 3. Structural and Projection Skeleton
 

tests/ShellSyntaxTree.Tests/DesignCorpus/README.md

@@ -36,3 +36,9 @@ publishing a truncated finite set. Supported heredocs and Bash here strings
 record data separately from executable substitutions and path-relevant
 redirects. Constructs deliberately deferred beyond stable v0.3 remain in the
 corpus as unparseable security boundaries.
+
+Resolver-provenance cases additionally pin selected current `Arg` fields. They
+record the v0.2 false path claim until its production slice lands, while the
+desired effective value records the literal shell value proved by the paired
+real-shell oracle. Those cases then move into the executable corpus instead of
+being treated as behavior that issue #69 must preserve.

tests/ShellSyntaxTree.Tests/DesignCorpus/V03DesignCorpusTests.cs

@@ -79,6 +79,18 @@ public void Current_expectations_match_the_v0_2_parsers()
                         actual.UnparseableReason ?? string.Empty,
                         StringComparison.OrdinalIgnoreCase);
                 }
+
+                if (designCase.Current.Argument is not null)
+                {
+                    var expected = designCase.Current.Argument;
+                    ValidateArgumentExpectation(designCase.Id, expected, actual.Clauses.Count);
+                    var clause = Assert.IsType<Clause>(actual.Clauses[expected.ClauseIndex]);
+                    var argument = Assert.IsType<Arg>(clause.Args[expected.ArgumentIndex]);
+                    Assert.Equal(expected.Raw, argument.Raw);
+                    Assert.Equal(expected.Kind, argument.Kind);
+                    Assert.Equal(expected.IsPath, argument.IsPath);
+                    Assert.Equal(expected.Resolved, argument.Resolved);
+                }
             }
         }
     }
@@ -177,6 +189,27 @@ or DesignRedirectOperation.CombinedOutput
         Assert.Equal(
             desired.Commands.Select(command => command.AuthoredVerb),
             desired.Compatibility.Verbs);
+
+        if (desired.Argument is not null)
+        {
+            ValidateArgumentExpectation(
+                designCase.Id,
+                desired.Argument,
+                desired.Compatibility.Verbs.Count);
+        }
+    }
+
+    private static void ValidateArgumentExpectation(
+        string caseId, ArgumentExpectation argument, int clauseCount)
+    {
+        Assert.InRange(argument.ClauseIndex, 0, clauseCount - 1);
+        Assert.True(argument.ArgumentIndex >= 0, $"{caseId}: argument index must be non-negative.");
+        Assert.False(string.IsNullOrWhiteSpace(argument.Raw));
+        if (argument.Resolved is not null)
+        {
+            Assert.True(argument.IsPath, $"{caseId}: a resolved argument must be path-relevant.");
+            Assert.NotEqual(ArgKind.DynamicSkip, argument.Kind);
+        }
     }
 
     private static void ValidateValue(
@@ -266,6 +299,23 @@ public sealed record CurrentBehaviorExpectation
     public bool IsUnparseable { get; init; }
 
     public string? ReasonContains { get; init; }
+
+    public ArgumentExpectation? Argument { get; init; }
+}
+
+public sealed record ArgumentExpectation
+{
+    public int ClauseIndex { get; init; }
+
+    public int ArgumentIndex { get; init; }
+
+    public string Raw { get; init; } = "";
+
+    public ArgKind Kind { get; init; }
+
+    public bool IsPath { get; init; }
+
+    public string? Resolved { get; init; }
 }
 
 public sealed record DesiredDesignExpectation
@@ -276,6 +326,8 @@ public sealed record DesiredDesignExpectation
 
     public IReadOnlyList<DesignCommandExpectation> Commands { get; init; } = [];
 
+    public ArgumentExpectation? Argument { get; init; }
+
     public CompatibilityExpectation Compatibility { get; init; } = new();
 
     public IReadOnlyList<SecurityInvariant> SecurityInvariants { get; init; } = [];
@@ -453,4 +505,5 @@ public enum SecurityInvariant
     ContextualKeywordNotControlFlow,
     ShellSpecificOptionSemantics,
     StaticRedirectNotDynamic,
+    LiteralExpansionProvenancePreserved,
 }