Merge pull request #81 from Aaronontheweb/feature/native-fragment-classifier

Refactor native argument fragment classification

Author: Aaronontheweb

IMPLEMENTATION_PLAN.md

@@ -194,11 +194,19 @@ priorities.
       Executable corpus cases preserve the corrected v0.2 compatibility
       projection; unknown facts remain `DynamicSkip` or unparseable, while
       completely proved mixed fragments resolve exactly.
-- [ ] Implement [issue #69](https://github.com/Aaronontheweb/ShellSyntaxTree/issues/69)
-      against the corrected fragment contract. Preserve raw, decoded, and span
-      facts plus unaffected classifications; explicitly document only
-      shell-oracle-proved compatibility corrections to false exact, `Glob`,
-      `Tilde`, provider, or path claims and avoidable `DynamicSkip` results.
+- [x] Implement [issue #69](https://github.com/Aaronontheweb/ShellSyntaxTree/issues/69)
+      against the corrected fragment contract. One shell-neutral classifier
+      now aggregates the complete raw span, decoded value, next-token index,
+      and ordered `ShellValue` provenance supplied by explicit Bash and
+      PowerShell adapters. It distinguishes literal-only, typed expansion,
+      and opaque/computed runs without rescanning decoded text; missing lexer
+      provenance fails closed as `Opaque` in both adapters. Direct adapter
+      tests pin spans, maximal consumption, expansion identity, opaque cause,
+      and fallback behavior. The full Bash and PowerShell corpora prove the
+      extraction leaves raw, decoded, span, path, and `DynamicSkip` results
+      unchanged. It introduces no new compatibility correction; the
+      shell-oracle-proved corrections remain the ones documented in the
+      preceding provenance item.
 - [ ] Add the structural and command-occurrence projections for the existing
       grammar before enabling any control-flow construct.
 - [ ] Deliver Bash `for ... in` and PowerShell `foreach` as the first two

openspec/changes/v0-3-structured-shell-analysis/tasks.md

@@ -15,8 +15,8 @@
 
 - [x] 2.1 Implement shell-specific lexical fragment provenance that distinguishes literal, typed recognized-expansion, and opaque resolver input; retains transform eligibility, expansion identity, cardinality, and opaque cause; aggregates complete argument and redirect-target fragment runs; and passes explicit Bash-argument, Bash-redirect, PowerShell-native, cmdlet-Path, cmdlet-LiteralPath, and PowerShell-redirect resolver context without changing the public API.
 - [x] 2.2 Add paired Bash and PowerShell shell-oracle regressions for standalone escapes, adjacent escaped values, all-static mixed quoting, within-token escapes, genuine literal-plus-expandable values, adjacent and wildcard redirect targets, runtime special/positional/numeric/Unicode variables, incomplete and escaped-literal braced interpolation, Bash provider-looking literals, and PowerShell native-versus-cmdlet, Path-versus-LiteralPath, and redirect-context divergence; require exact compatibility path results when every fragment, binding fact, and required resolver fact is exact, otherwise fail closed.
-- [ ] 2.3 Implement issue #69's shell-neutral native argument-fragment classifier with explicit Bash and PowerShell adapters that preserve the new provenance.
-- [ ] 2.4 Prove raw spelling, decoded logical values, source spans, and unaffected classifications remain unchanged; document each oracle-proved false exact, `Glob`, `Tilde`, provider, path, or avoidable `DynamicSkip` compatibility correction.
+- [x] 2.3 Implement issue #69's shell-neutral native argument-fragment classifier with explicit Bash and PowerShell adapters that preserve the new provenance.
+- [x] 2.4 Prove raw spelling, decoded logical values, source spans, and unaffected classifications remain unchanged; document each oracle-proved false exact, `Glob`, `Tilde`, provider, path, or avoidable `DynamicSkip` compatibility correction.
 - [ ] 2.5 Audit duplicated Bash and PowerShell path-normalization helpers and extract only rules with identical shell semantics.
 - [ ] 2.6 Run Release build, full tests, header verification, and the adversarial security corpus for the completed preparation.
 

src/ShellSyntaxTree/Internal/Bash/Parsing/BashCommandParser.cs

@@ -514,7 +514,8 @@ private static List<BashToken> FilterSignificant(IReadOnlyList<BashToken> tokens
             if (filtered.Count > 0
                 && IsNativeArgumentFragment(filtered[filtered.Count - 1])
                 && IsNativeArgumentFragment(t)
-                && IsAdjacent(filtered[filtered.Count - 1], t))
+                && IsAdjacent(filtered[filtered.Count - 1], t)
+                && !IsInlineNativeArgumentPrefix(filtered[filtered.Count - 1]))
             {
                 var previous = filtered[filtered.Count - 1];
                 var previousValue = previous.ResolverValue
@@ -1200,48 +1201,19 @@ private static void ExtractRedirectsAndArgs(
                         // `--data="@request file"` / `--data=$(generate)`.
                         if (NativeFlagSyntax.TrySplitEqualsPrefix(
                                 t.Value, out var adjacentFlagPart, out var adjacentValuePrefix)
-                            && i + 1 < segmentTokens.Count
-                            && IsAdjacent(t, segmentTokens[i + 1])
-                            && segmentTokens[i + 1].Kind is BashTokenKind.QuotedString
-                                or BashTokenKind.OpaqueSubstitution)
+                            && NativeArgumentFragmentClassifier.TryClassify(
+                                source,
+                                t.SourceStart,
+                                t.SourceStart + t.SourceLength,
+                                t.SourceStart + sourceRaw.IndexOf('=') + 1,
+                                adjacentFlagPart + "=",
+                                GetResolverValue(t, adjacentValuePrefix),
+                                segmentTokens,
+                                i + 1,
+                                new BashNativeArgumentFragmentAdapter(),
+                                out var fragmentClassification))
                         {
-                            var valueStart = i + 1;
-                            var valueEnd = valueStart;
-                            var valueBuilder = new StringBuilder(adjacentValuePrefix);
-                            var hasOpaqueFragment = false;
-                            var allFragmentsSingleQuoted = adjacentValuePrefix.Length == 0;
-                            var hasSingleQuotedFragment = false;
-                            var hasNonSingleQuotedFragment = adjacentValuePrefix.Length > 0;
-                            var hasSensitiveLiteralFragment = false;
-                            var previousFragment = t;
-                            while (valueEnd < segmentTokens.Count
-                                && IsAdjacent(previousFragment, segmentTokens[valueEnd])
-                                && IsNativeArgumentFragment(segmentTokens[valueEnd]))
-                            {
-                                var fragment = segmentTokens[valueEnd];
-                                valueBuilder.Append(fragment.Value);
-                                hasOpaqueFragment |= fragment.Kind == BashTokenKind.OpaqueSubstitution;
-                                hasSingleQuotedFragment |= fragment.Kind == BashTokenKind.QuotedString
-                                    && fragment.IsSingleQuoted;
-                                hasNonSingleQuotedFragment |= fragment.Kind != BashTokenKind.QuotedString
-                                    || !fragment.IsSingleQuoted;
-                                hasSensitiveLiteralFragment |= fragment.Kind == BashTokenKind.QuotedString
-                                    && fragment.IsSingleQuoted
-                                    && NativeFlagSyntax.ContainsResolverSensitiveLiteralSyntax(fragment.Value);
-                                allFragmentsSingleQuoted &= fragment.Kind == BashTokenKind.QuotedString
-                                    && fragment.IsSingleQuoted;
-                                previousFragment = fragment;
-                                valueEnd++;
-                            }
-
-                            var lastValueToken = segmentTokens[valueEnd - 1];
-                            var adjacentValue = valueBuilder.ToString();
-                            var equalsOffset = SourceSlice(source, t).IndexOf('=');
-                            var adjacentRawStart = t.SourceStart + equalsOffset + 1;
-                            var adjacentRaw = source.Substring(
-                                adjacentRawStart,
-                                lastValueToken.SourceStart + lastValueToken.SourceLength
-                                - adjacentRawStart);
+                            var adjacentValue = fragmentClassification.DecodedValue;
                             argList.Add(new Arg
                             {
                                 Raw = adjacentFlagPart,
@@ -1251,17 +1223,14 @@ private static void ExtractRedirectsAndArgs(
                             });
 
                             Arg valueArg;
-                            if (hasOpaqueFragment
-                                || (hasSingleQuotedFragment
-                                    && hasNonSingleQuotedFragment
-                                    && hasSensitiveLiteralFragment)
+                            if (fragmentClassification.HasOpaqueFragment
                                 || (verbKeyForFlagValuePaths is not null
                                     && BashPerVerbRules.ValueOfFlagIsOpaqueCommand(
                                         verbKeyForFlagValuePaths, adjacentFlagPart)))
                             {
                                 valueArg = new Arg
                                 {
-                                    Raw = adjacentRaw,
+                                    Raw = fragmentClassification.ValueRaw,
                                     Kind = ArgKind.DynamicSkip,
                                     IsPath = false,
                                 };
@@ -1276,7 +1245,7 @@ private static void ExtractRedirectsAndArgs(
                                         adjacentValue,
                                         out adjacentValueForResolution);
                                 var adjacentResolverValue = GetResolverValue(
-                                    t,
+                                    fragmentClassification.ResolverValue,
                                     adjacentValueForResolution);
                                 var (adjacentKind, adjacentResolved, adjacentIsPath) = BashResolver.Resolve(
                                     adjacentResolverValue,
@@ -1286,7 +1255,7 @@ private static void ExtractRedirectsAndArgs(
                                     ShellResolutionConsumer.BashArgument);
                                 valueArg = new Arg
                                 {
-                                    Raw = adjacentRaw,
+                                    Raw = fragmentClassification.ValueRaw,
                                     Resolved = adjacentResolved,
                                     Kind = adjacentKind,
                                     IsPath = adjacentIsPath,
@@ -1295,16 +1264,13 @@ private static void ExtractRedirectsAndArgs(
 
                             argList.Add(valueArg);
                             elementList.Add(CreateCombinedElement(
-                                source,
-                                t,
-                                lastValueToken,
-                                adjacentFlagPart + "=" + adjacentValue,
+                                fragmentClassification,
                                 precedingVerbTokenCount,
                                 valueArg.Kind,
                                 isFlag: true,
                                 valueArg.IsPath,
                                 valueArg.Resolved));
-                            i = valueEnd;
+                            i = fragmentClassification.NextTokenIndex;
                             continue;
                         }
 
@@ -1695,34 +1661,6 @@ private static ClauseElement CreateElement(
             Resolved = resolved,
         };
 
-    private static ClauseElement CreateCombinedElement(
-        string source,
-        BashToken first,
-        BashToken last,
-        string value,
-        int precedingVerbTokenCount,
-        ArgKind kind,
-        bool isFlag,
-        bool isPath,
-        string? resolved)
-    {
-        var sourceStart = first.SourceStart;
-        var sourceEnd = last.SourceStart + last.SourceLength;
-        return new ClauseElement
-        {
-            Raw = source.Substring(sourceStart, sourceEnd - sourceStart),
-            Value = value,
-            Role = ClauseElementRole.Argument,
-            SourceStart = sourceStart,
-            SourceLength = sourceEnd - sourceStart,
-            PrecedingVerbElementCount = precedingVerbTokenCount,
-            Kind = kind,
-            IsFlag = isFlag,
-            IsPath = isPath,
-            Resolved = resolved,
-        };
-    }
-
     private static ClauseElement CreateRedirectElement(
         string source,
         BashToken redirectOperator,
@@ -1753,6 +1691,11 @@ private static ShellValue GetResolverValue(BashToken token, string logicalValue)
     {
         var value = token.ResolverValue
             ?? ShellValue.Literal(token.Value, token.SourceStart, token.SourceLength);
+        return GetResolverValue(value, logicalValue);
+    }
+
+    private static ShellValue GetResolverValue(ShellValue value, string logicalValue)
+    {
         if (string.Equals(value.Decoded, logicalValue, StringComparison.Ordinal))
         {
             return value;
@@ -1768,6 +1711,26 @@ private static ShellValue GetResolverValue(BashToken token, string logicalValue)
         return ShellValue.Opaque(logicalValue, ShellOpaqueCause.Unsupported);
     }
 
+    private static ClauseElement CreateCombinedElement(
+        NativeArgumentFragmentClassification classification,
+        int precedingVerbTokenCount,
+        ArgKind kind,
+        bool isFlag,
+        bool isPath,
+        string? resolved) => new()
+        {
+            Raw = classification.Raw,
+            Value = classification.DecodedArgument,
+            Role = ClauseElementRole.Argument,
+            SourceStart = classification.SourceStart,
+            SourceLength = classification.SourceLength,
+            PrecedingVerbElementCount = precedingVerbTokenCount,
+            Kind = kind,
+            IsFlag = isFlag,
+            IsPath = isPath,
+            Resolved = resolved,
+        };
+
     private static bool TrySplitInlineFlag(
         BashToken token, out string flagPart, out string valuePart)
     {
@@ -1799,6 +1762,10 @@ token.Kind is BashTokenKind.Word
             or BashTokenKind.QuotedString
             or BashTokenKind.OpaqueSubstitution;
 
+    private static bool IsInlineNativeArgumentPrefix(BashToken token) =>
+        token.Kind == BashTokenKind.Word
+        && NativeFlagSyntax.TrySplitEqualsPrefix(token.Value, out _, out _);
+
     private static bool IsFdDupTarget(string value)
     {
         // Recognized shapes (POSIX `[n]>&word` / `[n]<&word`):

src/ShellSyntaxTree/Internal/Bash/Parsing/BashNativeArgumentFragmentAdapter.cs

@@ -0,0 +1,44 @@
+// -----------------------------------------------------------------------
+// <copyright file="BashNativeArgumentFragmentAdapter.cs" company="Aaron Stannard">
+//      Copyright (C) 2026 - 2026 Aaron Stannard <https://github.com/Aaronontheweb>
+// </copyright>
+// -----------------------------------------------------------------------
+using ShellSyntaxTree.Internal.Bash.Lexing;
+using ShellSyntaxTree.Internal.Parsing;
+using ShellSyntaxTree.Internal.Resolving;
+
+namespace ShellSyntaxTree.Internal.Bash.Parsing;
+
+internal readonly struct BashNativeArgumentFragmentAdapter
+    : INativeArgumentFragmentAdapter<BashToken>
+{
+    public bool CanStart(BashToken token) => token.Kind is
+        BashTokenKind.Word
+        or BashTokenKind.QuotedString
+        or BashTokenKind.OpaqueSubstitution;
+
+    public bool TryAdapt(BashToken token, out NativeArgumentFragment fragment)
+    {
+        if (token.Kind is not (BashTokenKind.Word
+            or BashTokenKind.QuotedString
+            or BashTokenKind.OpaqueSubstitution))
+        {
+            fragment = default;
+            return false;
+        }
+
+        var value = token.ResolverValue
+            ?? ShellValue.Opaque(
+                token.Value,
+                token.Kind == BashTokenKind.OpaqueSubstitution
+                    ? ShellOpaqueCause.CommandSubstitution
+                    : ShellOpaqueCause.Unsupported,
+                token.SourceStart,
+                token.SourceLength);
+        fragment = new NativeArgumentFragment(
+            value,
+            token.SourceStart,
+            token.SourceLength);
+        return true;
+    }
+}