feat: parse rustdoc item pages (decl, fields, variants, methods)

internal/app/rustdoc.go

@@ -193,6 +193,74 @@ func cleanRustDocHeading(text string) string {
 	return strings.TrimSpace(text)
 }
 
+// cleanRustDocSignature normalizes an impl/method signature extracted from a
+// docs.rs code-header via RawInnerText. It drops the "ⓘ" notable-trait marker
+// and flattens the internal newlines/tabs (e.g. from a where clause) into a
+// single spaced line so the signature fits in an inline code span.
+func cleanRustDocSignature(text string) string {
+	text = strings.ReplaceAll(text, "ⓘ", "")
+	text = strings.ReplaceAll(text, "\n", " ")
+	text = strings.ReplaceAll(text, "\t", " ")
+	for strings.Contains(text, "  ") {
+		text = strings.ReplaceAll(text, "  ", " ")
+	}
+	return strings.TrimSpace(text)
+}
+
+// writeRustDocBlock renders the full content of a div.docblock (paragraphs,
+// code blocks, lists and headings) into builder, prefixing every line with
+// indent so it nests under the enclosing item bullet. This mirrors the content
+// types the top-level overview matchers handle, so method, enum-variant and
+// struct-field docs no longer silently drop examples, lists or headings.
+// Headings are emitted as bold text rather than "#" headings to avoid clashing
+// with the document's section structure.
+func writeRustDocBlock(builder *strings.Builder, docblock *html.Node, indent string) {
+	headerMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("h1,h2,h3,h4,h5,h6"),
+		func(n *html.Node) {
+			text := cleanRustDocHeading(dq.InnerText(n, true))
+			if text != "" {
+				builder.WriteString(fmt.Sprintf("%s**%s**\n", indent, text))
+			}
+		},
+	)
+	pMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("p"),
+		func(n *html.Node) {
+			text := strings.TrimSpace(dq.InnerText(n, true))
+			if text != "" {
+				builder.WriteString(fmt.Sprintf("%s%s\n", indent, text))
+			}
+		},
+	)
+	preMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("pre"),
+		func(n *html.Node) {
+			code := strings.TrimRight(dq.RawInnerText(n, true), "\n")
+			builder.WriteString(indent + "```\n")
+			for _, line := range strings.Split(code, "\n") {
+				builder.WriteString(indent + line + "\n")
+			}
+			builder.WriteString(indent + "```\n")
+		},
+	)
+	listMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("ul,ol"),
+		nil,
+		dq.NewNodeMatcher(
+			dq.NewMatchFunc("li"),
+			func(n *html.Node) {
+				text := strings.TrimSpace(dq.InnerText(n, true))
+				if text != "" {
+					builder.WriteString(fmt.Sprintf("%s- %s\n", indent, text))
+				}
+			},
+		),
+	)
+
+	dq.Traverse(docblock, []dq.Matcher{headerMatcher, pMatcher, preMatcher, listMatcher})
+}
+
 func parseDocsRsSearchResult(doc *html.Node) (bool, string) {
 	builder := strings.Builder{}
 
@@ -259,6 +327,15 @@ func parseDocsRsSearchResult(doc *html.Node) (bool, string) {
 func parseDocsRsDocument(doc *html.Node) (bool, string) {
 	builder := strings.Builder{}
 
+	// Struct fields (span.structfield) and their docs (a following div.docblock)
+	// are direct children of section#main-content with no wrapping container,
+	// unlike the overview docblock (nested in details.top-doc) and method
+	// docblocks (nested in the impl lists). inFields, toggled by the section
+	// headers, scopes the top-level div.docblock matcher to the Fields section so
+	// it doesn't double-emit those other docblocks. fieldOpen pairs each field
+	// signature with its optional following docblock.
+	var inFields, fieldOpen bool
+
 	// Title from div.main-heading > h1
 	titleMatcher := dq.NewNodeMatcher(
 		dq.NewMatchFunc("div.main-heading"),
@@ -272,6 +349,19 @@ func parseDocsRsDocument(doc *html.Node) (bool, string) {
 		),
 	)
 
+	// Declaration block: pre.item-decl holds the canonical signature/declaration
+	// on item pages (struct, enum, fn, trait, type alias). Present on every item
+	// page and absent on module index pages, so this is a no-op for the latter.
+	itemDeclMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("pre.item-decl"),
+		func(n *html.Node) {
+			decl := strings.TrimSpace(dq.RawInnerText(n, true))
+			if decl != "" {
+				builder.WriteString(fmt.Sprintf("```rust\n%s\n```\n", decl))
+			}
+		},
+	)
+
 	// Matchers for content inside div.docblock
 	docblockHeaderMatcher := dq.NewNodeMatcher(
 		dq.NewMatchFunc("h1,h2,h3,h4,h5,h6"),
@@ -332,6 +422,17 @@ func parseDocsRsDocument(doc *html.Node) (bool, string) {
 		dq.NewMatchFunc("h2.section-header"),
 		func(n *html.Node) {
 			text := cleanRustDocHeading(dq.InnerText(n, true))
+			// Track whether we're inside the Fields section so the top-level
+			// div.docblock matcher only fires for field docs. Non-exhaustive
+			// structs render the header as "Fields (Non-exhaustive)", so match on
+			// the prefix rather than the exact string.
+			inFields = strings.HasPrefix(text, "Fields")
+			// Skip headers whose item lists we deliberately don't expand, so item
+			// pages don't end with dangling empty sections.
+			switch text {
+			case "Auto Trait Implementations", "Blanket Implementations":
+				return
+			}
 			builder.WriteString(fmt.Sprintf("\n## %s\n", text))
 		},
 	)
@@ -361,6 +462,93 @@ func parseDocsRsDocument(doc *html.Node) (bool, string) {
 		ddMatcher,
 	)
 
+	// Impl blocks on item pages: each impl carries an h3.code-header ("impl Foo")
+	// and its methods live in section.method > h4.code-header, each followed by a
+	// div.docblock. Scope to the inherent-impl and trait-impl lists; the auto-trait
+	// and blanket-impl lists are high-volume noise (Send/Sync/From<T>/…) and are
+	// intentionally skipped.
+	implHeaderMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("h3.code-header"),
+		func(n *html.Node) {
+			sig := cleanRustDocSignature(dq.RawInnerText(n, true))
+			if sig != "" {
+				builder.WriteString(fmt.Sprintf("\n### %s\n", sig))
+			}
+		},
+	)
+	methodSigMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("h4.code-header"),
+		func(n *html.Node) {
+			sig := cleanRustDocSignature(dq.RawInnerText(n, true))
+			if sig != "" {
+				builder.WriteString(fmt.Sprintf("- `%s`\n", sig))
+			}
+		},
+	)
+	methodDocMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("div.docblock"),
+		func(n *html.Node) {
+			writeRustDocBlock(&builder, n, "  ")
+		},
+	)
+	implsMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("div#implementations-list,div#trait-implementations-list"),
+		nil,
+		implHeaderMatcher,
+		methodSigMatcher,
+		methodDocMatcher,
+	)
+
+	// Enum variants: div.variants holds section.variant > h3.code-header (the
+	// variant signature, e.g. "Number(Number)") each followed by a sibling
+	// div.docblock. The "## Variants" header itself is already emitted by
+	// sectionHeaderMatcher. Reuse methodDocMatcher for the variant docs since the
+	// div.docblock > p shape is identical and the subtrees are disjoint.
+	variantSigMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("h3.code-header"),
+		func(n *html.Node) {
+			sig := cleanRustDocSignature(dq.RawInnerText(n, true))
+			if sig != "" {
+				builder.WriteString(fmt.Sprintf("- `%s`\n", sig))
+			}
+		},
+	)
+	variantsMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("div.variants"),
+		nil,
+		variantSigMatcher,
+		methodDocMatcher,
+	)
+
+	// Struct fields: span.structfield > code holds "name: Type"; a documented
+	// field is followed by a sibling div.docblock. Both are direct children of
+	// section#main-content, so fieldDocMatcher is gated on inFields (set by the
+	// Fields section header) to avoid double-emitting the overview/method docblocks.
+	structFieldMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("span.structfield"),
+		func(n *html.Node) {
+			code := dq.FindOne(n, "code")
+			if code == nil {
+				return
+			}
+			sig := cleanRustDocSignature(dq.RawInnerText(code, true))
+			if sig != "" {
+				builder.WriteString(fmt.Sprintf("- `%s`\n", sig))
+				fieldOpen = true
+			}
+		},
+	)
+	fieldDocMatcher := dq.NewNodeMatcher(
+		dq.NewMatchFunc("div.docblock"),
+		func(n *html.Node) {
+			if !inFields || !fieldOpen {
+				return
+			}
+			fieldOpen = false
+			writeRustDocBlock(&builder, n, "  ")
+		},
+	)
+
 	// Root: section#main-content
 	var matched bool
 	rootMatcher := dq.NewNodeMatcher(
@@ -369,9 +557,14 @@ func parseDocsRsDocument(doc *html.Node) (bool, string) {
 			matched = true
 		},
 		titleMatcher,
+		itemDeclMatcher,
 		overviewMatcher,
 		sectionHeaderMatcher,
 		itemTableMatcher,
+		structFieldMatcher,
+		fieldDocMatcher,
+		variantsMatcher,
+		implsMatcher,
 	)
 
 	dq.Traverse(doc, []dq.Matcher{rootMatcher})

internal/app/rustdoc_golden_test.go

@@ -24,6 +24,22 @@ func TestGolden_parseDocsRsDocument(t *testing.T) {
 		{"serde", "rustdoc_doc_serde.html", "rustdoc_doc_serde.golden"},
 		{"tokio", "rustdoc_doc_tokio.html", "rustdoc_doc_tokio.golden"},
 		{"regex", "rustdoc_doc_regex.html", "rustdoc_doc_regex.golden"},
+		{"serdejson_map", "rustdoc_doc_serdejson_map.html", "rustdoc_doc_serdejson_map.golden"},
+		{
+			"httparse_request",
+			"rustdoc_doc_httparse_request.html",
+			"rustdoc_doc_httparse_request.golden",
+		},
+		{
+			"serdejson_value",
+			"rustdoc_doc_serdejson_value.html",
+			"rustdoc_doc_serdejson_value.golden",
+		},
+		{
+			"serdejson_tostring",
+			"rustdoc_doc_serdejson_tostring.html",
+			"rustdoc_doc_serdejson_tostring.golden",
+		},
 	}
 
 	for _, tt := range tests {

internal/app/testdata/rustdoc_doc_httparse_request.golden

@@ -0,0 +1,68 @@
+# Struct Request
+```rust
+pub struct Request<'headers, 'buf> {
+    pub method: Option<&'buf str>,
+    pub path: Option<&'buf str>,
+    pub version: Option<u8>,
+    pub headers: &'headers mut [Header<'buf>],
+}
+```
+A parsed Request.
+The optional values will be None if a parse was not complete, and did not
+parse the associated property. This allows you to inspect the parts that
+could be parsed, before reading more, in case you wish to exit early.
+
+## Example
+```
+let buf = b"GET /404 HTTP/1.1\r\nHost:";
+let mut headers = [httparse::EMPTY_HEADER; 16];
+let mut req = httparse::Request::new(&mut headers);
+let res = req.parse(buf).unwrap();
+if res.is_partial() {
+    match req.path {
+        Some(ref path) => {
+            // check router for path.
+            // /404 doesn't exist? we could stop parsing
+        },
+        None => {
+            // must read more and parse again
+        }
+    }
+}
+```
+
+## Fields
+- `method: Option<&'buf str>`
+  The request method, such as GET .
+- `path: Option<&'buf str>`
+  The request path, such as /about-us .
+- `version: Option<u8>`
+  The request minor version, such as 1 for HTTP/1.1 .
+- `headers: &'headers mut [Header<'buf>]`
+  The request headers.
+
+## Implementations
+
+### impl<'h, 'b> Request<'h, 'b>
+- `pub fn new(headers: &'h mut [Header<'b>]) -> Request<'h, 'b>`
+  Creates a new Request, using a slice of headers you allocate.
+- `pub fn parse_with_uninit_headers( &mut self, buf: &'b [u8], headers: &'h mut [MaybeUninit<Header<'b>>], ) -> Result<usize>`
+  Try to parse a buffer of bytes into the Request,
+except use an uninitialized slice of Header s.
+  For more information, see parse
+- `pub fn parse(&mut self, buf: &'b [u8]) -> Result<usize>`
+  Try to parse a buffer of bytes into the Request.
+  Returns byte offset in buf to start of HTTP body.
+
+## Trait Implementations
+
+### impl<'headers, 'buf> Debug for Request<'headers, 'buf>
+- `fn fmt(&self, f: &mut Formatter<'_>) -> Result`
+
+### impl<'headers, 'buf> PartialEq for Request<'headers, 'buf>
+- `fn eq(&self, other: &Request<'headers, 'buf>) -> bool`
+- `fn ne(&self, other: &Rhs) -> bool`
+
+### impl<'headers, 'buf> Eq for Request<'headers, 'buf>
+
+### impl<'headers, 'buf> StructuralPartialEq for Request<'headers, 'buf>