Skip to content

Commit f258ecd

Browse files
aksOpsaksOps
committed
perf: 3x faster pipeline — drop goquery, eliminate regex bottlenecks
Before: 11.8ms/page, 23K allocs, 2.75MB (goquery + 5 regex passes) After: 3.9ms/page, 16K allocs, 1.57MB (direct x/net/html + string ops) - Replace goquery with direct *html.Node tree walking - Single-pass stripNoise (was 4 separate tree walks) - Replace noisePattern regex with simple string matching - Replace optimizeMarkdown regex chain with line scanner (4.3ms → 0.1ms) - Single dependency: golang.org/x/net/html
1 parent fae80c0 commit f258ecd

2 files changed

Lines changed: 133 additions & 122 deletions

File tree

‎convert.go‎

Lines changed: 48 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -479,38 +479,61 @@ func cleanMarkdown(s string) string {
479479
return strings.TrimSpace(s)
480480
}
481481

482-
// Patterns for output optimization
483-
var (
484-
// [¶](#section-anchor) — pilcrow anchors used by Go docs, MDN, etc.
485-
pilcrowPattern = regexp.MustCompile(`\s*\[¶\]\([^)]*\)`)
486-
// [text](#fragment) — fragment-only self-links, keep just the text
487-
fragmentLinkPattern = regexp.MustCompile(`\[([^\]]+)\]\(#[^)]*\)`)
488-
// Lines that are only whitespace
489-
whitespaceLinePattern = regexp.MustCompile(`(?m)^[ \t]+$`)
490-
// 3+ consecutive blank lines → 2
491-
excessiveBlanks = regexp.MustCompile(`\n{3,}`)
492-
// Trailing whitespace on lines
493-
trailingWhitespace = regexp.MustCompile(`(?m)[ \t]+$`)
494-
)
495-
496482
// optimizeMarkdown post-processes markdown to minimize token usage for AI agents.
483+
// Single-pass line scanner — no regex.
497484
func optimizeMarkdown(s string) string {
498-
// Strip pilcrow anchors: [¶](#...)
499-
s = pilcrowPattern.ReplaceAllString(s, "")
485+
var buf strings.Builder
486+
buf.Grow(len(s))
487+
blankCount := 0
488+
489+
for _, line := range strings.Split(s, "\n") {
490+
// Strip trailing whitespace
491+
line = strings.TrimRight(line, " \t")
500492

501-
// Convert fragment-only links to plain text: [text](#frag) → text
502-
s = fragmentLinkPattern.ReplaceAllString(s, "$1")
493+
// Strip pilcrow anchors: [¶](#...)
494+
if idx := strings.Index(line, "[¶]"); idx >= 0 {
495+
if end := strings.Index(line[idx:], ")"); end >= 0 {
496+
line = strings.TrimRight(line[:idx], " ") + line[idx+end+1:]
497+
}
498+
}
503499

504-
// Strip trailing whitespace from lines
505-
s = trailingWhitespace.ReplaceAllString(s, "")
500+
// Convert fragment-only links: [text](#frag) → text
501+
line = stripFragmentLinks(line)
506502

507-
// Collapse whitespace-only lines to empty lines
508-
s = whitespaceLinePattern.ReplaceAllString(s, "")
503+
// Collapse blank lines (max 2 consecutive)
504+
if line == "" {
505+
blankCount++
506+
if blankCount <= 2 {
507+
buf.WriteByte('\n')
508+
}
509+
continue
510+
}
511+
blankCount = 0
512+
buf.WriteString(line)
513+
buf.WriteByte('\n')
514+
}
509515

510-
// Collapse 3+ blank lines to 2
511-
s = excessiveBlanks.ReplaceAllString(s, "\n\n")
516+
return strings.TrimSpace(buf.String())
517+
}
512518

513-
return strings.TrimSpace(s)
519+
// stripFragmentLinks converts [text](#fragment) → text in a line.
520+
func stripFragmentLinks(line string) string {
521+
for {
522+
start := strings.Index(line, "](#")
523+
if start < 0 {
524+
return line
525+
}
526+
open := strings.LastIndex(line[:start], "[")
527+
if open < 0 {
528+
return line
529+
}
530+
close := strings.Index(line[start:], ")")
531+
if close < 0 {
532+
return line
533+
}
534+
close += start
535+
line = line[:open] + line[open+1:start] + line[close+1:]
536+
}
514537
}
515538

516539
// estimateTokens gives a rough token count using the ~4 chars/token heuristic

‎extract.go‎

Lines changed: 85 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package main
22

33
import (
4-
"regexp"
54
"strings"
65

76
"golang.org/x/net/html"
@@ -17,8 +16,43 @@ var killTags = []string{
1716
// landmarkTags are structural tags that should be removed as noise.
1817
var landmarkTags = []string{"nav", "footer", "aside"}
1918

20-
// noisePattern matches class/id values that indicate non-content elements.
21-
var noisePattern = regexp.MustCompile(`(?i)(cookie|consent|gdpr|banner|popup|modal|overlay|newsletter|subscribe|social|share|sidebar|advertisement|ad-|promo|related-posts)`)
19+
// Pre-built sets for fast tag lookup.
20+
var killSet = func() map[string]bool {
21+
m := make(map[string]bool, len(killTags))
22+
for _, t := range killTags {
23+
m[t] = true
24+
}
25+
return m
26+
}()
27+
28+
var landmarkSet = func() map[string]bool {
29+
m := make(map[string]bool, len(landmarkTags))
30+
for _, t := range landmarkTags {
31+
m[t] = true
32+
}
33+
return m
34+
}()
35+
36+
// noiseKeywords are substrings in class/id that indicate non-content elements.
37+
var noiseKeywords = []string{
38+
"cookie", "consent", "gdpr", "banner", "popup", "modal", "overlay",
39+
"newsletter", "subscribe", "social", "share", "sidebar",
40+
"advertisement", "ad-", "promo", "related-posts",
41+
}
42+
43+
// isNoisy checks if a string contains any noise keyword (case-insensitive).
44+
func isNoisy(s string) bool {
45+
if s == "" {
46+
return false
47+
}
48+
lower := strings.ToLower(s)
49+
for _, kw := range noiseKeywords {
50+
if strings.Contains(lower, kw) {
51+
return true
52+
}
53+
}
54+
return false
55+
}
2256

2357
// removeNode detaches a node from its parent.
2458
func removeNode(n *html.Node) {
@@ -28,117 +62,71 @@ func removeNode(n *html.Node) {
2862
}
2963

3064
// stripNoise removes non-content elements from the document in-place.
65+
// Single-pass tree walk — collects all removable nodes, then detaches them.
3166
func stripNoise(doc *html.Node) {
32-
// Build a set for fast lookup of kill tags
33-
killSet := make(map[string]bool, len(killTags))
34-
for _, tag := range killTags {
35-
killSet[tag] = true
36-
}
37-
landmarkSet := make(map[string]bool, len(landmarkTags))
38-
for _, tag := range landmarkTags {
39-
landmarkSet[tag] = true
40-
}
41-
42-
// 1 & 2. Kill tags and landmark tags — collect then remove
4367
var toRemove []*html.Node
44-
var collectKill func(*html.Node)
45-
collectKill = func(n *html.Node) {
68+
69+
var walk func(*html.Node)
70+
walk = func(n *html.Node) {
4671
for c := n.FirstChild; c != nil; c = c.NextSibling {
47-
if c.Type == html.ElementNode {
48-
if killSet[c.Data] || landmarkSet[c.Data] {
49-
toRemove = append(toRemove, c)
50-
continue // don't recurse into removed subtrees
51-
}
72+
if c.Type == html.CommentNode {
73+
toRemove = append(toRemove, c)
74+
continue
75+
}
76+
if c.Type != html.ElementNode {
77+
walk(c)
78+
continue
5279
}
53-
collectKill(c)
54-
}
55-
}
56-
collectKill(doc)
57-
for _, n := range toRemove {
58-
removeNode(n)
59-
}
6080

61-
// 3. Kill elements with noisy class/id
62-
toRemove = toRemove[:0]
63-
var collectNoisy func(*html.Node)
64-
collectNoisy = func(n *html.Node) {
65-
for c := n.FirstChild; c != nil; c = c.NextSibling {
66-
if c.Type == html.ElementNode {
67-
class := getAttr(c, "class")
68-
id := getAttr(c, "id")
69-
if noisePattern.MatchString(class) || noisePattern.MatchString(id) {
70-
toRemove = append(toRemove, c)
71-
continue
72-
}
81+
tag := c.Data
82+
83+
// Kill tags + landmark tags
84+
if killSet[tag] || landmarkSet[tag] {
85+
toRemove = append(toRemove, c)
86+
continue
7387
}
74-
collectNoisy(c)
75-
}
76-
}
77-
collectNoisy(doc)
78-
for _, n := range toRemove {
79-
removeNode(n)
80-
}
8188

82-
// 4. Kill header elements that do NOT contain an h1
83-
toRemove = toRemove[:0]
84-
var collectHeaders func(*html.Node)
85-
collectHeaders = func(n *html.Node) {
86-
for c := n.FirstChild; c != nil; c = c.NextSibling {
87-
if c.Type == html.ElementNode && c.Data == "header" {
88-
if findFirst(c, "h1") == nil {
89-
toRemove = append(toRemove, c)
90-
continue
91-
}
89+
// Header without h1
90+
if tag == "header" && findFirst(c, "h1") == nil {
91+
toRemove = append(toRemove, c)
92+
continue
9293
}
93-
collectHeaders(c)
94-
}
95-
}
96-
collectHeaders(doc)
97-
for _, n := range toRemove {
98-
removeNode(n)
99-
}
10094

101-
// 5. Kill hidden elements (aria-hidden=true and display:none)
102-
toRemove = toRemove[:0]
103-
var collectHidden func(*html.Node)
104-
collectHidden = func(n *html.Node) {
105-
for c := n.FirstChild; c != nil; c = c.NextSibling {
106-
if c.Type == html.ElementNode {
107-
if hasAttr(c, "aria-hidden", "true") {
108-
toRemove = append(toRemove, c)
109-
continue
95+
// Hidden elements
96+
if hasAttr(c, "aria-hidden", "true") {
97+
toRemove = append(toRemove, c)
98+
continue
99+
}
100+
101+
// Noisy class/id
102+
remove := false
103+
for _, a := range c.Attr {
104+
if a.Key == "class" || a.Key == "id" {
105+
if isNoisy(a.Val) {
106+
remove = true
107+
break
108+
}
110109
}
111-
style := getAttr(c, "style")
112-
if style != "" {
113-
normalized := strings.ReplaceAll(style, " ", "")
114-
if strings.Contains(normalized, "display:none") {
115-
toRemove = append(toRemove, c)
116-
continue
110+
if a.Key == "style" {
111+
if strings.Contains(strings.ReplaceAll(a.Val, " ", ""), "display:none") {
112+
remove = true
113+
break
117114
}
118115
}
119116
}
120-
collectHidden(c)
117+
if remove {
118+
toRemove = append(toRemove, c)
119+
continue
120+
}
121+
122+
walk(c)
121123
}
122124
}
123-
collectHidden(doc)
125+
walk(doc)
126+
124127
for _, n := range toRemove {
125128
removeNode(n)
126129
}
127-
128-
// 6. Remove HTML comments
129-
var removeComments func(*html.Node)
130-
removeComments = func(n *html.Node) {
131-
var next *html.Node
132-
for c := n.FirstChild; c != nil; c = next {
133-
next = c.NextSibling
134-
if c.Type == html.CommentNode {
135-
n.RemoveChild(c)
136-
} else {
137-
removeComments(c)
138-
}
139-
}
140-
}
141-
removeComments(doc)
142130
}
143131

144132
// extractContent picks the main content node from the document.
@@ -193,7 +181,7 @@ func extractContent(doc *html.Node, domain string) *html.Node {
193181
func readabilityScore(n *html.Node) int {
194182
class := getAttr(n, "class")
195183
id := getAttr(n, "id")
196-
if noisePattern.MatchString(class) || noisePattern.MatchString(id) {
184+
if isNoisy(class) || isNoisy(id) {
197185
return 0
198186
}
199187

0 commit comments

Comments
 (0)