Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
## Features

- **GitHub URL Support**: Accepts GitHub repository URLs directly (e.g., `https://github.com/user/repo`) and resolves them to `pkg.go.dev`.
- **Smart Defaults**: Automatically generates a sanitized output filename from the package name if not provided.
- **Source File Downloading**: Automatically identifies and downloads raw Go source files linked in the documentation, appending them to the output for full context.
- **Retry Logic**: Robust scraping with automatic retries and exponential backoff for transient network issues.
- **Recursive Scraping**: Automatically finds and scrapes sub-packages.
- **Markdown Conversion**: Converts documentation and READMEs into clean Markdown.
- **Sorted Output**: Organizes documentation alphabetically by package URL.
- **Library First**: Built as a reusable Go library with a CLI wrapper.
- **SOLID Architecture**: Refactored into modular components (Scraper, Converter, Source Handler) for better maintainability and extensibility.

## Installation

Expand Down Expand Up @@ -75,6 +76,14 @@ func main() {
}
```

## Tip: Using with NotebookLM

`flatten-go-doc` is an excellent companion for Google **NotebookLM**. By consolidating documentation and source code into a single Markdown file, you can:

1. **Full Library Context**: Upload the generated `.md` file as a source in NotebookLM.
2. **AI-Powered Guidance**: Ask NotebookLM to "explain how to use this library based on the source code" or "create a tutorial for [specific feature]".
3. **Code Synthesis**: Since the tool includes the actual `.go` source files, NotebookLM can reason about the implementation details that aren't always visible in the standard documentation, helping you write more accurate and idiomatic code.

## License

This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
43 changes: 43 additions & 0 deletions pkg/flattener/converter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package flattener

import (
"fmt"
"strings"

md "github.com/JohannesKaufmann/html-to-markdown"
"github.com/PuerkitoBio/goquery"
)

// ContentConverter handles HTML to Markdown conversion.
type ContentConverter struct {
mdConverter *md.Converter
}

// NewConverter creates a new ContentConverter.
func NewConverter() *ContentConverter {
converter := md.NewConverter("", true, nil)
converter.AddRules(
md.Rule{
Filter: []string{"details"},
Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string {
// Unwrap details, just return content
return &content
},
},
md.Rule{
Filter: []string{"summary"},
Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string {
// Make summary a header
text := strings.TrimSpace(content)
newContent := fmt.Sprintf("\n#### %s\n\n", text)
return &newContent
},
},
)
return &ContentConverter{mdConverter: converter}
}

// Convert converts a goquery selection to Markdown.
func (c *ContentConverter) Convert(selection *goquery.Selection) string {
return c.mdConverter.Convert(selection)
}
166 changes: 73 additions & 93 deletions pkg/flattener/flattener.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,37 +5,11 @@ import (
"sort"
"strings"
"sync"
"time"

md "github.com/JohannesKaufmann/html-to-markdown"
"github.com/PuerkitoBio/goquery"
"github.com/gocolly/colly/v2"
)

// PageResult represents a single scraped page's documentation.
type PageResult struct {
URL string
Content string
}

// Config holds configuration for the Flattener.
type Config struct {
UserAgent string
Parallelism int
RandomDelay time.Duration
AllowedDomains []string
}

// DefaultConfig returns a default configuration.
func DefaultConfig() Config {
return Config{
UserAgent: "Mozilla/5.0 (compatible; GeminiBot/1.0; +http://gemini.google.com)",
Parallelism: 2,
RandomDelay: 500 * time.Millisecond,
AllowedDomains: []string{"pkg.go.dev"},
}
}

// Flattener handles the scraping and conversion of documentation.
type Flattener struct {
config Config
Expand All @@ -52,76 +26,83 @@ func New(config Config) *Flattener {
func (f *Flattener) Flatten(rootURL string) ([]PageResult, error) {
rootURL = strings.TrimSuffix(rootURL, "/")

// Prepare the Markdown converter
converter := md.NewConverter("", true, nil)
converter.AddRules(
md.Rule{
Filter: []string{"details"},
Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string {
// Unwrap details, just return content
return &content
},
},
md.Rule{
Filter: []string{"summary"},
Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string {
// Make summary a header
text := strings.TrimSpace(content)
newContent := fmt.Sprintf("\n#### %s\n\n", text)
return &newContent
},
},
)
// Components
scraper, err := NewScraper(f.config)
if err != nil {
return nil, err
}
converter := NewConverter()

// Safe storage for results
var results []PageResult
var mu sync.Mutex
var wg sync.WaitGroup
visitedSources := make(map[string]bool)

// Initialize Colly
c := colly.NewCollector(
colly.AllowedDomains(f.config.AllowedDomains...),
colly.UserAgent(f.config.UserAgent),
colly.Async(true),
)

// Configure parallelism and delay
err := c.Limit(&colly.LimitRule{
DomainGlob: "*",
Parallelism: f.config.Parallelism,
RandomDelay: f.config.RandomDelay,
})
if err != nil {
return nil, fmt.Errorf("failed to set limit rule: %w", err)
}

// 1. Handle Documentation Content
c.OnHTML("main", func(e *colly.HTMLElement) {
// Handler for Main Content
scraper.Collector.OnHTML("main", func(e *colly.HTMLElement) {
docURL := e.Request.URL.String()

// Selectors updated to match pkg.go.dev structure
selection := e.DOM.Find(".UnitReadme, .Documentation-content")

// Remove noise
selection.Find(".Documentation-index").Remove()
selection.Find("script").Remove()
selection.Find("style").Remove()
selection.Find(".Documentation-exampleButtonsContainer").Remove()

markdown := converter.Convert(selection)

// Add a header for the file
header := fmt.Sprintf("# Package: %s\nInput URL: %s\n\n", docURL, docURL)

mu.Lock()
results = append(results, PageResult{
URL: docURL,
Content: header + markdown,
})
mu.Unlock()

// 1. Documentation & Readme
docSelection := e.DOM.Find(".UnitReadme, .Documentation-content")
if docSelection.Length() > 0 {
// Cleanup
docSelection.Find(".Documentation-index").Remove()
docSelection.Find("script").Remove()
docSelection.Find("style").Remove()
docSelection.Find(".Documentation-exampleButtonsContainer").Remove()

markdown := converter.Convert(docSelection)

// Add a header for the file
header := fmt.Sprintf("# Package: %s\nInput URL: %s\n\n", docURL, docURL)

mu.Lock()
results = append(results, PageResult{
URL: docURL,
Content: header + markdown,
})
mu.Unlock()

// Find Source File Links
e.DOM.Find(".UnitFiles-fileList li a").Each(func(_ int, s *goquery.Selection) {
href, exists := s.Attr("href")
if exists {
fullURL := e.Request.AbsoluteURL(href)

mu.Lock()
if visited := visitedSources[fullURL]; !visited && IsSourceLink(fullURL) {
visitedSources[fullURL] = true
mu.Unlock()

// Download source file
// We do this in a goroutine to not block the collector
wg.Add(1)
go func(u string) {
defer wg.Done()
content, err := DownloadSource(u, f.config.MaxRetries)
if err == nil {
header := fmt.Sprintf("# Source File: %s\nURL: %s\n\n```go\n", u, u)
footer := "\n```\n"

mu.Lock()
results = append(results, PageResult{
URL: u,
Content: header + content + footer,
})
mu.Unlock()
}
}(fullURL)
} else {
mu.Unlock()
}
}
})
}
})

// 2. Handle Subdirectories (Children links)
c.OnHTML("div.UnitDirectories table tr td a", func(e *colly.HTMLElement) {
// Handle Subdirectories (Children links)
scraper.Collector.OnHTML("div.UnitDirectories table tr td a", func(e *colly.HTMLElement) {
link := e.Attr("href")
absoluteURL := e.Request.AbsoluteURL(link)

Expand All @@ -132,16 +113,15 @@ func (f *Flattener) Flatten(rootURL string) ([]PageResult, error) {
})

// Start the scraping
if err := c.Visit(rootURL); err != nil {
if err := scraper.Visit(rootURL); err != nil {
return nil, fmt.Errorf("failed to start scraping: %w", err)
}

// Wait for all requests to finish
c.Wait()
scraper.Wait()
wg.Wait()

// Sort results by URL length (shortest first -> root) or alphabetically.
// Here we stick to alphabetical to correspond with original behavior's sort intention,
// but generally sorting by URL makes sense for a linear read.
// Sort results by URL
sort.Slice(results, func(i, j int) bool {
return results[i].URL < results[j].URL
})
Expand Down
42 changes: 31 additions & 11 deletions pkg/flattener/flattener_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ func TestFlatten(t *testing.T) {
<h1>Root Package</h1>
<p>This is the root readme.</p>
</div>

<div class="UnitFiles js-unitFiles">
<ul class="UnitFiles-fileList">
<li><a href="/pkg/file.go">file.go</a></li>
</ul>
</div>

<div class="UnitDirectories">
<table>
<tr>
Expand Down Expand Up @@ -54,6 +61,11 @@ func TestFlatten(t *testing.T) {
w.Write([]byte(html))
})

// Source file page (RAW)
mux.HandleFunc("/pkg/file.go", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("package pkg\n\nfunc Foo() {}"))
})

server := httptest.NewServer(mux)
defer server.Close()

Expand All @@ -69,6 +81,7 @@ func TestFlatten(t *testing.T) {
Parallelism: 1,
RandomDelay: 0, // No delay for tests
AllowedDomains: []string{u.Hostname()},
MaxRetries: 1,
}

flattener := New(config)
Expand All @@ -81,11 +94,15 @@ func TestFlatten(t *testing.T) {
}

// 5. Assertions
if len(results) != 2 {
t.Errorf("Expected 2 results, got %d", len(results))
// Expected: Root, Sub, SourceFile -> 3 results
if len(results) != 3 {
t.Errorf("Expected 3 results, got %d", len(results))
for _, r := range results {
t.Logf("Result: %s", r.URL)
}
}

// Helper to find result by URL suffix (since port changes)
// Helper to find result by URL suffix
findResult := func(suffix string) *PageResult {
for _, r := range results {
if strings.HasSuffix(r.URL, suffix) {
Expand All @@ -103,21 +120,24 @@ func TestFlatten(t *testing.T) {
if !strings.Contains(rootRes.Content, "# Root Package") {
t.Errorf("Root content missing header. Got:\n%s", rootRes.Content)
}
if !strings.Contains(rootRes.Content, "This is the root readme.") {
t.Errorf("Root content missing body. Got:\n%s", rootRes.Content)
}
}

// Check Sub
subRes := findResult("/pkg/sub")
if subRes == nil {
t.Error("Result for subpackage not found")
}

// Check Source File
sourceRes := findResult("/pkg/file.go")
if sourceRes == nil {
t.Error("Result for source file not found")
} else {
if !strings.Contains(subRes.Content, "## Subpackage Doc") {
t.Errorf("Sub content missing header. Got:\n%s", subRes.Content)
if !strings.Contains(sourceRes.Content, "func Foo() {}") {
t.Errorf("Source content missing code. Got:\n%s", sourceRes.Content)
}
if strings.Contains(subRes.Content, "Index (should be removed)") {
t.Error("Content cleaning failed: found .Documentation-index content")
if !strings.Contains(sourceRes.Content, "```go") {
t.Error("Source content missing code block formatting")
}
}
}
}
29 changes: 29 additions & 0 deletions pkg/flattener/models.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package flattener

import "time"

// PageResult represents a single scraped page's documentation.
type PageResult struct {
URL string
Content string
}

// Config holds configuration for the Flattener.
type Config struct {
UserAgent string
Parallelism int
RandomDelay time.Duration
AllowedDomains []string
MaxRetries int
}

// DefaultConfig returns a default configuration.
func DefaultConfig() Config {
return Config{
UserAgent: "Mozilla/5.0 (compatible; GeminiBot/1.0; +http://gemini.google.com)",
Parallelism: 2,
RandomDelay: 500 * time.Millisecond,
AllowedDomains: []string{"pkg.go.dev"},
MaxRetries: 3,
}
}
Loading