diff --git a/README.md b/README.md index 4c14f0c..360ff08 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,12 @@ ## Features - **GitHub URL Support**: Accepts GitHub repository URLs directly (e.g., `https://github.com/user/repo`) and resolves them to `pkg.go.dev`. -- **Smart Defaults**: Automatically generates a sanitized output filename from the package name if not provided. +- **Source File Downloading**: Automatically identifies and downloads raw Go source files linked in the documentation, appending them to the output for full context. +- **Retry Logic**: Robust scraping with automatic retries and exponential backoff for transient network issues. - **Recursive Scraping**: Automatically finds and scrapes sub-packages. - **Markdown Conversion**: Converts documentation and READMEs into clean Markdown. - **Sorted Output**: Organizes documentation alphabetically by package URL. -- **Library First**: Built as a reusable Go library with a CLI wrapper. +- **SOLID Architecture**: Refactored into modular components (Scraper, Converter, Source Handler) for better maintainability and extensibility. ## Installation @@ -75,6 +76,14 @@ func main() { } ``` +## Tip: Using with NotebookLM + +`flatten-go-doc` is an excellent companion for Google **NotebookLM**. By consolidating documentation and source code into a single Markdown file, you can: + +1. **Full Library Context**: Upload the generated `.md` file as a source in NotebookLM. +2. **AI-Powered Guidance**: Ask NotebookLM to "explain how to use this library based on the source code" or "create a tutorial for [specific feature]". +3. **Code Synthesis**: Since the tool includes the actual `.go` source files, NotebookLM can reason about the implementation details that aren't always visible in the standard documentation, helping you write more accurate and idiomatic code. + ## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/pkg/flattener/converter.go b/pkg/flattener/converter.go new file mode 100644 index 0000000..0d118b9 --- /dev/null +++ b/pkg/flattener/converter.go @@ -0,0 +1,43 @@ +package flattener + +import ( + "fmt" + "strings" + + md "github.com/JohannesKaufmann/html-to-markdown" + "github.com/PuerkitoBio/goquery" +) + +// ContentConverter handles HTML to Markdown conversion. +type ContentConverter struct { + mdConverter *md.Converter +} + +// NewConverter creates a new ContentConverter. +func NewConverter() *ContentConverter { + converter := md.NewConverter("", true, nil) + converter.AddRules( + md.Rule{ + Filter: []string{"details"}, + Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string { + // Unwrap details, just return content + return &content + }, + }, + md.Rule{ + Filter: []string{"summary"}, + Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string { + // Make summary a header + text := strings.TrimSpace(content) + newContent := fmt.Sprintf("\n#### %s\n\n", text) + return &newContent + }, + }, + ) + return &ContentConverter{mdConverter: converter} +} + +// Convert converts a goquery selection to Markdown. +func (c *ContentConverter) Convert(selection *goquery.Selection) string { + return c.mdConverter.Convert(selection) +} diff --git a/pkg/flattener/flattener.go b/pkg/flattener/flattener.go index ab5a8ea..5744a6b 100644 --- a/pkg/flattener/flattener.go +++ b/pkg/flattener/flattener.go @@ -5,37 +5,11 @@ import ( "sort" "strings" "sync" - "time" - md "github.com/JohannesKaufmann/html-to-markdown" "github.com/PuerkitoBio/goquery" "github.com/gocolly/colly/v2" ) -// PageResult represents a single scraped page's documentation. -type PageResult struct { - URL string - Content string -} - -// Config holds configuration for the Flattener. -type Config struct { - UserAgent string - Parallelism int - RandomDelay time.Duration - AllowedDomains []string -} - -// DefaultConfig returns a default configuration. -func DefaultConfig() Config { - return Config{ - UserAgent: "Mozilla/5.0 (compatible; GeminiBot/1.0; +http://gemini.google.com)", - Parallelism: 2, - RandomDelay: 500 * time.Millisecond, - AllowedDomains: []string{"pkg.go.dev"}, - } -} - // Flattener handles the scraping and conversion of documentation. type Flattener struct { config Config @@ -52,76 +26,83 @@ func New(config Config) *Flattener { func (f *Flattener) Flatten(rootURL string) ([]PageResult, error) { rootURL = strings.TrimSuffix(rootURL, "/") - // Prepare the Markdown converter - converter := md.NewConverter("", true, nil) - converter.AddRules( - md.Rule{ - Filter: []string{"details"}, - Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string { - // Unwrap details, just return content - return &content - }, - }, - md.Rule{ - Filter: []string{"summary"}, - Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string { - // Make summary a header - text := strings.TrimSpace(content) - newContent := fmt.Sprintf("\n#### %s\n\n", text) - return &newContent - }, - }, - ) + // Components + scraper, err := NewScraper(f.config) + if err != nil { + return nil, err + } + converter := NewConverter() // Safe storage for results var results []PageResult var mu sync.Mutex + var wg sync.WaitGroup + visitedSources := make(map[string]bool) - // Initialize Colly - c := colly.NewCollector( - colly.AllowedDomains(f.config.AllowedDomains...), - colly.UserAgent(f.config.UserAgent), - colly.Async(true), - ) - - // Configure parallelism and delay - err := c.Limit(&colly.LimitRule{ - DomainGlob: "*", - Parallelism: f.config.Parallelism, - RandomDelay: f.config.RandomDelay, - }) - if err != nil { - return nil, fmt.Errorf("failed to set limit rule: %w", err) - } - - // 1. Handle Documentation Content - c.OnHTML("main", func(e *colly.HTMLElement) { + // Handler for Main Content + scraper.Collector.OnHTML("main", func(e *colly.HTMLElement) { docURL := e.Request.URL.String() - - // Selectors updated to match pkg.go.dev structure - selection := e.DOM.Find(".UnitReadme, .Documentation-content") - - // Remove noise - selection.Find(".Documentation-index").Remove() - selection.Find("script").Remove() - selection.Find("style").Remove() - selection.Find(".Documentation-exampleButtonsContainer").Remove() - - markdown := converter.Convert(selection) - - // Add a header for the file - header := fmt.Sprintf("# Package: %s\nInput URL: %s\n\n", docURL, docURL) - - mu.Lock() - results = append(results, PageResult{ - URL: docURL, - Content: header + markdown, - }) - mu.Unlock() + + // 1. Documentation & Readme + docSelection := e.DOM.Find(".UnitReadme, .Documentation-content") + if docSelection.Length() > 0 { + // Cleanup + docSelection.Find(".Documentation-index").Remove() + docSelection.Find("script").Remove() + docSelection.Find("style").Remove() + docSelection.Find(".Documentation-exampleButtonsContainer").Remove() + + markdown := converter.Convert(docSelection) + + // Add a header for the file + header := fmt.Sprintf("# Package: %s\nInput URL: %s\n\n", docURL, docURL) + + mu.Lock() + results = append(results, PageResult{ + URL: docURL, + Content: header + markdown, + }) + mu.Unlock() + + // Find Source File Links + e.DOM.Find(".UnitFiles-fileList li a").Each(func(_ int, s *goquery.Selection) { + href, exists := s.Attr("href") + if exists { + fullURL := e.Request.AbsoluteURL(href) + + mu.Lock() + if visited := visitedSources[fullURL]; !visited && IsSourceLink(fullURL) { + visitedSources[fullURL] = true + mu.Unlock() + + // Download source file + // We do this in a goroutine to not block the collector + wg.Add(1) + go func(u string) { + defer wg.Done() + content, err := DownloadSource(u, f.config.MaxRetries) + if err == nil { + header := fmt.Sprintf("# Source File: %s\nURL: %s\n\n```go\n", u, u) + footer := "\n```\n" + + mu.Lock() + results = append(results, PageResult{ + URL: u, + Content: header + content + footer, + }) + mu.Unlock() + } + }(fullURL) + } else { + mu.Unlock() + } + } + }) + } }) - // 2. Handle Subdirectories (Children links) - c.OnHTML("div.UnitDirectories table tr td a", func(e *colly.HTMLElement) { + // Handle Subdirectories (Children links) + scraper.Collector.OnHTML("div.UnitDirectories table tr td a", func(e *colly.HTMLElement) { link := e.Attr("href") absoluteURL := e.Request.AbsoluteURL(link) @@ -132,16 +113,15 @@ func (f *Flattener) Flatten(rootURL string) ([]PageResult, error) { }) // Start the scraping - if err := c.Visit(rootURL); err != nil { + if err := scraper.Visit(rootURL); err != nil { return nil, fmt.Errorf("failed to start scraping: %w", err) } // Wait for all requests to finish - c.Wait() + scraper.Wait() + wg.Wait() - // Sort results by URL length (shortest first -> root) or alphabetically. - // Here we stick to alphabetical to correspond with original behavior's sort intention, - // but generally sorting by URL makes sense for a linear read. + // Sort results by URL sort.Slice(results, func(i, j int) bool { return results[i].URL < results[j].URL }) diff --git a/pkg/flattener/flattener_test.go b/pkg/flattener/flattener_test.go index bea1759..6248ab7 100644 --- a/pkg/flattener/flattener_test.go +++ b/pkg/flattener/flattener_test.go @@ -22,6 +22,13 @@ func TestFlatten(t *testing.T) {

Root Package

This is the root readme.

+ +
+ +
+
@@ -54,6 +61,11 @@ func TestFlatten(t *testing.T) { w.Write([]byte(html)) }) + // Source file page (RAW) + mux.HandleFunc("/pkg/file.go", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("package pkg\n\nfunc Foo() {}")) + }) + server := httptest.NewServer(mux) defer server.Close() @@ -69,6 +81,7 @@ func TestFlatten(t *testing.T) { Parallelism: 1, RandomDelay: 0, // No delay for tests AllowedDomains: []string{u.Hostname()}, + MaxRetries: 1, } flattener := New(config) @@ -81,11 +94,15 @@ func TestFlatten(t *testing.T) { } // 5. Assertions - if len(results) != 2 { - t.Errorf("Expected 2 results, got %d", len(results)) + // Expected: Root, Sub, SourceFile -> 3 results + if len(results) != 3 { + t.Errorf("Expected 3 results, got %d", len(results)) + for _, r := range results { + t.Logf("Result: %s", r.URL) + } } - // Helper to find result by URL suffix (since port changes) + // Helper to find result by URL suffix findResult := func(suffix string) *PageResult { for _, r := range results { if strings.HasSuffix(r.URL, suffix) { @@ -103,21 +120,24 @@ func TestFlatten(t *testing.T) { if !strings.Contains(rootRes.Content, "# Root Package") { t.Errorf("Root content missing header. Got:\n%s", rootRes.Content) } - if !strings.Contains(rootRes.Content, "This is the root readme.") { - t.Errorf("Root content missing body. Got:\n%s", rootRes.Content) - } } // Check Sub subRes := findResult("/pkg/sub") if subRes == nil { t.Error("Result for subpackage not found") + } + + // Check Source File + sourceRes := findResult("/pkg/file.go") + if sourceRes == nil { + t.Error("Result for source file not found") } else { - if !strings.Contains(subRes.Content, "## Subpackage Doc") { - t.Errorf("Sub content missing header. Got:\n%s", subRes.Content) + if !strings.Contains(sourceRes.Content, "func Foo() {}") { + t.Errorf("Source content missing code. Got:\n%s", sourceRes.Content) } - if strings.Contains(subRes.Content, "Index (should be removed)") { - t.Error("Content cleaning failed: found .Documentation-index content") + if !strings.Contains(sourceRes.Content, "```go") { + t.Error("Source content missing code block formatting") } } -} +} \ No newline at end of file diff --git a/pkg/flattener/models.go b/pkg/flattener/models.go new file mode 100644 index 0000000..b45e040 --- /dev/null +++ b/pkg/flattener/models.go @@ -0,0 +1,29 @@ +package flattener + +import "time" + +// PageResult represents a single scraped page's documentation. +type PageResult struct { + URL string + Content string +} + +// Config holds configuration for the Flattener. +type Config struct { + UserAgent string + Parallelism int + RandomDelay time.Duration + AllowedDomains []string + MaxRetries int +} + +// DefaultConfig returns a default configuration. +func DefaultConfig() Config { + return Config{ + UserAgent: "Mozilla/5.0 (compatible; GeminiBot/1.0; +http://gemini.google.com)", + Parallelism: 2, + RandomDelay: 500 * time.Millisecond, + AllowedDomains: []string{"pkg.go.dev"}, + MaxRetries: 3, + } +} diff --git a/pkg/flattener/scraper.go b/pkg/flattener/scraper.go new file mode 100644 index 0000000..d32614e --- /dev/null +++ b/pkg/flattener/scraper.go @@ -0,0 +1,72 @@ +package flattener + +import ( + "fmt" + "log" + + "github.com/gocolly/colly/v2" +) + +// Scraper wraps colly.Collector with custom configuration and retry logic. +type Scraper struct { + Collector *colly.Collector + config Config +} + +// NewScraper creates a new Scraper instance. +func NewScraper(config Config) (*Scraper, error) { + c := colly.NewCollector( + colly.AllowedDomains(config.AllowedDomains...), + colly.UserAgent(config.UserAgent), + colly.Async(true), + ) + + // Configure parallelism and delay + err := c.Limit(&colly.LimitRule{ + DomainGlob: "*", + Parallelism: config.Parallelism, + RandomDelay: config.RandomDelay, + }) + if err != nil { + return nil, fmt.Errorf("failed to set limit rule: %w", err) + } + + scraper := &Scraper{ + Collector: c, + config: config, + } + + scraper.setupRetry() + + return scraper, nil +} + +func (s *Scraper) setupRetry() { + s.Collector.OnError(func(r *colly.Response, err error) { + // StatusCode is 0 if the request failed before receiving a response + if r.StatusCode >= 400 || r.StatusCode == 0 { + retries := 0 + if v := r.Ctx.GetAny("retries"); v != nil { + retries = v.(int) + } + + if retries < s.config.MaxRetries { + log.Printf("Error requesting %s (attempt %d/%d): %v. Retrying...", r.Request.URL, retries+1, s.config.MaxRetries, err) + r.Ctx.Put("retries", retries+1) + r.Request.Retry() + } else { + log.Printf("Error requesting %s: %v. Max retries reached.", r.Request.URL, err) + } + } + }) +} + +// Visit starts the scraping process. +func (s *Scraper) Visit(url string) error { + return s.Collector.Visit(url) +} + +// Wait waits for all requests to finish. +func (s *Scraper) Wait() { + s.Collector.Wait() +} diff --git a/pkg/flattener/source.go b/pkg/flattener/source.go new file mode 100644 index 0000000..6b12e2f --- /dev/null +++ b/pkg/flattener/source.go @@ -0,0 +1,61 @@ +package flattener + +import ( + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// IsSourceLink checks if a link points to a source file. +func IsSourceLink(url string) bool { + return strings.HasSuffix(url, ".go") +} + +// GetRawURL converts a VCS provider URL to a raw content URL if possible. +func GetRawURL(originalURL string) string { + if strings.Contains(originalURL, "github.com") && strings.Contains(originalURL, "/blob/") { + // Convert https://github.com/user/repo/blob/branch/path/to/file.go + // To https://raw.githubusercontent.com/user/repo/branch/path/to/file.go + rawURL := strings.Replace(originalURL, "github.com", "raw.githubusercontent.com", 1) + rawURL = strings.Replace(rawURL, "/blob/", "/", 1) + return rawURL + } + // Add other providers if needed (e.g. gitlab, bitbucket) + return originalURL +} + +// DownloadSource downloads the raw content of a source file with retries. +func DownloadSource(url string, maxRetries int) (string, error) { + rawURL := GetRawURL(url) + + var lastErr error + for i := 0; i <= maxRetries; i++ { + if i > 0 { + time.Sleep(time.Duration(i) * 500 * time.Millisecond) // Exponential-ish backoff + } + + resp, err := http.Get(rawURL) + if err != nil { + lastErr = err + continue + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + lastErr = fmt.Errorf("unexpected status code: %d", resp.StatusCode) + continue + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + lastErr = err + continue + } + + return string(body), nil + } + + return "", fmt.Errorf("failed after %d retries: %w", maxRetries, lastErr) +} \ No newline at end of file