diff --git a/README.md b/README.md index 4c14f0c..360ff08 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,12 @@ ## Features - **GitHub URL Support**: Accepts GitHub repository URLs directly (e.g., `https://github.com/user/repo`) and resolves them to `pkg.go.dev`. -- **Smart Defaults**: Automatically generates a sanitized output filename from the package name if not provided. +- **Source File Downloading**: Automatically identifies and downloads raw Go source files linked in the documentation, appending them to the output for full context. +- **Retry Logic**: Robust scraping with automatic retries and exponential backoff for transient network issues. - **Recursive Scraping**: Automatically finds and scrapes sub-packages. - **Markdown Conversion**: Converts documentation and READMEs into clean Markdown. - **Sorted Output**: Organizes documentation alphabetically by package URL. -- **Library First**: Built as a reusable Go library with a CLI wrapper. +- **SOLID Architecture**: Refactored into modular components (Scraper, Converter, Source Handler) for better maintainability and extensibility. ## Installation @@ -75,6 +76,14 @@ func main() { } ``` +## Tip: Using with NotebookLM + +`flatten-go-doc` is an excellent companion for Google **NotebookLM**. By consolidating documentation and source code into a single Markdown file, you can: + +1. **Full Library Context**: Upload the generated `.md` file as a source in NotebookLM. +2. **AI-Powered Guidance**: Ask NotebookLM to "explain how to use this library based on the source code" or "create a tutorial for [specific feature]". +3. **Code Synthesis**: Since the tool includes the actual `.go` source files, NotebookLM can reason about the implementation details that aren't always visible in the standard documentation, helping you write more accurate and idiomatic code. + ## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/pkg/flattener/converter.go b/pkg/flattener/converter.go new file mode 100644 index 0000000..0d118b9 --- /dev/null +++ b/pkg/flattener/converter.go @@ -0,0 +1,43 @@ +package flattener + +import ( + "fmt" + "strings" + + md "github.com/JohannesKaufmann/html-to-markdown" + "github.com/PuerkitoBio/goquery" +) + +// ContentConverter handles HTML to Markdown conversion. +type ContentConverter struct { + mdConverter *md.Converter +} + +// NewConverter creates a new ContentConverter. +func NewConverter() *ContentConverter { + converter := md.NewConverter("", true, nil) + converter.AddRules( + md.Rule{ + Filter: []string{"details"}, + Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string { + // Unwrap details, just return content + return &content + }, + }, + md.Rule{ + Filter: []string{"summary"}, + Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string { + // Make summary a header + text := strings.TrimSpace(content) + newContent := fmt.Sprintf("\n#### %s\n\n", text) + return &newContent + }, + }, + ) + return &ContentConverter{mdConverter: converter} +} + +// Convert converts a goquery selection to Markdown. +func (c *ContentConverter) Convert(selection *goquery.Selection) string { + return c.mdConverter.Convert(selection) +} diff --git a/pkg/flattener/flattener.go b/pkg/flattener/flattener.go index ab5a8ea..5744a6b 100644 --- a/pkg/flattener/flattener.go +++ b/pkg/flattener/flattener.go @@ -5,37 +5,11 @@ import ( "sort" "strings" "sync" - "time" - md "github.com/JohannesKaufmann/html-to-markdown" "github.com/PuerkitoBio/goquery" "github.com/gocolly/colly/v2" ) -// PageResult represents a single scraped page's documentation. -type PageResult struct { - URL string - Content string -} - -// Config holds configuration for the Flattener. -type Config struct { - UserAgent string - Parallelism int - RandomDelay time.Duration - AllowedDomains []string -} - -// DefaultConfig returns a default configuration. -func DefaultConfig() Config { - return Config{ - UserAgent: "Mozilla/5.0 (compatible; GeminiBot/1.0; +http://gemini.google.com)", - Parallelism: 2, - RandomDelay: 500 * time.Millisecond, - AllowedDomains: []string{"pkg.go.dev"}, - } -} - // Flattener handles the scraping and conversion of documentation. type Flattener struct { config Config @@ -52,76 +26,83 @@ func New(config Config) *Flattener { func (f *Flattener) Flatten(rootURL string) ([]PageResult, error) { rootURL = strings.TrimSuffix(rootURL, "/") - // Prepare the Markdown converter - converter := md.NewConverter("", true, nil) - converter.AddRules( - md.Rule{ - Filter: []string{"details"}, - Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string { - // Unwrap details, just return content - return &content - }, - }, - md.Rule{ - Filter: []string{"summary"}, - Replacement: func(content string, selec *goquery.Selection, options *md.Options) *string { - // Make summary a header - text := strings.TrimSpace(content) - newContent := fmt.Sprintf("\n#### %s\n\n", text) - return &newContent - }, - }, - ) + // Components + scraper, err := NewScraper(f.config) + if err != nil { + return nil, err + } + converter := NewConverter() // Safe storage for results var results []PageResult var mu sync.Mutex + var wg sync.WaitGroup + visitedSources := make(map[string]bool) - // Initialize Colly - c := colly.NewCollector( - colly.AllowedDomains(f.config.AllowedDomains...), - colly.UserAgent(f.config.UserAgent), - colly.Async(true), - ) - - // Configure parallelism and delay - err := c.Limit(&colly.LimitRule{ - DomainGlob: "*", - Parallelism: f.config.Parallelism, - RandomDelay: f.config.RandomDelay, - }) - if err != nil { - return nil, fmt.Errorf("failed to set limit rule: %w", err) - } - - // 1. Handle Documentation Content - c.OnHTML("main", func(e *colly.HTMLElement) { + // Handler for Main Content + scraper.Collector.OnHTML("main", func(e *colly.HTMLElement) { docURL := e.Request.URL.String() - - // Selectors updated to match pkg.go.dev structure - selection := e.DOM.Find(".UnitReadme, .Documentation-content") - - // Remove noise - selection.Find(".Documentation-index").Remove() - selection.Find("script").Remove() - selection.Find("style").Remove() - selection.Find(".Documentation-exampleButtonsContainer").Remove() - - markdown := converter.Convert(selection) - - // Add a header for the file - header := fmt.Sprintf("# Package: %s\nInput URL: %s\n\n", docURL, docURL) - - mu.Lock() - results = append(results, PageResult{ - URL: docURL, - Content: header + markdown, - }) - mu.Unlock() + + // 1. Documentation & Readme + docSelection := e.DOM.Find(".UnitReadme, .Documentation-content") + if docSelection.Length() > 0 { + // Cleanup + docSelection.Find(".Documentation-index").Remove() + docSelection.Find("script").Remove() + docSelection.Find("style").Remove() + docSelection.Find(".Documentation-exampleButtonsContainer").Remove() + + markdown := converter.Convert(docSelection) + + // Add a header for the file + header := fmt.Sprintf("# Package: %s\nInput URL: %s\n\n", docURL, docURL) + + mu.Lock() + results = append(results, PageResult{ + URL: docURL, + Content: header + markdown, + }) + mu.Unlock() + + // Find Source File Links + e.DOM.Find(".UnitFiles-fileList li a").Each(func(_ int, s *goquery.Selection) { + href, exists := s.Attr("href") + if exists { + fullURL := e.Request.AbsoluteURL(href) + + mu.Lock() + if visited := visitedSources[fullURL]; !visited && IsSourceLink(fullURL) { + visitedSources[fullURL] = true + mu.Unlock() + + // Download source file + // We do this in a goroutine to not block the collector + wg.Add(1) + go func(u string) { + defer wg.Done() + content, err := DownloadSource(u, f.config.MaxRetries) + if err == nil { + header := fmt.Sprintf("# Source File: %s\nURL: %s\n\n```go\n", u, u) + footer := "\n```\n" + + mu.Lock() + results = append(results, PageResult{ + URL: u, + Content: header + content + footer, + }) + mu.Unlock() + } + }(fullURL) + } else { + mu.Unlock() + } + } + }) + } }) - // 2. Handle Subdirectories (Children links) - c.OnHTML("div.UnitDirectories table tr td a", func(e *colly.HTMLElement) { + // Handle Subdirectories (Children links) + scraper.Collector.OnHTML("div.UnitDirectories table tr td a", func(e *colly.HTMLElement) { link := e.Attr("href") absoluteURL := e.Request.AbsoluteURL(link) @@ -132,16 +113,15 @@ func (f *Flattener) Flatten(rootURL string) ([]PageResult, error) { }) // Start the scraping - if err := c.Visit(rootURL); err != nil { + if err := scraper.Visit(rootURL); err != nil { return nil, fmt.Errorf("failed to start scraping: %w", err) } // Wait for all requests to finish - c.Wait() + scraper.Wait() + wg.Wait() - // Sort results by URL length (shortest first -> root) or alphabetically. - // Here we stick to alphabetical to correspond with original behavior's sort intention, - // but generally sorting by URL makes sense for a linear read. + // Sort results by URL sort.Slice(results, func(i, j int) bool { return results[i].URL < results[j].URL }) diff --git a/pkg/flattener/flattener_test.go b/pkg/flattener/flattener_test.go index bea1759..6248ab7 100644 --- a/pkg/flattener/flattener_test.go +++ b/pkg/flattener/flattener_test.go @@ -22,6 +22,13 @@ func TestFlatten(t *testing.T) {
This is the root readme.
+ +