- AI Context
-
-
-
-
-
-
-
-
-
- 
+ AI Context
-
+ 
+ Capabilities • Installation • Usage • Tips & Notes +
|
-| CLI | 
|
+### Processing
-### Primary Usage
+Generate context from a single source.
```bash
# Process a single path (local directory) with additional ignore patterns
@@ -88,6 +52,20 @@ ai-context /path/to/directory -i "tests,docs,*doc.*"
# Process one URL (GitHub repo or YouTube Video or Webpage URL)
ai-context https://www.youtube.com/watch?v=video_id
+# Process private GitHub repository
+GH_TOKEN=$(cat /secrets/GH.PAT) ai-context https://github.com/ORG/REPO
+```
+
+**Flags:**
+- `--ignore, -i` - Additional patterns to ignore (e.g., 'tests,docs')
+- `--debug` - Enable debug logging
+- `--for-ai` - AI-friendly output (plain text, piped input)
+
+### Batch Processing
+
+Generate context from multiple sources listed in a file.
+
+```bash
# Make a list of paths
cat << EOF > listfile
../notif
@@ -98,44 +76,17 @@ EOF
# Process URL list concurrently
ai-context -f listfile
-
-# Process private GitHub repository
-GH_TOKEN=$(cat /secrets/GH.PAT) ai-context -u https://github.com/ORG/REPO
```
-> [!WARNING]
-> For directory path (in URL or listfile mode), the path should either start with `/` (absolute) or with `./` or `../` (relative). For current directory, always use `./` for correct regex matching.
-
-### Output
-
-- The tool creates a local folder called `context` and puts everything converted into `.md` files in that folder
-- The filenames have the syntax of `TYPE-PATHNAME.md` (example, `gh-ffuf_ffuf.md`)
-- Every single path in the `listfile` mode will result in a new context file
-- All images (only downloaded via webpages) are named as UUIDs and stored in the `context/images` directory (images are downloaded as a conenience, but doesn't take away from text-first context creation)
-
-### Command Line Options
-
-- CLI argument: provide a path (GitHub repo, YouTube video, WebPage link, or relative/absolute directory path) to process
-- `-f, --file`: provide a file with a list of paths (URLs or directory paths) to process
-- `-i, --ignore`: add additional patterns to ignore during processing (comma-separated)
-- `-t, --threads`: (*optional*) number of workers for concurrent file processing when passing list file (default = 10)
-
-> [!TIP]
-> - Do a `head -n 200 context/FILE.md` (or 500 lines) to view the content tree of the processed code base or directory to see what's been included. Then refine your `-i` flag arguments to ignore additional patterns.
-> - When processing a large number of items, it can look stalled due to thread limits and image download times; use `--debug` to enable verbose logs to know what's running.
-
-### Default Ignores
-
-The tool includes pre-defined and sensible ignore patterns, including common files and directories that typically don't add value to the context. These are:
+**Flags:**
+- `--file, -f` - File with list of URLs to process
+- `--threads, -t` - Number of threads to use for processing (default: 10)
-- Version control files (.git, .gitignore)
-- Dependencies (node_modules, vendor)
-- Compiled files (*.exe, *.dll)
-- Media files (images, videos, audio)
-- Lock files (package-lock.json, yarn.lock)
-- Build artifacts and caches
+## Tips and Notes
-For a full list, see `aicontext/ignores.go`.
+- For directory path (in URL or listfile mode), the path should either start with `/` (absolute) or with `./` or `../` (relative). For current directory, always use `./` for correct regex matching.
+- Do a `head -n 200 context/FILE.md` (or 500 lines) to view the content tree of the processed code base or directory to see what's been included. Then refine your `-i` flag arguments to ignore additional patterns.
+- The `--for-ai` flag produces plain text without ANSI colors, which is easier for AI agents to parse.
## Acknowledgments
diff --git a/cmd/ai-context.go b/cmd/root.go
similarity index 52%
rename from cmd/ai-context.go
rename to cmd/root.go
index 62003d7..b15d5e8 100644
--- a/cmd/ai-context.go
+++ b/cmd/root.go
@@ -4,10 +4,14 @@ import (
"bufio"
"os"
"strings"
+ "time"
"github.com/spf13/cobra"
- "github.com/tanq16/ai-context/aicontext"
- u "github.com/tanq16/ai-context/utils"
+ "github.com/rs/zerolog"
+ "github.com/rs/zerolog/log"
+
+ "github.com/tanq16/ai-context/internal/aicontext"
+ "github.com/tanq16/ai-context/utils"
)
var cmdFlags struct {
@@ -15,36 +19,36 @@ var cmdFlags struct {
url string
listFile string
ignoreList []string
- enableLog bool
}
-var AIContextVersion = "dev"
+var AppVersion = "dev-build"
+var debugFlag bool
+var forAIFlag bool
var rootCmd = &cobra.Command{
Use: "ai-context",
Short: "Produce AI context-file for GitHub project, directory, or YouTube video.",
- Version: AIContextVersion,
+ Version: AppVersion,
+ CompletionOptions: cobra.CompletionOptions{
+ HiddenDefaultCmd: true,
+ },
Args: cobra.MaximumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if len(args) > 0 && cmdFlags.listFile == "" {
cmdFlags.url = args[0]
} else if len(args) == 0 && cmdFlags.listFile == "" {
- u.PrintError("no URL argument or list file provided")
- os.Exit(1)
+ utils.PrintFatal("no URL argument or list file provided", nil)
} else if len(args) > 0 && cmdFlags.listFile != "" {
- u.PrintError("received both URL argument and list file")
- os.Exit(1)
+ utils.PrintFatal("received both URL argument and list file", nil)
}
- // Input URL processing
var urls []string
if cmdFlags.listFile == "" {
urls = append(urls, cmdFlags.url)
} else {
file, err := os.Open(cmdFlags.listFile)
if err != nil {
- u.PrintError("failed to open list file")
- os.Exit(1)
+ utils.PrintFatal("failed to open list file", err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
@@ -55,26 +59,48 @@ var rootCmd = &cobra.Command{
}
}
if scanner.Err() != nil {
- u.PrintError("failed to read list file")
- os.Exit(1)
+ utils.PrintFatal("failed to read list file", scanner.Err())
}
}
- aicontext.Handler(urls, cmdFlags.ignoreList, cmdFlags.threads, cmdFlags.enableLog)
+ aicontext.Handler(urls, cmdFlags.ignoreList, cmdFlags.threads, false)
},
}
func Execute() {
- rootCmd.CompletionOptions.DisableDefaultCmd = true
- rootCmd.SetHelpCommand(&cobra.Command{Hidden: true})
- err := rootCmd.Execute()
- if err != nil {
+ if err := rootCmd.Execute(); err != nil {
os.Exit(1)
}
}
+func setupLogs() {
+ zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
+ output := zerolog.ConsoleWriter{
+ Out: os.Stdout,
+ TimeFormat: time.DateTime,
+ NoColor: false,
+ }
+ zerolog.SetGlobalLevel(zerolog.InfoLevel)
+ if debugFlag {
+ zerolog.SetGlobalLevel(zerolog.DebugLevel)
+ log.Logger = zerolog.New(output).With().Timestamp().Logger()
+ utils.GlobalDebugFlag = true
+ }
+ if forAIFlag {
+ utils.GlobalForAIFlag = true
+ zerolog.SetGlobalLevel(zerolog.Disabled)
+ }
+}
+
func init() {
+ rootCmd.SetHelpCommand(&cobra.Command{Hidden: true})
+
+ rootCmd.PersistentFlags().BoolVar(&debugFlag, "debug", false, "Enable debug logging")
+ rootCmd.PersistentFlags().BoolVar(&forAIFlag, "for-ai", false, "AI-friendly output (plain text, piped input)")
+ rootCmd.MarkFlagsMutuallyExclusive("debug", "for-ai")
+
+ cobra.OnInitialize(setupLogs)
+
rootCmd.Flags().StringVarP(&cmdFlags.listFile, "file", "f", "", "File with list of URLs to process")
rootCmd.Flags().IntVarP(&cmdFlags.threads, "threads", "t", 10, "Number of threads to use for processing")
rootCmd.Flags().StringSliceVarP(&cmdFlags.ignoreList, "ignore", "i", []string{}, "Additional patterns to ignore (e.g., 'tests,docs'); helpful with GitHub or local directories")
- rootCmd.Flags().BoolVar(&cmdFlags.enableLog, "log", false, "Enable log-style output")
}
diff --git a/cmd/serve.go b/cmd/serve.go
deleted file mode 100644
index c22e027..0000000
--- a/cmd/serve.go
+++ /dev/null
@@ -1,246 +0,0 @@
-package cmd
-
-import (
- "archive/zip"
- "bytes"
- "embed"
- "encoding/json"
- "fmt"
- "io/fs"
- "log"
- "net/http"
- "os"
- "path/filepath"
- "strings"
-
- "github.com/spf13/cobra"
- "github.com/tanq16/ai-context/aicontext"
-)
-
-//go:embed all:web
-var webFS embed.FS
-
-var serveCmd = &cobra.Command{
- Use: "serve",
- Short: "Launch a web server to use the AI Context tool through a UI.",
- Run: runServer,
-}
-
-type generateRequest struct {
- URL string `json:"url"`
- Ignore []string `json:"ignore"`
-}
-
-type generateResponse struct {
- Content string `json:"content"`
-}
-
-func runServer(cmd *cobra.Command, args []string) {
- webContentFS, err := fs.Sub(webFS, "web")
- if err != nil {
- log.Fatalf("Failed to create web content file system: %v", err)
- }
- http.Handle("/static/", http.FileServer(http.FS(webContentFS)))
- http.HandleFunc("/generate", generateHandler)
- http.HandleFunc("/load", loadHandler)
- http.HandleFunc("/clear", clearHandler)
- http.HandleFunc("/download", downloadHandler)
- http.HandleFunc("/", rootHandler(webContentFS))
- port := "8080"
- fmt.Printf("Starting server at http://localhost:%s\n", port)
- if err := http.ListenAndServe(":"+port, nil); err != nil {
- log.Fatalf("Failed to start server: %v", err)
- }
-}
-
-func rootHandler(fs fs.FS) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path == "/" {
- indexHTML, err := webFS.ReadFile("web/index.html")
- if err != nil {
- http.Error(w, "Could not read index.html", http.StatusInternalServerError)
- log.Printf("Error reading embedded index.html: %v", err)
- return
- }
- w.Header().Set("Content-Type", "text/html; charset=utf-8")
- w.Write(indexHTML)
- return
- }
- http.FileServer(http.FS(fs)).ServeHTTP(w, r)
- }
-}
-
-func clearHandler(w http.ResponseWriter, r *http.Request) {
- if r.Method != http.MethodPost {
- http.Error(w, "Only POST method is allowed", http.StatusMethodNotAllowed)
- return
- }
- if err := cleanupContextDir(); err != nil {
- log.Printf("Error during cleanup: %v", err)
- http.Error(w, "Failed to clear context file", http.StatusInternalServerError)
- return
- }
- w.WriteHeader(http.StatusNoContent)
-}
-
-func loadHandler(w http.ResponseWriter, r *http.Request) {
- if r.Method != http.MethodGet {
- http.Error(w, "Only GET method is allowed", http.StatusMethodNotAllowed)
- return
- }
- outputFile, err := findGeneratedFile()
- if err != nil {
- w.Header().Set("Content-Type", "application/json")
- json.NewEncoder(w).Encode(generateResponse{Content: ""})
- return
- }
- content, err := os.ReadFile(outputFile)
- if err != nil {
- http.Error(w, "Failed to read context file", http.StatusInternalServerError)
- log.Printf("Error reading output file %s: %v", outputFile, err)
- return
- }
- resp := generateResponse{Content: string(content)}
- w.Header().Set("Content-Type", "application/json")
- json.NewEncoder(w).Encode(resp)
-}
-
-func generateHandler(w http.ResponseWriter, r *http.Request) {
- if r.Method != http.MethodPost {
- http.Error(w, "Only POST method is allowed", http.StatusMethodNotAllowed)
- return
- }
- var req generateRequest
- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
- http.Error(w, "Invalid request body", http.StatusBadRequest)
- return
- }
- if req.URL == "" {
- http.Error(w, "URL is required", http.StatusBadRequest)
- return
- }
- if err := cleanupContextDir(); err != nil {
- log.Printf("Warning: could not clean up context directory: %v", err)
- }
- aicontext.Handler([]string{req.URL}, req.Ignore, 1, true)
- outputFile, err := findGeneratedFile()
- if err != nil {
- http.Error(w, "Failed to find generated context file", http.StatusInternalServerError)
- log.Printf("Error finding generated file: %v", err)
- return
- }
- content, err := os.ReadFile(outputFile)
- if err != nil {
- http.Error(w, "Failed to read context file", http.StatusInternalServerError)
- log.Printf("Error reading output file %s: %v", outputFile, err)
- return
- }
- resp := generateResponse{Content: string(content)}
- w.Header().Set("Content-Type", "application/json")
- if err := json.NewEncoder(w).Encode(resp); err != nil {
- log.Printf("Error encoding response: %v", err)
- }
-}
-
-func downloadHandler(w http.ResponseWriter, r *http.Request) {
- if r.Method != http.MethodGet {
- http.Error(w, "Only GET method is allowed", http.StatusMethodNotAllowed)
- return
- }
- mdFile, err := findGeneratedFile()
- if err != nil {
- http.Error(w, "Could not find context file.", http.StatusNotFound)
- log.Printf("Error finding generated file for download: %v", err)
- return
- }
- mdContent, err := os.ReadFile(mdFile)
- if err != nil {
- http.Error(w, "Could not read context file.", http.StatusInternalServerError)
- log.Printf("Error reading context file %s for download: %v", mdFile, err)
- return
- }
- imagesDir := filepath.Join("context", "images")
- images, err := os.ReadDir(imagesDir)
- // Only MD file if no images pulled
- if err != nil || len(images) == 0 {
- w.Header().Set("Content-Type", "text/markdown")
- w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base(mdFile)))
- w.Write(mdContent)
- return
- }
- buf := new(bytes.Buffer)
- zipWriter := zip.NewWriter(buf)
- mdWriter, err := zipWriter.Create(filepath.Base(mdFile))
- if err != nil {
- http.Error(w, "Failed to create markdown entry in zip.", http.StatusInternalServerError)
- return
- }
- _, err = mdWriter.Write(mdContent)
- if err != nil {
- http.Error(w, "Failed to write markdown content to zip.", http.StatusInternalServerError)
- return
- }
- for _, image := range images {
- if !image.IsDir() {
- imgPath := filepath.Join(imagesDir, image.Name())
- imgData, err := os.ReadFile(imgPath)
- if err != nil {
- log.Printf("Warning: could not read image file %s: %v", imgPath, err)
- continue // Skip this file if it can't be read.
- }
- imgWriter, err := zipWriter.Create(filepath.Join("images", image.Name()))
- if err != nil {
- log.Printf("Warning: could not create image entry %s in zip: %v", image.Name(), err)
- continue
- }
- _, err = imgWriter.Write(imgData)
- if err != nil {
- log.Printf("Warning: could not write image data for %s to zip: %v", image.Name(), err)
- }
- }
- }
- if err := zipWriter.Close(); err != nil {
- http.Error(w, "Failed to finalize zip file.", http.StatusInternalServerError)
- return
- }
- zipName := strings.TrimSuffix(filepath.Base(mdFile), ".md") + ".zip"
- w.Header().Set("Content-Type", "application/zip")
- w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", zipName))
- w.Write(buf.Bytes())
-}
-
-func cleanupContextDir() error {
- dir := "context"
- files, err := filepath.Glob(filepath.Join(dir, "*.md"))
- if err != nil {
- return err
- }
- for _, file := range files {
- if err := os.Remove(file); err != nil {
- log.Printf("Failed to remove file %s: %v", file, err)
- }
- }
- imgPath := filepath.Join(dir, "images")
- if _, err := os.Stat(imgPath); !os.IsNotExist(err) {
- if err := os.RemoveAll(imgPath); err != nil {
- log.Printf("Failed to remove images directory %s: %v", imgPath, err)
- }
- }
- return nil
-}
-
-func findGeneratedFile() (string, error) {
- dir := "context"
- files, err := filepath.Glob(filepath.Join(dir, "*.md"))
- if err != nil {
- return "", fmt.Errorf("error searching for files: %w", err)
- }
- if len(files) == 0 {
- return "", fmt.Errorf("no markdown file found in context directory")
- }
- return files[0], nil
-}
-
-func init() {
- rootCmd.AddCommand(serveCmd)
-}
diff --git a/cmd/web/asset-download.sh b/cmd/web/asset-download.sh
deleted file mode 100644
index 61ed49b..0000000
--- a/cmd/web/asset-download.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-
-mkdir -p static/css
-mkdir -p static/js
-mkdir -p static/webfonts
-mkdir -p static/fonts
-
-echo "Downloading assets..."
-
-# Download Tailwind CSS
-curl -sL "https://cdn.tailwindcss.com" -o "static/js/tailwindcss.js"
-
-# Download Font Awesome CSS and its webfonts
-curl -sL "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/css/all.min.css" -o "static/css/all.min.css"
-curl -sL "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/webfonts/fa-brands-400.woff2" -o "static/webfonts/fa-brands-400.woff2"
-curl -sL "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/webfonts/fa-regular-400.woff2" -o "static/webfonts/fa-regular-400.woff2"
-curl -sL "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/webfonts/fa-solid-900.woff2" -o "static/webfonts/fa-solid-900.woff2"
-curl -sL "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/webfonts/fa-v4compatibility.woff2" -o "static/webfonts/fa-v4compatibility.woff2"
-
-# Update Font Awesome CSS to use local webfonts path
-sed -i.bak 's|https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/webfonts/|/static/webfonts/|g' static/css/all.min.css
-rm static/css/all.min.css.bak
-
-# Download Inter font CSS from Google Fonts
-curl -sL "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" -A "Mozilla/5.0" -o "static/css/inter.css"
-
-# Download font files referenced in the CSS
-grep -o 'https://fonts.gstatic.com/s/inter/[^)]*' static/css/inter.css | while read -r url; do
- # Remove the single quote from the end
- clean_url=$(echo "$url" | sed "s/'$//")
- filename=$(basename "$clean_url")
- curl -sL "$clean_url" -o "static/fonts/$filename"
-done
-
-# Update font CSS to use local font files
-sed -i.bak 's|https://fonts.gstatic.com/s/inter/v[0-9]*/|/static/fonts/|g' static/css/inter.css
-rm static/css/inter.css.bak
-
-echo "All assets downloaded successfully!"
diff --git a/cmd/web/index.html b/cmd/web/index.html
deleted file mode 100644
index cc100f8..0000000
--- a/cmd/web/index.html
+++ /dev/null
@@ -1,252 +0,0 @@
-
-
-
-
-
-
-