Repository navigation
Expand file tree
/
Copy pathstack.env
More file actions
181 lines (163 loc) · 5.68 KB
/
Copy pathstack.env
File metadata and controls
181 lines (163 loc) · 5.68 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
# ============================================
# Knowledge Extraction Orchestrator - Stack Environment
# ============================================
# Copy this to Portainer's environment variables section
# Variables marked [REQUIRED] must be set
# Variables marked [OPTIONAL] have defaults shown
# ===================
# REQUIRED - Container Registry
# ===================
REGISTRY_PREFIX=ghcr.io/tkontu
# ===================
# REQUIRED - Security (OVERRIDE IN PORTAINER)
# ===================
# API_KEY=<SET_IN_PORTAINER_SECRETS>
# IMPORTANT: Do NOT use this default in production!
# Override this in Portainer with a secure 32+ character key
API_KEY=defaultdevlopapikey32characterslong
# ===================
# REQUIRED - LLM Configuration (OVERRIDE IN PORTAINER)
# ===================
# OPENAI_BASE_URL=<SET_IN_PORTAINER_SECRETS>
# OPENAI_EMBEDDING_BASE_URL=<SET_IN_PORTAINER_SECRETS>
# IMPORTANT: Set these to your actual LLM server URLs in Portainer
OPENAI_BASE_URL=http://localhost:9003/v1
OPENAI_EMBEDDING_BASE_URL=http://localhost:9003/v1
OPENAI_API_KEY=ollama
LLM_MODEL=gemma3-12b-awq
RAG_EMBEDDING_MODEL=bge-m3
# ===================
# [OPTIONAL] Database (defaults: scristill)
# ===================
# DB_USER=scristill
# DB_PASSWORD=scristill
# DB_NAME=scristill
# ===================
# [OPTIONAL] LLM Timeouts
# ===================
# LLM_HTTP_TIMEOUT=900
# LLM_MAX_RETRIES=5
# LLM_RETRY_BACKOFF_MIN=2
# LLM_RETRY_BACKOFF_MAX=60
# ===================
# [OPTIONAL] Scraping Config (Individual Scrapes)
# ===================
# SCRAPE_DELAY_MIN=2
# SCRAPE_DELAY_MAX=5
# SCRAPE_MAX_CONCURRENT_PER_DOMAIN=2
# SCRAPE_DAILY_LIMIT_PER_DOMAIN=500
# SCRAPE_MAX_RETRIES=3
# SCRAPE_TIMEOUT=180 # 3 minutes for anti-bot protected sites
# ===================
# [OPTIONAL] Crawl Rate Limiting (Respectful Crawling)
# ===================
# Delay between crawl requests in milliseconds (2000 = 2 seconds)
# CRAWL_DELAY_MS=2000
# Max concurrent requests per crawl (2 = polite, won't overwhelm servers)
# CRAWL_MAX_CONCURRENCY=2
# Max parallel crawl jobs for different domains (adjust based on server resources)
# For 48GB RAM / 6 cores: MAX_CONCURRENT_CRAWLS=6
# For smaller servers: MAX_CONCURRENT_CRAWLS=1 or 2
# MAX_CONCURRENT_CRAWLS=6
# ===================
# [OPTIONAL] Network/Security
# ===================
# PIPELINE_PORT=8742
# ALLOWED_ORIGINS=*
# ===================
# [OPTIONAL] Anti-Bot Protection
# ===================
# USE_FLARESOLVERR=true
# FLARESOLVERR_LOG_LEVEL=info
# CAPTCHA_SOLVER=none
# ===================
# [OPTIONAL] FlareSolverr Proxy Adapter
# ===================
# PROXY_ADAPTER_ENABLED=true
# PROXY_ADAPTER_PORT=8192
# PROXY_ADAPTER_LOG_LEVEL=INFO
# FLARESOLVERR_URL=http://flaresolverr:8191
# FLARESOLVERR_MAX_TIMEOUT=60000
# FLARESOLVERR_BLOCKED_DOMAINS=weg.net,siemens.com,wattdrive.com
# ===================
# REQUIRED - Playwright Proxy Configuration (for Akamai bypass)
# ===================
PLAYWRIGHT_PROXY_SERVER=http://proxy-adapter:8192
# NOTE: Use hostname (proxy-adapter) not IP address.
# Only use IP address (http://172.x.x.x:8192) if DNS resolution fails in your environment.
# ===================
# [OPTIONAL] Camoufox Browser Service (anti-bot protected scraping)
# ===================
# Pool size: Max concurrent browser pages
# For 48GB RAM / 6 cores: CAMOUFOX_POOL_SIZE=40
# For smaller servers: CAMOUFOX_POOL_SIZE=10 or 20
# CAMOUFOX_POOL_SIZE=40
#
# Page load timeout in milliseconds (180000 = 3 minutes)
# CAMOUFOX_TIMEOUT=180000
#
# Network idle timeout (reduced for faster scraping)
# CAMOUFOX_NETWORKIDLE_TIMEOUT=5000
#
# Content stability detection (prevents unnecessary waiting)
# CAMOUFOX_CONTENT_STABILITY_CHECKS=2
# CAMOUFOX_CONTENT_STABILITY_INTERVAL=500
#
# Optional proxy URL (e.g., http://user:pass@host:port)
# CAMOUFOX_PROXY=
#
# Run headless (set false for debugging)
# CAMOUFOX_HEADLESS=true
#
# Migration: To switch Firecrawl from Playwright to Camoufox:
# 1. Build and test: docker compose build camoufox && docker compose up camoufox
# 2. Test endpoint: curl http://localhost:3004/health
# 3. Update firecrawl-api environment:
# PLAYWRIGHT_MICROSERVICE_URL=http://camoufox:3003/scrape
# 4. Remove HTTP_PROXY/HTTPS_PROXY from firecrawl-api (Camoufox handles anti-bot directly)
# ===================
# [OPTIONAL] Smart Crawl Settings
# ===================
# Smart crawl uses Map → Filter → Batch Scrape flow for intelligent URL discovery
# Enabled per-request with smart_crawl_enabled=true (disabled by default)
#
# Default embedding similarity threshold (0.0-1.0, higher = stricter filtering)
# SMART_CRAWL_DEFAULT_RELEVANCE_THRESHOLD=0.4
#
# Maximum URLs to discover from Firecrawl Map endpoint
# SMART_CRAWL_MAP_LIMIT=5000
#
# Max concurrent requests during batch scraping
# SMART_CRAWL_BATCH_MAX_CONCURRENCY=10
# ===================
# [OPTIONAL] Extraction Concurrency
# ===================
# Max concurrent chunk extractions (default: 80)
# EXTRACTION_MAX_CONCURRENT_CHUNKS=80
# ===================
# [OPTIONAL] LLM Worker Queue (Adaptive Concurrency)
# ===================
# Enable Redis-based LLM request queue for batching and adaptive concurrency
# LLM_QUEUE_ENABLED=false
#
# Initial concurrent LLM requests in flight (default: 10)
# LLM_WORKER_CONCURRENCY=10
#
# Maximum concurrent LLM requests (scales up on success) (default: 50)
# LLM_WORKER_MAX_CONCURRENCY=50
#
# Minimum concurrent LLM requests (scales down on timeouts) (default: 5)
# LLM_WORKER_MIN_CONCURRENCY=5
# ===================
# [OPTIONAL] File Paths (bind mounts)
# ===================
# INPUT_PATH=/path/to/input
# OUTPUT_PATH=/path/to/output
# ===================
# [OPTIONAL] Logging
# ===================
LOG_LEVEL=DEBUG
LOG_FORMAT=json
# ENABLE_METRICS=true
# Firecrawl worker logging (debug to trace sitemap hangs)
FIRECRAWL_LOGGING_LEVEL=debug