Repository navigation
Expand file tree
/
Copy pathknowledge_fetcher.py
More file actions
454 lines (383 loc) · 14.6 KB
/
Copy pathknowledge_fetcher.py
File metadata and controls
454 lines (383 loc) · 14.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
from __future__ import annotations
import argparse
from defusedxml import ElementTree as SafeET
from typing import Any, Dict, List
from urllib.parse import quote
from logger import get_logger
from http_utils import resilient_get, RateLimiter
from path_security import sanitize_cli_path
# Import config loader
try:
from config_loader import load_config, CounterscarpConfig
CONFIG_AVAILABLE = True
except ImportError:
CONFIG_AVAILABLE = False
logger = get_logger(__name__)
# Load configuration with fallback to defaults
_config = None
_rate_limiter = None
def get_config() -> CounterscarpConfig:
"""Get or load the configuration."""
global _config
if _config is None:
if CONFIG_AVAILABLE:
try:
_config = load_config()
except Exception:
_config = CounterscarpConfig()
else:
_config = CounterscarpConfig()
return _config
def get_rate_limiter() -> RateLimiter:
"""Get or create the rate limiter."""
global _rate_limiter
if _rate_limiter is None:
try:
rate_limit = get_config().threat_intel.api_rate_limit
except Exception:
rate_limit = 5
_rate_limiter = RateLimiter(requests_per_second=rate_limit)
return _rate_limiter
def get_c4_timeout() -> int:
"""Get Code4rena API timeout from config or use default."""
try:
return int(get_config().threat_intel.c4_timeout)
except Exception:
return 10
def get_immunefi_timeout() -> int:
"""Get Immunefi RSS timeout from config or use default."""
try:
return int(get_config().threat_intel.immunefi_timeout)
except Exception:
return 10
# --- CONFIGURATION ---
# GitHub API for Code4rena
C4_API_URL = "https://api.github.com/search/issues"
# Immunefi publishes reports on Medium. We use the RSS feed to fetch
# the latest "School of Rock" / Spotlights.
IMMUNEFI_RSS = "https://medium.com/feed/immunefi"
# Solodit URL constructor
SOLODIT_BASE = "https://solodit.xyz/search"
def get_contract_context_from_source(content: str) -> List[str]:
"""Infer context tags from Solidity source text."""
tags = set()
lower = content.lower()
if "erc20" in lower: tags.add("ERC20")
if "erc721" in lower or "nft" in lower: tags.add("NFT")
if "lending" in lower or "collateral" in lower: tags.add("Lending")
if "staking" in lower or "reward" in lower: tags.add("Staking")
if "vault" in lower or "erc4626" in lower: tags.add("ERC4626")
if "oracle" in lower or "chainlink" in lower: tags.add("Oracle")
if "swap" in lower or "amm" in lower: tags.add("AMM")
if "bridge" in lower: tags.add("Bridge")
if "governance" in lower or "voting" in lower: tags.add("Governance")
if "flash" in lower: tags.add("FlashLoan")
if "proxy" in lower or "upgradeable" in lower: tags.add("Upgradeable")
if not tags:
tags.add("Smart Contract")
return list(tags)
def get_contract_context(filepath: str) -> List[str]:
"""Scans the file to determine specific tags (e.g., "ERC4626", "Oracle").
Args:
filepath: Path to the Solidity file to analyze.
Returns:
List of detected context tags.
"""
tags = set()
try:
safe_file = sanitize_cli_path(filepath, allowed_suffixes={".sol"})
with open(safe_file, 'r', encoding='utf-8') as f:
return get_contract_context_from_source(f.read())
except (IOError, OSError, UnicodeDecodeError) as e:
logger.warning(f"Could not read contract file for context: {e}")
except Exception as e:
logger.error(f"Unexpected error scanning contract context: {e}")
return list(tags) if tags else ["Smart Contract"]
# Backwards compatibility alias
scan_file_for_context = get_contract_context
def _filter_bundled_db(query: str, source: str | None = None) -> List[Dict[str, Any]]:
"""Filter bundled threat intel entries by query keywords.
Args:
query: Space-separated keyword string to match against entries.
source: Optional source tag appended to result source field.
Returns:
List of matching threat intel entries from the bundled database.
"""
from threat_intel import load_bundled_db
entries = load_bundled_db()
if not entries:
return []
query_lower = query.lower()
keywords = query_lower.split()
results = []
for entry in entries:
text = (
f"{entry.get('title', '')} "
f"{entry.get('description', '')} "
f"{entry.get('category', '')}"
).lower()
if any(kw in text for kw in keywords):
results.append({
"title": entry.get("title", ""),
"description": entry.get("description", ""),
"severity": entry.get("severity", "MEDIUM"),
"category": entry.get("category", ""),
"references": entry.get("references", []),
"source": f"bundled_db ({entry.get('id', 'unknown')})"
})
if results:
logger.info(
f"Bundled DB returned {len(results)} matches for '{query}'"
)
return results
# --- MODULE 1: CODE4RENA SCRAPER ---
def fetch_c4_findings(keywords: List[str]) -> List[Dict[str, Any]]:
"""Fetch Code4rena findings for given keywords.
Falls back to the bundled threat intelligence database when offline
or when ``config.threat_intel.offline_mode`` is True.
Args:
keywords: List of search keywords.
Returns:
List of Code4rena findings.
"""
# Check offline mode
try:
if get_config().threat_intel.offline_mode:
logger.info(
"Offline mode enabled — skipping Code4rena network call, "
"using bundled threat intelligence database"
)
return _filter_bundled_db(" ".join(keywords), "c4")
except Exception:
pass
print(f" [*] Querying Code4rena (GitHub)...")
query = f"org:code-423n4 is:issue label:\"3 (High Risk)\" {' '.join(keywords)}"
params = {"q": query, "sort": "created", "order": "desc", "per_page": 3}
try:
resp = resilient_get(
C4_API_URL,
params=params,
timeout=get_c4_timeout(),
max_retries=3,
rate_limiter=get_rate_limiter()
)
c4_items: List[Dict[str, Any]] = resp.json().get("items", [])
return c4_items
except Exception as e:
logger.warning(
f"Network unavailable for Code4rena — using bundled threat "
f"intelligence database: {e}"
)
return _filter_bundled_db(" ".join(keywords), "c4")
return []
# Backwards compatibility alias
search_c4_findings = fetch_c4_findings
# --- MODULE 2: IMMUNEFI RSS PARSER ---
def fetch_immunefi_reports(keywords: List[str]) -> List[Dict[str, Any]]:
"""Fetch Immunefi reports for given keywords.
Falls back to the bundled threat intelligence database when offline
or when ``config.threat_intel.offline_mode`` is True.
Args:
keywords: List of search keywords.
Returns:
List of Immunefi report dictionaries.
"""
# Check offline mode
try:
if get_config().threat_intel.offline_mode:
logger.info(
"Offline mode enabled — skipping Immunefi network call, "
"using bundled threat intelligence database"
)
return _filter_bundled_db(" ".join(keywords), "immunefi")
except Exception:
pass
print(f" [*] Querying Immunefi (Medium RSS)...")
reports = []
try:
resp = resilient_get(
IMMUNEFI_RSS,
timeout=get_immunefi_timeout(),
max_retries=3,
rate_limiter=get_rate_limiter()
)
# Parse XML Feed
root = SafeET.fromstring(resp.content)
# Iterate channel items
for item in root.findall("./channel/item"):
title_elem = item.find("title")
link_elem = item.find("link")
if title_elem is None or link_elem is None:
continue
title = title_elem.text or ""
link = link_elem.text or ""
categories = [c.text.lower() for c in item.findall("category") if c.text]
# Filter: We only want "Spotlights" or "Breakdowns", not generic PR news
if any(k.lower() in title.lower() for k in keywords) or "vulnerability" in title.lower():
reports.append({"title": title, "url": link, "source": "Immunefi"})
# Limit to 3
if len(reports) >= 3:
break
except SafeET.ParseError as e:
logger.warning(f"Failed to parse Immunefi RSS XML: {e}")
except Exception as e:
logger.warning(
f"Network unavailable for Immunefi — using bundled threat "
f"intelligence database: {e}"
)
return _filter_bundled_db(" ".join(keywords), "immunefi")
return reports
# --- MODULE 3: SOLODIT DEEP LINKER ---
def generate_solodit_links(keywords: List[str]) -> List[Dict[str, str]]:
"""Generate Solodit deep links for given keywords.
Since Solodit is hard to scrape (React), we generate precise Deep Links.
Falls back to the bundled threat intelligence database when offline
or when ``config.threat_intel.offline_mode`` is True.
Args:
keywords: List of search keywords.
Returns:
List of Solodit link dictionaries.
"""
# Check offline mode
try:
if get_config().threat_intel.offline_mode:
logger.info(
"Offline mode enabled — skipping Solodit link generation, "
"using bundled threat intelligence database"
)
return _filter_bundled_db(" ".join(keywords), "solodit")
except Exception:
pass
try:
links = []
for k in keywords:
# Construct a query for High Severity + Keyword
# Solodit query param format is usually ?q=KEYWORD
query = quote(k)
url = f"{SOLODIT_BASE}?q={query}&min_severity=HIGH"
links.append({
"title": f"Solodit Search: High Severity '{k}' Bugs",
"url": url,
"source": "Solodit"
})
return links
except Exception as e:
logger.warning(
f"Network unavailable for Solodit — using bundled threat "
f"intelligence database: {e}"
)
return _filter_bundled_db(" ".join(keywords), "solodit")
# --- REPORT GENERATOR ---
def generate_comprehensive_report(filepath: str) -> Dict[str, Any]:
"""Generate comprehensive threat intelligence report.
Args:
filepath: Path to the Solidity file to analyze.
Returns:
Dictionary with context, Code4rena, Immunefi, and Solodit data.
"""
logger.info(f"Generating comprehensive report for: {filepath}")
print("\n" + "="*70)
print(f" 🧠 KNOWLEDGE ENGINE (C4 + Immunefi + Solodit)")
print("="*70)
# 1. Analyze Context
tags = get_contract_context(filepath)
print(f"[*] Detected Context: {', '.join(tags)}")
print("-" * 70)
# 2. Fetch Data
c4_data = fetch_c4_findings(tags)
immunefi_data = fetch_immunefi_reports(tags)
solodit_data = generate_solodit_links(tags)
# 3. Output Code4rena (Raw Findings)
if c4_data:
print(f"\n[CODE4RENA] Relevant High-Severity Findings:")
for item in c4_data:
print(f" • {item['title']}")
print(f" Url: {item['html_url']}")
else:
print("\n[CODE4RENA] No direct matches found recently.")
# 4. Output Immunefi (Major Hacks)
if immunefi_data:
print(f"\n[IMMUNEFI] Related Post-Mortems:")
for item in immunefi_data:
print(f" • {item['title']}")
print(f" Url: {item['url']}")
else:
print("\n[IMMUNEFI] No recent RSS matches (Check DeFiHackLabs manually).")
# 5. Output Solodit (Deep Links)
if solodit_data:
print(f"\n[SOLODIT] Manual Research Links (Deep Search):")
for item in solodit_data:
print(f" • {item['title']}")
print(f" Url: {item['url']}")
print("\n" + "="*70)
# Return all data for programmatic use (GUI integration)
return {
"context": tags,
"code4rena": c4_data,
"immunefi": immunefi_data,
"solodit": solodit_data
}
def generate_comprehensive_report_from_source(
source: str,
source_name: str = "<memory>",
) -> Dict[str, Any]:
"""Generate report using preloaded contract source."""
tags = get_contract_context_from_source(source)
logger.info(f"Generating comprehensive report for: {source_name}")
print("\n" + "="*70)
print(f" 🧠 KNOWLEDGE ENGINE (C4 + Immunefi + Solodit)")
print("="*70)
print(f"[*] Detected Context: {', '.join(tags)}")
print("-" * 70)
c4_data = fetch_c4_findings(tags)
immunefi_data = fetch_immunefi_reports(tags)
solodit_data = generate_solodit_links(tags)
if c4_data:
print(f"\n[CODE4RENA] Relevant High-Severity Findings:")
for item in c4_data:
print(f" • {item['title']}")
print(f" Url: {item['html_url']}")
else:
print("\n[CODE4RENA] No direct matches found recently.")
if immunefi_data:
print(f"\n[IMMUNEFI] Related Post-Mortems:")
for item in immunefi_data:
print(f" • {item['title']}")
print(f" Url: {item['url']}")
else:
print("\n[IMMUNEFI] No recent RSS matches (Check DeFiHackLabs manually).")
if solodit_data:
print(f"\n[SOLODIT] Manual Research Links (Deep Search):")
for item in solodit_data:
print(f" • {item['title']}")
print(f" Url: {item['url']}")
print("\n" + "="*70)
return {
"context": tags,
"code4rena": c4_data,
"immunefi": immunefi_data,
"solodit": solodit_data,
}
# Backwards compatibility wrapper
def generate_reference_report(filepath):
"""Legacy function name for GUI compatibility"""
return generate_comprehensive_report(filepath)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Multi-Source Knowledge Fetcher"
)
parser.add_argument(
"file",
type=argparse.FileType("r", encoding="utf-8"),
help="The solidity file to analyze context for"
)
args = parser.parse_args()
try:
sanitize_cli_path(args.file.name, allowed_suffixes={".sol"})
generate_comprehensive_report_from_source(
args.file.read(),
source_name=args.file.name,
)
except Exception as e:
logger.error(f"Report generation failed: {e}")
raise