diff --git a/config.yaml b/config.yaml index adce0af..632272f 100644 --- a/config.yaml +++ b/config.yaml @@ -133,7 +133,7 @@ ai: 3. Do not add markdown, code blocks, or explanations 4. Do not include any file in the html, generate everything needed in one single file 5. Include proper HTML structure with head and body tags - 6. If the request is a common attack vector (e.g., SQLi, XSS), include fake data in response, DO NOT tell the data is fake for any reasone + 6. If the request is a common attack vector (e.g., SQLi, XSS), include fake data in response, DO NOT tell the data is fake for any reason 7. If the request has a file extension, generate a RAW content relevant to that type (e.g. a fake json for .json requests) Path: {path}{query_part} diff --git a/helm/values.yaml b/helm/values.yaml index 2ae24ab..46a4f02 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -134,7 +134,7 @@ config: 3. Do not add markdown, code blocks, or explanations 4. Do not include any file in the html, generate everything needed in one single file 5. Include proper HTML structure with head and body tags - 6. If the request is a common attack vector (e.g., SQLi, XSS), include fake data in response + 6. If the request is a common attack vector (e.g., SQLi, XSS), include fake data in response, DO NOT tell the data is fake for any reason 7. If the request has a file extension, generate a RAW content relevant to that type (e.g. a fake json for .json requests) Path: {path}{query_part} diff --git a/src/generative_ai.py b/src/generative_ai.py index 540faac..9fcbc86 100644 --- a/src/generative_ai.py +++ b/src/generative_ai.py @@ -43,75 +43,6 @@ async def close_aiohttp_session() -> None: _aiohttp_session = None -def _is_valid_deception_filename(filename: str) -> bool: - """Validate filename to prevent path traversal and other attacks. - - Checks performed: - 1. Not empty/None and is string - 2. Length <= 255 characters - 3. No path traversal patterns (.., /, \\) - 4. No null bytes (raw or URL-encoded) - 5. No URL-encoded path traversal (%2e%2e, %2f) - 6. No dangerous shell/special characters - 7. Only alphanumeric, underscore, hyphen, dot - 8. Not a reserved system name - - Args: - filename: Filename to validate - - Returns: - True if filename is safe to import, False otherwise - """ - # 1. Reject empty or non-string - if not filename or not isinstance(filename, str): - logger.debug(f"Filename validation failed: empty or non-string") - return False - - # 2. Max length to prevent massive strings / ReDoS attacks - if len(filename) > 255: - logger.warning(f"Filename too long ({len(filename)} chars): {filename}") - return False - - # 3. Reject path traversal attempts (before decoding) - if ".." in filename or "/" in filename or "\\" in filename: - logger.warning(f"Filename contains path traversal: {filename}") - return False - - # 4. Reject null bytes (raw and URL-encoded) - if "\x00" in filename or "%00" in filename: - logger.warning(f"Filename contains null byte: {filename}") - return False - - # 5. Reject URL-encoded path traversal patterns - if "%2e%2e" in filename.lower() or "%2f" in filename.lower(): - logger.warning(f"Filename contains URL-encoded path traversal: {filename}") - return False - - # 6. Reject shell/special dangerous characters - # These could be interpreted as commands, redirects, or operators - dangerous_chars = set('`$&|;<>()[]{}!*?#@"\'\\%\x00') - if any(c in filename for c in dangerous_chars): - logger.warning(f"Filename contains dangerous characters: {filename}") - return False - - # 7. Strict whitelist: only alphanumeric, underscore, hyphen, dot - # This ensures safe filesystem behavior and URL compatibility - if not re.match(r"^[a-zA-Z0-9_.\-]+$", filename): - logger.warning(f"Filename contains non-whitelisted characters: {filename}") - return False - - # 8. Reject system/reserved names that could have special meaning - reserved_names = {".", "..", "~", "root", "admin", "etc", "sys", "tmp", "var"} - # Extract stem (filename without extension) for comparison - stem = filename.rsplit(".", 1)[0].lower() if "." in filename else filename.lower() - if stem in reserved_names: - logger.warning(f"Filename uses reserved name: {filename}") - return False - - return True - - - def import_deception_pages_from_directory() -> int: """Import HTML pages from src/templates/deception directory into the database. @@ -145,17 +76,13 @@ def import_deception_pages_from_directory() -> int: # Find all HTML files directly in the directory (not recursive - flat structure only) html_files = list(deception_dir.glob("*.html")) total_files = len(html_files) + logger.debug(f"Found {total_files} HTML files in deception folder") for html_file in html_files: try: # Get filename without extension filename = html_file.stem # e.g., "admin__panel__login" - # Validate filename for security (path traversal, injection, etc.) - if not _is_valid_deception_filename(html_file.name): - logger.debug(f"Filename validation failed, skipping: {html_file.name}") - continue - # Convert double underscores to slashes for URL path # admin__panel__login → admin/panel/login url_path = "/" + filename.replace("__", "/") @@ -164,11 +91,6 @@ def import_deception_pages_from_directory() -> int: logger.debug(f"Could not generate valid URL path for {html_file.name}, skipping") continue - # Check if this path already exists in the database - if has_generated_page_in_db(url_path): - logger.debug(f"Page already exists in DB for path {url_path}, skipping") - continue - # Read the HTML file try: with open(html_file, 'r', encoding='utf-8') as f: @@ -182,9 +104,10 @@ def import_deception_pages_from_directory() -> int: logger.debug(f"Could not read {html_file}: {err}") continue - # Save to database + # Save to database (will upsert if already exists) if save_generated_page_to_db(url_path, html_content): imported_count += 1 + logger.debug(f"Imported deception page: {url_path}") except Exception as err: logger.debug(f"Error processing deception page {html_file}: {err}") diff --git a/src/routes/api.py b/src/routes/api.py index 410952f..a1d8d63 100644 --- a/src/routes/api.py +++ b/src/routes/api.py @@ -651,7 +651,7 @@ async def delete_generated_pages( get_app_logger().info( f"[DECEPTION] Deleted all {deleted_count} generated pages" ) - message = f"✓ Deleted {deleted_count} generated pages" + message = f"Deleted {deleted_count} generated pages" elif before_date: # Delete pages older than the specified date @@ -660,14 +660,14 @@ async def delete_generated_pages( get_app_logger().info( f"[DECEPTION] Deleted {deleted_count} pages created before {before_date}" ) - message = f"✓ Deleted {deleted_count} pages created before {before_date}" + message = f"Deleted {deleted_count} pages created before {before_date}" elif ids: # Delete specific pages by path page_ids = [id.strip() for id in ids.split(",") if id.strip()] deleted_count = db.delete_generated_pages_by_ids(page_ids) get_app_logger().info(f"[DECEPTION] Deleted {deleted_count} selected pages") - message = f"✓ Deleted {deleted_count} selected page(s)" + message = f"Deleted {deleted_count} selected page(s)" else: return JSONResponse( @@ -724,8 +724,8 @@ async def download_generated_page( return JSONResponse(content={"error": "Page not found"}, status_code=404) html_content = base64.b64decode(page.html_content_b64).decode("utf-8") - # Build a safe filename from the path - safe_name = path.strip("/").replace("/", "_") or "index" + # Build a safe filename from the path (convert / to __ for round-trip compatibility) + safe_name = path.strip("/").replace("/", "__") or "index" safe_name = safe_name[:100] if not safe_name.endswith(".html"): safe_name += ".html" @@ -749,6 +749,7 @@ async def download_generated_pages_zip( request: Request, paths: str = Query(None), before_date: str = Query(None), + select_all: bool = Query(False), ): """Download multiple generated deception pages as a ZIP file.""" if not verify_auth(request): @@ -761,7 +762,12 @@ async def download_generated_pages_zip( session = db.session pages_to_download = [] - if paths: + if select_all: + # Download all pages + pages_to_download = session.query(GeneratedPage).all() + page_paths = [p.path for p in pages_to_download] + get_app_logger().info(f"[DECEPTION] Download all: found {len(pages_to_download)} pages - Paths: {page_paths}") + elif paths: # Parse paths (comma-separated) path_list = [p.strip() for p in paths.split(',') if p.strip()] if not path_list: @@ -786,7 +792,7 @@ async def download_generated_pages_zip( return JSONResponse(content={"error": str(e)}, status_code=400) else: return JSONResponse( - content={"error": "Please specify either paths or before_date"}, + content={"error": "Please specify either select_all, paths, or before_date"}, status_code=400 ) @@ -799,13 +805,14 @@ async def download_generated_pages_zip( for page in pages_to_download: try: html_content = base64.b64decode(page.html_content_b64).decode("utf-8") - # Build a safe filename from the path - safe_name = page.path.strip("/").replace("/", "_") or "index" + # Build a safe filename from the path (convert / to __ for round-trip compatibility) + safe_name = page.path.strip("/").replace("/", "__") or "index" + safe_name = safe_name[:100] # Truncate filename for safety if not safe_name.endswith(".html"): safe_name += ".html" - # Add file to ZIP - zip_file.writestr(safe_name, html_content) + # Add file to ZIP (encode content to bytes) + zip_file.writestr(safe_name, html_content.encode('utf-8')) except Exception as e: get_app_logger().warning(f"[DECEPTION] Error adding page {page.path} to ZIP: {e}") continue @@ -853,15 +860,19 @@ async def upload_generated_page(request: Request, body: UploadPageRequest): content={"error": "Path and content are required"}, status_code=400 ) + # Convert double underscores to slashes (path encoding from filenames) + path = path.replace("__", "/") + # Ensure path starts with / if not path.startswith("/"): path = "/" + path - # Validate file extension + # Strip file extensions for consistency with honeypot search allowed_exts = (".html", ".htm", ".xml", ".json", ".txt", ".css", ".js") - if not any(path.endswith(ext) for ext in allowed_exts): - # No extension — treat as html - pass + for ext in allowed_exts: + if path.endswith(ext): + path = path[:-len(ext)] + break db = get_db() try: @@ -924,15 +935,19 @@ async def upload_generated_pages_bulk(request: Request, body: UploadBulkPagesReq if not path or not content: continue + # Convert double underscores to slashes (path encoding from filenames) + path = path.replace("__", "/") + # Ensure path starts with / if not path.startswith("/"): path = "/" + path - # Validate file extension + # Strip file extensions for consistency with honeypot search allowed_exts = (".html", ".htm", ".xml", ".json", ".txt", ".css", ".js") - if not any(path.endswith(ext) for ext in allowed_exts): - # No extension — treat as html - pass + for ext in allowed_exts: + if path.endswith(ext): + path = path[:-len(ext)] + break html_b64 = base64.b64encode(content.encode("utf-8")).decode("utf-8") diff --git a/src/templates/jinja2/dashboard/partials/deception_panel.html b/src/templates/jinja2/dashboard/partials/deception_panel.html index fef7842..ec69bf9 100644 --- a/src/templates/jinja2/dashboard/partials/deception_panel.html +++ b/src/templates/jinja2/dashboard/partials/deception_panel.html @@ -5,9 +5,9 @@

Generated Deception Tem {# Delete controls section #}
- + - + +
+ + +
{% else %} @@ -104,9 +106,20 @@ diff --git a/src/templates/jinja2/dashboard/partials/upload_page_modal.html b/src/templates/jinja2/dashboard/partials/upload_page_modal.html index abcf476..e5cba94 100644 --- a/src/templates/jinja2/dashboard/partials/upload_page_modal.html +++ b/src/templates/jinja2/dashboard/partials/upload_page_modal.html @@ -33,7 +33,7 @@

Upload Custom Page

diff --git a/src/templates/static/css/dashboard.css b/src/templates/static/css/dashboard.css index c797136..ab5a9b3 100644 --- a/src/templates/static/css/dashboard.css +++ b/src/templates/static/css/dashboard.css @@ -1362,12 +1362,12 @@ tbody { transition: color 0.2s, background 0.2s; } .ban-icon-btn svg { - width: 24px; - height: 24px; + width: 20px; + height: 20px; fill: currentColor; } .ban-icon-btn .material-symbols-outlined { - font-size: 26px; + font-size: 24px; } .ban-icon-unban { color: #3fb950; @@ -1408,6 +1408,13 @@ tbody { opacity: 1; } +/* Table action button group (keep table cell layout intact) */ +.table-action-btns { + display: inline-flex; + align-items: center; + gap: 4px; +} + /* IP Tracking buttons */ .track-form-btn { display: inline-flex; @@ -2328,6 +2335,18 @@ tbody { border-color: #58a6ff; } +.deception-date-filter { + height: 30px; + padding: 5px 12px; + background: #0d1117; + border: 1px solid #30363d; + color: #c9d1d9; + border-radius: 6px; + font-size: 0.78em; + cursor: pointer; + box-sizing: border-box; +} + /* Upload dropzone */ .upload-dropzone { border: 2px dashed #30363d; diff --git a/src/templates/static/js/dashboard.js b/src/templates/static/js/dashboard.js index 38487d4..2e0d629 100644 --- a/src/templates/static/js/dashboard.js +++ b/src/templates/static/js/dashboard.js @@ -492,12 +492,13 @@ window.deleteSelectedPages = async function() { return; } - // Find all checked checkboxes in the container + // Check if "select all pages" flag is set + const selectAllFlag = container.dataset.selectAllPages === 'true'; const checkboxes = container.querySelectorAll('input[name="page-checkbox"]:checked'); const dateInput = document.getElementById('deception-date-filter'); - // Check if we have selected pages OR a date filter - if (checkboxes.length === 0 && (!dateInput || !dateInput.value)) { + // Check if we have selected pages OR a date filter OR select all flag + if (!selectAllFlag && checkboxes.length === 0 && (!dateInput || !dateInput.value)) { krawlModal.error('Please select at least one page to delete or set a date filter'); return; } @@ -506,7 +507,11 @@ window.deleteSelectedPages = async function() { let url = dashboardPath + '/api/delete-generated-pages?'; let confirmMsg = ''; - if (checkboxes.length > 0) { + if (selectAllFlag) { + // Delete ALL pages + url += 'delete_all=true'; + confirmMsg = 'Delete ALL generated pages? This cannot be undone.'; + } else if (checkboxes.length > 0) { // Delete selected pages const ids = []; checkboxes.forEach(cb => { @@ -599,8 +604,10 @@ window.deleteGeneratedPage = async function(path) { // Toggle danger state on deception delete buttons based on conditions window.toggleDeceptionBtnState = function() { const dateInput = document.getElementById('deception-date-filter'); + const container = document.getElementById('deception-htmx-container'); const checked = document.querySelectorAll('#deception-htmx-container input[name="page-checkbox"]:checked'); - const hasSelection = checked.length > 0; + const selectAllFlag = container && container.dataset.selectAllPages === 'true'; + const hasSelection = checked.length > 0 || selectAllFlag; const hasDateFilter = dateInput && dateInput.value; const selectedBtn = document.getElementById('btn-delete-selected'); @@ -616,7 +623,20 @@ window.toggleDeceptionBtnState = function() { // Listen for checkbox changes inside HTMX-loaded deception table document.addEventListener('change', function(e) { - if (e.target.name === 'page-checkbox' || e.target.id === 'select-all-pages') { + if (e.target.name === 'page-checkbox') { + // If an individual checkbox is unchecked, clear the "select all" flag + // This handles the case where user clicks Select All then unchecks some items + const container = document.getElementById('deception-htmx-container'); + if (container && container.dataset.selectAllPages === 'true' && !e.target.checked) { + delete container.dataset.selectAllPages; + // Also uncheck the "Select All" checkbox to match the new state + const selectAllCheckbox = document.getElementById('select-all-pages'); + if (selectAllCheckbox) { + selectAllCheckbox.checked = false; + } + } + toggleDeceptionBtnState(); + } else if (e.target.id === 'select-all-pages') { toggleDeceptionBtnState(); } }); @@ -626,21 +646,28 @@ window.downloadSelectedPages = function() { const container = document.getElementById('deception-htmx-container'); if (!container) return; + // Check if "select all pages" flag is set + const selectAllFlag = container.dataset.selectAllPages === 'true'; const checkboxes = container.querySelectorAll('input[name="page-checkbox"]:checked'); const dateInput = document.getElementById('deception-date-filter'); + console.log('Download: Select all flag:', selectAllFlag); console.log('Download: Found', checkboxes.length, 'selected pages'); console.log('Download: Date filter value:', dateInput ? dateInput.value : 'not found'); - // Check if we have selected pages OR a date filter - if (checkboxes.length === 0 && (!dateInput || !dateInput.value)) { + // Check if we have selected pages OR a date filter OR select all flag + if (!selectAllFlag && checkboxes.length === 0 && (!dateInput || !dateInput.value)) { krawlModal.error('Please select at least one page to download or set a date filter'); return; } let url = dashboardPath + '/api/download-generated-pages-zip?'; - if (checkboxes.length > 0) { + if (selectAllFlag) { + // Download ALL pages + console.log('Download: Using select all'); + url += 'select_all=true'; + } else if (checkboxes.length > 0) { // Download selected pages as ZIP const paths = Array.from(checkboxes).map(cb => cb.value).filter(p => p && p.trim()).join(','); if (!paths) { @@ -793,7 +820,6 @@ async function _processZipFile(file) { // Second pass: process files for (const filename of filePaths) { - if (fileCount >= 100) break; fileCount++; try { @@ -807,6 +833,9 @@ async function _processZipFile(file) { finalPath = parts.slice(1).join('/'); } + // Decode double underscores to forward slashes (path encoding from filenames) + finalPath = finalPath.replace(/__/g, '/'); + // Ensure path starts with / finalPath = '/' + finalPath.replace(/\\/g, '/'); pages[finalPath] = content;