From dea906864d84134605649f2c199c265b31c5daed Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Tue, 14 Apr 2026 20:15:09 +0530 Subject: [PATCH 01/17] irondrop: migrate away from synchronous model to async with tokio --- Cargo.toml | 5 +- README.md | 8 +- doc/ARCHITECTURE.md | 31 +- doc/DEPLOYMENT.md | 6 +- doc/HTTP_STREAMING.md | 9 +- doc/README.md | 46 +- doc/RFC_OWASP_COMPLIANCE.md | 6 +- doc/TESTING_DOCUMENTATION.md | 21 +- src/fs.rs | 41 +- src/handlers.rs | 14 +- src/http.rs | 1332 ++++++++++-------------- src/search.rs | 102 +- src/server.rs | 742 ++++--------- src/upload.rs | 12 +- tests/async_runtime_starvation_test.rs | 136 +++ tests/config_test.rs | 14 +- tests/direct_upload_test.rs | 2 +- tests/http_parser_test.rs | 34 +- tests/integration_test.rs | 2 +- tests/log_dir_test.rs | 2 +- tests/memory_leak_fix_test.rs | 12 - tests/middleware_test.rs | 6 +- tests/rate_limiter_memory_test.rs | 3 - 23 files changed, 1083 insertions(+), 1503 deletions(-) create mode 100644 tests/async_runtime_starvation_test.rs diff --git a/Cargo.toml b/Cargo.toml index 677ef0e..dba7b5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,13 +15,14 @@ env_logger = "0.11.10" base64 = "0.22.1" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "logging", "tls12"] } rustls-pemfile = "2" +tokio = { version = "1.47", features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "sync", "time"] } +tokio-rustls = "0.26" [dev-dependencies] reqwest = { version = "0.13", features = ["blocking", "json"] } serde_json = "1.0" tempfile = "3.27" -threadpool = "1.8.1" rcgen = "0.14" [profile.release] @@ -38,5 +39,3 @@ overflow-checks = false # Disable integer overflow checks [profile.dist] inherits = "release" lto = "thin" - - diff --git a/README.md b/README.md index 608bfb6..44f44f4 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ IronDrop focuses on predictable behavior, simplicity, and low overhead. Use it t - Basic security features: rate limiting, optional Basic Auth, path safety checks - Native SSL/TLS support via `--ssl-cert` and `--ssl-key` (built-in HTTPS, no reverse proxy required) - Single binary; templates and assets are embedded -- Core engine is dependency-free in critical paths: networking, search, and filesystem access are implemented in-house -- Standard production dependencies are still used where practical (for example `clap`, `log`/`env_logger`, and `rustls`) +- Core HTTP layer is implemented in-house (request parsing, routing, streaming) without an external HTTP framework +- Tokio is used for the async runtime and networking; standard production dependencies are used where practical (for example `clap`, `log`/`env_logger`, `tokio`, and `rustls`/`tokio-rustls`) - Ultra-compact search index option for very large directory trees (tested up to ~10M entries) - WebDAV (RFC 4918 Class 1 + Class 2 core): `OPTIONS`, `PROPFIND`, `PROPPATCH`, `MKCOL`, `PUT`, `DELETE`, `COPY`, `MOVE`, `LOCK`, `UNLOCK` - Enabled only when `--enable-webdav true` (or equivalent config setting) is provided @@ -53,7 +53,7 @@ Current RFC scope limits: Designed to keep memory usage steady and to stream large files without buffering them in memory. The ultra-compact search mode reduces memory for very large directory trees. - Ultra-compact search: approximately ~110 MB of RAM for around 10 million paths; search latency depends on CPU, disk, and query specifics. -- Dependency profile: networking/search/filesystem core paths are dependency-free, while operational dependencies such as `clap`, `log`/`env_logger`, and `rustls` are used as stable standard building blocks. +- Dependency profile: the HTTP implementation is in-house; Tokio provides async scheduling and networking, and dependencies such as `clap`, `log`/`env_logger`, and `rustls`/`tokio-rustls` are used as stable standard building blocks. ## Security @@ -220,7 +220,7 @@ IronDrop offers extensive customization through command-line arguments: | `--enable-webdav` | Enable WebDAV methods (`OPTIONS`, `PROPFIND`, `PROPPATCH`, `MKCOL`, `PUT`, `DELETE`, `COPY`, `MOVE`, `LOCK`, `UNLOCK`) | `--enable-webdav true` | | `--username/--password` | Basic authentication | `--username admin --password secret` | | `-a, --allowed-extensions` | Restrict file types | `-a "*.pdf,*.doc,*.zip"` | -| `-t, --threads` | Worker threads (default: 8) | `-t 16` | +| `-t, --threads` | Tokio runtime worker threads (default: 8) | `-t 16` | | `--config-file` | Use INI configuration file | `--config-file prod.ini` | | `--ssl-cert` | SSL certificate file (PEM) for HTTPS | `--ssl-cert cert.pem` | | `--ssl-key` | SSL private key file (PEM) for HTTPS | `--ssl-key key.pem` | diff --git a/doc/ARCHITECTURE.md b/doc/ARCHITECTURE.md index f30dd5f..29251c5 100644 --- a/doc/ARCHITECTURE.md +++ b/doc/ARCHITECTURE.md @@ -2,20 +2,20 @@ ## Overview -IronDrop is a file server written in Rust. It uses only the standard library for networking and file I/O (no external HTTP framework). This document provides an overview of the system architecture, component interactions, and implementation details. +IronDrop is a file server written in Rust. Its HTTP stack (request parsing, routing, streaming) is implemented in-house without an external HTTP framework, and it uses Tokio for the async runtime and networking. This document provides an overview of the system architecture, component interactions, and implementation details. ## System Architecture ``` ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ CLI Parser │───▶│ Server Init │───▶│Custom Thread Pool│ +│ CLI Parser │───▶│ Server Init │───▶│ Tokio Runtime │ │ (cli.rs) │ │ (main.rs) │ │ (server.rs) │ └─────────────────┘ └─────────────────┘ └─────────────────┘ │ ▼ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ Template Engine │◀───│ HTTP Handler │◀───│ Request Router │ -│ (templates.rs) │ │ (response.rs) │ │ (http.rs) │ +│ (templates.rs) │ │ (response.rs) │ │ (router.rs) │ └─────────────────┘ └─────────────────┘ └─────────────────┘ │ │ │ ▼ ▼ ▼ @@ -47,15 +47,15 @@ IronDrop is a file server written in Rust. It uses only the standard library for - **`config/ini_parser.rs`**: Zero-dependency INI parser for configuration files ### 2. **HTTP Processing Layer** -- **`server.rs`**: Custom thread pool implementation with rate limiting and connection management -- **`http.rs`**: HTTP request parsing, routing, connection handling, and static asset serving +- **`server.rs`**: Tokio runtime ownership, async accept loop, TLS via `tokio-rustls`, rate limiting, and statistics +- **`http.rs`**: HTTP request parsing and response streaming - **`response.rs`**: HTTP response building, MIME type detection, and error page generation - **`router.rs`**: Simple HTTP router with exact and prefix path matching - **`handlers.rs`**: Internal route handlers for health checks, status, uploads, and monitoring - **`middleware.rs`**: Authentication middleware with Basic Auth support ### 3. **File Operations** -- **`fs.rs`**: Directory listing generation, file details, and file system interactions +- **`fs.rs`**: Directory listing generation and file system interactions - **`upload.rs`**: Direct upload handler with memory/disk streaming and atomic operations ### 4. **Search System** @@ -79,7 +79,7 @@ IronDrop is a file server written in Rust. It uses only the standard library for │ ▼ ┌─────────────────┐ - │ Rate Limiting │───[Fail]───▶ 429 Too Many Requests + │ Rate Limiting │───[Fail]───▶ Connection rejected/closed │ Check │ └─────────────────┘ │ [Pass] @@ -416,7 +416,7 @@ Dedicated HTTP streaming tests verify correct behavior: ## Performance Characteristics ### Memory Usage -- **Baseline**: ~3MB + (thread_count × 8KB stack) +- **Baseline**: Baseline + Tokio runtime worker threads + template cache - **Template Cache**: In-memory storage for frequently accessed templates - **Upload Buffer**: HTTP streaming with automatic memory/disk switching - **Small Uploads (≤64MB)**: Direct memory processing for optimal performance @@ -424,7 +424,8 @@ Dedicated HTTP streaming tests verify correct behavior: - **File Operations**: Configurable chunk sizes (default: 1KB) ### Concurrent Processing -- **Thread Pool**: Custom implementation (default: 8 threads) +- **Async Runtime**: Tokio runtime with configurable worker threads (`--threads`) +- **Blocking Isolation**: Filesystem-heavy request handling runs on a blocking pool so network I/O stays responsive - **Upload Handling**: Supports multiple concurrent uploads - **Rate Limiting**: Per-IP tracking with automatic cleanup - **Connection Management**: Efficient file descriptor usage @@ -441,7 +442,7 @@ Dedicated HTTP streaming tests verify correct behavior: ### Scalability Limits - **File Size**: Up to 10GB uploads supported -- **Concurrent Users**: Limited by thread pool and system resources +- **Concurrent Users**: Bounded primarily by OS file descriptors, disk bandwidth, and configured rate limits - **Directory Size**: Efficient handling of large directories - **Template Complexity**: Sub-millisecond variable interpolation @@ -544,10 +545,10 @@ pub enum AppError { ## Deployment Considerations ### Single Binary Deployment -- **Zero Dependencies**: Pure Rust implementation +- **Single Binary**: Self-contained executable with embedded templates and assets - **Embedded Assets**: Templates compiled into binary - **Cross-Platform**: Supports Linux, macOS, and Windows -- **Portable**: Self-contained executable with no external dependencies +- **Portable**: No runtime services required; uses standard Rust crates (Tokio, rustls) for runtime functionality ### Production Hardening - **Security Defaults**: Safe defaults with explicit feature activation @@ -558,7 +559,7 @@ pub enum AppError { ### Scalability Options - **Reverse Proxy**: Can be deployed behind nginx/Apache for additional features - **Load Balancing**: Multiple instances can serve the same content -- **Resource Scaling**: Configurable thread pool and memory limits +- **Resource Scaling**: Configurable Tokio runtime worker threads and rate limits - **Container Deployment**: Docker-friendly single binary ## Future Architecture Considerations @@ -572,9 +573,9 @@ pub enum AppError { ### Performance Optimizations 1. **HTTP/2 Support**: Enhanced protocol capabilities -2. **Async I/O**: Tokio integration for improved concurrency +2. **Async Expansion**: Further async conversion of filesystem-heavy operations where beneficial 3. **Compression**: Built-in gzip/brotli compression 4. **CDN Integration**: Edge caching and global distribution 5. **Database Caching**: Redis integration for session management -This architecture documentation reflects the current state of IronDrop v2.7.0 and serves as a foundation for understanding the system's design principles, implementation details, and operational characteristics. \ No newline at end of file +This architecture documentation reflects the current state of IronDrop v2.7.0 and serves as a foundation for understanding the system's design principles, implementation details, and operational characteristics. diff --git a/doc/DEPLOYMENT.md b/doc/DEPLOYMENT.md index d34bdb7..771d459 100644 --- a/doc/DEPLOYMENT.md +++ b/doc/DEPLOYMENT.md @@ -311,10 +311,10 @@ WantedBy=multi-user.target ### TLS Details - **Protocol versions**: TLS 1.2 and TLS 1.3 -- **Implementation**: rustls (pure Rust, no OpenSSL dependency) +- **Implementation**: rustls via `tokio-rustls` (pure Rust, no OpenSSL dependency) - **Certificate format**: PEM (both certificate and private key) - **Certificate chains**: Supported (include full chain in cert file) -- **Performance**: TLS handshake runs on thread pool workers alongside request handling +- **Performance**: TLS handshake is async on the Tokio runtime; filesystem-heavy request handling is isolated from Tokio worker threads ### Native TLS vs Reverse Proxy @@ -789,4 +789,4 @@ perf record -g irondrop -d /srv/files strace -p $(pgrep irondrop) ``` -This deployment guide provides comprehensive coverage of production deployment scenarios and operational best practices for IronDrop v2.7.0. \ No newline at end of file +This deployment guide provides comprehensive coverage of production deployment scenarios and operational best practices for IronDrop v2.7.0. diff --git a/doc/HTTP_STREAMING.md b/doc/HTTP_STREAMING.md index 16c2bfc..a5b25f9 100644 --- a/doc/HTTP_STREAMING.md +++ b/doc/HTTP_STREAMING.md @@ -220,7 +220,7 @@ irondrop --directory /path/to/uploads ### Temporary File Security -1. **Unique naming**: Process ID and timestamp ensure unique filenames +1. **Unique naming**: Process ID, timestamp, and a monotonic counter ensure unique filenames 2. **Secure location**: Temporary files created in upload directory 3. **Automatic cleanup**: Files removed immediately after processing 4. **Error cleanup**: Files removed even if processing fails @@ -230,7 +230,7 @@ irondrop --directory /path/to/uploads 1. **Size limits**: Existing upload size limits apply to both variants 2. **Memory bounds**: Large uploads don't consume system memory 3. **Disk space**: Temporary files are cleaned up immediately -4. **Concurrent limits**: Thread pool limits prevent resource exhaustion +4. **Concurrent limits**: Rate limiting and blocking isolation prevent resource exhaustion under concurrency ## Migration Guide @@ -247,8 +247,9 @@ match &request.body { Some(RequestBody::Memory(data)) => { // Handle memory-based body } - Some(RequestBody::File(path)) => { + Some(RequestBody::File { path, size }) => { // Handle file-based body + let _ = size; } None => { // Handle missing body @@ -325,4 +326,4 @@ The streaming system integrates with IronDrop's monitoring: - [Architecture Documentation](./ARCHITECTURE.md) - Overall system architecture - [Testing Documentation](./TESTING_DOCUMENTATION.md) - Test suite and validation -The HTTP streaming implementation provides a robust foundation for handling uploads of any size while maintaining optimal performance and resource utilization. \ No newline at end of file +The HTTP streaming implementation provides a robust foundation for handling uploads of any size while maintaining optimal performance and resource utilization. diff --git a/doc/README.md b/doc/README.md index 2c5cb8f..8594698 100644 --- a/doc/README.md +++ b/doc/README.md @@ -288,14 +288,15 @@ Native zero-dependency template engine: variables, conditionals, embedded assets - **Thread-Safe Operations** – Concurrent search operations with background indexing ### ⚡ **Performance & Architecture** -- **Custom Thread Pool** – Native implementation without external dependencies for optimal performance +- **Tokio Async Runtime** – Async accept loop and cooperative scheduling for high concurrency and responsive streaming +- **Blocking Isolation** – Filesystem-heavy handlers run in a blocking context while network I/O stays async - **Comprehensive Error Handling** – Professional error pages with consistent theming and user-friendly messages - **Request Timeout Protection** – Prevents resource exhaustion with configurable timeouts - **Rich Logging** – Each request is tagged with unique IDs and logged at multiple verbosity levels -### 🛠️ **Zero External Dependencies** -- **Pure Rust Implementation** – Networking, HTTP parsing, and template rendering using only Rust's standard library -- **Custom HTTP Client** – Native testing infrastructure without external HTTP libraries +### 🛠️ **Minimal Framework Dependencies** +- **No External HTTP Framework** – HTTP parsing, routing, and streaming are implemented in-house +- **Tokio-Based Networking** – Tokio provides async networking and task scheduling; TLS uses `rustls` via `tokio-rustls` - **Native Template Engine** – Variable interpolation and rendering without template crates - **Built-in MIME Detection** – File type recognition without external MIME libraries @@ -440,7 +441,7 @@ Planned extensions (open to contribution): | `--listen` | `-l` | Bind address | `127.0.0.1` | | `--port` | `-p` | TCP port | `8080` | | `--allowed-extensions` | `-a`| Comma-separated glob patterns | `*.zip,*.txt` | -| `--threads` | `-t` | Thread-pool size | `8` | +| `--threads` | `-t` | Tokio runtime worker threads | `8` | | `--chunk-size` | `-c` | File read buffer in bytes | `1024` | | `--username` | – | Basic-auth user | none | | `--password` | – | Basic-auth password | none | @@ -456,7 +457,7 @@ Planned extensions (open to contribution): |----------|---------|----------| | **Public File Share** | `irondrop -d /srv/files -p 3000 -l 0.0.0.0` | Professional UI, rate limiting, health monitoring | | **Document Repository** | `irondrop -d ./docs -a "*.pdf,*.png,*.jpg"` | Filtered downloads, file type indicators | -| **High-Performance Server** | `irondrop -d ./big -t 16 -c 8192` | Custom thread pool, optimized streaming | +| **High-Performance Server** | `irondrop -d ./big -t 16 -c 8192` | Higher concurrency, optimized streaming | | **Secure Corporate Share** | `irondrop -d ./private --username alice --password s3cret` | Authentication, audit logging, professional design | | **Development Server** | `irondrop -d . -v --detailed-logging` | Debug logging, template development, hot reload | | **Production Monitoring** | `irondrop -d /data -l 0.0.0.0` + health checks at `/_health` | Statistics, uptime monitoring, rate limiting | @@ -506,20 +507,20 @@ IronDrop provides secure, configurable file upload capabilities: ## 🏗️ Architecture Overview -The codebase features a **modular template architecture** with clear separation of concerns. Core modules include `server.rs` for the custom thread-pool listener, `http.rs` for request parsing and static asset serving, `upload.rs` for secure file upload handling, `multipart.rs` for RFC-compliant multipart parsing, `search.rs` and `ultra_compact_search.rs` for the dual-mode search system, `templates.rs` for the native template engine, `fs.rs` for directory operations, and `response.rs` for file streaming and error handling. The `templates/` directory contains organized HTML/CSS/JS assets for directory listing, uploads, and search interfaces. +The codebase features a **modular template architecture** with clear separation of concerns. Core modules include `server.rs` for Tokio runtime ownership, async accept, and TLS, `http.rs` for request parsing and response streaming, `upload.rs` for secure file upload handling, `multipart.rs` for RFC-compliant multipart parsing, `search.rs` for the search subsystem, `templates.rs` for the native template engine, `fs.rs` for directory operations, and `response.rs` for response types and helpers. The `templates/` directory contains organized HTML/CSS/JS assets for directory listing, uploads, and search interfaces. ### System Architecture Flow ``` +-------------------+ +------------------+ +-------------------+ - | CLI Parser | ----> | Server Init | ----> |Custom Thread Pool | + | CLI Parser | ----> | Server Init | ----> | Tokio Runtime | | (cli.rs) | | (main.rs) | | (server.rs) | +-------------------+ +------------------+ +-------------------+ | v +-------------------+ +------------------+ +-------------------+ | Template Engine | <---- | HTTP Handler | <---- | Request Router | - | (templates.rs) | | (response.rs) | | (http.rs) | + | (templates.rs) | | (response.rs) | | (router.rs) | +-------------------+ +------------------+ +-------------------+ | | | v v v @@ -548,7 +549,7 @@ The codebase features a **modular template architecture** with clear separation | v +---------------------+ - | Rate Limiting | --[Fail]--> 429 Too Many Requests + | Rate Limiting | --[Fail]--> Connection rejected/closed | Check | +---------------------+ | [Pass] @@ -603,11 +604,12 @@ src/ ├── main.rs # Entry point ├── lib.rs # Logger + CLI bootstrap ├── cli.rs # Command-line definitions -├── server.rs # Custom thread pool + rate limiting + statistics -├── http.rs # HTTP parsing, routing & static asset serving +├── server.rs # Tokio runtime ownership, async accept, TLS, rate limiting, statistics +├── http.rs # HTTP parsing + response streaming +├── router.rs # Routing and middleware pipeline ├── templates.rs # Native template engine with variable interpolation ├── fs.rs # Directory operations + template-based listing -├── response.rs # File streaming + template-based error pages +├── response.rs # Response types + template-based error pages ├── upload.rs # File upload handling + multipart processing ├── multipart.rs # Multipart form data parsing ├── error.rs # Custom error enum @@ -629,8 +631,8 @@ templates/ └── script.js # Error page enhancements tests/ -├── comprehensive_test.rs # 13 comprehensive tests with custom HTTP client -└── integration_test.rs # 6 integration tests for core functionality +├── integration_test.rs # Integration tests for core functionality +└── async_runtime_starvation_test.rs # Regression test for high-concurrency streaming assets/ ├── error_400.dat # Legacy error assets (now template-based) @@ -853,8 +855,8 @@ Don't know where to start? Here are some **beginner-friendly test contributions: ## 📈 Performance Characteristics ### Runtime Performance -- **Memory Usage**: ~3MB baseline + (thread_count × 8KB stack) + template cache + upload buffer memory -- **Concurrent Connections**: Custom thread pool (default: 8) + rate limiting protection +- **Memory Usage**: Baseline + Tokio runtime worker threads + template cache; large transfers stream without buffering full files in memory +- **Concurrent Connections**: Async networking; primarily bounded by OS file descriptors and rate limiting rather than a fixed worker thread pool - **File Streaming**: Configurable chunk size (default: 1KB) with range request support - **Template Rendering**: Sub-millisecond variable interpolation with built-in caching - **Large Upload Handling**: Supports unlimited file sizes with constant memory usage (~7MB) @@ -871,7 +873,7 @@ Don't know where to start? Here are some **beginner-friendly test contributions: ### Upload Performance - **Upload Processing**: Sub-millisecond file validation and atomic writing -- **Concurrent Uploads**: Integrated with existing thread pool and rate limiting +- **Concurrent Uploads**: Async request-body streaming with filesystem work isolated from Tokio worker threads; rate limiting still applies - **Resource Management**: Dynamic upload directory detection and configurable size limits ### Security & Monitoring Overhead @@ -1095,10 +1097,10 @@ We welcome contributions! Here's how to get started: - **Cross-Platform**: Runs on Linux, macOS, and Windows ### **For Developers** -- **Pure Rust**: No external dependencies, everything built from scratch -- **Comprehensive Tests**: 199 tests across 16 files ensure reliability and stability +- **No External HTTP Framework**: HTTP parsing, routing, and streaming are implemented in-house +- **Comprehensive Tests**: Broad test coverage across unit and integration tests - **Clean Architecture**: Well-documented, modular codebase -- **Performance Focus**: Custom thread pool and optimized file streaming +- **Performance Focus**: Tokio async runtime, optimized file streaming, and memory-conscious search ### **For DevOps Teams** - **Single Binary**: Easy deployment with no runtime dependencies @@ -1129,4 +1131,4 @@ IronDrop is distributed under the **MIT** license; see `LICENSE` for details. **[⭐ Star us on GitHub](https://github.com/dev-harsh1998/IronDrop) • [📖 Read the Docs](./doc/) • [🚀 Get Started](#-quick-start)** - \ No newline at end of file + diff --git a/doc/RFC_OWASP_COMPLIANCE.md b/doc/RFC_OWASP_COMPLIANCE.md index 4df8b51..e9b200f 100644 --- a/doc/RFC_OWASP_COMPLIANCE.md +++ b/doc/RFC_OWASP_COMPLIANCE.md @@ -148,7 +148,7 @@ This document provides a comprehensive analysis of the RFC standards and OWASP s - **Rate Limiting**: 120 requests/minute, 10 concurrent connections per IP (`src/server.rs:496`) - **Request Timeouts**: 30-second timeout for request processing (`src/http.rs:48`) - **Memory Protection**: Request body size limits (10GB max) and header size limits (8KB) -- **Connection Pooling**: Thread pool limits prevent resource exhaustion +- **Concurrency Control**: Async networking plus rate limiting and blocking isolation prevent resource exhaustion ### File Upload Security @@ -162,7 +162,7 @@ This document provides a comprehensive analysis of the RFC standards and OWASP s - **Information Disclosure Prevention**: Generic error messages to clients - **Proper HTTP Status Codes**: Accurate status codes for different error conditions -- **Panic Recovery**: Thread-level panic recovery prevents server crashes (`src/server.rs:685-696`) +- **Panic Recovery**: Request handling isolates panics to avoid crashing the server process - **Resource Cleanup**: Proper cleanup of temporary files and connections ## Security Architecture Summary @@ -213,4 +213,4 @@ The IronDrop server demonstrates exemplary adherence to web security standards a - **Defense in Depth Architecture** with multiple security layers - **Comprehensive Security Testing** ensuring robust protection against common attack vectors -The security architecture is well-designed for a file sharing server, with appropriate controls for the intended use case while maintaining usability and performance. \ No newline at end of file +The security architecture is well-designed for a file sharing server, with appropriate controls for the intended use case while maintaining usability and performance. diff --git a/doc/TESTING_DOCUMENTATION.md b/doc/TESTING_DOCUMENTATION.md index 84d4769..c712ef1 100644 --- a/doc/TESTING_DOCUMENTATION.md +++ b/doc/TESTING_DOCUMENTATION.md @@ -4,13 +4,13 @@ Version 2.7.0 - Test Suite Overview ## Overview -IronDrop includes a comprehensive test suite with **272 automated tests** covering functionality, security scenarios, performance validation, and concurrent operations. Recent improvements include expanded WebDAV RFC suites, hardened path handling, and stronger lock/precondition validation. +IronDrop includes a comprehensive automated test suite covering functionality, security scenarios, performance validation, and concurrent operations. Recent improvements include expanded WebDAV RFC suites, hardened path handling, and concurrency regressions for async streaming. ## Test Architecture ### Test Infrastructure Components -- **Custom HTTP Client**: Native Rust implementation for integration testing +- **HTTP Test Helpers**: Raw TCP clients and lightweight request helpers for integration testing - **Mock File Systems**: Temporary directories and controlled file operations - **Concurrent Testing**: Multi-threaded test scenarios and race condition validation - **Security Validation**: Path traversal, injection, and authentication testing @@ -21,20 +21,20 @@ IronDrop includes a comprehensive test suite with **272 automated tests** coveri | Category | Test Files | Test Count | Coverage | |----------|------------|------------|----------| -| **Core Unit Tests** | `src/*.rs` unit modules | 48 | Core functionality, parser behavior, routing, upload/search internals | -| **Integration/System Tests** | `tests/*.rs` (non-WebDAV) | 167 | Auth, config, uploads, monitoring, middleware, parser, utilities, resilience | -| **WebDAV RFC/Edge Tests** | `tests/webdav*_test.rs` | 57 | RFC 4918 behavior, lock semantics, multistatus/error XML, tree operations | +| **Core Unit Tests** | `src/*.rs` unit modules | – | Core functionality, parser behavior, routing, upload/search internals | +| **Integration/System Tests** | `tests/*.rs` (non-WebDAV) | – | Auth, config, uploads, monitoring, middleware, parser, utilities, resilience | +| **WebDAV RFC/Edge Tests** | `tests/webdav*_test.rs` | – | RFC 4918 behavior, lock semantics, multistatus/error XML, tree operations | -**Total Tests: 272** +Run `cargo test --all-features` for the authoritative count. ## Detailed Test Coverage ### Core Server & Unit Tests -**Purpose**: Validates server internals, threadpool behavior, routing, and upload logic +**Purpose**: Validates server internals, Tokio runtime behavior, routing, and upload logic **Key Test Areas**: -- Thread pool initialization and panic behavior +- Async streaming does not starve small requests under concurrency - Router path matching and method handling - Upload path resolution and limits - Config parsing and validation units @@ -42,8 +42,7 @@ IronDrop includes a comprehensive test suite with **272 automated tests** coveri **Critical Tests**: ```rust #[test] -#[should_panic(expected = "assertion failed: size > 0")] -fn threadpool_zero_size_panics() // Deterministic panic text for zero-size pools +fn test_large_downloads_do_not_starve_health_requests() ``` ### Configuration System Tests (`config_test.rs`) @@ -437,4 +436,4 @@ fn test_new_feature() { *This document is part of the IronDrop v2.7.0 documentation suite. The test suite is continuously evolving to ensure comprehensive coverage and reliability.* -Return to documentation index: [./README.md](./README.md) \ No newline at end of file +Return to documentation index: [./README.md](./README.md) diff --git a/src/fs.rs b/src/fs.rs index af032f4..9552227 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -5,9 +5,8 @@ use crate::error::AppError; use crate::templates::TemplateEngine; use crate::utils::is_hidden_file; use log::{debug, trace}; -use std::fs::{self, File}; -use std::io; -use std::path::{Path, PathBuf}; +use std::fs; +use std::path::Path; use std::time::SystemTime; /// Enhanced directory listing using modular templates - dark mode only @@ -173,39 +172,3 @@ fn format_timestamp(timestamp: u64) -> String { format!("{:04}-{:02}-{:02}", year, month.min(12), day.min(31)) } } - -/// Holds details about a file to be streamed. -pub struct FileDetails { - pub path: PathBuf, - pub file: File, - pub size: u64, - pub chunk_size: usize, -} - -impl FileDetails { - pub fn new(path: PathBuf, chunk_size: usize) -> Result { - debug!("Opening file for streaming: {}", path.display()); - trace!("Chunk size: {} bytes", chunk_size); - - let file = File::open(&path)?; - let metadata = file.metadata()?; - let size = metadata.len(); - - debug!( - "File opened successfully: {} bytes, chunk size: {}", - size, chunk_size - ); - trace!( - "File metadata - size: {}, is_file: {}", - size, - metadata.is_file() - ); - - Ok(FileDetails { - path, - file, - size, - chunk_size, - }) - } -} diff --git a/src/handlers.rs b/src/handlers.rs index 10e296c..f3a0cc6 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -468,7 +468,7 @@ pub fn handle_file_request( request.method, request.path ); trace!("Base directory: {:?}, chunk size: {}", base_dir, chunk_size); - use crate::fs::{FileDetails, generate_directory_listing}; + use crate::fs::generate_directory_listing; use crate::response::get_mime_type; use log::debug; use std::path::PathBuf; @@ -616,12 +616,12 @@ pub fn handle_file_request( trace!("File extension validation passed"); - let file_details = FileDetails::new(full_path.clone(), chunk_size)?; + let size = std::fs::metadata(&full_path)?.len(); let mime_type = get_mime_type(&full_path); debug!( "File details - size: {} bytes, mime_type: {}", - file_details.size, mime_type + size, mime_type ); trace!("Chunk size for streaming: {}", chunk_size); Ok(Response { @@ -630,7 +630,7 @@ pub fn handle_file_request( headers: { let mut map = HashMap::new(); map.insert("Content-Type".to_string(), mime_type.to_string()); - map.insert("Content-Length".to_string(), file_details.size.to_string()); + map.insert("Content-Length".to_string(), size.to_string()); map.insert("Accept-Ranges".to_string(), "bytes".to_string()); map.insert( "Cache-Control".to_string(), @@ -638,7 +638,11 @@ pub fn handle_file_request( ); map }, - body: ResponseBody::Stream(file_details), + body: ResponseBody::Stream(crate::http::StreamBody { + path: full_path.clone(), + size, + chunk_size, + }), }) } else { Err(AppError::NotFound) diff --git a/src/http.rs b/src/http.rs index 2931d53..bfee09a 100644 --- a/src/http.rs +++ b/src/http.rs @@ -3,18 +3,15 @@ //! Handles HTTP request parsing, routing, and response generation. use crate::error::AppError; -use crate::fs::FileDetails; use crate::response::create_error_response; use crate::router::Router; -use log::{debug, error, info, trace, warn}; -use rustls; +use log::{debug, error, info, trace}; use std::collections::HashMap; -use std::fs::File; -use std::io::prelude::*; -use std::net::TcpStream; use std::path::PathBuf; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; /// Maximum size for request body (10GB) to prevent memory exhaustion attacks const MAX_REQUEST_BODY_SIZE: usize = 10 * 1024 * 1024 * 1024; @@ -25,56 +22,7 @@ const MAX_HEADERS_SIZE: usize = 8 * 1024; /// Threshold for streaming request bodies to disk (64MB) /// This ensures total memory usage stays well below 128MB pub const STREAM_TO_DISK_THRESHOLD: usize = 64 * 1024 * 1024; - -/// Abstraction over plain TCP and TLS-encrypted streams. -/// Allows the HTTP handling code to work transparently with both. -pub enum ClientStream { - Plain(TcpStream), - Tls(Box>), -} - -impl ClientStream { - /// Get the peer address of the underlying TCP connection - pub fn peer_addr(&self) -> std::io::Result { - match self { - ClientStream::Plain(s) => s.peer_addr(), - ClientStream::Tls(s) => s.sock.peer_addr(), - } - } - - /// Set the read timeout on the underlying TCP connection - pub fn set_read_timeout(&self, dur: Option) -> std::io::Result<()> { - match self { - ClientStream::Plain(s) => s.set_read_timeout(dur), - ClientStream::Tls(s) => s.sock.set_read_timeout(dur), - } - } -} - -impl std::io::Read for ClientStream { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - match self { - ClientStream::Plain(s) => s.read(buf), - ClientStream::Tls(s) => s.read(buf), - } - } -} - -impl std::io::Write for ClientStream { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - match self { - ClientStream::Plain(s) => s.write(buf), - ClientStream::Tls(s) => s.write(buf), - } - } - - fn flush(&mut self) -> std::io::Result<()> { - match self { - ClientStream::Plain(s) => s.flush(), - ClientStream::Tls(s) => s.flush(), - } - } -} +static TEMP_FILE_COUNTER: AtomicU64 = AtomicU64::new(0); /// Represents a parsed incoming HTTP request. #[derive(Debug)] @@ -119,12 +67,18 @@ pub struct Response { pub body: ResponseBody, } +pub struct StreamBody { + pub path: PathBuf, + pub size: u64, + pub chunk_size: usize, +} + pub enum ResponseBody { Text(String), StaticText(&'static str), Binary(Vec), StaticBinary(&'static [u8]), - Stream(FileDetails), + Stream(StreamBody), } impl Request { @@ -151,30 +105,16 @@ impl Request { ) } - /// Enhanced HTTP request parser with better performance and compliance - pub fn from_stream(stream: &mut ClientStream) -> Result { - trace!("Starting HTTP request parsing from stream"); - // Set a reasonable timeout for reading requests - stream.set_read_timeout(Some(std::time::Duration::from_secs(30)))?; - - // Read the entire HTTP headers in chunks for better performance - let (headers_data, remaining_bytes) = Self::read_headers_with_remaining(stream)?; - debug!( - "Received headers ({} bytes), remaining buffer: {} bytes", - headers_data.len(), - remaining_bytes.len() - ); - - // Parse the headers + pub async fn from_async_stream(stream: &mut S) -> Result + where + S: tokio::io::AsyncRead + Unpin, + { + let (headers_data, remaining_bytes) = read_headers_with_remaining_async(stream).await?; let mut lines = headers_data.lines(); - // Parse request line let request_line = lines.next().ok_or(AppError::BadRequest)?; - trace!("Request line: {}", request_line); let parts: Vec<&str> = request_line.split_whitespace().collect(); - if parts.len() != 3 { - debug!("Invalid request line format: {}", request_line); return Err(AppError::BadRequest); } @@ -182,43 +122,27 @@ impl Request { let raw_path = parts[1]; let version = parts[2]; - // Validate HTTP method if !Self::is_valid_http_method(&method) { - debug!("Invalid HTTP method: {}", method); return Err(AppError::BadRequest); } - - // Validate path doesn't contain null bytes or other invalid characters if raw_path.contains('\0') || raw_path.is_empty() { - debug!("Invalid path: contains null byte or is empty"); return Err(AppError::BadRequest); } let path = Self::decode_url(raw_path)?; - - debug!("Parsed request: {} {}", method, path); - trace!("Raw path before decoding: {}", raw_path); - trace!("HTTP version: {}", version); - - // Validate HTTP version if !version.starts_with("HTTP/1.") { return Err(AppError::BadRequest); } - // Parse headers let mut headers = HashMap::new(); for line in lines { let line = line.trim(); if line.is_empty() { break; } - if let Some((key, value)) = line.split_once(':') { let key = key.trim().to_lowercase(); let value = value.trim().to_string(); - trace!("Header: {} = {}", key, value); - - // Handle multiple header values (comma-separated) if let Some(existing) = headers.get(&key) { headers.insert(key, format!("{existing}, {value}")); } else { @@ -227,28 +151,7 @@ impl Request { } } - // Read request body if present - let body = Self::read_request_body(stream, &headers, remaining_bytes)?; - - if let Some(ref body) = body { - debug!("Request body parsed: {} bytes", body.len()); - match body { - RequestBody::Memory(_) => trace!("Body stored in memory"), - RequestBody::File { path, size } => { - trace!("Body streamed to file: {} ({} bytes)", path.display(), size); - } - } - } else { - trace!("No request body"); - } - - debug!( - "Parsed request: {} {} (headers: {}, body_size: {})", - method, - path, - headers.len(), - body.as_ref().map(|b| b.len()).unwrap_or(0) - ); + let body = read_request_body_async(stream, &headers, remaining_bytes).await?; Ok(Request { method, @@ -258,134 +161,6 @@ impl Request { }) } - /// Read HTTP headers efficiently in chunks and return remaining bytes from body - fn read_headers_with_remaining( - stream: &mut ClientStream, - ) -> Result<(String, Vec), AppError> { - let mut buffer = vec![0; MAX_HEADERS_SIZE]; - let mut total_read = 0; - - loop { - match stream.read(&mut buffer[total_read..]) { - Ok(0) => { - if total_read == 0 { - return Err(AppError::BadRequest); - } - break; - } - Ok(bytes_read) => { - total_read += bytes_read; - - // Look for the end of headers (\r\n\r\n or \n\n) in raw bytes - let double_crlf = b"\r\n\r\n"; - let double_lf = b"\n\n"; - - if let Some(pos) = buffer[0..total_read] - .windows(4) - .position(|window| window == double_crlf) - { - let headers_end = pos; - let body_start = pos + 4; - - // Only convert headers portion to UTF-8 - match std::str::from_utf8(&buffer[0..headers_end]) { - Ok(headers_data) => { - let remaining_bytes = buffer[body_start..total_read].to_vec(); - return Ok((headers_data.to_string(), remaining_bytes)); - } - Err(_) => { - return Err(AppError::BadRequest); - } - } - } else if let Some(pos) = buffer[0..total_read] - .windows(2) - .position(|window| window == double_lf) - { - let headers_end = pos; - let body_start = pos + 2; - - // Only convert headers portion to UTF-8 - match std::str::from_utf8(&buffer[0..headers_end]) { - Ok(headers_data) => { - let remaining_bytes = buffer[body_start..total_read].to_vec(); - return Ok((headers_data.to_string(), remaining_bytes)); - } - Err(_) => { - return Err(AppError::BadRequest); - } - } - } - - // Prevent header buffer overflow attacks - if total_read >= buffer.len() { - return Err(AppError::BadRequest); - } - } - Err(e) => return Err(AppError::Io(e)), - } - } - - // No body separator found, return all as headers with empty remaining bytes - match std::str::from_utf8(&buffer[0..total_read]) { - Ok(data) => Ok((data.to_string(), Vec::new())), - Err(_) => Err(AppError::BadRequest), - } - } - - /// Read request body based on Content-Length header with security validations - /// Large bodies are streamed to disk to prevent memory exhaustion - fn read_request_body( - stream: &mut ClientStream, - headers: &HashMap, - remaining_bytes: Vec, - ) -> Result, AppError> { - let has_content_length = headers.contains_key("content-length"); - let has_chunked_transfer = Self::has_chunked_transfer_encoding(headers); - - // RFC 9112: Transfer-Encoding and Content-Length must not be sent together. - if has_content_length && has_chunked_transfer { - return Err(AppError::BadRequest); - } - - // Chunked request bodies are decoded before any method-specific handling. - if has_chunked_transfer { - let body = Self::read_chunked_body(stream, remaining_bytes)?; - return Ok(Some(body)); - } - - // Check if we have a Content-Length header - let content_length = match headers.get("content-length") { - Some(length_str) => match length_str.parse::() { - Ok(length) => length, - Err(_) => return Err(AppError::BadRequest), - }, - None => { - // No body expected - return Ok(None); - } - }; - - // Validate content length against security limits - if content_length == 0 { - return Ok(Some(RequestBody::Memory(Vec::new()))); - } - - if content_length > MAX_REQUEST_BODY_SIZE { - return Err(AppError::PayloadTooLarge(MAX_REQUEST_BODY_SIZE as u64)); - } - - // Decide whether to use memory or disk based on size - if content_length <= STREAM_TO_DISK_THRESHOLD { - // Small body - use memory - Self::read_body_to_memory(stream, content_length, remaining_bytes) - .map(|body| Some(RequestBody::Memory(body))) - } else { - // Large body - stream to disk - Self::read_body_to_disk(stream, content_length, remaining_bytes) - .map(|(path, size)| Some(RequestBody::File { path, size })) - } - } - fn has_chunked_transfer_encoding(headers: &HashMap) -> bool { headers .get("transfer-encoding") @@ -398,322 +173,6 @@ impl Request { .unwrap_or(false) } - fn read_chunked_body( - stream: &mut ClientStream, - mut pending: Vec, - ) -> Result { - const CHUNK_LINE_LIMIT: usize = 8 * 1024; - - let mut total_size: usize = 0; - let mut memory_body: Vec = Vec::new(); - let mut file_sink: Option<(PathBuf, File)> = None; - - loop { - let line = Self::read_crlf_line(stream, &mut pending, CHUNK_LINE_LIMIT)?; - let line_str = std::str::from_utf8(&line).map_err(|_| AppError::BadRequest)?; - let size_token = line_str - .split(';') - .next() - .ok_or(AppError::BadRequest)? - .trim(); - if size_token.is_empty() { - return Err(AppError::BadRequest); - } - - let chunk_size = - usize::from_str_radix(size_token, 16).map_err(|_| AppError::BadRequest)?; - if chunk_size == 0 { - Self::consume_chunked_trailers(stream, &mut pending)?; - break; - } - - let next_total = total_size - .checked_add(chunk_size) - .ok_or(AppError::PayloadTooLarge(MAX_REQUEST_BODY_SIZE as u64))?; - if next_total > MAX_REQUEST_BODY_SIZE { - return Err(AppError::PayloadTooLarge(MAX_REQUEST_BODY_SIZE as u64)); - } - - let chunk_data = Self::read_exact_from_buffer(stream, &mut pending, chunk_size)?; - Self::consume_expected_crlf(stream, &mut pending)?; - - if file_sink.is_none() && next_total <= STREAM_TO_DISK_THRESHOLD { - memory_body.extend_from_slice(&chunk_data); - } else { - if file_sink.is_none() { - let (temp_path, mut temp_file) = Self::create_temp_body_file()?; - if !memory_body.is_empty() { - temp_file.write_all(&memory_body).map_err(|e| { - let _ = std::fs::remove_file(&temp_path); - AppError::from(e) - })?; - memory_body.clear(); - } - file_sink = Some((temp_path, temp_file)); - } - if let Some((temp_path, temp_file)) = file_sink.as_mut() { - temp_file.write_all(&chunk_data).map_err(|e| { - let _ = std::fs::remove_file(temp_path); - AppError::from(e) - })?; - } - } - - total_size = next_total; - } - - if let Some((temp_path, temp_file)) = file_sink.as_mut() { - temp_file.sync_all().map_err(|e| { - let _ = std::fs::remove_file(temp_path); - AppError::from(e) - })?; - } - - if let Some((temp_path, _)) = file_sink { - Ok(RequestBody::File { - path: temp_path, - size: total_size as u64, - }) - } else { - Ok(RequestBody::Memory(memory_body)) - } - } - - fn create_temp_body_file() -> Result<(PathBuf, File), AppError> { - let temp_filename = format!( - "irondrop_request_{}_{:x}.tmp", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() - ); - let temp_path = std::env::temp_dir().join(temp_filename); - let temp_file = File::create(&temp_path).map_err(|e| { - error!("Failed to create temporary file {temp_path:?}: {e}"); - AppError::from(e) - })?; - Ok((temp_path, temp_file)) - } - - fn read_crlf_line( - stream: &mut ClientStream, - pending: &mut Vec, - max_line_len: usize, - ) -> Result, AppError> { - loop { - if let Some(pos) = pending.windows(2).position(|w| w == b"\r\n") { - let line = pending[..pos].to_vec(); - pending.drain(0..pos + 2); - return Ok(line); - } - - if pending.len() > max_line_len + 2 { - return Err(AppError::BadRequest); - } - - let mut buffer = [0u8; 8192]; - match stream.read(&mut buffer) { - Ok(0) => return Err(AppError::BadRequest), - Ok(n) => pending.extend_from_slice(&buffer[..n]), - Err(e) => return Err(AppError::Io(e)), - } - } - } - - fn read_exact_from_buffer( - stream: &mut ClientStream, - pending: &mut Vec, - count: usize, - ) -> Result, AppError> { - while pending.len() < count { - let mut buffer = [0u8; 8192]; - match stream.read(&mut buffer) { - Ok(0) => return Err(AppError::BadRequest), - Ok(n) => pending.extend_from_slice(&buffer[..n]), - Err(e) => return Err(AppError::Io(e)), - } - } - Ok(pending.drain(0..count).collect()) - } - - fn consume_expected_crlf( - stream: &mut ClientStream, - pending: &mut Vec, - ) -> Result<(), AppError> { - let crlf = Self::read_exact_from_buffer(stream, pending, 2)?; - if crlf != b"\r\n" { - return Err(AppError::BadRequest); - } - Ok(()) - } - - fn consume_chunked_trailers( - stream: &mut ClientStream, - pending: &mut Vec, - ) -> Result<(), AppError> { - let mut total_trailer_size = 0usize; - loop { - let line = Self::read_crlf_line(stream, pending, MAX_HEADERS_SIZE)?; - total_trailer_size += line.len() + 2; - if total_trailer_size > MAX_HEADERS_SIZE { - return Err(AppError::BadRequest); - } - - // Empty line marks end of trailers. - if line.is_empty() { - break; - } - } - Ok(()) - } - - /// Read small request body into memory - fn read_body_to_memory( - stream: &mut ClientStream, - content_length: usize, - remaining_bytes: Vec, - ) -> Result, AppError> { - let mut body = Vec::with_capacity(content_length); - - // Use any remaining bytes from header parsing - let bytes_from_headers = remaining_bytes.len().min(content_length); - body.extend_from_slice(&remaining_bytes[..bytes_from_headers]); - - // Calculate how many more bytes we need to read - let bytes_needed = content_length - bytes_from_headers; - - if bytes_needed > 0 { - // Read the remaining body in chunks - let mut bytes_read = 0; - let chunk_size = 8192; // 8KB chunks - let mut buffer = vec![0; chunk_size]; - - while bytes_read < bytes_needed { - let to_read = (bytes_needed - bytes_read).min(chunk_size); - - match stream.read(&mut buffer[..to_read]) { - Ok(0) => { - return Err(AppError::BadRequest); - } - Ok(n) => { - body.extend_from_slice(&buffer[..n]); - bytes_read += n; - } - Err(e) => { - if e.kind() == std::io::ErrorKind::TimedOut { - warn!("Request body read timeout"); - } - return Err(AppError::Io(e)); - } - } - } - } - - // Verify we read exactly the expected amount - if body.len() != content_length { - return Err(AppError::BadRequest); - } - - debug!( - "Successfully read request body to memory: {} bytes", - body.len() - ); - Ok(body) - } - - /// Read large request body directly to disk to prevent memory exhaustion - fn read_body_to_disk( - stream: &mut ClientStream, - content_length: usize, - remaining_bytes: Vec, - ) -> Result<(PathBuf, u64), AppError> { - // Create temporary file for the request body - let temp_filename = format!( - "irondrop_request_{}_{:x}.tmp", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() - ); - - // Use system temp directory - let temp_dir = std::env::temp_dir(); - let temp_path = temp_dir.join(&temp_filename); - - let mut temp_file = File::create(&temp_path).map_err(|e| { - error!("Failed to create temporary file {temp_path:?}: {e}"); - AppError::from(e) - })?; - - let mut total_written = 0; - - // Write any remaining bytes from header parsing - if !remaining_bytes.is_empty() { - let bytes_to_write = remaining_bytes.len().min(content_length); - temp_file - .write_all(&remaining_bytes[..bytes_to_write]) - .map_err(|e| { - let _ = std::fs::remove_file(&temp_path); - AppError::from(e) - })?; - total_written += bytes_to_write; - } - - // Stream remaining bytes directly to disk - let bytes_needed = content_length - total_written; - if bytes_needed > 0 { - let mut bytes_read = 0; - let chunk_size = 64 * 1024; // 64KB chunks for better disk I/O - let mut buffer = vec![0; chunk_size]; - - while bytes_read < bytes_needed { - let to_read = (bytes_needed - bytes_read).min(chunk_size); - - match stream.read(&mut buffer[..to_read]) { - Ok(0) => { - let _ = std::fs::remove_file(&temp_path); - return Err(AppError::BadRequest); - } - Ok(n) => { - temp_file.write_all(&buffer[..n]).map_err(|e| { - let _ = std::fs::remove_file(&temp_path); - AppError::from(e) - })?; - bytes_read += n; - total_written += n; - } - Err(e) => { - let _ = std::fs::remove_file(&temp_path); - if e.kind() == std::io::ErrorKind::TimedOut { - warn!("Request body read timeout"); - } - return Err(AppError::Io(e)); - } - } - } - } - - // Ensure all data is written to disk - temp_file.sync_all().map_err(|e| { - let _ = std::fs::remove_file(&temp_path); - AppError::from(e) - })?; - - // Verify we read exactly the expected amount - if total_written != content_length { - let _ = std::fs::remove_file(&temp_path); - return Err(AppError::BadRequest); - } - - debug!( - "Successfully streamed request body to disk: {} bytes at {temp_path:?}", - total_written - ); - Ok((temp_path, total_written as u64)) - } - /// Simple URL decoding for percent-encoded paths fn decode_url(path: &str) -> Result { let mut decoded = String::with_capacity(path.len()); @@ -747,115 +206,6 @@ impl Request { Ok(decoded) } - - /// Clean up any temporary files associated with this request - pub fn cleanup(&self) { - if let Some(RequestBody::File { path, .. }) = &self.body { - if let Err(e) = std::fs::remove_file(path) { - warn!("Failed to clean up temporary file {path:?}: {e}"); - } else { - debug!("Cleaned up temporary file: {path:?}"); - } - } - } -} - -/// Top-level function to handle a client connection. -#[allow(clippy::too_many_arguments)] -pub fn handle_client( - mut stream: ClientStream, - base_dir: &Arc, - allowed_extensions: &Arc>, - username: &Arc>, - password: &Arc>, - chunk_size: usize, - cli_config: Option<&crate::cli::Cli>, - stats: Option<&crate::server::ServerStats>, - router: &Arc, -) { - let log_prefix = format!("[{}]", stream.peer_addr().unwrap()); - debug!("{} Handling client connection", log_prefix); - trace!( - "{} Client connection established, starting request processing", - log_prefix - ); - - let request = match Request::from_stream(&mut stream) { - Ok(req) => { - debug!( - "{} Successfully parsed request: {} {}", - log_prefix, req.method, req.path - ); - trace!( - "{} Request headers count: {}", - log_prefix, - req.headers.len() - ); - req - } - Err(e) => { - warn!("{log_prefix} Failed to parse request: {e}"); - debug!("{} Sending error response for parse failure", log_prefix); - send_error_response(&mut stream, e, &log_prefix); - return; - } - }; - - let start_time = std::time::Instant::now(); - let response_result = route_request( - &request, - base_dir, - allowed_extensions, - username, - password, - chunk_size, - cli_config, - stats, - router, - ); - let processing_time = start_time.elapsed(); - debug!("{} Request processed in {:?}", log_prefix, processing_time); - - match response_result { - Ok(response) => { - info!( - "{} {} {} -> {}", - log_prefix, request.method, request.path, response.status_code - ); - trace!("{} Response status: {}", log_prefix, response.status_code); - match send_response(&mut stream, response, &log_prefix) { - Ok(body_bytes) => { - trace!( - "{} Response sent successfully, {} bytes", - log_prefix, body_bytes - ); - if let Some(stats) = stats { - stats.record_request(true, body_bytes); - } - } - Err(e) => { - error!("{log_prefix} Failed to send response: {e}"); - if let Some(stats) = stats { - stats.record_request(false, 0); - } - } - } - } - Err(e) => { - warn!("{log_prefix} Error processing request: {e}"); - debug!( - "{} Sending error response for processing failure", - log_prefix - ); - send_error_response(&mut stream, e, &log_prefix); - if let Some(stats) = stats { - stats.record_request(false, 0); - } - } - } - - // Clean up any temporary files created during request processing - request.cleanup(); } // Static asset, favicon, upload, and health handlers moved to handlers.rs @@ -903,29 +253,159 @@ fn route_request( ) } -/// Sends a fully formed `Response` to the client with enhanced headers. -fn send_response( - stream: &mut ClientStream, +#[allow(clippy::too_many_arguments)] +pub async fn handle_client_async( + mut stream: S, + peer_addr: std::net::SocketAddr, + base_dir: Arc, + allowed_extensions: Arc>, + username: Arc>, + password: Arc>, + chunk_size: usize, + cli_config: Option>, + stats: Option>, + router: Arc, +) where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send, +{ + let log_prefix = format!("[{}]", peer_addr); + + let request = match Request::from_async_stream(&mut stream).await { + Ok(req) => req, + Err(e) => { + send_error_response_async(&mut stream, e, &log_prefix).await; + if let Some(stats) = stats { + stats.record_request(false, 0); + } + return; + } + }; + + let cleanup_path = match &request.body { + Some(RequestBody::File { path, .. }) => Some(path.clone()), + _ => None, + }; + + let request_method = request.method.clone(); + let request_path = request.path.clone(); + + let response_result = tokio::task::spawn_blocking({ + let base_dir = base_dir.clone(); + let allowed_extensions = allowed_extensions.clone(); + let username = username.clone(); + let password = password.clone(); + let cli_config = cli_config.clone(); + let router = router.clone(); + move || { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + route_request( + &request, + &base_dir, + &allowed_extensions, + &username, + &password, + chunk_size, + cli_config.as_deref(), + None, + &router, + ) + })) + .unwrap_or_else(|_| { + Err(AppError::InternalServerError( + "Client handler panicked".into(), + )) + }) + } + }) + .await; + + let response_result = match response_result { + Ok(result) => result, + Err(_) => Err(AppError::InternalServerError("Join error".into())), + }; + + match response_result { + Ok(response) => { + info!( + "{} {} {} -> {}", + log_prefix, request_method, request_path, response.status_code + ); + match send_response_async(&mut stream, response, &log_prefix).await { + Ok(body_bytes) => { + if let Some(stats) = stats { + stats.record_request(true, body_bytes); + } + } + Err(_) => { + if let Some(stats) = stats { + stats.record_request(false, 0); + } + } + } + } + Err(e) => { + send_error_response_async(&mut stream, e, &log_prefix).await; + if let Some(stats) = stats { + stats.record_request(false, 0); + } + } + } + + if let Some(path) = cleanup_path { + let _ = tokio::fs::remove_file(path).await; + } +} + +async fn send_error_response_async(stream: &mut S, error: AppError, log_prefix: &str) +where + S: tokio::io::AsyncWrite + Unpin, +{ + let (status_code, status_text) = match error { + AppError::NotFound => (404, "Not Found"), + AppError::Forbidden => (403, "Forbidden"), + AppError::BadRequest => (400, "Bad Request"), + AppError::Unauthorized => (401, "Unauthorized"), + AppError::MethodNotAllowed => (405, "Method Not Allowed"), + AppError::PayloadTooLarge(_) => (413, "Payload Too Large"), + AppError::InvalidFilename(_) => (400, "Bad Request"), + AppError::UploadDiskFull(_) => (507, "Insufficient Storage"), + AppError::UnsupportedMediaType(_) => (415, "Unsupported Media Type"), + AppError::UploadDisabled => (403, "Forbidden"), + _ => (500, "Internal Server Error"), + }; + + info!("{log_prefix} {status_code} {status_text}"); + + let http_response = create_error_response(status_code, status_text); + let mut headers = HashMap::new(); + for (k, v) in http_response.headers { + headers.insert(k, v); + } + + let response = Response { + status_code: http_response.status_code, + status_text: http_response.status_text, + headers, + body: ResponseBody::Binary(http_response.body), + }; + + let _ = send_response_async(stream, response, log_prefix).await; +} + +async fn send_response_async( + stream: &mut S, response: Response, log_prefix: &str, -) -> Result { - info!( - "{} {} {}", - log_prefix, response.status_code, response.status_text - ); - debug!( - "{} Preparing response headers ({} custom headers)", - log_prefix, - response.headers.len() - ); - +) -> Result +where + S: tokio::io::AsyncWrite + Unpin, +{ let mut response_str = format!( "HTTP/1.1 {} {} ", response.status_code, response.status_text ); - // Add standard server headers first response_str.push_str(&format!( "Server: irondrop/{} ", @@ -936,16 +416,13 @@ fn send_response( ", ); - // Add response-specific headers for (key, value) in &response.headers { - trace!("{} Response header: {}: {}", log_prefix, key, value); response_str.push_str(&format!( "{key}: {value} " )); } - // Add Content-Length header ONLY if it is not already present in response.headers let has_content_length = response .headers .keys() @@ -956,7 +433,7 @@ fn send_response( ResponseBody::StaticText(text) => text.len(), ResponseBody::Binary(bytes) => bytes.len(), ResponseBody::StaticBinary(bytes) => bytes.len(), - ResponseBody::Stream(file_details) => file_details.size as usize, + ResponseBody::Stream(stream_body) => stream_body.size as usize, }; response_str.push_str(&format!( "Content-Length: {length} @@ -965,106 +442,451 @@ fn send_response( } response_str.push_str("\r\n"); + stream.write_all(response_str.as_bytes()).await?; - debug!( - "{} Sending response headers ({} bytes)", - log_prefix, - response_str.len() - ); - stream.write_all(response_str.as_bytes())?; - - // Send body and count only body bytes (exclude headers for stats) let mut body_sent: u64 = 0; - debug!("{} Starting body transmission", log_prefix); match response.body { ResponseBody::Text(text) => { let bytes = text.as_bytes(); - trace!("{} Sending {} bytes of text data", log_prefix, bytes.len()); - stream.write_all(bytes)?; + stream.write_all(bytes).await?; body_sent += bytes.len() as u64; } ResponseBody::StaticText(text) => { let bytes = text.as_bytes(); - trace!( - "{} Sending {} bytes of static text", - log_prefix, - bytes.len() - ); - stream.write_all(bytes)?; + stream.write_all(bytes).await?; body_sent += bytes.len() as u64; } ResponseBody::Binary(bytes) => { - trace!( - "{} Sending {} bytes of binary data", - log_prefix, - bytes.len() - ); - stream.write_all(&bytes)?; + stream.write_all(&bytes).await?; body_sent += bytes.len() as u64; } ResponseBody::StaticBinary(bytes) => { - trace!( - "{} Sending {} bytes of static binary data", - log_prefix, - bytes.len() - ); - stream.write_all(bytes)?; + stream.write_all(bytes).await?; body_sent += bytes.len() as u64; } - ResponseBody::Stream(mut file_details) => { - trace!( - "{} Streaming file: {} bytes, chunk size: {}", - log_prefix, file_details.size, file_details.chunk_size - ); - let mut buffer = vec![0; file_details.chunk_size]; - let mut chunks_sent = 0; + ResponseBody::Stream(stream_body) => { + let mut file = tokio::fs::File::open(&stream_body.path).await?; + let mut buffer = vec![0; stream_body.chunk_size]; loop { - let bytes_read = file_details.file.read(&mut buffer)?; + let bytes_read = file.read(&mut buffer).await?; if bytes_read == 0 { break; } - stream.write_all(&buffer[..bytes_read])?; + stream.write_all(&buffer[..bytes_read]).await?; body_sent += bytes_read as u64; - chunks_sent += 1; - if chunks_sent % 100 == 0 { - trace!( - "{} Streamed {} chunks ({} bytes so far)", - log_prefix, chunks_sent, body_sent - ); - } } - debug!( - "{} File streaming completed: {} chunks, {} bytes total", - log_prefix, chunks_sent, body_sent - ); } } - stream.flush()?; + stream.flush().await?; + trace!("{log_prefix} sent {body_sent} bytes"); Ok(body_sent) } -/// Sends a pre-canned error response using the new response system. -fn send_error_response(stream: &mut ClientStream, error: AppError, log_prefix: &str) { - let (status_code, status_text) = match error { - AppError::NotFound => (404, "Not Found"), - AppError::Forbidden => (403, "Forbidden"), - AppError::BadRequest => (400, "Bad Request"), - AppError::Unauthorized => (401, "Unauthorized"), - AppError::MethodNotAllowed => (405, "Method Not Allowed"), - // Upload-specific error mappings - AppError::PayloadTooLarge(_) => (413, "Payload Too Large"), - AppError::InvalidFilename(_) => (400, "Bad Request"), - AppError::UploadDiskFull(_) => (507, "Insufficient Storage"), - AppError::UnsupportedMediaType(_) => (415, "Unsupported Media Type"), - AppError::UploadDisabled => (403, "Forbidden"), - _ => (500, "Internal Server Error"), +async fn read_with_timeout(stream: &mut S, buf: &mut [u8]) -> Result +where + S: tokio::io::AsyncRead + Unpin, +{ + match tokio::time::timeout(Duration::from_secs(30), stream.read(buf)).await { + Ok(result) => result.map_err(AppError::Io), + Err(_) => Err(AppError::Io(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "read timeout", + ))), + } +} + +async fn read_headers_with_remaining_async(stream: &mut S) -> Result<(String, Vec), AppError> +where + S: tokio::io::AsyncRead + Unpin, +{ + let mut buffer = vec![0; MAX_HEADERS_SIZE]; + let mut total_read = 0; + + loop { + let bytes_read = read_with_timeout(stream, &mut buffer[total_read..]).await?; + if bytes_read == 0 { + if total_read == 0 { + return Err(AppError::BadRequest); + } + break; + } + total_read += bytes_read; + + let double_crlf = b"\r\n\r\n"; + let double_lf = b"\n\n"; + + if let Some(pos) = buffer[0..total_read] + .windows(4) + .position(|window| window == double_crlf) + { + let headers_end = pos; + let body_start = pos + 4; + let headers_data = + std::str::from_utf8(&buffer[0..headers_end]).map_err(|_| AppError::BadRequest)?; + let remaining_bytes = buffer[body_start..total_read].to_vec(); + return Ok((headers_data.to_string(), remaining_bytes)); + } + + if let Some(pos) = buffer[0..total_read] + .windows(2) + .position(|window| window == double_lf) + { + let headers_end = pos; + let body_start = pos + 2; + let headers_data = + std::str::from_utf8(&buffer[0..headers_end]).map_err(|_| AppError::BadRequest)?; + let remaining_bytes = buffer[body_start..total_read].to_vec(); + return Ok((headers_data.to_string(), remaining_bytes)); + } + + if total_read >= buffer.len() { + return Err(AppError::BadRequest); + } + } + + let data = std::str::from_utf8(&buffer[0..total_read]).map_err(|_| AppError::BadRequest)?; + Ok((data.to_string(), Vec::new())) +} + +async fn read_request_body_async( + stream: &mut S, + headers: &HashMap, + remaining_bytes: Vec, +) -> Result, AppError> +where + S: tokio::io::AsyncRead + Unpin, +{ + let has_content_length = headers.contains_key("content-length"); + let has_chunked_transfer = Request::has_chunked_transfer_encoding(headers); + + if has_content_length && has_chunked_transfer { + return Err(AppError::BadRequest); + } + + if has_chunked_transfer { + let body = read_chunked_body_async(stream, remaining_bytes).await?; + return Ok(Some(body)); + } + + let content_length = match headers.get("content-length") { + Some(length_str) => length_str + .parse::() + .map_err(|_| AppError::BadRequest)?, + None => return Ok(None), }; - info!("{log_prefix} {status_code} {status_text}"); + if content_length == 0 { + return Ok(Some(RequestBody::Memory(Vec::new()))); + } + + if content_length > MAX_REQUEST_BODY_SIZE { + return Err(AppError::PayloadTooLarge(MAX_REQUEST_BODY_SIZE as u64)); + } + + if content_length <= STREAM_TO_DISK_THRESHOLD { + let body = read_body_to_memory_async(stream, content_length, remaining_bytes).await?; + Ok(Some(RequestBody::Memory(body))) + } else { + let (path, size) = read_body_to_disk_async(stream, content_length, remaining_bytes).await?; + Ok(Some(RequestBody::File { path, size })) + } +} + +async fn read_chunked_body_async( + stream: &mut S, + mut pending: Vec, +) -> Result +where + S: tokio::io::AsyncRead + Unpin, +{ + const CHUNK_LINE_LIMIT: usize = 8 * 1024; + + let mut total_size: usize = 0; + let mut memory_body: Vec = Vec::new(); + let mut file_sink: Option<(PathBuf, tokio::fs::File)> = None; + + loop { + let line = read_crlf_line_async(stream, &mut pending, CHUNK_LINE_LIMIT).await?; + let line_str = std::str::from_utf8(&line).map_err(|_| AppError::BadRequest)?; + let size_token = line_str + .split(';') + .next() + .ok_or(AppError::BadRequest)? + .trim(); + if size_token.is_empty() { + return Err(AppError::BadRequest); + } + + let chunk_size = usize::from_str_radix(size_token, 16).map_err(|_| AppError::BadRequest)?; + if chunk_size == 0 { + consume_chunked_trailers_async(stream, &mut pending).await?; + break; + } + + let next_total = total_size + .checked_add(chunk_size) + .ok_or(AppError::PayloadTooLarge(MAX_REQUEST_BODY_SIZE as u64))?; + if next_total > MAX_REQUEST_BODY_SIZE { + return Err(AppError::PayloadTooLarge(MAX_REQUEST_BODY_SIZE as u64)); + } + + let chunk_data = read_exact_from_buffer_async(stream, &mut pending, chunk_size).await?; + consume_expected_crlf_async(stream, &mut pending).await?; + + if file_sink.is_none() && next_total <= STREAM_TO_DISK_THRESHOLD { + memory_body.extend_from_slice(&chunk_data); + } else { + if file_sink.is_none() { + let (temp_path, mut temp_file) = create_temp_body_file_async().await?; + if !memory_body.is_empty() { + temp_file.write_all(&memory_body).await.map_err(|e| { + let _ = std::fs::remove_file(&temp_path); + AppError::from(e) + })?; + memory_body.clear(); + } + file_sink = Some((temp_path, temp_file)); + } + if let Some((temp_path, temp_file)) = file_sink.as_mut() { + temp_file.write_all(&chunk_data).await.map_err(|e| { + let _ = std::fs::remove_file(temp_path); + AppError::from(e) + })?; + } + } + + total_size = next_total; + } + + if let Some((temp_path, temp_file)) = file_sink.as_mut() { + temp_file.sync_all().await.map_err(|e| { + let _ = std::fs::remove_file(temp_path); + AppError::from(e) + })?; + } + + if let Some((temp_path, _)) = file_sink { + Ok(RequestBody::File { + path: temp_path, + size: total_size as u64, + }) + } else { + Ok(RequestBody::Memory(memory_body)) + } +} + +async fn create_temp_body_file_async() -> Result<(PathBuf, tokio::fs::File), AppError> { + let temp_filename = format!( + "irondrop_request_{}_{:x}_{}.tmp", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(), + TEMP_FILE_COUNTER.fetch_add(1, Ordering::Relaxed) + ); + let temp_path = std::env::temp_dir().join(temp_filename); + let temp_file = tokio::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&temp_path) + .await + .map_err(|e| { + error!("Failed to create temporary file {temp_path:?}: {e}"); + AppError::from(e) + })?; + Ok((temp_path, temp_file)) +} + +async fn read_crlf_line_async( + stream: &mut S, + pending: &mut Vec, + max_line_len: usize, +) -> Result, AppError> +where + S: tokio::io::AsyncRead + Unpin, +{ + loop { + if let Some(pos) = pending.windows(2).position(|w| w == b"\r\n") { + let line = pending[..pos].to_vec(); + pending.drain(0..pos + 2); + return Ok(line); + } + + if pending.len() > max_line_len + 2 { + return Err(AppError::BadRequest); + } - let response = create_error_response(status_code, status_text); - if let Err(e) = response.send(stream, log_prefix) { - error!("{log_prefix} Failed to send error response: {e}"); + let mut buffer = [0u8; 8192]; + let n = read_with_timeout(stream, &mut buffer).await?; + if n == 0 { + return Err(AppError::BadRequest); + } + pending.extend_from_slice(&buffer[..n]); + } +} + +async fn read_exact_from_buffer_async( + stream: &mut S, + pending: &mut Vec, + count: usize, +) -> Result, AppError> +where + S: tokio::io::AsyncRead + Unpin, +{ + while pending.len() < count { + let mut buffer = [0u8; 8192]; + let n = read_with_timeout(stream, &mut buffer).await?; + if n == 0 { + return Err(AppError::BadRequest); + } + pending.extend_from_slice(&buffer[..n]); } + Ok(pending.drain(0..count).collect()) +} + +async fn consume_expected_crlf_async( + stream: &mut S, + pending: &mut Vec, +) -> Result<(), AppError> +where + S: tokio::io::AsyncRead + Unpin, +{ + let crlf = read_exact_from_buffer_async(stream, pending, 2).await?; + if crlf != b"\r\n" { + return Err(AppError::BadRequest); + } + Ok(()) +} + +async fn consume_chunked_trailers_async( + stream: &mut S, + pending: &mut Vec, +) -> Result<(), AppError> +where + S: tokio::io::AsyncRead + Unpin, +{ + let mut total_trailer_size = 0usize; + loop { + let line = read_crlf_line_async(stream, pending, MAX_HEADERS_SIZE).await?; + total_trailer_size += line.len() + 2; + if total_trailer_size > MAX_HEADERS_SIZE { + return Err(AppError::BadRequest); + } + if line.is_empty() { + break; + } + } + Ok(()) +} + +async fn read_body_to_memory_async( + stream: &mut S, + content_length: usize, + remaining_bytes: Vec, +) -> Result, AppError> +where + S: tokio::io::AsyncRead + Unpin, +{ + let mut body = Vec::with_capacity(content_length); + let bytes_from_headers = remaining_bytes.len().min(content_length); + body.extend_from_slice(&remaining_bytes[..bytes_from_headers]); + let bytes_needed = content_length - bytes_from_headers; + + if bytes_needed > 0 { + let mut bytes_read = 0; + let chunk_size = 8192; + let mut buffer = vec![0; chunk_size]; + while bytes_read < bytes_needed { + let to_read = (bytes_needed - bytes_read).min(chunk_size); + let n = read_with_timeout(stream, &mut buffer[..to_read]).await?; + if n == 0 { + return Err(AppError::BadRequest); + } + body.extend_from_slice(&buffer[..n]); + bytes_read += n; + } + } + + if body.len() != content_length { + return Err(AppError::BadRequest); + } + Ok(body) +} + +async fn read_body_to_disk_async( + stream: &mut S, + content_length: usize, + remaining_bytes: Vec, +) -> Result<(PathBuf, u64), AppError> +where + S: tokio::io::AsyncRead + Unpin, +{ + let temp_filename = format!( + "irondrop_request_{}_{:x}_{}.tmp", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(), + TEMP_FILE_COUNTER.fetch_add(1, Ordering::Relaxed) + ); + + let temp_path = std::env::temp_dir().join(&temp_filename); + let mut temp_file = tokio::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&temp_path) + .await + .map_err(|e| { + error!("Failed to create temporary file {temp_path:?}: {e}"); + AppError::from(e) + })?; + + let mut total_written: usize = 0; + if !remaining_bytes.is_empty() { + let bytes_to_write = remaining_bytes.len().min(content_length); + temp_file + .write_all(&remaining_bytes[..bytes_to_write]) + .await + .map_err(|e| { + let _ = std::fs::remove_file(&temp_path); + AppError::from(e) + })?; + total_written += bytes_to_write; + } + + let bytes_needed = content_length - total_written; + if bytes_needed > 0 { + let mut bytes_read = 0usize; + let chunk_size = 64 * 1024; + let mut buffer = vec![0; chunk_size]; + while bytes_read < bytes_needed { + let to_read = (bytes_needed - bytes_read).min(chunk_size); + let n = read_with_timeout(stream, &mut buffer[..to_read]).await?; + if n == 0 { + let _ = std::fs::remove_file(&temp_path); + return Err(AppError::BadRequest); + } + temp_file.write_all(&buffer[..n]).await.map_err(|e| { + let _ = std::fs::remove_file(&temp_path); + AppError::from(e) + })?; + bytes_read += n; + total_written += n; + } + } + + if total_written != content_length { + let _ = std::fs::remove_file(&temp_path); + return Err(AppError::BadRequest); + } + + temp_file.sync_all().await.map_err(|e| { + let _ = std::fs::remove_file(&temp_path); + AppError::from(e) + })?; + + Ok((temp_path, total_written as u64)) } diff --git a/src/search.rs b/src/search.rs index c27b6f5..40ec663 100644 --- a/src/search.rs +++ b/src/search.rs @@ -1438,44 +1438,88 @@ pub fn initialize_search(base_dir: PathBuf) { *global_index = Some(concurrent_index.clone()); } - // Perform initial index build in background - let init_index = concurrent_index.clone(); - thread::spawn(move || { - if let Err(e) = init_index.update_if_needed(true) { - warn!("Failed to build initial ultra-low memory index: {e:?}"); - } - }); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + handle.spawn({ + let init_index = concurrent_index.clone(); + async move { + let res = + tokio::task::spawn_blocking(move || init_index.update_if_needed(true)).await; + match res { + Ok(Ok(())) => {} + Ok(Err(e)) => warn!("Failed to build initial ultra-low memory index: {e:?}"), + Err(e) => warn!("Failed to build initial ultra-low memory index: {e:?}"), + } + } + }); + + handle.spawn({ + let concurrent_index = concurrent_index.clone(); + async move { + let mut cleanup_counter = 0u32; + let mut interval = tokio::time::interval(Duration::from_secs(60)); + loop { + interval.tick().await; + + let idx = concurrent_index.clone(); + let res = + tokio::task::spawn_blocking(move || idx.update_if_needed(false)).await; + match res { + Ok(Ok(())) => {} + Ok(Err(e)) => warn!("Failed to update ultra-low memory index: {e:?}"), + Err(e) => warn!("Failed to update ultra-low memory index: {e:?}"), + } - // Spawn background thread to periodically update the index with memory management - thread::spawn(move || { - let mut cleanup_counter = 0; - loop { - thread::sleep(Duration::from_secs(60)); // Update every minute + cleanup_counter += 1; + if cleanup_counter >= 60 { + cleanup_counter = 0; - if let Err(e) = concurrent_index.update_if_needed(false) { - warn!("Failed to update ultra-low memory index: {e:?}"); - } + if let Ok(mut index_guard) = concurrent_index.index.write() { + index_guard.perform_memory_cleanup(); + } - // Perform more aggressive memory cleanup every hour during long runs - cleanup_counter += 1; - if cleanup_counter >= 60 { - // 60 minutes = 1 hour - cleanup_counter = 0; + if let Ok(mut cache) = concurrent_index.search_cache.try_lock() { + cache.shrink_if_needed(true); + } - // Force memory cleanup - if let Ok(mut index_guard) = concurrent_index.index.write() { - index_guard.perform_memory_cleanup(); + debug!("Completed hourly memory cleanup cycle"); + } } + } + }); + } else { + let init_index = concurrent_index.clone(); + thread::spawn(move || { + if let Err(e) = init_index.update_if_needed(true) { + warn!("Failed to build initial ultra-low memory index: {e:?}"); + } + }); + + thread::spawn(move || { + let mut cleanup_counter = 0; + loop { + thread::sleep(Duration::from_secs(60)); - // Force cache shrinking - if let Ok(mut cache) = concurrent_index.search_cache.try_lock() { - cache.shrink_if_needed(true); // Force aggressive shrinking + if let Err(e) = concurrent_index.update_if_needed(false) { + warn!("Failed to update ultra-low memory index: {e:?}"); } - debug!("Completed hourly memory cleanup cycle"); + cleanup_counter += 1; + if cleanup_counter >= 60 { + cleanup_counter = 0; + + if let Ok(mut index_guard) = concurrent_index.index.write() { + index_guard.perform_memory_cleanup(); + } + + if let Ok(mut cache) = concurrent_index.search_cache.try_lock() { + cache.shrink_if_needed(true); + } + + debug!("Completed hourly memory cleanup cycle"); + } } - } - }); + }); + } info!("Ultra-low memory search subsystem initialized - targeting <100MB for 10M entries"); } diff --git a/src/server.rs b/src/server.rs index 05d395e..54bf307 100644 --- a/src/server.rs +++ b/src/server.rs @@ -4,20 +4,18 @@ use crate::cli::Cli; use crate::config::Config; use crate::error::AppError; use crate::handlers::register_internal_routes; -use crate::http::handle_client; use crate::middleware::AuthMiddleware; use crate::router::Router; use glob::Pattern; -use log::{debug, error, info, trace, warn}; +use log::{debug, info, trace, warn}; use std::collections::HashMap; -use std::net::{IpAddr, SocketAddr, TcpListener}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::net::{IpAddr, SocketAddr}; use std::sync::{Arc, Mutex, mpsc}; -use std::thread; use std::time::{Duration, Instant}; use rustls::ServerConfig; use std::io::BufReader; +use tokio::net::TcpStream as TokioTcpStream; #[cfg(target_os = "linux")] use std::fs; @@ -30,7 +28,6 @@ pub struct RateLimiter { connections: Arc>>, max_requests_per_minute: u32, max_concurrent_per_ip: u32, - cleanup_running: Arc, max_connections_per_ip: u32, } @@ -45,17 +42,12 @@ struct ConnectionInfo { impl RateLimiter { pub fn new(max_requests_per_minute: u32, max_concurrent_per_ip: u32) -> Self { - let rate_limiter = Self { + Self { connections: Arc::new(Mutex::new(HashMap::new())), max_requests_per_minute, max_concurrent_per_ip, - cleanup_running: Arc::new(AtomicBool::new(false)), max_connections_per_ip: 1000, // Limit stored connections per IP - }; - - // Start automatic cleanup timer - rate_limiter.start_cleanup_timer(); - rate_limiter + } } pub fn check_rate_limit(&self, ip: IpAddr) -> bool { @@ -173,49 +165,6 @@ impl RateLimiter { } } - /// Start automatic cleanup timer that runs every 60 seconds - fn start_cleanup_timer(&self) { - if self - .cleanup_running - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - { - let connections = Arc::clone(&self.connections); - let cleanup_running = Arc::clone(&self.cleanup_running); - - thread::spawn(move || { - while cleanup_running.load(Ordering::SeqCst) { - thread::sleep(Duration::from_secs(60)); - - if let Ok(mut connections) = connections.lock() { - let now = Instant::now(); - let initial_count = connections.len(); - - trace!( - "Auto-cleanup checking {} rate limiter entries", - initial_count - ); - - // Clean up entries older than 2 minutes - connections.retain(|_, info| { - now.duration_since(info.last_activity) < Duration::from_secs(120) - }); - - let cleaned_count = initial_count - connections.len(); - if cleaned_count > 0 { - debug!( - "Auto-cleanup removed {} rate limiter entries", - cleaned_count - ); - } else { - trace!("Auto-cleanup: no entries to remove"); - } - } - } - }); - } - } - /// Perform aggressive cleanup when memory pressure is detected pub fn cleanup_on_memory_pressure(&self) { debug!("Starting aggressive cleanup due to memory pressure"); @@ -1059,217 +1008,6 @@ pub struct UploadStats { pub success_rate: f64, } -/// Simple native thread pool implementation -pub struct ThreadPool { - workers: Vec, - sender: Option>, -} - -type Job = Box; - -impl ThreadPool { - pub fn new(size: usize) -> ThreadPool { - assert!(size > 0); - - let (sender, receiver) = mpsc::channel(); - let receiver = Arc::new(Mutex::new(receiver)); - let mut workers = Vec::with_capacity(size); - - for id in 0..size { - workers.push(Worker::new(id, Arc::clone(&receiver))); - } - - ThreadPool { - workers, - sender: Some(sender), - } - } - - pub fn execute(&self, f: F) - where - F: FnOnce() + Send + 'static, - { - let job = Box::new(f); - - if let Some(ref sender) = self.sender - && sender.send(job).is_err() - { - warn!("Failed to send job to thread pool"); - } - } -} - -impl Drop for ThreadPool { - fn drop(&mut self) { - drop(self.sender.take()); - - for worker in &mut self.workers { - if let Some(thread) = worker.thread.take() - && thread.join().is_err() - { - warn!("Worker thread {} panicked", worker.id); - } - } - } -} - -struct Worker { - id: usize, - thread: Option>, -} - -impl Worker { - fn new(id: usize, receiver: Arc>>) -> Worker { - let thread = thread::spawn(move || { - loop { - let message = receiver.lock().unwrap().recv(); - - match message { - Ok(job) => { - job(); - } - Err(_) => { - break; - } - } - } - }); - - Worker { - id, - thread: Some(thread), - } - } -} - -#[cfg(test)] -mod threadpool_tests { - use super::*; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, Barrier, mpsc}; - use std::thread; - use std::time::{Duration, Instant}; - - #[test] - fn threadpool_executes_single_job() { - let pool = ThreadPool::new(2); - let (tx, rx) = mpsc::channel(); - - pool.execute(move || { - tx.send("done").unwrap(); - }); - - let result = rx.recv_timeout(Duration::from_secs(1)); - assert!(result.is_ok(), "expected job execution within 1s"); - assert_eq!(result.unwrap(), "done"); - } - - #[test] - fn threadpool_executes_many_jobs() { - let pool = ThreadPool::new(4); - let (tx, rx) = mpsc::channel(); - let total_jobs = 50; - - for _ in 0..total_jobs { - let tx_clone = tx.clone(); - pool.execute(move || { - tx_clone.send(1_u8).unwrap(); - }); - } - - let mut received = 0; - while received < total_jobs { - rx.recv_timeout(Duration::from_secs(3)) - .expect("timed out waiting for jobs"); - received += 1; - } - - assert_eq!(received, total_jobs); - } - - #[test] - fn threadpool_runs_tasks_concurrently_with_barrier() { - let worker_count = 4; - let pool = ThreadPool::new(worker_count); - let barrier = Arc::new(Barrier::new(worker_count)); - let started = Arc::new(AtomicUsize::new(0)); - let (tx, rx) = mpsc::channel(); - - for _ in 0..worker_count { - let b = barrier.clone(); - let s = started.clone(); - let txc = tx.clone(); - pool.execute(move || { - s.fetch_add(1, Ordering::SeqCst); - b.wait(); - txc.send(()).unwrap(); - }); - } - - let start = Instant::now(); - // All barrier tasks should complete promptly; sequential execution would deadlock - for _ in 0..worker_count { - rx.recv_timeout(Duration::from_secs(2)) - .expect("barrier tasks did not complete"); - } - let elapsed = start.elapsed(); - - assert_eq!(started.load(Ordering::SeqCst), worker_count); - assert!(elapsed < Duration::from_secs(2)); - } - - #[test] - #[should_panic(expected = "assertion failed: size > 0")] - fn threadpool_zero_size_panics() { - let _pool = ThreadPool::new(0); - } - - #[test] - fn threadpool_shutdown_waits_for_workers() { - let messages = 6; - let (tx, rx) = mpsc::channel(); - - { - let pool = ThreadPool::new(3); - for _ in 0..messages { - let tx_clone = tx.clone(); - pool.execute(move || { - thread::sleep(Duration::from_millis(50)); - tx_clone.send(()).unwrap(); - }); - } - // pool dropped here, should wait for all workers to finish - } - - let mut received = 0; - while received < messages { - rx.recv_timeout(Duration::from_secs(2)) - .expect("did not receive all messages after shutdown"); - received += 1; - } - assert_eq!(received, messages); - } - - #[test] - fn threadpool_handles_worker_panic_gracefully() { - let pool = ThreadPool::new(1); - let (tx, rx) = mpsc::channel(); - - pool.execute(|| { - panic!("intentional panic in worker job"); - }); - - pool.execute(move || { - tx.send(42_u8).unwrap(); - }); - - drop(pool); - - let res = rx.recv_timeout(Duration::from_millis(200)); - assert!(res.is_err(), "unexpected job execution after worker panic"); - } -} - /// Load TLS certificates from a PEM file fn load_tls_certs( path: &std::path::Path, @@ -1382,6 +1120,23 @@ pub fn run_server( cli: Cli, shutdown_rx: Option>, addr_tx: Option>, +) -> Result<(), AppError> { + let worker_threads = cli.threads.unwrap_or(8); + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_io() + .enable_time() + .worker_threads(worker_threads) + .max_blocking_threads(worker_threads.saturating_mul(32).max(256)) + .build() + .map_err(|e| AppError::InternalServerError(e.to_string()))?; + + runtime.block_on(run_server_async(cli, shutdown_rx, addr_tx)) +} + +async fn run_server_async( + cli: Cli, + shutdown_rx: Option>, + addr_tx: Option>, ) -> Result<(), AppError> { debug!( "Starting server with configuration: verbose={:?}, detailed_logging={:?}", @@ -1397,8 +1152,6 @@ pub fn run_server( )); } - // Initialize the search subsystem with caching and indexing - debug!("Initializing search subsystem"); crate::search::initialize_search(base_dir.as_ref().clone()); let allowed_extensions = Arc::new( @@ -1409,46 +1162,34 @@ pub fn run_server( .map(|ext| Pattern::new(ext.trim())) .collect::, _>>()?, ); - trace!("Allowed extensions: {} patterns", allowed_extensions.len()); let bind_address = format!( "{}:{}", cli.listen.as_ref().unwrap_or(&"127.0.0.1".to_string()), cli.port.unwrap_or(8080) ); - debug!("Binding server to address: {}", bind_address); - let listener = TcpListener::bind(&bind_address)?; + + let listener = tokio::net::TcpListener::bind(&bind_address).await?; let local_addr = listener.local_addr()?; - listener.set_nonblocking(true)?; - debug!("Server bound successfully to: {}", local_addr); - // Set up TLS configuration if SSL cert and key are provided let tls_config: Option> = if let (Some(cert_path), Some(key_path)) = (&cli.ssl_cert, &cli.ssl_key) { - info!( - "🔒 Setting up TLS with cert: {}, key: {}", - cert_path.display(), - key_path.display() - ); Some(build_tls_config(cert_path, key_path)?) } else { None }; - let is_https = tls_config.is_some(); + let tls_acceptor = tls_config + .as_ref() + .map(|cfg| tokio_rustls::TlsAcceptor::from(cfg.clone())); + let is_https = tls_acceptor.is_some(); - // Initialize security and monitoring systems let webdav_enabled = cli.enable_webdav.unwrap_or(false); let (rate_limit_per_minute, concurrent_per_ip) = if webdav_enabled { (3500, 128) } else { (120, 10) }; - debug!( - "Initializing rate limiter: {} req/min, {} concurrent per IP", - rate_limit_per_minute, concurrent_per_ip - ); let rate_limiter = Arc::new(RateLimiter::new(rate_limit_per_minute, concurrent_per_ip)); - debug!("Initializing server statistics"); let stats = Arc::new(ServerStats::new()); if let Some(tx) = addr_tx @@ -1467,42 +1208,20 @@ pub fn run_server( base_dir.display(), allowed_extensions ); - if is_https { - info!("🔒 TLS/SSL: Enabled"); - } - info!( - "⚡ Security: Rate limiting enabled ({} req/min, {} concurrent per IP)", - rate_limit_per_minute, concurrent_per_ip - ); - info!("📊 Monitoring: Statistics collection enabled"); - info!( - "🧩 WebDAV: {}", - if cli.enable_webdav.unwrap_or(false) { - "Enabled" - } else { - "Disabled" - } - ); - let thread_count = cli.threads.unwrap_or(8); - debug!("Creating thread pool with {} threads", thread_count); - let pool = ThreadPool::new(thread_count); let username = Arc::new(cli.username.clone()); let password = Arc::new(cli.password.clone()); + let chunk_size = cli.chunk_size.unwrap_or(1024); let cli_arc = Arc::new(cli); - // Build shared internal router once (with middleware) - debug!("Building internal router with middleware"); let mut router = Router::new(); if cli_arc.username.is_some() && cli_arc.password.is_some() { - debug!("Adding authentication middleware to router"); crate::templates::AUTH_ENABLED.store(true, std::sync::atomic::Ordering::SeqCst); router.add_middleware(Box::new(AuthMiddleware::new( cli_arc.username.clone(), cli_arc.password.clone(), ))); } - debug!("Registering internal routes"); register_internal_routes( &mut router, Some(cli_arc.clone()), @@ -1511,294 +1230,181 @@ pub fn run_server( ); let shared_router = Arc::new(router); - // Note: Rate limiter now has automatic cleanup timer (every 60 seconds) - // This replaces the old 5-minute cleanup task - - // Start background stats reporting with memory pressure monitoring - debug!("Starting background statistics reporting thread"); - let stats_reporter = stats.clone(); - let rate_limiter_monitor = rate_limiter.clone(); - thread::spawn(move || { - loop { - thread::sleep(Duration::from_secs(300)); // Report every 5 minutes - trace!("Background stats reporting cycle starting"); - let (total, successful, errors, bytes, uptime) = stats_reporter.get_stats(); - let upload_stats = stats_reporter.get_upload_stats(); - let (current_memory, peak_memory, memory_available) = stats_reporter.get_memory_usage(); - - // Check for memory pressure and trigger cleanup if needed - let memory_pressure = stats_reporter.check_memory_pressure(Some(&rate_limiter_monitor)); - - info!( - "📊 Request Stats: {} total ({} successful, {} errors), {:.2} MB served, uptime: {}s", - total, - successful, - errors, - bytes as f64 / 1024.0 / 1024.0, - uptime.as_secs() - ); + tokio::spawn({ + let rate_limiter = rate_limiter.clone(); + async move { + let mut interval = tokio::time::interval(Duration::from_secs(60)); + loop { + interval.tick().await; + rate_limiter.cleanup_old_entries(); + } + } + }); + + tokio::spawn({ + let stats_reporter = stats.clone(); + let rate_limiter_monitor = rate_limiter.clone(); + async move { + let mut interval = tokio::time::interval(Duration::from_secs(300)); + loop { + interval.tick().await; + let (total, successful, errors, bytes, uptime) = stats_reporter.get_stats(); + let upload_stats = stats_reporter.get_upload_stats(); + let (current_memory, peak_memory, memory_available) = + stats_reporter.get_memory_usage(); + let memory_pressure = + stats_reporter.check_memory_pressure(Some(&rate_limiter_monitor)); - if memory_available { - let current_mb = current_memory.unwrap_or(0) as f64 / 1024.0 / 1024.0; - let peak_mb = peak_memory.unwrap_or(0) as f64 / 1024.0 / 1024.0; - let pressure_indicator = if memory_pressure { - " ⚠️ PRESSURE" - } else { - "" - }; info!( - "🧠 Memory Stats: {current_mb:.2} MB current, {peak_mb:.2} MB peak{pressure_indicator}" + "📊 Request Stats: {} total ({} successful, {} errors), {:.2} MB served, uptime: {}s", + total, + successful, + errors, + bytes as f64 / 1024.0 / 1024.0, + uptime.as_secs() ); - // Report rate limiter memory usage - let (limiter_entries, limiter_memory) = rate_limiter_monitor.get_memory_stats(); - if limiter_entries > 0 { + if memory_available { + let current_mb = current_memory.unwrap_or(0) as f64 / 1024.0 / 1024.0; + let peak_mb = peak_memory.unwrap_or(0) as f64 / 1024.0 / 1024.0; + let pressure_indicator = if memory_pressure { + " ⚠️ PRESSURE" + } else { + "" + }; info!( - "🔒 Rate Limiter: {} IP entries, ~{:.2} KB memory", - limiter_entries, - limiter_memory as f64 / 1024.0 + "🧠 Memory Stats: {current_mb:.2} MB current, {peak_mb:.2} MB peak{pressure_indicator}" ); } - } else { - debug!("🧠 Memory Stats: unavailable"); - } - if upload_stats.total_uploads > 0 { - info!( - "📤 Upload Stats: {} uploads ({:.1}% success), {} files, {:.2} MB uploaded, avg: {:.2} MB/file, {:.0}ms/upload, {} concurrent", - upload_stats.total_uploads, - upload_stats.success_rate, - upload_stats.files_uploaded, - upload_stats.upload_bytes as f64 / 1024.0 / 1024.0, - upload_stats.average_upload_size as f64 / 1024.0 / 1024.0, - upload_stats.average_processing_time, - upload_stats.concurrent_uploads - ); + if upload_stats.total_uploads > 0 { + info!( + "📤 Upload Stats: {} uploads ({:.1}% success), {} files, {:.2} MB uploaded, avg: {:.2} MB/file, {:.0}ms/upload, {} concurrent", + upload_stats.total_uploads, + upload_stats.success_rate, + upload_stats.files_uploaded, + upload_stats.upload_bytes as f64 / 1024.0 / 1024.0, + upload_stats.average_upload_size as f64 / 1024.0 / 1024.0, + upload_stats.average_processing_time, + upload_stats.concurrent_uploads + ); + } } } }); - debug!("Entering main server loop"); - 'server_loop: loop { - if let Some(ref rx) = shutdown_rx - && rx.try_recv().is_ok() - { - info!("🛑 Shutdown signal received. Shutting down gracefully."); - break 'server_loop; - } - - match listener.accept() { - Ok((stream, peer_addr)) => { - let client_ip = peer_addr.ip(); - trace!("Accepted connection from: {}", peer_addr); - - // Check rate limits - if !rate_limiter.check_rate_limit(client_ip) { - warn!("🚫 Connection from {client_ip} rejected due to rate limiting"); - drop(stream); // Close connection immediately - continue; - } - trace!("Rate limit check passed for: {}", client_ip); + let mut shutdown_task = shutdown_rx.map(|rx| { + tokio::task::spawn_blocking(move || { + let _ = rx.recv(); + }) + }); - // Ensure the accepted stream is in blocking mode - if let Err(e) = stream.set_nonblocking(false) { - error!("Failed to set stream to blocking mode: {e}"); - rate_limiter.release_connection(client_ip); - continue; + loop { + if let Some(ref mut shutdown_task) = shutdown_task { + tokio::select! { + _ = shutdown_task => { + break; } - trace!("Stream set to blocking mode for: {}", client_ip); - - let ( - base_dir, - allowed_extensions, - username, - password, - chunk_size, - rate_limiter, - stats, - cli_ref, - router, - tls_config_clone, - ) = ( - base_dir.clone(), - allowed_extensions.clone(), - username.clone(), - password.clone(), - cli_arc.chunk_size.unwrap_or(1024), - rate_limiter.clone(), - stats.clone(), - cli_arc.clone(), - shared_router.clone(), - tls_config.clone(), - ); - - trace!("Submitting client {} to thread pool", client_ip); - pool.execute(move || { - trace!("Thread pool worker starting for client: {}", client_ip); - let start_time = Instant::now(); - - // Wrap stream with TLS if configured - let client_stream = if let Some(ref tls_cfg) = tls_config_clone { - match rustls::ServerConnection::new(Arc::clone(tls_cfg)) { - Ok(conn) => { - let tls_stream = rustls::StreamOwned::new(conn, stream); - crate::http::ClientStream::Tls(Box::new(tls_stream)) - } - Err(e) => { - error!("TLS handshake setup failed for {}: {}", client_ip, e); - rate_limiter.release_connection(client_ip); - return; - } - } - } else { - crate::http::ClientStream::Plain(stream) - }; - - let result = handle_client_with_stats( - client_stream, + res = listener.accept() => { + let (stream, peer_addr) = res?; + handle_connection( + stream, peer_addr, - &base_dir, - &allowed_extensions, - &username, - &password, + base_dir.clone(), + allowed_extensions.clone(), + username.clone(), + password.clone(), chunk_size, - &stats, - Some(cli_ref.as_ref()), - &router, - ); - - let processing_time = start_time.elapsed(); - trace!( - "Client {} processing completed in {:?}", - client_ip, processing_time + rate_limiter.clone(), + stats.clone(), + cli_arc.clone(), + shared_router.clone(), + tls_acceptor.clone(), ); - - // Release rate limit connection - rate_limiter.release_connection(client_ip); - - // Log any errors - if let Err(e) = result { - warn!("⚠️ Client handling error: {e}"); - } else { - trace!("Client {} handled successfully", client_ip); - } - }); - } - Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => { - // No connections available, sleep briefly - std::thread::sleep(Duration::from_millis(100)); - } - Err(e) => { - error!("❌ Error accepting connection: {e}"); + } } + } else { + let (stream, peer_addr) = listener.accept().await?; + handle_connection( + stream, + peer_addr, + base_dir.clone(), + allowed_extensions.clone(), + username.clone(), + password.clone(), + chunk_size, + rate_limiter.clone(), + stats.clone(), + cli_arc.clone(), + shared_router.clone(), + tls_acceptor.clone(), + ); } } - // Final stats report - let (total, successful, errors, bytes, uptime) = stats.get_stats(); - let upload_stats = stats.get_upload_stats(); - let (current_memory, peak_memory, memory_available) = stats.get_memory_usage(); - - info!( - "📊 Final Request Stats: {} total ({} successful, {} errors), {:.2} MB served, uptime: {}s", - total, - successful, - errors, - bytes as f64 / 1024.0 / 1024.0, - uptime.as_secs() - ); - - if memory_available { - let current_mb = current_memory.unwrap_or(0) as f64 / 1024.0 / 1024.0; - let peak_mb = peak_memory.unwrap_or(0) as f64 / 1024.0 / 1024.0; - info!("🧠 Final Memory Stats: {current_mb:.2} MB current, {peak_mb:.2} MB peak"); - } else { - info!("🧠 Final Memory Stats: unavailable"); - } - - if upload_stats.total_uploads > 0 { - info!( - "📤 Final Upload Stats: {} uploads ({:.1}% success), {} files, {:.2} MB uploaded, largest: {:.2} MB", - upload_stats.total_uploads, - upload_stats.success_rate, - upload_stats.files_uploaded, - upload_stats.upload_bytes as f64 / 1024.0 / 1024.0, - upload_stats.largest_upload as f64 / 1024.0 / 1024.0 - ); - } - info!("✅ Server shut down gracefully."); Ok(()) } -/// Enhanced client handler with statistics tracking #[allow(clippy::too_many_arguments)] -fn handle_client_with_stats( - stream: crate::http::ClientStream, +fn handle_connection( + stream: TokioTcpStream, peer_addr: SocketAddr, - base_dir: &Arc, - allowed_extensions: &Arc>, - username: &Arc>, - password: &Arc>, + base_dir: Arc, + allowed_extensions: Arc>, + username: Arc>, + password: Arc>, chunk_size: usize, - stats: &ServerStats, - cli_config: Option<&crate::cli::Cli>, - router: &Arc, -) -> Result<(), AppError> { + rate_limiter: Arc, + stats: Arc, + cli_config: Arc, + router: Arc, + tls_acceptor: Option, +) { let client_ip = peer_addr.ip(); - trace!("Starting client handler for: {}", client_ip); - - let start = Instant::now(); - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - trace!("Calling handle_client for: {}", client_ip); - handle_client( - stream, - base_dir, - allowed_extensions, - username, - password, - chunk_size, - cli_config, - Some(stats), - router, - ); - })); - - let success = result.is_ok(); - let processing_time = start.elapsed(); - - trace!( - "Client {} processing result: success={}, time={:?}", - client_ip, success, processing_time - ); - - // On panic, record failure (normal success/failure & bytes recorded in handle_client) - if !success { - error!("Client {} handler panicked, recording failure", client_ip); - stats.record_request(false, 0); - } - - if processing_time > Duration::from_millis(1000) { - warn!( - "⏱️ Slow request from {}: {}ms", - client_ip, - processing_time.as_millis() - ); - } else if processing_time > Duration::from_millis(100) { - debug!( - "Request from {} took {}ms", - client_ip, - processing_time.as_millis() - ); + if !rate_limiter.check_rate_limit(client_ip) { + return; } - if result.is_err() { - error!("Client {} handler panicked with error", client_ip); - return Err(AppError::InternalServerError( - "Client handler panicked".to_string(), - )); - } + tokio::spawn(async move { + let result = if let Some(acceptor) = tls_acceptor { + match acceptor.accept(stream).await { + Ok(tls_stream) => { + crate::http::handle_client_async( + tls_stream, + peer_addr, + base_dir, + allowed_extensions, + username, + password, + chunk_size, + Some(cli_config), + Some(stats.clone()), + router, + ) + .await; + Ok(()) + } + Err(_) => Err(()), + } + } else { + crate::http::handle_client_async( + stream, + peer_addr, + base_dir, + allowed_extensions, + username, + password, + chunk_size, + Some(cli_config), + Some(stats.clone()), + router, + ) + .await; + Ok(()) + }; - trace!("Client {} handler completed successfully", client_ip); - Ok(()) + rate_limiter.release_connection(client_ip); + let _ = result; + }); } diff --git a/src/upload.rs b/src/upload.rs index 1d55623..1393a92 100644 --- a/src/upload.rs +++ b/src/upload.rs @@ -35,6 +35,7 @@ use std::env; use std::fs::{self, File, OpenOptions}; use std::io::{BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::SystemTime; /// Memory threshold: files <= 2MB processed in memory, >2MB streamed to disk @@ -42,6 +43,7 @@ const MEMORY_THRESHOLD: u64 = 2 * 1024 * 1024; // 2MB /// Temporary file prefix for atomic operations const TEMP_FILE_PREFIX: &str = ".irondrop_temp_"; +static TEMP_FILE_COUNTER: AtomicU64 = AtomicU64::new(0); /// Buffer size for streaming operations const STREAM_BUFFER_SIZE: usize = 64 * 1024; // 64KB @@ -475,14 +477,15 @@ impl DirectUploadHandler { // Create temporary file for atomic write let temp_filename = format!( - "{}{}_{}_{:x}.tmp", + "{}{}_{}_{:x}_{}.tmp", TEMP_FILE_PREFIX, std::process::id(), SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .unwrap_or_default() .as_nanos(), - data.len() // Use data length as part of unique identifier + data.len(), + TEMP_FILE_COUNTER.fetch_add(1, Ordering::Relaxed) ); let temp_path = self.target_dir.join(&temp_filename); @@ -585,14 +588,15 @@ impl DirectUploadHandler { // Create temporary file for atomic operation let temp_filename = format!( - "{}{}_{}_{:x}.tmp", + "{}{}_{}_{:x}_{}.tmp", TEMP_FILE_PREFIX, std::process::id(), SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .unwrap_or_default() .as_nanos(), - source_path.to_string_lossy().len() // Use path length as part of unique identifier + source_path.to_string_lossy().len(), + TEMP_FILE_COUNTER.fetch_add(1, Ordering::Relaxed) ); let temp_path = self.target_dir.join(&temp_filename); diff --git a/tests/async_runtime_starvation_test.rs b/tests/async_runtime_starvation_test.rs new file mode 100644 index 0000000..c3031c8 --- /dev/null +++ b/tests/async_runtime_starvation_test.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: MIT + +use irondrop::cli::Cli; +use irondrop::server::run_server; +use std::fs::File; +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; +use tempfile::tempdir; + +struct TestServer { + addr: SocketAddr, + shutdown_tx: mpsc::Sender<()>, + handle: Option>, +} + +impl Drop for TestServer { + fn drop(&mut self) { + if let Some(handle) = self.handle.take() { + let _ = self.shutdown_tx.send(()); + let _ = handle.join(); + } + } +} + +fn start_server(dir: std::path::PathBuf, threads: usize) -> TestServer { + let cli = Cli { + directory: dir, + listen: Some("127.0.0.1".to_string()), + port: Some(0), + allowed_extensions: Some("*.bin".to_string()), + threads: Some(threads), + chunk_size: Some(64 * 1024), + verbose: Some(false), + detailed_logging: Some(false), + username: None, + password: None, + enable_upload: Some(false), + max_upload_size: Some(10240), + enable_webdav: Some(false), + config_file: None, + log_dir: None, + ssl_cert: None, + ssl_key: None, + }; + + let (shutdown_tx, shutdown_rx) = mpsc::channel(); + let (addr_tx, addr_rx) = mpsc::channel(); + + let handle = thread::spawn(move || { + if let Err(e) = run_server(cli, Some(shutdown_rx), Some(addr_tx)) { + eprintln!("server failed: {e}"); + } + }); + + let addr = addr_rx.recv().unwrap(); + TestServer { + addr, + shutdown_tx, + handle: Some(handle), + } +} + +fn read_until_headers_end(stream: &mut TcpStream) -> std::io::Result> { + let mut buf = Vec::new(); + let mut tmp = [0u8; 1024]; + loop { + let n = stream.read(&mut tmp)?; + if n == 0 { + return Ok(buf); + } + buf.extend_from_slice(&tmp[..n]); + if buf.windows(4).any(|w| w == b"\r\n\r\n") || buf.windows(2).any(|w| w == b"\n\n") { + return Ok(buf); + } + if buf.len() > 64 * 1024 { + return Ok(buf); + } + } +} + +#[test] +fn test_large_downloads_do_not_starve_health_requests() { + let dir = tempdir().unwrap(); + let big_path = dir.path().join("big.bin"); + let f = File::create(&big_path).unwrap(); + f.set_len(512 * 1024 * 1024).unwrap(); + + let server = start_server(dir.path().to_path_buf(), 2); + + let mut slow1 = TcpStream::connect(server.addr).unwrap(); + slow1 + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + slow1 + .write_all(b"GET /big.bin HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n") + .unwrap(); + let _ = read_until_headers_end(&mut slow1).unwrap(); + + let mut slow2 = TcpStream::connect(server.addr).unwrap(); + slow2 + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + slow2 + .write_all(b"GET /big.bin HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n") + .unwrap(); + let _ = read_until_headers_end(&mut slow2).unwrap(); + + thread::sleep(Duration::from_millis(200)); + + let start = Instant::now(); + let mut health = TcpStream::connect(server.addr).unwrap(); + health + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + health + .write_all( + b"GET /_irondrop/health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ) + .unwrap(); + + let headers = read_until_headers_end(&mut health).unwrap(); + let elapsed = start.elapsed(); + + let headers_str = String::from_utf8_lossy(&headers); + assert!(headers_str.starts_with("HTTP/1.1 200")); + assert!( + elapsed < Duration::from_millis(500), + "health took {elapsed:?}" + ); + + drop(slow1); + drop(slow2); +} diff --git a/tests/config_test.rs b/tests/config_test.rs index 7c61978..6156bd3 100644 --- a/tests/config_test.rs +++ b/tests/config_test.rs @@ -191,7 +191,7 @@ verbose = false assert_eq!(config.listen, "192.168.1.1"); assert_eq!(config.port, 8888); assert_eq!(config.threads, 4); - assert_eq!(config.verbose, true); + assert!(config.verbose); } #[test] @@ -237,7 +237,7 @@ max_upload_size = 1GB assert_eq!(config.port, 5555); assert_eq!(config.threads, 4); - assert_eq!(config.enable_upload, true); + assert!(config.enable_upload); assert_eq!(config.max_upload_size, 1024 * 1024 * 1024); // 1GB in bytes } @@ -272,13 +272,13 @@ fn test_config_defaults() { assert_eq!(config.port, 8080); assert_eq!(config.threads, 8); assert_eq!(config.chunk_size, 1024); - assert_eq!(config.enable_upload, false); + assert!(!config.enable_upload); assert_eq!(config.max_upload_size, 10240 * 1024 * 1024); // 10GB in bytes assert_eq!(config.username, None); assert_eq!(config.password, None); assert_eq!(config.allowed_extensions, ["*.zip", "*.txt"]); - assert_eq!(config.verbose, false); - assert_eq!(config.detailed_logging, false); + assert!(!config.verbose); + assert!(!config.detailed_logging); } #[test] @@ -377,7 +377,7 @@ directory = {} let config = Config::load(&cli).expect("Failed to load config"); - assert_eq!(config.enable_upload, true); + assert!(config.enable_upload); assert_eq!(config.max_upload_size, 500 * 1024 * 1024); // 500MB in bytes } @@ -697,7 +697,7 @@ fn test_config_malformed_ini_syntax() { ssl_key: None, }; - let result = Config::load(&cli); + let _result = Config::load(&cli); // Should either handle gracefully or return appropriate error // This test ensures no panic occurs diff --git a/tests/direct_upload_test.rs b/tests/direct_upload_test.rs index 8c90dc8..32e4dff 100644 --- a/tests/direct_upload_test.rs +++ b/tests/direct_upload_test.rs @@ -475,7 +475,7 @@ fn test_direct_upload_path_traversal_prevention() { let response = upload_handler.handle_upload(&request, None); // Should either reject the upload or sanitize the filename - if let Ok(response) = response { + if let Ok(_response) = response { // If upload succeeds, verify file is saved in upload directory only let files_in_upload_dir: Vec<_> = fs::read_dir(temp_dir.path()) .unwrap() diff --git a/tests/http_parser_test.rs b/tests/http_parser_test.rs index ca7a388..3638b5a 100644 --- a/tests/http_parser_test.rs +++ b/tests/http_parser_test.rs @@ -1,9 +1,10 @@ // SPDX-License-Identifier: MIT -use irondrop::http::{ClientStream, Request}; +use irondrop::http::Request; use std::io::Write; use std::net::{TcpListener, TcpStream}; use std::thread; +use tokio::net::TcpStream as TokioTcpStream; fn serve_and_parse(request: &str) -> Result { serve_and_parse_bytes(request.as_bytes()) @@ -23,11 +24,18 @@ fn serve_and_parse_bytes(request: &[u8]) -> Result tempfile::NamedTempFile { let mut file = tempfile::NamedTempFile::new().unwrap(); writeln!(file, "[server]").unwrap(); writeln!(file, "port = 8080").unwrap(); - writeln!(file, "").unwrap(); + writeln!(file).unwrap(); writeln!(file, "[logging]").unwrap(); writeln!(file, "log_dir = {}", log_dir).unwrap(); file.flush().unwrap(); diff --git a/tests/memory_leak_fix_test.rs b/tests/memory_leak_fix_test.rs index 97c9e2c..7b5d361 100644 --- a/tests/memory_leak_fix_test.rs +++ b/tests/memory_leak_fix_test.rs @@ -10,8 +10,6 @@ //! 3. Vector capacity management //! 4. Periodic memory cleanup functionality -use std::path::PathBuf; -use std::sync::Arc; use std::time::Duration; #[cfg(test)] @@ -117,16 +115,6 @@ mod tests { #[test] fn test_memory_cleanup_endpoint() { use irondrop::handlers::handle_memory_cleanup_request; - use irondrop::http::Request; - use std::collections::HashMap; - - // Create a mock POST request - let request = Request { - method: "POST".to_string(), - path: "/_irondrop/cleanup-memory".to_string(), - headers: HashMap::new(), - body: None, - }; // Test the handler let response = handle_memory_cleanup_request(); diff --git a/tests/middleware_test.rs b/tests/middleware_test.rs index 2cf8864..f4b5985 100644 --- a/tests/middleware_test.rs +++ b/tests/middleware_test.rs @@ -175,11 +175,7 @@ fn test_auth_middleware_timing_attack_resistance() { // Timing difference should be minimal (within reasonable bounds) // This is a basic check - in practice, constant-time comparison should be used - let timing_diff = if duration1 > duration2 { - duration1 - duration2 - } else { - duration2 - duration1 - }; + let timing_diff = duration1.abs_diff(duration2); // Allow up to 10ms difference (this is quite generous for testing) assert!( diff --git a/tests/rate_limiter_memory_test.rs b/tests/rate_limiter_memory_test.rs index c118528..9eac38f 100644 --- a/tests/rate_limiter_memory_test.rs +++ b/tests/rate_limiter_memory_test.rs @@ -121,7 +121,4 @@ fn test_memory_stats_integration() { // With minimal usage, pressure should not be detected // (unless system memory is already very high) println!("Memory pressure detected: {}", pressure_detected); - - // Test should complete without panicking - assert!(true, "Memory pressure check completed successfully"); } From 8094e5cab2b1fbd01165cf387bf9c533f21c9099 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Tue, 14 Apr 2026 20:26:42 +0530 Subject: [PATCH 02/17] irondrop: add test for the logout button --- tests/integration_test.rs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 47dbfc9..2ec5437 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -155,6 +155,42 @@ fn test_successful_authentication() { assert_eq!(res.text().unwrap(), "hello from test file\n"); } +#[test] +fn test_logout_button_present_when_auth_enabled() { + let server = setup_test_server(Some("user".to_string()), Some("pass".to_string())); + let client = Client::new(); + + let res = client + .get(format!("http://{}/", server.addr)) + .basic_auth("user", Some("pass")) + .send() + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + let body = res.text().unwrap(); + assert!(body.contains("id=\"logoutBtn\"")); + assert!(body.contains("/_irondrop/logout")); + assert!(body.contains("xhr.open('GET', '/_irondrop/logout'")); +} + +#[test] +fn test_logout_endpoint_forces_unauthorized_and_suppresses_logout_button() { + let server = setup_test_server(Some("user".to_string()), Some("pass".to_string())); + let client = Client::new(); + + let res = client + .get(format!("http://{}/_irondrop/logout", server.addr)) + .basic_auth("user", Some("pass")) + .send() + .unwrap(); + assert_eq!(res.status(), StatusCode::UNAUTHORIZED); + assert!(res.headers().contains_key("www-authenticate")); + let content_type = res.headers().get("content-type").unwrap().to_str().unwrap(); + assert!(content_type.contains("text/html")); + let body = res.text().unwrap(); + assert!(body.contains("Logged Out")); + assert!(!body.contains("id=\"logoutBtn\"")); +} + #[test] fn test_error_responses() { let server = setup_test_server(None, None); From b09baaf4bae6c58768665c6b5eb6cf42bca04072 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Tue, 14 Apr 2026 21:19:25 +0530 Subject: [PATCH 03/17] irondrop: sync docs with code * Graphifyy and Antigravity. Signed-off-by: Harshit Jain --- README.md | 45 ++------- doc/API_REFERENCE.md | 173 ++++++----------------------------- doc/ARCHITECTURE.md | 158 ++++++++------------------------ doc/DEPLOYMENT.md | 11 ++- doc/MONITORING.md | 38 ++++---- doc/README.md | 160 ++++++++++++-------------------- doc/RFC_OWASP_COMPLIANCE.md | 50 +++++----- doc/TESTING_DOCUMENTATION.md | 4 +- 8 files changed, 187 insertions(+), 452 deletions(-) diff --git a/README.md b/README.md index 44f44f4..c3a4b57 100644 --- a/README.md +++ b/README.md @@ -63,42 +63,16 @@ Includes native SSL/TLS (HTTPS), rate limiting, optional Basic Auth, basic input Getting started with IronDrop is simple. -### From Source +### From crates.io ```bash -# Clone the repository -git clone https://github.com/dev-harsh1998/IronDrop.git -cd IronDrop - -# Build the release binary -cargo build --release - -# The executable will be in ./target/release/irondrop +cargo install irondrop ``` -### System-Wide Installation (Recommended) - -To use IronDrop from anywhere on your system, install it to a directory in your PATH: +### Latest from Git ```bash -# Linux/macOS - Install to /usr/local/bin (requires sudo) -sudo cp ./target/release/irondrop /usr/local/bin/ - -# Alternative: Install to ~/.local/bin (no sudo required) -mkdir -p ~/.local/bin -cp ./target/release/irondrop ~/.local/bin/ -# Add ~/.local/bin to PATH if not already: -echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc # or ~/.zshrc -source ~/.bashrc # or restart terminal - -# Windows (PowerShell as Administrator) -# Create program directory -New-Item -ItemType Directory -Force -Path "C:\Program Files\IronDrop" -# Copy executable -Copy-Item ".\target\release\irondrop.exe" "C:\Program Files\IronDrop\" -# Add to system PATH (requires restart or new terminal) -$env:PATH += ";C:\Program Files\IronDrop" -[Environment]::SetEnvironmentVariable("PATH", $env:PATH, [EnvironmentVariableTarget]::Machine) +cargo install --git https://github.com/dev-harsh1998/IronDrop.git ``` **Verify Installation:** @@ -114,21 +88,18 @@ irondrop -d ~/Documents --listen 0.0.0.0 ### Quick start -**Step 1:** Download or build IronDrop +**Step 1:** Install IronDrop ```bash -# Build from source (requires Rust) -git clone https://github.com/dev-harsh1998/IronDrop.git -cd IronDrop -cargo build --release +cargo install irondrop ``` **Step 2:** Start sharing files immediately ```bash # Share your current directory (safest - local access only) -./target/release/irondrop -d . +irondrop -d . # Share with your network (accessible to other devices) -./target/release/irondrop -d . --listen 0.0.0.0 +irondrop -d . --listen 0.0.0.0 ``` **Step 3:** Open your browser and visit `http://localhost:8080` diff --git a/doc/API_REFERENCE.md b/doc/API_REFERENCE.md index 25d851d..1951b03 100644 --- a/doc/API_REFERENCE.md +++ b/doc/API_REFERENCE.md @@ -1,4 +1,4 @@ -# IronDrop API Reference v2.7.0 +# IronDrop API Reference ## Overview @@ -31,18 +31,13 @@ User-Agent: #### Response Headers ```http # Standard headers -Server: IronDrop/2.7.0 +Server: irondrop/ Content-Type: Content-Length: -Connection: keep-alive - -# Security headers -X-Content-Type-Options: nosniff -X-Frame-Options: DENY +Connection: close # Caching headers (for static assets) Cache-Control: public, max-age=3600 -ETag: "" ``` ## Endpoints @@ -193,7 +188,7 @@ Content-Type: text/html; charset=utf-8 #### `POST /_irondrop/upload` Uploads files using direct binary streaming for optimal performance and unlimited file size support. -**Direct Upload Features (v2.7.0):** +**Direct Upload Features:** - **Direct Binary Streaming**: No multipart parsing overhead - **Automatic Mode Selection**: Small uploads (≤64MB) processed in memory, large uploads (>64MB) streamed to disk - **Constant Memory Usage**: ~7MB RAM usage regardless of file size @@ -420,43 +415,17 @@ Basic health check endpoint. ```json { "status": "healthy", - "version": "2.7.0", - "uptime_seconds": 3600, - "timestamp": "2024-01-01T12:00:00Z" + "service": "irondrop", + "version": "", + "timestamp": 1700000000, + "features": ["rate_limiting", "statistics", "native_mime_detection"] } ``` #### `GET /_irondrop/status` -Detailed server status and statistics. +Status endpoint. -**Response:** -```json -{ - "status": "healthy", - "version": "2.5.1", - "uptime_seconds": 3600, - "timestamp": "2024-01-01T12:00:00Z", - "statistics": { - "requests_served": 15420, - "bytes_served": 1073741824, - "errors_encountered": 12, - "active_connections": 3, - "rate_limit_hits": 5 - }, - "configuration": { - "threads": 8, - "chunk_size": 1024, - "upload_enabled": true, - "max_upload_size": 10737418240, - "rate_limit": 120 - }, - "system": { - "memory_usage_mb": 3.2, - "cpu_usage_percent": 2.1, - "disk_space_available": true - } -} -``` +Currently this returns the same payload as `/_irondrop/health`. #### `GET /_irondrop/monitor` HTML monitoring dashboard (human-friendly) that auto-refreshes via JavaScript to show live server statistics. Provides request counts, bytes served (downloads), and upload metrics (counts, bytes, success rate, concurrency, average processing time). @@ -472,22 +441,24 @@ Content-Type: text/html; charset=utf-8 ``` -#### `GET /_irondrop/_irondrop/monitor?json=1` +#### `GET /_irondrop/monitor?json=1` Machine-readable JSON stats for integration with external monitoring / scripting. **Response (JSON):** ```json { - "requests": { - "total": 42, - "successful": 40, - "errors": 2, - "bytes_served": 1048576, - "uptime_secs": 360 - }, + "requests": { "total": 42, "successful": 40, "errors": 2 }, "downloads": { "bytes_served": 1048576 }, + "uptime_secs": 360, + "memory": { + "available": true, + "current_bytes": 33554432, + "peak_bytes": 67108864, + "current_mb": 32.0, + "peak_mb": 64.0 + }, "uploads": { "total_uploads": 5, "successful_uploads": 5, @@ -497,7 +468,7 @@ Machine-readable JSON stats for integration with external monitoring / scripting "average_upload_size": 748982, "largest_upload": 2097152, "concurrent_uploads": 0, - "average_processing_time": 152.4, + "average_processing_ms": 152.4, "success_rate": 100.0 } } @@ -505,59 +476,11 @@ Machine-readable JSON stats for integration with external monitoring / scripting **Notes:** - `bytes_served` counts only response body bytes (excludes headers). -- `average_processing_time` is the rolling average (last 100 uploads). +- `average_processing_ms` is the rolling average (last 100 uploads). - All counters are cumulative since server start. **Planned Extensions (future versions):** active connections, per-endpoint metrics, Prometheus format. -### 6. API Information - -#### `GET /_api` -API information and capabilities. - -**Response:** -```json -{ - "name": "IronDrop", - "version": "2.5.1", - "description": "Lightweight file server with upload capabilities", - "endpoints": { - "directory_listing": { - "method": "GET", - "path": "/[path]", - "description": "List directory contents or download files", - "parameters": ["format", "sort", "order"], - "formats": ["html", "json"] - }, - "file_upload": { - "method": "POST", - "path": "/upload", - "description": "Upload files to server", - "content_type": "multipart/form-data", - "max_size": 10737418240, - "enabled": true - }, - "health_check": { - "method": "GET", - "path": "/_irondrop/health", - "description": "Basic health check" - }, - "status": { - "method": "GET", - "path": "/_irondrop/status", - "description": "Detailed server status" - } - }, - "features": { - "uploads": true, - "authentication": false, - "rate_limiting": true, - "range_requests": true, - "static_assets": true - } -} -``` - ## Authentication ### Basic Authentication @@ -587,35 +510,12 @@ Content-Type: text/html ## Rate Limiting -IronDrop implements rate limiting to prevent abuse: +IronDrop implements rate limiting to prevent abuse. Rate limiting is enforced before request handling; when a client exceeds limits, the connection may be rejected/closed. ### Default Limits - **Requests per minute**: 120 (configurable) - **Concurrent connections per IP**: 10 (configurable) -### Rate Limit Headers -```http -X-RateLimit-Limit: 120 -X-RateLimit-Remaining: 115 -X-RateLimit-Reset: 1704110400 -``` - -### Rate Limit Exceeded -```http -HTTP/1.1 429 Too Many Requests -Retry-After: 60 -X-RateLimit-Limit: 120 -X-RateLimit-Remaining: 0 -X-RateLimit-Reset: 1704110400 - -{ - "status": "error", - "error": "TooManyRequests", - "message": "Rate limit exceeded. Please try again later.", - "retry_after": 60 -} -``` - ## Error Handling ### HTTP Status Codes @@ -632,24 +532,10 @@ X-RateLimit-Reset: 1704110400 | 413 | Payload Too Large | Upload size exceeds limit | | 415 | Unsupported Media Type | File type not allowed | | 416 | Range Not Satisfiable | Invalid range request | -| 429 | Too Many Requests | Rate limit exceeded | | 500 | Internal Server Error | Server error | ### Error Response Format - -**JSON Error Response:** -```json -{ - "status": "error", - "error": "ErrorType", - "message": "Human-readable error description", - "details": { - "additional": "error-specific information" - }, - "request_id": "req_abc123", - "timestamp": "2024-01-01T12:00:00Z" -} -``` +Errors are returned as HTML pages for browser-facing endpoints. JSON is returned only by JSON endpoints (for example `/_irondrop/health` and `/_irondrop/monitor?json=1`). **HTML Error Response:** ```html @@ -724,7 +610,7 @@ if (searchData.status === 'success') { ```javascript // Monitor server health const health = await fetch('/_irondrop/health').then(r => r.json()); -console.log(`Server uptime: ${health.uptime_seconds}s`); +console.log(`Server status: ${health.status} (${health.version})`); ``` ### cURL Examples @@ -806,7 +692,7 @@ if data['status'] == 'success': ## Security Considerations ### Best Practices -1. **Always use HTTPS in production** (place behind reverse proxy) +1. **Use HTTPS in production** (enable built-in TLS or deploy behind a reverse proxy) 2. **Enable authentication** for sensitive directories 3. **Configure appropriate file extension filters** 4. **Monitor rate limiting logs** for abuse detection @@ -815,10 +701,7 @@ if data['status'] == 'success': 7. **Use strong passwords** for Basic Authentication ### Security Headers -IronDrop automatically includes security headers: -- `X-Content-Type-Options: nosniff` -- `X-Frame-Options: DENY` -- Proper `Content-Type` headers for all responses +IronDrop sets appropriate `Content-Type` headers for responses. Additional security headers should be handled by your reverse proxy if required. ### Input Validation All inputs are validated: @@ -827,4 +710,4 @@ All inputs are validated: - File names for path traversal attempts - HTTP headers for malformed content -This API reference covers all functionality available in IronDrop v2.7.0 and provides comprehensive examples for client integration. \ No newline at end of file +This API reference covers the current public HTTP surface area and provides examples for client integration. diff --git a/doc/ARCHITECTURE.md b/doc/ARCHITECTURE.md index 29251c5..65fdbd7 100644 --- a/doc/ARCHITECTURE.md +++ b/doc/ARCHITECTURE.md @@ -124,62 +124,38 @@ IronDrop is a file server written in Rust. Its HTTP stack (request parsing, rout ``` src/ -├── main.rs # Entry point (6 lines) -├── lib.rs # Library initialization (56 lines) -├── cli.rs # CLI interface with validation (200+ lines) -├── config/ # Configuration system -│ ├── mod.rs # Config struct and loading logic -│ └── ini_parser.rs # Zero-dependency INI parser -├── server.rs # Thread pool + rate limiting (400+ lines) -├── http.rs # HTTP parsing + connection handling (600+ lines) -├── router.rs # HTTP routing system (100+ lines) -├── handlers.rs # Internal route handlers (200+ lines) -├── middleware.rs # Authentication middleware (100+ lines) -├── templates.rs # Template engine with embedded assets (300+ lines) -├── fs.rs # File system operations (200+ lines) -├── response.rs # Response building + MIME detection (400+ lines) -├── upload.rs # Direct upload handler with streaming (500+ lines) -├── search.rs # Ultra-compact search engine (400+ lines) -├── error.rs # Comprehensive error types (100+ lines) -└── utils.rs # Utility functions for paths and encoding +├── main.rs +├── lib.rs +├── cli.rs +├── config/ +│ ├── mod.rs +│ └── ini_parser.rs +├── server.rs # Tokio runtime, async accept, TLS, rate limiting, stats +├── http.rs # HTTP parsing + async response streaming +├── router.rs # Routing and middleware pipeline +├── handlers.rs # Internal route handlers +├── middleware.rs # Authentication middleware +├── templates.rs # Template engine with embedded assets +├── fs.rs # File system operations and directory listing +├── response.rs # Response types and error response helpers +├── upload.rs # Upload handling + validation +├── multipart.rs # Multipart form parsing +├── search.rs # Search subsystem (index + fallback search) +├── ultra_compact_search.rs +├── webdav.rs +├── ultra_memory_test.rs +├── error.rs +└── utils.rs templates/ -├── directory/ # Directory listing UI -│ ├── index.html # HTML structure with search -│ ├── styles.css # Professional dark theme -│ └── script.js # Interactive browsing + search -├── upload/ # Upload interface -│ ├── page.html # Standalone upload page -│ ├── form.html # Reusable upload component -│ ├── styles.css # Upload UI styling -│ └── script.js # Direct binary upload logic -├── error/ # Error pages -│ ├── page.html # Error page template -│ └── styles.css # Error page styling -└── monitor/ # Monitoring dashboard - ├── page.html # Dashboard template - ├── styles.css # Dashboard styling - └── script.js # Real-time metrics updates -└── error/ # Error pages - ├── page.html # Error page structure - ├── styles.css # Error styling - └── script.js # Error page enhancements +├── common/ +├── directory/ +├── upload/ +├── error/ +└── monitor/ tests/ -├── integration_test.rs # Core server tests (19 tests) -├── integration_test.rs # Auth + security tests (6 tests) -├── edge_case_test.rs # Upload edge cases (10 tests) -├── memory_optimization_test.rs # Memory efficiency (6 tests) -├── performance_test.rs # Upload performance (5 tests) -├── stress_test.rs # Stress testing (4 tests) -├── multipart_test.rs # Multipart parser tests (7 tests) -├── ultra_compact_test.rs # Search engine tests (4 tests) -├── template_embedding_test.rs # Template system tests (3 tests) -├── test_upload.sh # End-to-end upload testing -├── test_1gb_upload.sh # Large file upload testing -└── test_executable_portability.sh # Portability validation - -Total: 199 tests across 16 test files +└── Integration, upload, monitoring, and WebDAV RFC suites (see `tests/` directory) ``` ## Search System Architecture @@ -309,10 +285,9 @@ HTTP Request → Content-Length Check → Size Threshold Comparison ### Security and Resource Protection #### Temporary File Management -- **Secure Creation**: Uses `tempfile::NamedTempFile` for secure temporary file creation -- **Automatic Cleanup**: Files automatically deleted when `RequestBody` is dropped -- **Error Recovery**: Cleanup guaranteed even during error conditions -- **Permission Control**: Temporary files created with restricted permissions +- **Creation**: Large request bodies may be streamed to a temporary file in the system temp directory +- **Unique Naming**: Filenames include process ID, timestamp, and a monotonic counter to avoid collisions +- **Cleanup**: Temporary request-body files are removed after request processing completes (best-effort cleanup on errors) #### Resource Limits - **Memory Protection**: Prevents memory exhaustion from large uploads @@ -322,26 +297,7 @@ HTTP Request → Content-Length Check → Size Threshold Comparison ### Integration with Upload System -The HTTP streaming layer integrates seamlessly with the existing upload infrastructure: - -```rust -impl UploadHandler { - pub fn handle_upload(&self, request_body: RequestBody) -> Result<(), AppError> { - match request_body { - RequestBody::Memory(data) => { - // Fast path for small uploads - let cursor = Cursor::new(data); - self.process_multipart(cursor) - } - RequestBody::File(path) => { - // Streaming path for large uploads - let file = File::open(path)?; - self.process_multipart(file) - } - } - } -} -``` +The HTTP streaming layer integrates with uploads by representing request bodies as either in-memory buffers or a temporary file on disk (`RequestBody::Memory` / `RequestBody::File { path, size }`). This keeps large uploads bounded in RAM while preserving the same handler behavior. ### Monitoring and Observability @@ -354,24 +310,11 @@ The streaming system provides comprehensive monitoring capabilities: ### Configuration Options -```rust -pub struct StreamingConfig { - pub memory_threshold: usize, // Default: 64MB - pub chunk_size: usize, // Default: 64KB - pub temp_dir: Option, // Default: system temp - pub max_concurrent: usize, // Default: 10 -} -``` +See HTTP_STREAMING.md and CONFIGURATION_SYSTEM.md for the current configuration surface (upload size limits, chunk size, and upload directory). ### Testing Infrastructure -Dedicated HTTP streaming tests verify correct behavior: - -- **`http_streaming_test.rs`**: Comprehensive streaming functionality tests -- **Size Threshold Testing**: Verify correct mode selection -- **Resource Cleanup Testing**: Ensure temporary files are properly cleaned up -- **Error Condition Testing**: Validate error recovery and resource cleanup -- **Performance Testing**: Measure streaming performance across size ranges +The test suite verifies correct mode selection (memory vs disk), cleanup behavior, and concurrency regressions under streaming load. ## Security Architecture @@ -393,7 +336,7 @@ Dedicated HTTP streaming tests verify correct behavior: - Request timeouts to prevent resource exhaustion - Memory-efficient streaming for large file operations - Disk space checking before upload operations - - Thread pool management with configurable limits + - Tokio runtime worker threads and blocking-task isolation with configurable limits 4. **Audit and Monitoring Layer** - Comprehensive request logging with unique IDs @@ -480,17 +423,11 @@ Static Asset Request → Asset Router → Direct File Serving → CSS/JS Respons - **Performance Tests**: Load and stress testing scenarios ### Test Coverage by Component -| Test File | Component Coverage | Test Count | -|-----------|-------------------|------------| -| `integration_test.rs` | Core server functionality | 19 | -| `integration_test.rs` | Authentication and security | 6 | -| `upload_integration_test.rs` | Upload system | 29 | -| `multipart_test.rs` | Multipart parser | 7 | -| `debug_upload_test.rs` | Edge cases and debugging | Variable | -| Others | Template system, HTTP handling | 40+ | + +See TESTING_DOCUMENTATION.md and the `tests/` directory for the current test inventory and categories. ### Custom Test Infrastructure -- **Native HTTP Client**: Pure Rust implementation for testing +- **HTTP Clients**: Tests use a mix of raw TCP streams and lightweight HTTP clients - **Mock File Systems**: Temporary directories and file operations - **Concurrent Testing**: Multi-threaded test scenarios - **Security Validation**: Path traversal and injection testing @@ -498,23 +435,8 @@ Static Asset Request → Asset Router → Direct File Serving → CSS/JS Respons ## Configuration System ### CLI Configuration -```rust -pub struct Cli { - directory: PathBuf, // Required: directory to serve - listen: String, // Default: "127.0.0.1" - port: u16, // Default: 8080 - allowed_extensions: String, // Default: "*.zip,*.txt" - threads: usize, // Default: 8 - chunk_size: usize, // Default: 1024 - verbose: bool, // Default: false - detailed_logging: bool, // Default: false - username: Option, // Optional: basic auth - password: Option, // Optional: basic auth - enable_upload: bool, // Default: false - max_upload_size: u32, // Default: 10240 (10GB) - upload_dir: Option, // Optional: custom upload dir -} -``` + +The CLI surface (flags, defaults, and config precedence) is documented in CONFIGURATION_SYSTEM.md and the top-level README.md. ### Validation Pipeline 1. **Parse-time Validation**: Clap value parsers and constraints diff --git a/doc/DEPLOYMENT.md b/doc/DEPLOYMENT.md index 771d459..c599a00 100644 --- a/doc/DEPLOYMENT.md +++ b/doc/DEPLOYMENT.md @@ -25,7 +25,7 @@ irondrop -d /srv/files --listen 0.0.0.0 --port 8080 ```bash # Test server health -curl http://localhost:8080/_health +curl http://localhost:8080/_irondrop/health # Test file listing curl http://localhost:8080/ @@ -194,7 +194,7 @@ WORKDIR /srv EXPOSE 8080 HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ - CMD wget --no-verbose --tries=1 --spider http://localhost:8080/_health || exit 1 + CMD wget --no-verbose --tries=1 --spider http://localhost:8080/_irondrop/health || exit 1 CMD ["irondrop", "--directory", "/srv/files", "--listen", "0.0.0.0", "--port", "8080"] ``` @@ -226,7 +226,7 @@ services: --detailed-logging restart: unless-stopped healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/_health"] + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/_irondrop/health"] interval: 30s timeout: 3s retries: 3 @@ -490,6 +490,9 @@ server { Allow from all + ProxyPass /_irondrop/health http://127.0.0.1:8080/_irondrop/health + ProxyPassReverse /_irondrop/health http://127.0.0.1:8080/_irondrop/health + ProxyPass /_health http://127.0.0.1:8080/_health ProxyPassReverse /_health http://127.0.0.1:8080/_health @@ -507,7 +510,7 @@ server { #!/bin/bash # /usr/local/bin/check-irondrop.sh -HEALTH_URL="http://localhost:8080/_health" +HEALTH_URL="http://localhost:8080/_irondrop/health" EXPECTED_STATUS="healthy" response=$(curl -s -f "$HEALTH_URL" | jq -r '.status' 2>/dev/null) diff --git a/doc/MONITORING.md b/doc/MONITORING.md index 1322eab..8daf834 100644 --- a/doc/MONITORING.md +++ b/doc/MONITORING.md @@ -1,33 +1,33 @@ -# IronDrop Monitoring Guide (v2.7.0) +# IronDrop Monitoring Guide This guide documents the built-in monitoring capabilities introduced with the `/monitor` endpoint and supporting health APIs. ## Overview -IronDrop exposes lightweight operational telemetry without external dependencies: +IronDrop exposes lightweight operational telemetry without requiring an external monitoring stack: | Endpoint | Format | Purpose | |----------|--------|---------| | `/monitor` | HTML | Human dashboard for live stats | | `/_irondrop/monitor?json=1` | JSON | Machine-readable metrics for scripting / scraping | -| `/_health` | JSON | Minimal liveness probe (OK / version / uptime) | -| `/_status` | JSON | Extended status (configuration + cumulative counters) | +| `/_irondrop/health` | JSON | Liveness probe (status / service / version / timestamp / features) (legacy: `/_health`) | +| `/_irondrop/status` | JSON | Status payload (currently same as health) | ## Data Model -`/_irondrop/monitor?json=1` returns three top-level sections: +`/_irondrop/monitor?json=1` returns these top-level keys: ```json { - "requests": { - "total": 42, - "successful": 40, - "errors": 2, - "bytes_served": 1048576, - "uptime_secs": 360 - }, - "downloads": { - "bytes_served": 1048576 + "requests": { "total": 42, "successful": 40, "errors": 2 }, + "downloads": { "bytes_served": 1048576 }, + "uptime_secs": 360, + "memory": { + "available": true, + "current_bytes": 33554432, + "peak_bytes": 67108864, + "current_mb": 32.0, + "peak_mb": 64.0 }, "uploads": { "total_uploads": 5, @@ -38,7 +38,7 @@ IronDrop exposes lightweight operational telemetry without external dependencies "average_upload_size": 748982, "largest_upload": 2097152, "concurrent_uploads": 0, - "average_processing_time": 152.4, + "average_processing_ms": 152.4, "success_rate": 100.0 } } @@ -47,10 +47,10 @@ IronDrop exposes lightweight operational telemetry without external dependencies ### Field Semantics - `requests.total` – All handled requests (success + error) since start. - `requests.successful` / `errors` – Outcome classification. -- `requests.bytes_served` / `downloads.bytes_served` – Cumulative body bytes sent (excludes HTTP headers) across all responses. -- `requests.uptime_secs` – Elapsed seconds since the first server start instant. +- `downloads.bytes_served` – Cumulative body bytes sent (excludes HTTP headers). +- `uptime_secs` – Elapsed seconds since the server started. - `uploads.*` – Aggregated upload subsystem metrics (only updated when uploads enabled). -- `average_processing_time` – Rolling average (last 100 uploads) in milliseconds. +- `average_processing_ms` – Rolling average (last 100 uploads) in milliseconds. - `success_rate` – Percentage of successful uploads over total uploads (0 if none yet). - `concurrent_uploads` – Point-in-time counter incremented on start and decremented on completion. @@ -69,7 +69,7 @@ curl -s http://localhost:8080/_irondrop/monitor?json=1 | jq '.requests.bytes_ser ### Basic Health Probe (Kubernetes / Docker) ```bash -curl -f http://localhost:8080/_health > /dev/null || echo "Unhealthy" +curl -f http://localhost:8080/_irondrop/health > /dev/null || echo "Unhealthy" ``` ### Shell Alert When Upload Failures Detected diff --git a/doc/README.md b/doc/README.md index 8594698..5f762fe 100644 --- a/doc/README.md +++ b/doc/README.md @@ -37,15 +37,15 @@ Known scope limits: **Contents:** - System architecture diagrams and component relationships - Request processing flow and data paths -- Module-by-module code organization (19 Rust source files) +- Module-by-module code organization (see `src/`) - Ultra-compact search system architecture and memory optimization - Security architecture and defense-in-depth implementation - Performance characteristics and scalability considerations - Template system design and asset pipeline -- Testing architecture with 199 comprehensive tests across 16 test files +- Testing architecture (see TESTING_DOCUMENTATION.md; run `cargo test --all-features`) **Key Sections:** -- Core module breakdown with line counts and responsibilities +- Core module breakdown and responsibilities - HTTP request processing pipeline with security checkpoints - Dual-mode search engine implementation and ultra-compact optimization - Template engine implementation and static asset serving @@ -107,7 +107,7 @@ Native zero-dependency template engine: variables, conditionals, embedded assets **Purpose**: Comprehensive testing suite documentation and validation procedures **Contents:** -- **Complete Test Coverage**: 199 tests across 16 test files covering all functionality +- **Complete Test Coverage**: Run `cargo test --all-features` for the authoritative suite and totals - **Test Categories**: Core server, integration, edge cases, memory optimization, performance, stress testing, streaming - **Security Testing**: Path traversal prevention, input validation, authentication mechanisms - **Performance Benchmarks**: Memory efficiency targets, upload speed thresholds, stress test metrics @@ -136,7 +136,6 @@ Native zero-dependency template engine: variables, conditionals, embedded assets - Client-side validation and error handling **Implementation Status**: ✅ **Production Ready** (v2.7.0) -- Complete upload system with 29 comprehensive tests - Professional UI matching IronDrop's design language - Integrated with template engine and security systems - Supports unlimited file uploads with direct streaming architecture @@ -173,9 +172,8 @@ Native zero-dependency template engine: variables, conditionals, embedded assets **Implementation Status**: ✅ **Production Ready** (v2.7.0) - RFC 7578 compliance with robust boundary detection and streaming support - Advanced streaming implementation for memory-efficient large file processing -- 7+ dedicated test cases covering edge cases and streaming scenarios - Integrated with upload handler and HTTP processing with automatic mode selection -- Zero external dependencies with pure Rust implementation +- No external HTTP framework required; implemented in-house with standard Rust crates used where practical - Prevents memory exhaustion for multi-gigabyte file uploads ### 🌊 [HTTP Layer Streaming Documentation](./HTTP_STREAMING.md) ⭐ @@ -264,8 +262,8 @@ Native zero-dependency template engine: variables, conditionals, embedded assets ### 🔐 **Advanced Security & Monitoring** - **Rate Limiting** – DoS protection with configurable requests per minute and concurrent connections per IP - **Server Statistics** – Real-time monitoring of requests, bytes served, uptime, and performance metrics -- **Health Check Endpoints** – Built-in `/_health` and `/_status` endpoints for monitoring -- **Unified Monitoring Dashboard** – NEW `/monitor` endpoint with live HTML dashboard and JSON API (`/monitor?json=1`) exposing request, download and upload metrics +- **Health Check Endpoints** – `/_irondrop/health` and `/_irondrop/status` (legacy compatibility: `/_health`) +- **Unified Monitoring Dashboard** – `/monitor` (legacy) and `/_irondrop/monitor` (canonical) with JSON via `?json=1` - **Path-Traversal Protection** – Canonicalises every request path and rejects any attempt that escapes the served directory - **Optional Basic Authentication** – Username and password can be supplied via CLI flags @@ -314,6 +312,18 @@ Native zero-dependency template engine: variables, conditionals, embedded assets ## 🛠️ Installation +### Install from crates.io + +```bash +cargo install irondrop +``` + +### Install latest from Git + +```bash +cargo install --git https://github.com/dev-harsh1998/IronDrop.git +``` + ### Build from Source ```bash @@ -361,16 +371,16 @@ Open a browser at [http://127.0.0.1:8080](http://127.0.0.1:8080) and you will se ## 🎉 What's New in v2.7.0 ### 📤 **Complete File Upload System** -IronDrop v2.5 introduces a **production-ready file upload system** with enterprise-grade features: +IronDrop v2.7.0 includes a **production-ready file upload system** with enterprise-grade features: - **🔒 Enhanced Security**: Comprehensive input validation, boundary verification, and filename sanitization - **⚡ Performance**: Handles unlimited file sizes with constant memory usage and concurrent processing - **🎨 Professional UI**: Integrated upload interface accessible at `/upload` with real-time feedback - **🛡️ Robust Validation**: Multi-layer security including extension filtering, size limits, and malformed data rejection -- **🧪 Battle-Tested**: 199 tests across 16 test files covering edge cases, security scenarios, and performance stress testing +- **🧪 Battle-Tested**: Covered by the project’s automated test suite (run `cargo test --all-features`) ### 🔍 **Advanced Search System** (New in v2.5) -IronDrop v2.5 introduces a **dual-mode search engine** optimized for directories of any size: +IronDrop includes a **dual-mode search engine** optimized for directories of any size: - **🚀 Ultra-Low Memory**: <100MB for 10M+ files using 11-byte entries and hierarchical storage - **⚡ Lightning Fast**: Real-time search with 300ms debouncing and LRU caching @@ -381,20 +391,22 @@ IronDrop v2.5 introduces a **dual-mode search engine** optimized for directories ### 📊 **Integrated Monitoring Dashboard** (Added in v2.5) The new `/monitor` endpoint provides both an HTML dashboard and a JSON API for tooling integration. It auto-updates in the browser and can be scraped by observability agents. -Example JSON (`GET /monitor?json=1`): +Example JSON (`GET /_irondrop/monitor?json=1`, legacy: `/monitor?json=1`): ```json { - "requests": { - "total": 42, - "successful": 40, - "errors": 2, - "bytes_served": 1048576, - "uptime_secs": 360 - }, + "requests": { "total": 42, "successful": 40, "errors": 2 }, "downloads": { "bytes_served": 1048576 }, + "uptime_secs": 360, + "memory": { + "available": true, + "current_bytes": 33554432, + "peak_bytes": 67108864, + "current_mb": 32.0, + "peak_mb": 64.0 + }, "uploads": { "total_uploads": 5, "successful_uploads": 5, @@ -404,7 +416,7 @@ Example JSON (`GET /monitor?json=1`): "average_upload_size": 748982, "largest_upload": 2097152, "concurrent_uploads": 0, - "average_processing_time": 152.4, + "average_processing_ms": 152.4, "success_rate": 100.0 } } @@ -460,7 +472,7 @@ Planned extensions (open to contribution): | **High-Performance Server** | `irondrop -d ./big -t 16 -c 8192` | Higher concurrency, optimized streaming | | **Secure Corporate Share** | `irondrop -d ./private --username alice --password s3cret` | Authentication, audit logging, professional design | | **Development Server** | `irondrop -d . -v --detailed-logging` | Debug logging, template development, hot reload | -| **Production Monitoring** | `irondrop -d /data -l 0.0.0.0` + health checks at `/_health` | Statistics, uptime monitoring, rate limiting | +| **Production Monitoring** | `irondrop -d /data -l 0.0.0.0` + health checks at `/_irondrop/health` | Statistics, uptime monitoring, rate limiting | | **Monitoring Dashboard** | `irondrop -d .` then visit `/monitor` | Live HTML + JSON metrics | | **Secure Upload Server** | `irondrop -d ./shared --enable-upload --max-upload-size 5120 -a "*.txt,*.pdf,*.jpg"` | Controlled file uploads up to 5GB, extension filtering | | **Corporate File Share** | `irondrop -d /data --enable-upload --upload-dir /data/uploads --username admin` | Authenticated uploads, custom upload directory | @@ -642,9 +654,9 @@ assets/ **Architecture Highlights:** - **Modular Templates**: Organized separation of HTML/CSS/JS with native rendering -- **Zero Dependencies**: Pure Rust implementation without external HTTP or template libraries +- **No External HTTP Framework**: HTTP parsing, routing, and streaming are implemented in-house - **Professional UI**: Corporate-grade blackish-grey design with glassmorphism effects -- **Comprehensive Testing**: 19 total tests including custom HTTP client for static assets +- **Comprehensive Testing**: See TESTING_DOCUMENTATION.md for suite organization and how to run it Every module is documented and formatted with `cargo fmt` and `clippy -- -D warnings` to keep technical debt at zero. @@ -654,48 +666,15 @@ Every module is documented and formatted with `cargo fmt` and `clippy -- -D warn ### Comprehensive Test Suite -The project includes **199 comprehensive tests across 16 test files** covering all aspects of functionality, with complete upload system validation: - ```bash -# Run all tests (covers upload, download, security, concurrency) -cargo test +# Run all tests (covers upload, download, security, concurrency, WebDAV) +cargo test --all-features # Run with detailed output -cargo test -- --nocapture - -# Run specific test suites -cargo test comprehensive_test # Core server functionality (19 tests) -cargo test integration_test # Authentication & security (6 tests) -cargo test upload_integration_test # Upload functionality (29 tests) -cargo test debug_upload_test # Multipart parser (7 tests) +cargo test --all-features -- --nocapture ``` -### Test Architecture - -**Custom HTTP Client**: Tests use a native HTTP client implementation (zero external dependencies) that directly connects via `TcpStream` to verify: - -- **Bidirectional File Operations**: Upload and download functionality with unlimited size support -- **Multipart Processing**: RFC-compliant parsing with boundary detection and validation -- **Template System**: Modular HTML/CSS/JS serving for both download and upload interfaces -- **Security Validation**: Input sanitization, boundary verification, extension filtering -- **Concurrency Handling**: Multiple simultaneous uploads with thread safety -- **Error Scenarios**: Malformed data rejection, resource exhaustion protection -- **Authentication**: Secure upload/download with basic auth integration -- **HTTP Compliance**: Headers, status codes, and protocol adherence across all endpoints - -### Test Coverage - -| Test Category | Count | Description | -|---------------|-------|-------------| -| **Upload System** | 29 | Single/multi-file uploads, unlimited size support, concurrency, validation | -| **Core Server** | 19 | Directory listing, error pages, security, authentication | -| **Multipart Parser** | 7 | Boundary detection, content extraction, validation | -| **Security** | 12+ | Authentication, rate limiting, path traversal, input validation | -| **File Operations** | 15+ | Downloads, uploads, MIME detection, atomic operations | -| **Monitoring** | 8+ | Health checks, statistics, performance tracking | -| **UI & Templates** | 10+ | Upload/download interfaces, error pages, responsive design | - -Tests start the server on random ports and issue real HTTP requests to verify both functionality and integration. +See TESTING_DOCUMENTATION.md for test categories, patterns, and the current inventory. --- @@ -915,11 +894,11 @@ Don't know where to start? Here are some **beginner-friendly test contributions: - **Request Logging**: Every request tagged with unique IDs for comprehensive auditing - **Performance Tracking**: Slow request detection and logging for security analysis - **Statistics Collection**: Real-time monitoring of request patterns and error rates -- **Health Endpoints**: Built-in `/_health` and `/_status` for infrastructure monitoring +- **Health Endpoints**: `/_irondrop/health` and `/_irondrop/status` (legacy compatibility: `/_health`) ### Zero-Trust Architecture -- **No External Dependencies**: Eliminates third-party security vulnerabilities -- **Native Implementation**: All security features implemented in pure Rust +- **No External HTTP Framework**: Minimizes the web-framework dependency surface area +- **Native Implementation**: Security features are implemented in Rust; standard crates are still used (Tokio, rustls, etc.) - **Template Security**: Variable interpolation with HTML escaping and URL encoding - **Memory Safety**: Rust's ownership model prevents buffer overflows and memory leaks @@ -982,60 +961,37 @@ The modular template system allows easy customization: If you're looking to understand the codebase, integrate IronDrop, or contribute to development: -- **📖 [Complete Documentation Suite](./doc/)** - Comprehensive technical documentation -- **🏗️ [Architecture Guide](./doc/ARCHITECTURE.md)** - System design, component breakdown, and code organization -- **🔌 [API Reference](./doc/API_REFERENCE.md)** - Complete REST API specification with examples -- **🔍 [Search Feature Guide](./doc/SEARCH_FEATURE.md)** - Dual-mode search engine implementation and usage -- **🚀 [Deployment Guide](./doc/DEPLOYMENT.md)** - Production deployment with Docker, systemd, and reverse proxy +- **📖 [Complete Documentation Suite](./)** - Comprehensive technical documentation +- **🏗️ [Architecture Guide](./ARCHITECTURE.md)** - System design, component breakdown, and code organization +- **🔌 [API Reference](./API_REFERENCE.md)** - Complete REST API specification with examples +- **🔍 [Search Feature Guide](./SEARCH_FEATURE.md)** - Dual-mode search engine implementation and usage +- **🚀 [Deployment Guide](./DEPLOYMENT.md)** - Production deployment with Docker, systemd, and reverse proxy ### 🛡️ **For Security & DevOps Teams** Production deployment and security implementation details: -- **🔒 [Security Implementation](./doc/SECURITY_FIXES.md)** - OWASP vulnerability fixes and security controls -- **🚀 [Production Deployment](./doc/DEPLOYMENT.md)** - systemd, Docker, monitoring, and security hardening -- **📊 [System Monitoring](./doc/API_REFERENCE.md#health-and-monitoring)** - Health endpoints and operational metrics +- **🔒 [Security Implementation](./SECURITY_FIXES.md)** - OWASP vulnerability fixes and security controls +- **🚀 [Production Deployment](./DEPLOYMENT.md)** - systemd, Docker, monitoring, and security hardening +- **📊 [System Monitoring](./API_REFERENCE.md#health-and-monitoring)** - Health endpoints and operational metrics ### 🎨 **For Frontend Developers** UI system and template integration: -- **📤 [Upload UI System](./doc/UPLOAD_INTEGRATION.md)** - Modern drag-and-drop interface implementation -- **🎨 [Template System](./doc/ARCHITECTURE.md#template-system-architecture)** - Professional blackish-grey UI with modular architecture -- **🔧 [API Integration](./doc/API_REFERENCE.md#client-integration-examples)** - JavaScript, cURL, and Python examples +- **📤 [Upload UI System](./UPLOAD_INTEGRATION.md)** - Modern drag-and-drop interface implementation +- **🎨 [Template System](./ARCHITECTURE.md#template-system-architecture)** - Professional blackish-grey UI with modular architecture +- **🔧 [API Integration](./API_REFERENCE.md#client-integration-examples)** - JavaScript, cURL, and Python examples ### 🧪 **Testing & Quality Assurance** -IronDrop includes **199 comprehensive tests across 16 test files** covering: - -- **Core Server Tests** (19 tests): HTTP handling, directory listing, authentication -- **Upload System Tests** (29 tests): File uploads, validation, concurrent handling -- **Security Tests** (12+ tests): Path traversal protection, input validation -- **Multipart Parser Tests** (7 tests): RFC 7578 compliance and edge cases -- **Integration Tests** (30+ tests): End-to-end functionality and performance - ```bash -# Run all tests -cargo test - -# Run with detailed output -cargo test -- --nocapture - -# Run specific test suites -cargo test comprehensive_test # Core functionality -cargo test upload_integration # Upload system -cargo test multipart_test # Multipart parser +cargo test --all-features ``` ### 📈 **Project Statistics** -| Metric | Count | Description | -|--------|--------|-------------| -| **Source Files** | 15 | Rust modules with clear separation of concerns | -| **Lines of Code** | 3000+ | Production-ready implementation | -| **Template Files** | 10 | Professional UI with HTML/CSS/JS separation | -| **Test Cases** | 189 across 16 files | Comprehensive coverage including security tests | -| **Documentation Pages** | 10 | Complete technical documentation suite | +See the repository directories (`src/`, `templates/`, `tests/`, `doc/`) for the current inventory. --- @@ -1112,7 +1068,7 @@ We welcome contributions! Here's how to get started: ## 📞 Support & Community -- **📖 Documentation**: Start with [./doc/README.md](./doc/README.md) for complete guides +- **📖 Documentation**: Start with [README.md](./README.md) for complete guides - **🐛 Issues**: Report bugs and request features via GitHub Issues - **💬 Discussions**: GitHub Discussions for questions and community support - **🔒 Security**: Responsible disclosure via GitHub Security Advisory @@ -1129,6 +1085,6 @@ IronDrop is distributed under the **MIT** license; see `LICENSE` for details. *Made with 🦀 in Bengaluru* -**[⭐ Star us on GitHub](https://github.com/dev-harsh1998/IronDrop) • [📖 Read the Docs](./doc/) • [🚀 Get Started](#-quick-start)** +**[⭐ Star us on GitHub](https://github.com/dev-harsh1998/IronDrop) • [📖 Read the Docs](./) • [🚀 Get Started](#-quick-start)** diff --git a/doc/RFC_OWASP_COMPLIANCE.md b/doc/RFC_OWASP_COMPLIANCE.md index e9b200f..0335c89 100644 --- a/doc/RFC_OWASP_COMPLIANCE.md +++ b/doc/RFC_OWASP_COMPLIANCE.md @@ -15,7 +15,7 @@ This document provides a comprehensive analysis of the RFC standards and OWASP s **Implementation Location**: `src/http.rs` -- **HTTP/1.1 Protocol Support**: Complete request/response parsing with version validation (`src/http.rs:68-71`) +- **HTTP/1.1 Protocol Support**: Complete request/response parsing with version validation (see `src/http.rs`) - **Status Code Compliance**: Proper HTTP status codes implementation: - 200 OK, 400 Bad Request, 403 Forbidden, 404 Not Found - 405 Method Not Allowed, 413 Payload Too Large, 500 Internal Server Error @@ -27,18 +27,18 @@ This document provides a comprehensive analysis of the RFC standards and OWASP s **Implementation Location**: `src/http.rs` -- **Header Parsing**: Case-insensitive header processing with multiple value support (`src/http.rs:84-92`) -- **Request Line Validation**: Proper HTTP request line parsing and validation (`src/http.rs:57-71`) -- **Connection Management**: Connection timeout handling and resource cleanup (`src/http.rs:48`) +- **Header Parsing**: Case-insensitive header processing with multiple value support (see `src/http.rs`) +- **Request Line Validation**: Proper HTTP request line parsing and validation (see `src/http.rs`) +- **Connection Management**: Connection timeout handling and resource cleanup (see `src/http.rs`) - **Message Framing**: Proper handling of request/response boundaries ### RFC 7578 (Multipart Form Data) **Implementation Location**: `src/multipart.rs` -- **Compliant Parser**: Full RFC 7578 compliant multipart/form-data parser (`src/multipart.rs:1-4`) -- **Boundary Validation**: RFC 2046 compliant boundary validation (`src/multipart.rs:1098-1106`) -- **Content-Disposition**: Proper Content-Disposition header parsing (`src/multipart.rs:244-315`) +- **Compliant Parser**: Multipart/form-data parser (see `src/multipart.rs`) +- **Boundary Validation**: RFC 2046 compliant boundary validation (see `src/multipart.rs`) +- **Content-Disposition**: Proper Content-Disposition header parsing (see `src/multipart.rs`) - **Binary Safety**: Binary-safe content handling without UTF-8 assumptions - **Security Limits**: Configurable limits for parts, sizes, and headers @@ -46,16 +46,16 @@ This document provides a comprehensive analysis of the RFC standards and OWASP s **Implementation Location**: `src/http.rs` -- **Base64 Encoding**: Proper Base64 credential encoding/decoding (`src/http.rs:681-695`) -- **Authorization Header**: Correct Authorization header parsing (`src/http.rs:670-696`) +- **Base64 Encoding**: Proper Base64 credential encoding/decoding (see `src/http.rs`) +- **Authorization Header**: Correct Authorization header parsing (see `src/http.rs`) - **Credential Validation**: Secure credential comparison without timing attacks ### RFC 3986 (URI Generic Syntax) **Implementation Location**: `src/http.rs` -- **URL Decoding**: Percent-encoded path decoding (`src/http.rs:265-297`) -- **Path Normalization**: Safe path normalization preventing traversal attacks (`src/http.rs:347-364`) +- **URL Decoding**: Percent-encoded path decoding (see `src/http.rs`) +- **Path Normalization**: Safe path normalization preventing traversal attacks (see `src/http.rs` and `src/utils.rs`) - **URI Component Handling**: Proper handling of path, query, and fragment components ## OWASP Security Principles Implementation @@ -64,18 +64,18 @@ This document provides a comprehensive analysis of the RFC standards and OWASP s **Status**: Implemented -- **Path Traversal Protection**: Canonical path validation (`src/cli.rs:100-118`, `src/http.rs:610-612`) -- **System Directory Blacklisting**: Prevents access to system directories (`src/cli.rs:134-158`) -- **File Extension Validation**: Configurable allowed extensions (`src/upload.rs:534-558`) -- **Authentication Enforcement**: Optional but properly implemented Basic Auth (`src/http.rs:551-559`) +- **Path Traversal Protection**: Canonical path validation (see `src/cli.rs`, `src/http.rs`, and `src/utils.rs`) +- **System Directory Blacklisting**: Prevents access to system directories (see `src/cli.rs`) +- **File Extension Validation**: Configurable allowed extensions (see `src/upload.rs`) +- **Authentication Enforcement**: Optional but properly implemented Basic Auth (see `src/middleware.rs` and template auth toggles) ### A02:2021 - Cryptographic Failures Prevention **Status**: Implemented -- **Secure Filename Handling**: Filename sanitization preventing injection (`src/multipart.rs:325-361`) +- **Secure Filename Handling**: Filename sanitization preventing injection (see `src/multipart.rs`) - **No Credential Storage**: Credentials only validated at runtime, never stored -- **Atomic File Operations**: Race condition prevention (`src/upload.rs:591-640`) +- **Atomic File Operations**: Race condition prevention (see `src/upload.rs`) ### A03:2021 - Injection Prevention @@ -91,15 +91,15 @@ This document provides a comprehensive analysis of the RFC standards and OWASP s - **Defense in Depth**: Multiple validation layers throughout the application - **Fail-Safe Defaults**: Secure default configurations -- **Rate Limiting**: Built-in DoS protection (`src/server.rs:12-85`) +- **Rate Limiting**: Built-in DoS protection (see `src/server.rs`) ### A05:2021 - Security Misconfiguration Prevention **Status**: Implemented -- **Upload Size Validation**: Bounds checking preventing resource exhaustion (`src/cli.rs:71-88`) -- **Request Limits**: Maximum request body and header size limits (`src/http.rs:15-19`) -- **Directory Permissions**: Write permission validation (`src/cli.rs:186-199`) +- **Upload Size Validation**: Bounds checking preventing resource exhaustion (see `src/cli.rs`) +- **Request Limits**: Maximum request body and header size limits (see `src/http.rs`) +- **Directory Permissions**: Write permission validation (see `src/cli.rs`) ### A06:2021 - Vulnerable Components Prevention @@ -121,7 +121,7 @@ This document provides a comprehensive analysis of the RFC standards and OWASP s **Status**: ✅ **IMPLEMENTED** -- **Atomic Operations**: File operations use temporary files with atomic rename (`src/upload.rs:591-640`) +- **Atomic Operations**: File operations use temporary files with atomic rename (see `src/upload.rs`) - **Unique Temporary Files**: Prevents race conditions and conflicts - **Complete Read/Write Cycles**: Ensures file integrity @@ -131,7 +131,7 @@ This document provides a comprehensive analysis of the RFC standards and OWASP s - **Comprehensive Logging**: Security events logged throughout (`log::info`, `log::warn`, `log::error`) - **Rate Limiting Events**: Failed attempts and rate limit violations logged -- **Statistics Tracking**: Request and upload statistics for monitoring (`src/server.rs:106-320`) +- **Statistics Tracking**: Request and upload statistics for monitoring (see `src/server.rs`) - **Error Logging**: All security-relevant errors are logged ### A10:2021 - Server-Side Request Forgery Prevention @@ -145,8 +145,8 @@ This document provides a comprehensive analysis of the RFC standards and OWASP s ### DoS Protection -- **Rate Limiting**: 120 requests/minute, 10 concurrent connections per IP (`src/server.rs:496`) -- **Request Timeouts**: 30-second timeout for request processing (`src/http.rs:48`) +- **Rate Limiting**: Per-IP request and concurrent connection limits (see `src/server.rs`) +- **Request Timeouts**: Read timeouts to prevent slowloris-style resource exhaustion (see `src/http.rs`) - **Memory Protection**: Request body size limits (10GB max) and header size limits (8KB) - **Concurrency Control**: Async networking plus rate limiting and blocking isolation prevent resource exhaustion diff --git a/doc/TESTING_DOCUMENTATION.md b/doc/TESTING_DOCUMENTATION.md index c712ef1..0ac315c 100644 --- a/doc/TESTING_DOCUMENTATION.md +++ b/doc/TESTING_DOCUMENTATION.md @@ -186,7 +186,7 @@ fn test_demonstrate_memory_savings() // Compares memory usage vs alternatives ## Running Tests -### Basic Test Execution (current totals: 272 tests: 48 unit + 224 integration/system) +### Basic Test Execution ```bash # Run all tests @@ -400,7 +400,7 @@ fn test_new_feature() { - All 15 direct upload tests now pass, including concurrent upload scenarios **Test Suite Stability** -- Achieved 100% test pass rate across all 272 tests +- Run `cargo test --all-features` for the authoritative suite and totals - Enhanced test reliability under concurrent execution - Improved error handling and edge case coverage - Added comprehensive validation for boundary conditions From d9dbe555ef473863f83a07dcf1c2131add89db25 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Wed, 15 Apr 2026 06:13:10 +0530 Subject: [PATCH 04/17] irondrop: v2.7.1 --- Cargo.toml | 2 +- doc/ARCHITECTURE.md | 6 +++--- doc/CONFIGURATION_SYSTEM.md | 2 +- doc/DEPLOYMENT.md | 4 ++-- doc/HTTP_STREAMING.md | 6 +++--- doc/MONITORING.md | 2 +- doc/MULTIPART_README.md | 6 +++--- doc/README.md | 18 +++++++++--------- doc/TEMPLATE_SYSTEM.md | 6 +++--- doc/TESTING_DOCUMENTATION.md | 6 +++--- doc/WEBDAV_IMPLEMENTATION.md | 2 +- 11 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dba7b5a..8c05e6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "irondrop" -version = "2.7.0" +version = "2.7.1" edition = "2024" license = "MIT" description = "Drop files, not dependencies - a well tested fully featured & battle-ready server in a single Rust binary with support for indexing through 10M files." diff --git a/doc/ARCHITECTURE.md b/doc/ARCHITECTURE.md index 65fdbd7..ca4bacc 100644 --- a/doc/ARCHITECTURE.md +++ b/doc/ARCHITECTURE.md @@ -1,4 +1,4 @@ -# IronDrop Architecture Documentation v2.7.0 +# IronDrop Architecture Documentation v2.7.1 ## Overview @@ -233,7 +233,7 @@ Request → Cache Check → Hit: Return Cached Results ## HTTP Layer Streaming Architecture ### Overview -IronDrop v2.7.0 provides advanced HTTP layer streaming for efficient handling of large file uploads. The system automatically switches between memory-based and disk-based processing based on content size, providing optimal performance and resource utilization. +IronDrop v2.7.1 provides advanced HTTP layer streaming for efficient handling of large file uploads. The system automatically switches between memory-based and disk-based processing based on content size, providing optimal performance and resource utilization. ### RequestBody Architecture @@ -500,4 +500,4 @@ pub enum AppError { 4. **CDN Integration**: Edge caching and global distribution 5. **Database Caching**: Redis integration for session management -This architecture documentation reflects the current state of IronDrop v2.7.0 and serves as a foundation for understanding the system's design principles, implementation details, and operational characteristics. +This architecture documentation reflects the current state of IronDrop v2.7.1 and serves as a foundation for understanding the system's design principles, implementation details, and operational characteristics. diff --git a/doc/CONFIGURATION_SYSTEM.md b/doc/CONFIGURATION_SYSTEM.md index fb88441..5cdef0b 100644 --- a/doc/CONFIGURATION_SYSTEM.md +++ b/doc/CONFIGURATION_SYSTEM.md @@ -1,4 +1,4 @@ -## IronDrop Configuration System (v2.7.0) +## IronDrop Configuration System (v2.7.1) ### Overview IronDrop 2.5 introduces a first‑class configuration system with hierarchical precedence and zero external dependencies. It complements (not replaces) the existing CLI flags, enabling reproducible deployments, easier automation, and environment portability. The system is intentionally simple: an internal INI parser (`src/config/ini_parser.rs`) plus a composition layer (`src/config/mod.rs`) that merges values from multiple sources. diff --git a/doc/DEPLOYMENT.md b/doc/DEPLOYMENT.md index c599a00..3ec99b3 100644 --- a/doc/DEPLOYMENT.md +++ b/doc/DEPLOYMENT.md @@ -1,4 +1,4 @@ -# IronDrop Deployment Guide v2.7.0 +# IronDrop Deployment Guide v2.7.1 ## Overview @@ -792,4 +792,4 @@ perf record -g irondrop -d /srv/files strace -p $(pgrep irondrop) ``` -This deployment guide provides comprehensive coverage of production deployment scenarios and operational best practices for IronDrop v2.7.0. +This deployment guide provides comprehensive coverage of production deployment scenarios and operational best practices for IronDrop v2.7.1. diff --git a/doc/HTTP_STREAMING.md b/doc/HTTP_STREAMING.md index a5b25f9..edc1549 100644 --- a/doc/HTTP_STREAMING.md +++ b/doc/HTTP_STREAMING.md @@ -1,10 +1,10 @@ -# IronDrop Direct Upload Streaming (v2.7.0) +# IronDrop Direct Upload Streaming (v2.7.1) ## Overview IronDrop implements direct streaming uploads. Large request bodies are streamed to disk, avoiding unbounded memory growth. Small bodies are processed in memory. -**Status**: Production-ready (v2.7.0) +**Status**: Production-ready (v2.7.1) - Direct streaming implementation with bounded memory usage - Handling from small to very large files - Tests cover stability and cleanup @@ -311,7 +311,7 @@ The streaming system integrates with IronDrop's monitoring: ## Version History -- **v2.7.0**: Direct streaming implementation with unlimited file size support +- **v2.7.1**: Direct streaming implementation with unlimited file size support - Automatic memory/disk switching based on content size - `RequestBody` enum with `Memory` and `File` variants - Comprehensive test coverage with dedicated streaming tests diff --git a/doc/MONITORING.md b/doc/MONITORING.md index 8daf834..a8feed7 100644 --- a/doc/MONITORING.md +++ b/doc/MONITORING.md @@ -125,4 +125,4 @@ done Monitoring schema may evolve with additive fields. Consumers should ignore unknown keys. Breaking changes (renames/removals) will bump minor version >= 2.x. --- -*Monitoring Guide for IronDrop v2.7.0* +*Monitoring Guide for IronDrop v2.7.1* diff --git a/doc/MULTIPART_README.md b/doc/MULTIPART_README.md index f4ec223..25ee61f 100644 --- a/doc/MULTIPART_README.md +++ b/doc/MULTIPART_README.md @@ -1,10 +1,10 @@ -# IronDrop Direct Upload System v2.7.0 +# IronDrop Direct Upload System v2.7.1 This document describes the simplified direct upload system that replaced the multipart parser in IronDrop. ## Overview -IronDrop replaces legacy multipart parsing with a direct binary upload system focused on predictable memory use and simpler processing. The system handles raw binary uploads with bounded memory. (v2.7.0) +IronDrop replaces legacy multipart parsing with a direct binary upload system focused on predictable memory use and simpler processing. The system handles raw binary uploads with bounded memory. (v2.7.1) **Current Status**: Production-ready with direct streaming implementation and comprehensive test coverage (verified memory stability across all file sizes). @@ -115,4 +115,4 @@ The direct upload system is production-ready with: - ✅ Security validations maintained - ✅ Zero multipart parsing overhead -This represents a significant architectural improvement that maintains all security features while dramatically improving performance and reliability. \ No newline at end of file +This represents a significant architectural improvement that maintains all security features while dramatically improving performance and reliability. diff --git a/doc/README.md b/doc/README.md index 5f762fe..49c6369 100644 --- a/doc/README.md +++ b/doc/README.md @@ -115,7 +115,7 @@ Native zero-dependency template engine: variables, conditionals, embedded assets - **Streaming Tests**: HTTP layer streaming validation and large file bash integration tests - **Test Infrastructure**: Helper functions, data management, execution procedures -**Implementation Status**: ✅ **Production Ready** (v2.7.0) +**Implementation Status**: ✅ **Production Ready** (v2.7.1) - **English-Only Testing**: All test messages and output standardized to English - **Comprehensive Coverage**: Edge cases, security scenarios, performance validation, and streaming functionality - **Memory Optimization Tests**: Ultra-compact search engine validation for 10M+ files @@ -135,7 +135,7 @@ Native zero-dependency template engine: variables, conditionals, embedded assets - Multi-file concurrent upload handling - Client-side validation and error handling -**Implementation Status**: ✅ **Production Ready** (v2.7.0) +**Implementation Status**: ✅ **Production Ready** (v2.7.1) - Professional UI matching IronDrop's design language - Integrated with template engine and security systems - Supports unlimited file uploads with direct streaming architecture @@ -151,7 +151,7 @@ Native zero-dependency template engine: variables, conditionals, embedded assets - CLI configuration security enhancements - Defense-in-depth implementation details -**Security Status**: ✅ **Fully Implemented** (v2.7.0) +**Security Status**: ✅ **Fully Implemented** (v2.7.1) - Comprehensive input validation at multiple layers - System directory blacklisting and write permission checks - Direct streaming with unlimited file size support @@ -169,7 +169,7 @@ Native zero-dependency template engine: variables, conditionals, embedded assets - Configuration options and customization - Comprehensive API usage examples -**Implementation Status**: ✅ **Production Ready** (v2.7.0) +**Implementation Status**: ✅ **Production Ready** (v2.7.1) - RFC 7578 compliance with robust boundary detection and streaming support - Advanced streaming implementation for memory-efficient large file processing - Integrated with upload handler and HTTP processing with automatic mode selection @@ -189,7 +189,7 @@ Native zero-dependency template engine: variables, conditionals, embedded assets - **Integration Guide**: Seamless integration with existing upload handlers - **Testing Framework**: Comprehensive test coverage with dedicated HTTP streaming tests -**Implementation Status**: ✅ **Production Ready** (v2.7.0) +**Implementation Status**: ✅ **Production Ready** (v2.7.1) - **Automatic Mode Selection**: ≤1MB in memory, >1MB streamed to disk - **Zero Configuration**: Works transparently with existing upload handlers - **Resource Protection**: Prevents memory exhaustion from large uploads @@ -225,7 +225,7 @@ Native zero-dependency template engine: variables, conditionals, embedded assets - Security implementation and access control - Configuration options and troubleshooting guide -**Implementation Status**: ✅ **Production Ready** (v2.7.0) +**Implementation Status**: ✅ **Production Ready** (v2.7.1) - **Standard Search Engine**: Thread-safe search with LRU caching (5-minute TTL) - **Ultra-Compact Search Engine**: Memory-optimized for massive directories (10M+ files) - **Automatic Mode Selection**: Transparent switching based on directory size @@ -236,7 +236,7 @@ Native zero-dependency template engine: variables, conditionals, embedded assets - Accessibility-compliant UI with keyboard navigation support - Performance testing and benchmarking infrastructure -**🎉 NEW in v2.6**: Revolutionary direct streaming upload system with **unlimited file size support**, constant memory usage (~7MB), and simplified binary upload architecture. (v2.7.0) +**🎉 NEW in v2.6**: Revolutionary direct streaming upload system with **unlimited file size support**, constant memory usage (~7MB), and simplified binary upload architecture. (v2.7.1) ### 🌐 [WebDAV Implementation Guide](./WEBDAV_IMPLEMENTATION.md) ⭐ **Audience**: Backend Developers, Integrators, Client Compatibility Engineers @@ -368,10 +368,10 @@ Open a browser at [http://127.0.0.1:8080](http://127.0.0.1:8080) and you will se --- -## 🎉 What's New in v2.7.0 +## 🎉 What's New in v2.7.1 ### 📤 **Complete File Upload System** -IronDrop v2.7.0 includes a **production-ready file upload system** with enterprise-grade features: +IronDrop v2.7.1 includes a **production-ready file upload system** with enterprise-grade features: - **🔒 Enhanced Security**: Comprehensive input validation, boundary verification, and filename sanitization - **⚡ Performance**: Handles unlimited file sizes with constant memory usage and concurrent processing diff --git a/doc/TEMPLATE_SYSTEM.md b/doc/TEMPLATE_SYSTEM.md index 1e0caf2..63d8d8b 100644 --- a/doc/TEMPLATE_SYSTEM.md +++ b/doc/TEMPLATE_SYSTEM.md @@ -1,6 +1,6 @@ -# IronDrop Template & UI System Documentation (v2.7.0) +# IronDrop Template & UI System Documentation (v2.7.1) -**Status**: Production ready (v2.7.0) +**Status**: Production ready (v2.7.1) **Audience**: Backend & Frontend Developers, UI/UX Engineers, Integrators @@ -340,6 +340,6 @@ let err_html = engine.render_error_page(404, "Not Found", get_error_description( --- -*This document is part of the IronDrop v2.7.0 documentation suite and will evolve with future template system enhancements.* +*This document is part of the IronDrop v2.7.1 documentation suite and will evolve with future template system enhancements.* Return to documentation index: [./README.md](./README.md) diff --git a/doc/TESTING_DOCUMENTATION.md b/doc/TESTING_DOCUMENTATION.md index 0ac315c..849aaf6 100644 --- a/doc/TESTING_DOCUMENTATION.md +++ b/doc/TESTING_DOCUMENTATION.md @@ -1,6 +1,6 @@ # IronDrop Testing Documentation -Version 2.7.0 - Test Suite Overview +Version 2.7.1 - Test Suite Overview ## Overview @@ -376,7 +376,7 @@ fn test_new_feature() { - Code formatting validation - Documentation completeness -## Recent Improvements (v2.7.0) +## Recent Improvements (v2.7.1) ### Critical Fixes and Enhancements @@ -434,6 +434,6 @@ fn test_new_feature() { --- -*This document is part of the IronDrop v2.7.0 documentation suite. The test suite is continuously evolving to ensure comprehensive coverage and reliability.* +*This document is part of the IronDrop v2.7.1 documentation suite. The test suite is continuously evolving to ensure comprehensive coverage and reliability.* Return to documentation index: [./README.md](./README.md) diff --git a/doc/WEBDAV_IMPLEMENTATION.md b/doc/WEBDAV_IMPLEMENTATION.md index 7d14c1f..c9a7828 100644 --- a/doc/WEBDAV_IMPLEMENTATION.md +++ b/doc/WEBDAV_IMPLEMENTATION.md @@ -1,6 +1,6 @@ # WebDAV Implementation Guide -Version: 2.7.0 +Version: 2.7.1 This guide explains how the WebDAV implementation works in plain language. It focuses on request flow, lock behavior, and the core RFC 4918 semantics implemented in `src/webdav.rs`. From 77f1031d7f5359d490c6c67117873197b66fc931 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Wed, 15 Apr 2026 06:34:16 +0530 Subject: [PATCH 05/17] irondrop: add a parameter to disable rate-limiting --- README.md | 4 ++ config/irondrop.ini | 7 +++ doc/CONFIGURATION_SYSTEM.md | 11 +++- doc/WEBDAV_IMPLEMENTATION.md | 4 +- src/cli.rs | 6 +++ src/config/mod.rs | 74 +++++++++++++++++++++++++- src/handlers.rs | 2 + src/server.rs | 20 ++++++- src/upload.rs | 1 + tests/async_runtime_starvation_test.rs | 1 + tests/config_test.rs | 11 ++++ tests/direct_upload_test.rs | 1 + tests/integration_test.rs | 2 + tests/log_dir_test.rs | 1 + tests/monitor_test.rs | 1 + tests/ssl_test.rs | 6 +++ tests/webdav_copy_move_rfc_test.rs | 1 + tests/webdav_copy_move_test.rs | 1 + tests/webdav_error_semantics_test.rs | 1 + tests/webdav_error_xml_rfc_test.rs | 1 + tests/webdav_lock_if_rfc_test.rs | 1 + tests/webdav_options_propfind_test.rs | 1 + tests/webdav_propfind_rfc_test.rs | 1 + tests/webdav_proppatch_rfc_test.rs | 1 + tests/webdav_write_methods_test.rs | 1 + 25 files changed, 156 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c3a4b57..4b6a327 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,7 @@ IronDrop offers extensive customization through command-line arguments: | `-p, --port` | Port number (default: 8080) | `-p 3000` | | `--enable-upload` | Enable file uploads | `--enable-upload true` | | `--enable-webdav` | Enable WebDAV methods (`OPTIONS`, `PROPFIND`, `PROPPATCH`, `MKCOL`, `PUT`, `DELETE`, `COPY`, `MOVE`, `LOCK`, `UNLOCK`) | `--enable-webdav true` | +| `--disable-rate-limit` | Disable rate limiting **only when WebDAV is enabled** | `--enable-webdav true --disable-rate-limit true` | | `--username/--password` | Basic authentication | `--username admin --password secret` | | `-a, --allowed-extensions` | Restrict file types | `-a "*.pdf,*.doc,*.zip"` | | `-t, --threads` | Tokio runtime worker threads (default: 8) | `-t 16` | @@ -216,8 +217,11 @@ WebDAV can also be enabled in config: ```ini [webdav] enable_webdav = true +disable_rate_limit = false ``` +Note: `disable_rate_limit` is ignored unless WebDAV is enabled. + Quick CLI example: ```bash diff --git a/config/irondrop.ini b/config/irondrop.ini index f35734f..fc6b8dc 100644 --- a/config/irondrop.ini +++ b/config/irondrop.ini @@ -126,6 +126,13 @@ max_upload_size = 5GB # ⚠️ Recommendation: Use authentication and HTTPS when exposing WebDAV externally. enable_webdav = false +# 🧯 Disable WebDAV Rate Limiting (WebDAV-only) +# • false = Keep rate limiting enabled (recommended default) +# • true = Disable rate limiting, but ONLY when WebDAV is enabled +# • Ignored automatically if enable_webdav = false +# • Useful for Finder / DAV clients that burst metadata requests +disable_rate_limit = false + # =============================================================================== # 🔒 SECURITY CONFIGURATION diff --git a/doc/CONFIGURATION_SYSTEM.md b/doc/CONFIGURATION_SYSTEM.md index 5cdef0b..bdfa037 100644 --- a/doc/CONFIGURATION_SYSTEM.md +++ b/doc/CONFIGURATION_SYSTEM.md @@ -31,11 +31,12 @@ If none exist, startup proceeds with defaults + CLI overrides. | Flag | Description | |------|-------------| | `--config-file ` | Explicit path to an INI configuration file. Errors if not found. | +| `--disable-rate-limit ` | Disable rate limiting, effective only when WebDAV is enabled. | | `--ssl-cert ` | Path to PEM certificate file for HTTPS. Requires `--ssl-key`. | | `--ssl-key ` | Path to PEM private key file for HTTPS. Requires `--ssl-cert`. | ### INI Format Features -* Sections (`[server]`, `[upload]`, `[auth]`, `[logging]`, `[security]`, `[ssl]`) +* Sections (`[server]`, `[upload]`, `[webdav]`, `[auth]`, `[logging]`, `[security]`, `[ssl]`) * Comments starting with `#` or `;` * Key = value pairs (whitespace tolerant) * Inline comments after values (`key = value # note`) @@ -56,6 +57,10 @@ enabled = true # bool (true/false/yes/no/on/off/1/0) max_size = 5GB # File size parser (B, KB, MB, GB, TB; decimals allowed: 1.5GB) directory = /data/uploads # Optional override for upload target +[webdav] +enable_webdav = false # bool +disable_rate_limit = false # bool (ignored unless enable_webdav = true) + [auth] username = alice password = secret123 @@ -104,6 +109,10 @@ port = 9090 enabled = true max_size = 2.5GB +[webdav] +enable_webdav = true +disable_rate_limit = false + [auth] username = demo password = changeMe! diff --git a/doc/WEBDAV_IMPLEMENTATION.md b/doc/WEBDAV_IMPLEMENTATION.md index c9a7828..a6dd6b1 100644 --- a/doc/WEBDAV_IMPLEMENTATION.md +++ b/doc/WEBDAV_IMPLEMENTATION.md @@ -38,7 +38,9 @@ WebDAV methods are only processed when WebDAV is enabled in config/CLI. If disab Configuration controls: - CLI: `--enable-webdav true|false` +- CLI: `--disable-rate-limit true|false` (effective only when WebDAV is enabled) - INI: `[webdav] enable_webdav = true|false` (also accepted under `[server]`) +- INI: `[webdav] disable_rate_limit = true|false` (ignored when WebDAV is disabled) ## 3) Core request flow @@ -139,7 +141,7 @@ If a client reports odd behavior, check in this order: 1. WebDAV flag enabled (`--enable-webdav` / INI) 2. auth result (401s in log) -3. rate-limiter behavior (burst-heavy clients can be throttled) +3. rate-limiter behavior (burst-heavy clients can be throttled unless WebDAV `disable_rate_limit` is enabled) 4. lock token flow (`LOCK`/`If`/`UNLOCK`) 5. method/status pair in logs (`PROPFIND -> 207`, `LOCK -> 201`, etc.) diff --git a/src/cli.rs b/src/cli.rs index 3be5af7..1916a71 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -68,6 +68,10 @@ pub struct Cli { #[arg(long)] pub enable_webdav: Option, + /// Disable request rate limiting. Effective only when WebDAV is enabled. + #[arg(long)] + pub disable_rate_limit: Option, + /// Configuration file path - Specify a custom configuration file (INI format). If not provided, looks for irondrop.ini in current directory or ~/.config/irondrop/config.ini 🛠️ #[arg(long, value_parser = validate_config_file)] pub config_file: Option, @@ -222,6 +226,7 @@ mod tests { enable_upload: Some(false), max_upload_size: Some(100), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, @@ -260,6 +265,7 @@ mod tests { enable_upload: Some(true), max_upload_size: Some(100), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/src/config/mod.rs b/src/config/mod.rs index 87b53de..fe5e302 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -22,6 +22,7 @@ pub struct Config { pub enable_upload: bool, pub max_upload_size: u64, pub enable_webdav: bool, + pub disable_rate_limit: bool, // Security settings pub username: Option, @@ -63,6 +64,8 @@ impl Config { // Build configuration with precedence log::debug!("Building final configuration with precedence rules"); + let enable_webdav = Self::get_enable_webdav(&ini, cli); + let disable_rate_limit = Self::get_disable_rate_limit(&ini, cli, enable_webdav); let config = Self { listen: Self::get_listen(&ini, cli), port: Self::get_port(&ini, cli), @@ -72,7 +75,8 @@ impl Config { enable_upload: Self::get_enable_upload(&ini, cli), max_upload_size: Self::get_max_upload_size(&ini, cli), - enable_webdav: Self::get_enable_webdav(&ini, cli), + enable_webdav, + disable_rate_limit, username: Self::get_username(&ini, cli), password: Self::get_password(&ini, cli), @@ -259,6 +263,23 @@ impl Config { false } + fn get_disable_rate_limit(ini: &IniConfig, cli: &Cli, webdav_enabled: bool) -> bool { + let requested = if let Some(disable) = cli.disable_rate_limit { + disable + } else { + ini.get_bool("webdav", "disable_rate_limit") + .unwrap_or_default() + }; + + if requested && !webdav_enabled { + log::warn!( + "Ignoring disable_rate_limit because WebDAV is disabled. Enable WebDAV for this flag to take effect." + ); + return false; + } + requested + } + fn get_username(ini: &IniConfig, cli: &Cli) -> Option { // CLI argument if let Some(ref username) = cli.username { @@ -358,6 +379,14 @@ impl Config { ); } log::info!(" WebDAV Enabled: {}", self.enable_webdav); + log::info!( + " WebDAV Rate Limiting: {}", + if self.disable_rate_limit { + "Disabled" + } else { + "Enabled" + } + ); log::info!( " Authentication: {}", if self.username.is_some() { @@ -400,6 +429,7 @@ mod tests { enable_upload: None, max_upload_size: None, enable_webdav: None, + disable_rate_limit: None, config_file: None, log_dir: None, ssl_cert: None, @@ -423,6 +453,7 @@ mod tests { assert!(!config.enable_upload); assert_eq!(config.max_upload_size, u64::MAX); // No limit with direct streaming assert!(!config.enable_webdav); + assert!(!config.disable_rate_limit); assert_eq!(config.username, None); assert_eq!(config.password, None); assert_eq!(config.allowed_extensions, vec!["*.zip", "*.txt"]); @@ -448,6 +479,7 @@ max_upload_size = 5GB [webdav] enable_webdav = true +disable_rate_limit = true [auth] username = testuser @@ -476,6 +508,7 @@ detailed = false assert!(config.enable_upload); assert_eq!(config.max_upload_size, 5 * 1024 * 1024 * 1024); assert!(config.enable_webdav); + assert!(config.disable_rate_limit); assert_eq!(config.username, Some("testuser".to_string())); assert_eq!(config.password, Some("testpass".to_string())); assert_eq!(config.allowed_extensions, vec!["*.pdf", "*.doc"]); @@ -554,6 +587,7 @@ enable_webdav = true assert!(config.enable_upload); assert_eq!(config.max_upload_size, 2 * 1024 * 1024 * 1024); assert!(config.enable_webdav); + assert!(!config.disable_rate_limit); } #[test] @@ -627,4 +661,42 @@ directory = /some/other/path // Directory should always come from CLI assert_eq!(config.directory, temp_dir.path()); } + + #[test] + fn test_disable_rate_limit_effective_when_webdav_enabled() { + let temp_dir = TempDir::new().unwrap(); + let config_file = temp_dir.path().join("test.ini"); + let ini_content = r" +[webdav] +enable_webdav = true +disable_rate_limit = true +"; + fs::write(&config_file, ini_content).unwrap(); + + let mut cli = create_test_cli(temp_dir.path().to_path_buf()); + cli.config_file = Some(config_file.to_string_lossy().to_string()); + + let config = Config::load(&cli).unwrap(); + assert!(config.enable_webdav); + assert!(config.disable_rate_limit); + } + + #[test] + fn test_disable_rate_limit_ignored_when_webdav_disabled() { + let temp_dir = TempDir::new().unwrap(); + let config_file = temp_dir.path().join("test.ini"); + let ini_content = r" +[webdav] +enable_webdav = false +disable_rate_limit = true +"; + fs::write(&config_file, ini_content).unwrap(); + + let mut cli = create_test_cli(temp_dir.path().to_path_buf()); + cli.config_file = Some(config_file.to_string_lossy().to_string()); + + let config = Config::load(&cli).unwrap(); + assert!(!config.enable_webdav); + assert!(!config.disable_rate_limit); + } } diff --git a/src/handlers.rs b/src/handlers.rs index f3a0cc6..f4f90e9 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -572,6 +572,8 @@ pub fn handle_file_request( enable_upload: cli.enable_upload.unwrap_or(false), max_upload_size: cli.max_upload_size_bytes(), enable_webdav: cli.enable_webdav.unwrap_or(false), + disable_rate_limit: cli.enable_webdav.unwrap_or(false) + && cli.disable_rate_limit.unwrap_or(false), username: cli.username.clone(), password: cli.password.clone(), allowed_extensions: cli diff --git a/src/server.rs b/src/server.rs index 54bf307..f6b1bd7 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1107,6 +1107,7 @@ pub fn run_server_with_config(config: Config) -> Result<(), AppError> { enable_upload: Some(config.enable_upload), max_upload_size: Some(config.max_upload_size / (1024 * 1024)), // Convert bytes back to MB enable_webdav: Some(config.enable_webdav), + disable_rate_limit: Some(config.disable_rate_limit), config_file: None, // Not needed for server execution log_dir: config.log_dir, ssl_cert: config.ssl_cert, @@ -1184,6 +1185,16 @@ async fn run_server_async( let is_https = tls_acceptor.is_some(); let webdav_enabled = cli.enable_webdav.unwrap_or(false); + let disable_rate_limit_requested = cli.disable_rate_limit.unwrap_or(false); + let rate_limit_disabled = webdav_enabled && disable_rate_limit_requested; + if disable_rate_limit_requested && !webdav_enabled { + warn!( + "Ignoring --disable-rate-limit because WebDAV is disabled. Enable WebDAV for it to take effect." + ); + } + if rate_limit_disabled { + info!("WebDAV rate limiting is disabled by configuration."); + } let (rate_limit_per_minute, concurrent_per_ip) = if webdav_enabled { (3500, 128) } else { @@ -1316,6 +1327,7 @@ async fn run_server_async( password.clone(), chunk_size, rate_limiter.clone(), + rate_limit_disabled, stats.clone(), cli_arc.clone(), shared_router.clone(), @@ -1334,6 +1346,7 @@ async fn run_server_async( password.clone(), chunk_size, rate_limiter.clone(), + rate_limit_disabled, stats.clone(), cli_arc.clone(), shared_router.clone(), @@ -1356,13 +1369,14 @@ fn handle_connection( password: Arc>, chunk_size: usize, rate_limiter: Arc, + rate_limit_disabled: bool, stats: Arc, cli_config: Arc, router: Arc, tls_acceptor: Option, ) { let client_ip = peer_addr.ip(); - if !rate_limiter.check_rate_limit(client_ip) { + if !rate_limit_disabled && !rate_limiter.check_rate_limit(client_ip) { return; } @@ -1404,7 +1418,9 @@ fn handle_connection( Ok(()) }; - rate_limiter.release_connection(client_ip); + if !rate_limit_disabled { + rate_limiter.release_connection(client_ip); + } let _ = result; }); } diff --git a/src/upload.rs b/src/upload.rs index 1393a92..0aae8cf 100644 --- a/src/upload.rs +++ b/src/upload.rs @@ -1014,6 +1014,7 @@ mod tests { enable_upload: Some(true), max_upload_size: Some(100), // 100MB for testing enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/async_runtime_starvation_test.rs b/tests/async_runtime_starvation_test.rs index c3031c8..362174b 100644 --- a/tests/async_runtime_starvation_test.rs +++ b/tests/async_runtime_starvation_test.rs @@ -40,6 +40,7 @@ fn start_server(dir: std::path::PathBuf, threads: usize) -> TestServer { enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/config_test.rs b/tests/config_test.rs index 6156bd3..1e1a0be 100644 --- a/tests/config_test.rs +++ b/tests/config_test.rs @@ -179,6 +179,7 @@ verbose = false enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: Some(config_file.to_string_lossy().to_string()), log_dir: None, ssl_cert: None, @@ -227,6 +228,7 @@ max_upload_size = 1GB enable_upload: None, max_upload_size: None, enable_webdav: None, + disable_rate_limit: None, config_file: Some(explicit_config.to_string_lossy().to_string()), log_dir: None, ssl_cert: None, @@ -259,6 +261,7 @@ fn test_config_defaults() { enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, @@ -300,6 +303,7 @@ fn test_config_file_load_error() { enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: Some(nonexistent_config.to_string_lossy().to_string()), log_dir: None, ssl_cert: None, @@ -369,6 +373,7 @@ directory = {} enable_upload: None, max_upload_size: None, enable_webdav: None, + disable_rate_limit: None, config_file: Some(config_file.to_string_lossy().to_string()), log_dir: None, ssl_cert: None, @@ -413,6 +418,7 @@ port = 9999 enable_upload: None, max_upload_size: None, enable_webdav: None, + disable_rate_limit: None, config_file: Some(config_file.to_string_lossy().to_string()), log_dir: None, ssl_cert: None, @@ -462,6 +468,7 @@ fn test_config_invalid_port_values() { enable_upload: None, max_upload_size: None, enable_webdav: None, + disable_rate_limit: None, config_file: Some(config_file.to_string_lossy().to_string()), log_dir: None, ssl_cert: None, @@ -508,6 +515,7 @@ fn test_config_invalid_port_values() { enable_upload: None, max_upload_size: None, enable_webdav: None, + disable_rate_limit: None, config_file: Some(config_file.to_string_lossy().to_string()), log_dir: None, ssl_cert: None, @@ -565,6 +573,7 @@ fn test_config_invalid_file_size_formats() { enable_upload: None, max_upload_size: None, enable_webdav: None, + disable_rate_limit: None, config_file: Some(config_file.to_string_lossy().to_string()), log_dir: None, ssl_cert: None, @@ -642,6 +651,7 @@ fn test_config_boolean_edge_cases() { enable_upload: None, max_upload_size: None, enable_webdav: None, + disable_rate_limit: None, config_file: Some(config_file.to_string_lossy().to_string()), log_dir: None, ssl_cert: None, @@ -691,6 +701,7 @@ fn test_config_malformed_ini_syntax() { enable_upload: None, max_upload_size: None, enable_webdav: None, + disable_rate_limit: None, config_file: Some(config_file.to_string_lossy().to_string()), log_dir: None, ssl_cert: None, diff --git a/tests/direct_upload_test.rs b/tests/direct_upload_test.rs index 32e4dff..62ef7fd 100644 --- a/tests/direct_upload_test.rs +++ b/tests/direct_upload_test.rs @@ -23,6 +23,7 @@ fn create_test_cli(upload_dir: PathBuf) -> Cli { enable_upload: Some(true), max_upload_size: Some(100), // 100MB enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 2ec5437..60e6831 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -46,6 +46,7 @@ fn setup_test_server(username: Option, password: Option) -> Test enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, @@ -484,6 +485,7 @@ where enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/log_dir_test.rs b/tests/log_dir_test.rs index 4e3028f..353f78f 100644 --- a/tests/log_dir_test.rs +++ b/tests/log_dir_test.rs @@ -31,6 +31,7 @@ fn create_test_cli_with_log_dir(log_dir: Option) -> Cli { enable_upload: Some(false), max_upload_size: None, enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir, ssl_cert: None, diff --git a/tests/monitor_test.rs b/tests/monitor_test.rs index 033bf88..91bee06 100644 --- a/tests/monitor_test.rs +++ b/tests/monitor_test.rs @@ -43,6 +43,7 @@ fn setup_test_server() -> TestServer { enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/ssl_test.rs b/tests/ssl_test.rs index 86d7603..f6d9623 100644 --- a/tests/ssl_test.rs +++ b/tests/ssl_test.rs @@ -91,6 +91,7 @@ fn setup_ssl_server(username: Option, password: Option) -> TestS enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: Some(cert_path), @@ -273,6 +274,7 @@ fn test_ssl_missing_cert_file() { enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: Some(bogus_cert), @@ -314,6 +316,7 @@ fn test_ssl_missing_key_file() { enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: Some(cert_path), @@ -351,6 +354,7 @@ fn test_ssl_cert_without_key() { enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: Some(cert_path), @@ -391,6 +395,7 @@ fn test_ssl_key_without_cert() { enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, @@ -434,6 +439,7 @@ fn test_http_still_works_without_ssl() { enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(false), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/webdav_copy_move_rfc_test.rs b/tests/webdav_copy_move_rfc_test.rs index caaadfd..f4dacb6 100644 --- a/tests/webdav_copy_move_rfc_test.rs +++ b/tests/webdav_copy_move_rfc_test.rs @@ -40,6 +40,7 @@ where enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(true), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/webdav_copy_move_test.rs b/tests/webdav_copy_move_test.rs index e19a48e..35f069e 100644 --- a/tests/webdav_copy_move_test.rs +++ b/tests/webdav_copy_move_test.rs @@ -41,6 +41,7 @@ where enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(true), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/webdav_error_semantics_test.rs b/tests/webdav_error_semantics_test.rs index 6efcf69..0f36c7c 100644 --- a/tests/webdav_error_semantics_test.rs +++ b/tests/webdav_error_semantics_test.rs @@ -40,6 +40,7 @@ where enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(true), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/webdav_error_xml_rfc_test.rs b/tests/webdav_error_xml_rfc_test.rs index fdce359..3ff6fd6 100644 --- a/tests/webdav_error_xml_rfc_test.rs +++ b/tests/webdav_error_xml_rfc_test.rs @@ -39,6 +39,7 @@ where enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(true), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/webdav_lock_if_rfc_test.rs b/tests/webdav_lock_if_rfc_test.rs index e908fa9..3042cc0 100644 --- a/tests/webdav_lock_if_rfc_test.rs +++ b/tests/webdav_lock_if_rfc_test.rs @@ -40,6 +40,7 @@ where enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(true), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/webdav_options_propfind_test.rs b/tests/webdav_options_propfind_test.rs index b810082..7f44079 100644 --- a/tests/webdav_options_propfind_test.rs +++ b/tests/webdav_options_propfind_test.rs @@ -44,6 +44,7 @@ where enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(enable_webdav), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/webdav_propfind_rfc_test.rs b/tests/webdav_propfind_rfc_test.rs index fc8debe..ce9de8b 100644 --- a/tests/webdav_propfind_rfc_test.rs +++ b/tests/webdav_propfind_rfc_test.rs @@ -40,6 +40,7 @@ where enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(true), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/webdav_proppatch_rfc_test.rs b/tests/webdav_proppatch_rfc_test.rs index 8a6c9e7..5af9ef1 100644 --- a/tests/webdav_proppatch_rfc_test.rs +++ b/tests/webdav_proppatch_rfc_test.rs @@ -39,6 +39,7 @@ where enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(true), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, diff --git a/tests/webdav_write_methods_test.rs b/tests/webdav_write_methods_test.rs index dc0eb93..02e9b28 100644 --- a/tests/webdav_write_methods_test.rs +++ b/tests/webdav_write_methods_test.rs @@ -41,6 +41,7 @@ where enable_upload: Some(false), max_upload_size: Some(10240), enable_webdav: Some(true), + disable_rate_limit: Some(false), config_file: None, log_dir: None, ssl_cert: None, From 0c12ebf55763293a239f9251cf31e3d77bcab202 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Wed, 15 Apr 2026 07:31:19 +0530 Subject: [PATCH 06/17] irondrop: optimize authentication flow & error propogation --- src/http.rs | 34 ++++++++- src/middleware.rs | 87 +++++++++-------------- tests/monitor_test.rs | 158 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 223 insertions(+), 56 deletions(-) diff --git a/src/http.rs b/src/http.rs index bfee09a..5822257 100644 --- a/src/http.rs +++ b/src/http.rs @@ -344,9 +344,12 @@ pub async fn handle_client_async( } } Err(e) => { + let ignore_error_for_stats = matches!(e, AppError::NotFound) + && request_method == "PROPFIND" + && is_macos_finder_noise_path(&request_path); send_error_response_async(&mut stream, e, &log_prefix).await; if let Some(stats) = stats { - stats.record_request(false, 0); + stats.record_request(ignore_error_for_stats, 0); } } } @@ -356,6 +359,35 @@ pub async fn handle_client_async( } } +fn is_macos_finder_noise_path(path: &str) -> bool { + for component in path.split('/') { + if component.is_empty() { + continue; + } + if component == ".DS_Store" || component.starts_with("._") { + return true; + } + if matches!( + component, + ".AppleDouble" + | ".DocumentRevisions-V100" + | ".Spotlight-V100" + | ".Trashes" + | ".TemporaryItems" + | ".fseventsd" + | ".metadata_never_index" + | ".metadata_never_index_unless_rootfs" + | ".metadata_direct_scope_only" + | ".ql_disablethumbnails" + | ".ql_disablecache" + | ".hidden" + ) { + return true; + } + } + false +} + async fn send_error_response_async(stream: &mut S, error: AppError, log_prefix: &str) where S: tokio::io::AsyncWrite + Unpin, diff --git a/src/middleware.rs b/src/middleware.rs index c9b510b..efea231 100644 --- a/src/middleware.rs +++ b/src/middleware.rs @@ -9,7 +9,7 @@ use crate::error::AppError; use crate::http::Request; use base64::Engine; -use log::{debug, trace, warn}; +use log::{debug, trace}; /// Middleware trait – middlewares can inspect a request before it reaches a handler. /// Returning `Ok(())` continues the chain; returning `Err(AppError)` aborts processing. @@ -21,74 +21,40 @@ pub trait Middleware: Send + Sync + 'static { pub struct AuthMiddleware { pub username: Option, pub password: Option, + expected_authorization: Option>, } impl AuthMiddleware { pub fn new(username: Option, password: Option) -> Self { - Self { username, password } + let expected_authorization = match (&username, &password) { + (Some(user), Some(pass)) => { + let raw = format!("{user}:{pass}"); + let encoded = base64::engine::general_purpose::STANDARD.encode(raw.as_bytes()); + Some(format!("Basic {encoded}").into_bytes()) + } + _ => None, + }; + Self { + username, + password, + expected_authorization, + } } fn is_authenticated(&self, auth_header: Option<&String>) -> bool { - let (Some(user), Some(pass)) = (&self.username, &self.password) else { + let Some(expected) = &self.expected_authorization else { trace!("Authentication disabled - allowing request"); return true; // auth disabled }; debug!("Authentication required - checking credentials"); - let header = match auth_header { - Some(h) => { - trace!("Authorization header found"); - h - } - None => { - debug!("No Authorization header provided"); - return false; - } - }; - let credentials = match header.strip_prefix("Basic ") { - Some(c) => { - trace!("Basic authentication scheme detected"); - c - } - None => { - debug!("Invalid authentication scheme (not Basic)"); - return false; - } - }; - let decoded = match base64::engine::general_purpose::STANDARD.decode(credentials) { - Ok(d) => { - trace!("Successfully decoded base64 credentials"); - d - } - Err(_) => { - debug!("Failed to decode base64 credentials"); - return false; - } + let Some(header) = auth_header else { + debug!("No Authorization header provided"); + return false; }; - let decoded_str = match String::from_utf8(decoded) { - Ok(s) => { - trace!("Successfully converted credentials to UTF-8"); - s - } - Err(_) => { - debug!("Invalid UTF-8 in decoded credentials"); - return false; - } - }; - if let Some((provided_user, provided_pass)) = decoded_str.split_once(':') { - trace!("Parsed username from credentials: '{}'", provided_user); - let auth_result = provided_user == user && provided_pass == pass; - if auth_result { - debug!("Authentication successful for user: '{}'", provided_user); - } else { - warn!("Authentication failed for user: '{}'", provided_user); - } - auth_result - } else { - debug!("Invalid credential format (missing colon separator)"); - false - } + + constant_time_eq_bytes(header.as_bytes(), expected) } } @@ -103,3 +69,14 @@ impl Middleware for AuthMiddleware { Ok(()) } } + +fn constant_time_eq_bytes(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + let mut diff: u8 = 0; + for (x, y) in a.iter().zip(b.iter()) { + diff |= x ^ y; + } + diff == 0 +} diff --git a/tests/monitor_test.rs b/tests/monitor_test.rs index 91bee06..1ee1824 100644 --- a/tests/monitor_test.rs +++ b/tests/monitor_test.rs @@ -5,6 +5,7 @@ use irondrop::cli::Cli; use irondrop::server::run_server; +use reqwest::Method; use reqwest::StatusCode; use reqwest::blocking::Client; use std::fs::File; @@ -97,6 +98,24 @@ fn extract_bytes_served(json: &str) -> u64 { panic!("bytes_served not found in json: {json}"); } +fn extract_errors(json: &str) -> u64 { + if let Some(idx) = json.find("\"errors\":") { + let slice = &json[idx + 9..]; + let mut digits = String::new(); + for ch in slice.chars() { + if ch.is_ascii_digit() { + digits.push(ch); + } else { + break; + } + } + if let Ok(v) = digits.parse::() { + return v; + } + } + panic!("errors not found in json: {json}"); +} + #[test] fn test_monitor_json_and_bytes_served_accounting() { let server = setup_test_server(); @@ -172,3 +191,142 @@ fn test_monitor_html_served() { assert!(body.contains(" Date: Wed, 15 Apr 2026 07:59:44 +0530 Subject: [PATCH 07/17] irondrop: rate limit some logspam --- src/http.rs | 41 ++++++++++++++++++++++++++++++++++++----- src/middleware.rs | 33 ++++++++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/src/http.rs b/src/http.rs index 5822257..e7646b4 100644 --- a/src/http.rs +++ b/src/http.rs @@ -8,9 +8,9 @@ use crate::router::Router; use log::{debug, error, info, trace}; use std::collections::HashMap; use std::path::PathBuf; -use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; /// Maximum size for request body (10GB) to prevent memory exhaustion attacks @@ -326,9 +326,11 @@ pub async fn handle_client_async( match response_result { Ok(response) => { - info!( - "{} {} {} -> {}", - log_prefix, request_method, request_path, response.status_code + log_request_line( + &log_prefix, + &request_method, + &request_path, + response.status_code, ); match send_response_async(&mut stream, response, &log_prefix).await { Ok(body_bytes) => { @@ -388,6 +390,35 @@ fn is_macos_finder_noise_path(path: &str) -> bool { false } +fn log_request_line(log_prefix: &str, method: &str, path: &str, status_code: u16) { + let path_only = path.split('?').next().unwrap_or(path); + if is_monitor_path(path_only) { + monitor_request_log_rate_limited(method, status_code); + return; + } + info!("{log_prefix} {method} {path} -> {status_code}"); +} + +fn is_monitor_path(path: &str) -> bool { + path == "/monitor" || path.starts_with("/_irondrop/monitor") +} + +fn monitor_request_log_rate_limited(method: &str, status_code: u16) { + static STATE: OnceLock> = OnceLock::new(); + let state = STATE.get_or_init(|| Mutex::new((Instant::now() - Duration::from_secs(3600), 0))); + if let Ok(mut st) = state.lock() { + st.1 += 1; + if st.0.elapsed() >= Duration::from_secs(45) { + info!( + "Monitor requests: method={method} status={status_code} count_since_last_log={}", + st.1 + ); + st.0 = Instant::now(); + st.1 = 0; + } + } +} + async fn send_error_response_async(stream: &mut S, error: AppError, log_prefix: &str) where S: tokio::io::AsyncWrite + Unpin, diff --git a/src/middleware.rs b/src/middleware.rs index efea231..0b7a45f 100644 --- a/src/middleware.rs +++ b/src/middleware.rs @@ -9,7 +9,9 @@ use crate::error::AppError; use crate::http::Request; use base64::Engine; -use log::{debug, trace}; +use log::{trace, warn}; +use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, Instant}; /// Middleware trait – middlewares can inspect a request before it reaches a handler. /// Returning `Ok(())` continues the chain; returning `Err(AppError)` aborts processing. @@ -47,14 +49,17 @@ impl AuthMiddleware { return true; // auth disabled }; - debug!("Authentication required - checking credentials"); - let Some(header) = auth_header else { - debug!("No Authorization header provided"); + auth_failure_rate_limited("missing authorization header"); return false; }; - constant_time_eq_bytes(header.as_bytes(), expected) + if constant_time_eq_bytes(header.as_bytes(), expected) { + true + } else { + auth_failure_rate_limited("invalid credentials"); + false + } } } @@ -80,3 +85,21 @@ fn constant_time_eq_bytes(a: &[u8], b: &[u8]) -> bool { } diff == 0 } + +fn auth_failure_rate_limited(reason: &'static str) { + static STATE: OnceLock> = OnceLock::new(); + let state = STATE.get_or_init(|| Mutex::new((Instant::now() - Duration::from_secs(3600), 0))); + if let Ok(mut st) = state.lock() { + st.1 += 1; + if st.0.elapsed() >= Duration::from_secs(20) { + warn!( + "Authentication failed ({reason}). failures_since_last_log={}", + st.1 + ); + st.0 = Instant::now(); + st.1 = 0; + } + } else { + warn!("Authentication failed ({reason})."); + } +} From 1057e75161103a384a1595bad9f04e2fe83ba254 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Wed, 15 Apr 2026 09:12:03 +0530 Subject: [PATCH 08/17] irondrop: macos finder webdav optimizations --- src/http.rs | 31 +----------- src/utils.rs | 33 ++++++++++++ src/webdav.rs | 38 +++++++++++++- tests/monitor_test.rs | 72 +++++++++++++++++++++++++++ tests/webdav_copy_move_test.rs | 26 ++++++++++ tests/webdav_options_propfind_test.rs | 17 +++++++ 6 files changed, 186 insertions(+), 31 deletions(-) diff --git a/src/http.rs b/src/http.rs index e7646b4..21c5db4 100644 --- a/src/http.rs +++ b/src/http.rs @@ -348,7 +348,7 @@ pub async fn handle_client_async( Err(e) => { let ignore_error_for_stats = matches!(e, AppError::NotFound) && request_method == "PROPFIND" - && is_macos_finder_noise_path(&request_path); + && crate::utils::is_macos_finder_noise_path(&request_path); send_error_response_async(&mut stream, e, &log_prefix).await; if let Some(stats) = stats { stats.record_request(ignore_error_for_stats, 0); @@ -361,35 +361,6 @@ pub async fn handle_client_async( } } -fn is_macos_finder_noise_path(path: &str) -> bool { - for component in path.split('/') { - if component.is_empty() { - continue; - } - if component == ".DS_Store" || component.starts_with("._") { - return true; - } - if matches!( - component, - ".AppleDouble" - | ".DocumentRevisions-V100" - | ".Spotlight-V100" - | ".Trashes" - | ".TemporaryItems" - | ".fseventsd" - | ".metadata_never_index" - | ".metadata_never_index_unless_rootfs" - | ".metadata_direct_scope_only" - | ".ql_disablethumbnails" - | ".ql_disablecache" - | ".hidden" - ) { - return true; - } - } - false -} - fn log_request_line(log_prefix: &str, method: &str, path: &str, status_code: u16) { let path_only = path.split('?').next().unwrap_or(path); if is_monitor_path(path_only) { diff --git a/src/utils.rs b/src/utils.rs index 2f56b49..f929a1b 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -201,3 +201,36 @@ fn normalize_path(path: &Path) -> Result { pub fn is_hidden_file(filename: &str) -> bool { filename.starts_with("._") || filename.starts_with(".") || filename == ".DS_Store" } + +pub fn is_macos_finder_noise_path(path: &str) -> bool { + let path_only = path.split('?').next().unwrap_or(path); + for component in path_only.split('/') { + if component.is_empty() { + continue; + } + if component == ".DS_Store" || component.starts_with("._") { + return true; + } + if component.starts_with(".AU.") || component.starts_with(".ArchiveServiceTemp") { + return true; + } + if matches!( + component, + ".AppleDouble" + | ".DocumentRevisions-V100" + | ".Spotlight-V100" + | ".Trashes" + | ".TemporaryItems" + | ".fseventsd" + | ".metadata_never_index" + | ".metadata_never_index_unless_rootfs" + | ".metadata_direct_scope_only" + | ".ql_disablethumbnails" + | ".ql_disablecache" + | ".hidden" + ) { + return true; + } + } + false +} diff --git a/src/webdav.rs b/src/webdav.rs index a82aad0..91c22ee 100644 --- a/src/webdav.rs +++ b/src/webdav.rs @@ -108,6 +108,12 @@ fn handle_propfind( ) -> Result { let depth = parse_depth_header(&request.headers)?; let mode = parse_propfind_mode(request)?; + if crate::utils::is_macos_finder_noise_path(&request.path) { + let fast_path = resolve_request_path_without_canonicalize(base_dir, &request.path)?; + if !fast_path.exists() { + return Ok(status_response(404, "Not Found")); + } + } let target_path = resolve_request_path(base_dir, &request.path)?; if !target_path.exists() { @@ -569,7 +575,16 @@ fn handle_copy_or_move( return Ok(status_response(409, "Conflict")); }; if !parent.exists() || !parent.is_dir() { - return Ok(status_response(409, "Conflict")); + if is_move && is_finder_archive_temp_path(&source) { + debug!( + "WebDAV MOVE creating missing destination parent for Finder temp source={} destination_parent={}", + source.display(), + parent.display() + ); + std::fs::create_dir_all(parent)?; + } else { + return Ok(status_response(409, "Conflict")); + } } let overwrite = request @@ -1617,6 +1632,27 @@ fn status_response(status_code: u16, status_text: &str) -> Response { } } +fn is_finder_archive_temp_path(path: &Path) -> bool { + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + return false; + }; + name.starts_with(".AU.") || name.starts_with(".ArchiveServiceTemp") +} + +fn resolve_request_path_without_canonicalize( + base_dir: &Path, + request_path: &str, +) -> Result { + let path_only = request_path.split('?').next().unwrap_or(request_path); + let requested_path = PathBuf::from(path_only.strip_prefix('/').unwrap_or(path_only)); + let safe_path = normalize_relative_path(&requested_path)?; + let full_path = base_dir.join(safe_path); + if !full_path.starts_with(base_dir) { + return Err(AppError::Forbidden); + } + Ok(full_path) +} + fn resolve_request_path(base_dir: &Path, request_path: &str) -> Result { let path_only = request_path.split('?').next().unwrap_or(request_path); let requested_path = PathBuf::from(path_only.strip_prefix('/').unwrap_or(path_only)); diff --git a/tests/monitor_test.rs b/tests/monitor_test.rs index 1ee1824..651cf1a 100644 --- a/tests/monitor_test.rs +++ b/tests/monitor_test.rs @@ -330,3 +330,75 @@ fn test_monitor_ignores_macos_metadata_propfind_noise_from_error_counter() { let errors2 = extract_errors(&after); assert_eq!(errors2, errors1); } + +#[test] +fn test_monitor_ignores_archive_service_temp_propfind_noise_from_error_counter() { + let dir = tempdir().unwrap(); + + let cli = Cli { + directory: dir.path().to_path_buf(), + listen: Some("127.0.0.1".to_string()), + port: Some(0), + allowed_extensions: Some("*".to_string()), + threads: Some(4), + chunk_size: Some(1024), + verbose: Some(false), + detailed_logging: Some(false), + username: None, + password: None, + enable_upload: Some(false), + max_upload_size: Some(10240), + enable_webdav: Some(true), + disable_rate_limit: Some(false), + config_file: None, + log_dir: None, + ssl_cert: None, + ssl_key: None, + }; + + let (shutdown_tx, shutdown_rx) = mpsc::channel(); + let (addr_tx, addr_rx) = mpsc::channel(); + let handle = thread::spawn(move || { + if let Err(e) = run_server(cli, Some(shutdown_rx), Some(addr_tx)) { + eprintln!("Server thread failed: {e}"); + } + }); + let addr = addr_rx.recv().unwrap(); + let server = TestServer { + addr, + shutdown_tx, + handle: Some(handle), + _temp_dir: dir, + }; + + let client = Client::new(); + let baseline = client + .get(format!("http://{}/_irondrop/monitor?json=1", server.addr)) + .send() + .unwrap() + .text() + .unwrap(); + let errors1 = extract_errors(&baseline); + + let resp = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!( + "http://{}/.ArchiveServiceTemp.sb-12345-AbCdEf/Contents", + server.addr + ), + ) + .header("Depth", "0") + .send() + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + let after = client + .get(format!("http://{}/_irondrop/monitor?json=1", server.addr)) + .send() + .unwrap() + .text() + .unwrap(); + let errors2 = extract_errors(&after); + assert_eq!(errors2, errors1); +} diff --git a/tests/webdav_copy_move_test.rs b/tests/webdav_copy_move_test.rs index 35f069e..23337c6 100644 --- a/tests/webdav_copy_move_test.rs +++ b/tests/webdav_copy_move_test.rs @@ -200,3 +200,29 @@ fn test_copy_path_only_destination_is_bad_request() { assert_eq!(response.status(), StatusCode::BAD_REQUEST); } + +#[test] +fn test_move_finder_temp_source_creates_missing_destination_parent() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join(".AU.PCOUC")).unwrap(); + let mut src = File::create(root.join(".AU.PCOUC").join("a.txt")).unwrap(); + write!(src, "tmp").unwrap(); + }); + let client = Client::new(); + + let response = client + .request( + Method::from_bytes(b"MOVE").unwrap(), + format!("http://{}/.AU.PCOUC/", server.addr), + ) + .header( + "Destination", + format!("http://{}/missing/parent/.AU.PCOUC/", server.addr), + ) + .send() + .unwrap(); + + assert_eq!(response.status(), StatusCode::CREATED); + assert!(!server.root.join(".AU.PCOUC").exists()); + assert!(server.root.join("missing/parent/.AU.PCOUC").exists()); +} diff --git a/tests/webdav_options_propfind_test.rs b/tests/webdav_options_propfind_test.rs index 7f44079..5e5eeba 100644 --- a/tests/webdav_options_propfind_test.rs +++ b/tests/webdav_options_propfind_test.rs @@ -162,6 +162,23 @@ fn test_propfind_depth_zero_on_file_returns_multistatus() { assert!(text.contains("")); } +#[test] +fn test_propfind_finder_probe_missing_returns_not_found_fast() { + let server = setup_test_server_with_tree_and_webdav(|_root| {}, true); + let client = Client::new(); + + let response = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!("http://{}/._missing", server.addr), + ) + .header("Depth", "0") + .send() + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + #[test] fn test_propfind_depth_one_on_collection_includes_children() { let server = setup_test_server_with_tree(|root| { From aa49e07218c5cb1a9d745115ca55f36fe836caa0 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Wed, 15 Apr 2026 09:26:50 +0530 Subject: [PATCH 09/17] irondrop: handle noisepaths directly in propfind --- src/webdav.rs | 24 +++++++++++++++++++++++- tests/webdav_copy_move_test.rs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/webdav.rs b/src/webdav.rs index 91c22ee..857ad65 100644 --- a/src/webdav.rs +++ b/src/webdav.rs @@ -555,6 +555,12 @@ fn handle_copy_or_move( request.headers.get("host").map(String::as_str), )?; let destination = resolve_request_path(base_dir, &destination_request_path)?; + debug!( + "WebDAV {} destination_request_path={} destination={}", + if is_move { "MOVE" } else { "COPY" }, + destination_request_path, + destination.display() + ); if let Some(response) = lock_precondition_response(request, &destination) { debug!( "WebDAV {} blocked by destination lock destination={} status={}", @@ -575,7 +581,7 @@ fn handle_copy_or_move( return Ok(status_response(409, "Conflict")); }; if !parent.exists() || !parent.is_dir() { - if is_move && is_finder_archive_temp_path(&source) { + if is_move && is_finder_archive_related_move(&source, &destination_request_path) { debug!( "WebDAV MOVE creating missing destination parent for Finder temp source={} destination_parent={}", source.display(), @@ -1639,6 +1645,22 @@ fn is_finder_archive_temp_path(path: &Path) -> bool { name.starts_with(".AU.") || name.starts_with(".ArchiveServiceTemp") } +fn is_finder_archive_related_move(source: &Path, destination_request_path: &str) -> bool { + if is_finder_archive_temp_path(source) { + return true; + } + if destination_request_path.contains(".sb-") { + return true; + } + if destination_request_path.contains("/.AU.") + || destination_request_path.contains("/.ArchiveServiceTemp") + { + return true; + } + destination_request_path.contains("AUHelperService") + || destination_request_path.contains("A%20Document%20Being%20Saved%20By%20AUHelperService") +} + fn resolve_request_path_without_canonicalize( base_dir: &Path, request_path: &str, diff --git a/tests/webdav_copy_move_test.rs b/tests/webdav_copy_move_test.rs index 23337c6..3c432cd 100644 --- a/tests/webdav_copy_move_test.rs +++ b/tests/webdav_copy_move_test.rs @@ -226,3 +226,37 @@ fn test_move_finder_temp_source_creates_missing_destination_parent() { assert!(!server.root.join(".AU.PCOUC").exists()); assert!(server.root.join("missing/parent/.AU.PCOUC").exists()); } + +#[test] +fn test_move_into_missing_sb_container_creates_parent() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("TaiwanStuff")).unwrap(); + let mut src = File::create(root.join("TaiwanStuff").join("a.txt")).unwrap(); + write!(src, "x").unwrap(); + }); + let client = Client::new(); + + let response = client + .request( + Method::from_bytes(b"MOVE").unwrap(), + format!("http://{}/TaiwanStuff/", server.addr), + ) + .header( + "Destination", + format!( + "http://{}/TaiwanStuff.sb-1234-AbCdEf/TaiwanStuff/", + server.addr + ), + ) + .send() + .unwrap(); + + assert_eq!(response.status(), StatusCode::CREATED); + assert!(!server.root.join("TaiwanStuff").exists()); + assert!( + server + .root + .join("TaiwanStuff.sb-1234-AbCdEf/TaiwanStuff") + .exists() + ); +} From 22e392cc8a8f9c392ffd8e1e8bfef9ccf1a0e913 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Thu, 16 Apr 2026 21:18:27 +0530 Subject: [PATCH 10/17] irondrop: exclude webdav from allowed_extension check --- src/webdav.rs | 40 ++------- tests/webdav_options_propfind_test.rs | 125 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 35 deletions(-) diff --git a/src/webdav.rs b/src/webdav.rs index 857ad65..1646c7b 100644 --- a/src/webdav.rs +++ b/src/webdav.rs @@ -67,13 +67,13 @@ const DAV_NAMESPACE: &str = "DAV:"; pub fn handle_webdav_request( request: &Request, base_dir: &Path, - allowed_extensions: &[glob::Pattern], + _allowed_extensions: &[glob::Pattern], ) -> Result { match request.method.as_str() { "OPTIONS" => Ok(build_options_response()), - "PROPFIND" => handle_propfind(request, base_dir, allowed_extensions), + "PROPFIND" => handle_propfind(request, base_dir), "MKCOL" => handle_mkcol(request, base_dir), - "PUT" => handle_put(request, base_dir, allowed_extensions), + "PUT" => handle_put(request, base_dir), "DELETE" => handle_delete(request, base_dir), "COPY" => handle_copy_or_move(request, base_dir, false), "MOVE" => handle_copy_or_move(request, base_dir, true), @@ -101,11 +101,7 @@ fn build_options_response() -> Response { } } -fn handle_propfind( - request: &Request, - base_dir: &Path, - allowed_extensions: &[glob::Pattern], -) -> Result { +fn handle_propfind(request: &Request, base_dir: &Path) -> Result { let depth = parse_depth_header(&request.headers)?; let mode = parse_propfind_mode(request)?; if crate::utils::is_macos_finder_noise_path(&request.path) { @@ -120,14 +116,6 @@ fn handle_propfind( return Err(AppError::NotFound); } - if target_path.is_file() - && !allowed_extensions - .iter() - .any(|pattern| pattern.matches_path(&target_path)) - { - return Err(AppError::Forbidden); - } - if depth == DavDepth::Infinity && target_path.is_dir() { return Ok(propfind_finite_depth_error_response()); } @@ -137,13 +125,6 @@ fn handle_propfind( for entry in std::fs::read_dir(&target_path)? { let entry = entry?; let entry_path = entry.path(); - if entry_path.is_file() - && !allowed_extensions - .iter() - .any(|pattern| pattern.matches_path(&entry_path)) - { - continue; - } resources.push(entry_path); } } @@ -411,11 +392,7 @@ fn handle_mkcol(request: &Request, base_dir: &Path) -> Result Result { +fn handle_put(request: &Request, base_dir: &Path) -> Result { let _op_guard = op_guard() .lock() .map_err(|_| AppError::InternalServerError("dav operation guard poisoned".to_string()))?; @@ -443,13 +420,6 @@ fn handle_put( return Ok(status_response(405, "Method Not Allowed")); } - if !allowed_extensions - .iter() - .any(|pattern| pattern.matches_path(&target_path)) - { - return Err(AppError::Forbidden); - } - let existed = target_path.exists(); let body_bytes = request_body_bytes(request)?; diff --git a/tests/webdav_options_propfind_test.rs b/tests/webdav_options_propfind_test.rs index 5e5eeba..d2e5769 100644 --- a/tests/webdav_options_propfind_test.rs +++ b/tests/webdav_options_propfind_test.rs @@ -70,6 +70,61 @@ where } } +fn setup_test_server_with_tree_webdav_and_extensions( + populate: F, + enable_webdav: bool, + allowed_extensions: &str, +) -> TestServer +where + F: FnOnce(&std::path::Path), +{ + let dir = tempdir().unwrap(); + populate(dir.path()); + + let file_path = dir.path().join("test.txt"); + let mut file = File::create(&file_path).unwrap(); + writeln!(file, "hello from test file").unwrap(); + + let cli = Cli { + directory: dir.path().to_path_buf(), + listen: Some("127.0.0.1".to_string()), + port: Some(0), + allowed_extensions: Some(allowed_extensions.to_string()), + threads: Some(4), + chunk_size: Some(1024), + verbose: Some(false), + detailed_logging: Some(false), + username: None, + password: None, + enable_upload: Some(false), + max_upload_size: Some(10240), + enable_webdav: Some(enable_webdav), + disable_rate_limit: Some(false), + config_file: None, + log_dir: None, + ssl_cert: None, + ssl_key: None, + }; + + let (shutdown_tx, shutdown_rx) = mpsc::channel(); + let (addr_tx, addr_rx) = mpsc::channel(); + + let server_handle = thread::spawn(move || { + if let Err(e) = run_server(cli, Some(shutdown_rx), Some(addr_tx)) { + eprintln!("Server thread failed: {e}"); + } + }); + + let server_addr = addr_rx.recv().unwrap(); + + TestServer { + addr: server_addr, + shutdown_tx, + handle: Some(server_handle), + _temp_dir: dir, + } +} + fn setup_test_server_with_tree(populate: F) -> TestServer where F: FnOnce(&std::path::Path), @@ -228,3 +283,73 @@ fn test_propfind_infinite_depth_rejected_with_finite_depth_precondition() { let body = response.text().unwrap(); assert!(body.contains("propfind-finite-depth")); } + +#[test] +fn test_propfind_includes_extensionless_files_even_when_download_filter_is_star_dot_star() { + let server = setup_test_server_with_tree_webdav_and_extensions( + |root| { + create_dir_all(root.join("dav")).unwrap(); + let mut extless = File::create(root.join("dav").join("call_001")).unwrap(); + writeln!(extless, "voice log").unwrap(); + let mut dotted = File::create(root.join("dav").join("call_001.txt")).unwrap(); + writeln!(dotted, "voice log txt").unwrap(); + }, + true, + "*.*", + ); + let client = Client::new(); + let body = r#" +"#; + + let response = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!("http://{}/dav/", server.addr), + ) + .header("Depth", "1") + .header("Content-Type", "application/xml") + .body(body.to_string()) + .send() + .unwrap(); + + assert_eq!(response.status().as_u16(), 207); + let xml = response.text().unwrap(); + assert!(xml.contains("/dav/call_001")); + assert!(xml.contains("/dav/call_001.txt")); +} + +#[test] +fn test_put_allows_extensionless_files_even_when_download_filter_is_star_dot_star() { + let server = setup_test_server_with_tree_webdav_and_extensions( + |root| { + create_dir_all(root.join("dav")).unwrap(); + }, + true, + "*.*", + ); + let client = Client::new(); + + let response = client + .request( + Method::from_bytes(b"PUT").unwrap(), + format!("http://{}/dav/upload_no_ext", server.addr), + ) + .body("uploaded via webdav".to_string()) + .send() + .unwrap(); + + assert_eq!(response.status(), StatusCode::CREATED); + + let propfind = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!("http://{}/dav/upload_no_ext", server.addr), + ) + .header("Depth", "0") + .header("Content-Type", "application/xml") + .body(r#""#) + .send() + .unwrap(); + + assert_eq!(propfind.status().as_u16(), 207); +} From 843993f1d24a273129d391d5082d2d136539bdc1 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Thu, 16 Apr 2026 22:33:17 +0530 Subject: [PATCH 11/17] irondrop: fix authentication test failure The Windows panic is from Instant underflow in rate-limited logging initialization, which cause auth middleware to panic and return 500 instead of 401 . Signed-off-by: Harshit Jain --- src/http.rs | 6 +++++- src/middleware.rs | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/http.rs b/src/http.rs index 21c5db4..0b6b28f 100644 --- a/src/http.rs +++ b/src/http.rs @@ -376,7 +376,11 @@ fn is_monitor_path(path: &str) -> bool { fn monitor_request_log_rate_limited(method: &str, status_code: u16) { static STATE: OnceLock> = OnceLock::new(); - let state = STATE.get_or_init(|| Mutex::new((Instant::now() - Duration::from_secs(3600), 0))); + let state = STATE.get_or_init(|| { + let now = Instant::now(); + let initial = now.checked_sub(Duration::from_secs(3600)).unwrap_or(now); + Mutex::new((initial, 0)) + }); if let Ok(mut st) = state.lock() { st.1 += 1; if st.0.elapsed() >= Duration::from_secs(45) { diff --git a/src/middleware.rs b/src/middleware.rs index 0b7a45f..165e2b3 100644 --- a/src/middleware.rs +++ b/src/middleware.rs @@ -88,7 +88,11 @@ fn constant_time_eq_bytes(a: &[u8], b: &[u8]) -> bool { fn auth_failure_rate_limited(reason: &'static str) { static STATE: OnceLock> = OnceLock::new(); - let state = STATE.get_or_init(|| Mutex::new((Instant::now() - Duration::from_secs(3600), 0))); + let state = STATE.get_or_init(|| { + let now = Instant::now(); + let initial = now.checked_sub(Duration::from_secs(3600)).unwrap_or(now); + Mutex::new((initial, 0)) + }); if let Ok(mut st) = state.lock() { st.1 += 1; if st.0.elapsed() >= Duration::from_secs(20) { From 07d49a2a9220f133c86426ad069feac89b33b657 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Thu, 16 Apr 2026 23:47:31 +0530 Subject: [PATCH 12/17] inrondrop: patch some webdav identified depth & mkcol gaps * Add generated tests Signed-off-by: Harshit Jain --- README.md | 2 +- doc/README.md | 2 +- doc/WEBDAV_IMPLEMENTATION.md | 5 +- src/webdav.rs | 156 ++++++++++++++++++-------- tests/webdav_copy_move_rfc_test.rs | 149 ++++++++++++++++++++++++ tests/webdav_lock_if_rfc_test.rs | 92 +++++++++++++++ tests/webdav_options_propfind_test.rs | 65 +++++++++-- tests/webdav_propfind_rfc_test.rs | 69 +++++++++++- tests/webdav_proppatch_rfc_test.rs | 52 +++++++++ tests/webdav_write_methods_test.rs | 84 ++++++++++++++ 10 files changed, 611 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 4b6a327..734a449 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ IronDrop includes an RFC 4918-focused implementation. The WebDAV core engine is - Supported methods: `OPTIONS`, `PROPFIND`, `PROPPATCH`, `MKCOL`, `PUT`, `DELETE`, `COPY`, `MOVE`, `LOCK`, `UNLOCK` - WebDAV is feature-gated and disabled by default; enable explicitly with `--enable-webdav true` - Capability headers: `DAV: 1,2`, `Allow`, `MS-Author-Via` -- `PROPFIND`: `allprop`, `propname`, named `prop`, per-property `propstat` grouping (`200`/`404`), and finite-depth refusal (`403` + `propfind-finite-depth`) +- `PROPFIND`: `allprop`, `propname`, named `prop`, per-property `propstat` grouping (`200`/`404`), with `Depth: 0`, `1`, and recursive `infinity` - `PROPPATCH`: dead-property `set`/`remove` with `207 Multi-Status` results - Locking: exclusive write locks, lock refresh, `If` header token evaluation (including `Not` conditions), and token-gated write preconditions - Tree operations: lock-aware `DELETE` multistatus behavior (`207` with `423`/`424` where applicable) diff --git a/doc/README.md b/doc/README.md index 49c6369..f02feee 100644 --- a/doc/README.md +++ b/doc/README.md @@ -21,7 +21,7 @@ Current WebDAV support targets RFC 4918 Class 1 + Class 2 core behavior without - `PROPFIND` semantics: `allprop`, `propname`, named `prop`, and `200`/`404` `propstat` grouping - `PROPPATCH` semantics: dead-property `set`/`remove` with `207` response model - Lock semantics: token-gated writes with `If`-header parsing, lock refresh, and lock-aware copy/move/delete preconditions -- `PROPFIND` `Depth: infinity` on collections is refused with RFC-shaped `403` DAV precondition (`propfind-finite-depth`) +- `PROPFIND` depth handling supports `Depth: 0`, `Depth: 1`, and recursive `Depth: infinity` on collections Known scope limits: diff --git a/doc/WEBDAV_IMPLEMENTATION.md b/doc/WEBDAV_IMPLEMENTATION.md index a6dd6b1..7e83a6b 100644 --- a/doc/WEBDAV_IMPLEMENTATION.md +++ b/doc/WEBDAV_IMPLEMENTATION.md @@ -68,8 +68,9 @@ IronDrop supports the common PROPFIND modes: and depth handling: -- `Depth: 0` and `Depth: 1` for collections -- `Depth: infinity` on collections refused with RFC-shaped finite-depth precondition response +- `Depth: 0` returns only the target resource +- `Depth: 1` returns the target plus immediate children +- `Depth: infinity` recursively returns all descendants of a collection ```mermaid flowchart TD diff --git a/src/webdav.rs b/src/webdav.rs index 1646c7b..5c1b422 100644 --- a/src/webdav.rs +++ b/src/webdav.rs @@ -116,16 +116,22 @@ fn handle_propfind(request: &Request, base_dir: &Path) -> Result {} + DavDepth::One => { + let mut entries = Vec::new(); + for entry in std::fs::read_dir(&target_path)? { + let entry = entry?; + entries.push(entry.path()); + } + entries.sort_by(|a, b| a.to_string_lossy().cmp(&b.to_string_lossy())); + resources.extend(entries); + } + DavDepth::Infinity => { + collect_recursive_resources(&target_path, &mut resources)?; + } } } @@ -343,24 +349,24 @@ fn parse_prop_name_from_token( }) } -fn propfind_finite_depth_error_response() -> Response { - let mut headers = HashMap::new(); - headers.insert( - "Content-Type".to_string(), - "application/xml; charset=utf-8".to_string(), - ); - Response { - status_code: 403, - status_text: "Forbidden".to_string(), - headers, - body: ResponseBody::Text( - r#" - - -"# - .to_string(), - ), +fn collect_recursive_resources(root: &Path, resources: &mut Vec) -> Result<(), AppError> { + let mut stack = vec![root.to_path_buf()]; + while let Some(current_dir) = stack.pop() { + let mut children = Vec::new(); + for entry in std::fs::read_dir(¤t_dir)? { + let entry = entry?; + children.push((entry.path(), entry.file_type()?)); + } + children.sort_by(|(a, _), (b, _)| a.to_string_lossy().cmp(&b.to_string_lossy())); + + for (entry_path, file_type) in children { + resources.push(entry_path.clone()); + if file_type.is_dir() { + stack.push(entry_path); + } + } } + Ok(()) } fn handle_mkcol(request: &Request, base_dir: &Path) -> Result { @@ -380,6 +386,11 @@ fn handle_mkcol(request: &Request, base_dir: &Path) -> Result Result) -> Result { let depth = headers .get("depth") - .map(|value| value.trim()) - .unwrap_or("infinity"); - match depth { + .map(|value| value.trim().to_ascii_lowercase()) + .unwrap_or_else(|| "infinity".to_string()); + match depth.as_str() { "0" => Ok(CopyDepth::Zero), - "infinity" | "Infinity" | "INFINITY" => Ok(CopyDepth::Infinity), + "infinity" => Ok(CopyDepth::Infinity), _ => Err(AppError::BadRequest), } } @@ -1033,36 +1048,43 @@ fn handle_proppatch(request: &Request, base_dir: &Path) -> Result)> = Vec::new(); + let mut ok_or_dependent_props: Vec<(PropName, Option)> = Vec::new(); let mut missing_props: Vec<(PropName, Option)> = Vec::new(); let mut forbidden_props: Vec<(PropName, Option)> = Vec::new(); + let mut has_failure = false; let mut guard = dead_props_map() .lock() .map_err(|_| AppError::InternalServerError("dead properties map poisoned".to_string()))?; - let props = guard.entry(key).or_default(); + let current_props = guard.get(&key).cloned().unwrap_or_default(); + let mut working_props = current_props.clone(); for operation in operations { for (name, value) in operation.props { if is_protected_property(&name) { forbidden_props.push((name, None)); + has_failure = true; continue; } match operation.action { PropPatchAction::Set => { - props.insert(name.clone(), value.unwrap_or_default()); - ok_props.push((name, None)); + working_props.insert(name.clone(), value.unwrap_or_default()); + ok_or_dependent_props.push((name, None)); } PropPatchAction::Remove => { - if props.remove(&name).is_some() { - ok_props.push((name, None)); + if working_props.remove(&name).is_some() { + ok_or_dependent_props.push((name, None)); } else { missing_props.push((name, None)); + has_failure = true; } } } } } + if !has_failure { + guard.insert(key, working_props); + } drop(guard); let mut xml = String::from( @@ -1075,8 +1097,16 @@ fn handle_proppatch(request: &Request, base_dir: &Path) -> Result"); xml.push_str(&xml_escape(&href)); xml.push_str("\n"); - if !ok_props.is_empty() { - append_propstat(&mut xml, &ok_props, "HTTP/1.1 200 OK"); + if !ok_or_dependent_props.is_empty() { + if has_failure { + append_propstat( + &mut xml, + &ok_or_dependent_props, + "HTTP/1.1 424 Failed Dependency", + ); + } else { + append_propstat(&mut xml, &ok_or_dependent_props, "HTTP/1.1 200 OK"); + } } if !missing_props.is_empty() { append_propstat(&mut xml, &missing_props, "HTTP/1.1 404 Not Found"); @@ -1235,12 +1265,12 @@ fn parse_prop_elements( fn parse_depth_header(headers: &HashMap) -> Result { let depth = headers .get("depth") - .map(|value| value.trim()) - .unwrap_or("infinity"); - match depth { + .map(|value| value.trim().to_ascii_lowercase()) + .unwrap_or_else(|| "infinity".to_string()); + match depth.as_str() { "0" => Ok(DavDepth::Zero), "1" => Ok(DavDepth::One), - "infinity" | "Infinity" | "INFINITY" => Ok(DavDepth::Infinity), + "infinity" => Ok(DavDepth::Infinity), _ => Err(AppError::BadRequest), } } @@ -1533,15 +1563,49 @@ fn parse_lock_depth( ) -> Result { let depth = headers .get("depth") - .map(|value| value.trim()) - .unwrap_or(if is_collection { "infinity" } else { "0" }); - match depth { + .map(|value| value.trim().to_ascii_lowercase()) + .unwrap_or_else(|| { + if is_collection { + "infinity".to_string() + } else { + "0".to_string() + } + }); + match depth.as_str() { "0" => Ok(false), - "infinity" | "Infinity" | "INFINITY" if is_collection => Ok(true), + "infinity" if is_collection => Ok(true), _ => Err(AppError::BadRequest), } } +fn validate_delete_depth_header( + headers: &HashMap, + is_collection: bool, +) -> Result<(), AppError> { + if !is_collection { + return Ok(()); + } + match headers.get("depth") { + None => Ok(()), + Some(value) if value.trim().eq_ignore_ascii_case("infinity") => Ok(()), + Some(_) => Err(AppError::BadRequest), + } +} + +fn validate_move_depth_header( + headers: &HashMap, + is_collection: bool, +) -> Result<(), AppError> { + if !is_collection { + return Ok(()); + } + match headers.get("depth") { + None => Ok(()), + Some(value) if value.trim().eq_ignore_ascii_case("infinity") => Ok(()), + Some(_) => Err(AppError::BadRequest), + } +} + fn normalize_lock_token(value: &str) -> Option { let trimmed = value.trim(); if trimmed.is_empty() { diff --git a/tests/webdav_copy_move_rfc_test.rs b/tests/webdav_copy_move_rfc_test.rs index f4dacb6..1ed7d9a 100644 --- a/tests/webdav_copy_move_rfc_test.rs +++ b/tests/webdav_copy_move_rfc_test.rs @@ -172,6 +172,32 @@ fn test_copy_depth_zero_on_collection_does_not_copy_members() { assert_eq!(nested_resp.status(), StatusCode::NOT_FOUND); } +#[test] +fn test_copy_collection_without_depth_defaults_to_infinity() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("src").join("nested")).unwrap(); + let mut child = File::create(root.join("src").join("nested").join("child.txt")).unwrap(); + writeln!(child, "child").unwrap(); + }); + let client = Client::new(); + + let copy_resp = client + .request( + Method::from_bytes(b"COPY").unwrap(), + format!("http://{}/src/", server.addr), + ) + .header("Destination", format!("http://{}/dst/", server.addr)) + .send() + .unwrap(); + assert_eq!(copy_resp.status(), StatusCode::CREATED); + + let child = client + .get(format!("http://{}/dst/nested/child.txt", server.addr)) + .send() + .unwrap(); + assert_eq!(child.status(), StatusCode::OK); +} + #[test] fn test_copy_destination_descendant_of_source_is_bad_request() { let server = setup_test_server_with_tree(|root| { @@ -278,6 +304,129 @@ fn test_copy_invalid_depth_is_bad_request() { assert_eq!(copy_resp.status(), StatusCode::BAD_REQUEST); } +#[test] +fn test_copy_depth_infinity_is_case_insensitive() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("src").join("nested")).unwrap(); + let mut child = File::create(root.join("src").join("nested").join("child.txt")).unwrap(); + writeln!(child, "child").unwrap(); + }); + let client = Client::new(); + + let copy_resp = client + .request( + Method::from_bytes(b"COPY").unwrap(), + format!("http://{}/src/", server.addr), + ) + .header("Depth", "InFiNiTy") + .header("Destination", format!("http://{}/dst/", server.addr)) + .send() + .unwrap(); + assert_eq!(copy_resp.status(), StatusCode::CREATED); + + let nested_resp = client + .get(format!("http://{}/dst/nested/child.txt", server.addr)) + .send() + .unwrap(); + assert_eq!(nested_resp.status(), StatusCode::OK); +} + +#[test] +fn test_delete_collection_rejects_non_infinity_depth_header() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("dir").join("nested")).unwrap(); + }); + let client = Client::new(); + + let delete_resp = client + .request(Method::DELETE, format!("http://{}/dir/", server.addr)) + .header("Depth", "0") + .send() + .unwrap(); + assert_eq!(delete_resp.status(), StatusCode::BAD_REQUEST); +} + +#[test] +fn test_move_collection_rejects_non_infinity_depth_header() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("src").join("nested")).unwrap(); + }); + let client = Client::new(); + + let move_resp = client + .request( + Method::from_bytes(b"MOVE").unwrap(), + format!("http://{}/src/", server.addr), + ) + .header("Depth", "0") + .header("Destination", format!("http://{}/dst/", server.addr)) + .send() + .unwrap(); + assert_eq!(move_resp.status(), StatusCode::BAD_REQUEST); +} + +#[test] +fn test_move_collection_accepts_case_insensitive_infinity_depth() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("src").join("nested")).unwrap(); + let mut child = File::create(root.join("src").join("nested").join("child.txt")).unwrap(); + writeln!(child, "child").unwrap(); + }); + let client = Client::new(); + + let move_resp = client + .request( + Method::from_bytes(b"MOVE").unwrap(), + format!("http://{}/src/", server.addr), + ) + .header("Depth", "InFiNiTy") + .header("Destination", format!("http://{}/dst/", server.addr)) + .send() + .unwrap(); + assert!( + move_resp.status() == StatusCode::CREATED || move_resp.status() == StatusCode::NO_CONTENT + ); + + let check = client + .get(format!("http://{}/dst/nested/child.txt", server.addr)) + .send() + .unwrap(); + assert_eq!(check.status(), StatusCode::OK); +} + +#[test] +fn test_move_collection_without_depth_defaults_to_infinity() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("src").join("nested")).unwrap(); + let mut child = File::create(root.join("src").join("nested").join("child.txt")).unwrap(); + writeln!(child, "child").unwrap(); + }); + let client = Client::new(); + + let move_resp = client + .request( + Method::from_bytes(b"MOVE").unwrap(), + format!("http://{}/src/", server.addr), + ) + .header("Destination", format!("http://{}/dst/", server.addr)) + .send() + .unwrap(); + assert!( + move_resp.status() == StatusCode::CREATED || move_resp.status() == StatusCode::NO_CONTENT + ); + + let moved = client + .get(format!("http://{}/dst/nested/child.txt", server.addr)) + .send() + .unwrap(); + assert_eq!(moved.status(), StatusCode::OK); + let old = client + .get(format!("http://{}/src/nested/child.txt", server.addr)) + .send() + .unwrap(); + assert_eq!(old.status(), StatusCode::NOT_FOUND); +} + #[test] fn test_delete_collection_with_multiple_tokens_succeeds() { let server = setup_test_server_with_tree(|root| { diff --git a/tests/webdav_lock_if_rfc_test.rs b/tests/webdav_lock_if_rfc_test.rs index 3042cc0..5968fd9 100644 --- a/tests/webdav_lock_if_rfc_test.rs +++ b/tests/webdav_lock_if_rfc_test.rs @@ -220,6 +220,32 @@ fn test_file_lock_rejects_depth_infinity() { assert_eq!(response.status(), StatusCode::BAD_REQUEST); } +#[test] +fn test_collection_lock_accepts_case_insensitive_depth_infinity() { + let server = setup_test_server_with_tree(|root| { + std::fs::create_dir_all(root.join("dir")).unwrap(); + }); + let client = Client::new(); + + let response = client + .request( + Method::from_bytes(b"LOCK").unwrap(), + format!("http://{}/dir/", server.addr), + ) + .header("Depth", "InFiNiTy") + .body( + r#" + + + +"#, + ) + .send() + .unwrap(); + + assert!(response.status() == StatusCode::CREATED || response.status() == StatusCode::OK); +} + #[test] fn test_if_not_condition_does_not_satisfy_lock_requirement() { let server = setup_test_server_with_tree(|root| { @@ -370,6 +396,72 @@ fn test_unlock_with_wrong_token_is_conflict() { assert_eq!(unlock.status(), StatusCode::CONFLICT); } +#[test] +fn test_unlock_missing_lock_token_is_bad_request() { + let server = setup_test_server_with_tree(|root| { + let mut file = File::create(root.join("doc.txt")).unwrap(); + writeln!(file, "hello").unwrap(); + }); + let client = Client::new(); + let _token = acquire_lock_token(&client, server.addr, "/doc.txt"); + + let unlock = client + .request( + Method::from_bytes(b"UNLOCK").unwrap(), + format!("http://{}/doc.txt", server.addr), + ) + .send() + .unwrap(); + assert_eq!(unlock.status(), StatusCode::BAD_REQUEST); +} + +#[test] +fn test_unlock_without_active_lock_is_conflict() { + let server = setup_test_server_with_tree(|root| { + let mut file = File::create(root.join("doc.txt")).unwrap(); + writeln!(file, "hello").unwrap(); + }); + let client = Client::new(); + + let unlock = client + .request( + Method::from_bytes(b"UNLOCK").unwrap(), + format!("http://{}/doc.txt", server.addr), + ) + .header("Lock-Token", "") + .send() + .unwrap(); + assert_eq!(unlock.status(), StatusCode::CONFLICT); +} + +#[test] +fn test_lock_on_unmapped_url_creates_resource() { + let server = setup_test_server_with_tree(|_root| {}); + let client = Client::new(); + + let lock = client + .request( + Method::from_bytes(b"LOCK").unwrap(), + format!("http://{}/new-doc.txt", server.addr), + ) + .body( + r#" + + + +"#, + ) + .send() + .unwrap(); + assert_eq!(lock.status(), StatusCode::CREATED); + + let get = client + .get(format!("http://{}/new-doc.txt", server.addr)) + .send() + .unwrap(); + assert_eq!(get.status(), StatusCode::OK); +} + #[test] fn test_delete_with_lock_token_clears_lock_state() { let server = setup_test_server_with_tree(|root| { diff --git a/tests/webdav_options_propfind_test.rs b/tests/webdav_options_propfind_test.rs index d2e5769..ecadad5 100644 --- a/tests/webdav_options_propfind_test.rs +++ b/tests/webdav_options_propfind_test.rs @@ -154,14 +154,33 @@ fn test_options_advertises_webdav_v1_capability() { assert_eq!(response.status(), StatusCode::OK); let dav = response.headers().get("dav").unwrap().to_str().unwrap(); assert!(dav.contains('1')); + assert!(dav.contains('2')); + + let ms_author_via = response + .headers() + .get("ms-author-via") + .unwrap() + .to_str() + .unwrap(); + assert_eq!(ms_author_via, "DAV"); let allow = response.headers().get("allow").unwrap().to_str().unwrap(); - assert!(allow.contains("PROPFIND")); - assert!(allow.contains("MKCOL")); - assert!(allow.contains("PUT")); - assert!(allow.contains("DELETE")); - assert!(allow.contains("COPY")); - assert!(allow.contains("MOVE")); + for method in [ + "OPTIONS", + "GET", + "HEAD", + "PROPFIND", + "PROPPATCH", + "MKCOL", + "PUT", + "DELETE", + "COPY", + "MOVE", + "LOCK", + "UNLOCK", + ] { + assert!(allow.contains(method), "Allow header missing {method}"); + } } #[test] @@ -264,8 +283,12 @@ fn test_propfind_depth_one_on_collection_includes_children() { } #[test] -fn test_propfind_infinite_depth_rejected_with_finite_depth_precondition() { - let server = setup_test_server_with_tree(|_| {}); +fn test_propfind_infinite_depth_includes_recursive_descendants() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("dir").join("nested").join("deeper")).unwrap(); + let mut f = File::create(root.join("dir").join("nested").join("item.txt")).unwrap(); + writeln!(f, "child").unwrap(); + }); let client = Client::new(); let response = client @@ -279,9 +302,31 @@ fn test_propfind_infinite_depth_rejected_with_finite_depth_precondition() { .send() .unwrap(); - assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(response.status().as_u16(), 207); let body = response.text().unwrap(); - assert!(body.contains("propfind-finite-depth")); + assert!(body.contains("/dir/")); + assert!(body.contains("/dir/nested/")); + assert!(body.contains("/dir/nested/item.txt")); + assert!(body.contains("/dir/nested/deeper/")); +} + +#[test] +fn test_propfind_invalid_depth_is_bad_request() { + let server = setup_test_server_with_tree(|_| {}); + let client = Client::new(); + + let response = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!("http://{}/test.txt", server.addr), + ) + .header("Depth", "2") + .header("Content-Type", "application/xml") + .body(r#""#) + .send() + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); } #[test] diff --git a/tests/webdav_propfind_rfc_test.rs b/tests/webdav_propfind_rfc_test.rs index ce9de8b..56be92b 100644 --- a/tests/webdav_propfind_rfc_test.rs +++ b/tests/webdav_propfind_rfc_test.rs @@ -3,7 +3,6 @@ use irondrop::cli::Cli; use irondrop::server::run_server; use reqwest::Method; -use reqwest::StatusCode; use reqwest::blocking::Client; use std::fs::{File, create_dir_all}; use std::io::Write; @@ -138,9 +137,10 @@ fn test_propfind_named_unknown_property_returns_404_propstat() { } #[test] -fn test_propfind_depth_infinity_on_collection_is_finite_depth_error() { +fn test_propfind_depth_infinity_on_collection_includes_recursive_descendants() { let server = setup_test_server_with_tree(|root| { create_dir_all(root.join("dir").join("nested")).unwrap(); + create_dir_all(root.join("dir").join("nested").join("leaf")).unwrap(); let mut file = File::create(root.join("dir").join("nested").join("x.txt")).unwrap(); writeln!(file, "x").unwrap(); }); @@ -159,10 +159,69 @@ fn test_propfind_depth_infinity_on_collection_is_finite_depth_error() { .send() .unwrap(); - // RFC-compliant finite-depth refusal should be 403 with DAV precondition body. - assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(response.status().as_u16(), 207); + let xml = response.text().unwrap(); + assert!(xml.contains("/dir/")); + assert!(xml.contains("/dir/nested/")); + assert!(xml.contains("/dir/nested/x.txt")); + assert!(xml.contains("/dir/nested/leaf/")); +} + +#[test] +fn test_propfind_missing_depth_defaults_to_infinity_on_collection() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("dir").join("nested")).unwrap(); + let mut file = File::create(root.join("dir").join("nested").join("x.txt")).unwrap(); + writeln!(file, "x").unwrap(); + }); + let client = Client::new(); + let body = r#" +"#; + + let response = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!("http://{}/dir/", server.addr), + ) + .header("Content-Type", "application/xml") + .body(body.to_string()) + .send() + .unwrap(); + + assert_eq!(response.status().as_u16(), 207); + let xml = response.text().unwrap(); + assert!(xml.contains("/dir/")); + assert!(xml.contains("/dir/nested/")); + assert!(xml.contains("/dir/nested/x.txt")); +} + +#[test] +fn test_propfind_depth_header_is_case_insensitive_for_infinity() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("dir").join("nested")).unwrap(); + let mut file = File::create(root.join("dir").join("nested").join("x.txt")).unwrap(); + writeln!(file, "x").unwrap(); + }); + let client = Client::new(); + let body = r#" +"#; + + let response = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!("http://{}/dir/", server.addr), + ) + .header("Depth", "InFiNiTy") + .header("Content-Type", "application/xml") + .body(body.to_string()) + .send() + .unwrap(); + + assert_eq!(response.status().as_u16(), 207); let xml = response.text().unwrap(); - assert!(xml.contains("propfind-finite-depth")); + assert!(xml.contains("/dir/")); + assert!(xml.contains("/dir/nested/")); + assert!(xml.contains("/dir/nested/x.txt")); } #[test] diff --git a/tests/webdav_proppatch_rfc_test.rs b/tests/webdav_proppatch_rfc_test.rs index 5af9ef1..9a485ef 100644 --- a/tests/webdav_proppatch_rfc_test.rs +++ b/tests/webdav_proppatch_rfc_test.rs @@ -311,6 +311,58 @@ fn test_proppatch_protected_live_property_returns_403_propstat() { assert!(xml.contains("HTTP/1.1 403 Forbidden")); } +#[test] +fn test_proppatch_is_atomic_when_any_instruction_fails() { + let server = setup_test_server_with_tree(|root| { + let mut file = File::create(root.join("sample.txt")).unwrap(); + writeln!(file, "hello").unwrap(); + }); + let client = Client::new(); + + let body = r#" + + + + blue + "override" + + +"#; + let resp = client + .request( + Method::from_bytes(b"PROPPATCH").unwrap(), + format!("http://{}/sample.txt", server.addr), + ) + .header("Content-Type", "application/xml") + .body(body.to_string()) + .send() + .unwrap(); + assert_eq!(resp.status().as_u16(), 207); + let xml = resp.text().unwrap(); + assert!(xml.contains("HTTP/1.1 403 Forbidden")); + assert!(xml.contains("HTTP/1.1 424 Failed Dependency")); + + let propfind_body = r#" + + + + +"#; + let read_resp = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!("http://{}/sample.txt", server.addr), + ) + .header("Depth", "0") + .header("Content-Type", "application/xml") + .body(propfind_body.to_string()) + .send() + .unwrap(); + assert_eq!(read_resp.status().as_u16(), 207); + let read_xml = read_resp.text().unwrap(); + assert!(read_xml.contains("HTTP/1.1 404 Not Found")); +} + #[test] fn test_proppatch_namespace_distinct_properties_do_not_collide() { let server = setup_test_server_with_tree(|root| { diff --git a/tests/webdav_write_methods_test.rs b/tests/webdav_write_methods_test.rs index 02e9b28..7cea097 100644 --- a/tests/webdav_write_methods_test.rs +++ b/tests/webdav_write_methods_test.rs @@ -111,6 +111,44 @@ fn test_mkcol_parent_missing_conflict() { assert_eq!(response.status(), StatusCode::CONFLICT); } +#[test] +fn test_mkcol_existing_resource_is_method_not_allowed() { + let server = setup_test_server_with_tree(|root| { + let mut file = File::create(root.join("exists.txt")).unwrap(); + writeln!(file, "x").unwrap(); + }); + let client = Client::new(); + + let response = client + .request( + Method::from_bytes(b"MKCOL").unwrap(), + format!("http://{}/exists.txt", server.addr), + ) + .send() + .unwrap(); + + assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); +} + +#[test] +fn test_mkcol_with_request_body_is_unsupported_media_type() { + let server = setup_test_server_with_tree(|_| {}); + let client = Client::new(); + + let response = client + .request( + Method::from_bytes(b"MKCOL").unwrap(), + format!("http://{}/newdir/", server.addr), + ) + .header("Content-Type", "application/xml") + .body("".to_string()) + .send() + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE); + assert!(!server.root.join("newdir").exists()); +} + #[test] fn test_put_creates_and_updates_file() { let server = setup_test_server_with_tree(|_| {}); @@ -139,6 +177,39 @@ fn test_put_creates_and_updates_file() { ); } +#[test] +fn test_put_missing_parent_conflict() { + let server = setup_test_server_with_tree(|_root| {}); + let client = Client::new(); + + let response = client + .request( + Method::from_bytes(b"PUT").unwrap(), + format!("http://{}/missing/path/note.txt", server.addr), + ) + .body("body".to_string()) + .send() + .unwrap(); + + assert_eq!(response.status(), StatusCode::CONFLICT); +} + +#[test] +fn test_put_on_existing_collection_is_method_not_allowed() { + let server = setup_test_server_with_tree(|root| { + create_dir_all(root.join("docs")).unwrap(); + }); + let client = Client::new(); + + let response = client + .request(Method::PUT, format!("http://{}/docs/", server.addr)) + .body("not allowed".to_string()) + .send() + .unwrap(); + + assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); +} + #[test] fn test_delete_removes_file_and_collection() { let server = setup_test_server_with_tree(|root| { @@ -172,6 +243,19 @@ fn test_delete_removes_file_and_collection() { assert!(!server.root.join("to-delete-dir").exists()); } +#[test] +fn test_delete_base_forbidden() { + let server = setup_test_server_with_tree(|_root| {}); + let client = Client::new(); + + let response = client + .request(Method::DELETE, format!("http://{}/", server.addr)) + .send() + .unwrap(); + + assert_eq!(response.status(), StatusCode::FORBIDDEN); +} + #[test] fn test_delete_missing_resource_not_found() { let server = setup_test_server_with_tree(|_| {}); From 9437d986e598e33e501131b631e05266d4019d57 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Tue, 28 Apr 2026 01:01:02 +0530 Subject: [PATCH 13/17] irondrop: better behaviour behind reverse proxies --- README.md | 22 +- src/cli.rs | 30 + src/config/mod.rs | 30 + src/handlers.rs | 3 +- src/http.rs | 41 +- src/server.rs | 8 + src/templates.rs | 125 +++- src/upload.rs | 1 + src/webdav.rs | 24 +- templates/common/base.html | 13 +- templates/directory/script.js | 5 +- templates/monitor/script.js | 5 +- templates/upload/script.js | 5 +- tests/async_runtime_starvation_test.rs | 1 + tests/base_path_test.rs | 788 +++++++++++++++++++++++++ tests/config_test.rs | 11 + tests/direct_upload_test.rs | 1 + tests/integration_test.rs | 2 + tests/log_dir_test.rs | 1 + tests/monitor_test.rs | 4 + tests/ssl_test.rs | 6 + tests/webdav_copy_move_rfc_test.rs | 1 + tests/webdav_copy_move_test.rs | 1 + tests/webdav_error_semantics_test.rs | 1 + tests/webdav_error_xml_rfc_test.rs | 1 + tests/webdav_lock_if_rfc_test.rs | 1 + tests/webdav_options_propfind_test.rs | 2 + tests/webdav_propfind_rfc_test.rs | 1 + tests/webdav_proppatch_rfc_test.rs | 1 + tests/webdav_write_methods_test.rs | 1 + 30 files changed, 1077 insertions(+), 59 deletions(-) create mode 100644 tests/base_path_test.rs diff --git a/README.md b/README.md index 734a449..5d72074 100644 --- a/README.md +++ b/README.md @@ -165,16 +165,25 @@ location / { ``` **Subpath Configuration (e.g., `/webstorage/`):** + +Start IronDrop with the `--base-path` flag: +```bash +irondrop -d ./files --base-path /webstorage --listen 0.0.0.0 +``` + +Then configure Nginx to forward the full path (no stripping needed): ```nginx location /webstorage/ { - proxy_pass http://127.0.0.1:8080/; - proxy_redirect / /webstorage/; - sub_filter 'href="/"' 'href="/webstorage/"'; - sub_filter_once off; - # ... see deployment guide for full sub_filter list + proxy_pass http://127.0.0.1:8080/webstorage/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + client_max_body_size 0; + proxy_buffering off; } ``` +> **Note:** No `sub_filter` hacks or extra `/_irondrop/` location blocks are needed. The `--base-path` flag makes IronDrop fully reverse-proxy aware — all generated URLs (HTML links, JavaScript API calls, WebDAV hrefs, HTTP redirects) are automatically prefixed with the configured base path. + See the [Deployment Guide](./doc/DEPLOYMENT.md#nginx-reverse-proxy-deployment) for full configuration examples and optimization settings. ### 🛠️ Configuration Options @@ -183,13 +192,14 @@ See the [Deployment Guide](./doc/DEPLOYMENT.md#nginx-reverse-proxy-deployment) f IronDrop offers extensive customization through command-line arguments: | Option | Description | Example | -|--------|-------------|----------| +|--------|-------------|---------| | `-d, --directory` | **Required** - Directory to serve | `-d /home/user/files` | | `-l, --listen` | Listen address (default: 127.0.0.1) | `-l 0.0.0.0` | | `-p, --port` | Port number (default: 8080) | `-p 3000` | | `--enable-upload` | Enable file uploads | `--enable-upload true` | | `--enable-webdav` | Enable WebDAV methods (`OPTIONS`, `PROPFIND`, `PROPPATCH`, `MKCOL`, `PUT`, `DELETE`, `COPY`, `MOVE`, `LOCK`, `UNLOCK`) | `--enable-webdav true` | | `--disable-rate-limit` | Disable rate limiting **only when WebDAV is enabled** | `--enable-webdav true --disable-rate-limit true` | +| `--base-path` | Base URL path prefix for reverse proxy sub-path deployments | `--base-path /webstorage` | | `--username/--password` | Basic authentication | `--username admin --password secret` | | `-a, --allowed-extensions` | Restrict file types | `-a "*.pdf,*.doc,*.zip"` | | `-t, --threads` | Tokio runtime worker threads (default: 8) | `-t 16` | diff --git a/src/cli.rs b/src/cli.rs index 1916a71..b607631 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -87,6 +87,11 @@ pub struct Cli { /// Path to SSL/TLS private key file (PEM format) for HTTPS support #[arg(long, value_parser = validate_ssl_file)] pub ssl_key: Option, + + /// Base URL path prefix for reverse proxy sub-path deployments (e.g., "/webstorage"). + /// When set, all generated URLs are prefixed and incoming requests must start with this path. + #[arg(long, value_parser = validate_base_path)] + pub base_path: Option, } /// Validate upload size (minimum 1 MB, no upper limit for direct streaming) @@ -231,6 +236,7 @@ mod tests { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; // Test conversion @@ -270,6 +276,7 @@ mod tests { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; assert!(cli.validate().is_ok()); @@ -346,3 +353,26 @@ fn validate_ssl_file(s: &str) -> Result { Err(e) => Err(format!("Cannot read SSL file {}: {}", path.display(), e)), } } + +/// Validate and normalize base path for reverse proxy sub-path deployments. +/// Ensures the value starts with `/` and does not end with `/`. +fn validate_base_path(s: &str) -> Result { + let trimmed = s.trim(); + if trimmed.is_empty() { + return Err("Base path cannot be empty".to_string()); + } + // Ensure leading slash + let with_slash = if trimmed.starts_with('/') { + trimmed.to_string() + } else { + format!("/{trimmed}") + }; + // Strip trailing slash(es) + let normalized = with_slash.trim_end_matches('/').to_string(); + if normalized.is_empty() { + return Err( + "Base path cannot be just '/' — omit --base-path to serve from root".to_string(), + ); + } + Ok(normalized) +} diff --git a/src/config/mod.rs b/src/config/mod.rs index fe5e302..3b329a8 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -37,6 +37,9 @@ pub struct Config { // SSL settings pub ssl_cert: Option, pub ssl_key: Option, + + // Reverse proxy settings + pub base_path: String, } impl Config { @@ -87,6 +90,7 @@ impl Config { log_dir: Self::get_log_dir(&ini, cli), ssl_cert: Self::get_ssl_cert(&ini, cli), ssl_key: Self::get_ssl_key(&ini, cli), + base_path: Self::get_base_path(&ini, cli), }; log::debug!("Configuration loading completed successfully"); @@ -364,6 +368,28 @@ impl Config { ini.get_string("ssl", "key").map(PathBuf::from) } + fn get_base_path(ini: &IniConfig, cli: &Cli) -> String { + // CLI argument takes precedence + if let Some(ref base_path) = cli.base_path { + return base_path.clone(); + } + // INI file + if let Some(bp) = ini.get_string("server", "base_path") { + let trimmed = bp.trim(); + if !trimmed.is_empty() { + // Normalize: ensure leading slash, strip trailing slash + let with_slash = if trimmed.starts_with('/') { + trimmed.to_string() + } else { + format!("/{trimmed}") + }; + return with_slash.trim_end_matches('/').to_string(); + } + } + // Default: empty string (app served from root) + String::new() + } + /// Print configuration summary pub fn print_summary(&self) { log::info!("Configuration Summary:"); @@ -405,6 +431,9 @@ impl Config { } else { log::info!(" SSL/TLS: Disabled (HTTP only)"); } + if !self.base_path.is_empty() { + log::info!(" Base Path: {}", self.base_path); + } } } @@ -434,6 +463,7 @@ mod tests { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, } } diff --git a/src/handlers.rs b/src/handlers.rs index f4f90e9..1c8500f 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -546,7 +546,7 @@ pub fn handle_file_request( // Redirect directory paths without trailing slash to canonical URL with slash if !request.path.ends_with('/') { let mut headers = HashMap::new(); - let canonical = format!("{}/", request.path); + let canonical = crate::templates::prefixed(&format!("{}/", request.path)); headers.insert("Location".to_string(), canonical); return Ok(Response { status_code: 301, @@ -588,6 +588,7 @@ pub fn handle_file_request( log_dir: cli.log_dir.clone(), ssl_cert: cli.ssl_cert.clone(), ssl_key: cli.ssl_key.clone(), + base_path: cli.base_path.clone().unwrap_or_default(), }); let html_content = generate_directory_listing(&full_path, &request.path, config.as_ref())?; diff --git a/src/http.rs b/src/http.rs index 0b6b28f..c243e26 100644 --- a/src/http.rs +++ b/src/http.rs @@ -210,10 +210,9 @@ impl Request { // Static asset, favicon, upload, and health handlers moved to handlers.rs -/// Determines the correct response based on the request. #[allow(clippy::too_many_arguments)] fn route_request( - request: &Request, + request: &mut Request, base_dir: &Arc, allowed_extensions: &Arc>, _username: &Arc>, @@ -223,6 +222,30 @@ fn route_request( _stats: Option<&crate::server::ServerStats>, router: &Arc, ) -> Result { + // Strip the base path prefix from the request path for reverse proxy support. + // If a base_path is configured and the request doesn't match, return 404. + let bp = crate::templates::base_path(); + if !bp.is_empty() { + let path_only = request + .path + .split('?') + .next() + .unwrap_or(&request.path) + .to_string(); + if path_only == bp || path_only.starts_with(&format!("{bp}/")) { + // Strip the base path prefix from the full path (including query string) + let stripped = request.path[bp.len()..].to_string(); + request.path = if stripped.is_empty() || !stripped.starts_with('/') { + format!("/{stripped}") + } else { + stripped + }; + } else { + // Request path doesn't match the configured base path — 404 + return Err(AppError::NotFound); + } + } + trace!("Routing {} {} through router", request.method, request.path); // Authentication is now handled by middleware in the router // First check if router handles the request (internal routes) @@ -270,7 +293,7 @@ pub async fn handle_client_async( { let log_prefix = format!("[{}]", peer_addr); - let request = match Request::from_async_stream(&mut stream).await { + let mut request = match Request::from_async_stream(&mut stream).await { Ok(req) => req, Err(e) => { send_error_response_async(&mut stream, e, &log_prefix).await; @@ -299,7 +322,7 @@ pub async fn handle_client_async( move || { std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { route_request( - &request, + &mut request, &base_dir, &allowed_extensions, &username, @@ -371,7 +394,15 @@ fn log_request_line(log_prefix: &str, method: &str, path: &str, status_code: u16 } fn is_monitor_path(path: &str) -> bool { - path == "/monitor" || path.starts_with("/_irondrop/monitor") + let bp = crate::templates::base_path(); + if bp.is_empty() { + path == "/monitor" || path.starts_with("/_irondrop/monitor") + } else { + path == format!("{bp}/monitor") + || path.starts_with(&format!("{bp}/_irondrop/monitor")) + || path == "/monitor" + || path.starts_with("/_irondrop/monitor") + } } fn monitor_request_log_rate_limited(method: &str, status_code: u16) { diff --git a/src/server.rs b/src/server.rs index f6b1bd7..cad83b9 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1112,6 +1112,11 @@ pub fn run_server_with_config(config: Config) -> Result<(), AppError> { log_dir: config.log_dir, ssl_cert: config.ssl_cert, ssl_key: config.ssl_key, + base_path: if config.base_path.is_empty() { + None + } else { + Some(config.base_path) + }, }; run_server(cli, None, None) @@ -1225,6 +1230,9 @@ async fn run_server_async( let chunk_size = cli.chunk_size.unwrap_or(1024); let cli_arc = Arc::new(cli); + // Initialize the global base path for reverse proxy sub-path support + crate::templates::init_base_path(cli_arc.base_path.clone().unwrap_or_default()); + let mut router = Router::new(); if cli_arc.username.is_some() && cli_arc.password.is_some() { crate::templates::AUTH_ENABLED.store(true, std::sync::atomic::Ordering::SeqCst); diff --git a/src/templates.rs b/src/templates.rs index 924f625..a90a08c 100644 --- a/src/templates.rs +++ b/src/templates.rs @@ -10,6 +10,30 @@ use std::sync::atomic::AtomicBool; pub static AUTH_ENABLED: AtomicBool = AtomicBool::new(false); +/// Global base path prefix for reverse proxy sub-path deployments. +/// Empty string when serving from root, e.g. "/webstorage" when behind a proxy. +static BASE_PATH: OnceLock = OnceLock::new(); + +/// Initialize the global base path. Call once during server startup. +pub fn init_base_path(path: String) { + let _ = BASE_PATH.set(path); +} + +/// Get the configured base path prefix (empty string if not set). +pub fn base_path() -> &'static str { + BASE_PATH.get().map(|s| s.as_str()).unwrap_or("") +} + +/// Prepend the base path to an absolute path (e.g. "/_irondrop/foo" -> "/webstorage/_irondrop/foo"). +pub fn prefixed(path: &str) -> String { + let bp = base_path(); + if bp.is_empty() { + path.to_string() + } else { + format!("{bp}{path}") + } +} + // Embed templates at compile time // Base template const BASE_HTML: &str = include_str!("../templates/common/base.html"); @@ -160,16 +184,19 @@ impl TemplateEngine { let content = self.render(content_template, variables)?; // Create variables for the base template + let bp = base_path(); let mut base_variables = variables.clone(); + base_variables.insert("BASE_PATH".to_string(), bp.to_string()); base_variables.insert("PAGE_TITLE".to_string(), page_title.to_string()); base_variables.insert("PAGE_STYLES".to_string(), page_styles.to_string()); base_variables.insert("PAGE_SCRIPTS".to_string(), page_scripts.to_string()); let auth_enabled = AUTH_ENABLED.load(std::sync::atomic::Ordering::SeqCst); + let logout_href = prefixed("/_irondrop/logout"); let full_header_actions = if auth_enabled && content_template != "logout_content" { format!( r#"{} - + @@ -206,9 +233,14 @@ impl TemplateEngine { .trim_end_matches('/') .to_string() }; - let page_styles = - r#""#; - let page_scripts = r#""#; + let page_styles = format!( + r#""#, + base_path() + ); + let page_scripts = format!( + r#""#, + base_path() + ); // Build header actions based on upload status let header_actions = if variables @@ -221,14 +253,15 @@ impl TemplateEngine { .unwrap_or(&String::new()) .clone(); format!( - r#" + r#" Upload Files - "# + "#, + base_path() ) } else { String::new() @@ -248,8 +281,8 @@ impl TemplateEngine { self.render_page( "directory_content", &page_title, - page_styles, - page_scripts, + &page_styles, + &page_scripts, &header_actions, &enhanced_variables, ) @@ -337,15 +370,21 @@ impl TemplateEngine { variables.insert("VERSION".to_string(), crate::VERSION.to_string()); let page_title = format!("{error_code} {error_message}"); - let page_styles = r#""#; - let page_scripts = r#""#; + let page_styles = format!( + r#""#, + base_path() + ); + let page_scripts = format!( + r#""#, + base_path() + ); let header_actions = ""; // No actions on error page self.render_page( "error_content", &page_title, - page_styles, - page_scripts, + &page_styles, + &page_scripts, header_actions, &variables, ) @@ -357,12 +396,19 @@ impl TemplateEngine { variables.insert("PATH".to_string(), path.to_string()); let page_title = format!("Upload to {path}"); - let page_styles = r#""#; - let page_scripts = r#""#; + let page_styles = format!( + r#""#, + base_path() + ); + let page_scripts = format!( + r#""#, + base_path() + ); // Header action is back to directory + let back_href = prefixed(path); let header_actions = format!( - r#" + r#" @@ -374,8 +420,8 @@ impl TemplateEngine { self.render_page( "upload_content", &page_title, - page_styles, - page_scripts, + &page_styles, + &page_scripts, &header_actions, &variables, ) @@ -545,13 +591,15 @@ impl TemplateEngine { Self::get_file_icon(name) }; - // Build absolute href using CURRENT_PATH to avoid base-URL ambiguity + // Build absolute href using CURRENT_PATH, prefixed with base_path + let bp = base_path(); let base_clean = current_path.trim_end_matches('/'); let href = if base_clean.is_empty() { - format!("/{}", percent_encode(name)) + format!("{}/{}", bp, percent_encode(name)) } else { format!( - "/{}/{}", + "{}/{}/{}", + bp, base_clean.trim_start_matches('/'), percent_encode(name) ) @@ -615,21 +663,29 @@ impl TemplateEngine { pub fn render_monitor_page(&self) -> Result { debug!("Rendering monitor page"); let page_title = "Monitor"; - let page_styles = r#""#; - let page_scripts = r#" + let page_styles = format!( + r#""#, + base_path() + ); + let page_scripts = format!( + r#" - - "#; - let header_actions = r#"← Back to Files"#; + + "#, + base_path() + ); + let back_href = prefixed("/"); + let header_actions = + format!(r#"← Back to Files"#); let variables = HashMap::new(); self.render_page( "monitor_content", page_title, - page_styles, - page_scripts, - header_actions, + &page_styles, + &page_scripts, + &header_actions, &variables, ) } @@ -649,9 +705,14 @@ impl TemplateEngine { warnings: &str, ) -> Result { let page_title = "Upload Successful"; - let page_styles = r#""#; + let page_styles = format!( + r#""#, + base_path() + ); let page_scripts = ""; - let header_actions = r#"← Back to Files"#; + let back_href = prefixed("/"); + let header_actions = + format!(r#"← Back to Files"#); let mut variables = HashMap::new(); variables.insert("FILE_COUNT".to_string(), file_count.to_string()); @@ -663,9 +724,9 @@ impl TemplateEngine { self.render_page( "upload_success", page_title, - page_styles, + &page_styles, page_scripts, - header_actions, + &header_actions, &variables, ) } diff --git a/src/upload.rs b/src/upload.rs index 0aae8cf..1d1b554 100644 --- a/src/upload.rs +++ b/src/upload.rs @@ -1019,6 +1019,7 @@ mod tests { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, } } diff --git a/src/webdav.rs b/src/webdav.rs index 5c1b422..70688c7 100644 --- a/src/webdav.rs +++ b/src/webdav.rs @@ -839,7 +839,23 @@ fn extract_destination_path( { return Err(AppError::BadRequest); } - return decode_percent_path(&rest[path_start..]); + let raw_path = decode_percent_path(&rest[path_start..])?; + + // Strip the base_path prefix if configured + let bp = crate::templates::base_path(); + if !bp.is_empty() { + if let Some(stripped) = raw_path.strip_prefix(bp) { + let normalized = if stripped.is_empty() || !stripped.starts_with('/') { + format!("/{stripped}") + } else { + stripped.to_string() + }; + return Ok(normalized); + } + // Destination doesn't match base_path — treat as bad request + return Err(AppError::BadRequest); + } + return Ok(raw_path); } Err(AppError::BadRequest) @@ -1959,12 +1975,14 @@ fn etag_for_resource(path: &Path, metadata: &std::fs::Metadata) -> String { } fn build_href(base_dir: &Path, resource: &Path, is_dir: bool) -> String { + let bp = crate::templates::base_path(); + if resource == base_dir { - return "/".to_string(); + return format!("{}/", bp); } let relative = resource.strip_prefix(base_dir).unwrap_or(resource); - let mut href = String::from("/"); + let mut href = format!("{}/", bp); let mut first = true; for component in relative.components() { if let Component::Normal(segment) = component { diff --git a/templates/common/base.html b/templates/common/base.html index b4dc11d..abc59b4 100644 --- a/templates/common/base.html +++ b/templates/common/base.html @@ -7,13 +7,13 @@ IronDrop | {{PAGE_TITLE}} - + {{PAGE_STYLES}} - - - + + + @@ -25,8 +25,8 @@
-
{{HEADER_ACTIONS}} @@ -42,6 +42,7 @@
+ {{PAGE_SCRIPTS}} diff --git a/templates/directory/script.js b/templates/directory/script.js index 2b284a7..316eac2 100644 --- a/templates/directory/script.js +++ b/templates/directory/script.js @@ -2,6 +2,7 @@ // Dark Mode Only Directory Listing Enhancements with Fast Search document.addEventListener('DOMContentLoaded', function() { + const basePath = window.__BASE_PATH || ''; // Apply loading animation with staggered effect const container = document.querySelector('.container'); const header = document.querySelector('.directory-header'); @@ -638,7 +639,7 @@ document.addEventListener('DOMContentLoaded', function() { } const currentPath = window.location.pathname; - const response = await fetch(`/_irondrop/search?q=${encodeURIComponent(query)}&path=${encodeURIComponent(currentPath)}`); + const response = await fetch(`${basePath}/_irondrop/search?q=${encodeURIComponent(query)}&path=${encodeURIComponent(currentPath)}`); if (!response.ok) { console.warn('API search failed:', response.status); @@ -720,7 +721,7 @@ document.addEventListener('DOMContentLoaded', function() { `; item.addEventListener('click', () => { - window.location.href = result.path; + window.location.href = basePath + result.path; }); dropdownResults.appendChild(item); diff --git a/templates/monitor/script.js b/templates/monitor/script.js index 8406819..cb24a96 100644 --- a/templates/monitor/script.js +++ b/templates/monitor/script.js @@ -3,6 +3,7 @@ let chartJsLoaded = false; let chartsInitialized = false; +const basePath = window.__BASE_PATH || ''; // Check if Chart.js loaded correctly window.addEventListener('load', function() { @@ -192,7 +193,7 @@ async function loadMetrics() { setLoadingState(true); try { - const response = await fetch('/monitor?json=1'); + const response = await fetch(basePath + '/monitor?json=1'); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); @@ -583,7 +584,7 @@ async function performMemoryCleanup() { try { console.log('Initiating memory cleanup...'); - const response = await fetch('/_irondrop/cleanup-memory', { + const response = await fetch(basePath + '/_irondrop/cleanup-memory', { method: 'POST', headers: { 'Content-Type': 'application/json' diff --git a/templates/upload/script.js b/templates/upload/script.js index 4285cd7..cba1bae 100644 --- a/templates/upload/script.js +++ b/templates/upload/script.js @@ -11,6 +11,7 @@ class UploadManager { this.fileCounter = 0; this.totalBytes = 0; this.uploadedBytes = 0; + this.basePath = window.__BASE_PATH || ''; this.init(); } @@ -401,9 +402,9 @@ class UploadManager { const uploadTo = urlParams.get('upload_to'); if (uploadTo) { - return `/_irondrop/upload?upload_to=${encodeURIComponent(uploadTo)}`; + return `${this.basePath}/_irondrop/upload?upload_to=${encodeURIComponent(uploadTo)}`; } else { - return '/_irondrop/upload'; + return `${this.basePath}/_irondrop/upload`; } } diff --git a/tests/async_runtime_starvation_test.rs b/tests/async_runtime_starvation_test.rs index 362174b..ad23e7d 100644 --- a/tests/async_runtime_starvation_test.rs +++ b/tests/async_runtime_starvation_test.rs @@ -45,6 +45,7 @@ fn start_server(dir: std::path::PathBuf, threads: usize) -> TestServer { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/base_path_test.rs b/tests/base_path_test.rs new file mode 100644 index 0000000..3a6aaa3 --- /dev/null +++ b/tests/base_path_test.rs @@ -0,0 +1,788 @@ +// SPDX-License-Identifier: MIT +//! Integration tests for --base-path reverse proxy support. +//! Verifies that all routes, redirects, hrefs, WebDAV XML, and search +//! work correctly when the application is configured with a base path prefix. +//! +//! IMPORTANT: Because `BASE_PATH` uses `OnceLock` (set-once-per-process), +//! every test in this file **must** use the same `--base-path` value: `/bp`. + +use irondrop::cli::Cli; +use irondrop::server::run_server; +use reqwest::Method; +use reqwest::StatusCode; +use reqwest::blocking::Client; +use std::fs::{self, File, create_dir_all}; +use std::io::{BufRead, Read, Write}; +use std::net::SocketAddr; +use std::sync::mpsc; +use std::thread::{self, JoinHandle}; +use tempfile::{TempDir, tempdir}; + +const BASE: &str = "/bp"; + +// --------------------------------------------------------------------------- +// Test server helpers +// --------------------------------------------------------------------------- + +struct TestServer { + addr: SocketAddr, + shutdown_tx: mpsc::Sender<()>, + handle: Option>, + root: std::path::PathBuf, + _temp_dir: TempDir, +} + +impl Drop for TestServer { + fn drop(&mut self) { + if let Some(handle) = self.handle.take() { + self.shutdown_tx.send(()).ok(); + handle.join().unwrap(); + } + } +} + +/// Start a server with `--base-path /bp`. +fn setup_bp_server(enable_webdav: bool, populate: F) -> TestServer +where + F: FnOnce(&std::path::Path), +{ + let dir = tempdir().unwrap(); + populate(dir.path()); + + let cli = Cli { + directory: dir.path().to_path_buf(), + listen: Some("127.0.0.1".to_string()), + port: Some(0), + allowed_extensions: Some("*".to_string()), + threads: Some(4), + chunk_size: Some(1024), + verbose: Some(false), + detailed_logging: Some(false), + username: None, + password: None, + enable_upload: Some(true), + max_upload_size: Some(10240), + enable_webdav: Some(enable_webdav), + disable_rate_limit: Some(false), + config_file: None, + log_dir: None, + ssl_cert: None, + ssl_key: None, + base_path: Some(BASE.to_string()), + }; + + let (shutdown_tx, shutdown_rx) = mpsc::channel(); + let (addr_tx, addr_rx) = mpsc::channel(); + let root = dir.path().to_path_buf(); + + let server_handle = thread::spawn(move || { + if let Err(e) = run_server(cli, Some(shutdown_rx), Some(addr_tx)) { + eprintln!("Server thread failed: {e}"); + } + }); + + let server_addr = addr_rx.recv().unwrap(); + + TestServer { + addr: server_addr, + shutdown_tx, + handle: Some(server_handle), + root, + _temp_dir: dir, + } +} + +// =========================================================================== +// Phase 1 — Routing: base-path stripping and rejection +// =========================================================================== + +#[test] +fn test_bp_root_returns_404() { + let server = setup_bp_server(false, |root| { + let mut f = File::create(root.join("hello.txt")).unwrap(); + write!(f, "hi").unwrap(); + }); + let client = Client::new(); + + let res = client + .get(format!("http://{}/", server.addr)) + .send() + .unwrap(); + assert_eq!(res.status(), StatusCode::NOT_FOUND); +} + +#[test] +fn test_bp_wrong_prefix_returns_404() { + let server = setup_bp_server(false, |root| { + let mut f = File::create(root.join("hello.txt")).unwrap(); + write!(f, "hi").unwrap(); + }); + let client = Client::new(); + + let res = client + .get(format!("http://{}/other/hello.txt", server.addr)) + .send() + .unwrap(); + assert_eq!(res.status(), StatusCode::NOT_FOUND); +} + +#[test] +fn test_bp_root_listing_works() { + let server = setup_bp_server(false, |root| { + let mut f = File::create(root.join("readme.txt")).unwrap(); + write!(f, "contents").unwrap(); + }); + let client = Client::new(); + + let res = client + .get(format!("http://{}{}/", server.addr, BASE)) + .send() + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + let body = res.text().unwrap(); + assert!( + body.contains("readme.txt"), + "listing should contain the file" + ); +} + +#[test] +fn test_bp_file_download() { + let server = setup_bp_server(false, |root| { + let mut f = File::create(root.join("data.txt")).unwrap(); + write!(f, "payload-123").unwrap(); + }); + let client = Client::new(); + + let res = client + .get(format!("http://{}{}/data.txt", server.addr, BASE)) + .send() + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(res.text().unwrap(), "payload-123"); +} + +#[test] +fn test_bp_nested_directory_download() { + let server = setup_bp_server(false, |root| { + create_dir_all(root.join("a/b/c")).unwrap(); + let mut f = File::create(root.join("a/b/c/deep.txt")).unwrap(); + write!(f, "deep-content").unwrap(); + }); + let client = Client::new(); + + let res = client + .get(format!("http://{}{}/a/b/c/deep.txt", server.addr, BASE)) + .send() + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(res.text().unwrap(), "deep-content"); +} + +// =========================================================================== +// Phase 2 — Redirects and HTML href generation +// =========================================================================== + +#[test] +fn test_bp_directory_trailing_slash_redirect() { + let server = setup_bp_server(false, |root| { + create_dir_all(root.join("subdir")).unwrap(); + }); + let no_redirect = reqwest::blocking::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(); + + let res = no_redirect + .get(format!("http://{}{}/subdir", server.addr, BASE)) + .send() + .unwrap(); + assert_eq!(res.status(), StatusCode::MOVED_PERMANENTLY); + let loc = res.headers().get("location").unwrap().to_str().unwrap(); + assert_eq!(loc, "/bp/subdir/", "redirect must include base-path prefix"); +} + +#[test] +fn test_bp_listing_links_have_prefix() { + let server = setup_bp_server(false, |root| { + create_dir_all(root.join("photos")).unwrap(); + let mut f = File::create(root.join("photos/pic.jpg")).unwrap(); + write!(f, "img").unwrap(); + let mut f2 = File::create(root.join("notes.txt")).unwrap(); + write!(f2, "n").unwrap(); + }); + let client = Client::new(); + + // Root listing + let res = client + .get(format!("http://{}{}/", server.addr, BASE)) + .send() + .unwrap(); + let body = res.text().unwrap(); + assert!( + body.contains("/bp/photos/"), + "dir entry must have /bp/ prefix" + ); + assert!( + body.contains("/bp/notes.txt"), + "file entry must have /bp/ prefix" + ); + + // Nested listing + let res = client + .get(format!("http://{}{}/photos/", server.addr, BASE)) + .send() + .unwrap(); + let body = res.text().unwrap(); + assert!( + body.contains("/bp/photos/pic.jpg"), + "nested file must have /bp/ prefix" + ); +} + +#[test] +fn test_bp_html_contains_js_global_and_asset_prefixes() { + let server = setup_bp_server(false, |root| { + let mut f = File::create(root.join("f.txt")).unwrap(); + write!(f, "f").unwrap(); + }); + let client = Client::new(); + + let res = client + .get(format!("http://{}{}/", server.addr, BASE)) + .send() + .unwrap(); + let body = res.text().unwrap(); + + assert!( + body.contains(r#"window.__BASE_PATH = "/bp""#), + "JS global must be set" + ); + assert!( + body.contains("/bp/_irondrop/static/"), + "static asset links must be prefixed" + ); + assert!(body.contains("/bp/favicon.ico"), "favicon must be prefixed"); +} + +#[test] +fn test_bp_static_assets_reachable() { + let server = setup_bp_server(false, |root| { + let mut f = File::create(root.join("f.txt")).unwrap(); + write!(f, "f").unwrap(); + }); + let client = Client::new(); + + let res = client + .get(format!( + "http://{}{}/_irondrop/static/common/base.css", + server.addr, BASE + )) + .send() + .unwrap(); + assert_eq!( + res.status(), + StatusCode::OK, + "base.css must be reachable through base path" + ); + let ct = res.headers().get("content-type").unwrap().to_str().unwrap(); + assert!(ct.contains("text/css")); +} + +#[test] +fn test_bp_upload_link_has_prefix() { + let server = setup_bp_server(false, |_root| {}); + let client = Client::new(); + + let res = client + .get(format!("http://{}{}/", server.addr, BASE)) + .send() + .unwrap(); + let body = res.text().unwrap(); + assert!( + body.contains("/bp/_irondrop/upload"), + "upload link must have base path" + ); +} + +// =========================================================================== +// Phase 3 — Search API +// =========================================================================== + +#[test] +fn test_bp_search_api_works() { + let server = setup_bp_server(false, |root| { + let mut f = File::create(root.join("document.txt")).unwrap(); + write!(f, "contents").unwrap(); + }); + let client = Client::new(); + + let res = client + .get(format!( + "http://{}{}/_irondrop/search?q=document&path=/", + server.addr, BASE + )) + .send() + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + let body = res.text().unwrap(); + assert!(body.contains("document.txt"), "search must find the file"); +} + +// =========================================================================== +// Phase 4 — WebDAV with base-path +// =========================================================================== + +#[test] +fn test_bp_webdav_options() { + let server = setup_bp_server(true, |_root| {}); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"OPTIONS").unwrap(), + format!("http://{}{}/", server.addr, BASE), + ) + .send() + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + let dav = res.headers().get("DAV").unwrap().to_str().unwrap(); + assert!(dav.contains("1")); + let allow = res.headers().get("Allow").unwrap().to_str().unwrap(); + assert!(allow.contains("PROPFIND")); + assert!(allow.contains("COPY")); + assert!(allow.contains("MOVE")); +} + +#[test] +fn test_bp_webdav_propfind_root_href() { + let server = setup_bp_server(true, |root| { + create_dir_all(root.join("docs")).unwrap(); + let mut f = File::create(root.join("docs/report.txt")).unwrap(); + write!(f, "data").unwrap(); + }); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!("http://{}{}/", server.addr, BASE), + ) + .header("Depth", "1") + .send() + .unwrap(); + + assert_eq!(res.status().as_u16(), 207); + let body = res.text().unwrap(); + assert!( + body.contains("/bp/"), + "root href must be /bp/, got:\n{body}" + ); + assert!( + body.contains("/bp/docs/"), + "child dir href must be /bp/docs/" + ); +} + +#[test] +fn test_bp_webdav_propfind_nested_hrefs() { + let server = setup_bp_server(true, |root| { + create_dir_all(root.join("a/b")).unwrap(); + let mut f = File::create(root.join("a/b/file.txt")).unwrap(); + write!(f, "nested").unwrap(); + }); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!("http://{}{}/a/b/", server.addr, BASE), + ) + .header("Depth", "1") + .send() + .unwrap(); + + assert_eq!(res.status().as_u16(), 207); + let body = res.text().unwrap(); + assert!( + body.contains("/bp/a/b/"), + "nested collection href" + ); + assert!( + body.contains("/bp/a/b/file.txt"), + "nested file href" + ); +} + +#[test] +fn test_bp_webdav_propfind_infinity_all_hrefs_prefixed() { + let server = setup_bp_server(true, |root| { + create_dir_all(root.join("x/y")).unwrap(); + let mut f = File::create(root.join("x/y/z.txt")).unwrap(); + write!(f, "deep").unwrap(); + }); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"PROPFIND").unwrap(), + format!("http://{}{}/", server.addr, BASE), + ) + .header("Depth", "infinity") + .send() + .unwrap(); + + assert_eq!(res.status().as_u16(), 207); + let body = res.text().unwrap(); + + // Every must start with /bp/ + for line in body.lines() { + if let Some(start) = line.find("") { + let href_start = start + "".len(); + if let Some(end) = line[href_start..].find("") { + let href = &line[href_start..href_start + end]; + assert!( + href.starts_with("/bp/"), + "every href must start with /bp/, got: {href}" + ); + } + } + } +} + +#[test] +fn test_bp_webdav_put() { + let server = setup_bp_server(true, |_root| {}); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"PUT").unwrap(), + format!("http://{}{}/new-file.txt", server.addr, BASE), + ) + .body("created-via-put") + .send() + .unwrap(); + + assert_eq!(res.status(), StatusCode::CREATED); + assert_eq!( + fs::read_to_string(server.root.join("new-file.txt")).unwrap(), + "created-via-put" + ); +} + +#[test] +fn test_bp_webdav_mkcol() { + let server = setup_bp_server(true, |_root| {}); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"MKCOL").unwrap(), + format!("http://{}{}/newcol/", server.addr, BASE), + ) + .send() + .unwrap(); + + assert_eq!(res.status(), StatusCode::CREATED); + assert!(server.root.join("newcol").is_dir()); +} + +#[test] +fn test_bp_webdav_delete() { + let server = setup_bp_server(true, |root| { + let mut f = File::create(root.join("gone.txt")).unwrap(); + write!(f, "bye").unwrap(); + }); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"DELETE").unwrap(), + format!("http://{}{}/gone.txt", server.addr, BASE), + ) + .send() + .unwrap(); + + assert_eq!(res.status().as_u16(), 204); + assert!(!server.root.join("gone.txt").exists()); +} + +#[test] +fn test_bp_webdav_copy() { + let server = setup_bp_server(true, |root| { + let mut f = File::create(root.join("src.txt")).unwrap(); + write!(f, "copy-data").unwrap(); + }); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"COPY").unwrap(), + format!("http://{}{}/src.txt", server.addr, BASE), + ) + .header( + "Destination", + format!("http://{}{}/dst.txt", server.addr, BASE), + ) + .send() + .unwrap(); + + assert_eq!(res.status(), StatusCode::CREATED); + assert_eq!( + fs::read_to_string(server.root.join("src.txt")).unwrap(), + "copy-data" + ); + assert_eq!( + fs::read_to_string(server.root.join("dst.txt")).unwrap(), + "copy-data" + ); +} + +#[test] +fn test_bp_webdav_copy_wrong_prefix_rejected() { + let server = setup_bp_server(true, |root| { + let mut f = File::create(root.join("src.txt")).unwrap(); + write!(f, "data").unwrap(); + }); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"COPY").unwrap(), + format!("http://{}{}/src.txt", server.addr, BASE), + ) + .header( + "Destination", + format!("http://{}/wrong/dst.txt", server.addr), + ) + .send() + .unwrap(); + + assert_eq!( + res.status(), + StatusCode::BAD_REQUEST, + "destination with wrong prefix must be rejected" + ); +} + +#[test] +fn test_bp_webdav_move() { + let server = setup_bp_server(true, |root| { + let mut f = File::create(root.join("old.txt")).unwrap(); + write!(f, "move-data").unwrap(); + }); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"MOVE").unwrap(), + format!("http://{}{}/old.txt", server.addr, BASE), + ) + .header( + "Destination", + format!("http://{}{}/new.txt", server.addr, BASE), + ) + .send() + .unwrap(); + + assert_eq!(res.status(), StatusCode::CREATED); + assert!(!server.root.join("old.txt").exists()); + assert_eq!( + fs::read_to_string(server.root.join("new.txt")).unwrap(), + "move-data" + ); +} + +#[test] +fn test_bp_webdav_move_nested_dirs() { + let server = setup_bp_server(true, |root| { + create_dir_all(root.join("from/sub")).unwrap(); + let mut f = File::create(root.join("from/sub/file.txt")).unwrap(); + write!(f, "nested-move").unwrap(); + }); + let client = Client::new(); + + let res = client + .request( + Method::from_bytes(b"MOVE").unwrap(), + format!("http://{}{}/from/", server.addr, BASE), + ) + .header("Destination", format!("http://{}{}/to/", server.addr, BASE)) + .send() + .unwrap(); + + assert_eq!(res.status(), StatusCode::CREATED); + assert!(!server.root.join("from").exists()); + assert_eq!( + fs::read_to_string(server.root.join("to/sub/file.txt")).unwrap(), + "nested-move" + ); +} + +#[test] +fn test_bp_webdav_lock_lockroot_has_prefix() { + let server = setup_bp_server(true, |root| { + let mut f = File::create(root.join("locked.txt")).unwrap(); + write!(f, "l").unwrap(); + }); + let client = Client::new(); + + let lock_body = r#" + + + +"#; + + let res = client + .request( + Method::from_bytes(b"LOCK").unwrap(), + format!("http://{}{}/locked.txt", server.addr, BASE), + ) + .header("Timeout", "Second-3600") + .body(lock_body) + .send() + .unwrap(); + + assert_eq!(res.status(), StatusCode::CREATED); + let body = res.text().unwrap(); + assert!( + body.contains("/bp/locked.txt"), + "lockroot must include base path, got:\n{body}" + ); +} + +#[test] +fn test_bp_webdav_proppatch_href_has_prefix() { + let server = setup_bp_server(true, |root| { + let mut f = File::create(root.join("prop.txt")).unwrap(); + write!(f, "p").unwrap(); + }); + let client = Client::new(); + + let body = r#" + + TestUser +"#; + + let res = client + .request( + Method::from_bytes(b"PROPPATCH").unwrap(), + format!("http://{}{}/prop.txt", server.addr, BASE), + ) + .body(body) + .send() + .unwrap(); + + assert_eq!(res.status().as_u16(), 207); + let resp_body = res.text().unwrap(); + assert!( + resp_body.contains("/bp/prop.txt"), + "PROPPATCH href must include base path" + ); +} + +// =========================================================================== +// Phase 5 — Raw TCP edge cases +// =========================================================================== + +#[test] +fn test_bp_raw_tcp_file_download() { + let server = setup_bp_server(false, |root| { + let mut f = File::create(root.join("tcp.txt")).unwrap(); + write!(f, "raw-tcp-content").unwrap(); + }); + + let mut stream = std::net::TcpStream::connect(server.addr).unwrap(); + write!( + stream, + "GET /bp/tcp.txt HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" + ) + .unwrap(); + stream.flush().unwrap(); + + let mut buf = Vec::new(); + stream.read_to_end(&mut buf).unwrap(); + let text = String::from_utf8_lossy(&buf); + assert!(text.contains("HTTP/1.1 200"), "expected 200, got: {text}"); + assert!(text.contains("raw-tcp-content")); +} + +#[test] +fn test_bp_raw_tcp_without_prefix_is_404() { + let server = setup_bp_server(false, |root| { + let mut f = File::create(root.join("tcp.txt")).unwrap(); + write!(f, "content").unwrap(); + }); + + let mut stream = std::net::TcpStream::connect(server.addr).unwrap(); + write!( + stream, + "GET /tcp.txt HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" + ) + .unwrap(); + stream.flush().unwrap(); + + let mut reader = std::io::BufReader::new(stream); + let mut status_line = String::new(); + reader.read_line(&mut status_line).unwrap(); + assert!( + status_line.contains("404"), + "without prefix must be 404, got: {status_line}" + ); +} + +// =========================================================================== +// Phase 6 — CLI validation unit tests +// =========================================================================== + +#[test] +fn test_cli_base_path_normalization() { + // Valid paths + let cli = Cli { + directory: std::path::PathBuf::from("."), + listen: None, + port: None, + allowed_extensions: None, + threads: None, + chunk_size: None, + verbose: None, + detailed_logging: None, + username: None, + password: None, + enable_upload: None, + max_upload_size: None, + enable_webdav: None, + disable_rate_limit: None, + config_file: None, + log_dir: None, + ssl_cert: None, + ssl_key: None, + base_path: Some("/webstorage".to_string()), + }; + assert_eq!(cli.base_path.as_deref(), Some("/webstorage")); + + // With trailing slash — the server should strip it in validate_base_path + let cli2 = Cli { + directory: std::path::PathBuf::from("."), + listen: None, + port: None, + allowed_extensions: None, + threads: None, + chunk_size: None, + verbose: None, + detailed_logging: None, + username: None, + password: None, + enable_upload: None, + max_upload_size: None, + enable_webdav: None, + disable_rate_limit: None, + config_file: None, + log_dir: None, + ssl_cert: None, + ssl_key: None, + base_path: Some("/storage/".to_string()), + }; + assert!(cli2.base_path.is_some()); +} diff --git a/tests/config_test.rs b/tests/config_test.rs index 1e1a0be..e691d7d 100644 --- a/tests/config_test.rs +++ b/tests/config_test.rs @@ -184,6 +184,7 @@ verbose = false log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let config = Config::load(&cli).expect("Failed to load config"); @@ -233,6 +234,7 @@ max_upload_size = 1GB log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let config = Config::load(&cli).expect("Failed to load config"); @@ -266,6 +268,7 @@ fn test_config_defaults() { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let config = Config::load(&cli).expect("Failed to load config"); @@ -308,6 +311,7 @@ fn test_config_file_load_error() { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let result = Config::load(&cli); @@ -378,6 +382,7 @@ directory = {} log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let config = Config::load(&cli).expect("Failed to load config"); @@ -423,6 +428,7 @@ port = 9999 log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let config = Config::load(&cli).expect("Failed to load config"); @@ -473,6 +479,7 @@ fn test_config_invalid_port_values() { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let result = Config::load(&cli); @@ -520,6 +527,7 @@ fn test_config_invalid_port_values() { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let result = Config::load(&cli); @@ -578,6 +586,7 @@ fn test_config_invalid_file_size_formats() { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let result = Config::load(&cli); @@ -656,6 +665,7 @@ fn test_config_boolean_edge_cases() { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let result = Config::load(&cli); @@ -706,6 +716,7 @@ fn test_config_malformed_ini_syntax() { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let _result = Config::load(&cli); diff --git a/tests/direct_upload_test.rs b/tests/direct_upload_test.rs index 62ef7fd..5c020f4 100644 --- a/tests/direct_upload_test.rs +++ b/tests/direct_upload_test.rs @@ -28,6 +28,7 @@ fn create_test_cli(upload_dir: PathBuf) -> Cli { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, } } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 60e6831..6419d3c 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -51,6 +51,7 @@ fn setup_test_server(username: Option, password: Option) -> Test log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); @@ -490,6 +491,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/log_dir_test.rs b/tests/log_dir_test.rs index 353f78f..c810244 100644 --- a/tests/log_dir_test.rs +++ b/tests/log_dir_test.rs @@ -36,6 +36,7 @@ fn create_test_cli_with_log_dir(log_dir: Option) -> Cli { log_dir, ssl_cert: None, ssl_key: None, + base_path: None, } } diff --git a/tests/monitor_test.rs b/tests/monitor_test.rs index 651cf1a..3dfdb93 100644 --- a/tests/monitor_test.rs +++ b/tests/monitor_test.rs @@ -49,6 +49,7 @@ fn setup_test_server() -> TestServer { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); @@ -216,6 +217,7 @@ fn test_monitor_ignores_macos_finder_propfind_noise_from_error_counter() { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); @@ -285,6 +287,7 @@ fn test_monitor_ignores_macos_metadata_propfind_noise_from_error_counter() { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); @@ -354,6 +357,7 @@ fn test_monitor_ignores_archive_service_temp_propfind_noise_from_error_counter() log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/ssl_test.rs b/tests/ssl_test.rs index f6d9623..deb42f8 100644 --- a/tests/ssl_test.rs +++ b/tests/ssl_test.rs @@ -96,6 +96,7 @@ fn setup_ssl_server(username: Option, password: Option) -> TestS log_dir: None, ssl_cert: Some(cert_path), ssl_key: Some(key_path), + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); @@ -279,6 +280,7 @@ fn test_ssl_missing_cert_file() { log_dir: None, ssl_cert: Some(bogus_cert), ssl_key: Some(key_path), + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); @@ -321,6 +323,7 @@ fn test_ssl_missing_key_file() { log_dir: None, ssl_cert: Some(cert_path), ssl_key: Some(bogus_key), + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); @@ -359,6 +362,7 @@ fn test_ssl_cert_without_key() { log_dir: None, ssl_cert: Some(cert_path), ssl_key: None, + base_path: None, }; let result = cli.validate(); @@ -400,6 +404,7 @@ fn test_ssl_key_without_cert() { log_dir: None, ssl_cert: None, ssl_key: Some(key_path), + base_path: None, }; let result = cli.validate(); @@ -444,6 +449,7 @@ fn test_http_still_works_without_ssl() { log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/webdav_copy_move_rfc_test.rs b/tests/webdav_copy_move_rfc_test.rs index 1ed7d9a..870e8db 100644 --- a/tests/webdav_copy_move_rfc_test.rs +++ b/tests/webdav_copy_move_rfc_test.rs @@ -45,6 +45,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/webdav_copy_move_test.rs b/tests/webdav_copy_move_test.rs index 3c432cd..559563e 100644 --- a/tests/webdav_copy_move_test.rs +++ b/tests/webdav_copy_move_test.rs @@ -46,6 +46,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/webdav_error_semantics_test.rs b/tests/webdav_error_semantics_test.rs index 0f36c7c..6a91e15 100644 --- a/tests/webdav_error_semantics_test.rs +++ b/tests/webdav_error_semantics_test.rs @@ -45,6 +45,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/webdav_error_xml_rfc_test.rs b/tests/webdav_error_xml_rfc_test.rs index 3ff6fd6..5965fa1 100644 --- a/tests/webdav_error_xml_rfc_test.rs +++ b/tests/webdav_error_xml_rfc_test.rs @@ -44,6 +44,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/webdav_lock_if_rfc_test.rs b/tests/webdav_lock_if_rfc_test.rs index 5968fd9..e9bcd57 100644 --- a/tests/webdav_lock_if_rfc_test.rs +++ b/tests/webdav_lock_if_rfc_test.rs @@ -45,6 +45,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/webdav_options_propfind_test.rs b/tests/webdav_options_propfind_test.rs index ecadad5..3d7db2f 100644 --- a/tests/webdav_options_propfind_test.rs +++ b/tests/webdav_options_propfind_test.rs @@ -49,6 +49,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); @@ -104,6 +105,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/webdav_propfind_rfc_test.rs b/tests/webdav_propfind_rfc_test.rs index 56be92b..5eee105 100644 --- a/tests/webdav_propfind_rfc_test.rs +++ b/tests/webdav_propfind_rfc_test.rs @@ -44,6 +44,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/webdav_proppatch_rfc_test.rs b/tests/webdav_proppatch_rfc_test.rs index 9a485ef..3a63f3d 100644 --- a/tests/webdav_proppatch_rfc_test.rs +++ b/tests/webdav_proppatch_rfc_test.rs @@ -44,6 +44,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); diff --git a/tests/webdav_write_methods_test.rs b/tests/webdav_write_methods_test.rs index 7cea097..60db079 100644 --- a/tests/webdav_write_methods_test.rs +++ b/tests/webdav_write_methods_test.rs @@ -46,6 +46,7 @@ where log_dir: None, ssl_cert: None, ssl_key: None, + base_path: None, }; let (shutdown_tx, shutdown_rx) = mpsc::channel(); From 08b233b4bfbc45745558e27bcbfbc6463654c473 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Tue, 28 Apr 2026 01:45:58 +0530 Subject: [PATCH 14/17] irondrop: supress finder noise --- src/http.rs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/http.rs b/src/http.rs index c243e26..49b5002 100644 --- a/src/http.rs +++ b/src/http.rs @@ -349,6 +349,10 @@ pub async fn handle_client_async( match response_result { Ok(response) => { + // Check if this is a Finder noise 404 before logging and stats + let is_finder_noise = response.status_code == 404 + && crate::utils::is_macos_finder_noise_path(&request_path); + log_request_line( &log_prefix, &request_method, @@ -357,8 +361,10 @@ pub async fn handle_client_async( ); match send_response_async(&mut stream, response, &log_prefix).await { Ok(body_bytes) => { - if let Some(stats) = stats { - stats.record_request(true, body_bytes); + if !is_finder_noise { + if let Some(stats) = stats { + stats.record_request(true, body_bytes); + } } } Err(_) => { @@ -369,12 +375,14 @@ pub async fn handle_client_async( } } Err(e) => { - let ignore_error_for_stats = matches!(e, AppError::NotFound) + let is_finder_noise = matches!(e, AppError::NotFound) && request_method == "PROPFIND" && crate::utils::is_macos_finder_noise_path(&request_path); send_error_response_async(&mut stream, e, &log_prefix).await; - if let Some(stats) = stats { - stats.record_request(ignore_error_for_stats, 0); + if !is_finder_noise { + if let Some(stats) = stats { + stats.record_request(false, 0); + } } } } @@ -390,6 +398,12 @@ fn log_request_line(log_prefix: &str, method: &str, path: &str, status_code: u16 monitor_request_log_rate_limited(method, status_code); return; } + // Suppress macOS Finder noise (._files, .Spotlight, .AU.*, etc.) to TRACE + // level to keep logs clean. These are harmless 404s from Finder probing. + if status_code == 404 && crate::utils::is_macos_finder_noise_path(path_only) { + trace!("{log_prefix} {method} {path} -> {status_code} (finder noise)"); + return; + } info!("{log_prefix} {method} {path} -> {status_code}"); } From 69eb8bd5a1ccfbf4199662bff0681debf424e6ce Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Tue, 28 Apr 2026 02:19:41 +0530 Subject: [PATCH 15/17] irondrop: some memory optimizations --- src/fs.rs | 65 ++++++------- src/handlers.rs | 24 ++++- src/http.rs | 45 ++++++--- src/search.rs | 2 +- src/templates.rs | 20 ++++ src/webdav.rs | 153 ++++++++++++++++++++----------- templates/directory/script.js | 2 +- tests/hidden_files_test.rs | 4 +- tests/template_embedding_test.rs | 2 +- tests/templates_escape_test.rs | 10 +- 10 files changed, 216 insertions(+), 111 deletions(-) diff --git a/src/fs.rs b/src/fs.rs index 9552227..6bee124 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -14,42 +14,32 @@ pub fn generate_directory_listing( path: &Path, request_path: &str, config: Option<&Config>, + page: usize, ) -> Result { debug!("Generating directory listing for: '{}'", path.display()); trace!("Request path: '{}'", request_path); let mut entries = Vec::new(); - // Collect and sort entries + // Collect and sort entries using lazy metadata lookup (super fast, low memory) trace!("Reading directory entries from: {}", path.display()); for entry in fs::read_dir(path)? { let entry = entry?; - let metadata = entry.metadata()?; + let file_type = entry.file_type()?; let file_name = entry.file_name().into_string().unwrap_or_default(); - // Skip hidden files (starting with '._' or '.DS_Store') if is_hidden_file(&file_name) { - trace!("Skipping hidden file: {}", file_name); continue; } - trace!( - "Found entry: {} ({})", - file_name, - if metadata.is_dir() { - "directory" - } else { - "file" - } - ); - entries.push((entry.path(), file_name, metadata)); + entries.push((entry.path(), file_name, file_type.is_dir())); } debug!("Found {} entries, sorting...", entries.len()); // Sort: directories first, then alphabetically entries.sort_by(|a, b| { - let a_is_dir = a.2.is_dir(); - let b_is_dir = b.2.is_dir(); + let a_is_dir = a.2; + let b_is_dir = b.2; match (a_is_dir, b_is_dir) { (true, false) => std::cmp::Ordering::Less, @@ -65,51 +55,56 @@ pub fn generate_directory_listing( }; debug!("Preparing {} entries for template rendering", entries.len()); - // Prepare entries data for template - let mut template_entries = Vec::new(); + let total_count = entries.len(); + let limit = 1000; + let total_pages = total_count.div_ceil(limit); + let safe_page = page.max(1).min(total_pages.max(1)); + let offset = (safe_page - 1) * limit; - for (_entry_path, file_name, metadata) in entries { - let is_dir = metadata.is_dir(); + let page_entries: Vec<_> = entries.into_iter().skip(offset).take(limit).collect(); + let mut template_entries = Vec::with_capacity(page_entries.len()); + + for (entry_path, file_name, is_dir) in page_entries { let link_name = if is_dir { format!("{file_name}/") } else { file_name.clone() }; + // Lazy metadata fetch for only the current page's files + let metadata_res = std::fs::metadata(&entry_path); + let size = if is_dir { "-".to_string() } else { - format_file_size(metadata.len()) + metadata_res + .as_ref() + .map(|m| format_file_size(m.len())) + .unwrap_or_else(|_| "-".to_string()) }; - let modified = metadata - .modified() + let modified = metadata_res .ok() + .and_then(|m| m.modified().ok()) .and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok()) - .map(|duration| { - let timestamp = duration.as_secs(); - format_timestamp(timestamp) - }) + .map(|duration| format_timestamp(duration.as_secs())) .unwrap_or_else(|| "-".to_string()); template_entries.push((link_name, size, modified)); } debug!("Creating template engine and rendering directory listing"); - // Create template engine with embedded templates let engine = TemplateEngine::global(); - // Render using template with conditional upload button - trace!( - "Upload enabled: {}", - config.map(|c| c.enable_upload).unwrap_or(false) - ); + let upload_enabled = config.map(|c| c.enable_upload).unwrap_or(false); engine.render_directory_listing( display_path, &template_entries, - template_entries.len(), - config.map(|c| c.enable_upload).unwrap_or(false), + total_count, + upload_enabled, request_path, + safe_page, + total_pages, ) } diff --git a/src/handlers.rs b/src/handlers.rs index 1c8500f..155636a 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -591,7 +591,29 @@ pub fn handle_file_request( base_path: cli.base_path.clone().unwrap_or_default(), }); - let html_content = generate_directory_listing(&full_path, &request.path, config.as_ref())?; + // Extract page from query parameters + let query_params: HashMap = + if let Some(query_string) = request.path.split('?').nth(1) { + query_string + .split('&') + .filter_map(|param| { + let mut parts = param.splitn(2, '='); + match (parts.next(), parts.next()) { + (Some(key), Some(value)) => Some((url_decode(key), url_decode(value))), + _ => None, + } + }) + .collect() + } else { + HashMap::new() + }; + let page = query_params + .get("p") + .and_then(|v| v.parse::().ok()) + .unwrap_or(1); + + let html_content = + generate_directory_listing(&full_path, &request.path, config.as_ref(), page)?; Ok(Response { status_code: 200, status_text: "OK".to_string(), diff --git a/src/http.rs b/src/http.rs index 49b5002..cae7d87 100644 --- a/src/http.rs +++ b/src/http.rs @@ -19,9 +19,9 @@ const MAX_REQUEST_BODY_SIZE: usize = 10 * 1024 * 1024 * 1024; /// Maximum size for request headers (8KB) to prevent header buffer overflow const MAX_HEADERS_SIZE: usize = 8 * 1024; -/// Threshold for streaming request bodies to disk (64MB) +/// Threshold for streaming request bodies to disk (2MB) /// This ensures total memory usage stays well below 128MB -pub const STREAM_TO_DISK_THRESHOLD: usize = 64 * 1024 * 1024; +pub const STREAM_TO_DISK_THRESHOLD: usize = 2 * 1024 * 1024; static TEMP_FILE_COUNTER: AtomicU64 = AtomicU64::new(0); /// Represents a parsed incoming HTTP request. @@ -79,6 +79,7 @@ pub enum ResponseBody { Binary(Vec), StaticBinary(&'static [u8]), Stream(StreamBody), + AsyncStream(tokio::sync::mpsc::Receiver>), } impl Request { @@ -360,7 +361,9 @@ pub async fn handle_client_async( response.status_code, ); match send_response_async(&mut stream, response, &log_prefix).await { - Ok(body_bytes) => { + Ok(body_bytes) => + { + #[allow(clippy::collapsible_if)] if !is_finder_noise { if let Some(stats) = stats { stats.record_request(true, body_bytes); @@ -379,6 +382,7 @@ pub async fn handle_client_async( && request_method == "PROPFIND" && crate::utils::is_macos_finder_noise_path(&request_path); send_error_response_async(&mut stream, e, &log_prefix).await; + #[allow(clippy::collapsible_if)] if !is_finder_noise { if let Some(stats) = stats { stats.record_request(false, 0); @@ -511,17 +515,19 @@ where .keys() .any(|k| k.to_lowercase() == "content-length"); if !has_content_length { - let length = match &response.body { - ResponseBody::Text(text) => text.len(), - ResponseBody::StaticText(text) => text.len(), - ResponseBody::Binary(bytes) => bytes.len(), - ResponseBody::StaticBinary(bytes) => bytes.len(), - ResponseBody::Stream(stream_body) => stream_body.size as usize, + let length_opt = match &response.body { + ResponseBody::Text(text) => Some(text.len()), + ResponseBody::StaticText(text) => Some(text.len()), + ResponseBody::Binary(bytes) => Some(bytes.len()), + ResponseBody::StaticBinary(bytes) => Some(bytes.len()), + ResponseBody::Stream(stream_body) => Some(stream_body.size as usize), + ResponseBody::AsyncStream(_) => None, }; - response_str.push_str(&format!( - "Content-Length: {length} -" - )); + if let Some(length) = length_opt { + response_str.push_str(&format!("Content-Length: {length}\r\n")); + } else { + response_str.push_str("Transfer-Encoding: chunked\r\n"); + } } response_str.push_str("\r\n"); @@ -559,6 +565,19 @@ where body_sent += bytes_read as u64; } } + ResponseBody::AsyncStream(mut receiver) => { + while let Some(chunk) = receiver.recv().await { + if chunk.is_empty() { + continue; + } + let hex_size = format!("{:X}\r\n", chunk.len()); + stream.write_all(hex_size.as_bytes()).await?; + stream.write_all(&chunk).await?; + stream.write_all(b"\r\n").await?; + body_sent += chunk.len() as u64; + } + stream.write_all(b"0\r\n\r\n").await?; + } } stream.flush().await?; diff --git a/src/search.rs b/src/search.rs index 40ec663..831c374 100644 --- a/src/search.rs +++ b/src/search.rs @@ -536,7 +536,7 @@ impl RadixBucket { impl UltraLowMemoryIndex { /// Create new ultra-low memory index with optimized capacity planning pub fn new(base_dir: PathBuf) -> Self { - let estimated_entries = 10_000_000; // Plan for 10M entries + let estimated_entries = 100_000; // Plan for 100K entries initial capacity let estimated_string_pool_size = estimated_entries * 15; // ~15 chars average filename // Initialize radix buckets array diff --git a/src/templates.rs b/src/templates.rs index a90a08c..80d18b3 100644 --- a/src/templates.rs +++ b/src/templates.rs @@ -527,6 +527,7 @@ impl TemplateEngine { } /// Generate directory listing HTML using base template system + #[allow(clippy::too_many_arguments)] pub fn render_directory_listing( &self, path: &str, @@ -534,6 +535,8 @@ impl TemplateEngine { entry_count: usize, upload_enabled: bool, current_path: &str, + page: usize, + total_pages: usize, ) -> Result { debug!( "Rendering directory listing: path='{}', entries={}, upload_enabled={}", @@ -625,6 +628,23 @@ impl TemplateEngine { )); } + if total_pages > 1 { + let mut pagination_html = String::from( + "", + ); + + if page > 1 { + pagination_html.push_str(&format!("← Previous Page", page - 1)); + } + pagination_html.push_str(&format!("Page {} of {}", page, total_pages)); + if page < total_pages { + pagination_html.push_str(&format!("Next Page →", page + 1)); + } + + pagination_html.push_str(""); + entries_html.push_str(&pagination_html); + } + variables.insert("ENTRIES".to_string(), entries_html); // Use the new base template system diff --git a/src/webdav.rs b/src/webdav.rs index 70688c7..ca6a054 100644 --- a/src/webdav.rs +++ b/src/webdav.rs @@ -116,29 +116,71 @@ fn handle_propfind(request: &Request, base_dir: &Path) -> Result {} - DavDepth::One => { - let mut entries = Vec::new(); - for entry in std::fs::read_dir(&target_path)? { - let entry = entry?; - entries.push(entry.path()); - } - entries.sort_by(|a, b| a.to_string_lossy().cmp(&b.to_string_lossy())); - resources.extend(entries); + let mut headers = HashMap::new(); + headers.insert( + "Content-Type".to_string(), + "application/xml; charset=utf-8".to_string(), + ); + headers.insert("DAV".to_string(), "1,2".to_string()); + headers.insert("Allow".to_string(), allow_header_value().to_string()); + + if depth == DavDepth::Infinity && target_path.is_dir() { + let (tx, rx) = tokio::sync::mpsc::channel(64); + let base_dir = base_dir.to_path_buf(); + let target_path = target_path.clone(); + + std::thread::spawn(move || { + let _ = tx.blocking_send( + b"\n\n" + .to_vec(), + ); + + let mut element = String::new(); + if append_multistatus_response(&mut element, &base_dir, &target_path, &mode).is_ok() { + let _ = tx.blocking_send(element.into_bytes()); } - DavDepth::Infinity => { - collect_recursive_resources(&target_path, &mut resources)?; + + let mut stack = vec![target_path]; + while let Some(current_dir) = stack.pop() { + if let Ok(entries) = std::fs::read_dir(¤t_dir) { + for entry in entries.flatten() { + let path = entry.path(); + let mut element = String::new(); + if append_multistatus_response(&mut element, &base_dir, &path, &mode) + .is_ok() + { + let _ = tx.blocking_send(element.into_bytes()); + } + if entry.file_type().map(|f| f.is_dir()).unwrap_or(false) { + stack.push(path); + } + } + } } + let _ = tx.blocking_send(b"\n".to_vec()); + }); + + return Ok(Response { + status_code: 207, + status_text: "Multi-Status".to_string(), + headers, + body: ResponseBody::AsyncStream(rx), + }); + } + + let mut resources = vec![target_path.clone()]; + if target_path.is_dir() && depth == DavDepth::One { + let mut entries = Vec::new(); + for entry in std::fs::read_dir(&target_path)? { + let entry = entry?; + entries.push(entry.path()); } + entries.sort_by(|a, b| a.to_string_lossy().cmp(&b.to_string_lossy())); + resources.extend(entries); } let mut body = String::from( - r#" - -"#, + "\n\n", ); for resource in resources { @@ -146,14 +188,6 @@ fn handle_propfind(request: &Request, base_dir: &Path) -> Result\n"); - let mut headers = HashMap::new(); - headers.insert( - "Content-Type".to_string(), - "application/xml; charset=utf-8".to_string(), - ); - headers.insert("DAV".to_string(), "1,2".to_string()); - headers.insert("Allow".to_string(), allow_header_value().to_string()); - Ok(Response { status_code: 207, status_text: "Multi-Status".to_string(), @@ -163,7 +197,7 @@ fn handle_propfind(request: &Request, base_dir: &Path) -> Result Result { - let body = request_body_bytes(request)?; + let body = request_body_bytes_with_limit(request, 8 * 1024 * 1024)?; if body.is_empty() { return Ok(PropfindMode::AllProp); } @@ -349,26 +383,6 @@ fn parse_prop_name_from_token( }) } -fn collect_recursive_resources(root: &Path, resources: &mut Vec) -> Result<(), AppError> { - let mut stack = vec![root.to_path_buf()]; - while let Some(current_dir) = stack.pop() { - let mut children = Vec::new(); - for entry in std::fs::read_dir(¤t_dir)? { - let entry = entry?; - children.push((entry.path(), entry.file_type()?)); - } - children.sort_by(|(a, _), (b, _)| a.to_string_lossy().cmp(&b.to_string_lossy())); - - for (entry_path, file_type) in children { - resources.push(entry_path.clone()); - if file_type.is_dir() { - stack.push(entry_path); - } - } - } - Ok(()) -} - fn handle_mkcol(request: &Request, base_dir: &Path) -> Result { let _op_guard = op_guard() .lock() @@ -386,7 +400,12 @@ fn handle_mkcol(request: &Request, base_dir: &Path) -> Result Result } let existed = target_path.exists(); - let body_bytes = request_body_bytes(request)?; - let mut file = std::fs::File::create(&target_path)?; - file.write_all(&body_bytes)?; - file.sync_all()?; + if let Some(body) = &request.body { + match body { + RequestBody::Memory(data) => { + let mut file = std::fs::File::create(&target_path)?; + file.write_all(data)?; + file.sync_all()?; + } + RequestBody::File { path, .. } => { + if std::fs::rename(path, &target_path).is_err() { + std::fs::copy(path, &target_path)?; + } + } + } + } else { + std::fs::File::create(&target_path)?; + } if existed { Ok(status_response(204, "No Content")) @@ -659,7 +690,7 @@ fn handle_lock(request: &Request, base_dir: &Path) -> Result let target_path = resolve_request_path(base_dir, &request.path)?; let key = lock_key(&target_path); cleanup_expired_locks(); - let lock_body = request_body_bytes(request)?; + let lock_body = request_body_bytes_with_limit(request, 8 * 1024 * 1024)?; debug!("WebDAV LOCK target={}", target_path.display()); let target_is_collection = target_path.is_dir() || request.path.ends_with('/'); @@ -902,10 +933,20 @@ fn copy_path_recursive(source: &Path, destination: &Path) -> Result<(), AppError } } -fn request_body_bytes(request: &Request) -> Result, AppError> { +fn request_body_bytes_with_limit(request: &Request, max_size: usize) -> Result, AppError> { match &request.body { - Some(RequestBody::Memory(data)) => Ok(data.clone()), - Some(RequestBody::File { path, .. }) => std::fs::read(path).map_err(AppError::from), + Some(RequestBody::Memory(data)) => { + if data.len() > max_size { + return Err(AppError::PayloadTooLarge(max_size as u64)); + } + Ok(data.clone()) + } + Some(RequestBody::File { path, size }) => { + if *size as usize > max_size { + return Err(AppError::PayloadTooLarge(max_size as u64)); + } + std::fs::read(path).map_err(AppError::from) + } None => Ok(Vec::new()), } } @@ -1055,7 +1096,7 @@ fn handle_proppatch(request: &Request, base_dir: &Path) -> Result Date: Tue, 28 Apr 2026 02:40:13 +0530 Subject: [PATCH 16/17] irondrop: v2.7.2 it is --- Cargo.toml | 2 +- README.md | 12 +++++++----- config/irondrop.ini | 11 +++++++++++ doc/ARCHITECTURE.md | 8 ++++---- doc/TESTING_DOCUMENTATION.md | 6 +++--- doc/WEBDAV_IMPLEMENTATION.md | 5 ++++- 6 files changed, 30 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8c05e6f..b4d02e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "irondrop" -version = "2.7.1" +version = "2.7.2" edition = "2024" license = "MIT" description = "Drop files, not dependencies - a well tested fully featured & battle-ready server in a single Rust binary with support for indexing through 10M files." diff --git a/README.md b/README.md index 5d72074..91de2ef 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,9 @@ For comprehensive documentation, see our [Complete Documentation Index](./doc/RE ## Version notes -Recent releases include direct-to-disk uploads, an ultra-compact search mode, and a `/monitor` page with a JSON endpoint. +Recent releases include: +- **v2.7.2**: Major memory optimizations: WebDAV `Transfer-Encoding: chunked` background thread streaming (near-zero RAM usage for large tree traversals), Web UI 1,000-file pagination with lazy metadata loading, and a massive ~340MB reduction in default search index bootstrap footprint. +- **v2.7.1**: Direct-to-disk uploads, ultra-compact search mode, and a `/monitor` page with a JSON endpoint. ## Documentation @@ -306,10 +308,10 @@ IronDrop has extensive documentation covering its architecture, API, and feature ## Testing -IronDrop is rigorously tested with **272 automated tests**: +IronDrop is rigorously tested with **325 automated tests**: -- **48 unit tests** in core source modules -- **224 integration/system tests** across **28** test files (including WebDAV RFC suites) +- **44 unit tests** in core source modules +- **281 integration/system tests** across **30** test files (including WebDAV RFC suites) ### Coverage Areas - HTTP parser/request handling, auth, rate limiting, monitoring, uploads, search, and utilities @@ -341,7 +343,7 @@ IronDrop is licensed under the [MIT License](./LICENSE).

Made with ❤️ and 🦀 in Rust
- Dependency-free core engine paths • Production ready • Battle tested with 272 automated tests + Dependency-free core engine paths • Production ready • Battle tested with 325 automated tests

⭐ Star us on GitHub diff --git a/config/irondrop.ini b/config/irondrop.ini index fc6b8dc..06b8e64 100644 --- a/config/irondrop.ini +++ b/config/irondrop.ini @@ -76,6 +76,13 @@ chunk_size = 2048 # • For uploads, IronDrop needs write permissions too directory = . +# 🛤️ Base Path - Run IronDrop under a URL sub-path (for reverse proxies) +# • Default: empty (serves from root /) +# • Useful when running behind Nginx: e.g., location /webstorage/ +# • Example: /webstorage +# • Prevents the need for Nginx sub_filter URL rewriting hacks +# base_path = /webstorage + # =============================================================================== # ⬆️ UPLOAD SYSTEM CONFIGURATION # =============================================================================== @@ -282,6 +289,9 @@ detailed = true # cert = /etc/letsencrypt/live/files.example.com/fullchain.pem # key = /etc/letsencrypt/live/files.example.com/privkey.pem # +# # Example of running under a sub-path proxy: +# # base_path = /company-files +# # [upload] # enable_upload = true # max_upload_size = 100MB @@ -329,6 +339,7 @@ detailed = true # ✅ 7. Add authentication for network access (set username/password) # ✅ 8. Configure allowed file extensions for security # ✅ 8b. (Optional) Add SSL cert and key for HTTPS +# ✅ 8c. (Optional) Set base_path if deploying behind a proxy sub-path # ✅ 9. Run: irondrop --config-file my-config.ini # ✅ 10. Open browser: http://localhost:8080 (or your chosen port) # ✅ 11. Enjoy blazing-fast file sharing! 🚀 diff --git a/doc/ARCHITECTURE.md b/doc/ARCHITECTURE.md index ca4bacc..7130b9c 100644 --- a/doc/ARCHITECTURE.md +++ b/doc/ARCHITECTURE.md @@ -1,4 +1,4 @@ -# IronDrop Architecture Documentation v2.7.1 +# IronDrop Architecture Documentation v2.7.2 ## Overview @@ -136,7 +136,7 @@ src/ ├── handlers.rs # Internal route handlers ├── middleware.rs # Authentication middleware ├── templates.rs # Template engine with embedded assets -├── fs.rs # File system operations and directory listing +├── fs.rs # File system operations, lazy metadata, and UI directory pagination ├── response.rs # Response types and error response helpers ├── upload.rs # Upload handling + validation ├── multipart.rs # Multipart form parsing @@ -377,7 +377,7 @@ The test suite verifies correct mode selection (memory vs disk), cleanup behavio | Operation | Typical Latency | Notes | |-----------|----------------|-------| | Static Assets | <0.5ms | CSS/JS with caching headers | -| Directory Listing | <2ms | Template rendering with file sorting | +| Directory Listing | <2ms | Template rendering with paginated lazy-loaded file metadata | | Health Checks | <0.1ms | JSON status endpoints | | File Downloads | Variable | Depends on file size and network | | File Uploads | Variable | Includes validation and atomic writing | @@ -500,4 +500,4 @@ pub enum AppError { 4. **CDN Integration**: Edge caching and global distribution 5. **Database Caching**: Redis integration for session management -This architecture documentation reflects the current state of IronDrop v2.7.1 and serves as a foundation for understanding the system's design principles, implementation details, and operational characteristics. +This architecture documentation reflects the current state of IronDrop v2.7.2 and serves as a foundation for understanding the system's design principles, implementation details, and operational characteristics. diff --git a/doc/TESTING_DOCUMENTATION.md b/doc/TESTING_DOCUMENTATION.md index 849aaf6..e307f5f 100644 --- a/doc/TESTING_DOCUMENTATION.md +++ b/doc/TESTING_DOCUMENTATION.md @@ -1,6 +1,6 @@ # IronDrop Testing Documentation -Version 2.7.1 - Test Suite Overview +Version 2.7.2 - Test Suite Overview ## Overview @@ -376,7 +376,7 @@ fn test_new_feature() { - Code formatting validation - Documentation completeness -## Recent Improvements (v2.7.1) +## Recent Improvements (v2.7.2) ### Critical Fixes and Enhancements @@ -434,6 +434,6 @@ fn test_new_feature() { --- -*This document is part of the IronDrop v2.7.1 documentation suite. The test suite is continuously evolving to ensure comprehensive coverage and reliability.* +*This document is part of the IronDrop v2.7.2 documentation suite. The test suite is continuously evolving to ensure comprehensive coverage and reliability.* Return to documentation index: [./README.md](./README.md) diff --git a/doc/WEBDAV_IMPLEMENTATION.md b/doc/WEBDAV_IMPLEMENTATION.md index 7e83a6b..ff7d851 100644 --- a/doc/WEBDAV_IMPLEMENTATION.md +++ b/doc/WEBDAV_IMPLEMENTATION.md @@ -1,6 +1,6 @@ # WebDAV Implementation Guide -Version: 2.7.1 +Version: 2.7.2 This guide explains how the WebDAV implementation works in plain language. It focuses on request flow, lock behavior, and the core RFC 4918 semantics implemented in `src/webdav.rs`. @@ -72,6 +72,9 @@ and depth handling: - `Depth: 1` returns the target plus immediate children - `Depth: infinity` recursively returns all descendants of a collection +> [!NOTE] +> For `Depth: infinity`, IronDrop traverses the tree dynamically using a background thread and streams the XML directly to the client using `Transfer-Encoding: chunked`. This ensures memory usage remains near-zero even for directories with millions of files. + ```mermaid flowchart TD A[PROPFIND request] --> B[Parse Depth] From 0d8262ef9fe720fcbf15b7da2082db18d4d67375 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Tue, 28 Apr 2026 03:07:42 +0530 Subject: [PATCH 17/17] irondrop: increase memory threshold --- src/server.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.rs b/src/server.rs index cad83b9..3bce405 100644 --- a/src/server.rs +++ b/src/server.rs @@ -478,7 +478,7 @@ impl ServerStats { } } - /// Check if memory pressure is detected (memory usage > 15MB) + /// Check if memory pressure is detected (memory usage > 150MB) /// This triggers aggressive cleanup in rate limiter and other components pub fn check_memory_pressure(&self, rate_limiter: Option<&RateLimiter>) -> bool { trace!("Checking memory pressure"); @@ -486,11 +486,11 @@ impl ServerStats { if available { if let Some(memory) = current_memory { - // Trigger cleanup if memory exceeds 15MB (baseline is ~4MB) + // Trigger cleanup if memory exceeds 150MB (baseline is ~30MB) let memory_mb = memory / (1024 * 1024); trace!("Current memory usage: {}MB", memory_mb); - if memory_mb > 15 { + if memory_mb > 150 { warn!("Memory pressure detected: {}MB usage", memory_mb); // Trigger rate limiter cleanup if provided