Optz

src/fs.rs

@@ -5,10 +5,31 @@ use crate::error::AppError;
 use crate::templates::TemplateEngine;
 use crate::utils::is_hidden_file;
 use log::{debug, trace};
+use std::cmp::Ordering;
+use std::collections::BinaryHeap;
 use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::time::SystemTime;
 
+#[derive(Eq, PartialEq)]
+struct ListingEntry {
+    path: PathBuf,
+    file_name: String,
+    is_dir: bool,
+}
+
+impl Ord for ListingEntry {
+    fn cmp(&self, other: &Self) -> Ordering {
+        compare_listing_entries(self, other)
+    }
+}
+
+impl PartialOrd for ListingEntry {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 /// Enhanced directory listing using modular templates - dark mode only
 pub fn generate_directory_listing(
     path: &Path,
@@ -19,9 +40,13 @@ pub fn generate_directory_listing(
     debug!("Generating directory listing for: '{}'", path.display());
     trace!("Request path: '{}'", request_path);
 
-    let mut entries = Vec::new();
+    let limit = 1000;
+    let page_size = page.max(1);
+    let selection_size = page_size.saturating_mul(limit);
+    let mut selected_entries = BinaryHeap::with_capacity(selection_size);
+    let mut total_count = 0usize;
 
-    // Collect and sort entries using lazy metadata lookup (super fast, low memory)
+    // Keep only the best entries for the requested page instead of sorting the entire directory.
     trace!("Reading directory entries from: {}", path.display());
     for entry in fs::read_dir(path)? {
         let entry = entry?;
@@ -32,39 +57,54 @@ pub fn generate_directory_listing(
             continue;
         }
 
-        entries.push((entry.path(), file_name, file_type.is_dir()));
-    }
-
-    debug!("Found {} entries, sorting...", entries.len());
-    // Sort: directories first, then alphabetically
-    entries.sort_by(|a, b| {
-        let a_is_dir = a.2;
-        let b_is_dir = b.2;
+        total_count += 1;
+        let listing_entry = ListingEntry {
+            path: entry.path(),
+            file_name,
+            is_dir: file_type.is_dir(),
+        };
 
-        match (a_is_dir, b_is_dir) {
-            (true, false) => std::cmp::Ordering::Less,
-            (false, true) => std::cmp::Ordering::Greater,
-            _ => cmp_case_insensitive_ascii(&a.1, &b.1),
+        if selection_size == 0 {
+            continue;
+        }
+        if selected_entries.len() < selection_size {
+            selected_entries.push(listing_entry);
+            continue;
+        }
+        if let Some(current_max) = selected_entries.peek()
+            && compare_listing_entries(&listing_entry, current_max) == Ordering::Less
+        {
+            selected_entries.pop();
+            selected_entries.push(listing_entry);
         }
-    });
+    }
+
+    let mut entries = selected_entries.into_vec();
+    entries.sort_unstable_by(compare_listing_entries);
 
     let display_path = if request_path.is_empty() || request_path == "/" {
         "/"
     } else {
         request_path
     };
 
-    debug!("Preparing {} entries for template rendering", entries.len());
-    let total_count = entries.len();
-    let limit = 1000;
+    debug!(
+        "Preparing {} selected entries for template rendering",
+        entries.len()
+    );
     let total_pages = total_count.div_ceil(limit);
     let safe_page = page.max(1).min(total_pages.max(1));
     let offset = (safe_page - 1) * limit;
 
     let page_entries: Vec<_> = entries.into_iter().skip(offset).take(limit).collect();
     let mut template_entries = Vec::with_capacity(page_entries.len());
 
-    for (entry_path, file_name, is_dir) in page_entries {
+    for entry in page_entries {
+        let ListingEntry {
+            path: entry_path,
+            file_name,
+            is_dir,
+        } = entry;
         let link_name = if is_dir {
             format!("{file_name}/")
         } else {
@@ -152,6 +192,48 @@ fn cmp_case_insensitive_ascii(a: &str, b: &str) -> std::cmp::Ordering {
     }
 }
 
+fn compare_listing_entries(a: &ListingEntry, b: &ListingEntry) -> Ordering {
+    match (a.is_dir, b.is_dir) {
+        (true, false) => Ordering::Less,
+        (false, true) => Ordering::Greater,
+        _ => cmp_case_insensitive_ascii(&a.file_name, &b.file_name),
+    }
+}
+
+#[cfg(test)]
+mod perf_tests {
+    use super::*;
+    use tempfile::tempdir;
+
+    #[test]
+    #[ignore]
+    fn perf_directory_listing_first_page() {
+        let temp_dir = tempdir().unwrap();
+
+        for dir_idx in 0..200 {
+            std::fs::create_dir_all(temp_dir.path().join(format!("dir_{dir_idx:03}"))).unwrap();
+        }
+        for file_idx in 0..9_800 {
+            std::fs::write(
+                temp_dir.path().join(format!("file_{file_idx:05}.txt")),
+                b"irondrop",
+            )
+            .unwrap();
+        }
+
+        let start = std::time::Instant::now();
+        let html = generate_directory_listing(temp_dir.path(), "/", None, 1).unwrap();
+        let elapsed_ms = start.elapsed().as_millis();
+
+        assert!(html.contains("file_"));
+        println!(
+            "PERF directory_listing entries=10000 page=1 render_ms={} html_bytes={}",
+            elapsed_ms,
+            html.len()
+        );
+    }
+}
+
 /// Format Unix timestamp to human-readable date
 fn format_timestamp(timestamp: u64) -> String {
     // Simple date formatting without external dependencies

src/http.rs

@@ -82,6 +82,59 @@ pub enum ResponseBody {
     AsyncStream(tokio::sync::mpsc::Receiver<Vec<u8>>),
 }
 
+struct PendingBuffer {
+    data: Vec<u8>,
+    start: usize,
+}
+
+impl PendingBuffer {
+    fn new(data: Vec<u8>) -> Self {
+        Self { data, start: 0 }
+    }
+
+    fn len(&self) -> usize {
+        self.data.len().saturating_sub(self.start)
+    }
+
+    fn as_slice(&self) -> &[u8] {
+        &self.data[self.start..]
+    }
+
+    fn extend_from_slice(&mut self, bytes: &[u8]) {
+        if self.start > 0 && (self.start >= 8192 || self.start * 2 >= self.data.len().max(1)) {
+            self.compact();
+        }
+        self.data.extend_from_slice(bytes);
+    }
+
+    fn take(&mut self, count: usize) -> Vec<u8> {
+        let end = self.start + count;
+        let chunk = self.data[self.start..end].to_vec();
+        self.start = end;
+        if self.start == self.data.len() {
+            self.data.clear();
+            self.start = 0;
+        }
+        chunk
+    }
+
+    fn discard(&mut self, count: usize) {
+        self.start += count;
+        if self.start == self.data.len() {
+            self.data.clear();
+            self.start = 0;
+        }
+    }
+
+    fn compact(&mut self) {
+        if self.start == 0 {
+            return;
+        }
+        self.data.drain(..self.start);
+        self.start = 0;
+    }
+}
+
 impl Request {
     /// Validates if the given method is a valid HTTP method
     fn is_valid_http_method(method: &str) -> bool {
@@ -659,13 +712,14 @@ where
 
 async fn read_chunked_body_async<S>(
     stream: &mut S,
-    mut pending: Vec<u8>,
+    remaining_bytes: Vec<u8>,
 ) -> Result<RequestBody, AppError>
 where
     S: tokio::io::AsyncRead + Unpin,
 {
     const CHUNK_LINE_LIMIT: usize = 8 * 1024;
 
+    let mut pending = PendingBuffer::new(remaining_bytes);
     let mut total_size: usize = 0;
     let mut memory_body: Vec<u8> = Vec::new();
     let mut file_sink: Option<(PathBuf, tokio::fs::File)> = None;
@@ -765,16 +819,16 @@ async fn create_temp_body_file_async() -> Result<(PathBuf, tokio::fs::File), App
 
 async fn read_crlf_line_async<S>(
     stream: &mut S,
-    pending: &mut Vec<u8>,
+    pending: &mut PendingBuffer,
     max_line_len: usize,
 ) -> Result<Vec<u8>, AppError>
 where
     S: tokio::io::AsyncRead + Unpin,
 {
     loop {
-        if let Some(pos) = pending.windows(2).position(|w| w == b"\r\n") {
-            let line = pending[..pos].to_vec();
-            pending.drain(0..pos + 2);
+        if let Some(pos) = pending.as_slice().windows(2).position(|w| w == b"\r\n") {
+            let line = pending.take(pos);
+            pending.discard(2);
             return Ok(line);
         }
 
@@ -793,7 +847,7 @@ where
 
 async fn read_exact_from_buffer_async<S>(
     stream: &mut S,
-    pending: &mut Vec<u8>,
+    pending: &mut PendingBuffer,
     count: usize,
 ) -> Result<Vec<u8>, AppError>
 where
@@ -807,12 +861,12 @@ where
         }
         pending.extend_from_slice(&buffer[..n]);
     }
-    Ok(pending.drain(0..count).collect())
+    Ok(pending.take(count))
 }
 
 async fn consume_expected_crlf_async<S>(
     stream: &mut S,
-    pending: &mut Vec<u8>,
+    pending: &mut PendingBuffer,
 ) -> Result<(), AppError>
 where
     S: tokio::io::AsyncRead + Unpin,
@@ -826,7 +880,7 @@ where
 
 async fn consume_chunked_trailers_async<S>(
     stream: &mut S,
-    pending: &mut Vec<u8>,
+    pending: &mut PendingBuffer,
 ) -> Result<(), AppError>
 where
     S: tokio::io::AsyncRead + Unpin,
@@ -954,3 +1008,45 @@ where
 
     Ok((temp_path, total_written as u64))
 }
+
+#[cfg(test)]
+mod perf_tests {
+    use super::*;
+    use tokio::io::{AsyncWriteExt, duplex};
+
+    #[tokio::test]
+    #[ignore]
+    async fn perf_chunked_body_parsing() {
+        let chunk = vec![b'a'; 64 * 1024];
+        let mut payload = Vec::new();
+        for _ in 0..128 {
+            payload.extend_from_slice(format!("{:X}\r\n", chunk.len()).as_bytes());
+            payload.extend_from_slice(&chunk);
+            payload.extend_from_slice(b"\r\n");
+        }
+        payload.extend_from_slice(b"0\r\n\r\n");
+
+        let (mut writer, mut reader) = duplex(payload.len() + 1024);
+        writer.write_all(&payload).await.unwrap();
+        drop(writer);
+
+        let start = Instant::now();
+        let body = read_chunked_body_async(&mut reader, Vec::new())
+            .await
+            .unwrap();
+        let elapsed_ms = start.elapsed().as_millis();
+
+        match body {
+            RequestBody::File { size, .. } => {
+                assert_eq!(size, (64 * 1024 * 128) as u64);
+            }
+            RequestBody::Memory(data) => panic!("expected file-backed body, got {}", data.len()),
+        }
+
+        println!(
+            "PERF chunked_body_parse size_bytes={} elapsed_ms={}",
+            64 * 1024 * 128,
+            elapsed_ms
+        );
+    }
+}