extend archive index with lists, use in source view for tree

crates/bin/docs_rs_web/src/handlers/rustdoc.rs

@@ -1132,7 +1132,7 @@ mod test {
     ) -> Result<String, anyhow::Error> {
         try_latest_version_redirect(krate, path, web, config)
             .await?
-            .with_context(|| anyhow::anyhow!("no redirect found for {}", path))
+            .with_context(|| anyhow!("no redirect found for {}", path))
     }
 
     #[test_case(true)]

crates/bin/docs_rs_web/src/handlers/source.rs

@@ -18,42 +18,40 @@ use askama::Template;
 use axum::{Extension, response::IntoResponse};
 use axum_extra::{TypedHeader, headers::HeaderMapExt};
 use docs_rs_headers::{CanonicalUrl, IfNoneMatch};
-use docs_rs_storage::{AsyncStorage, PathNotFoundError};
+use docs_rs_storage::{AsyncStorage, FolderEntry, PathNotFoundError, source_archive_path};
 use docs_rs_types::{BuildId, KrateName, ReqVersion, Version};
-use mime::Mime;
-use std::{cmp::Ordering, sync::Arc};
+use futures_util::TryStreamExt as _;
+use std::sync::Arc;
 use tracing::instrument;
 
-/// A source file's name and mime type
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd)]
-struct File {
-    name: String,
-    mime: String,
-}
-
-impl File {
-    fn from_path_and_mime(path: &str, mime: &Mime) -> File {
-        let (name, mime) = if let Some((dir, _)) = path.split_once('/') {
-            (dir, "dir")
-        } else {
-            (path, mime.as_ref())
-        };
-
-        Self {
-            name: name.to_owned(),
-            mime: mime.to_owned(),
-        }
-    }
-}
-
 /// A list of source files
 #[derive(Debug, Clone, PartialEq, Default)]
 struct FileList {
-    files: Vec<File>,
+    files: Vec<FolderEntry>,
 }
 
 impl FileList {
-    /// Gets FileList from a request path
+    async fn from_archive_index(
+        storage: &AsyncStorage,
+        name: &KrateName,
+        version: &Version,
+        latest_build_id: Option<BuildId>,
+        folder: Option<&str>,
+    ) -> Result<FileList> {
+        let mut files = storage
+            .find_archive_index(&source_archive_path(name, version), latest_build_id)
+            .await?
+            .folder_contents(folder)
+            .try_filter(|entry| std::future::ready(entry.name() != ".cargo-ok"))
+            .try_collect::<Vec<_>>()
+            .await?;
+
+        files.sort_unstable();
+
+        Ok(Self { files })
+    }
+
+    /// Gets legacy FileList from a request path
     ///
     /// All paths stored in database have this format:
     ///
@@ -104,11 +102,6 @@ impl FileList {
 
             for file in files {
                 if let Some(file) = file.as_array() {
-                    let mime: Mime = file[0]
-                        .as_str()
-                        .unwrap()
-                        .parse()
-                        .unwrap_or(mime::APPLICATION_OCTET_STREAM);
                     let path = file[1].as_str().unwrap();
 
                     // skip .cargo-ok generated by cargo
@@ -118,7 +111,7 @@ impl FileList {
 
                     // look only files for req_path
                     if let Some(path) = path.strip_prefix(folder) {
-                        let file = File::from_path_and_mime(path, &mime);
+                        let file = FolderEntry::from_path(path);
 
                         // avoid adding duplicates, a directory may occur more than once
                         if !file_list.contains(&file) {
@@ -132,16 +125,7 @@ impl FileList {
                 return Ok(None);
             }
 
-            file_list.sort_by(|a, b| {
-                // directories must be listed first
-                if a.mime == "dir" && b.mime != "dir" {
-                    Ordering::Less
-                } else if a.mime != "dir" && b.mime == "dir" {
-                    Ordering::Greater
-                } else {
-                    a.name.to_lowercase().cmp(&b.name.to_lowercase())
-                }
-            });
+            file_list.sort_unstable();
 
             Ok(Some(FileList { files: file_list }))
         } else {
@@ -157,7 +141,7 @@ struct SourcePage {
     file_list: FileList,
     metadata: MetaData,
     show_parent_link: bool,
-    file: Option<File>,
+    file: Option<FolderEntry>,
     file_content: Option<String>,
     canonical_url: CanonicalUrl,
     is_file_too_large: bool,
@@ -246,16 +230,31 @@ pub(crate) async fn source_browser_handler(
 
     let inner_path = params.inner_path();
 
-    let current_folder = if let Some(last_slash_pos) = inner_path.rfind('/') {
-        &inner_path[..last_slash_pos + 1]
-    } else {
-        ""
-    };
-    let show_parent_link = !current_folder.is_empty();
+    let current_folder = inner_path
+        .rfind('/')
+        .map(|last_slash_pos| &inner_path[..last_slash_pos + 1]);
 
-    let file_list = FileList::from_path(&mut conn, params.name(), version, current_folder)
+    let show_parent_link = current_folder.is_some();
+
+    let file_list = if row.archive_storage {
+        FileList::from_archive_index(
+            &storage,
+            params.name(),
+            version,
+            row.latest_build_id,
+            current_folder,
+        )
+        .await?
+    } else {
+        FileList::from_path(
+            &mut conn,
+            params.name(),
+            version,
+            current_folder.unwrap_or(""),
+        )
         .await?
-        .unwrap_or_default();
+        .unwrap_or_default()
+    };
 
     let metadata = MetaData::from_crate(
         &mut conn,
@@ -327,7 +326,7 @@ pub(crate) async fn source_browser_handler(
                         .map(|(_, path)| path)
                         .unwrap_or(&blob.path);
                     (
-                        Some(File::from_path_and_mime(path, &blob.mime)),
+                        Some(FolderEntry::from_path(path)),
                         Some(String::from_utf8_lossy(&blob.content).to_string()),
                     )
                 }

crates/bin/docs_rs_web/src/utils/mod.rs

@@ -3,10 +3,42 @@ pub(crate) mod html_rewrite;
 pub(crate) mod licenses;
 pub(crate) mod markdown;
 
+use crate::{
+    icons::{
+        IconFile, IconFileLines, IconFolderOpen, IconGitAlt, IconLock, IconMarkdown, IconRust,
+    },
+    page::templates::{RenderBrands, RenderRegular, RenderSolid},
+};
 use anyhow::Result;
+use askama::filters::Safe;
 use chrono::{DateTime, NaiveDate, Utc};
+use docs_rs_mimes as mimes;
+use docs_rs_storage::FolderEntry;
 use docs_rs_utils::rustc_version::parse_rustc_date;
 
+pub fn folder_entry_icon(entry: &FolderEntry) -> Safe<String> {
+    if entry.is_dir() {
+        return IconFolderOpen.render_regular(false, false, "");
+    }
+
+    let mime = entry.mime().expect("files always have mime");
+    let name = entry.name();
+
+    if *mime == *mimes::TEXT_RUST {
+        IconRust.render_brands(false, false, "")
+    } else if *mime == mime::TEXT_PLAIN && name == "Cargo.lock" {
+        IconLock.render_solid(false, false, "")
+    } else if *mime == *mimes::TEXT_MARKDOWN {
+        IconMarkdown.render_brands(false, false, "")
+    } else if *mime == mime::TEXT_PLAIN && name == ".gitignore" {
+        IconGitAlt.render_brands(false, false, "")
+    } else if *mime == mime::TEXT_PLAIN || mime.type_() == "text" {
+        IconFileLines.render_regular(false, false, "")
+    } else {
+        IconFile.render_regular(false, false, "")
+    }
+}
+
 /// Picks the correct "rustdoc.css" static file depending on which rustdoc version was used to
 /// generate this version of this crate.
 pub(crate) fn get_correct_docsrs_style_file(version: &str) -> Result<String> {

crates/bin/docs_rs_web/templates/crate/source.html

@@ -44,55 +44,9 @@
                                     Show a link to the file with a fancy icon. If the file is a directory,
                                     `/` is appended to show the contents of the folder
                                 #}
-                                <a href="./{{ file.name }}{% if file.mime == "dir" %}/{% endif %}" class="pure-menu-link">
-                                    {# Directories #}
-                                    {%- if file.mime == "dir" -%}
-                                        {{ crate::icons::IconFolderOpen.render_regular(false, false, "") }}
-
-                                    {# Rust files #}
-                                    {%- elif file.mime == "text/rust" -%}
-                                        {{ crate::icons::IconRust.render_brands(false, false, "") }}
-
-                                    {# Cargo.lock #}
-                                    {%- elif file.mime == "text/plain" && file.name == "Cargo.lock" -%}
-                                        {{ crate::icons::IconLock.render_solid(false, false, "") }}
-
-                                    {# Markdown files #}
-                                    {% elif file.mime == "text/markdown" %}
-                                        {{ crate::icons::IconMarkdown.render_brands(false, false, "") }}
-
-                                    {# .gitignore #}
-                                    {% elif file.mime == "text/plain" && file.name == ".gitignore" %}
-                                        {{ crate::icons::IconGitAlt.render_brands(false, false, "") }}
-
-                                    {#
-                                        More ideas
-                                            FontAwesome v5:
-                                                ".application/x-bzip"
-                                               |"application/gzip"
-                                               |"application/x-bzip2"
-                                               |"application/vnd.rar"
-                                               |"application/x-tar"
-                                               |"application/zip"
-                                               |"application/x-7z-compressed" => https://fontawesome.com/icons/file-archive
-                                                "text/javascript" => https://fontawesome.com/icons/js
-                                                "application/java-archive" => https://fontawesome.com/icons/java
-                                            DevOpticons (https://github.com/file-icons/DevOpicons):
-                                                "text/rust" => https://github.com/file-icons/DevOpicons/blob/master/charmap.md#Rust
-                                                "text/css" => https://github.com/file-icons/DevOpicons/blob/master/charmap.md#CSS3,%20Full
-                                                "text/html" => https://github.com/file-icons/DevOpicons/blob/master/charmap.md#HTML5
-                                    #}
-
-                                    {# Text files or files which mime starts with `text` #}
-                                    {%- elif file.mime == "text/plain" || file.mime.split('/').next() == Some("text") -%}
-                                        {{ crate::icons::IconFileLines.render_regular(false, false, "") }}
-
-                                        {# Binary files and any unrecognized types #}
-                                    {% else -%}
-                                        {{ crate::icons::IconFile.render_regular(false, false, "") }}
-                                    {%- endif -%}
-
-                                    <span class="text">{{ file.name }}</span>
+                                <a href="./{{ file.name() }}{% if file.is_dir() %}/{% endif %}" class="pure-menu-link">
+                                    {{ crate::utils::folder_entry_icon(file) }}
+                                    <span class="text">{{ file.name() }}</span>
                                 </a>
                             </li>
                         {%- endfor -%}
@@ -115,7 +69,7 @@
             {%- if let Some(file_content) = file_content -%}
                 {% decl file_name %}
                 {% if let Some(file) = file %}
-                    {% set file_name = file.name.as_str() %}
+                    {% set file_name = file.name() %}
                 {% else %}
                     {% set file_name = "" %}
                 {% endif %}

crates/lib/docs_rs_database/src/releases.rs

@@ -25,7 +25,7 @@ use tracing::{debug, error, info, instrument};
 /// NOTE: `source_files` refers to the files originally in the crate,
 /// not the files generated by rustdoc.
 #[allow(clippy::too_many_arguments)]
-#[instrument(skip(conn, compression_algorithms))]
+#[instrument(skip(conn, compression_algorithms, source_files))]
 pub async fn finish_release(
     conn: &mut sqlx::PgConnection,
     crate_id: CrateId,