Add the imgproxy delivery layer

Cargo.toml

@@ -51,6 +51,9 @@ lazy_static = "1.5.0"
 governor = "0.10.4"
 bytes = "1.12.1"
 metrics = "0.24.6"
+# Already in the tree via hyper and axum-extra; naming it directly costs no
+# extra compilation and replaces a hand-rolled HTTP-date parser.
+httpdate = "1.0.3"
 
 [dev-dependencies]
 tempfile = "3.27.0"

src/app.rs

@@ -59,7 +59,7 @@ impl Imgforge {
         let cache = Cache::new(cache_config.clone()).await?;
         let metadata_cache = MetadataCache::new(cache_config).await?;
         let vips_app = Arc::new(init_vips()?);
-        let http_client = build_http_client(config.download_timeout)?;
+        let http_client = build_http_client(&config)?;
         let rate_limiter = build_rate_limiter(config.rate_limit_per_minute);
         let watermark_cache = OnceCell::new();
 
@@ -108,7 +108,11 @@ impl Imgforge {
         path: &str,
         bearer_token: Option<&str>,
     ) -> Result<crate::service::ProcessedImage, crate::service::ServiceError> {
-        let request = crate::service::ProcessRequest { path, bearer_token };
+        let request = crate::service::ProcessRequest {
+            path,
+            bearer_token,
+            hints: crate::negotiation::RequestHints::default(),
+        };
         crate::service::process_path(self.state.clone(), request).await
     }
 
@@ -123,7 +127,11 @@ impl Imgforge {
         path: &str,
         bearer_token: Option<&str>,
     ) -> Result<crate::service::ImageInfo, crate::service::ServiceError> {
-        let request = crate::service::ProcessRequest { path, bearer_token };
+        let request = crate::service::ProcessRequest {
+            path,
+            bearer_token,
+            hints: crate::negotiation::RequestHints::default(),
+        };
         crate::service::image_info(self.state.clone(), request).await
     }
 }
@@ -158,9 +166,40 @@ fn init_vips() -> Result<VipsApp, InitError> {
     VipsApp::new("imgforge", false).map_err(InitError::Libvips)
 }
 
-fn build_http_client(timeout_secs: u64) -> Result<reqwest::Client, reqwest::Error> {
-    let timeout = Duration::from_secs(timeout_secs);
-    reqwest::Client::builder().timeout(timeout).build()
+/// Builds the outbound HTTP client, including the redirect policy that holds
+/// every hop to the source allow list.
+///
+/// Public so integration tests exercise the real policy rather than a
+/// look-alike that would not catch a regression in it.
+pub fn build_http_client(config: &Config) -> Result<reqwest::Client, reqwest::Error> {
+    reqwest::Client::builder()
+        .timeout(Duration::from_secs(config.download_timeout))
+        .user_agent(config.user_agent.clone())
+        .redirect(redirect_policy(config))
+        .build()
+}
+
+/// Bounds the redirect chain, and holds it to the same allow list as the
+/// original URL.
+///
+/// Checking only the requested URL is checking the wrong thing: an allowed
+/// origin that redirects to `http://169.254.169.254/` would be followed
+/// straight past the restriction, which is the classic way an image proxy
+/// becomes an SSRF gadget. Every destination is revalidated.
+fn redirect_policy(config: &Config) -> reqwest::redirect::Policy {
+    let max_redirects = config.max_redirects;
+    let rules = config.source_rules.clone();
+
+    reqwest::redirect::Policy::custom(move |attempt| {
+        if attempt.previous().len() >= max_redirects {
+            return attempt.error("too many redirects");
+        }
+        if !rules.permits(attempt.url().as_str()) {
+            warn!("Refusing a redirect to a source outside IMGFORGE_ALLOWED_SOURCES");
+            return attempt.stop();
+        }
+        attempt.follow()
+    })
 }
 
 fn build_rate_limiter(limit_per_minute: Option<u32>) -> Option<RequestRateLimiter> {

src/caching/cache.rs

@@ -35,6 +35,28 @@ fn block_size_for_capacity(capacity: usize) -> usize {
 pub struct CachedImage {
     pub bytes: Bytes,
     pub content_type: &'static str,
+    /// The URL these bytes were fetched from, after any redirects.
+    ///
+    /// Kept so a hit can be checked against the allow list as it stands now.
+    /// The request's own URL is validated before the lookup, but a redirect can
+    /// have moved the actual source somewhere that is no longer permitted, and
+    /// an entry outlives the policy that admitted it.
+    pub source_url: String,
+    /// Where a `watermark_url` watermark was fetched from, after any redirects,
+    /// or empty when the entry composites none. Its pixels are in the bytes
+    /// above, so its source is rechecked on a hit exactly as the image's own.
+    pub watermark_source_url: String,
+    /// The entity tag of `bytes`, computed when the entry was stored so a hit
+    /// does not hash the whole body again on the async worker.
+    pub etag: String,
+    /// The origin's own `Cache-Control`, empty when it sent none.
+    ///
+    /// Kept so a hit under passthrough keeps saying what the origin said —
+    /// losing a `no-store` the moment the cache answered invited shared caches
+    /// to store exactly what the origin forbade.
+    pub origin_cache_control: String,
+    /// The origin's `Last-Modified`, for the same reason. Empty when absent.
+    pub origin_last_modified: String,
 }
 
 impl Code for CachedImage {
@@ -46,6 +68,26 @@ impl Code for CachedImage {
         let content_type_bytes = self.content_type.as_bytes();
         content_type_bytes.len().encode(writer)?;
         writer.write_all(content_type_bytes).map_err(FoyerError::io_error)?;
+
+        let source_bytes = self.source_url.as_bytes();
+        source_bytes.len().encode(writer)?;
+        writer.write_all(source_bytes).map_err(FoyerError::io_error)?;
+
+        let watermark_bytes = self.watermark_source_url.as_bytes();
+        watermark_bytes.len().encode(writer)?;
+        writer.write_all(watermark_bytes).map_err(FoyerError::io_error)?;
+
+        let etag_bytes = self.etag.as_bytes();
+        etag_bytes.len().encode(writer)?;
+        writer.write_all(etag_bytes).map_err(FoyerError::io_error)?;
+
+        let cache_control_bytes = self.origin_cache_control.as_bytes();
+        cache_control_bytes.len().encode(writer)?;
+        writer.write_all(cache_control_bytes).map_err(FoyerError::io_error)?;
+
+        let last_modified_bytes = self.origin_last_modified.as_bytes();
+        last_modified_bytes.len().encode(writer)?;
+        writer.write_all(last_modified_bytes).map_err(FoyerError::io_error)?;
         Ok(())
     }
 
@@ -61,14 +103,60 @@ impl Code for CachedImage {
             .map_err(|_| FoyerError::new(ErrorKind::Parse, "invalid utf8 in content type"))?;
         let content_type = format_to_content_type(content_str);
 
+        let source_len = usize::decode(reader)?;
+        let mut source_buf = vec![0u8; source_len];
+        reader.read_exact(&mut source_buf).map_err(FoyerError::io_error)?;
+        let source_url = String::from_utf8(source_buf)
+            .map_err(|_| FoyerError::new(ErrorKind::Parse, "invalid utf8 in source url"))?;
+
+        let watermark_len = usize::decode(reader)?;
+        let mut watermark_buf = vec![0u8; watermark_len];
+        reader.read_exact(&mut watermark_buf).map_err(FoyerError::io_error)?;
+        let watermark_source_url = String::from_utf8(watermark_buf)
+            .map_err(|_| FoyerError::new(ErrorKind::Parse, "invalid utf8 in watermark source url"))?;
+
+        let etag_len = usize::decode(reader)?;
+        let mut etag_buf = vec![0u8; etag_len];
+        reader.read_exact(&mut etag_buf).map_err(FoyerError::io_error)?;
+        let etag =
+            String::from_utf8(etag_buf).map_err(|_| FoyerError::new(ErrorKind::Parse, "invalid utf8 in etag"))?;
+
+        let cache_control_len = usize::decode(reader)?;
+        let mut cache_control_buf = vec![0u8; cache_control_len];
+        reader
+            .read_exact(&mut cache_control_buf)
+            .map_err(FoyerError::io_error)?;
+        let origin_cache_control = String::from_utf8(cache_control_buf)
+            .map_err(|_| FoyerError::new(ErrorKind::Parse, "invalid utf8 in origin cache control"))?;
+
+        let last_modified_len = usize::decode(reader)?;
+        let mut last_modified_buf = vec![0u8; last_modified_len];
+        reader
+            .read_exact(&mut last_modified_buf)
+            .map_err(FoyerError::io_error)?;
+        let origin_last_modified = String::from_utf8(last_modified_buf)
+            .map_err(|_| FoyerError::new(ErrorKind::Parse, "invalid utf8 in origin last modified"))?;
+
         Ok(CachedImage {
             bytes: Bytes::from(data),
             content_type,
+            source_url,
+            watermark_source_url,
+            etag,
+            origin_cache_control,
+            origin_last_modified,
         })
     }
 
     fn estimated_size(&self) -> usize {
-        self.bytes.len() + self.content_type.len() + std::mem::size_of::<usize>() * 2
+        self.bytes.len()
+            + self.content_type.len()
+            + self.source_url.len()
+            + self.watermark_source_url.len()
+            + self.etag.len()
+            + self.origin_cache_control.len()
+            + self.origin_last_modified.len()
+            + std::mem::size_of::<usize>() * 7
     }
 }
 
@@ -84,6 +172,10 @@ pub struct CachedMetadata {
     pub orientation: u32,
     /// Frames or pages the source carries; 1 for a still image.
     pub pages: u32,
+    /// The URL the description was read from, after any redirects, so a hit
+    /// can be rechecked against the allow list as it stands now — the same
+    /// reason `CachedImage` remembers its own.
+    pub source_url: String,
 }
 
 impl Code for CachedMetadata {
@@ -104,6 +196,10 @@ impl Code for CachedMetadata {
         self.has_alpha.encode(writer)?;
         self.orientation.encode(writer)?;
         self.pages.encode(writer)?;
+
+        let source_bytes = self.source_url.as_bytes();
+        source_bytes.len().encode(writer)?;
+        writer.write_all(source_bytes).map_err(FoyerError::io_error)?;
         Ok(())
     }
 
@@ -131,6 +227,12 @@ impl Code for CachedMetadata {
         let orientation = u32::decode(reader)?;
         let pages = u32::decode(reader)?;
 
+        let source_len = usize::decode(reader)?;
+        let mut source_buf = vec![0u8; source_len];
+        reader.read_exact(&mut source_buf).map_err(FoyerError::io_error)?;
+        let source_url = String::from_utf8(source_buf)
+            .map_err(|_| FoyerError::new(ErrorKind::Parse, "invalid utf8 in source url"))?;
+
         Ok(CachedMetadata {
             width,
             height,
@@ -141,15 +243,17 @@ impl Code for CachedMetadata {
             has_alpha,
             orientation,
             pages,
+            source_url,
         })
     }
 
     fn estimated_size(&self) -> usize {
         std::mem::size_of::<u32>() * 5
-            + std::mem::size_of::<usize>() * 2
+            + std::mem::size_of::<usize>() * 3
             + std::mem::size_of::<bool>()
             + self.format.len()
             + self.content_type.len()
+            + self.source_url.len()
     }
 }
 
@@ -394,6 +498,11 @@ mod tests {
         let value = CachedImage {
             bytes: Bytes::from(vec![1, 2, 3]),
             content_type: "image/jpeg",
+            source_url: "https://example.test/cached.png".to_string(),
+            watermark_source_url: "https://cdn.example.test/mark.png".to_string(),
+            etag: "\"abc123\"".to_string(),
+            origin_cache_control: "no-store".to_string(),
+            origin_last_modified: "Wed, 21 Oct 2015 07:28:00 GMT".to_string(),
         };
 
         cache.insert(key.clone(), value.clone()).unwrap();
@@ -412,10 +521,43 @@ mod tests {
         let value = CachedImage {
             bytes: Bytes::from(vec![1, 2, 3]),
             content_type: "image/jpeg",
+            source_url: "https://example.test/cached.png".to_string(),
+            watermark_source_url: "https://cdn.example.test/mark.png".to_string(),
+            etag: "\"abc123\"".to_string(),
+            origin_cache_control: "no-store".to_string(),
+            origin_last_modified: "Wed, 21 Oct 2015 07:28:00 GMT".to_string(),
         };
         cache.insert(key.clone(), value.clone()).unwrap();
         let retrieved = cache.get(&key).await.unwrap();
         assert_eq!(retrieved.bytes, value.bytes);
         assert_eq!(retrieved.content_type, value.content_type);
+        // Both provenance URLs have to survive the disk round trip, or the
+        // hit-time allow-list check silently checks nothing.
+        assert_eq!(retrieved.source_url, value.source_url);
+        assert_eq!(retrieved.watermark_source_url, value.watermark_source_url);
+        assert_eq!(retrieved.etag, value.etag);
+        assert_eq!(retrieved.origin_cache_control, value.origin_cache_control);
+        assert_eq!(retrieved.origin_last_modified, value.origin_last_modified);
+    }
+
+    #[test]
+    fn cached_metadata_round_trips_through_its_encoding() {
+        let metadata = CachedMetadata {
+            width: 800,
+            height: 600,
+            format: "jpeg".to_string(),
+            content_type: "image/jpeg".to_string(),
+            size_bytes: 1234,
+            channels: 3,
+            has_alpha: false,
+            orientation: 6,
+            pages: 4,
+            source_url: "https://cdn.example.test/real.jpg".to_string(),
+        };
+
+        let mut buf = Vec::new();
+        metadata.encode(&mut buf).unwrap();
+        let decoded = CachedMetadata::decode(&mut buf.as_slice()).unwrap();
+        assert_eq!(decoded, metadata);
     }
 }

src/config/env_vars.rs

@@ -63,3 +63,15 @@ where
         .map(Some)
         .map_err(|source| ConfigError::InvalidSecurityLimit { name, value, source })
 }
+
+/// Reads a comma-separated list, dropping empty entries.
+pub(super) fn list_var(name: &'static str) -> Result<Option<Vec<String>>, ConfigError> {
+    Ok(optional_var(name)?.map(|value| {
+        value
+            .split(',')
+            .map(str::trim)
+            .filter(|entry| !entry.is_empty())
+            .map(str::to_string)
+            .collect()
+    }))
+}