From 303ff7fc5bd06487f3ead274a802a65873c70b75 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 12 Jun 2026 11:49:53 +0200 Subject: [PATCH 1/2] Extract read-only OCI operations into an OciRead trait Move read_blob, read_index, has_blob, and all derived read-only methods (fsck, find_referrers, open_image_this_platform, etc.) from impl OciDir into a new OciRead trait with default methods. OciDir implements the trait, so all existing call sites work once OciRead is in scope. A prelude module re-exports OciRead for convenient importing. This prepares for adding OciArchive (read-only tar-based backend) that will share the same read logic via the trait without any code duplication. It changes none of the behaviour, or the API (other than the requirement to import that prelude to use the trait). Signed-off-by: Alexander Larsson Assisted-by: Claude Code (Claude Opus 4.6) --- README.md | 1 + examples/ocidir.rs | 1 + src/lib.rs | 1469 ++++++++++++++++++++++---------------------- 3 files changed, 747 insertions(+), 724 deletions(-) diff --git a/README.md b/README.md index 36daa2a..502fd0f 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ To access an existing OCI directory: ```rust,no_run # use ocidir::cap_std; +# use ocidir::prelude::*; # use anyhow::{anyhow, Result}; # fn main() -> anyhow::Result<()> { let d = cap_std::fs::Dir::open_ambient_dir("/path/to/ocidir", cap_std::ambient_authority())?; diff --git a/examples/ocidir.rs b/examples/ocidir.rs index fe4c640..1c181b6 100644 --- a/examples/ocidir.rs +++ b/examples/ocidir.rs @@ -11,6 +11,7 @@ use oci_distribution::secrets::RegistryAuth; use ocidir::OciDir; use ocidir::cap_std::fs::Dir; use ocidir::oci_spec::image::{self as oci_image, Descriptor, ImageManifest, MediaType, Platform}; +use ocidir::prelude::*; const OCI_TAG_ANNOTATION: &str = "org.opencontainers.image.ref.name"; diff --git a/src/lib.rs b/src/lib.rs index 52644a1..cdcc301 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,13 @@ #![doc = include_str!("../README.md")] #![deny(missing_docs)] +/// Re-exports of commonly used traits. +/// +/// brings OciRead into scope so that its methods are available on OciDir +pub mod prelude { + pub use crate::OciRead; +} + use canon_json::CanonicalFormatter; use cap_std::fs::{Dir, DirBuilderExt}; use cap_std_ext::cap_tempfile; @@ -229,499 +236,263 @@ fn sha256_of_descriptor(desc: &Descriptor) -> Result<&str> { }) } -impl OciDir { - /// Create an empty config descriptor. - /// See https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor - /// Our API right now always mutates a manifest, which means we need - /// a "valid" manifest, which requires a "valid" config descriptor. - fn empty_config_descriptor(&self) -> Result { - let empty_descriptor = oci_image::DescriptorBuilder::default() - .media_type(MediaType::EmptyJSON) - .size(2_u32) - .digest(Sha256Digest::from_str( - "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - )?) - .data("e30=") - .build()?; - - if !self - .dir - .exists(OciDir::parse_descriptor_to_path(&empty_descriptor)?) - { - let mut blob = self.create_blob()?; - serde_json::to_writer(&mut blob, &EmptyDescriptor {})?; - blob.complete_verified_as(&empty_descriptor)?; - } - - Ok(empty_descriptor) - } +fn descriptor_is_tagged(d: &Descriptor, tag: &str) -> bool { + d.annotations() + .as_ref() + .and_then(|annos| annos.get(OCI_TAG_ANNOTATION)) + .filter(|tagval| tagval.as_str() == tag) + .is_some() +} - /// Generate a valid empty manifest. See above. - pub fn new_empty_manifest(&self) -> Result { - Ok(oci_image::ImageManifestBuilder::default() - .schema_version(oci_image::SCHEMA_VERSION) - .config(self.empty_config_descriptor()?) - .layers(Vec::new())) - } +/// Check if a platform is compatible with the native platform. +/// +/// Platform has additional optional fields (variant, os_version, +/// os_features, features) which are primarily used for Windows images. +/// We only compare architecture and OS for compatibility. +fn platform_compatible(platform: &Platform, native: &Platform) -> bool { + platform.architecture() == native.architecture() && platform.os() == native.os() +} - /// Open the OCI directory at the target path; if it does not already - /// have the standard OCI metadata, it is created. - pub fn ensure(dir: Dir) -> Result { - let mut db = cap_std::fs::DirBuilder::new(); - db.recursive(true).mode(0o755); - dir.ensure_dir_with(BLOBDIR, &db)?; - if !dir.try_exists("oci-layout")? { - dir.atomic_write("oci-layout", r#"{"imageLayoutVersion":"1.0.0"}"#)?; - } - Self::open(dir) - } +/// Format the available platforms from a list of descriptors for error messages. +/// Limits output to 10 platforms to prevent excessive memory usage. +fn format_available_platforms<'a>(manifests: impl Iterator) -> Box { + const MAX_PLATFORMS_IN_ERROR: usize = 10; - /// Clone an OCI directory, using reflinks for blobs. - pub fn clone_to(&self, destdir: &Dir, p: impl AsRef) -> Result { - let p = p.as_ref(); - destdir.create_dir(p)?; - let cloned = Self::ensure(destdir.open_dir(p)?)?; - for blob in self.blobs_dir.entries()? { - let blob = blob?; - let path = Path::new(BLOBDIR).join(blob.file_name()); - let mut src = self.dir.open(&path).map(BufReader::new)?; - self.dir - .atomic_replace_with(&path, |w| std::io::copy(&mut src, w))?; - } - Ok(cloned) - } + let platforms: Vec<_> = manifests + .filter_map(|d| { + d.platform() + .as_ref() + .map(|p| format!("{}/{}", p.os(), p.architecture())) + }) + .take(MAX_PLATFORMS_IN_ERROR + 1) // Take one extra to detect truncation + .collect(); - /// Open an existing OCI directory. - pub fn open(dir: Dir) -> Result { - let blobs_dir = dir.open_dir(BLOBDIR)?; - Self::open_with_external_blobs(dir, blobs_dir) + if platforms.is_empty() { + return "(no platform info)".into(); } - /// Open an existing OCI directory with a separate cap_std::Dir for blobs/sha256 - /// This is useful when `blobs/sha256` might contain symlinks pointing outside the oci - /// directory, e.g. when sharing blobs across OCI repositories. The LXC OCI template uses this - /// feature. - pub fn open_with_external_blobs(dir: Dir, blobs_dir: Dir) -> Result { - Ok(Self { dir, blobs_dir }) + if platforms.len() > MAX_PLATFORMS_IN_ERROR { + let truncated: Vec<_> = platforms.into_iter().take(MAX_PLATFORMS_IN_ERROR).collect(); + format!("{}, ...", truncated.join(", ")).into() + } else { + platforms.join(", ").into() } +} - /// Return the underlying directory. - pub fn dir(&self) -> &Dir { - &self.dir - } +/// Read-only access to an OCI image store. +/// +/// This trait abstracts over different backends (directory layout, tar archive) +/// for reading OCI images. Implementors provide three primitives; all higher-level +/// operations are provided as default methods. +pub trait OciRead { + /// The concrete reader type returned by [`read_blob`](Self::read_blob). + type BlobReader: Read + Seek + Send + 'static; - /// Return the underlying directory for blobs. - pub fn blobs_dir(&self) -> &Dir { - &self.blobs_dir - } + /// Open a blob for reading, validating its size against the descriptor. + fn read_blob(&self, desc: &Descriptor) -> Result; - /// Write a serializable data (JSON) as an OCI blob - pub fn write_json_blob( - &self, - v: &S, - media_type: oci_image::MediaType, - ) -> Result { - let mut w = BlobWriter::new(&self.dir)?; - let mut ser = serde_json::Serializer::with_formatter(&mut w, CanonicalFormatter::new()); - v.serialize(&mut ser)?; - let blob = w.complete()?; - Ok(blob.descriptor().media_type(media_type)) - } + /// Read the image index (`index.json`). + fn read_index(&self) -> Result; - /// Create a blob (can be anything). - pub fn create_blob(&self) -> Result> { - BlobWriter::new(&self.dir) - } + /// Returns `true` if the blob with this digest is already present. + fn has_blob(&self, desc: &Descriptor) -> Result; - /// Create a layer writer with a custom encoder and - /// media type - pub fn create_custom_layer<'a, W: WriteComplete>>( - &'a self, - create: impl FnOnce(BlobWriter<'a>) -> std::io::Result, - media_type: MediaType, - ) -> Result> { - let bw = BlobWriter::new(&self.dir)?; - Ok(LayerWriter::new(create(bw)?, media_type)) + /// Returns `true` if the manifest is already present. + fn has_manifest(&self, desc: &Descriptor) -> Result { + let index = self.read_index()?; + Ok(index + .manifests() + .iter() + .any(|m| m.digest() == desc.digest())) } - /// Create a writer for a new uncompressed layer. - /// - /// This skips computing a separate uncompressed digest (diffid) since the - /// blob content is identical to the uncompressed content. - pub fn create_uncompressed_layer(&self) -> Result>> { - let bw = BlobWriter::new(&self.dir)?; - Ok(LayerWriter::new_uncompressed(bw, MediaType::ImageLayer)) + /// Read a JSON blob. + fn read_json_blob(&self, desc: &Descriptor) -> Result { + let blob = BufReader::new(self.read_blob(desc)?); + serde_json::from_reader(blob).map_err(Into::into) } - /// Create a writer for a new gzip+tar blob; the contents - /// are not parsed, but are expected to be a tarball. - pub fn create_gzip_layer<'a>( - &'a self, - c: Option, - ) -> Result>>> { - let creator = |bw: BlobWriter<'a>| Ok(GzEncoder::new(bw, c.unwrap_or_default())); - self.create_custom_layer(creator, MediaType::ImageLayerGzip) + /// Find the manifest with the provided tag + fn find_manifest_with_tag(&self, tag: &str) -> Result> { + let desc = self.find_manifest_descriptor_with_tag(tag)?; + desc.map(|img| self.read_json_blob(&img)).transpose() } - /// Create a tar output stream, backed by a blob - pub fn create_layer( - &'_ self, - c: Option, - ) -> Result>>>> { - Ok(tar::Builder::new(self.create_gzip_layer(c)?)) + /// Find the manifest descriptor with the provided tag + fn find_manifest_descriptor_with_tag(&self, tag: &str) -> Result> { + let idx = self.read_index()?; + Ok(idx + .manifests() + .iter() + .find(|desc| descriptor_is_tagged(desc, tag)) + .cloned()) } - #[cfg(feature = "zstd")] - /// Create a writer for a new zstd+tar blob; the contents - /// are not parsed, but are expected to be a tarball. + /// Open an image manifest for the current platform. /// - /// This method is only available when the `zstd` feature is enabled. - pub fn create_layer_zstd<'a>( - &'a self, - compression_level: Option, - ) -> Result>>> { - let creator = |bw: BlobWriter<'a>| zstd::Encoder::new(bw, compression_level.unwrap_or(0)); - self.create_custom_layer(creator, MediaType::ImageLayerZstd) - } - - #[cfg(feature = "zstdmt")] - /// Create a writer for a new zstd+tar blob; the contents - /// are not parsed, but are expected to be a tarball. - /// The compression is multithreaded. + /// This resolves the appropriate manifest from the index for the native + /// platform (OS and architecture). If `tag` is provided, only manifests + /// with that tag annotation are considered. /// - /// The `n_workers` parameter specifies the number of threads to use for compression, per - /// [zstd::Encoder::multithread]] + /// If the index contains an image index (manifest list), it is "peeled" + /// to get the underlying manifests. Nested image indices are not supported. /// - /// This method is only available when the `zstdmt` feature is enabled. - pub fn create_layer_zstd_multithread<'a>( - &'a self, - compression_level: Option, - n_workers: u32, - ) -> Result>>> { - let creator = |bw: BlobWriter<'a>| { - let mut encoder = zstd::Encoder::new(bw, compression_level.unwrap_or(0))?; - encoder.multithread(n_workers)?; - Ok(encoder) + /// Returns a [`ResolvedManifest`] containing the manifest, its digest, + /// and optionally the image index it was resolved from with its digest. + /// + /// # Errors + /// + /// Returns an error if: + /// - The index cannot be read + /// - The index is empty + /// - A tag is specified but not found + /// - No manifest matches the native platform + /// - A nested image index is encountered + fn open_image_this_platform(&self, tag: Option<&str>) -> Result { + let index = self.read_index()?; + let manifests = index.manifests(); + + // Filter by tag if specified, returning early on empty results + let candidates: Vec<_> = if let Some(tag) = tag { + let tagged: Vec<_> = manifests + .iter() + .filter(|d| descriptor_is_tagged(d, tag)) + .collect(); + if tagged.is_empty() { + return Err(Error::TagNotFound { tag: tag.into() }); + } + tagged + } else { + if manifests.is_empty() { + return Err(Error::EmptyImageIndex); + } + manifests.iter().collect() }; - self.create_custom_layer(creator, MediaType::ImageLayerZstd) - } - /// Add a layer to the top of the image stack. The firsh pushed layer becomes the root. - pub fn push_layer( - &self, - manifest: &mut oci_image::ImageManifest, - config: &mut oci_image::ImageConfiguration, - layer: Layer, - description: &str, - annotations: Option>, - ) { - self.push_layer_annotated(manifest, config, layer, annotations, description); - } + // Get the native platform + let native_platform = Platform::default(); - /// Add a layer to the top of the image stack with optional annotations. - /// - /// This is otherwise equivalent to [`Self::push_layer`]. - pub fn push_layer_annotated( - &self, - manifest: &mut oci_image::ImageManifest, - config: &mut oci_image::ImageConfiguration, - layer: Layer, - annotations: Option>>, - description: &str, - ) { - let created = chrono::offset::Utc::now(); - self.push_layer_full(manifest, config, layer, annotations, description, created) - } + // Collect all found candidate descriptors for error reporting + let mut found_candidates: Vec = Vec::new(); - /// Add a layer to the top of the image stack with optional annotations and desired timestamp. - /// - /// This is otherwise equivalent to [`Self::push_layer_annotated`]. - pub fn push_layer_full( - &self, - manifest: &mut oci_image::ImageManifest, - config: &mut oci_image::ImageConfiguration, - layer: Layer, - annotations: Option>>, - description: &str, - created: chrono::DateTime, - ) { - let history = oci_image::HistoryBuilder::default() - .created(created.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)) - .created_by(description.to_string()) - .build() - .unwrap(); - self.push_layer_with_history_annotated(manifest, config, layer, annotations, Some(history)); - } - - /// Add a layer to the top of the image stack with optional annotations and desired history entry. - /// - /// This is otherwise equivalent to [`Self::push_layer_annotated`]. - pub fn push_layer_with_history_annotated( - &self, - manifest: &mut oci_image::ImageManifest, - config: &mut oci_image::ImageConfiguration, - layer: Layer, - annotations: Option>>, - history: Option, - ) { - let mut builder = layer.descriptor(); - if let Some(annotations) = annotations { - builder = builder.annotations(annotations); - } - let blobdesc = builder.build().unwrap(); - manifest.layers_mut().push(blobdesc); - let mut rootfs = config.rootfs().clone(); - rootfs - .diff_ids_mut() - .push(layer.uncompressed_sha256_as_digest().to_string()); - config.set_rootfs(rootfs); - let history = if let Some(history) = history { - history - } else { - oci_image::HistoryBuilder::default().build().unwrap() - }; - config.history_mut().get_or_insert_default().push(history); - } - - /// Add a layer to the top of the image stack with desired history entry. - /// - /// This is otherwise equivalent to [`Self::push_layer`]. - pub fn push_layer_with_history( - &self, - manifest: &mut oci_image::ImageManifest, - config: &mut oci_image::ImageConfiguration, - layer: Layer, - history: Option, - ) { - let annotations: Option> = None; - self.push_layer_with_history_annotated(manifest, config, layer, annotations, history); - } - - fn parse_descriptor_to_path(desc: &oci_spec::image::Descriptor) -> Result { - let digest = sha256_of_descriptor(desc)?; - Ok(PathBuf::from(digest)) - } + for desc in candidates { + match desc.media_type() { + MediaType::ImageManifest => { + if let Some(manifest) = + self.resolve_descriptor_for_platform(desc, &native_platform)? + { + return Ok(ResolvedManifest { + manifest, + manifest_descriptor: desc.clone(), + source_index: None, + }); + } + found_candidates.push(desc.clone()); + } + MediaType::ImageIndex => { + // Peel the manifest list + let nested: ImageIndex = self.read_json_blob(desc)?; + let index_descriptor = desc.clone(); - /// Open a blob; its size is validated as a sanity check. - pub fn read_blob(&self, desc: &oci_spec::image::Descriptor) -> Result { - let path = Self::parse_descriptor_to_path(desc)?; - let f = self.blobs_dir.open(path).map(|f| f.into_std())?; - let expected: u64 = desc.size(); - let found = f.metadata()?.len(); - if expected != found { - return Err(Error::SizeMismatch { expected, found }); + if let Some(resolved) = self.resolve_manifest_list( + nested, + index_descriptor, + &native_platform, + &mut found_candidates, + )? { + return Ok(resolved); + } + } + other => { + return Err(Error::UnexpectedMediaType { + media_type: other.clone(), + }); + } + } } - Ok(f) - } - - /// Returns `true` if the blob with this digest is already present. - pub fn has_blob(&self, desc: &oci_spec::image::Descriptor) -> Result { - let path = Self::parse_descriptor_to_path(desc)?; - self.blobs_dir.try_exists(path).map_err(Into::into) - } - - /// Returns `true` if the manifest is already present. - pub fn has_manifest(&self, desc: &oci_spec::image::Descriptor) -> Result { - let index = self.read_index()?; - Ok(index - .manifests() - .iter() - .any(|m| m.digest() == desc.digest())) - } - - /// Read a JSON blob. - pub fn read_json_blob( - &self, - desc: &oci_spec::image::Descriptor, - ) -> Result { - let blob = BufReader::new(self.read_blob(desc)?); - serde_json::from_reader(blob).map_err(Into::into) - } - - /// Write a configuration blob. - pub fn write_config( - &self, - config: oci_image::ImageConfiguration, - ) -> Result { - Ok(self - .write_json_blob(&config, MediaType::ImageConfig)? - .build()?) - } - /// Read the image index. - pub fn read_index(&self) -> Result { - let r = if let Some(index) = self.dir.open_optional("index.json")?.map(BufReader::new) { - oci_image::ImageIndex::from_reader(index)? - } else { - return Err(Error::MissingImageIndex); - }; - Ok(r) + // No match found + Err(Error::NoMatchingPlatform { + os: native_platform.os().to_string().into(), + architecture: native_platform.architecture().to_string().into(), + available: format_available_platforms(found_candidates.iter()), + }) } - /// Write a manifest as a blob, and replace the index with a reference to it. + /// Resolve a manifest from an image index (manifest list) for a given platform. /// - /// When the manifest has a `subject` field (i.e. it is a referrer artifact), - /// the `artifact_type` and `annotations` from the manifest are automatically - /// propagated to the descriptor in the index, as required by the OCI - /// distribution spec's Referrers API. + /// Iterates the manifests within the index, returning the first one that + /// matches `native_platform`. Non-matching descriptors are appended to + /// `found_candidates` so callers can include them in error messages. /// - /// If the manifest has an explicit `artifact_type`, that value is used on the - /// descriptor. Otherwise, if the manifest has a `subject`, the descriptor's - /// `artifact_type` falls back to `config.mediaType` (per the spec). - pub fn insert_manifest( + /// Returns `Ok(None)` if no manifest in this index matched. + fn resolve_manifest_list( &self, - manifest: oci_image::ImageManifest, - tag: Option<&str>, - platform: oci_image::Platform, - ) -> Result { - let mut desc_builder = self - .write_json_blob(&manifest, MediaType::ImageManifest)? - .platform(platform); - - // Per the OCI distribution spec, descriptors in the index for manifests - // with a `subject` must carry `artifactType` and all annotations from - // the manifest. This enables the Referrers API to work without fetching - // each manifest blob. - if manifest.subject().is_some() { - let effective_artifact_type = manifest - .artifact_type() - .clone() - .unwrap_or_else(|| manifest.config().media_type().clone()); - desc_builder = desc_builder.artifact_type(effective_artifact_type); - - // Copy manifest-level annotations to the descriptor - if let Some(annos) = manifest.annotations() { - desc_builder = desc_builder.annotations(annos.clone()); + index: ImageIndex, + index_descriptor: Descriptor, + native_platform: &Platform, + found_candidates: &mut Vec, + ) -> Result> { + for desc in index.manifests() { + match desc.media_type() { + MediaType::ImageIndex => { + return Err(Error::NestedImageIndex); + } + MediaType::ImageManifest => { + if let Some(manifest) = + self.resolve_descriptor_for_platform(desc, native_platform)? + { + return Ok(Some(ResolvedManifest { + manifest, + manifest_descriptor: desc.clone(), + source_index: Some((index, index_descriptor)), + })); + } + found_candidates.push(desc.clone()); + } + other => { + return Err(Error::UnexpectedMediaType { + media_type: other.clone(), + }); + } } - } else if let Some(at) = manifest.artifact_type() { - // Even without a subject, propagate artifact_type if set - desc_builder = desc_builder.artifact_type(at.clone()); } - - let mut manifest_desc = desc_builder.build()?; - if let Some(tag) = tag { - let mut annotations = manifest_desc.annotations().clone().unwrap_or_default(); - annotations.insert(OCI_TAG_ANNOTATION.to_string(), tag.to_string()); - manifest_desc.set_annotations(Some(annotations)); - } - - self.append_to_index(manifest_desc.clone(), tag)?; - Ok(manifest_desc) - } - - /// Write an `ImageIndex` to `index.json` using canonical JSON formatting. - fn write_index(&self, index: &oci_image::ImageIndex) -> Result<()> { - self.dir - .atomic_replace_with("index.json", |mut w| -> Result<()> { - let mut ser = - serde_json::Serializer::with_formatter(&mut w, CanonicalFormatter::new()); - index.serialize(&mut ser)?; - Ok(()) - })?; - Ok(()) - } - - /// Convenience helper to write the provided config, update the manifest to use it, then call [`insert_manifest`]. - pub fn insert_manifest_and_config( - &self, - mut manifest: oci_image::ImageManifest, - config: oci_image::ImageConfiguration, - tag: Option<&str>, - platform: oci_image::Platform, - ) -> Result { - let config = self.write_config(config)?; - manifest.set_config(config); - self.insert_manifest(manifest, tag, platform) + Ok(None) } - /// Create and insert an artifact manifest that references another manifest - /// via the `subject` field. - /// - /// This creates an OCI artifact manifest with the given `artifact_type`, - /// pointing to `subject` as the referenced manifest. The artifact's config - /// is set to the [empty descriptor][empty] and layers contain the provided - /// content blobs (or a single empty descriptor if no layers are provided). - /// - /// Per the [OCI image spec][artifact-usage], when `config.mediaType` is set - /// to the empty value, `artifact_type` MUST be defined. - /// - /// The resulting descriptor in the index carries `artifact_type` and all - /// manifest annotations, enabling the [Referrers API][referrers] to list - /// this artifact without fetching the manifest blob. - /// - /// Unlike [`insert_manifest`](Self::insert_manifest), the descriptor in - /// the index does not carry a `platform` field, since artifacts are not - /// platform-specific. + /// Resolve a manifest descriptor for the given platform, reading the config + /// blob when the descriptor has no explicit `platform` annotation. /// - /// [empty]: https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor - /// [artifact-usage]: https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage - /// [referrers]: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers - pub fn insert_artifact_manifest( + /// Returns `Ok(Some(manifest))` when `desc` is compatible with `native`, + /// `Ok(None)` when it is not, and `Err(_)` on I/O or parse errors. + fn resolve_descriptor_for_platform( &self, - subject: Descriptor, - artifact_type: MediaType, - layers: Vec, - annotations: Option>, - ) -> Result { - let empty_descriptor = self.empty_config_descriptor()?; - - // Per the spec, if no layers are provided, use a single empty - // descriptor as a placeholder layer. - let layers = if layers.is_empty() { - vec![empty_descriptor.clone()] - } else { - layers - }; - - let mut manifest_builder = oci_image::ImageManifestBuilder::default() - .schema_version(oci_image::SCHEMA_VERSION) - .config(empty_descriptor) - .layers(layers) - .artifact_type(artifact_type.clone()) - .subject(subject); - - if let Some(annos) = annotations { - manifest_builder = manifest_builder.annotations(annos); - } - - let manifest = manifest_builder.build()?; - - // Write the manifest blob and build a descriptor without a platform - // field. We propagate artifact_type and annotations to the descriptor - // for the Referrers API, as required by the OCI distribution spec. - let mut desc_builder = self - .write_json_blob(&manifest, MediaType::ImageManifest)? - .artifact_type(artifact_type); - - if let Some(annos) = manifest.annotations() { - desc_builder = desc_builder.annotations(annos.clone()); - } - - let manifest_desc = desc_builder.build()?; - self.append_to_index(manifest_desc.clone(), None)?; - Ok(manifest_desc) - } - - /// Append a descriptor to the index, optionally replacing any existing - /// entry with the same tag. - fn append_to_index(&self, desc: Descriptor, tag: Option<&str>) -> Result<()> { - let index = match self.read_index() { - Ok(mut index) => { - let mut manifests = index.manifests().clone(); - if let Some(tag) = tag { - manifests.retain(|d| !Self::descriptor_is_tagged(d, tag)); - } - manifests.push(desc); - index.set_manifests(manifests); - index - } - Err(Error::MissingImageIndex) => oci_image::ImageIndexBuilder::default() - .schema_version(oci_image::SCHEMA_VERSION) - .manifests(vec![desc]) - .build()?, - Err(e) => return Err(e), - }; - self.write_index(&index) + desc: &Descriptor, + native: &Platform, + ) -> Result> { + // Fast path: explicit platform annotation — no blob I/O needed. + if let Some(platform) = desc.platform().as_ref() { + if platform_compatible(platform, native) { + return Ok(Some(self.read_json_blob::(desc)?)); + } + return Ok(None); + } + + // If there's no annotation then read the manifest and config. + let manifest = self.read_json_blob::(desc)?; + + // Only image manifests (not OCI artifact manifests) carry a platform in + // their config blob. Skip the read entirely for anything else. + if manifest.config().media_type() != &MediaType::ImageConfig { + return Ok(None); + } + + let config: ImageConfiguration = self.read_json_blob(manifest.config())?; + if config.architecture() == native.architecture() && config.os() == native.os() { + Ok(Some(manifest)) + } else { + Ok(None) + } } /// Find all descriptors in the index that reference the given subject @@ -738,7 +509,7 @@ impl OciDir { /// field, so the cost scales with the number of manifests in the index. /// /// [referrers]: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers - pub fn find_referrers( + fn find_referrers( &self, subject_digest: &Digest, artifact_type_filter: Option<&MediaType>, @@ -775,326 +546,576 @@ impl OciDir { } } - referrers.push(desc.clone()); - } + referrers.push(desc.clone()); + } + + Ok(referrers) + } + + /// Verify a blob's SHA-256 digest matches its descriptor. + fn verify_blob_digest(&self, desc: &Descriptor) -> Result<()> { + let expected = sha256_of_descriptor(desc)?; + let mut f = self.read_blob(desc)?; + let mut hasher = Hasher::new(MessageDigest::sha256())?; + std::io::copy(&mut f, &mut hasher)?; + let found = hex::encode(hasher.finish()?); + if expected != found { + return Err(Error::DigestMismatch { + expected: expected.into(), + found: found.into(), + }); + } + Ok(()) + } + + /// Verify a single manifest and all of its referenced objects. + /// Skips already validated blobs referenced by digest in `validated`, + /// and updates that set with ones we did validate. + fn fsck_one_manifest( + &self, + manifest: &ImageManifest, + validated: &mut HashSet>, + ) -> Result<()> { + let config_digest = sha256_of_descriptor(manifest.config())?; + if !validated.contains(config_digest) { + // Always verify the config blob digest, regardless of media type. + self.verify_blob_digest(manifest.config())?; + // Additionally validate the content structure for known types. + match manifest.config().media_type() { + MediaType::ImageConfig => { + let _: ImageConfiguration = self.read_json_blob(manifest.config())?; + } + MediaType::EmptyJSON => { + let _: EmptyDescriptor = self.read_json_blob(manifest.config())?; + } + // Per the OCI image spec, implementations MUST NOT error on + // encountering an unknown config mediaType. + _ => {} + } + validated.insert(config_digest.into()); + } + for layer in manifest.layers() { + let expected = sha256_of_descriptor(layer)?; + if validated.contains(expected) { + continue; + } + self.verify_blob_digest(layer)?; + validated.insert(expected.into()); + } + Ok(()) + } + + /// Verify consistency of the index, its manifests, the config and blobs (all the latter) + /// by verifying their descriptor. + fn fsck(&self) -> Result { + let index = self.read_index()?; + let mut validated_blobs = HashSet::new(); + for manifest_descriptor in index.manifests() { + let expected_sha256 = sha256_of_descriptor(manifest_descriptor)?; + let manifest: ImageManifest = self.read_json_blob(manifest_descriptor)?; + validated_blobs.insert(expected_sha256.into()); + self.fsck_one_manifest(&manifest, &mut validated_blobs)?; + } + Ok(validated_blobs.len().try_into().unwrap()) + } +} + +impl OciDir { + /// Create an empty config descriptor. + /// See https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor + /// Our API right now always mutates a manifest, which means we need + /// a "valid" manifest, which requires a "valid" config descriptor. + fn empty_config_descriptor(&self) -> Result { + let empty_descriptor = oci_image::DescriptorBuilder::default() + .media_type(MediaType::EmptyJSON) + .size(2_u32) + .digest(Sha256Digest::from_str( + "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + )?) + .data("e30=") + .build()?; + + if !self + .dir + .exists(OciDir::parse_descriptor_to_path(&empty_descriptor)?) + { + let mut blob = self.create_blob()?; + serde_json::to_writer(&mut blob, &EmptyDescriptor {})?; + blob.complete_verified_as(&empty_descriptor)?; + } + + Ok(empty_descriptor) + } + + /// Generate a valid empty manifest. See above. + pub fn new_empty_manifest(&self) -> Result { + Ok(oci_image::ImageManifestBuilder::default() + .schema_version(oci_image::SCHEMA_VERSION) + .config(self.empty_config_descriptor()?) + .layers(Vec::new())) + } + + /// Open the OCI directory at the target path; if it does not already + /// have the standard OCI metadata, it is created. + pub fn ensure(dir: Dir) -> Result { + let mut db = cap_std::fs::DirBuilder::new(); + db.recursive(true).mode(0o755); + dir.ensure_dir_with(BLOBDIR, &db)?; + if !dir.try_exists("oci-layout")? { + dir.atomic_write("oci-layout", r#"{"imageLayoutVersion":"1.0.0"}"#)?; + } + Self::open(dir) + } + + /// Clone an OCI directory, using reflinks for blobs. + pub fn clone_to(&self, destdir: &Dir, p: impl AsRef) -> Result { + let p = p.as_ref(); + destdir.create_dir(p)?; + let cloned = Self::ensure(destdir.open_dir(p)?)?; + for blob in self.blobs_dir.entries()? { + let blob = blob?; + let path = Path::new(BLOBDIR).join(blob.file_name()); + let mut src = self.dir.open(&path).map(BufReader::new)?; + self.dir + .atomic_replace_with(&path, |w| std::io::copy(&mut src, w))?; + } + Ok(cloned) + } + + /// Open an existing OCI directory. + pub fn open(dir: Dir) -> Result { + let blobs_dir = dir.open_dir(BLOBDIR)?; + Self::open_with_external_blobs(dir, blobs_dir) + } + + /// Open an existing OCI directory with a separate cap_std::Dir for blobs/sha256 + /// This is useful when `blobs/sha256` might contain symlinks pointing outside the oci + /// directory, e.g. when sharing blobs across OCI repositories. The LXC OCI template uses this + /// feature. + pub fn open_with_external_blobs(dir: Dir, blobs_dir: Dir) -> Result { + Ok(Self { dir, blobs_dir }) + } + + /// Return the underlying directory. + pub fn dir(&self) -> &Dir { + &self.dir + } + + /// Return the underlying directory for blobs. + pub fn blobs_dir(&self) -> &Dir { + &self.blobs_dir + } + + /// Write a serializable data (JSON) as an OCI blob + pub fn write_json_blob( + &self, + v: &S, + media_type: oci_image::MediaType, + ) -> Result { + let mut w = BlobWriter::new(&self.dir)?; + let mut ser = serde_json::Serializer::with_formatter(&mut w, CanonicalFormatter::new()); + v.serialize(&mut ser)?; + let blob = w.complete()?; + Ok(blob.descriptor().media_type(media_type)) + } + + /// Create a blob (can be anything). + pub fn create_blob(&self) -> Result> { + BlobWriter::new(&self.dir) + } + + /// Create a layer writer with a custom encoder and + /// media type + pub fn create_custom_layer<'a, W: WriteComplete>>( + &'a self, + create: impl FnOnce(BlobWriter<'a>) -> std::io::Result, + media_type: MediaType, + ) -> Result> { + let bw = BlobWriter::new(&self.dir)?; + Ok(LayerWriter::new(create(bw)?, media_type)) + } + + /// Create a writer for a new uncompressed layer. + /// + /// This skips computing a separate uncompressed digest (diffid) since the + /// blob content is identical to the uncompressed content. + pub fn create_uncompressed_layer(&self) -> Result>> { + let bw = BlobWriter::new(&self.dir)?; + Ok(LayerWriter::new_uncompressed(bw, MediaType::ImageLayer)) + } + + /// Create a writer for a new gzip+tar blob; the contents + /// are not parsed, but are expected to be a tarball. + pub fn create_gzip_layer<'a>( + &'a self, + c: Option, + ) -> Result>>> { + let creator = |bw: BlobWriter<'a>| Ok(GzEncoder::new(bw, c.unwrap_or_default())); + self.create_custom_layer(creator, MediaType::ImageLayerGzip) + } + + /// Create a tar output stream, backed by a blob + pub fn create_layer( + &'_ self, + c: Option, + ) -> Result>>>> { + Ok(tar::Builder::new(self.create_gzip_layer(c)?)) + } + + #[cfg(feature = "zstd")] + /// Create a writer for a new zstd+tar blob; the contents + /// are not parsed, but are expected to be a tarball. + /// + /// This method is only available when the `zstd` feature is enabled. + pub fn create_layer_zstd<'a>( + &'a self, + compression_level: Option, + ) -> Result>>> { + let creator = |bw: BlobWriter<'a>| zstd::Encoder::new(bw, compression_level.unwrap_or(0)); + self.create_custom_layer(creator, MediaType::ImageLayerZstd) + } + + #[cfg(feature = "zstdmt")] + /// Create a writer for a new zstd+tar blob; the contents + /// are not parsed, but are expected to be a tarball. + /// The compression is multithreaded. + /// + /// The `n_workers` parameter specifies the number of threads to use for compression, per + /// [zstd::Encoder::multithread]] + /// + /// This method is only available when the `zstdmt` feature is enabled. + pub fn create_layer_zstd_multithread<'a>( + &'a self, + compression_level: Option, + n_workers: u32, + ) -> Result>>> { + let creator = |bw: BlobWriter<'a>| { + let mut encoder = zstd::Encoder::new(bw, compression_level.unwrap_or(0))?; + encoder.multithread(n_workers)?; + Ok(encoder) + }; + self.create_custom_layer(creator, MediaType::ImageLayerZstd) + } + + /// Add a layer to the top of the image stack. The firsh pushed layer becomes the root. + pub fn push_layer( + &self, + manifest: &mut oci_image::ImageManifest, + config: &mut oci_image::ImageConfiguration, + layer: Layer, + description: &str, + annotations: Option>, + ) { + self.push_layer_annotated(manifest, config, layer, annotations, description); + } - Ok(referrers) + /// Add a layer to the top of the image stack with optional annotations. + /// + /// This is otherwise equivalent to [`Self::push_layer`]. + pub fn push_layer_annotated( + &self, + manifest: &mut oci_image::ImageManifest, + config: &mut oci_image::ImageConfiguration, + layer: Layer, + annotations: Option>>, + description: &str, + ) { + let created = chrono::offset::Utc::now(); + self.push_layer_full(manifest, config, layer, annotations, description, created) } - /// Write a manifest as a blob, and replace the index with a reference to it. - pub fn replace_with_single_manifest( + /// Add a layer to the top of the image stack with optional annotations and desired timestamp. + /// + /// This is otherwise equivalent to [`Self::push_layer_annotated`]. + pub fn push_layer_full( &self, - manifest: oci_image::ImageManifest, - platform: oci_image::Platform, - ) -> Result<()> { - let manifest = self - .write_json_blob(&manifest, MediaType::ImageManifest)? - .platform(platform) + manifest: &mut oci_image::ImageManifest, + config: &mut oci_image::ImageConfiguration, + layer: Layer, + annotations: Option>>, + description: &str, + created: chrono::DateTime, + ) { + let history = oci_image::HistoryBuilder::default() + .created(created.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)) + .created_by(description.to_string()) .build() .unwrap(); + self.push_layer_with_history_annotated(manifest, config, layer, annotations, Some(history)); + } - let index_data = oci_image::ImageIndexBuilder::default() - .schema_version(oci_image::SCHEMA_VERSION) - .manifests(vec![manifest]) - .build() - .unwrap(); - self.write_index(&index_data) + /// Add a layer to the top of the image stack with optional annotations and desired history entry. + /// + /// This is otherwise equivalent to [`Self::push_layer_annotated`]. + pub fn push_layer_with_history_annotated( + &self, + manifest: &mut oci_image::ImageManifest, + config: &mut oci_image::ImageConfiguration, + layer: Layer, + annotations: Option>>, + history: Option, + ) { + let mut builder = layer.descriptor(); + if let Some(annotations) = annotations { + builder = builder.annotations(annotations); + } + let blobdesc = builder.build().unwrap(); + manifest.layers_mut().push(blobdesc); + let mut rootfs = config.rootfs().clone(); + rootfs + .diff_ids_mut() + .push(layer.uncompressed_sha256_as_digest().to_string()); + config.set_rootfs(rootfs); + let history = if let Some(history) = history { + history + } else { + oci_image::HistoryBuilder::default().build().unwrap() + }; + config.history_mut().get_or_insert_default().push(history); } - fn descriptor_is_tagged(d: &Descriptor, tag: &str) -> bool { - d.annotations() - .as_ref() - .and_then(|annos| annos.get(OCI_TAG_ANNOTATION)) - .filter(|tagval| tagval.as_str() == tag) - .is_some() + /// Add a layer to the top of the image stack with desired history entry. + /// + /// This is otherwise equivalent to [`Self::push_layer`]. + pub fn push_layer_with_history( + &self, + manifest: &mut oci_image::ImageManifest, + config: &mut oci_image::ImageConfiguration, + layer: Layer, + history: Option, + ) { + let annotations: Option> = None; + self.push_layer_with_history_annotated(manifest, config, layer, annotations, history); } - /// Find the manifest with the provided tag - pub fn find_manifest_with_tag(&self, tag: &str) -> Result> { - let desc = self.find_manifest_descriptor_with_tag(tag)?; - desc.map(|img| self.read_json_blob(&img)).transpose() + fn parse_descriptor_to_path(desc: &oci_spec::image::Descriptor) -> Result { + let digest = sha256_of_descriptor(desc)?; + Ok(PathBuf::from(digest)) } - /// Find the manifest descriptor with the provided tag - pub fn find_manifest_descriptor_with_tag( + /// Write a configuration blob. + pub fn write_config( &self, - tag: &str, - ) -> Result> { - let idx = self.read_index()?; - Ok(idx - .manifests() - .iter() - .find(|desc| Self::descriptor_is_tagged(desc, tag)) - .cloned()) + config: oci_image::ImageConfiguration, + ) -> Result { + Ok(self + .write_json_blob(&config, MediaType::ImageConfig)? + .build()?) } - /// Open an image manifest for the current platform. - /// - /// This resolves the appropriate manifest from the index for the native - /// platform (OS and architecture). If `tag` is provided, only manifests - /// with that tag annotation are considered. - /// - /// If the index contains an image index (manifest list), it is "peeled" - /// to get the underlying manifests. Nested image indices are not supported. - /// - /// Returns a [`ResolvedManifest`] containing the manifest, its digest, - /// and optionally the image index it was resolved from with its digest. + /// Write a manifest as a blob, and replace the index with a reference to it. /// - /// # Errors + /// When the manifest has a `subject` field (i.e. it is a referrer artifact), + /// the `artifact_type` and `annotations` from the manifest are automatically + /// propagated to the descriptor in the index, as required by the OCI + /// distribution spec's Referrers API. /// - /// Returns an error if: - /// - The index cannot be read - /// - The index is empty - /// - A tag is specified but not found - /// - No manifest matches the native platform - /// - A nested image index is encountered - pub fn open_image_this_platform(&self, tag: Option<&str>) -> Result { - let index = self.read_index()?; - let manifests = index.manifests(); + /// If the manifest has an explicit `artifact_type`, that value is used on the + /// descriptor. Otherwise, if the manifest has a `subject`, the descriptor's + /// `artifact_type` falls back to `config.mediaType` (per the spec). + pub fn insert_manifest( + &self, + manifest: oci_image::ImageManifest, + tag: Option<&str>, + platform: oci_image::Platform, + ) -> Result { + let mut desc_builder = self + .write_json_blob(&manifest, MediaType::ImageManifest)? + .platform(platform); - // Filter by tag if specified, returning early on empty results - let candidates: Vec<_> = if let Some(tag) = tag { - let tagged: Vec<_> = manifests - .iter() - .filter(|d| Self::descriptor_is_tagged(d, tag)) - .collect(); - if tagged.is_empty() { - return Err(Error::TagNotFound { tag: tag.into() }); - } - tagged - } else { - if manifests.is_empty() { - return Err(Error::EmptyImageIndex); - } - manifests.iter().collect() - }; + // Per the OCI distribution spec, descriptors in the index for manifests + // with a `subject` must carry `artifactType` and all annotations from + // the manifest. This enables the Referrers API to work without fetching + // each manifest blob. + if manifest.subject().is_some() { + let effective_artifact_type = manifest + .artifact_type() + .clone() + .unwrap_or_else(|| manifest.config().media_type().clone()); + desc_builder = desc_builder.artifact_type(effective_artifact_type); - // Get the native platform - let native_platform = Platform::default(); + // Copy manifest-level annotations to the descriptor + if let Some(annos) = manifest.annotations() { + desc_builder = desc_builder.annotations(annos.clone()); + } + } else if let Some(at) = manifest.artifact_type() { + // Even without a subject, propagate artifact_type if set + desc_builder = desc_builder.artifact_type(at.clone()); + } - // Collect all found candidate descriptors for error reporting - let mut found_candidates: Vec = Vec::new(); + let mut manifest_desc = desc_builder.build()?; + if let Some(tag) = tag { + let mut annotations = manifest_desc.annotations().clone().unwrap_or_default(); + annotations.insert(OCI_TAG_ANNOTATION.to_string(), tag.to_string()); + manifest_desc.set_annotations(Some(annotations)); + } - for desc in candidates { - match desc.media_type() { - MediaType::ImageManifest => { - if let Some(manifest) = - self.resolve_descriptor_for_platform(desc, &native_platform)? - { - return Ok(ResolvedManifest { - manifest, - manifest_descriptor: desc.clone(), - source_index: None, - }); - } - found_candidates.push(desc.clone()); - } - MediaType::ImageIndex => { - // Peel the manifest list - let nested: ImageIndex = self.read_json_blob(desc)?; - let index_descriptor = desc.clone(); + self.append_to_index(manifest_desc.clone(), tag)?; + Ok(manifest_desc) + } - if let Some(resolved) = self.resolve_manifest_list( - nested, - index_descriptor, - &native_platform, - &mut found_candidates, - )? { - return Ok(resolved); - } - } - other => { - return Err(Error::UnexpectedMediaType { - media_type: other.clone(), - }); - } - } - } + /// Write an `ImageIndex` to `index.json` using canonical JSON formatting. + fn write_index(&self, index: &oci_image::ImageIndex) -> Result<()> { + self.dir + .atomic_replace_with("index.json", |mut w| -> Result<()> { + let mut ser = + serde_json::Serializer::with_formatter(&mut w, CanonicalFormatter::new()); + index.serialize(&mut ser)?; + Ok(()) + })?; + Ok(()) + } - // No match found - Err(Error::NoMatchingPlatform { - os: native_platform.os().to_string().into(), - architecture: native_platform.architecture().to_string().into(), - available: Self::format_available_platforms(found_candidates.iter()), - }) + /// Convenience helper to write the provided config, update the manifest to use it, then call [`insert_manifest`]. + pub fn insert_manifest_and_config( + &self, + mut manifest: oci_image::ImageManifest, + config: oci_image::ImageConfiguration, + tag: Option<&str>, + platform: oci_image::Platform, + ) -> Result { + let config = self.write_config(config)?; + manifest.set_config(config); + self.insert_manifest(manifest, tag, platform) } - /// Resolve a manifest from an image index (manifest list) for a given platform. + /// Create and insert an artifact manifest that references another manifest + /// via the `subject` field. /// - /// Iterates the manifests within the index, returning the first one that - /// matches `native_platform`. Non-matching descriptors are appended to - /// `found_candidates` so callers can include them in error messages. + /// This creates an OCI artifact manifest with the given `artifact_type`, + /// pointing to `subject` as the referenced manifest. The artifact's config + /// is set to the [empty descriptor][empty] and layers contain the provided + /// content blobs (or a single empty descriptor if no layers are provided). /// - /// Returns `Ok(None)` if no manifest in this index matched. - fn resolve_manifest_list( + /// Per the [OCI image spec][artifact-usage], when `config.mediaType` is set + /// to the empty value, `artifact_type` MUST be defined. + /// + /// The resulting descriptor in the index carries `artifact_type` and all + /// manifest annotations, enabling the [Referrers API][referrers] to list + /// this artifact without fetching the manifest blob. + /// + /// Unlike [`insert_manifest`](Self::insert_manifest), the descriptor in + /// the index does not carry a `platform` field, since artifacts are not + /// platform-specific. + /// + /// [empty]: https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor + /// [artifact-usage]: https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage + /// [referrers]: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers + pub fn insert_artifact_manifest( &self, - index: ImageIndex, - index_descriptor: Descriptor, - native_platform: &Platform, - found_candidates: &mut Vec, - ) -> Result> { - for desc in index.manifests() { - match desc.media_type() { - MediaType::ImageIndex => { - return Err(Error::NestedImageIndex); - } - MediaType::ImageManifest => { - if let Some(manifest) = - self.resolve_descriptor_for_platform(desc, native_platform)? - { - return Ok(Some(ResolvedManifest { - manifest, - manifest_descriptor: desc.clone(), - source_index: Some((index, index_descriptor)), - })); - } - found_candidates.push(desc.clone()); - } - other => { - return Err(Error::UnexpectedMediaType { - media_type: other.clone(), - }); - } - } - } - Ok(None) - } + subject: Descriptor, + artifact_type: MediaType, + layers: Vec, + annotations: Option>, + ) -> Result { + let empty_descriptor = self.empty_config_descriptor()?; - /// Format the available platforms from a list of descriptors for error messages. - /// Limits output to 10 platforms to prevent excessive memory usage. - fn format_available_platforms<'a>(manifests: impl Iterator) -> Box { - const MAX_PLATFORMS_IN_ERROR: usize = 10; + // Per the spec, if no layers are provided, use a single empty + // descriptor as a placeholder layer. + let layers = if layers.is_empty() { + vec![empty_descriptor.clone()] + } else { + layers + }; - let platforms: Vec<_> = manifests - .filter_map(|d| { - d.platform() - .as_ref() - .map(|p| format!("{}/{}", p.os(), p.architecture())) - }) - .take(MAX_PLATFORMS_IN_ERROR + 1) // Take one extra to detect truncation - .collect(); + let mut manifest_builder = oci_image::ImageManifestBuilder::default() + .schema_version(oci_image::SCHEMA_VERSION) + .config(empty_descriptor) + .layers(layers) + .artifact_type(artifact_type.clone()) + .subject(subject); - if platforms.is_empty() { - return "(no platform info)".into(); + if let Some(annos) = annotations { + manifest_builder = manifest_builder.annotations(annos); } - if platforms.len() > MAX_PLATFORMS_IN_ERROR { - let truncated: Vec<_> = platforms.into_iter().take(MAX_PLATFORMS_IN_ERROR).collect(); - format!("{}, ...", truncated.join(", ")).into() - } else { - platforms.join(", ").into() + let manifest = manifest_builder.build()?; + + // Write the manifest blob and build a descriptor without a platform + // field. We propagate artifact_type and annotations to the descriptor + // for the Referrers API, as required by the OCI distribution spec. + let mut desc_builder = self + .write_json_blob(&manifest, MediaType::ImageManifest)? + .artifact_type(artifact_type); + + if let Some(annos) = manifest.annotations() { + desc_builder = desc_builder.annotations(annos.clone()); } - } - /// Check if a platform is compatible with the native platform. - /// - /// Platform has additional optional fields (variant, os_version, - /// os_features, features) which are primarily used for Windows images. - /// We only compare architecture and OS for compatibility. - fn platform_compatible(platform: &Platform, native: &Platform) -> bool { - platform.architecture() == native.architecture() && platform.os() == native.os() + let manifest_desc = desc_builder.build()?; + self.append_to_index(manifest_desc.clone(), None)?; + Ok(manifest_desc) } - /// Resolve a manifest descriptor for the given platform, reading the config - /// blob when the descriptor has no explicit `platform` annotation. - /// - /// Returns `Ok(Some(manifest))` when `desc` is compatible with `native`, - /// `Ok(None)` when it is not, and `Err(_)` on I/O or parse errors. - fn resolve_descriptor_for_platform( - &self, - desc: &Descriptor, - native: &Platform, - ) -> Result> { - // Fast path: explicit platform annotation — no blob I/O needed. - if let Some(platform) = desc.platform().as_ref() { - if Self::platform_compatible(platform, native) { - return Ok(Some(self.read_json_blob::(desc)?)); + /// Append a descriptor to the index, optionally replacing any existing + /// entry with the same tag. + fn append_to_index(&self, desc: Descriptor, tag: Option<&str>) -> Result<()> { + let index = match self.read_index() { + Ok(mut index) => { + let mut manifests = index.manifests().clone(); + if let Some(tag) = tag { + manifests.retain(|d| !descriptor_is_tagged(d, tag)); + } + manifests.push(desc); + index.set_manifests(manifests); + index } - return Ok(None); - } - - // If there's no annotation then read the manifest and config. - let manifest = self.read_json_blob::(desc)?; + Err(Error::MissingImageIndex) => oci_image::ImageIndexBuilder::default() + .schema_version(oci_image::SCHEMA_VERSION) + .manifests(vec![desc]) + .build()?, + Err(e) => return Err(e), + }; + self.write_index(&index) + } - // Only image manifests (not OCI artifact manifests) carry a platform in - // their config blob. Skip the read entirely for anything else. - if manifest.config().media_type() != &MediaType::ImageConfig { - return Ok(None); - } + /// Write a manifest as a blob, and replace the index with a reference to it. + pub fn replace_with_single_manifest( + &self, + manifest: oci_image::ImageManifest, + platform: oci_image::Platform, + ) -> Result<()> { + let manifest = self + .write_json_blob(&manifest, MediaType::ImageManifest)? + .platform(platform) + .build() + .unwrap(); - let config: ImageConfiguration = self.read_json_blob(manifest.config())?; - if config.architecture() == native.architecture() && config.os() == native.os() { - Ok(Some(manifest)) - } else { - Ok(None) - } + let index_data = oci_image::ImageIndexBuilder::default() + .schema_version(oci_image::SCHEMA_VERSION) + .manifests(vec![manifest]) + .build() + .unwrap(); + self.write_index(&index_data) } +} - /// Verify a blob's SHA-256 digest matches its descriptor. - fn verify_blob_digest(&self, desc: &Descriptor) -> Result<()> { - let expected = sha256_of_descriptor(desc)?; - let mut f = self.read_blob(desc)?; - let mut hasher = Hasher::new(MessageDigest::sha256())?; - std::io::copy(&mut f, &mut hasher)?; - let found = hex::encode(hasher.finish()?); +impl OciRead for OciDir { + type BlobReader = File; + + fn read_blob(&self, desc: &Descriptor) -> Result { + let path = Self::parse_descriptor_to_path(desc)?; + let f = self.blobs_dir.open(path).map(|f| f.into_std())?; + let expected: u64 = desc.size(); + let found = f.metadata()?.len(); if expected != found { - return Err(Error::DigestMismatch { - expected: expected.into(), - found: found.into(), - }); + return Err(Error::SizeMismatch { expected, found }); } - Ok(()) + Ok(f) } - /// Verify a single manifest and all of its referenced objects. - /// Skips already validated blobs referenced by digest in `validated`, - /// and updates that set with ones we did validate. - fn fsck_one_manifest( - &self, - manifest: &ImageManifest, - validated: &mut HashSet>, - ) -> Result<()> { - let config_digest = sha256_of_descriptor(manifest.config())?; - if !validated.contains(config_digest) { - // Always verify the config blob digest, regardless of media type. - self.verify_blob_digest(manifest.config())?; - // Additionally validate the content structure for known types. - match manifest.config().media_type() { - MediaType::ImageConfig => { - let _: ImageConfiguration = self.read_json_blob(manifest.config())?; - } - MediaType::EmptyJSON => { - let _: EmptyDescriptor = self.read_json_blob(manifest.config())?; - } - // Per the OCI image spec, implementations MUST NOT error on - // encountering an unknown config mediaType. - _ => {} - } - validated.insert(config_digest.into()); - } - for layer in manifest.layers() { - let expected = sha256_of_descriptor(layer)?; - if validated.contains(expected) { - continue; - } - self.verify_blob_digest(layer)?; - validated.insert(expected.into()); - } - Ok(()) + fn read_index(&self) -> Result { + let r = if let Some(index) = self.dir.open_optional("index.json")?.map(BufReader::new) { + oci_image::ImageIndex::from_reader(index)? + } else { + return Err(Error::MissingImageIndex); + }; + Ok(r) } - /// Verify consistency of the index, its manifests, the config and blobs (all the latter) - /// by verifying their descriptor. - pub fn fsck(&self) -> Result { - let index = self.read_index()?; - let mut validated_blobs = HashSet::new(); - for manifest_descriptor in index.manifests() { - let expected_sha256 = sha256_of_descriptor(manifest_descriptor)?; - let manifest: ImageManifest = self.read_json_blob(manifest_descriptor)?; - validated_blobs.insert(expected_sha256.into()); - self.fsck_one_manifest(&manifest, &mut validated_blobs)?; - } - Ok(validated_blobs.len().try_into().unwrap()) + fn has_blob(&self, desc: &Descriptor) -> Result { + let path = OciDir::parse_descriptor_to_path(desc)?; + self.blobs_dir.try_exists(path).map_err(Into::into) } } From dd59af5d1267011e409ebd7cd99fdbcfdcb8273f Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 12 Jun 2026 11:50:13 +0200 Subject: [PATCH 2/2] Add OciArchive: read-only OCI image access from tar files Add OciArchive, a read-only OCI image backend that reads directly from tar archives using pread for concurrent access. It implements OciRead so all manifest resolution, fsck, and referrer logic works without code duplication. Key changes: - ArchiveReader: a Read+Seek adapter over a region of a file using pread - OciArchive: scans tar entries on open, serves reads via ArchiveReader Assisted-by: Claude Code (Claude Opus 4.6) Signed-off-by: Alexander Larsson --- src/lib.rs | 224 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index cdcc301..3cb889b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,8 +25,15 @@ use std::fmt::Debug; use std::fs::File; use std::io::{BufReader, BufWriter, prelude::*}; use std::marker::PhantomData; +#[cfg(unix)] +use std::os::unix::fs::FileExt; +#[cfg(windows)] +use std::os::windows::fs::FileExt; +#[cfg(not(any(unix, windows)))] +compile_error!("ocidir requires pread support (unix read_at or windows seek_read)"); use std::path::{Path, PathBuf}; use std::str::FromStr; +use std::sync::Arc; use thiserror::Error; // Re-export our dependencies that are used as part of the public API. @@ -1119,6 +1126,152 @@ impl OciRead for OciDir { } } +/// A reader for a region of a file, using pread for concurrent access. +#[derive(Debug)] +pub struct ArchiveReader { + file: Arc, + start: u64, + offset: u64, + size: u64, +} + +impl Read for ArchiveReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let remaining = (self.size - self.offset) as usize; + let to_read = buf.len().min(remaining); + if to_read == 0 { + return Ok(0); + } + #[cfg(unix)] + let n = self + .file + .read_at(&mut buf[..to_read], self.start + self.offset)?; + #[cfg(windows)] + let n = self + .file + .seek_read(&mut buf[..to_read], self.start + self.offset)?; + self.offset += n as u64; + Ok(n) + } +} + +impl Seek for ArchiveReader { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + let new_pos = match pos { + std::io::SeekFrom::Start(n) => n.min(self.size), + std::io::SeekFrom::End(n) => { + if n >= 0 { + self.size + } else { + self.size.saturating_sub((-n) as u64) + } + } + std::io::SeekFrom::Current(n) => { + if n >= 0 { + self.offset.saturating_add(n as u64).min(self.size) + } else { + self.offset.saturating_sub((-n) as u64) + } + } + }; + self.offset = new_pos; + Ok(new_pos) + } +} + +/// A read-only OCI image stored as a tar archive. +/// +/// This type provides read access to OCI images packaged as tar files +/// (the OCI archive format). It scans the tar entries on open and uses +/// pread-based access for concurrent blob reads. +#[derive(Debug)] +pub struct OciArchive { + file: Arc, + /// Map from tar entry path (e.g. "blobs/sha256/abcd...") to (offset, size). + entries: HashMap, +} + +impl OciArchive { + /// Open an OCI archive from a filesystem path. + pub fn open(path: impl AsRef) -> Result { + let file = File::open(path)?; + Self::from_file(file) + } + + /// Open an OCI archive from an already-opened file. + pub fn from_file(file: File) -> Result { + let mut archive = tar::Archive::new(&file); + let mut entries = HashMap::new(); + for entry in archive.entries()? { + let entry = entry?; + let path: PathBuf = entry + .path()? + .components() + .filter(|c| *c != std::path::Component::CurDir) + .collect(); + let offset = entry.raw_file_position(); + let size = entry.size(); + entries.insert(path, (offset, size)); + } + Ok(Self { + file: Arc::new(file), + entries, + }) + } + + fn open_entry(&self, path: &Path) -> Result { + let &(offset, size) = self.entries.get(path).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("not found in archive: {}", path.display()), + ) + })?; + Ok(ArchiveReader { + file: Arc::clone(&self.file), + start: offset, + offset: 0, + size, + }) + } +} + +impl OciRead for OciArchive { + type BlobReader = ArchiveReader; + + fn read_blob(&self, desc: &Descriptor) -> Result { + let sha256 = sha256_of_descriptor(desc)?; + let path = Path::new(BLOBDIR).join(sha256); + let reader = self.open_entry(&path)?; + let expected = desc.size(); + if expected != reader.size { + return Err(Error::SizeMismatch { + expected, + found: reader.size, + }); + } + Ok(reader) + } + + fn read_index(&self) -> Result { + let reader = self + .open_entry(Path::new("index.json")) + .map_err(|e| match e { + Error::Io(ref io) if io.kind() == std::io::ErrorKind::NotFound => { + Error::MissingImageIndex + } + other => other, + })?; + let index = oci_image::ImageIndex::from_reader(BufReader::new(reader))?; + Ok(index) + } + + fn has_blob(&self, desc: &Descriptor) -> Result { + let digest = sha256_of_descriptor(desc)?; + let path = Path::new(BLOBDIR).join(digest); + Ok(self.entries.contains_key(&path)) + } +} + impl<'a> BlobWriter<'a> { fn new(ocidir: &'a Dir) -> Result { Ok(Self { @@ -2354,4 +2507,75 @@ mod tests { Ok(()) } + + fn append_cap_file( + tar: &mut tar::Builder, + path: &str, + f: cap_std::fs::File, + ) -> Result<()> { + let f = f.into_std(); + let len = f.metadata()?.len(); + let mut header = tar::Header::new_gnu(); + header.set_size(len); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data(&mut header, path, &f)?; + Ok(()) + } + + fn ocidir_to_tar_file(w: &OciDir) -> Result { + let archive_path = std::env::temp_dir().join("test_oci_archive.tar"); + let archive_file = std::fs::File::create(&archive_path)?; + let mut tar_builder = tar::Builder::new(archive_file); + + append_cap_file(&mut tar_builder, "oci-layout", w.dir().open("oci-layout")?)?; + append_cap_file(&mut tar_builder, "index.json", w.dir().open("index.json")?)?; + + for entry in w.blobs_dir().entries()? { + let entry = entry?; + let name = entry.file_name(); + let path = format!("{BLOBDIR}/{}", name.to_string_lossy()); + append_cap_file(&mut tar_builder, &path, w.blobs_dir().open(name)?)?; + } + + tar_builder.finish()?; + let f = std::fs::File::open(&archive_path)?; + std::fs::remove_file(&archive_path)?; + Ok(f) + } + + #[test] + fn test_open_archive() -> Result<()> { + let (_td, w) = new_ocidir()?; + + let root_layer = create_test_layer(&w, b"pretend this is a tarball")?; + let mut manifest = w.new_empty_manifest()?.build()?; + let mut config = new_empty_config(); + w.push_layer(&mut manifest, &mut config, root_layer, "root", None); + w.insert_manifest_and_config( + manifest, + config, + Some("latest"), + oci_image::Platform::default(), + )?; + + let expected_fsck = w.fsck()?; + + let tar_file = ocidir_to_tar_file(&w)?; + let archive = OciArchive::from_file(tar_file)?; + + let index = archive.read_index()?; + assert_eq!(index.manifests().len(), 1); + + let resolved = archive.open_image_this_platform(Some("latest"))?; + assert_eq!(resolved.manifest.layers().len(), 1); + + assert_eq!(archive.fsck()?, expected_fsck); + + let found = archive.find_manifest_with_tag("latest")?; + assert!(found.is_some()); + assert!(archive.find_manifest_with_tag("noent")?.is_none()); + + Ok(()) + } }