diff --git a/api/src/config.rs b/api/src/config.rs index 810640e7e88..40bbecbe3bc 100644 --- a/api/src/config.rs +++ b/api/src/config.rs @@ -145,9 +145,12 @@ impl ConfigV2 { /// Get configuration information for RAFS filesystem. pub fn get_rafs_config(&self) -> Result<&RafsConfigV2> { - self.rafs - .as_ref() - .ok_or_else(|| einval!("no configuration information for rafs")) + self.rafs.as_ref().ok_or_else(|| { + Error::new( + ErrorKind::InvalidInput, + "no configuration information for rafs", + ) + }) } /// Clone the object with all secrets removed. @@ -644,6 +647,9 @@ pub struct RafsConfigV2 { /// Filesystem prefetching configuration. #[serde(default)] pub prefetch: PrefetchConfigV2, + /// Probe TOC list included in the blob. + #[serde(default)] + pub probe_toc: bool, } impl RafsConfigV2 { @@ -969,6 +975,9 @@ struct RafsConfig { // ZERO value means, amplifying user io is not enabled. #[serde(default = "default_batch_size")] pub amplify_io: usize, + /// Probe TOC list included in the blob. + #[serde(default)] + pub probe_toc: bool, } impl TryFrom for ConfigV2 { @@ -986,6 +995,7 @@ impl TryFrom for ConfigV2 { access_pattern: v.access_pattern, latest_read_files: v.latest_read_files, prefetch: v.fs_prefetch.into(), + probe_toc: v.probe_toc, }; if !cache.prefetch.enable && rafs.prefetch.enable { cache.prefetch = rafs.prefetch.clone(); diff --git a/rafs/src/fs.rs b/rafs/src/fs.rs index 6eb24aca15e..c4524bbacb5 100644 --- a/rafs/src/fs.rs +++ b/rafs/src/fs.rs @@ -86,7 +86,10 @@ impl Rafs { let blob_infos = sb.superblock.get_blob_infos(); let device = BlobDevice::new(cfg, &blob_infos).map_err(RafsError::CreateDevice)?; - if cfg.is_chunk_validation_enabled() && sb.meta.has_inlined_chunk_digest() { + if rafs_cfg.probe_toc + && cfg.is_chunk_validation_enabled() + && sb.meta.has_inlined_chunk_digest() + { sb.superblock.set_blob_device(device.clone()); } diff --git a/rafs/src/metadata/direct_v6.rs b/rafs/src/metadata/direct_v6.rs index fbb63fc4f0a..2d394c3020c 100644 --- a/rafs/src/metadata/direct_v6.rs +++ b/rafs/src/metadata/direct_v6.rs @@ -1152,7 +1152,7 @@ impl RafsInodeExt for OndiskInodeWrapper { + (idx as usize * size_of::()); let chunk_addr = state.map.get_ref::(offset)?; - if state.meta.has_inlined_chunk_digest() { + if state.meta.probe_toc && state.meta.has_inlined_chunk_digest() { let blob_index = chunk_addr.blob_index(); let chunk_index = chunk_addr.blob_ci_index(); let device = self.mapping.device.lock().unwrap(); diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs index aa849901528..5efde38466b 100644 --- a/rafs/src/metadata/mod.rs +++ b/rafs/src/metadata/mod.rs @@ -431,6 +431,9 @@ pub struct RafsSuperMeta { pub chunk_table_offset: u64, /// Size of the chunk table for RAFS v6. pub chunk_table_size: u64, + + /// Probe TOC list included in the blob. + pub probe_toc: bool, } impl RafsSuperMeta { @@ -519,6 +522,7 @@ impl Default for RafsSuperMeta { is_chunk_dict: false, chunk_table_offset: 0, chunk_table_size: 0, + probe_toc: false, } } } @@ -634,9 +638,12 @@ impl Default for RafsSuper { impl RafsSuper { /// Create a new `RafsSuper` instance from a `RafsConfigV2` object. pub fn new(conf: &RafsConfigV2) -> Result { + let mut meta = RafsSuperMeta::default(); + meta.probe_toc = conf.probe_toc; Ok(Self { mode: RafsMode::from_str(conf.mode.as_str())?, validate_digest: conf.validate, + meta, ..Default::default() }) } @@ -655,12 +662,18 @@ impl RafsSuper { is_chunk_dict: bool, config: Arc, ) -> Result<(Self, RafsIoReader)> { + let mut probe_toc = false; + if let Ok(rafs_config) = config.get_rafs_config() { + probe_toc = rafs_config.probe_toc; + } + let mut rs = RafsSuper { mode: RafsMode::Direct, validate_digest, ..Default::default() }; rs.meta.is_chunk_dict = is_chunk_dict; + rs.meta.probe_toc = probe_toc; // open bootstrap file let file = OpenOptions::new() @@ -669,25 +682,28 @@ impl RafsSuper { .open(path.as_ref())?; let mut reader = Box::new(file) as RafsIoReader; - if let Err(e) = rs.load(&mut reader) { - let id = BlobInfo::get_blob_id_from_meta_path(path.as_ref())?; - let new_path = match TocEntryList::extract_rafs_meta(&id, config.clone()) { - Ok(v) => v, - Err(_e) => { - debug!("failed to load inlined RAFS meta, {}", _e); - return Err(e); - } - }; - let file = OpenOptions::new().read(true).write(false).open(new_path)?; - reader = Box::new(file) as RafsIoReader; + if probe_toc { + if let Err(e) = rs.load(&mut reader) { + let id = BlobInfo::get_blob_id_from_meta_path(path.as_ref())?; + let new_path = match TocEntryList::extract_rafs_meta(&id, config.clone()) { + Ok(v) => v, + Err(_e) => { + debug!("failed to load inlined RAFS meta, {}", _e); + return Err(e); + } + }; + let file = OpenOptions::new().read(true).write(false).open(new_path)?; + reader = Box::new(file) as RafsIoReader; + rs.load(&mut reader)?; + rs.set_blob_id_from_meta_path(path.as_ref())?; + } + if (validate_digest || config.is_chunk_validation_enabled()) + && rs.meta.has_inlined_chunk_digest() + { + rs.create_blob_device(config)?; + } + } else { rs.load(&mut reader)?; - rs.set_blob_id_from_meta_path(path.as_ref())?; - } - - if (validate_digest || config.is_chunk_validation_enabled()) - && rs.meta.has_inlined_chunk_digest() - { - rs.create_blob_device(config)?; } Ok((rs, reader)) diff --git a/src/bin/nydus-image/builder/mod.rs b/src/bin/nydus-image/builder/mod.rs index 4a42b64e4d4..536ea67c380 100644 --- a/src/bin/nydus-image/builder/mod.rs +++ b/src/bin/nydus-image/builder/mod.rs @@ -84,11 +84,7 @@ fn dump_bootstrap( } } if !ctx.conversion_type.is_to_ref() { - if ctx.blob_inline_meta { - blob_ctx.compressed_blob_size = 0; - } else { - blob_ctx.compressed_blob_size = blob_writer.pos()?; - } + blob_ctx.compressed_blob_size = blob_writer.pos()?; } } @@ -174,11 +170,7 @@ fn finalize_blob( dump_toc(ctx, blob_ctx, blob_writer)?; if !ctx.conversion_type.is_to_ref() { - if ctx.blob_inline_meta { - blob_ctx.compressed_blob_size = 0; - } else { - blob_ctx.compressed_blob_size = blob_writer.pos()?; - } + blob_ctx.compressed_blob_size = blob_writer.pos()?; } if ctx.blob_inline_meta && blob_ctx.blob_id == "x".repeat(64) { blob_ctx.blob_id = String::new(); diff --git a/src/bin/nydus-image/core/chunk_dict.rs b/src/bin/nydus-image/core/chunk_dict.rs index d627036a617..cb578011793 100644 --- a/src/bin/nydus-image/core/chunk_dict.rs +++ b/src/bin/nydus-image/core/chunk_dict.rs @@ -133,7 +133,7 @@ impl HashChunkDict { }; rafs_config.check_compatibility(&rs.meta)?; - if rs.meta.is_v5() || rs.meta.has_inlined_chunk_digest() { + if rs.meta.is_v5() || rs.meta.probe_toc && rs.meta.has_inlined_chunk_digest() { Tree::from_bootstrap(&rs, &mut d).context("failed to build tree from bootstrap")?; } else if rs.meta.is_v6() { d.load_chunk_table(&rs) diff --git a/src/bin/nydus-image/core/context.rs b/src/bin/nydus-image/core/context.rs index c966e632669..1478c9f990e 100644 --- a/src/bin/nydus-image/core/context.rs +++ b/src/bin/nydus-image/core/context.rs @@ -472,8 +472,14 @@ impl BlobContext { let mut blob_id = blob.raw_blob_id().to_string(); let mut features = blob.features(); + let mut probe_toc = false; + if let Ok(rafs_config) = ctx.configuration.get_rafs_config() { + probe_toc = rafs_config.probe_toc; + } + // Fixes up blob info objects from inlined-meta blobs. - if features.contains(BlobFeatures::INLINED_META) + if probe_toc + && features.contains(BlobFeatures::INLINED_META) && (chunk_source == ChunkSource::Dict || chunk_source == ChunkSource::Parent) { let backend_config = ctx diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs index 2ec65328da1..bc8e9b14b2e 100644 --- a/src/bin/nydus-image/merge.rs +++ b/src/bin/nydus-image/merge.rs @@ -31,7 +31,6 @@ use crate::core::tree::{MetadataTreeBuilder, Tree}; pub struct Merger {} impl Merger { - /* /// Merge assumes the bootstrap name is in `$blob_digest` format. fn get_raw_blob_digest(bootstrap_path: &Path) -> Result { let file_name = bootstrap_path @@ -41,7 +40,6 @@ impl Merger { .ok_or_else(|| anyhow!("convert to string"))?; Ok(file_name.to_string()) } - */ fn get_digest_from_list(digests: &Option>, idx: usize) -> Result> { Ok(if let Some(digests) = &digests { @@ -163,7 +161,7 @@ impl Merger { // The blob id (blob sha256 hash) in parent bootstrap is invalid for nydusd // runtime, should change it to the hash of whole tar blob. - blob_ctx.blob_id = blob.blob_id().to_owned(); + blob_ctx.blob_id = Self::get_raw_blob_digest(bootstrap_path)?; if let Some(digest) = Self::get_digest_from_list(&blob_digests, layer_idx)? { if blob.has_feature(BlobFeatures::ZRAN) { blob_ctx.rafs_blob_digest = digest; diff --git a/storage/src/device.rs b/storage/src/device.rs index 6a75799db4d..358607ae038 100644 --- a/storage/src/device.rs +++ b/storage/src/device.rs @@ -196,9 +196,7 @@ impl BlobInfo { /// Get the id of the blob, with special handling of `inlined-meta` case. pub fn blob_id(&self) -> String { - if self.has_feature(BlobFeatures::INLINED_META) && !self.has_feature(BlobFeatures::ZRAN) - || !self.meta_ci_is_valid() - { + if self.has_feature(BlobFeatures::INLINED_META) && !self.has_feature(BlobFeatures::ZRAN) { let guard = self.meta_path.lock().unwrap(); if !guard.is_empty() { return guard.deref().clone(); @@ -451,16 +449,13 @@ impl BlobInfo { /// Get RAFS blob id for ZRan. pub fn get_rafs_blob_id(&self) -> Result { assert!(self.has_feature(BlobFeatures::ZRAN)); - let id = if self.has_feature(BlobFeatures::INLINED_META) { + if self.has_feature(BlobFeatures::INLINED_META) { let guard = self.meta_path.lock().unwrap(); - if guard.is_empty() { - return Err(einval!("failed to get blob id from meta file name")); + if !guard.is_empty() { + return Ok(guard.deref().clone()); } - guard.deref().clone() - } else { - hex::encode(&self.rafs_blob_digest) - }; - Ok(id) + } + Ok(hex::encode(&self.rafs_blob_digest)) } }