//! Big files are split into content-defined chunks, so a small edit to a big //! file stores and moves only the chunks that changed. use crate::object::{Kind, Object, decode_chunks, encode_chunks}; use crate::{Id, Result, Source, corrupt}; /// Files bigger than this are chunked. pub const CHUNK_OVER: usize = 1 << 20; const MIN: u32 = 16 * 1024; const AVG: u32 = 64 * 1024; const MAX: u32 = 256 * 1024; /// The objects for one file's content: one blob, or a chunk list plus its /// chunk blobs. The first object's ID is the file's ID. pub fn file_objects(content: &[u8]) -> Vec { if content.len() <= CHUNK_OVER { return vec![Object::blob(content.to_vec())]; } let mut chunks = Vec::new(); let mut list = Vec::new(); for c in fastcdc::v2020::FastCDC::new(content, MIN, AVG, MAX) { let blob = Object::blob(content[c.offset..c.offset + c.length].to_vec()); list.push((blob.id(), c.length as u64)); chunks.push(blob); } let mut out = vec![Object::new(Kind::Chunks, encode_chunks(&list))]; out.extend(chunks); out } /// The file ID for `content` without building every object. pub fn file_id(content: &[u8]) -> Id { if content.len() <= CHUNK_OVER { return Id::of(Kind::Blob as u8, content); } file_objects(content)[0].id() } /// A file's content from its ID (a blob or a chunk list). pub fn read_file(src: &dyn Source, id: &Id) -> Result> { let object = src.must(id)?; match object.kind { Kind::Blob => Ok(object.data), Kind::Chunks => { let list = decode_chunks(&object.data)?; let total: u64 = list.iter().map(|(_, n)| n).sum(); let mut out = Vec::with_capacity(total as usize); for (chunk, len) in list { let blob = src.must(&chunk)?; if blob.data.len() as u64 != len { return Err(corrupt("chunk length mismatch")); } out.extend_from_slice(&blob.data); } Ok(out) } Kind::Tree => Err(corrupt(format!("{} is a tree, not a file", id.short()))), } } /// A file's size from its ID without reading chunk contents. pub fn file_size(src: &dyn Source, id: &Id) -> Result { let object = src.must(id)?; match object.kind { Kind::Blob => Ok(object.data.len() as u64), Kind::Chunks => Ok(decode_chunks(&object.data)?.iter().map(|(_, n)| n).sum()), Kind::Tree => Err(corrupt("a tree has no size")), } } #[cfg(test)] mod tests { use super::*; use crate::MemSource; #[test] fn big_files_round_trip() { let mut content = Vec::new(); for i in 0..400_000u32 { content.extend_from_slice(&i.to_le_bytes()); } let objects = file_objects(&content); assert!(objects.len() > 3); assert_eq!(objects[0].kind, Kind::Chunks); assert_eq!(file_id(&content), objects[0].id()); let mut src = MemSource::default(); for o in objects { src.put(o); } assert_eq!(read_file(&src, &file_id(&content)).unwrap(), content); // A one-byte edit in the middle changes few chunks. let mut edited = content.clone(); edited[800_000] ^= 1; let before: std::collections::HashSet = file_objects(&content).iter().map(|o| o.id()).collect(); let after = file_objects(&edited); let new = after.iter().filter(|o| !before.contains(&o.id())).count(); assert!(new <= 3, "{new} new objects"); } #[test] fn small_files_are_blobs() { assert_eq!(file_objects(b"hi").len(), 1); assert_eq!(file_id(b"hi"), Object::blob(b"hi".to_vec()).id()); } }