//! Workspaces on disk: finding one, locking it, its stat index, saving the //! folder into a tree, and writing a tree back into the folder. use std::collections::BTreeMap; use std::fs; use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::path::{Path, PathBuf}; use anyhow::{Context, Result, anyhow}; use rayon::prelude::*; use vcs_core::Id; use vcs_core::tree::{self, Entry, EntryKind, Files}; use crate::objects::Store; pub const DIR: &str = ".vcs"; /// The workspace folder containing `start`, if any. pub fn find(start: &Path) -> Option { let mut dir = Some(start); while let Some(d) = dir { if d.join(DIR).join("workspace").is_file() { return Some(d.to_path_buf()); } dir = d.parent(); } None } pub fn read_id(root: &Path) -> Result { Ok(fs::read_to_string(root.join(DIR).join("workspace"))?.trim().to_string()) } pub fn write_id(root: &Path, id: &str) -> Result<()> { let dir = root.join(DIR); fs::create_dir_all(&dir)?; fs::write(dir.join("workspace"), format!("{id}\n"))?; // Keep the folder out of anything that walks the tree. fs::write(dir.join(".gitignore"), "*\n")?; Ok(()) } /// An exclusive lock on the workspace, held while a command or the saver /// works in it. pub struct Lock { _file: fs::File, } pub fn lock(root: &Path) -> Result { use fs4::fs_std::FileExt; let path = root.join(DIR).join("lock"); let file = fs::OpenOptions::new().create(true).truncate(false).write(true).open(&path)?; file.lock_exclusive().context("locking the workspace")?; Ok(Lock { _file: file }) } /// Like [`lock`], but gives up at once if someone else holds it. pub fn try_lock(root: &Path) -> Result> { use fs4::fs_std::FileExt; let path = root.join(DIR).join("lock"); let file = fs::OpenOptions::new().create(true).truncate(false).write(true).open(&path)?; match file.try_lock_exclusive() { Ok(true) => Ok(Some(Lock { _file: file })), Ok(false) => Ok(None), Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => Ok(None), Err(e) => Err(e.into()), } } // The stat index: what each tracked file looked like when we last knew its // blob, so unchanged files are never read again. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Stat { pub size: u64, pub mtime_ns: i64, pub ino: u64, } impl Stat { fn of(meta: &fs::Metadata) -> Stat { Stat { size: meta.len(), mtime_ns: meta.mtime() * 1_000_000_000 + meta.mtime_nsec(), ino: meta.ino(), } } } #[derive(Debug, Clone, Copy)] pub struct IndexEntry { pub entry: Entry, pub stat: Stat, } #[derive(Debug, Default, Clone)] pub struct Index { pub entries: BTreeMap, /// When the index was written (ns); files changed within a second of it /// are re-read next time (git's "racy" rule). pub written_ns: i64, } const MAGIC: &[u8] = b"VCSI2\n"; impl Index { pub fn load(root: &Path) -> Index { Self::try_load(root).unwrap_or_default() } fn try_load(root: &Path) -> Result { let bytes = fs::read(root.join(DIR).join("index"))?; if !bytes.starts_with(MAGIC) { return Err(anyhow!("bad index")); } let mut at = MAGIC.len(); let take = |at: &mut usize, n: usize| -> Result<&[u8]> { let slice = bytes.get(*at..*at + n).ok_or_else(|| anyhow!("short index"))?; *at += n; Ok(slice) }; let written_ns = i64::from_le_bytes(take(&mut at, 8)?.try_into()?); let count = u32::from_le_bytes(take(&mut at, 4)?.try_into()?); let mut entries = BTreeMap::new(); for _ in 0..count { let len = u16::from_le_bytes(take(&mut at, 2)?.try_into()?) as usize; let path = String::from_utf8(take(&mut at, len)?.to_vec())?; let id = Id::from_slice(take(&mut at, 32)?).ok_or_else(|| anyhow!("bad index"))?; let kind = EntryKind::from_u8(take(&mut at, 1)?[0]).ok_or_else(|| anyhow!("bad index"))?; let size = u64::from_le_bytes(take(&mut at, 8)?.try_into()?); let mtime_ns = i64::from_le_bytes(take(&mut at, 8)?.try_into()?); let ino = u64::from_le_bytes(take(&mut at, 8)?.try_into()?); entries.insert(path, IndexEntry { entry: Entry { kind, id }, stat: Stat { size, mtime_ns, ino } }); } Ok(Index { entries, written_ns }) } pub fn save(&mut self, root: &Path) -> Result<()> { self.written_ns = now_ns(); let mut out = Vec::with_capacity(self.entries.len() * 80 + 32); out.extend_from_slice(MAGIC); out.extend_from_slice(&self.written_ns.to_le_bytes()); out.extend_from_slice(&(self.entries.len() as u32).to_le_bytes()); for (path, e) in &self.entries { out.extend_from_slice(&(path.len() as u16).to_le_bytes()); out.extend_from_slice(path.as_bytes()); out.extend_from_slice(&e.entry.id.0); out.push(e.entry.kind as u8); out.extend_from_slice(&e.stat.size.to_le_bytes()); out.extend_from_slice(&e.stat.mtime_ns.to_le_bytes()); out.extend_from_slice(&e.stat.ino.to_le_bytes()); } let tmp = root.join(DIR).join("index.tmp"); let mut file = fs::File::create(&tmp)?; file.write_all(&out)?; file.sync_data()?; fs::rename(tmp, root.join(DIR).join("index"))?; Ok(()) } /// Whether `stat` proves the file still has the indexed content. fn unchanged(&self, entry: &IndexEntry, stat: Stat) -> bool { entry.stat == stat && stat.mtime_ns < self.written_ns - 1_000_000_000 } } fn now_ns() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_nanos() as i64) .unwrap_or(0) } // Rules for what a workspace keeps out of its trees. /// What `vcs.toml` says. #[derive(Debug, Default, Clone, serde::Deserialize)] pub struct Config { #[serde(default)] pub workspace: WorkspaceConfig, #[serde(default)] pub mine: MineConfig, } #[derive(Debug, Default, Clone, serde::Deserialize)] pub struct WorkspaceConfig { #[serde(default)] pub copy: Vec, } #[derive(Debug, Default, Clone, serde::Deserialize)] pub struct MineConfig { #[serde(default)] pub paths: Vec, } pub fn config(root: &Path) -> Config { fs::read_to_string(root.join("vcs.toml")) .ok() .and_then(|text| toml::from_str(&text).ok()) .unwrap_or_default() } pub const DEFAULT_MINE: &[&str] = &[".env", ".env.*", "!.env.example", "!.env.sample", "!.env.template"]; /// Paths that stay the user's own: never saved into a change, never landed. pub struct Mine { include: globset::GlobSet, exclude: globset::GlobSet, } impl Mine { /// The project's rules (vcs.toml, or the defaults) plus the person's own. pub fn new(config: &Config, personal: &[String]) -> Mine { let mut include = globset::GlobSetBuilder::new(); let mut exclude = globset::GlobSetBuilder::new(); let mut patterns: Vec = if config.mine.paths.is_empty() { DEFAULT_MINE.iter().map(|s| s.to_string()).collect() } else { config.mine.paths.clone() }; patterns.extend(personal.iter().cloned()); for pattern in patterns { let (set, p) = match pattern.strip_prefix('!') { Some(p) => (&mut exclude, p.to_string()), None => (&mut include, pattern), }; // A bare name matches at any depth, as in .gitignore. let p = p.trim_end_matches('/').to_string(); let globs = if p.contains('/') { vec![p.trim_start_matches('/').to_string()] } else { vec![p.clone(), format!("**/{p}")] }; for g in globs { for g in [g.clone(), format!("{g}/**")] { if let Ok(glob) = globset::GlobBuilder::new(&g).literal_separator(true).build() { set.add(glob); } } } } Mine { include: include.build().unwrap_or_else(|_| globset::GlobSet::empty()), exclude: exclude.build().unwrap_or_else(|_| globset::GlobSet::empty()), } } pub fn matches(&self, path: &str) -> bool { self.include.is_match(path) && !self.exclude.is_match(path) } } /// What saving a workspace found. pub struct Snapshot { pub files: Files, pub tree: Id, pub index: Index, /// Mine files in the folder (kept out of the tree): path, file ID, exec. pub mine: Vec<(String, Id, bool)>, } struct Seen { stat: Stat, kind: EntryKind, abs: PathBuf, } fn kind_of(meta: &fs::Metadata) -> EntryKind { if meta.file_type().is_symlink() { EntryKind::Link } else if meta.permissions().mode() & 0o111 != 0 { EntryKind::Exec } else { EntryKind::File } } fn in_scope(only: &[String], path: &str) -> bool { only.is_empty() || only.iter().any(|o| tree::under(path, o)) } /// Saves the folder's content into the store and returns its tree. `base` /// is the tree the change starts from: files it tracks are kept even when /// .gitignore matches them, and with `only` set every path outside those /// folders comes from it unchanged. /// `main_paths` are the files the change's base tracks: rules for mine /// files never take a file main already has. pub fn snapshot( root: &Path, store: &Store, base: &Files, only: &[String], index: &Index, personal: &[String], main_paths: &std::collections::HashSet, ) -> Result { let config = config(root); let mine = Mine::new(&config, personal); // Walk the folder, honouring .gitignore files inside it. let mut walk = ignore::WalkBuilder::new(root); walk.hidden(false) .git_ignore(true) .git_global(false) .git_exclude(false) .ignore(false) .parents(false) .require_git(false) .follow_links(false) .filter_entry(|e| { let name = e.file_name(); !(e.depth() == 1 && (name == DIR || name == ".git")) }); let found = std::sync::Mutex::new(Vec::<(String, Seen)>::new()); walk.build_parallel().run(|| { let found = &found; Box::new(move |result| { let Ok(entry) = result else { return ignore::WalkState::Continue }; if entry.depth() == 0 { return ignore::WalkState::Continue; } let Some(kind) = entry.file_type() else { return ignore::WalkState::Continue }; if kind.is_dir() { return ignore::WalkState::Continue; } let Ok(rel) = entry.path().strip_prefix(root) else { return ignore::WalkState::Continue }; let Some(rel) = rel.to_str() else { return ignore::WalkState::Continue }; let Ok(meta) = entry.path().symlink_metadata() else { return ignore::WalkState::Continue }; let seen = Seen { stat: Stat::of(&meta), kind: kind_of(&meta), abs: entry.path().to_path_buf() }; found.lock().unwrap().push((rel.to_string(), seen)); ignore::WalkState::Continue }) }); let mut seen: BTreeMap = found.into_inner().unwrap().into_iter().collect(); // Tracked files the walk skipped (ignored but tracked) still count. for path in base.keys().chain(index.entries.keys()) { if seen.contains_key(path) || !in_scope(only, path) { continue; } let abs = root.join(path); if let Ok(meta) = abs.symlink_metadata() && !meta.is_dir() { seen.insert(path.clone(), Seen { stat: Stat::of(&meta), kind: kind_of(&meta), abs }); } } // Mine files stay out of trees unless main already tracks them. let mut mine_paths = Vec::new(); seen.retain(|path, s| { if !main_paths.contains(path) && mine.matches(path) { mine_paths.push((path.clone(), s.abs.clone(), s.kind == EntryKind::Exec)); false } else { in_scope(only, path) } }); // Read and store what might have changed, in parallel. Objects are // files named by their IDs, so parallel writes are safe. let jobs: Vec<(&String, &Seen)> = seen .iter() .filter(|(path, s)| match index.entries.get(*path) { Some(e) => !(index.unchanged(e, s.stat) && e.entry.kind == s.kind), None => true, }) .collect(); let stored: BTreeMap = jobs .par_iter() .map(|(path, s)| -> Result<(String, Id)> { let bytes = read_content(&s.abs, s.kind)?; Ok(((*path).clone(), store.put_file(&bytes)?)) }) .collect::>>()?; let mut files = Files::new(); for (path, entry) in base { if !in_scope(only, path) { files.insert(path.clone(), *entry); } } let mut next = Index::default(); for (path, s) in &seen { let id = match stored.get(path) { Some(id) => *id, None => index.entries[path].entry.id, }; let entry = Entry { kind: s.kind, id }; files.insert(path.clone(), entry); next.entries.insert(path.clone(), IndexEntry { entry, stat: s.stat }); } let (tree, objects) = tree::build(&files); for o in &objects { store.put(o)?; } let mut mine_out = Vec::new(); for (path, abs, exec) in mine_paths { let bytes = read_content(&abs, EntryKind::File)?; mine_out.push((path, store.put_file(&bytes)?, exec)); } Ok(Snapshot { files, tree, index: next, mine: mine_out }) } fn read_content(path: &Path, kind: EntryKind) -> Result> { if kind == EntryKind::Link { let target = fs::read_link(path)?; return Ok(target.to_string_lossy().into_owned().into_bytes()); } let mut bytes = Vec::new(); fs::File::open(path) .with_context(|| format!("reading {}", path.display()))? .read_to_end(&mut bytes)?; Ok(bytes) } /// Another workspace to copy unchanged files from. pub struct CowSource<'a> { pub root: &'a Path, pub index: &'a Index, } /// Makes the folder hold exactly `target` (within `only`), changing only /// what differs from `index`. Untracked and ignored files are left alone. /// Returns the new index. pub fn materialize(root: &Path, store: &Store, index: &Index, target: &Files, only: &[String], cow: Option) -> Result { let wanted: BTreeMap<&String, &Entry> = target.iter().filter(|(p, _)| in_scope(only, p)).collect(); // Remove tracked files that the target doesn't have. let mut removed_dirs = Vec::new(); for path in index.entries.keys() { if !wanted.contains_key(path) { let abs = root.join(path); if abs.symlink_metadata().is_ok_and(|m| !m.is_dir()) { fs::remove_file(&abs)?; } if let Some(parent) = abs.parent() { removed_dirs.push(parent.to_path_buf()); } } } // Empty folders left behind go too (never the root). removed_dirs.sort_by_key(|d| std::cmp::Reverse(d.components().count())); removed_dirs.dedup(); for dir in removed_dirs { let mut d = dir.as_path(); while d != root && d.starts_with(root) { if fs::remove_dir(d).is_err() { break; } match d.parent() { Some(p) => d = p, None => break, } } } // Write what's new or different. let mut keep = Index::default(); let mut jobs = Vec::new(); for (path, entry) in &wanted { if let Some(existing) = index.entries.get(*path) && existing.entry == **entry && let Ok(meta) = root.join(path).symlink_metadata() && Stat::of(&meta) == existing.stat { keep.entries.insert((*path).clone(), *existing); continue; } jobs.push(((*path).clone(), **entry)); } let written: Vec<(String, IndexEntry)> = jobs .par_iter() .map(|(path, entry)| -> Result<(String, IndexEntry)> { let abs = root.join(path); if let Some(parent) = abs.parent() { make_dirs(root, parent)?; } if let Ok(meta) = abs.symlink_metadata() { if meta.is_dir() { fs::remove_dir_all(&abs)?; } else { fs::remove_file(&abs)?; } } let copied = match &cow { Some(src) if entry.kind != EntryKind::Link => copy_from(src, path, entry, &abs), _ => false, }; if !copied { let content = store.read_file(&entry.id).with_context(|| format!("missing content for {path}"))?; if entry.kind == EntryKind::Link { std::os::unix::fs::symlink(String::from_utf8_lossy(&content).into_owned(), &abs)?; } else { fs::write(&abs, content)?; } } if entry.kind != EntryKind::Link { let mode = if entry.kind == EntryKind::Exec { 0o755 } else { 0o644 }; fs::set_permissions(&abs, fs::Permissions::from_mode(mode))?; } let meta = abs.symlink_metadata()?; Ok((path.clone(), IndexEntry { entry: *entry, stat: Stat::of(&meta) })) }) .collect::>>()?; for (path, e) in written { keep.entries.insert(path, e); } Ok(keep) } /// Creates `dir` and its parents, replacing any file in the way. fn make_dirs(root: &Path, dir: &Path) -> Result<()> { if dir.is_dir() { return Ok(()); } let mut missing = Vec::new(); let mut d = dir; while d != root && !d.is_dir() { missing.push(d.to_path_buf()); match d.parent() { Some(p) => d = p, None => break, } } for m in missing.into_iter().rev() { if m.symlink_metadata().is_ok_and(|meta| !meta.is_dir()) { fs::remove_file(&m)?; } match fs::create_dir(&m) { Ok(()) => {} Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} Err(e) => return Err(e.into()), } } Ok(()) } /// Copies one file from another workspace if it still has the same content. fn copy_from(src: &CowSource, path: &str, entry: &Entry, dest: &Path) -> bool { let Some(theirs) = src.index.entries.get(path) else { return false }; if theirs.entry != *entry { return false; } let from = src.root.join(path); let Ok(meta) = from.symlink_metadata() else { return false }; if Stat::of(&meta) != theirs.stat { return false; } reflink_copy::reflink_or_copy(&from, dest).is_ok() } /// Copies a build folder (like node_modules) from another workspace, /// copy-on-write where the disk supports it. Returns how many files. pub fn copy_folder(from: &Path, to: &Path) -> Result { let mut count = 0; if !from.is_dir() || to.exists() { return Ok(0); } let mut stack = vec![from.to_path_buf()]; while let Some(dir) = stack.pop() { let rel = dir.strip_prefix(from)?; fs::create_dir_all(to.join(rel))?; for entry in fs::read_dir(&dir)? { let entry = entry?; let path = entry.path(); let dest = to.join(path.strip_prefix(from)?); let kind = entry.file_type()?; if kind.is_dir() { stack.push(path); } else if kind.is_symlink() { std::os::unix::fs::symlink(fs::read_link(&path)?, &dest)?; count += 1; } else { reflink_copy::reflink_or_copy(&path, &dest)?; count += 1; } } } Ok(count) } /// Whether text still holds conflict markers. pub fn has_markers(bytes: &[u8]) -> bool { let text = String::from_utf8_lossy(bytes); let mut open = false; let mut middle = false; for line in text.lines() { if line.starts_with("<<<<<<< ") || line == "<<<<<<<" { open = true; } else if open && line == "=======" { middle = true; } else if open && middle && (line.starts_with(">>>>>>> ") || line == ">>>>>>>") { return true; } } false } #[cfg(test)] mod tests { use super::*; #[test] fn markers() { assert!(has_markers(b"a\n<<<<<<< main 4\nx\n=======\ny\n>>>>>>> your change bcdf\nb\n")); assert!(!has_markers(b"a\n=======\nb\n")); } #[test] fn mine_defaults() { let mine = Mine::new(&Config::default(), &[]); assert!(mine.matches(".env")); assert!(mine.matches(".env.production")); assert!(mine.matches("apps/web/.env.local")); assert!(!mine.matches(".env.example")); assert!(!mine.matches("src/env.ts")); } #[test] fn snapshot_and_materialize() { let objects = tempfile::tempdir().unwrap(); let store = Store::open(objects.path()).unwrap(); let ws = tempfile::tempdir().unwrap(); let root = ws.path(); write_id(root, "w1").unwrap(); fs::create_dir_all(root.join("src")).unwrap(); fs::write(root.join("README.md"), "hi\n").unwrap(); fs::write(root.join("src/a.rs"), "fn a() {}\n").unwrap(); fs::write(root.join(".gitignore"), "target/\n").unwrap(); fs::create_dir_all(root.join("target")).unwrap(); fs::write(root.join("target/junk"), "x").unwrap(); fs::write(root.join(".env"), "SECRET=1\n").unwrap(); let snap = snapshot(root, &store, &Files::new(), &[], &Index::default(), &[], &Default::default()).unwrap(); assert!(snap.files.contains_key("README.md")); assert!(snap.files.contains_key("src/a.rs")); assert!(snap.files.contains_key(".gitignore")); assert!(!snap.files.contains_key("target/junk")); assert!(!snap.files.contains_key(".env")); assert_eq!(snap.mine.len(), 1); let ws2 = tempfile::tempdir().unwrap(); write_id(ws2.path(), "w2").unwrap(); let index2 = materialize(ws2.path(), &store, &Index::default(), &snap.files, &[], Some(CowSource { root, index: &snap.index })).unwrap(); assert_eq!(fs::read_to_string(ws2.path().join("src/a.rs")).unwrap(), "fn a() {}\n"); let mut fewer = snap.files.clone(); fewer.remove("src/a.rs"); materialize(ws2.path(), &store, &index2, &fewer, &[], None).unwrap(); assert!(!ws2.path().join("src").exists()); } }