826 lines · 32 KBtext/plain; charset=utf-8
Onset: version control where nobody waits and nothing gets lost. The on command, its core and onset-server.
| 5 |
| use std::fs; |
| 6 | use std::io::Read; |
| 7 | use std::path::{Path, PathBuf}; |
| 8 |
| 9 | use anyhow::{Context, Result, anyhow}; |
| 10 | use rayon::prelude::*; |
| 11 | use vcs_core::Id; |
| 12 | use vcs_core::tree::{self, Entry, EntryKind, Files}; |
| 13 |
| 14 | use crate::objects::Store; |
| 15 | use crate::sys; |
| 16 |
| 17 | pub const DIR: &str = ".vcs"; |
| 18 |
| 19 | /// The workspace folder containing `start`, if any. |
| 20 | pub fn find(start: &Path) -> Option<PathBuf> { |
| 21 | let mut dir = Some(start); |
| 22 | while let Some(d) = dir { |
| 23 | if d.join(DIR).join("workspace").is_file() { |
| 24 | return Some(d.to_path_buf()); |
| 25 | } |
| 26 | dir = d.parent(); |
| 27 | } |
| 28 | None |
| 29 | } |
| 30 |
| 31 | pub fn read_id(root: &Path) -> Result<String> { |
| 32 | Ok(fs::read_to_string(root.join(DIR).join("workspace"))?.trim().to_string()) |
| 33 | } |
| 34 |
| 35 | pub fn write_id(root: &Path, id: &str) -> Result<()> { |
| 36 | let dir = root.join(DIR); |
| 37 | fs::create_dir_all(&dir)?; |
| 38 | fs::write(dir.join("workspace"), format!("{id}\n"))?; |
| 39 | // Keep the folder out of anything that walks the tree. |
| 40 | fs::write(dir.join(".gitignore"), "*\n")?; |
| 41 | Ok(()) |
| 42 | } |
| 43 |
| 44 | /// An exclusive lock on the workspace, held while a command or the saver |
| 45 | /// works in it. |
| 46 | pub struct Lock { |
| 47 | _file: fs::File, |
| 48 | } |
| 49 |
| 50 | pub fn lock(root: &Path) -> Result<Lock> { |
| 51 | use fs4::fs_std::FileExt; |
| 52 | let path = root.join(DIR).join("lock"); |
| 53 | let file = fs::OpenOptions::new().create(true).truncate(false).write(true).open(&path)?; |
| 54 | file.lock_exclusive().context("locking the workspace")?; |
| 55 | Ok(Lock { _file: file }) |
| 56 | } |
| 57 |
| 58 | /// Like [`lock`], but gives up at once if someone else holds it. |
| 59 | pub fn try_lock(root: &Path) -> Result<Option<Lock>> { |
| 60 | use fs4::fs_std::FileExt; |
| 61 | let path = root.join(DIR).join("lock"); |
| 62 | let file = fs::OpenOptions::new().create(true).truncate(false).write(true).open(&path)?; |
| 63 | match file.try_lock_exclusive() { |
| 64 | Ok(true) => Ok(Some(Lock { _file: file })), |
| 65 | Ok(false) => Ok(None), |
| 66 | Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => Ok(None), |
| 67 | Err(e) => Err(e.into()), |
| 68 | } |
| 69 | } |
| 70 |
| 71 | // The stat index: what each tracked file looked like when we last knew its |
| 72 | // blob, so unchanged files are never read again. |
| 73 |
| 74 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 75 | pub struct Stat { |
| 76 | pub size: u64, |
| 77 | pub mtime_ns: i64, |
| 78 | pub ino: u64, |
| 79 | } |
| 80 |
| 81 | impl Stat { |
| 82 | fn of(meta: &fs::Metadata) -> Stat { |
| 83 | let (mtime_ns, ino) = sys::times(meta); |
| 84 | Stat { size: meta.len(), mtime_ns, ino } |
| 85 | } |
| 86 | } |
| 87 |
| 88 | #[derive(Debug, Clone, Copy)] |
| 89 | pub struct IndexEntry { |
| 90 | pub entry: Entry, |
| 91 | pub stat: Stat, |
| 92 | } |
| 93 |
| 94 | #[derive(Debug, Default, Clone)] |
| 95 | pub struct Index { |
| 96 | pub entries: BTreeMap<String, IndexEntry>, |
| 97 | /// When the index was written (ns); files changed just before it are |
| 98 | /// re-read next time (git's "racy" rule, see `racy_window`). |
| 99 | pub written_ns: i64, |
| 100 | /// The tree these entries make, so a save that finds nothing new |
| 101 | /// doesn't build it again. |
| 102 | pub tree: Option<Id>, |
| 103 | } |
| 104 |
| 105 | const MAGIC: &[u8] = b"VCSI3\n"; |
| 106 |
| 107 | impl Index { |
| 108 | pub fn load(root: &Path) -> Index { |
| 109 | Self::try_load(root).unwrap_or_default() |
| 110 | } |
| 111 |
| 112 | fn try_load(root: &Path) -> Result<Index> { |
| 113 | let bytes = fs::read(root.join(DIR).join("index"))?; |
| 114 | if !bytes.starts_with(MAGIC) { |
| 115 | return Err(anyhow!("bad index")); |
| 116 | } |
| 117 | let mut at = MAGIC.len(); |
| 118 | let take = |at: &mut usize, n: usize| -> Result<&[u8]> { |
| 119 | let slice = bytes.get(*at..*at + n).ok_or_else(|| anyhow!("short index"))?; |
| 120 | *at += n; |
| 121 | Ok(slice) |
| 122 | }; |
| 123 | let written_ns = i64::from_le_bytes(take(&mut at, 8)?.try_into()?); |
| 124 | let tree = Id::from_slice(take(&mut at, 32)?).filter(|t| *t != Id::ZERO); |
| 125 | let count = u32::from_le_bytes(take(&mut at, 4)?.try_into()?); |
| 126 | // Written in order, so the map is built in one go, not one insert at a time. |
| 127 | let mut list = Vec::with_capacity(count as usize); |
| 128 | for _ in 0..count { |
| 129 | let len = u16::from_le_bytes(take(&mut at, 2)?.try_into()?) as usize; |
| 130 | let path = std::str::from_utf8(take(&mut at, len)?)?.to_string(); |
| 131 | let id = Id::from_slice(take(&mut at, 32)?).ok_or_else(|| anyhow!("bad index"))?; |
| 132 | let kind = EntryKind::from_u8(take(&mut at, 1)?[0]).ok_or_else(|| anyhow!("bad index"))?; |
| 133 | let size = u64::from_le_bytes(take(&mut at, 8)?.try_into()?); |
| 134 | let mtime_ns = i64::from_le_bytes(take(&mut at, 8)?.try_into()?); |
| 135 | let ino = u64::from_le_bytes(take(&mut at, 8)?.try_into()?); |
| 136 | list.push((path, IndexEntry { entry: Entry { kind, id }, stat: Stat { size, mtime_ns, ino } })); |
| 137 | } |
| 138 | Ok(Index { entries: list.into_iter().collect(), written_ns, tree }) |
| 139 | } |
| 140 |
| 141 | /// Writes the index. It's a cache (a lost one only means reading files |
| 142 | /// again), so it isn't synced to disk. |
| 143 | pub fn save(&mut self, root: &Path) -> Result<()> { |
| 144 | self.written_ns = now_ns(); |
| 145 | let mut out = Vec::with_capacity(self.entries.len() * 80 + 64); |
| 146 | out.extend_from_slice(MAGIC); |
| 147 | out.extend_from_slice(&self.written_ns.to_le_bytes()); |
| 148 | out.extend_from_slice(&self.tree.unwrap_or(Id::ZERO).0); |
| 149 | out.extend_from_slice(&(self.entries.len() as u32).to_le_bytes()); |
| 150 | for (path, e) in &self.entries { |
| 151 | out.extend_from_slice(&(path.len() as u16).to_le_bytes()); |
| 152 | out.extend_from_slice(path.as_bytes()); |
| 153 | out.extend_from_slice(&e.entry.id.0); |
| 154 | out.push(e.entry.kind as u8); |
| 155 | out.extend_from_slice(&e.stat.size.to_le_bytes()); |
| 156 | out.extend_from_slice(&e.stat.mtime_ns.to_le_bytes()); |
| 157 | out.extend_from_slice(&e.stat.ino.to_le_bytes()); |
| 158 | } |
| 159 | let tmp = root.join(DIR).join(format!("index.tmp{}", std::process::id())); |
| 160 | fs::write(&tmp, &out)?; |
| 161 | fs::rename(tmp, root.join(DIR).join("index"))?; |
| 162 | Ok(()) |
| 163 | } |
| 164 |
| 165 | /// Whether writing `next` over this index would change anything: new |
| 166 | /// entries, or entries still inside the racy second. |
| 167 | pub fn same_as(&self, next: &Index) -> bool { |
| 168 | self.tree.is_some() |
| 169 | && self.tree == next.tree |
| 170 | && self.entries.len() == next.entries.len() |
| 171 | && self.entries.iter().zip(next.entries.iter()).all(|((p, a), (q, b))| { |
| 172 | p == q && a.entry == b.entry && a.stat == b.stat && a.stat.mtime_ns < self.written_ns - racy_window(a.stat.mtime_ns) |
| 173 | }) |
| 174 | } |
| 175 |
| 176 | /// Whether `stat` proves the file still has the indexed content. |
| 177 | fn unchanged(&self, entry: &IndexEntry, stat: Stat) -> bool { |
| 178 | entry.stat == stat && stat.mtime_ns < self.written_ns - racy_window(stat.mtime_ns) |
| 179 | } |
| 180 | } |
| 181 |
| 182 | /// How close to the index's writing a change can hide: a file written again |
| 183 | /// within the same timestamp tick looks unchanged. Timestamps here have |
| 184 | /// nanoseconds, and the kernel moves them every few milliseconds; a whole |
| 185 | /// second means a disk with coarse timestamps (FAT, old HFS+). |
| 186 | fn racy_window(mtime_ns: i64) -> i64 { |
| 187 | if mtime_ns % 1_000_000_000 == 0 { 2_000_000_000 } else { 50_000_000 } |
| 188 | } |
| 189 |
| 190 | fn now_ns() -> i64 { |
| 191 | std::time::SystemTime::now() |
| 192 | .duration_since(std::time::UNIX_EPOCH) |
| 193 | .map(|d| d.as_nanos() as i64) |
| 194 | .unwrap_or(0) |
| 195 | } |
| 196 |
| 197 | // Rules for what a workspace keeps out of its trees. |
| 198 |
| 199 | /// What `vcs.toml` says. |
| 200 | #[derive(Debug, Default, Clone, serde::Deserialize)] |
| 201 | pub struct Config { |
| 202 | #[serde(default)] |
| 203 | pub workspace: WorkspaceConfig, |
| 204 | #[serde(default)] |
| 205 | pub mine: MineConfig, |
| 206 | } |
| 207 |
| 208 | #[derive(Debug, Default, Clone, serde::Deserialize)] |
| 209 | pub struct WorkspaceConfig { |
| 210 | #[serde(default)] |
| 211 | pub copy: Vec<String>, |
| 212 | } |
| 213 |
| 214 | #[derive(Debug, Default, Clone, serde::Deserialize)] |
| 215 | pub struct MineConfig { |
| 216 | #[serde(default)] |
| 217 | pub paths: Vec<String>, |
| 218 | } |
| 219 |
| 220 | pub fn config(root: &Path) -> Config { |
| 221 | fs::read_to_string(root.join("vcs.toml")) |
| 222 | .ok() |
| 223 | .and_then(|text| toml::from_str(&text).ok()) |
| 224 | .unwrap_or_default() |
| 225 | } |
| 226 |
| 227 | pub const DEFAULT_MINE: &[&str] = &[".env", ".env.*", "!.env.example", "!.env.sample", "!.env.template"]; |
| 228 |
| 229 | /// Paths that stay the user's own: never saved into a change, never landed. |
| 230 | pub struct Mine { |
| 231 | include: globset::GlobSet, |
| 232 | exclude: globset::GlobSet, |
| 233 | /// The rules as written, to tell when they change. |
| 234 | pub rules: String, |
| 235 | } |
| 236 |
| 237 | impl Mine { |
| 238 | /// The project's rules (vcs.toml, or the defaults) plus the person's own. |
| 239 | pub fn new(config: &Config, personal: &[String]) -> Mine { |
| 240 | let mut include = globset::GlobSetBuilder::new(); |
| 241 | let mut exclude = globset::GlobSetBuilder::new(); |
| 242 | let mut patterns: Vec<String> = if config.mine.paths.is_empty() { |
| 243 | DEFAULT_MINE.iter().map(|s| s.to_string()).collect() |
| 244 | } else { |
| 245 | config.mine.paths.clone() |
| 246 | }; |
| 247 | patterns.extend(personal.iter().cloned()); |
| 248 | let rules = patterns.join("\n"); |
| 249 | for pattern in patterns { |
| 250 | let (set, p) = match pattern.strip_prefix('!') { |
| 251 | Some(p) => (&mut exclude, p.to_string()), |
| 252 | None => (&mut include, pattern), |
| 253 | }; |
| 254 | // A bare name matches at any depth, as in .gitignore. |
| 255 | let p = p.trim_end_matches('/').to_string(); |
| 256 | let globs = if p.contains('/') { vec![p.trim_start_matches('/').to_string()] } else { vec![p.clone(), format!("**/{p}")] }; |
| 257 | for g in globs { |
| 258 | for g in [g.clone(), format!("{g}/**")] { |
| 259 | if let Ok(glob) = globset::GlobBuilder::new(&g).literal_separator(true).build() { |
| 260 | set.add(glob); |
| 261 | } |
| 262 | } |
| 263 | } |
| 264 | } |
| 265 | Mine { |
| 266 | include: include.build().unwrap_or_else(|_| globset::GlobSet::empty()), |
| 267 | exclude: exclude.build().unwrap_or_else(|_| globset::GlobSet::empty()), |
| 268 | rules, |
| 269 | } |
| 270 | } |
| 271 |
| 272 | pub fn matches(&self, path: &str) -> bool { |
| 273 | self.include.is_match(path) && !self.exclude.is_match(path) |
| 274 | } |
| 275 | } |
| 276 |
| 277 |
| 278 | /// What saving a workspace found. |
| 279 | pub struct Snapshot { |
| 280 | pub tree: Id, |
| 281 | /// The index to keep (None: the one given, unchanged). |
| 282 | pub index: Option<Index>, |
| 283 | /// Mine files in the folder (kept out of the tree): path, file ID, exec. |
| 284 | pub mine: Vec<(String, Id, bool)>, |
| 285 | } |
| 286 |
| 287 | struct Seen { |
| 288 | stat: Stat, |
| 289 | kind: EntryKind, |
| 290 | /// Not known to the index as it is now: its content is read. |
| 291 | changed: bool, |
| 292 | } |
| 293 |
| 294 | /// Every file under `root` its .gitignore files leave in, never vcs's or |
| 295 | /// git's own folders, as "a/b" paths with their stats. Folders are read in |
| 296 | /// parallel, and each folder's files are handed over at once. |
| 297 | fn walk(root: &Path) -> Vec<(String, fs::Metadata)> { |
| 298 | use ignore::gitignore::{Gitignore, GitignoreBuilder}; |
| 299 | use std::sync::{Arc, Mutex}; |
| 300 | type Rules = Arc<Vec<Arc<Gitignore>>>; |
| 301 | type Found = Mutex<Vec<(String, fs::Metadata)>>; |
| 302 |
| 303 | // The deepest .gitignore with something to say about a path decides. |
| 304 | fn ignored(rules: &Rules, path: &Path, is_dir: bool) -> bool { |
| 305 | for g in rules.iter().rev() { |
| 306 | match g.matched(path, is_dir) { |
| 307 | ignore::Match::Ignore(_) => return true, |
| 308 | ignore::Match::Whitelist(_) => return false, |
| 309 | ignore::Match::None => {} |
| 310 | } |
| 311 | } |
| 312 | false |
| 313 | } |
| 314 |
| 315 | fn visit<'s>(scope: &rayon::Scope<'s>, dir: PathBuf, rel: String, rules: Rules, found: &'s Found) { |
| 316 | let Ok(read) = fs::read_dir(&dir) else { return }; |
| 317 | let entries: Vec<fs::DirEntry> = read.flatten().collect(); |
| 318 | let rules = match entries.iter().find(|e| e.file_name() == ".gitignore" && e.file_type().is_ok_and(|t| t.is_file())) { |
| 319 | Some(e) => { |
| 320 | let mut builder = GitignoreBuilder::new(&dir); |
| 321 | let _ = builder.add(e.path()); |
| 322 | match builder.build() { |
| 323 | Ok(g) => Arc::new(rules.iter().cloned().chain([Arc::new(g)]).collect()), |
| 324 | Err(_) => rules, |
| 325 | } |
| 326 | } |
| 327 | None => rules, |
| 328 | }; |
| 329 | let mut files = Vec::new(); |
| 330 | for entry in entries { |
| 331 | let name = entry.file_name(); |
| 332 | // Names no project can hold, and vcs's and git's own folders at |
| 333 | // any depth (a nested git checkout too). |
| 334 | let Some(name) = name.to_str().filter(|n| tree::safe_name(n)) else { continue }; |
| 335 | let Ok(kind) = entry.file_type() else { continue }; |
| 336 | let child = if rel.is_empty() { name.to_string() } else { format!("{rel}/{name}") }; |
| 337 | let path = entry.path(); |
| 338 | if kind.is_dir() { |
| 339 | if !ignored(&rules, &path, true) { |
| 340 | let rules = rules.clone(); |
| 341 | scope.spawn(move |s| visit(s, path, child, rules, found)); |
| 342 | } |
| 343 | } else if !ignored(&rules, &path, false) |
| 344 | && let Ok(meta) = entry.metadata() |
| 345 | { |
| 346 | files.push((child, meta)); |
| 347 | } |
| 348 | } |
| 349 | if !files.is_empty() { |
| 350 | found.lock().unwrap().extend(files); |
| 351 | } |
| 352 | } |
| 353 |
| 354 | let found = Mutex::new(Vec::new()); |
| 355 | rayon::scope(|s| visit(s, root.to_path_buf(), String::new(), Arc::new(Vec::new()), &found)); |
| 356 | found.into_inner().unwrap() |
| 357 | } |
| 358 |
| 359 |
| 360 | fn in_scope(only: &[String], path: &str) -> bool { |
| 361 | only.is_empty() || only.iter().any(|o| tree::under(path, o)) |
| 362 | } |
| 363 |
| 364 | /// Saves the folder's content into the store and returns its tree. `base` |
| 365 | /// is the tree the change starts from: files it tracks are kept even when |
| 366 | /// .gitignore matches them, and with `only` set every path outside those |
| 367 | /// folders comes from it unchanged. |
| 368 | /// `main_paths` answers the files the change's base tracks (asked only when |
| 369 | /// a file looks like a mine file): rules for mine files never take a file |
| 370 | /// main already has. |
| 371 | pub fn snapshot( |
| 372 | root: &Path, |
| 373 | store: &Store, |
| 374 | base: &Files, |
| 375 | only: &[String], |
| 376 | index: &Index, |
| 377 | personal: &[String], |
| 378 | main_paths: &dyn Fn() -> Result<std::collections::HashSet<String>>, |
| 379 | ) -> Result<Snapshot> { |
| 380 | let config = config(root); |
| 381 | let mine = Mine::new(&config, personal); |
| 382 |
| 383 | // What a file on disk is now, and whether it changed since the index. |
| 384 | let look = |path: &str, meta: &fs::Metadata| -> Seen { |
| 385 | let stat = Stat::of(meta); |
| 386 | match index.entries.get(path) { |
| 387 | Some(e) => { |
| 388 | let kind = sys::kind_of(meta, Some(e.entry.kind)); |
| 389 | Seen { stat, kind, changed: !(index.unchanged(e, stat) && e.entry.kind == kind) } |
| 390 | } |
| 391 | None => Seen { stat, kind: sys::kind_of(meta, base.get(path).map(|e| e.kind)), changed: true }, |
| 392 | } |
| 393 | }; |
| 394 | // One pass over the walk and the index, both sorted: each file's state, |
| 395 | // the files the index doesn't have yet, and tracked files the walk |
| 396 | // skipped (ignored but tracked, which still count). |
| 397 | let mut walked = walk(root); |
| 398 | walked.par_sort_unstable_by(|a, b| a.0.cmp(&b.0)); |
| 399 | let mut list: Vec<(String, Seen)> = Vec::with_capacity(walked.len()); |
| 400 | let mut fresh: Vec<String> = Vec::new(); |
| 401 | let mut skipped: Vec<&String> = Vec::new(); |
| 402 | let mut tracked = index.entries.iter().peekable(); |
| 403 | for (path, meta) in walked { |
| 404 | while let Some((p, _)) = tracked.next_if(|(p, _)| p.as_str() < path.as_str()) { |
| 405 | skipped.push(p); |
| 406 | } |
| 407 | let stat = Stat::of(&meta); |
| 408 | let seen = match tracked.next_if(|(p, _)| **p == path) { |
| 409 | Some((_, e)) => { |
| 410 | let kind = sys::kind_of(&meta, Some(e.entry.kind)); |
| 411 | Seen { stat, kind, changed: !(index.unchanged(e, stat) && e.entry.kind == kind) } |
| 412 | } |
| 413 | None => { |
| 414 | fresh.push(path.clone()); |
| 415 | look(&path, &meta) |
| 416 | } |
| 417 | }; |
| 418 | list.push((path, seen)); |
| 419 | } |
| 420 | skipped.extend(tracked.map(|(p, _)| p)); |
| 421 | let mut seen: BTreeMap<String, Seen> = list.into_iter().collect(); |
| 422 | if !only.is_empty() { |
| 423 | skipped.extend(base.keys().filter(|p| !seen.contains_key(*p) && !index.entries.contains_key(*p))); |
| 424 | } |
| 425 | for path in skipped { |
| 426 | if !in_scope(only, path) { |
| 427 | continue; |
| 428 | } |
| 429 | if let Ok(meta) = root.join(path).symlink_metadata() |
| 430 | && !meta.is_dir() |
| 431 | { |
| 432 | seen.insert(path.clone(), look(path, &meta)); |
| 433 | } |
| 434 | } |
| 435 |
| 436 | // Submodules are pointers, not files here: they carry over as they are, |
| 437 | // and nothing inside their folders is tracked. |
| 438 | let links: Vec<(String, Entry)> = if only.is_empty() { |
| 439 | index.entries.iter().filter(|(_, e)| e.entry.kind == EntryKind::Gitlink).map(|(p, e)| (p.clone(), e.entry)).collect() |
| 440 | } else { |
| 441 | base.iter().filter(|(p, e)| e.kind == EntryKind::Gitlink && in_scope(only, p)).map(|(p, e)| (p.clone(), *e)).collect() |
| 442 | }; |
| 443 | if !links.is_empty() { |
| 444 | seen.retain(|path, _| !links.iter().any(|(l, _)| tree::under(path, l))); |
| 445 | } |
| 446 |
| 447 | // Mine files stay out of trees unless main already tracks them. Files |
| 448 | // the index has were sorted out when they came, so only new ones are |
| 449 | // looked at, unless the rules changed since; main's paths are read only |
| 450 | // when some file looks like one. |
| 451 | let rules_file = root.join(DIR).join("mine-rules"); |
| 452 | let rules_changed = fs::read_to_string(&rules_file).ok().as_deref() != Some(mine.rules.as_str()); |
| 453 | let candidates: Vec<String> = |
| 454 | if rules_changed { seen.keys().filter(|p| mine.matches(p)).cloned().collect() } else { fresh.into_iter().filter(|p| mine.matches(p)).collect() }; |
| 455 | let main_paths = if candidates.is_empty() { Default::default() } else { main_paths()? }; |
| 456 | let mut mine_paths = Vec::new(); |
| 457 | for path in candidates.into_iter().filter(|p| !main_paths.contains(p)) { |
| 458 | if let Some(s) = seen.remove(&path) { |
| 459 | mine_paths.push((root.join(&path), s.kind == EntryKind::Exec, path)); |
| 460 | } |
| 461 | } |
| 462 | if rules_changed { |
| 463 | let _ = fs::write(&rules_file, &mine.rules); |
| 464 | } |
| 465 | if !only.is_empty() { |
| 466 | seen.retain(|path, _| in_scope(only, path)); |
| 467 | } |
| 468 |
| 469 | // Read and store what might have changed, in parallel. Objects are |
| 470 | // files named by their IDs, so parallel writes are safe. |
| 471 | let jobs: Vec<(&String, &Seen)> = seen.iter().filter(|(_, s)| s.changed).collect(); |
| 472 | let mut mine_out = Vec::new(); |
| 473 | for (abs, exec, path) in mine_paths { |
| 474 | let bytes = read_content(&abs, EntryKind::File)?; |
| 475 | mine_out.push((path, store.put_file(&bytes)?, exec)); |
| 476 | } |
| 477 | // Nothing changed since the index (what a status or a land usually |
| 478 | // finds): its tree stands, and nothing is rebuilt. |
| 479 | if jobs.is_empty() |
| 480 | && only.is_empty() |
| 481 | && links.is_empty() |
| 482 | && seen.len() == index.entries.len() |
| 483 | && let Some(tree) = index.tree.filter(|t| store.has(t)) |
| 484 | && seen.keys().eq(index.entries.keys()) |
| 485 | { |
| 486 | return Ok(Snapshot { tree, index: None, mine: mine_out }); |
| 487 | } |
| 488 | let stored: BTreeMap<String, Id> = jobs |
| 489 | .par_iter() |
| 490 | .map(|(path, s)| -> Result<(String, Id)> { |
| 491 | let bytes = read_content(&root.join(path), s.kind)?; |
| 492 | Ok(((*path).clone(), store.put_file(&bytes)?)) |
| 493 | }) |
| 494 | .collect::<Result<BTreeMap<_, _>>>()?; |
| 495 |
| 496 | let mut files = Files::new(); |
| 497 | for (path, entry) in base { |
| 498 | if !in_scope(only, path) { |
| 499 | files.insert(path.clone(), *entry); |
| 500 | } |
| 501 | } |
| 502 | let mut next = Index::default(); |
| 503 | for (path, entry) in &links { |
| 504 | files.insert(path.clone(), *entry); |
| 505 | let stat = index.entries.get(path).map(|e| e.stat).unwrap_or(Stat { size: 0, mtime_ns: 0, ino: 0 }); |
| 506 | next.entries.insert(path.clone(), IndexEntry { entry: *entry, stat }); |
| 507 | } |
| 508 | for (path, s) in &seen { |
| 509 | let id = match stored.get(path) { |
| 510 | Some(id) => *id, |
| 511 | None => index.entries[path].entry.id, |
| 512 | }; |
| 513 | let entry = Entry { kind: s.kind, id }; |
| 514 | files.insert(path.clone(), entry); |
| 515 | next.entries.insert(path.clone(), IndexEntry { entry, stat: s.stat }); |
| 516 | } |
| 517 | // Nothing new since the index's tree: it's still the tree. |
| 518 | let unchanged = stored.is_empty() && only.is_empty() && next.entries.len() == index.entries.len() && next.entries.keys().eq(index.entries.keys()); |
| 519 | let tree = match index.tree { |
| 520 | Some(t) if unchanged && store.has(&t) => t, |
| 521 | _ => { |
| 522 | let (tree, objects) = tree::build(&files); |
| 523 | for o in &objects { |
| 524 | store.put(o)?; |
| 525 | } |
| 526 | tree |
| 527 | } |
| 528 | }; |
| 529 | next.tree = Some(tree); |
| 530 | Ok(Snapshot { tree, index: Some(next), mine: mine_out }) |
| 531 | } |
| 532 |
| 533 | fn read_content(path: &Path, kind: EntryKind) -> Result<Vec<u8>> { |
| 534 | if kind == EntryKind::Link { |
| 535 | return Ok(sys::read_link(path)?); |
| 536 | } |
| 537 | let mut bytes = Vec::new(); |
| 538 | fs::File::open(path) |
| 539 | .with_context(|| format!("reading {}", path.display()))? |
| 540 | .read_to_end(&mut bytes)?; |
| 541 | Ok(bytes) |
| 542 | } |
| 543 |
| 544 | /// Another workspace to copy unchanged files from. |
| 545 | pub struct CowSource<'a> { |
| 546 | pub root: &'a Path, |
| 547 | pub index: &'a Index, |
| 548 | } |
| 549 |
| 550 | /// Makes the folder hold exactly `target` (within `only`), changing only |
| 551 | /// what differs from `index`. Untracked and ignored files are left alone. |
| 552 | /// Returns the new index. |
| 553 | pub fn materialize(root: &Path, store: &Store, index: &Index, target: &Files, only: &[String], cow: Option<CowSource>) -> Result<Index> { |
| 554 | let wanted: BTreeMap<&String, &Entry> = target.iter().filter(|(p, _)| in_scope(only, p)).collect(); |
| 555 |
| 556 | // Remove tracked files that the target doesn't have. |
| 557 | let mut removed_dirs = Vec::new(); |
| 558 | for path in index.entries.keys() { |
| 559 | if !wanted.contains_key(path) { |
| 560 | let abs = root.join(path); |
| 561 | if abs.symlink_metadata().is_ok_and(|m| !m.is_dir()) { |
| 562 | fs::remove_file(&abs)?; |
| 563 | } |
| 564 | if let Some(parent) = abs.parent() { |
| 565 | removed_dirs.push(parent.to_path_buf()); |
| 566 | } |
| 567 | } |
| 568 | } |
| 569 | // Empty folders left behind go too (never the root). |
| 570 | removed_dirs.sort_by_key(|d| std::cmp::Reverse(d.components().count())); |
| 571 | removed_dirs.dedup(); |
| 572 | for dir in removed_dirs { |
| 573 | let mut d = dir.as_path(); |
| 574 | while d != root && d.starts_with(root) { |
| 575 | if fs::remove_dir(d).is_err() { |
| 576 | break; |
| 577 | } |
| 578 | match d.parent() { |
| 579 | Some(p) => d = p, |
| 580 | None => break, |
| 581 | } |
| 582 | } |
| 583 | } |
| 584 |
| 585 | // Never vcs's or git's own folders, or outside the workspace, whatever a |
| 586 | // host sends; and nothing this system can't have a file by. |
| 587 | if let Some(bad) = wanted.keys().find(|p| !tree::valid_path(p) || !p.split('/').all(sys::fits)) { |
| 588 | return Err(anyhow!( |
| 589 | "{bad} can't be written here: {}", |
| 590 | if tree::valid_path(bad) { "this system can't have a file by that name" } else { "it's vcs's or git's own folder, or not a safe path" } |
| 591 | )); |
| 592 | } |
| 593 |
| 594 | // Write what's new or different. |
| 595 | let mut keep = Index::default(); |
| 596 | let mut jobs = Vec::new(); |
| 597 | for (path, entry) in &wanted { |
| 598 | if let Some(existing) = index.entries.get(*path) |
| 599 | && existing.entry == **entry |
| 600 | && let Ok(meta) = root.join(path).symlink_metadata() |
| 601 | && Stat::of(&meta) == existing.stat |
| 602 | { |
| 603 | keep.entries.insert((*path).clone(), *existing); |
| 604 | continue; |
| 605 | } |
| 606 | jobs.push(((*path).clone(), **entry)); |
| 607 | } |
| 608 | // Links go last, so what they point at is there first (Windows makes a |
| 609 | // link to a folder differently from one to a file). |
| 610 | let (links, files): (Vec<(String, Entry)>, Vec<(String, Entry)>) = jobs.into_iter().partition(|(_, e)| e.kind == EntryKind::Link); |
| 611 | let write = |(path, entry): &(String, Entry)| -> Result<(String, IndexEntry)> { |
| 612 | let abs = root.join(path); |
| 613 | if let Some(parent) = abs.parent() { |
| 614 | make_dirs(root, parent)?; |
| 615 | } |
| 616 | if entry.kind == EntryKind::Gitlink { |
| 617 | // A submodule's folder, left as it is (checked out or not), or |
| 618 | // made empty, as git leaves one it hasn't checked out. |
| 619 | if abs.symlink_metadata().is_ok_and(|m| !m.is_dir()) { |
| 620 | fs::remove_file(&abs)?; |
| 621 | } |
| 622 | make_dirs(root, &abs)?; |
| 623 | let meta = abs.symlink_metadata()?; |
| 624 | return Ok((path.clone(), IndexEntry { entry: *entry, stat: Stat::of(&meta) })); |
| 625 | } |
| 626 | if let Ok(meta) = abs.symlink_metadata() { |
| 627 | if meta.is_dir() { |
| 628 | fs::remove_dir_all(&abs)?; |
| 629 | } else { |
| 630 | fs::remove_file(&abs)?; |
| 631 | } |
| 632 | } |
| 633 | let copied = match &cow { |
| 634 | Some(src) if entry.kind != EntryKind::Link => copy_from(src, path, entry, &abs), |
| 635 | _ => false, |
| 636 | }; |
| 637 | if !copied { |
| 638 | let content = store.read_file(&entry.id).with_context(|| format!("missing content for {path}"))?; |
| 639 | if entry.kind == EntryKind::Link { |
| 640 | sys::write_link(&String::from_utf8_lossy(&content), &abs)?; |
| 641 | } else { |
| 642 | fs::write(&abs, content)?; |
| 643 | } |
| 644 | } |
| 645 | if entry.kind != EntryKind::Link { |
| 646 | sys::set_exec(&abs, entry.kind == EntryKind::Exec)?; |
| 647 | } |
| 648 | let meta = abs.symlink_metadata()?; |
| 649 | Ok((path.clone(), IndexEntry { entry: *entry, stat: Stat::of(&meta) })) |
| 650 | }; |
| 651 | let mut written: Vec<(String, IndexEntry)> = files.par_iter().map(write).collect::<Result<Vec<_>>>()?; |
| 652 | written.extend(links.par_iter().map(write).collect::<Result<Vec<_>>>()?); |
| 653 | for (path, e) in written { |
| 654 | keep.entries.insert(path, e); |
| 655 | } |
| 656 | Ok(keep) |
| 657 | } |
| 658 |
| 659 | /// Creates `dir` and its parents, replacing any file in the way. |
| 660 | fn make_dirs(root: &Path, dir: &Path) -> Result<()> { |
| 661 | if dir.is_dir() { |
| 662 | return Ok(()); |
| 663 | } |
| 664 | let mut missing = Vec::new(); |
| 665 | let mut d = dir; |
| 666 | while d != root && !d.is_dir() { |
| 667 | missing.push(d.to_path_buf()); |
| 668 | match d.parent() { |
| 669 | Some(p) => d = p, |
| 670 | None => break, |
| 671 | } |
| 672 | } |
| 673 | for m in missing.into_iter().rev() { |
| 674 | if m.symlink_metadata().is_ok_and(|meta| !meta.is_dir()) { |
| 675 | fs::remove_file(&m)?; |
| 676 | } |
| 677 | match fs::create_dir(&m) { |
| 678 | Ok(()) => {} |
| 679 | Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} |
| 680 | Err(e) => return Err(e.into()), |
| 681 | } |
| 682 | } |
| 683 | Ok(()) |
| 684 | } |
| 685 |
| 686 | /// Copies one file from another workspace if it still has the same content. |
| 687 | fn copy_from(src: &CowSource, path: &str, entry: &Entry, dest: &Path) -> bool { |
| 688 | let Some(theirs) = src.index.entries.get(path) else { return false }; |
| 689 | if theirs.entry != *entry { |
| 690 | return false; |
| 691 | } |
| 692 | let from = src.root.join(path); |
| 693 | let Ok(meta) = from.symlink_metadata() else { return false }; |
| 694 | if Stat::of(&meta) != theirs.stat { |
| 695 | return false; |
| 696 | } |
| 697 | reflink_copy::reflink_or_copy(&from, dest).is_ok() |
| 698 | } |
| 699 |
| 700 | /// Copies a build folder (like node_modules) from another workspace, |
| 701 | /// copy-on-write where the disk supports it. Returns how many files. |
| 702 | pub fn copy_folder(from: &Path, to: &Path) -> Result<u64> { |
| 703 | let mut count = 0; |
| 704 | if !from.is_dir() || to.exists() { |
| 705 | return Ok(0); |
| 706 | } |
| 707 | let mut stack = vec![from.to_path_buf()]; |
| 708 | while let Some(dir) = stack.pop() { |
| 709 | let rel = dir.strip_prefix(from)?; |
| 710 | fs::create_dir_all(to.join(rel))?; |
| 711 | for entry in fs::read_dir(&dir)? { |
| 712 | let entry = entry?; |
| 713 | let path = entry.path(); |
| 714 | let dest = to.join(path.strip_prefix(from)?); |
| 715 | let kind = entry.file_type()?; |
| 716 | if kind.is_dir() { |
| 717 | stack.push(path); |
| 718 | } else if kind.is_symlink() { |
| 719 | sys::write_link(&String::from_utf8_lossy(&sys::read_link(&path)?), &dest)?; |
| 720 | count += 1; |
| 721 | } else { |
| 722 | reflink_copy::reflink_or_copy(&path, &dest)?; |
| 723 | count += 1; |
| 724 | } |
| 725 | } |
| 726 | } |
| 727 | Ok(count) |
| 728 | } |
| 729 |
| 730 | /// Whether text still holds conflict markers. |
| 731 | pub fn has_markers(bytes: &[u8]) -> bool { |
| 732 | let text = String::from_utf8_lossy(bytes); |
| 733 | let mut open = false; |
| 734 | let mut middle = false; |
| 735 | for line in text.lines() { |
| 736 | if line.starts_with("<<<<<<< ") || line == "<<<<<<<" { |
| 737 | open = true; |
| 738 | } else if open && line == "=======" { |
| 739 | middle = true; |
| 740 | } else if open && middle && (line.starts_with(">>>>>>> ") || line == ">>>>>>>") { |
| 741 | return true; |
| 742 | } |
| 743 | } |
| 744 | false |
| 745 | } |
| 746 |
| 747 | #[cfg(test)] |
| 748 | mod tests { |
| 749 | use super::*; |
| 750 |
| 751 | #[test] |
| 752 | fn markers() { |
| 753 | assert!(has_markers(b"a\n<<<<<<< main 4\nx\n=======\ny\n>>>>>>> your change bcdf\nb\n")); |
| 754 | assert!(!has_markers(b"a\n=======\nb\n")); |
| 755 | } |
| 756 |
| 757 | #[test] |
| 758 | fn mine_defaults() { |
| 759 | let mine = Mine::new(&Config::default(), &[]); |
| 760 | assert!(mine.matches(".env")); |
| 761 | assert!(mine.matches(".env.production")); |
| 762 | assert!(mine.matches("apps/web/.env.local")); |
| 763 | assert!(!mine.matches(".env.example")); |
| 764 | assert!(!mine.matches("src/env.ts")); |
| 765 | } |
| 766 |
| 767 | #[test] |
| 768 | fn snapshot_and_materialize() { |
| 769 | let objects = tempfile::tempdir().unwrap(); |
| 770 | let store = Store::open(objects.path()).unwrap(); |
| 771 | let ws = tempfile::tempdir().unwrap(); |
| 772 | let root = ws.path(); |
| 773 | write_id(root, "w1").unwrap(); |
| 774 | fs::create_dir_all(root.join("src")).unwrap(); |
| 775 | fs::write(root.join("README.md"), "hi\n").unwrap(); |
| 776 | fs::write(root.join("src/a.rs"), "fn a() {}\n").unwrap(); |
| 777 | fs::write(root.join(".gitignore"), "target/\n").unwrap(); |
| 778 | fs::create_dir_all(root.join("target")).unwrap(); |
| 779 | fs::write(root.join("target/junk"), "x").unwrap(); |
| 780 | fs::write(root.join(".env"), "SECRET=1\n").unwrap(); |
| 781 | let snap = snapshot(root, &store, &Files::new(), &[], &Index::default(), &[], &|| Ok(Default::default())).unwrap(); |
| 782 | let files = tree::flatten(&store, &snap.tree).unwrap(); |
| 783 | assert!(files.contains_key("README.md")); |
| 784 | assert!(files.contains_key("src/a.rs")); |
| 785 | assert!(files.contains_key(".gitignore")); |
| 786 | assert!(!files.contains_key("target/junk")); |
| 787 | assert!(!files.contains_key(".env")); |
| 788 | assert_eq!(snap.mine.len(), 1); |
| 789 | let index = snap.index.unwrap(); |
| 790 |
| 791 | // Nothing new: the same tree, and the index stands. |
| 792 | let again = snapshot(root, &store, &Files::new(), &[], &index, &[], &|| Ok(Default::default())).unwrap(); |
| 793 | assert_eq!(again.tree, snap.tree); |
| 794 |
| 795 | let ws2 = tempfile::tempdir().unwrap(); |
| 796 | write_id(ws2.path(), "w2").unwrap(); |
| 797 | let index2 = materialize(ws2.path(), &store, &Index::default(), &files, &[], Some(CowSource { root, index: &index })).unwrap(); |
| 798 | assert_eq!(fs::read_to_string(ws2.path().join("src/a.rs")).unwrap(), "fn a() {}\n"); |
| 799 | let mut fewer = files.clone(); |
| 800 | fewer.remove("src/a.rs"); |
| 801 | materialize(ws2.path(), &store, &index2, &fewer, &[], None).unwrap(); |
| 802 | assert!(!ws2.path().join("src").exists()); |
| 803 | } |
| 804 |
| 805 | /// An executable or a link stays one when it's edited, on a disk that |
| 806 | /// can't mark executables or make links (Windows). |
| 807 | #[test] |
| 808 | fn kinds_survive_edits() { |
| 809 | let objects = tempfile::tempdir().unwrap(); |
| 810 | let store = Store::open(objects.path()).unwrap(); |
| 811 | let ws = tempfile::tempdir().unwrap(); |
| 812 | let root = ws.path(); |
| 813 | write_id(root, "w1").unwrap(); |
| 814 | let mut files = Files::new(); |
| 815 | files.insert("run.sh".into(), Entry { kind: EntryKind::Exec, id: store.put_file(b"echo 1\n").unwrap() }); |
| 816 | files.insert("link".into(), Entry { kind: EntryKind::Link, id: store.put_file(b"run.sh").unwrap() }); |
| 817 | let index = materialize(root, &store, &Index::default(), &files, &[], None).unwrap(); |
| 818 | fs::write(root.join("run.sh"), "echo 2\n").unwrap(); |
| 819 | let snap = snapshot(root, &store, &Files::new(), &[], &index, &[], &|| Ok(Default::default())).unwrap(); |
| 820 | let got = tree::flatten(&store, &snap.tree).unwrap(); |
| 821 | assert_eq!(got["run.sh"].kind, EntryKind::Exec); |
| 822 | assert_eq!(got["run.sh"].id, store.put_file(b"echo 2\n").unwrap()); |
| 823 | assert_eq!(got["link"], files["link"]); |
| 824 | } |
| 825 | } |
| 826 |