| 1 | diff --git a/lib/src/eol.rs b/lib/src/eol.rs |
| 2 | index bfb111d3de..5eacfa4133 100644 |
| 3 | --- a/lib/src/eol.rs |
| 4 | +++ b/lib/src/eol.rs |
| 5 | @@ -49,6 +49,11 @@ |
| 6 | } |
| 7 | } |
| 8 | |
| 9 | + /// Whether [`Self::convert_eol_for_update()`] can change file content. |
| 10 | + pub(crate) fn converts_on_update(&self) -> bool { |
| 11 | + self.eol_conversion_mode == EolConversionMode::InputOutput |
| 12 | + } |
| 13 | + |
| 14 | /// The limit to probe for whether the file is binary is 8KB. |
| 15 | /// All files strictly smaller than the limit are always |
| 16 | /// evaluated correctly and in full. |
| 17 | diff --git a/lib/src/file_cache.rs b/lib/src/file_cache.rs |
| 18 | new file mode 100644 |
| 19 | index 0000000000..be03d9ca74 |
| 20 | --- /dev/null |
| 21 | +++ b/lib/src/file_cache.rs |
| 22 | @@ -0,0 +1,169 @@ |
| 23 | +// Copyright 2026 The Jujutsu Authors |
| 24 | +// |
| 25 | +// Licensed under the Apache License, Version 2.0 (the "License"); |
| 26 | +// you may not use this file except in compliance with the License. |
| 27 | +// You may obtain a copy of the License at |
| 28 | +// |
| 29 | +// https://www.apache.org/licenses/LICENSE-2.0 |
| 30 | +// |
| 31 | +// Unless required by applicable law or agreed to in writing, software |
| 32 | +// distributed under the License is distributed on an "AS IS" BASIS, |
| 33 | +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 34 | +// See the License for the specific language governing permissions and |
| 35 | +// limitations under the License. |
| 36 | + |
| 37 | +//! Content-addressed cache of materialized files, which checkouts clone |
| 38 | +//! instead of rewriting. |
| 39 | + |
| 40 | +use std::collections::HashMap; |
| 41 | +use std::fs; |
| 42 | +use std::fs::File; |
| 43 | +use std::io; |
| 44 | +use std::io::Seek as _; |
| 45 | +use std::path::Path; |
| 46 | +use std::path::PathBuf; |
| 47 | + |
| 48 | +use futures::AsyncRead; |
| 49 | +use tempfile::TempPath; |
| 50 | + |
| 51 | +use crate::file_util::copy_async_to_sync; |
| 52 | + |
| 53 | +/// Cache entries for one checkout. Entries it adds stay pending until |
| 54 | +/// [`Self::commit()`] makes them durable with a single device flush, as Git's |
| 55 | +/// `core.fsyncMethod=batch` does. |
| 56 | +pub(crate) struct FileCache { |
| 57 | + dir: PathBuf, |
| 58 | + pending: HashMap<String, TempPath>, |
| 59 | +} |
| 60 | + |
| 61 | +impl FileCache { |
| 62 | + /// Opens the cache of the repo at `repo_path`. Returns `None` where files |
| 63 | + /// can't be cloned. |
| 64 | + pub fn open(repo_path: &Path) -> Option<Self> { |
| 65 | + let dir = repo_path.join("file_cache"); |
| 66 | + let supported = cfg!(target_vendor = "apple") && fs::create_dir_all(&dir).is_ok(); |
| 67 | + supported.then(|| Self { |
| 68 | + dir, |
| 69 | + pending: HashMap::new(), |
| 70 | + }) |
| 71 | + } |
| 72 | + |
| 73 | + /// Opens the entry `name`, adding it from `contents` if missing. |
| 74 | + pub async fn entry(&mut self, name: String, contents: impl AsyncRead) -> io::Result<File> { |
| 75 | + let path = match self.pending.get(&name) { |
| 76 | + Some(temp_path) => temp_path.to_path_buf(), |
| 77 | + None => self.dir.join(&name), |
| 78 | + }; |
| 79 | + match File::open(path) { |
| 80 | + Err(err) if err.kind() == io::ErrorKind::NotFound => {} |
| 81 | + result => return result, |
| 82 | + } |
| 83 | + |
| 84 | + let mut temp_file = platform::new_entry(&self.dir)?; |
| 85 | + copy_async_to_sync(contents, temp_file.as_file_mut()).await?; |
| 86 | + platform::write_out(temp_file.as_file())?; |
| 87 | + let (mut file, temp_path) = temp_file.into_parts(); |
| 88 | + file.rewind()?; |
| 89 | + self.pending.insert(name, temp_path); |
| 90 | + Ok(file) |
| 91 | + } |
| 92 | + |
| 93 | + /// Publishes pending entries once they are durable. |
| 94 | + pub fn commit(self) -> io::Result<()> { |
| 95 | + if self.pending.is_empty() { |
| 96 | + return Ok(()); |
| 97 | + } |
| 98 | + platform::flush_device(&File::open(&self.dir)?)?; |
| 99 | + for (name, temp_path) in self.pending { |
| 100 | + temp_path.persist(self.dir.join(name))?; |
| 101 | + } |
| 102 | + Ok(()) |
| 103 | + } |
| 104 | +} |
| 105 | + |
| 106 | +pub(crate) use platform::clone_dir; |
| 107 | +pub(crate) use platform::clone_file; |
| 108 | + |
| 109 | +#[cfg(target_vendor = "apple")] |
| 110 | +mod platform { |
| 111 | + use std::fs::File; |
| 112 | + use std::fs::Permissions; |
| 113 | + use std::io; |
| 114 | + use std::os::unix::fs::PermissionsExt as _; |
| 115 | + use std::path::Path; |
| 116 | + |
| 117 | + use rustix::fs::CloneFlags; |
| 118 | + use rustix::fs::Mode; |
| 119 | + use rustix::fs::OFlags; |
| 120 | + use tempfile::NamedTempFile; |
| 121 | + |
| 122 | + /// Creates a temporary entry with the mode of a checked-out file, so |
| 123 | + /// clones of it need no `chmod`. |
| 124 | + pub fn new_entry(dir: &Path) -> io::Result<NamedTempFile> { |
| 125 | + tempfile::Builder::new() |
| 126 | + .permissions(Permissions::from_mode(0o644)) |
| 127 | + .tempfile_in(dir) |
| 128 | + } |
| 129 | + |
| 130 | + /// Creates `dst` as a copy-on-write clone of `src`, failing if `dst` |
| 131 | + /// exists. The clone keeps the mode and mtime of `src`. |
| 132 | + pub fn clone_file(src: &File, dst: &Path) -> io::Result<()> { |
| 133 | + Ok(rustix::fs::fclonefileat( |
| 134 | + src, |
| 135 | + rustix::fs::CWD, |
| 136 | + dst, |
| 137 | + CloneFlags::empty(), |
| 138 | + )?) |
| 139 | + } |
| 140 | + |
| 141 | + /// Like [`clone_file()`], but clones the directory hierarchy at `src` |
| 142 | + /// without following a symlink there. |
| 143 | + pub fn clone_dir(src: &Path, dst: &Path) -> io::Result<()> { |
| 144 | + let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW; |
| 145 | + let src = rustix::fs::open(src, flags, Mode::empty())?; |
| 146 | + Ok(rustix::fs::fclonefileat( |
| 147 | + src, |
| 148 | + rustix::fs::CWD, |
| 149 | + dst, |
| 150 | + CloneFlags::empty(), |
| 151 | + )?) |
| 152 | + } |
| 153 | + |
| 154 | + /// Writes `file` to the device, but not through the device's own cache. |
| 155 | + pub fn write_out(file: &File) -> io::Result<()> { |
| 156 | + Ok(rustix::fs::fsync(file)?) |
| 157 | + } |
| 158 | + |
| 159 | + pub fn flush_device(file: &File) -> io::Result<()> { |
| 160 | + Ok(rustix::fs::fcntl_fullfsync(file)?) |
| 161 | + } |
| 162 | +} |
| 163 | + |
| 164 | +#[cfg(not(target_vendor = "apple"))] |
| 165 | +mod platform { |
| 166 | + use std::fs::File; |
| 167 | + use std::io; |
| 168 | + use std::path::Path; |
| 169 | + |
| 170 | + use tempfile::NamedTempFile; |
| 171 | + |
| 172 | + pub fn new_entry(dir: &Path) -> io::Result<NamedTempFile> { |
| 173 | + NamedTempFile::new_in(dir) |
| 174 | + } |
| 175 | + |
| 176 | + pub fn clone_file(_src: &File, _dst: &Path) -> io::Result<()> { |
| 177 | + Err(io::ErrorKind::Unsupported.into()) |
| 178 | + } |
| 179 | + |
| 180 | + pub fn clone_dir(_src: &Path, _dst: &Path) -> io::Result<()> { |
| 181 | + Err(io::ErrorKind::Unsupported.into()) |
| 182 | + } |
| 183 | + |
| 184 | + pub fn write_out(file: &File) -> io::Result<()> { |
| 185 | + file.sync_data() |
| 186 | + } |
| 187 | + |
| 188 | + pub fn flush_device(file: &File) -> io::Result<()> { |
| 189 | + file.sync_all() |
| 190 | + } |
| 191 | +} |
| 192 | diff --git a/lib/src/lib.rs b/lib/src/lib.rs |
| 193 | index c97831a80a..8466d009d1 100644 |
| 194 | --- a/lib/src/lib.rs |
| 195 | +++ b/lib/src/lib.rs |
| 196 | @@ -49,6 +49,7 @@ |
| 197 | pub(crate) mod eol; |
| 198 | pub mod evolution; |
| 199 | pub mod extensions_map; |
| 200 | +mod file_cache; |
| 201 | pub mod file_util; |
| 202 | pub mod files; |
| 203 | pub mod fileset; |
| 204 | diff --git a/lib/src/local_working_copy.rs b/lib/src/local_working_copy.rs |
| 205 | index d7da1716fc..7c1294ff6f 100644 |
| 206 | --- a/lib/src/local_working_copy.rs |
| 207 | +++ b/lib/src/local_working_copy.rs |
| 208 | @@ -53,6 +53,7 @@ |
| 209 | use pollster::FutureExt as _; |
| 210 | use prost::Message as _; |
| 211 | use rayon::iter::IntoParallelIterator as _; |
| 212 | +use rayon::iter::IntoParallelRefIterator as _; |
| 213 | use rayon::prelude::IndexedParallelIterator as _; |
| 214 | use rayon::prelude::ParallelIterator as _; |
| 215 | use tempfile::NamedTempFile; |
| 216 | @@ -80,6 +81,9 @@ |
| 217 | use crate::conflicts::materialize_tree_value; |
| 218 | pub use crate::eol::EolConversionMode; |
| 219 | use crate::eol::TargetEolStrategy; |
| 220 | +use crate::file_cache::FileCache; |
| 221 | +use crate::file_cache::clone_dir; |
| 222 | +use crate::file_cache::clone_file; |
| 223 | use crate::file_util::FileIdentity; |
| 224 | use crate::file_util::check_symlink_support; |
| 225 | use crate::file_util::copy_async_to_sync; |
| 226 | @@ -99,6 +103,7 @@ |
| 227 | use crate::matchers::Matcher; |
| 228 | use crate::matchers::PrefixMatcher; |
| 229 | use crate::matchers::UnionMatcher; |
| 230 | +use crate::matchers::Visit; |
| 231 | use crate::merge::Merge; |
| 232 | use crate::merge::MergeBuilder; |
| 233 | use crate::merge::MergedTreeValue; |
| 234 | @@ -127,6 +132,7 @@ |
| 235 | use crate::working_copy::WorkingCopy; |
| 236 | use crate::working_copy::WorkingCopyFactory; |
| 237 | use crate::working_copy::WorkingCopyStateError; |
| 238 | +use crate::workspace_store::SimpleWorkspaceStore; |
| 239 | |
| 240 | fn symlink_target_convert_to_store(path: &Path) -> Option<Cow<'_, str>> { |
| 241 | let path = path.to_str()?; |
| 242 | @@ -818,6 +824,14 @@ |
| 243 | |
| 244 | const RESERVED_DIR_NAMES: &[&str] = &[".git", ".jj"]; |
| 245 | |
| 246 | +/// Smaller files are written directly, since cloning a file costs more than |
| 247 | +/// writing a few blocks. |
| 248 | +const MIN_CACHED_FILE_SIZE: u64 = 16 << 10; |
| 249 | + |
| 250 | +/// Untracked directories that tend to be huge, so their parents are never |
| 251 | +/// cloned from a sibling workspace whole. |
| 252 | +const UNCLONED_DIR_NAMES: &[&str] = &["node_modules", "target"]; |
| 253 | + |
| 254 | fn file_identity_from_symlink_path(disk_path: &Path) -> io::Result<Option<FileIdentity>> { |
| 255 | match FileIdentity::from_symlink_path(disk_path) { |
| 256 | Ok(identity) => Ok(Some(identity)), |
| 257 | @@ -2071,6 +2085,82 @@ |
| 258 | .map_err(|err| checkout_error_for_mtime_out_of_range(err, disk_path)) |
| 259 | } |
| 260 | |
| 261 | + /// Like [`Self::write_file()`], but clones `disk_path` from `file_cache` |
| 262 | + /// if the content is large enough to be worth it. Drops `file_cache` and |
| 263 | + /// copies instead if the file system can't clone. `fresh` means the |
| 264 | + /// working copy held nothing before this checkout. |
| 265 | + async fn write_cached_file( |
| 266 | + &self, |
| 267 | + disk_path: &Path, |
| 268 | + id: &FileId, |
| 269 | + contents: impl AsyncRead + Send + Unpin, |
| 270 | + exec_bit: ExecBit, |
| 271 | + file_cache: &mut Option<FileCache>, |
| 272 | + fresh: bool, |
| 273 | + ) -> Result<FileState, CheckoutError> { |
| 274 | + let Some(cache) = file_cache else { |
| 275 | + return self.write_file(disk_path, contents, exec_bit, true).await; |
| 276 | + }; |
| 277 | + |
| 278 | + let mut contents = self |
| 279 | + .target_eol_strategy |
| 280 | + .convert_eol_for_update(contents) |
| 281 | + .await |
| 282 | + .map_err(|err| CheckoutError::Other { |
| 283 | + message: "Failed to convert the EOL for the content".to_string(), |
| 284 | + err: err.into(), |
| 285 | + })?; |
| 286 | + let mut head = Vec::new(); |
| 287 | + futures::AsyncReadExt::take(&mut contents, MIN_CACHED_FILE_SIZE) |
| 288 | + .read_to_end(&mut head) |
| 289 | + .await |
| 290 | + .map_err(|err| CheckoutError::Other { |
| 291 | + message: format!("Failed to read the content for {}", disk_path.display()), |
| 292 | + err: err.into(), |
| 293 | + })?; |
| 294 | + let is_small = (head.len() as u64) < MIN_CACHED_FILE_SIZE; |
| 295 | + let contents = futures::AsyncReadExt::chain(head.as_slice(), contents); |
| 296 | + if is_small { |
| 297 | + return self.write_file(disk_path, contents, exec_bit, false).await; |
| 298 | + } |
| 299 | + |
| 300 | + let entry_name = if self.target_eol_strategy.converts_on_update() { |
| 301 | + format!("{}.crlf", id.hex()) |
| 302 | + } else { |
| 303 | + id.hex() |
| 304 | + }; |
| 305 | + let entry = |
| 306 | + cache |
| 307 | + .entry(entry_name, contents) |
| 308 | + .await |
| 309 | + .map_err(|err| CheckoutError::Other { |
| 310 | + message: format!("Failed to cache {}", disk_path.display()), |
| 311 | + err: err.into(), |
| 312 | + })?; |
| 313 | + // A clone keeps the entry's mtime, which would look stale to build tools |
| 314 | + // unless nothing in the working copy predates this checkout. |
| 315 | + if !fresh { |
| 316 | + entry |
| 317 | + .set_modified(SystemTime::now()) |
| 318 | + .map_err(|err| checkout_error_for_stat_error(err, disk_path))?; |
| 319 | + } |
| 320 | + let metadata = entry |
| 321 | + .metadata() |
| 322 | + .map_err(|err| checkout_error_for_stat_error(err, disk_path))?; |
| 323 | + if clone_file(&entry, disk_path).is_err() { |
| 324 | + *file_cache = None; |
| 325 | + return self |
| 326 | + .write_file(disk_path, AllowStdIo::new(entry), exec_bit, false) |
| 327 | + .await; |
| 328 | + } |
| 329 | + if exec_bit.0 { |
| 330 | + set_executable(exec_bit, disk_path) |
| 331 | + .map_err(|err| checkout_error_for_stat_error(err, disk_path))?; |
| 332 | + } |
| 333 | + FileState::for_file(exec_bit, metadata.len(), &metadata) |
| 334 | + .map_err(|err| checkout_error_for_mtime_out_of_range(err, disk_path)) |
| 335 | + } |
| 336 | + |
| 337 | fn write_symlink(&self, disk_path: &Path, target: String) -> Result<FileState, CheckoutError> { |
| 338 | let target = symlink_target_convert_to_disk(&target); |
| 339 | |
| 340 | @@ -2183,6 +2273,180 @@ |
| 341 | }) |
| 342 | } |
| 343 | |
| 344 | + /// Loads the tree state of the sibling workspace whose tree differs least |
| 345 | + /// from `new_tree`, along with the paths that differ. |
| 346 | + async fn closest_sibling(&self, new_tree: &MergedTree) -> Option<(Self, HashSet<RepoPathBuf>)> { |
| 347 | + let workspace_paths = SimpleWorkspaceStore::load(self.store.repo_path()) |
| 348 | + .and_then(|store| store.workspace_paths()) |
| 349 | + .ok()?; |
| 350 | + let own_path = dunce::canonicalize(&self.working_copy_path).ok()?; |
| 351 | + let settings = TreeStateSettings { |
| 352 | + conflict_marker_style: self.conflict_marker_style, |
| 353 | + eol_conversion_mode: EolConversionMode::None, |
| 354 | + exec_change_setting: ExecChangeSetting::Auto, |
| 355 | + fsmonitor_settings: FsmonitorSettings::None, |
| 356 | + }; |
| 357 | + |
| 358 | + let mut closest: Option<(Self, HashSet<RepoPathBuf>)> = None; |
| 359 | + for path in workspace_paths { |
| 360 | + let state_path = path.join(".jj").join("working_copy"); |
| 361 | + // Loading a missing tree state would create one in the sibling. |
| 362 | + if !state_path.join("tree_state").is_file() |
| 363 | + || dunce::canonicalize(&path).is_ok_and(|path| path == own_path) |
| 364 | + { |
| 365 | + continue; |
| 366 | + } |
| 367 | + let Ok(sibling) = Self::load(self.store.clone(), path, state_path, &settings) else { |
| 368 | + continue; |
| 369 | + }; |
| 370 | + if sibling.sparse_patterns != [RepoPathBuf::root()] { |
| 371 | + continue; |
| 372 | + } |
| 373 | + let limit = closest |
| 374 | + .as_ref() |
| 375 | + .map_or(usize::MAX, |(_, paths)| paths.len()); |
| 376 | + let differing_paths: HashSet<RepoPathBuf> = sibling |
| 377 | + .tree |
| 378 | + .diff_stream(new_tree, &EverythingMatcher) |
| 379 | + .map(|entry| entry.path) |
| 380 | + .take(limit) |
| 381 | + .collect() |
| 382 | + .await; |
| 383 | + if differing_paths.len() < limit { |
| 384 | + closest = Some((sibling, differing_paths)); |
| 385 | + } |
| 386 | + } |
| 387 | + closest |
| 388 | + } |
| 389 | + |
| 390 | + /// Seeds a fresh working copy with directories cloned from the sibling |
| 391 | + /// workspace closest to `new_tree`, then keeps only the cloned files whose |
| 392 | + /// content the sibling's tree state vouches for. Returns their states. |
| 393 | + async fn seed_from_sibling( |
| 394 | + &self, |
| 395 | + new_tree: &MergedTree, |
| 396 | + matcher: &dyn Matcher, |
| 397 | + ) -> Result<Vec<(RepoPathBuf, FileState)>, CheckoutError> { |
| 398 | + let Some((sibling, differing_paths)) = self.closest_sibling(new_tree).await else { |
| 399 | + return Ok(vec![]); |
| 400 | + }; |
| 401 | + let sibling_states = sibling.file_states(); |
| 402 | + |
| 403 | + let mut tracked_dirs: HashSet<&RepoPath> = HashSet::new(); |
| 404 | + for path in sibling_states.paths() { |
| 405 | + for dir in path.ancestors().skip(1) { |
| 406 | + if !tracked_dirs.insert(dir) { |
| 407 | + break; |
| 408 | + } |
| 409 | + } |
| 410 | + } |
| 411 | + let mut child_dirs: HashMap<&RepoPath, Vec<&RepoPath>> = HashMap::new(); |
| 412 | + for &dir in &tracked_dirs { |
| 413 | + if let Some(parent) = dir.parent() { |
| 414 | + child_dirs.entry(parent).or_default().push(dir); |
| 415 | + } |
| 416 | + } |
| 417 | + let dirs_holding_uncloned: Vec<&RepoPath> = tracked_dirs |
| 418 | + .par_iter() |
| 419 | + .copied() |
| 420 | + .filter(|dir| { |
| 421 | + let disk_dir = dir.to_fs_path_unchecked(&sibling.working_copy_path); |
| 422 | + UNCLONED_DIR_NAMES.iter().any(|&name| { |
| 423 | + let child = dir.join(RepoPathComponent::new(name).unwrap()); |
| 424 | + !tracked_dirs.contains(&*child) |
| 425 | + && disk_dir.join(name).symlink_metadata().is_ok() |
| 426 | + }) |
| 427 | + }) |
| 428 | + .collect(); |
| 429 | + let split_dirs: HashSet<&RepoPath> = dirs_holding_uncloned |
| 430 | + .into_iter() |
| 431 | + .chain(differing_paths.iter().filter_map(|path| path.parent())) |
| 432 | + .flat_map(RepoPath::ancestors) |
| 433 | + .collect(); |
| 434 | + |
| 435 | + // Never the root, which holds .jj and usually build output. |
| 436 | + let mut clone_roots = vec![]; |
| 437 | + let mut dirs_to_split = vec![RepoPath::root()]; |
| 438 | + while let Some(dir) = dirs_to_split.pop() { |
| 439 | + for &child in child_dirs.get(dir).into_iter().flatten() { |
| 440 | + if !split_dirs.contains(child) && matcher.visit(child) == Visit::AllRecursively { |
| 441 | + clone_roots.push(child); |
| 442 | + } else { |
| 443 | + dirs_to_split.push(child); |
| 444 | + } |
| 445 | + } |
| 446 | + } |
| 447 | + |
| 448 | + // Clone roots hold the same tree in both workspaces, so a file is |
| 449 | + // vouched for if the sibling would see it as unchanged. |
| 450 | + let vouched_state = |path: &RepoPath, metadata: &Metadata| { |
| 451 | + let old_state = sibling_states.get(path)?; |
| 452 | + let new_state = file_state(metadata).ok()??; |
| 453 | + let clean = old_state.materialized_conflict_data.is_none() |
| 454 | + && new_state.is_clean(&old_state) |
| 455 | + && old_state.mtime < sibling.own_mtime; |
| 456 | + clean.then_some(new_state) |
| 457 | + }; |
| 458 | + let seed = |root: &RepoPath| -> Result<Vec<(RepoPathBuf, FileState)>, CheckoutError> { |
| 459 | + let disk_root = root.to_fs_path(&self.working_copy_path)?; |
| 460 | + let created_parent = fs::create_dir_all(disk_root.parent().unwrap()); |
| 461 | + let source = root.to_fs_path(&sibling.working_copy_path)?; |
| 462 | + if created_parent |
| 463 | + .and_then(|()| clone_dir(&source, &disk_root)) |
| 464 | + .is_err() |
| 465 | + { |
| 466 | + fs::remove_dir_all(&disk_root).ok(); |
| 467 | + return Ok(vec![]); |
| 468 | + } |
| 469 | + |
| 470 | + let mut seeded = vec![]; |
| 471 | + let mut dirs_to_walk = vec![root.to_owned()]; |
| 472 | + while let Some(dir) = dirs_to_walk.pop() { |
| 473 | + let disk_dir = dir.to_fs_path(&self.working_copy_path)?; |
| 474 | + let entries = fs::read_dir(&disk_dir) |
| 475 | + .map_err(|err| checkout_error_for_stat_error(err, &disk_dir))?; |
| 476 | + for entry in entries { |
| 477 | + let entry = |
| 478 | + entry.map_err(|err| checkout_error_for_stat_error(err, &disk_dir))?; |
| 479 | + let disk_path = entry.path(); |
| 480 | + let metadata = entry |
| 481 | + .metadata() |
| 482 | + .map_err(|err| checkout_error_for_stat_error(err, &disk_path))?; |
| 483 | + let path = entry |
| 484 | + .file_name() |
| 485 | + .to_str() |
| 486 | + .and_then(|name| RepoPathComponent::new(name).ok()) |
| 487 | + .map(|name| dir.join(name)); |
| 488 | + let removed = match path { |
| 489 | + Some(path) if metadata.is_dir() && tracked_dirs.contains(&*path) => { |
| 490 | + dirs_to_walk.push(path); |
| 491 | + Ok(()) |
| 492 | + } |
| 493 | + _ if metadata.is_dir() => fs::remove_dir_all(&disk_path), |
| 494 | + Some(path) => match vouched_state(&path, &metadata) { |
| 495 | + Some(state) => { |
| 496 | + seeded.push((path, state)); |
| 497 | + Ok(()) |
| 498 | + } |
| 499 | + None => fs::remove_file(&disk_path), |
| 500 | + }, |
| 501 | + None => fs::remove_file(&disk_path), |
| 502 | + }; |
| 503 | + removed.map_err(|err| CheckoutError::Other { |
| 504 | + message: format!("Failed to remove {}", disk_path.display()), |
| 505 | + err: err.into(), |
| 506 | + })?; |
| 507 | + } |
| 508 | + } |
| 509 | + Ok(seeded) |
| 510 | + }; |
| 511 | + let seeded: Vec<Vec<_>> = clone_roots |
| 512 | + .into_par_iter() |
| 513 | + .map(seed) |
| 514 | + .collect::<Result<_, _>>()?; |
| 515 | + Ok(seeded.into_iter().flatten().collect()) |
| 516 | + } |
| 517 | + |
| 518 | async fn update( |
| 519 | &mut self, |
| 520 | old_tree: &MergedTree, |
| 521 | @@ -2200,6 +2464,22 @@ |
| 522 | let mut changed_file_states = Vec::new(); |
| 523 | let mut deleted_files = HashSet::new(); |
| 524 | let mut prev_created_path: RepoPathBuf = RepoPathBuf::root(); |
| 525 | + let mut file_cache = FileCache::open(self.store.repo_path()); |
| 526 | + let fresh = *old_tree.tree_ids() == self.store.empty_merged_tree_id() |
| 527 | + && fs::read_dir(&self.working_copy_path).is_ok_and(|mut entries| { |
| 528 | + entries.all(|entry| entry.is_ok_and(|entry| entry.file_name() == ".jj")) |
| 529 | + }); |
| 530 | + let seeded = if fresh |
| 531 | + && file_cache.is_some() |
| 532 | + && *new_tree.tree_ids() != self.store.empty_merged_tree_id() |
| 533 | + && !self.target_eol_strategy.converts_on_update() |
| 534 | + { |
| 535 | + self.seed_from_sibling(new_tree, matcher).await? |
| 536 | + } else { |
| 537 | + vec![] |
| 538 | + }; |
| 539 | + let seeded_matcher = FilesMatcher::new(seeded.iter().map(|(path, _)| path)); |
| 540 | + let matcher: &dyn Matcher = &DifferenceMatcher::new(matcher, &seeded_matcher); |
| 541 | |
| 542 | let mut process_diff_entry = async |path: RepoPathBuf, |
| 543 | before: MergedTreeValue, |
| 544 | @@ -2336,8 +2616,15 @@ |
| 545 | MaterializedTreeValue::File(file) => { |
| 546 | let exec_bit = |
| 547 | ExecBit::new_from_repo(file.executable, self.exec_policy, get_prev_exec); |
| 548 | - self.write_file(&disk_path, file.reader, exec_bit, true) |
| 549 | - .await? |
| 550 | + self.write_cached_file( |
| 551 | + &disk_path, |
| 552 | + &file.id, |
| 553 | + file.reader, |
| 554 | + exec_bit, |
| 555 | + &mut file_cache, |
| 556 | + fresh, |
| 557 | + ) |
| 558 | + .await? |
| 559 | } |
| 560 | MaterializedTreeValue::Symlink { id: _, target } => { |
| 561 | if self.symlink_support { |
| 562 | @@ -2451,6 +2738,16 @@ |
| 563 | changed_file_states.sort_unstable_by(|(path1, _), (path2, _)| path1.cmp(path2)); |
| 564 | } |
| 565 | |
| 566 | + if !seeded.is_empty() { |
| 567 | + stats.added_files += u32::try_from(seeded.len()).unwrap(); |
| 568 | + changed_file_states.extend(seeded); |
| 569 | + changed_file_states.sort_unstable_by(|(path1, _), (path2, _)| path1.cmp(path2)); |
| 570 | + } |
| 571 | + |
| 572 | + // The files are in place; a cache that fails to update only costs speed. |
| 573 | + if let Some(file_cache) = file_cache { |
| 574 | + file_cache.commit().ok(); |
| 575 | + } |
| 576 | self.file_states |
| 577 | .merge_in(changed_file_states, &deleted_files); |
| 578 | Ok(stats) |
| 579 | diff --git a/lib/src/repo.rs b/lib/src/repo.rs |
| 580 | index 61e5a6e093..b69a8ae85f 100644 |
| 581 | --- a/lib/src/repo.rs |
| 582 | +++ b/lib/src/repo.rs |
| 583 | @@ -223,7 +223,7 @@ |
| 584 | fs::write(&backend_path, backend.name()).context(&backend_path)?; |
| 585 | let merge_options = |
| 586 | MergeOptions::from_settings(settings).map_err(|err| BackendInitError(err.into()))?; |
| 587 | - let store = Store::new(backend, signer, merge_options); |
| 588 | + let store = Store::new(backend, signer, merge_options, repo_path.clone()); |
| 589 | |
| 590 | let op_store_path = repo_path.join("op_store"); |
| 591 | fs::create_dir(&op_store_path).context(&op_store_path)?; |
| 592 | @@ -711,6 +711,7 @@ |
| 593 | store_factories.load_backend(settings, &repo_path.join("store"))?, |
| 594 | Signer::from_settings(settings)?, |
| 595 | merge_options, |
| 596 | + repo_path.to_owned(), |
| 597 | ); |
| 598 | let root_op_data = RootOperationData { |
| 599 | root_commit_id: store.root_commit_id().clone(), |
| 600 | diff --git a/lib/src/store.rs b/lib/src/store.rs |
| 601 | index 813d1eef87..e812b6b701 100644 |
| 602 | --- a/lib/src/store.rs |
| 603 | +++ b/lib/src/store.rs |
| 604 | @@ -16,6 +16,8 @@ |
| 605 | |
| 606 | use std::fmt::Debug; |
| 607 | use std::fmt::Formatter; |
| 608 | +use std::path::Path; |
| 609 | +use std::path::PathBuf; |
| 610 | use std::pin::Pin; |
| 611 | use std::sync::Arc; |
| 612 | use std::sync::Mutex; |
| 613 | @@ -59,6 +61,7 @@ |
| 614 | commit_cache: Mutex<CLruCache<CommitId, Arc<backend::Commit>>>, |
| 615 | tree_cache: Mutex<CLruCache<(RepoPathBuf, TreeId), Arc<backend::Tree>>>, |
| 616 | merge_options: MergeOptions, |
| 617 | + repo_path: PathBuf, |
| 618 | } |
| 619 | |
| 620 | impl Debug for Store { |
| 621 | @@ -74,6 +77,7 @@ |
| 622 | backend: Box<dyn Backend>, |
| 623 | signer: Signer, |
| 624 | merge_options: MergeOptions, |
| 625 | + repo_path: PathBuf, |
| 626 | ) -> Arc<Self> { |
| 627 | Arc::new(Self { |
| 628 | backend, |
| 629 | @@ -81,6 +85,7 @@ |
| 630 | commit_cache: Mutex::new(CLruCache::new(COMMIT_CACHE_CAPACITY.try_into().unwrap())), |
| 631 | tree_cache: Mutex::new(CLruCache::new(TREE_CACHE_CAPACITY.try_into().unwrap())), |
| 632 | merge_options, |
| 633 | + repo_path, |
| 634 | }) |
| 635 | } |
| 636 | |
| 637 | @@ -102,6 +107,11 @@ |
| 638 | &self.merge_options |
| 639 | } |
| 640 | |
| 641 | + /// The repo directory, which holds state shared by its workspaces. |
| 642 | + pub fn repo_path(&self) -> &Path { |
| 643 | + &self.repo_path |
| 644 | + } |
| 645 | + |
| 646 | pub fn get_copy_records( |
| 647 | &self, |
| 648 | paths: Option<&[RepoPathBuf]>, |
| 649 | diff --git a/lib/src/workspace_store.rs b/lib/src/workspace_store.rs |
| 650 | index 32e6072b4b..a5ad1bf614 100644 |
| 651 | --- a/lib/src/workspace_store.rs |
| 652 | +++ b/lib/src/workspace_store.rs |
| 653 | @@ -127,6 +127,18 @@ |
| 654 | Ok(store) |
| 655 | } |
| 656 | |
| 657 | + /// Returns the paths of all registered workspaces. |
| 658 | + pub fn workspace_paths(&self) -> Result<Vec<PathBuf>, WorkspaceStoreError> { |
| 659 | + let paths = self |
| 660 | + .read_store()? |
| 661 | + .workspaces |
| 662 | + .iter() |
| 663 | + .map(|w| path_from_bytes(&w.path).map(|path| self.repo_path.join(path))) |
| 664 | + .collect::<Result<_, _>>() |
| 665 | + .map_err(SimpleWorkspaceStoreError::BadPathEncoding)?; |
| 666 | + Ok(paths) |
| 667 | + } |
| 668 | + |
| 669 | fn lock(&self) -> Result<FileLock, SimpleWorkspaceStoreError> { |
| 670 | Ok(FileLock::lock(self.lock_file.clone())?) |
| 671 | } |
| 672 | diff --git a/lib/tests/test_local_working_copy.rs b/lib/tests/test_local_working_copy.rs |
| 673 | index 55aaffd9c9..00a12f87bc 100644 |
| 674 | --- a/lib/tests/test_local_working_copy.rs |
| 675 | +++ b/lib/tests/test_local_working_copy.rs |
| 676 | @@ -51,8 +51,10 @@ |
| 677 | use jj_lib::merge::SameChange; |
| 678 | use jj_lib::merged_tree::MergedTree; |
| 679 | use jj_lib::merged_tree_builder::MergedTreeBuilder; |
| 680 | +use jj_lib::object_id::ObjectId as _; |
| 681 | use jj_lib::op_store::OperationId; |
| 682 | use jj_lib::ref_name::WorkspaceName; |
| 683 | +use jj_lib::ref_name::WorkspaceNameBuf; |
| 684 | use jj_lib::repo::ReadonlyRepo; |
| 685 | use jj_lib::repo::Repo as _; |
| 686 | use jj_lib::repo_path::RepoPath; |
| 687 | @@ -68,6 +70,7 @@ |
| 688 | use jj_lib::working_copy::WorkingCopy as _; |
| 689 | use jj_lib::workspace::Workspace; |
| 690 | use jj_lib::workspace::default_working_copy_factories; |
| 691 | +use jj_lib::workspace::default_working_copy_factory; |
| 692 | use pollster::FutureExt as _; |
| 693 | use test_case::test_case; |
| 694 | use testutils::CommitBuilderExt as _; |
| 695 | @@ -84,6 +87,7 @@ |
| 696 | use testutils::repo_path_buf; |
| 697 | use testutils::repo_path_component; |
| 698 | use testutils::write_random_commit; |
| 699 | +use testutils::write_working_copy_file; |
| 700 | |
| 701 | fn check_icase_fs(dir: &Path) -> bool { |
| 702 | let test_file = tempfile::Builder::new() |
| 703 | @@ -822,6 +826,125 @@ |
| 704 | } |
| 705 | |
| 706 | #[test] |
| 707 | +fn test_checkout_clones_large_files_from_file_cache() -> TestResult { |
| 708 | + // Edits to a cloned file must not reach the cache entry or later checkouts. |
| 709 | + let mut test_workspace = TestWorkspace::init(); |
| 710 | + let repo = test_workspace.repo.clone(); |
| 711 | + let workspace_root = test_workspace.workspace.workspace_root().to_owned(); |
| 712 | + let cache_dir = repo.store().repo_path().join("file_cache"); |
| 713 | + let clones = cfg!(target_vendor = "apple"); |
| 714 | + |
| 715 | + let large_path = repo_path("large"); |
| 716 | + let small_path = repo_path("small"); |
| 717 | + let large_contents = "large\n".repeat(16 << 10); |
| 718 | + let tree = create_tree( |
| 719 | + &repo, |
| 720 | + &[(large_path, &large_contents), (small_path, "small")], |
| 721 | + ); |
| 722 | + let commit = commit_with_tree(repo.store(), tree.clone()); |
| 723 | + let Ok(Some(TreeValue::File { id: large_id, .. })) = |
| 724 | + tree.path_value(large_path).block_on()?.into_resolved() |
| 725 | + else { |
| 726 | + panic!("large file should be resolved"); |
| 727 | + }; |
| 728 | + let entry_path = cache_dir.join(large_id.hex()); |
| 729 | + let large_disk_path = large_path.to_fs_path_unchecked(&workspace_root); |
| 730 | + |
| 731 | + test_workspace |
| 732 | + .workspace |
| 733 | + .check_out(repo.op_id().clone(), None, &commit) |
| 734 | + .block_on()?; |
| 735 | + assert_eq!(std::fs::read_to_string(&large_disk_path)?, large_contents); |
| 736 | + if clones { |
| 737 | + assert_eq!(std::fs::read_to_string(&entry_path)?, large_contents); |
| 738 | + assert_eq!(std::fs::read_dir(&cache_dir)?.count(), 1); |
| 739 | + } else { |
| 740 | + assert!(!cache_dir.exists()); |
| 741 | + } |
| 742 | + |
| 743 | + write_working_copy_file(&workspace_root, large_path, "edited"); |
| 744 | + assert_ne!(test_workspace.snapshot()?.tree_ids(), tree.tree_ids()); |
| 745 | + if clones { |
| 746 | + assert_eq!(std::fs::read_to_string(&entry_path)?, large_contents); |
| 747 | + File::open(&entry_path)?.set_modified(SystemTime::UNIX_EPOCH)?; |
| 748 | + } |
| 749 | + |
| 750 | + test_workspace |
| 751 | + .workspace |
| 752 | + .check_out(repo.op_id().clone(), None, &commit) |
| 753 | + .block_on()?; |
| 754 | + assert_eq!(std::fs::read_to_string(&large_disk_path)?, large_contents); |
| 755 | + let age = SystemTime::now().duration_since(large_disk_path.metadata()?.modified()?); |
| 756 | + assert!(age.is_ok_and(|age| age < Duration::from_secs(60))); |
| 757 | + assert_eq!(test_workspace.snapshot()?.tree_ids(), tree.tree_ids()); |
| 758 | + Ok(()) |
| 759 | +} |
| 760 | + |
| 761 | +#[test] |
| 762 | +fn test_checkout_fresh_workspace_seeds_from_sibling() -> TestResult { |
| 763 | + // Unchanged directories come from the sibling, but not its unsnapshotted |
| 764 | + // edits, untracked files, or node_modules. |
| 765 | + let mut test_workspace = TestWorkspace::init(); |
| 766 | + let repo = test_workspace.repo.clone(); |
| 767 | + let root1 = test_workspace.workspace.workspace_root().to_owned(); |
| 768 | + let tree = create_tree( |
| 769 | + &repo, |
| 770 | + &[ |
| 771 | + (repo_path("dir/clean"), "clean"), |
| 772 | + (repo_path("dir/edited"), "original"), |
| 773 | + (repo_path("pkg/src/lib"), "lib"), |
| 774 | + (repo_path("top"), "top"), |
| 775 | + ], |
| 776 | + ); |
| 777 | + let commit = commit_with_tree(repo.store(), tree.clone()); |
| 778 | + test_workspace |
| 779 | + .workspace |
| 780 | + .check_out(repo.op_id().clone(), None, &commit) |
| 781 | + .block_on()?; |
| 782 | + // Re-save the tree state so no file is racily clean. |
| 783 | + std::thread::sleep(Duration::from_millis(10)); |
| 784 | + test_workspace.snapshot()?; |
| 785 | + std::fs::write(root1.join("dir/edited"), "edited")?; |
| 786 | + std::fs::write(root1.join("dir/untracked"), "untracked")?; |
| 787 | + std::fs::create_dir_all(root1.join("pkg/node_modules/dep"))?; |
| 788 | + std::fs::write(root1.join("pkg/node_modules/dep/index.js"), "dep")?; |
| 789 | + |
| 790 | + let root2 = test_workspace.root_dir().join("ws2_root"); |
| 791 | + std::fs::create_dir(&root2)?; |
| 792 | + let (mut ws2, repo) = Workspace::init_workspace_with_existing_repo( |
| 793 | + &root2, |
| 794 | + test_workspace.repo_path(), |
| 795 | + &repo, |
| 796 | + &*default_working_copy_factory(), |
| 797 | + WorkspaceNameBuf::from("ws2"), |
| 798 | + ) |
| 799 | + .block_on()?; |
| 800 | + ws2.check_out(repo.op_id().clone(), None, &commit) |
| 801 | + .block_on()?; |
| 802 | + |
| 803 | + assert_eq!(std::fs::read_to_string(root2.join("dir/clean"))?, "clean"); |
| 804 | + assert_eq!( |
| 805 | + std::fs::read_to_string(root2.join("dir/edited"))?, |
| 806 | + "original" |
| 807 | + ); |
| 808 | + assert_eq!(std::fs::read_to_string(root2.join("pkg/src/lib"))?, "lib"); |
| 809 | + assert!(!root2.join("dir/untracked").exists()); |
| 810 | + assert!(!root2.join("pkg/node_modules").exists()); |
| 811 | + let mtime = |path: PathBuf| path.metadata().and_then(|metadata| metadata.modified()); |
| 812 | + let seeded = cfg!(target_vendor = "apple"); |
| 813 | + for path in ["dir/clean", "pkg/src/lib"] { |
| 814 | + assert_eq!(mtime(root2.join(path))? == mtime(root1.join(path))?, seeded); |
| 815 | + } |
| 816 | + let mut locked_ws = ws2.start_working_copy_mutation().block_on()?; |
| 817 | + let (snapshot, _stats) = locked_ws |
| 818 | + .locked_wc() |
| 819 | + .snapshot(&empty_snapshot_options()) |
| 820 | + .block_on()?; |
| 821 | + assert_eq!(snapshot.tree_ids(), tree.tree_ids()); |
| 822 | + Ok(()) |
| 823 | +} |
| 824 | + |
| 825 | +#[test] |
| 826 | fn test_snapshot_file_directory_transition() -> TestResult { |
| 827 | let mut test_workspace = TestWorkspace::init(); |
| 828 | let repo = test_workspace.repo.clone(); |