diff --git a/cli/Cargo.toml b/cli/Cargo.toml index bb129fb41f6..8f658bee066 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -115,6 +115,7 @@ packaging = ["gix-max-performance"] test-fakes = ["jj-lib/testing"] vendored-openssl = ["git2/vendored-openssl", "jj-lib/vendored-openssl"] watchman = ["jj-lib/watchman"] +shell = [] [package.metadata.binstall] # The archive name is jj, not jj-cli. Also, `cargo binstall` gets diff --git a/cli/src/commands/git/clone.rs b/cli/src/commands/git/clone.rs index 743fd5e8447..a05cedafc5d 100644 --- a/cli/src/commands/git/clone.rs +++ b/cli/src/commands/git/clone.rs @@ -226,11 +226,23 @@ fn do_git_clone( ) }) .map_err(|err| match err { - GitFetchError::NoSuchRemote(_) => { - panic!("shouldn't happen as we just created the git remote") + GitFetchError::NoSuchRemote(repo_name) => { + user_error(format!("could not find repository at '{repo_name}'")) } GitFetchError::GitImportError(err) => CommandError::from(err), GitFetchError::InternalGitError(err) => map_git_error(err), + GitFetchError::GitForkError(err) => CommandError::with_message( + crate::command_error::CommandErrorKind::Internal, + "external git process failed", + err, + ), + GitFetchError::ExternalGitError(err) => { + CommandError::new(crate::command_error::CommandErrorKind::Internal, err) + } + GitFetchError::PathConversionError(path) => CommandError::new( + crate::command_error::CommandErrorKind::Internal, + format!("failed to convert path {} to string", path.display()), + ), GitFetchError::InvalidBranchPattern => { unreachable!("we didn't provide any globs") } diff --git a/cli/src/git_util.rs b/cli/src/git_util.rs index 00d97d6288e..702d0ad0311 100644 --- a/cli/src/git_util.rs +++ b/cli/src/git_util.rs @@ -261,25 +261,30 @@ pub fn with_remote_git_callbacks( ) -> T { let mut callbacks = git::RemoteCallbacks::default(); let mut progress_callback = None; - if let Some(mut output) = ui.progress_output() { - let mut progress = Progress::new(Instant::now()); - progress_callback = Some(move |x: &git::Progress| { - _ = progress.update(Instant::now(), x, &mut output); - }); + if !cfg!(feature = "shell") { + if let Some(mut output) = ui.progress_output() { + let mut progress = Progress::new(Instant::now()); + progress_callback = Some(move |x: &git::Progress| { + _ = progress.update(Instant::now(), x, &mut output); + }); + } + callbacks.progress = progress_callback + .as_mut() + .map(|x| x as &mut dyn FnMut(&git::Progress)); + callbacks.sideband_progress = + sideband_progress_callback.map(|x| x as &mut dyn FnMut(&[u8])); + let mut get_ssh_keys = get_ssh_keys; // Coerce to unit fn type + callbacks.get_ssh_keys = Some(&mut get_ssh_keys); + let mut get_pw = + |url: &str, _username: &str| pinentry_get_pw(url).or_else(|| terminal_get_pw(ui, url)); + callbacks.get_password = Some(&mut get_pw); + let mut get_user_pw = + |url: &str| Some((terminal_get_username(ui, url)?, terminal_get_pw(ui, url)?)); + callbacks.get_username_password = Some(&mut get_user_pw); + f(callbacks) + } else { + f(callbacks) } - callbacks.progress = progress_callback - .as_mut() - .map(|x| x as &mut dyn FnMut(&git::Progress)); - callbacks.sideband_progress = sideband_progress_callback.map(|x| x as &mut dyn FnMut(&[u8])); - let mut get_ssh_keys = get_ssh_keys; // Coerce to unit fn type - callbacks.get_ssh_keys = Some(&mut get_ssh_keys); - let mut get_pw = - |url: &str, _username: &str| pinentry_get_pw(url).or_else(|| terminal_get_pw(ui, url)); - callbacks.get_password = Some(&mut get_pw); - let mut get_user_pw = - |url: &str| Some((terminal_get_username(ui, url)?, terminal_get_pw(ui, url)?)); - callbacks.get_username_password = Some(&mut get_user_pw); - f(callbacks) } pub fn print_git_import_stats( diff --git a/cli/tests/test_git_clone.rs b/cli/tests/test_git_clone.rs index d0ebdedca6e..24b3fe3cff9 100644 --- a/cli/tests/test_git_clone.rs +++ b/cli/tests/test_git_clone.rs @@ -96,10 +96,10 @@ fn test_git_clone() { let stdout = test_env.normalize_output(&get_stdout_string(&assert)); let stderr = test_env.normalize_output(&get_stderr_string(&assert)); insta::assert_snapshot!(stdout, @""); - insta::assert_snapshot!(stderr, @r###" + insta::assert_snapshot!(stderr, @r#" Fetching into new repo in "$TEST_ENV/failed" - Error: could not find repository at '$TEST_ENV/bad'; class=Repository (6) - "###); + Error: could not find repository at '$TEST_ENV/bad' + "#); assert!(!test_env.env_root().join("failed").exists()); // Failed clone shouldn't remove the existing destination directory @@ -111,10 +111,10 @@ fn test_git_clone() { let stdout = test_env.normalize_output(&get_stdout_string(&assert)); let stderr = test_env.normalize_output(&get_stderr_string(&assert)); insta::assert_snapshot!(stdout, @""); - insta::assert_snapshot!(stderr, @r###" + insta::assert_snapshot!(stderr, @r#" Fetching into new repo in "$TEST_ENV/failed" - Error: could not find repository at '$TEST_ENV/bad'; class=Repository (6) - "###); + Error: could not find repository at '$TEST_ENV/bad' + "#); assert!(test_env.env_root().join("failed").exists()); assert!(!test_env.env_root().join("failed").join(".jj").exists()); @@ -284,10 +284,10 @@ fn test_git_clone_colocate() { let stdout = test_env.normalize_output(&get_stdout_string(&assert)); let stderr = test_env.normalize_output(&get_stderr_string(&assert)); insta::assert_snapshot!(stdout, @""); - insta::assert_snapshot!(stderr, @r###" + insta::assert_snapshot!(stderr, @r#" Fetching into new repo in "$TEST_ENV/failed" - Error: could not find repository at '$TEST_ENV/bad'; class=Repository (6) - "###); + Error: could not find repository at '$TEST_ENV/bad' + "#); assert!(!test_env.env_root().join("failed").exists()); // Failed clone shouldn't remove the existing destination directory @@ -302,10 +302,10 @@ fn test_git_clone_colocate() { let stdout = test_env.normalize_output(&get_stdout_string(&assert)); let stderr = test_env.normalize_output(&get_stderr_string(&assert)); insta::assert_snapshot!(stdout, @""); - insta::assert_snapshot!(stderr, @r###" + insta::assert_snapshot!(stderr, @r#" Fetching into new repo in "$TEST_ENV/failed" - Error: could not find repository at '$TEST_ENV/bad'; class=Repository (6) - "###); + Error: could not find repository at '$TEST_ENV/bad' + "#); assert!(test_env.env_root().join("failed").exists()); assert!(!test_env.env_root().join("failed").join(".git").exists()); assert!(!test_env.env_root().join("failed").join(".jj").exists()); @@ -586,6 +586,7 @@ fn test_git_clone_trunk_deleted() { "#); } +#[cfg(not(feature = "shell"))] #[test] fn test_git_clone_with_depth() { let test_env = TestEnvironment::default(); @@ -606,6 +607,43 @@ fn test_git_clone_with_depth() { "#); } +#[cfg(feature = "shell")] +#[test] +fn test_git_clone_with_depth() { + let test_env = TestEnvironment::default(); + test_env.add_config("git.auto-local-bookmark = true"); + let git_repo_path = test_env.env_root().join("source"); + let clone_path = test_env.env_root().join("clone"); + let git_repo = git2::Repository::init(git_repo_path).unwrap(); + set_up_non_empty_git_repo(&git_repo); + + // local transport *does* work in normal git + // we check everything works + let (stdout, stderr) = test_env.jj_cmd_ok( + test_env.env_root(), + &["git", "clone", "--depth", "1", "source", "clone"], + ); + insta::assert_snapshot!(stdout, @""); + insta::assert_snapshot!(stderr, @r#" + Fetching into new repo in "$TEST_ENV/clone" + bookmark: main@origin [new] tracked + Setting the revset alias "trunk()" to "main@origin" + Working copy now at: sqpuoqvx cad212e1 (empty) (no description set) + Parent commit : mzyxwzks 9f01a0e0 main | message + Added 1 files, modified 0 files, removed 0 files + "#); + + let (stdout, stderr) = test_env.jj_cmd_ok(&clone_path, &["log"]); + insta::assert_snapshot!(stdout, @r" + @ sqpuoqvx test.user@example.com 2001-02-03 08:05:07 cad212e1 + │ (empty) (no description set) + ◆ mzyxwzks some.one@example.com 1970-01-01 11:00:00 main 9f01a0e0 + │ message + ~ + "); + insta::assert_snapshot!(stderr, @""); +} + #[test] fn test_git_clone_invalid_immutable_heads() { let test_env = TestEnvironment::default(); diff --git a/lib/Cargo.toml b/lib/Cargo.toml index ee21c254d1c..ed45da8278b 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -107,6 +107,7 @@ gix-max-performance = [ vendored-openssl = ["git2/vendored-openssl"] watchman = ["dep:tokio", "dep:watchman_client"] testing = ["git"] +shell = [] [lints] workspace = true diff --git a/lib/src/git.rs b/lib/src/git.rs index e52e9baa912..dfab8530298 100644 --- a/lib/src/git.rs +++ b/lib/src/git.rs @@ -19,10 +19,15 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; use std::default::Default; +use std::ffi::OsStr; use std::fmt; use std::io::Read; use std::num::NonZeroU32; +use std::path::Path; use std::path::PathBuf; +use std::process::Command; +use std::process::Output; +use std::process::Stdio; use std::str; use git2::Oid; @@ -110,6 +115,37 @@ fn to_remote_branch<'a>(parsed_ref: &'a RefName, remote_name: &str) -> Option<&' } } +// there are two options: +// +// a bare git repo, which has a parent named .jj that sits on the workspace root +// or +// a colocated .git dir, which is already on the workspace root +fn work_tree_from_git_dir(git_dir: &Path) -> Result { + if git_dir.file_name() == Some(OsStr::new(".git")) { + git_dir + .parent() + .map(|x| x.to_path_buf()) + .ok_or(format!("git repo had no parent: {}", git_dir.display())) + } else if git_dir.file_name() == Some(OsStr::new("git")) { + let mut it = git_dir.ancestors(); + for path in it.by_ref() { + if path.file_name() == Some(OsStr::new(".jj")) { + break; + } + } + + it.next().map(|x| x.to_path_buf()).ok_or(format!( + "could not find .jj dir in git dir path: {}", + git_dir.display() + )) + } else { + Err(format!( + "git dir is not named `git` nor `.git`: {}", + git_dir.display() + )) + } +} + /// Returns true if the `parsed_ref` won't be imported because its remote name /// is reserved. /// @@ -1078,6 +1114,21 @@ fn is_remote_not_found_err(err: &git2::Error) -> bool { ) } +fn repository_not_found_err(err: &git2::Error) -> Option<&str> { + let mut s = None; + if matches!( + (err.class(), err.code()), + (git2::ErrorClass::Repository, git2::ErrorCode::GenericError) + ) && err.message().starts_with("could not find repository at ") + { + let mut sp = err.message().split('\''); + sp.next(); + s = sp.next(); + } + + s +} + fn is_remote_exists_err(err: &git2::Error) -> bool { matches!( (err.class(), err.code()), @@ -1227,8 +1278,33 @@ pub enum GitFetchError { // TODO: I'm sure there are other errors possible, such as transport-level errors. #[error("Unexpected git error when fetching")] InternalGitError(#[from] git2::Error), + #[error("Failed to fork git process")] + GitForkError(#[from] std::io::Error), + #[error("git process failed: {0}")] + ExternalGitError(String), + #[error("failed to convert path")] + PathConversionError(std::path::PathBuf), +} + +#[cfg(feature = "shell")] +fn fetch_options( + callbacks: RemoteCallbacks<'_>, + depth: Option, +) -> git2::FetchOptions<'_> { + let mut proxy_options = git2::ProxyOptions::new(); + proxy_options.auto(); + + let mut fetch_options = git2::FetchOptions::new(); + fetch_options.proxy_options(proxy_options); + fetch_options.remote_callbacks(callbacks.into_git()); + if let Some(depth) = depth { + fetch_options.depth(depth.get().try_into().unwrap_or(i32::MAX)); + } + + fetch_options } +#[cfg(not(feature = "shell"))] fn fetch_options( callbacks: RemoteCallbacks<'_>, depth: Option, @@ -1257,6 +1333,7 @@ struct GitFetch<'a> { git_settings: &'a GitSettings, fetch_options: git2::FetchOptions<'a>, fetched: Vec, + depth: Option, } impl<'a> GitFetch<'a> { @@ -1265,6 +1342,7 @@ impl<'a> GitFetch<'a> { git_repo: &'a git2::Repository, git_settings: &'a GitSettings, fetch_options: git2::FetchOptions<'a>, + depth: Option, ) -> Self { GitFetch { mut_repo, @@ -1272,14 +1350,36 @@ impl<'a> GitFetch<'a> { git_settings, fetch_options, fetched: vec![], + depth, } } + fn expand_refspecs( + remote_name: &str, + branch_names: &[StringPattern], + ) -> Result, GitFetchError> { + branch_names + .iter() + .map(|pattern| { + pattern + .to_glob() + .filter( + /* This triggered by non-glob `*`s in addition to INVALID_REFSPEC_CHARS + * because `to_glob()` escapes such `*`s as `[*]`. */ + |glob| !glob.contains(INVALID_REFSPEC_CHARS), + ) + .map(|glob| format!("+refs/heads/{glob}:refs/remotes/{remote_name}/{glob}")) + }) + .collect::>() + .ok_or(GitFetchError::InvalidBranchPattern) + } + /// Perform a `git fetch` on the local git repo, updating the /// remote-tracking branches in the git repo. /// /// Keeps track of the {branch_names, remote_name} pair the refs can be /// subsequently imported into the `jj` repo by calling `import_refs()`. + #[cfg(not(feature = "shell"))] fn fetch( &mut self, branch_names: &[StringPattern], @@ -1294,27 +1394,22 @@ impl<'a> GitFetch<'a> { })?; // At this point, we are only updating Git's remote tracking branches, not the // local branches. - let refspecs: Vec<_> = branch_names - .iter() - .map(|pattern| { - pattern - .to_glob() - .filter( - /* This triggered by non-glob `*`s in addition to INVALID_REFSPEC_CHARS - * because `to_glob()` escapes such `*`s as `[*]`. */ - |glob| !glob.contains(INVALID_REFSPEC_CHARS), - ) - .map(|glob| format!("+refs/heads/{glob}:refs/remotes/{remote_name}/{glob}")) - }) - .collect::>() - .ok_or(GitFetchError::InvalidBranchPattern)?; + let refspecs: Vec<_> = Self::expand_refspecs(remote_name, branch_names)?; if refspecs.is_empty() { // Don't fall back to the base refspecs. return Ok(None); } tracing::debug!("remote.download"); - remote.download(&refspecs, Some(&mut self.fetch_options))?; + remote + .download(&refspecs, Some(&mut self.fetch_options)) + .map_err(|err| { + if let Some(s) = repository_not_found_err(&err) { + GitFetchError::NoSuchRemote(s.to_string()) + } else { + GitFetchError::InternalGitError(err) + } + })?; tracing::debug!("remote.prune"); remote.prune(None)?; tracing::debug!("remote.update_tips"); @@ -1348,6 +1443,68 @@ impl<'a> GitFetch<'a> { Ok(default_branch) } + /// Perform a `git fetch` on the local git repo, updating the + /// remote-tracking branches in the git repo. + /// + /// Keeps track of the {branch_names, remote_name} pair the refs can be + /// subsequently imported into the `jj` repo by calling `import_refs()`. + #[cfg(feature = "shell")] + fn fetch( + &mut self, + branch_names: &[StringPattern], + remote_name: &str, + ) -> Result, GitFetchError> { + // At this point, we are only updating Git's remote tracking branches, not the + // local branches. + let refspecs: Vec<_> = Self::expand_refspecs(remote_name, branch_names)?; + if refspecs.is_empty() { + // Don't fall back to the base refspecs. + return Ok(None); + } + + let git_dir = self.git_repo.path().to_str().ok_or_else(|| { + GitFetchError::PathConversionError(self.git_repo.path().to_path_buf()) + })?; + + // TODO: there is surely a better way to get this? it seems like it's stored in the + // workspace, but that seems inaccessible here + let work_tree = work_tree_from_git_dir(self.git_repo.path()) + .map(|p| { + p.to_str() + .map(|x| x.to_string()) + .ok_or(GitFetchError::PathConversionError(p)) + }) + .map_err(|s| GitFetchError::ExternalGitError(s))??; + + let mut prunes = Vec::new(); + for refspec in refspecs { + fork_git_fetch( + git_dir, + &work_tree, + remote_name, + self.depth, + &refspec, + &mut prunes, + )?; + } + for branch in prunes { + fork_git_branch_prune(git_dir, &work_tree, remote_name, &branch)?; + } + + self.fetched.push(FetchedBranches { + branches: branch_names.to_vec(), + remote: remote_name.to_string(), + }); + + // TODO: We could make it optional to get the default branch since we only care + // about it on clone. + + let default_branch = fork_git_remote_show(git_dir, &work_tree, remote_name)?; + tracing::debug!(default_branch = default_branch); + + Ok(default_branch) + } + /// Import the previously fetched remote-tracking branches into the jj repo /// and update jj's local branches. We also import local tags since remote /// tags should have been merged by Git. @@ -1404,11 +1561,13 @@ pub fn fetch( git_settings: &GitSettings, depth: Option, ) -> Result { + // git fetch remote_name branch_names let mut git_fetch = GitFetch::new( mut_repo, git_repo, git_settings, fetch_options(callbacks, depth), + depth, ); let default_branch = git_fetch.fetch(branch_names, remote_name)?; let import_stats = git_fetch.import_refs()?; @@ -1436,6 +1595,40 @@ pub enum GitPushError { // and errors caused by the remote rejecting the push. #[error("Unexpected git error when pushing")] InternalGitError(#[from] git2::Error), + #[error("Failed to fork git process")] + GitForkError(#[from] IoError), + #[error("git process failed: {0}")] + ExternalGitError(String), + #[error("failed to convert path")] + PathConversionError(std::path::PathBuf), +} +impl From for GitPushError { + fn from(value: std::io::Error) -> Self { + GitPushError::GitForkError(IoError(value)) + } +} + +/// New-type for comparable io errors +pub struct IoError(std::io::Error); +impl PartialEq for IoError { + fn eq(&self, other: &Self) -> bool { + self.0.kind().eq(&other.0.kind()) + } +} +impl std::error::Error for IoError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } +} +impl std::fmt::Display for IoError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} +impl std::fmt::Debug for IoError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } } #[derive(Clone, Debug)] @@ -1527,6 +1720,58 @@ pub fn push_updates( ) } +fn check_allow_push( + repo: &dyn Repo, + dst_refname: &str, + actual_remote_location: Option<&CommitId>, + expected_remote_location: Option<&CommitId>, + local_location: Option<&CommitId>, + failed_ref_matches: &mut Vec, +) -> bool { + match allow_push( + repo.index(), + actual_remote_location, + expected_remote_location, + local_location, + ) { + Ok(PushAllowReason::NormalMatch) => {} + Ok(PushAllowReason::UnexpectedNoop) => { + tracing::info!( + "The push of {dst_refname} is unexpectedly a no-op, the remote branch \ + is already at {actual_remote_location:?}. We expected it to be at \ + {expected_remote_location:?}. We don't consider this an error.", + ); + } + Ok(PushAllowReason::ExceptionalFastforward) => { + // TODO(ilyagr): We could consider printing a user-facing message at + // this point. + tracing::info!( + "We allow the push of {dst_refname} to {local_location:?}, even \ + though it is unexpectedly at {actual_remote_location:?} on the \ + server rather than the expected {expected_remote_location:?}. The \ + desired location is a descendant of the actual location, and the \ + actual location is a descendant of the expected location.", + ); + } + Err(()) => { + // While we show debug info in the message with `--debug`, + // there's probably no need to show the detailed commit + // locations to the user normally. They should do a `jj git + // fetch`, and the resulting branch conflicts should contain + // all the information they need. + tracing::info!( + "Cannot push {dst_refname} to {local_location:?}; it is at \ + unexpectedly at {actual_remote_location:?} on the server as opposed \ + to the expected {expected_remote_location:?}", + ); + failed_ref_matches.push(dst_refname.to_string()); + return false; + } + } + true +} + +#[cfg(not(feature = "shell"))] fn push_refs( repo: &dyn Repo, git_repo: &git2::Repository, @@ -1568,47 +1813,14 @@ fn push_refs( |oid: git2::Oid| (!oid.is_zero()).then(|| CommitId::from_bytes(oid.as_bytes())); let actual_remote_location = oid_to_maybe_commitid(update.src()); let local_location = oid_to_maybe_commitid(update.dst()); - - match allow_push( - repo.index(), + check_allow_push( + repo, + dst_refname, actual_remote_location.as_ref(), expected_remote_location, local_location.as_ref(), - ) { - Ok(PushAllowReason::NormalMatch) => {} - Ok(PushAllowReason::UnexpectedNoop) => { - tracing::info!( - "The push of {dst_refname} is unexpectedly a no-op, the remote branch \ - is already at {actual_remote_location:?}. We expected it to be at \ - {expected_remote_location:?}. We don't consider this an error.", - ); - } - Ok(PushAllowReason::ExceptionalFastforward) => { - // TODO(ilyagr): We could consider printing a user-facing message at - // this point. - tracing::info!( - "We allow the push of {dst_refname} to {local_location:?}, even \ - though it is unexpectedly at {actual_remote_location:?} on the \ - server rather than the expected {expected_remote_location:?}. The \ - desired location is a descendant of the actual location, and the \ - actual location is a descendant of the expected location.", - ); - } - Err(()) => { - // While we show debug info in the message with `--debug`, - // there's probably no need to show the detailed commit - // locations to the user normally. They should do a `jj git - // fetch`, and the resulting branch conflicts should contain - // all the information they need. - tracing::info!( - "Cannot push {dst_refname} to {local_location:?}; it is at \ - unexpectedly at {actual_remote_location:?} on the server as opposed \ - to the expected {expected_remote_location:?}", - ); - - failed_push_negotiations.push(dst_refname.to_string()); - } - } + &mut failed_push_negotiations, + ); } if failed_push_negotiations.is_empty() { Ok(()) @@ -1654,6 +1866,94 @@ fn push_refs( } } +#[cfg(feature = "shell")] +fn push_refs( + repo: &dyn Repo, + git_repo: &git2::Repository, + remote_name: &str, + qualified_remote_refs_expected_locations: &HashMap<&str, Option<&CommitId>>, + refspecs: &[String], + _callbacks: RemoteCallbacks<'_>, +) -> Result<(), GitPushError> { + if remote_name == REMOTE_NAME_FOR_LOCAL_GIT_REPO { + return Err(GitPushError::RemoteReservedForLocalGitRepo); + } + let git_dir = git_repo + .path() + .to_str() + .ok_or_else(|| GitPushError::PathConversionError(git_repo.path().to_path_buf()))?; + let work_tree = work_tree_from_git_dir(git_repo.path()) + .map(|p| { + p.to_str() + .map(|x| x.to_string()) + .ok_or(GitPushError::PathConversionError(p)) + }) + .map_err(|s| GitPushError::ExternalGitError(s))??; + + let mut remaining_remote_refs: HashSet<_> = qualified_remote_refs_expected_locations + .keys() + .copied() + .collect(); + let mut failed_ref_matches = vec![]; + for remote_refspec in refspecs { + let remote_ref_opt = remote_refspec.split(":").last(); + let dest_local_location = remote_refspec + .split(":") + .next() + .map(|x| { + if x.starts_with("+") { + let mut chars = x.chars(); + chars.next(); + chars.as_str() + } else { + x + } + }) + .map(|x| if x.trim().is_empty() { None } else { Some(x) }) + .flatten() + .map(|x| CommitId::try_from_hex(x).expect("non hex")); + if let Some(remote_ref) = remote_ref_opt { + if let Some(expected_remote_location) = + qualified_remote_refs_expected_locations.get(remote_ref) + { + let actual_remote_location = + fork_git_ls_remote(git_dir, &work_tree, remote_name, remote_ref)?; + if !check_allow_push( + repo, + remote_ref, + actual_remote_location.as_ref(), + *expected_remote_location, + dest_local_location.as_ref(), + &mut failed_ref_matches, + ) { + continue; + } + } + } + fork_git_push(git_dir, &work_tree, remote_name, remote_refspec)?; + if let Some(remote_ref) = remote_ref_opt { + remaining_remote_refs.remove(remote_ref); + } + } + + if !failed_ref_matches.is_empty() { + failed_ref_matches.sort(); + Err(GitPushError::RefInUnexpectedLocation(failed_ref_matches)) + } else { + if remaining_remote_refs.is_empty() { + Ok(()) + } else { + Err(GitPushError::RefUpdateRejected( + remaining_remote_refs + .iter() + .sorted() + .map(|name| name.to_string()) + .collect(), + )) + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] enum PushAllowReason { NormalMatch, @@ -1905,3 +2205,500 @@ pub fn parse_gitmodules( .collect(); Ok(ret) } + +fn convert_git_fetch_output_to_fetch_result( + output: Output, + prunes: &mut Vec, +) -> Result { + // None means signal termination: + // 128 is the base for the signal exit codes (128 + signo) + let code = output.status.code(); + if code == Some(0) { + return Ok(output); + } + + let lossy_err = String::from_utf8_lossy(&output.stderr).to_string(); + let stderr = String::from_utf8(output.stderr.clone()).map_err(|e| { + GitFetchError::ExternalGitError(format!( + "external git program failed with non-utf8 output: {e:?}\n--- stderr ---\n{lossy_err}", + )) + })?; + + // There are some git errors we want to parse out + + // GitFetchError::NoSuchRemote + // + // To say this, git prints out a lot of things, but the first line is of the form: + // `fatal: '' does not appear to be a git repository` + let git_err = { + let mut git_err = None; + if let Some(first_line) = stderr.lines().next() { + if first_line.starts_with("fatal: '") + && first_line.ends_with("' does not appear to be a git repository") + { + let mut split = first_line.split('\''); + split.next(); // ignore prefix + let branch_name = split.next(); + if let Some(bname) = branch_name { + git_err = Some(Err(GitFetchError::NoSuchRemote(bname.to_string()))); + } + } + } + + git_err + }; + if let Some(e) = git_err { + return e; + } + + // even though --prune is specified, if a particular refspec is asked for but not present + // in the remote, git will error out. + // we ignore it explicitely + // + // The first line is of the form: + // `fatal: couldn't find remote ref refs/heads/` + if let Some(first_line) = stderr.lines().next() { + if first_line.starts_with("fatal: couldn't find remote ref refs/heads/") { + let mut sp = first_line.split("refs/heads/"); + sp.next(); + prunes.push(sp.next().unwrap().to_string()); + return Ok(output); + } + } + + Err(GitFetchError::ExternalGitError(format!( + "external git program failed:\n{stderr}", + ))) +} + +fn fork_git_fetch( + git_dir: &str, + work_tree: &str, + remote_name: &str, + depth: Option, + refspec: &str, + prunes: &mut Vec, +) -> Result<(), GitFetchError> { + let depth_arg = depth.map(|x| format!("--depth={x}")); + tracing::debug!( + "shelling out to `git --git-dir={} --work-tree={} fetch --prune{} {} {}", + git_dir, + work_tree, + depth_arg + .as_ref() + .map(|x| format!(" {x}")) + .unwrap_or("".to_string()), + remote_name, + refspec + ); + + let remote_git = { + let mut cmd = Command::new("git"); + + cmd.arg(format!("--git-dir={git_dir}")) + .arg(format!("--work-tree={work_tree}")) + .arg("fetch") + .arg("--prune"); + + if let Some(depth) = depth_arg { + cmd.arg(depth); + } + + cmd.arg(remote_name) + .arg(refspec) + .stdin(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + }?; + + let output = remote_git.wait_with_output()?; + let _ = convert_git_fetch_output_to_fetch_result(output, prunes)?; + + Ok(()) +} + +fn convert_git_remote_show_output_to_fetch_result( + output: Output, + prunes: &mut Vec, +) -> Result { + // None means signal termination: + // 128 is the base for the signal exit codes (128 + signo) + let code = output.status.code(); + if code == Some(0) { + return Ok(output); + } + + let lossy_err = String::from_utf8_lossy(&output.stderr).to_string(); + let stderr = String::from_utf8(output.stderr.clone()).map_err(|e| { + GitFetchError::ExternalGitError(format!( + "external git program failed with non-utf8 output: {e:?}\n--- stderr ---\n{lossy_err}", + )) + })?; + + // There are some git errors we want to parse out + + // GitFetchError::NoSuchRemote + // + // To say this, git prints out a lot of things, but the first line is of the form: + // `fatal: '' does not appear to be a git repository` + let git_err = { + let mut git_err = None; + if let Some(first_line) = stderr.lines().next() { + if first_line.starts_with("fatal: '") + && first_line.ends_with("' does not appear to be a git repository") + { + let mut split = first_line.split('\''); + split.next(); // ignore prefix + let branch_name = split.next(); + if let Some(bname) = branch_name { + git_err = Some(Err(GitFetchError::NoSuchRemote(bname.to_string()))); + } + } + } + + git_err + }; + if let Some(e) = git_err { + return e; + } + + // even though --prune is specified, if a particular refspec is asked for but not present + // in the remote, git will error out. + // we ignore it explicitely + // + // The first line is of the form: + // `fatal: couldn't find remote ref refs/heads/` + if let Some(first_line) = stderr.lines().next() { + if first_line.starts_with("fatal: couldn't find remote ref refs/heads/") { + let mut sp = first_line.split("refs/heads/"); + sp.next(); + prunes.push(sp.next().unwrap().to_string()); + return Ok(output); + } + } + + Err(GitFetchError::ExternalGitError(format!( + "external git program failed:\n{stderr}", + ))) +} + +// How we retrieve the remote's default branch: +// +// `git remote show ` +// +// dumps a lot of information about the remote, with a line such as: +// ` HEAD branch: ` +fn fork_git_remote_show( + git_dir: &str, + work_tree: &str, + remote_name: &str, +) -> Result, GitFetchError> { + tracing::debug!("shelling out to `git --git-dir={git_dir} --work-tree={work_tree} remote show {remote_name}`"); + + let remote_git = Command::new("git") + .arg(format!("--git-dir={git_dir}")) + .arg(format!("--work-tree={work_tree}")) + .arg("remote") + .arg("show") + .arg(remote_name) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + let output = remote_git.wait_with_output()?; + let output = convert_git_remote_show_output_to_fetch_result(output, &mut vec![])?; + + // find the HEAD branch line in the output + let branch_name = String::from_utf8(output.stdout) + .map_err(|e| { + GitFetchError::ExternalGitError(format!("git remote output is not utf-8: {e:?}")) + })? + .lines() + .map(|x| x.trim()) + .find(|x| x.starts_with("HEAD branch:")) + .and_then(|x| x.split(" ").last().map(|y| y.trim().to_string())); + + // git will ouptut (unknown) if there is no default branch. we want it to be a none value + if let Some(x) = branch_name.as_deref() { + if x == "(unknown)" { + return Ok(None); + } + } + Ok(branch_name) +} + +fn convert_git_branch_prune_output_to_fetch_result( + output: Output, +) -> Result { + // None means signal termination: + // 128 is the base for the signal exit codes (128 + signo) + let code = output.status.code(); + if code == Some(0) { + return Ok(output); + } + + let lossy_err = String::from_utf8_lossy(&output.stderr).to_string(); + let stderr = String::from_utf8(output.stderr.clone()).map_err(|e| { + GitFetchError::ExternalGitError(format!( + "external git program failed with non-utf8 output: {e:?}\n--- stderr ---\n{lossy_err}", + )) + })?; + + // There are some git errors we want to parse out + + // if a branch is asked for but is not present, jj will detect it post-hoc + // so, we want to ignore these particular errors with git + // + // The first line is of the form: + // error: remote-tracking branch '' not found + if let Some(first_line) = stderr.lines().next() { + if first_line.starts_with("error: remote-tracking branch '") + && first_line.ends_with("' not found") + { + return Ok(output); + } + } + + Err(GitFetchError::ExternalGitError(format!( + "external git program failed:\n{stderr}", + ))) +} + +fn fork_git_branch_prune( + git_dir: &str, + work_tree: &str, + remote_name: &str, + branch_name: &str, +) -> Result<(), GitFetchError> { + tracing::debug!("shelling out to `git --git-dir={git_dir} --work-tree={work_tree} branch --remotes --delete {remote_name}/{branch_name}`"); + let branch_git = Command::new("git") + .arg(format!("--git-dir={git_dir}")) + .arg(format!("--work-tree={work_tree}")) + .arg("branch") + .arg("--remotes") + .arg("--delete") + .arg(format!("{remote_name}/{branch_name}")) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + + let output = branch_git.wait_with_output()?; + convert_git_branch_prune_output_to_fetch_result(output)?; + + Ok(()) +} + +fn convert_git_push_output_to_push_result(output: Output) -> Result { + // None means signal termination: + // 128 is the base for the signal exit codes (128 + signo) + let code = output.status.code(); + if code == Some(0) { + return Ok(output); + } + + let lossy_err = String::from_utf8_lossy(&output.stderr).to_string(); + let stderr = String::from_utf8(output.stderr.clone()).map_err(|e| { + GitPushError::ExternalGitError(format!( + "external git program failed with non-utf8 output: {e:?}\n--- stderr ---\n{lossy_err}", + )) + })?; + + // GitPushError::NoSuchRemote + // + // To say this, git prints out a lot of things, but the first line is of the form: + // `fatal: '' does not appear to be a git repository` + let git_err = { + let mut git_err = None; + if let Some(first_line) = stderr.lines().next() { + if first_line.starts_with("fatal: '") + && first_line.ends_with("' does not appear to be a git repository") + { + let mut split = first_line.split('\''); + split.next(); // ignore prefix + let branch_name = split.next(); + if let Some(bname) = branch_name { + git_err = Some(Err(GitPushError::NoSuchRemote(bname.to_string()))); + } + } + } + + git_err + }; + if let Some(e) = git_err { + return e; + } + + // GitPushError::NoSuchRemote + // + // If the remote repo is specified with the URL directly it gives a slightly different error + // `fatal: '' does not appear to be a git repository` + let git_err = { + let mut git_err = None; + if let Some(first_line) = stderr.lines().next() { + if first_line.starts_with("fatal: unable to access '") + && first_line.ends_with("': Could not resolve host: invalid-remote") + { + let mut split = first_line.split('\''); + split.next(); // ignore prefix + let branch_name = split.next(); + if let Some(bname) = branch_name { + git_err = Some(Err(GitPushError::NoSuchRemote(bname.to_string()))); + } + } + } + + git_err + }; + if let Some(e) = git_err { + return e; + } + + Err(GitPushError::ExternalGitError(format!( + "external git program failed:\n{stderr}" + ))) +} + +fn fork_git_push( + git_dir: &str, + work_tree: &str, + remote_name: &str, + refspec: &str, +) -> Result<(), GitPushError> { + tracing::debug!( + "shelling out to `git --git-dir={} --work-tree={} push {} {}", + git_dir, + work_tree, + remote_name, + refspec + ); + + let remote_git = Command::new("git") + .arg(format!("--git-dir={git_dir}")) + .arg(format!("--work-tree={work_tree}")) + .arg("push") + .arg(remote_name) + .arg(refspec) + .stdin(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + + let output = remote_git.wait_with_output()?; + let _ = convert_git_push_output_to_push_result(output)?; + + Ok(()) +} + +fn convert_git_ls_remote_to_push_result(output: Output) -> Result { + // None means signal termination: + // 128 is the base for the signal exit codes (128 + signo) + let code = output.status.code(); + if code == Some(0) { + return Ok(output); + } + + let lossy_err = String::from_utf8_lossy(&output.stderr).to_string(); + let stderr = String::from_utf8(output.stderr.clone()).map_err(|e| { + GitPushError::ExternalGitError(format!( + "external git program failed with non-utf8 output: {e:?}\n--- stderr ---\n{lossy_err}", + )) + })?; + + // GitPushError::NoSuchRemote + // + // To say this, git prints out a lot of things, but the first line is of the form: + // `fatal: '' does not appear to be a git repository` + let git_err = { + let mut git_err = None; + if let Some(first_line) = stderr.lines().next() { + if first_line.starts_with("fatal: '") + && first_line.ends_with("' does not appear to be a git repository") + { + let mut split = first_line.split('\''); + split.next(); // ignore prefix + let branch_name = split.next(); + if let Some(bname) = branch_name { + git_err = Some(Err(GitPushError::NoSuchRemote(bname.to_string()))); + } + } + } + + git_err + }; + if let Some(e) = git_err { + return e; + } + + // GitPushError::NoSuchRemote + // + // If the remote repo is specified with the URL directly it gives a slightly different error + // `fatal: '' does not appear to be a git repository` + let git_err = { + let mut git_err = None; + if let Some(first_line) = stderr.lines().next() { + if first_line.starts_with("fatal: unable to access '") + && first_line.ends_with("': Could not resolve host: invalid-remote") + { + let mut split = first_line.split('\''); + split.next(); // ignore prefix + let branch_name = split.next(); + if let Some(bname) = branch_name { + git_err = Some(Err(GitPushError::NoSuchRemote(bname.to_string()))); + } + } + } + + git_err + }; + if let Some(e) = git_err { + return e; + } + + Err(GitPushError::ExternalGitError(format!( + "external git program failed:\n{stderr}" + ))) +} + +fn fork_git_ls_remote( + git_dir: &str, + work_tree: &str, + remote_name: &str, + remote_ref: &str, +) -> Result, GitPushError> { + tracing::debug!( + "shelling out to `git --git-dir={} --work-tree={} ls-remote {} {}", + git_dir, + work_tree, + remote_name, + remote_ref + ); + + let remote_git = Command::new("git") + .arg(format!("--git-dir={git_dir}")) + .arg(format!("--work-tree={work_tree}")) + .arg("ls-remote") + .arg(remote_name) + .arg(remote_ref) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + let output = remote_git.wait_with_output()?; + let output = convert_git_ls_remote_to_push_result(output)?; + + let hex_str = String::from_utf8(output.stdout).map_err(|e| { + GitPushError::ExternalGitError(format!("git rev-parse gave non-utf8 output: {e:?}")) + })?; + + if let Some(commit_str) = hex_str.split_whitespace().next() { + let commit_id = CommitId::try_from_hex(commit_str).map_err(|e| { + GitPushError::ExternalGitError(format!("git ls-remote gave non hex output: {e:?}")) + })?; + + return Ok(Some(commit_id)); + } + + Ok(None) +}