git: spawn a separate git process for network operations

.github/workflows/build.yml

@@ -86,9 +86,20 @@ jobs:
         tool: nextest
     - name: Build
       run: cargo build --workspace --all-targets --verbose ${{ matrix.cargo_flags }}
-    - name: Test
+    - name: Test [non-windows]
+      if: ${{ matrix.os != 'windows-latest' }}
+      run: |
+        cargo nextest run --workspace --all-targets --verbose ${{ matrix.cargo_flags }}
+      env:
+        RUST_BACKTRACE: 1
+        CARGO_TERM_COLOR: always
+    - name: Test [windows]
+      if: ${{ matrix.os == 'windows-latest' }}
       run: cargo nextest run --workspace --all-targets --verbose ${{ matrix.cargo_flags }}
       env:
+        # tests clear the PATH variable (but is only felt on windows),
+        # so we need to set the git executable
+        TEST_GIT_EXECUTABLE_PATH: 'C:\Program Files\Git\bin\git.exe'
         RUST_BACKTRACE: 1
         CARGO_TERM_COLOR: always
 

CHANGELOG.md

@@ -68,6 +68,11 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### New features
 
+* `jj git {push,clone,fetch}` can now spawn an external `git` subprocess, via
+   the `git.subprocess = true` config knob. This provides an alternative that,
+   when turned on, fixes SSH bugs when interacting with Git remotes due to
+   `libgit2`s limitations [#4979](https://github.com/jj-vcs/jj/issues/4979).
+
 * `jj describe` now accepts `--edit`.
 
 * `jj evolog` and `jj op log` now accept `--reversed`.

cli/src/commands/git/clone.rs

@@ -204,8 +204,7 @@ fn fetch_new_remote(
     )?;
     let git_settings = workspace_command.settings().git_settings()?;
     let mut fetch_tx = workspace_command.start_transaction();
-
-    let stats = with_remote_git_callbacks(ui, None, |cb| {
+    let stats = with_remote_git_callbacks(ui, None, &git_settings, |cb| {
         git::fetch(
             fetch_tx.repo_mut(),
             &git_repo,
@@ -222,6 +221,7 @@ fn fetch_new_remote(
         }
         GitFetchError::GitImportError(err) => CommandError::from(err),
         GitFetchError::InternalGitError(err) => map_git_error(err),
+        GitFetchError::Subprocess(err) => user_error(err),
         GitFetchError::InvalidBranchPattern => {
             unreachable!("we didn't provide any globs")
         }

cli/src/commands/git/push.rs

@@ -374,9 +374,23 @@ pub fn cmd_git_push(
     let mut sideband_progress_callback = |progress_message: &[u8]| {
         _ = writer.write(ui, progress_message);
     };
-    with_remote_git_callbacks(ui, Some(&mut sideband_progress_callback), |cb| {
-        git::push_branches(tx.repo_mut(), &git_repo, &remote, &targets, cb)
-    })
+
+    let git_settings = tx.settings().git_settings()?;
+    with_remote_git_callbacks(
+        ui,
+        Some(&mut sideband_progress_callback),
+        &git_settings,
+        |cb| {
+            git::push_branches(
+                tx.repo_mut(),
+                &git_repo,
+                &git_settings,
+                &remote,
+                &targets,
+                cb,
+            )
+        },
+    )
     .map_err(|err| match err {
         GitPushError::InternalGitError(err) => map_git_error(err),
         GitPushError::RefInUnexpectedLocation(refs) => user_error_with_hint(

cli/src/config-schema.json

@@ -380,6 +380,16 @@
                     "type": "boolean",
                     "description": "Whether jj should sign commits before pushing",
                     "default": "false"
+                },
+                "subprocess": {
+                    "type": "boolean",
+                    "description": "Whether jj spawns a git subprocess for network operations (push/fetch/clone)",
+                    "default": false
+                },
+                "executable-path": {
+                    "type": "string",
+                    "description": "Path to the git executable",
+                    "default": "git"
                 }
             }
         },

cli/src/git_util.rs

@@ -41,6 +41,7 @@ use jj_lib::op_store::RefTarget;
 use jj_lib::op_store::RemoteRef;
 use jj_lib::repo::ReadonlyRepo;
 use jj_lib::repo::Repo;
+use jj_lib::settings::GitSettings;
 use jj_lib::store::Store;
 use jj_lib::str_util::StringPattern;
 use jj_lib::workspace::Workspace;
@@ -286,29 +287,40 @@ type SidebandProgressCallback<'a> = &'a mut dyn FnMut(&[u8]);
 pub fn with_remote_git_callbacks<T>(
     ui: &Ui,
     sideband_progress_callback: Option<SidebandProgressCallback<'_>>,
+    git_settings: &GitSettings,
     f: impl FnOnce(git::RemoteCallbacks<'_>) -> T,
 ) -> T {
-    let mut callbacks = git::RemoteCallbacks::default();
-    let mut progress_callback = None;
-    if let Some(mut output) = ui.progress_output() {
-        let mut progress = Progress::new(Instant::now());
-        progress_callback = Some(move |x: &git::Progress| {
-            _ = progress.update(Instant::now(), x, &mut output);
-        });
+    if git_settings.subprocess {
+        // TODO: with git2, we are able to display progress from the data that is given
+        // With the git processes themselves, this is significantly harder, as it
+        // requires parsing the output directly
+        //
+        // In any case, this would be the place to add that funcionalty
+        f(git::RemoteCallbacks::default())
+    } else {
+        let mut callbacks = git::RemoteCallbacks::default();
+        let mut progress_callback = None;
+        if let Some(mut output) = ui.progress_output() {
+            let mut progress = Progress::new(Instant::now());
+            progress_callback = Some(move |x: &git::Progress| {
+                _ = progress.update(Instant::now(), x, &mut output);
+            });
+        }
+        callbacks.progress = progress_callback
+            .as_mut()
+            .map(|x| x as &mut dyn FnMut(&git::Progress));
+        callbacks.sideband_progress =
+            sideband_progress_callback.map(|x| x as &mut dyn FnMut(&[u8]));
+        let mut get_ssh_keys = get_ssh_keys; // Coerce to unit fn type
+        callbacks.get_ssh_keys = Some(&mut get_ssh_keys);
+        let mut get_pw =
+            |url: &str, _username: &str| pinentry_get_pw(url).or_else(|| terminal_get_pw(ui, url));
+        callbacks.get_password = Some(&mut get_pw);
+        let mut get_user_pw =
+            |url: &str| Some((terminal_get_username(ui, url)?, terminal_get_pw(ui, url)?));
+        callbacks.get_username_password = Some(&mut get_user_pw);
+        f(callbacks)
     }
-    callbacks.progress = progress_callback
-        .as_mut()
-        .map(|x| x as &mut dyn FnMut(&git::Progress));
-    callbacks.sideband_progress = sideband_progress_callback.map(|x| x as &mut dyn FnMut(&[u8]));
-    let mut get_ssh_keys = get_ssh_keys; // Coerce to unit fn type
-    callbacks.get_ssh_keys = Some(&mut get_ssh_keys);
-    let mut get_pw =
-        |url: &str, _username: &str| pinentry_get_pw(url).or_else(|| terminal_get_pw(ui, url));
-    callbacks.get_password = Some(&mut get_pw);
-    let mut get_user_pw =
-        |url: &str| Some((terminal_get_username(ui, url)?, terminal_get_pw(ui, url)?));
-    callbacks.get_username_password = Some(&mut get_user_pw);
-    f(callbacks)
 }
 
 pub fn print_git_import_stats(
@@ -639,7 +651,7 @@ pub fn git_fetch(
     let git_settings = tx.settings().git_settings()?;
 
     for remote in remotes {
-        let stats = with_remote_git_callbacks(ui, None, |cb| {
+        let stats = with_remote_git_callbacks(ui, None, &git_settings, |cb| {
             git::fetch(
                 tx.repo_mut(),
                 git_repo,

cli/tests/common/mod.rs

@@ -80,6 +80,7 @@ impl Default for TestEnvironment {
 'format_time_range(time_range)' = 'time_range.start() ++ " - " ++ time_range.end()'
         "#,
         );
+
         env
     }
 }
@@ -280,6 +281,18 @@ impl TestEnvironment {
         self.env_vars.insert(key.into(), val.into());
     }
 
+    pub fn set_up_git_subprocessing(&self) {
+        self.add_config("git.subprocess = true");
+
+        // add a git path from env: this is only used if git.subprocess = true
+        if let Ok(git_executable_path) = std::env::var("TEST_GIT_EXECUTABLE_PATH") {
+            self.add_config(format!(
+                "git.executable-path = {}",
+                to_toml_value(git_executable_path)
+            ));
+        }
+    }
+
     pub fn current_operation_id(&self, repo_path: &Path) -> String {
         let id_and_newline =
             self.jj_cmd_success(repo_path, &["debug", "operation", "--display=id"]);