Skip to content

Fix GitDagBundle submodule clone not using SSH settings from connection #64879

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
shunsuke-sugita wants to merge 26 commits into
base: main
Choose a base branch
from
+54 −6
Draft

Fix GitDagBundle submodule clone not using SSH settings from connection #64879

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
4987c26
Fix Git bundle submodule fetch not using SSH settings from connection
shunsuke-sugita Apr 8, 2026
46f66a4
Merge branch 'main' into provider-git-fix-fetch-submodul
shunsuke-sugita Apr 9, 2026
fd68b54
Rename cm to ssh_env_cm in _fetch_bare_repo and _fetch_submodules.
shunsuke-sugita Apr 12, 2026
7e67b4c
Test GitDagBundle._fetch_submodules applies GIT_SSH_COMMAND via custo…
shunsuke-sugita Apr 12, 2026
ffb5e5a
Merge branch 'main' into provider-git-fix-fetch-submodul
shunsuke-sugita Apr 12, 2026
799e084
Merge branch 'main' into provider-git-fix-fetch-submodul
shunsuke-sugita Apr 15, 2026
21098f9
Merge branch 'main' into provider-git-fix-fetch-submodul
shunsuke-sugita Apr 17, 2026
6346cb2
Use autospec for Git/Repo mocks in submodule fetch unit tests
shunsuke-sugita Apr 17, 2026
2dcfd1d
Drop redundant mock_calls assertions in submodule fetch test
shunsuke-sugita Apr 20, 2026
fe31907
GitDagBundle: submodule custom_environment uses full hook.env
shunsuke-sugita Apr 20, 2026
62d9068
Merge branch 'main' into provider-git-fix-fetch-submodul
shunsuke-sugita Apr 20, 2026
6d27243
Rename submodule test for empty hook.env (custom_environment skip)
shunsuke-sugita Apr 20, 2026
4f0d7fc
Update providers/git/tests/unit/git/bundles/test_git.py
shunsuke-sugita Apr 20, 2026
9b6ca63
rm airflow
shunsuke-sugita Apr 20, 2026
76fd8ca
Add missing blank line between submodule fetch tests
shunsuke-sugita Apr 20, 2026
51f14ca
Fix submodule test call-order assertion (use mock_git.mock_calls, not…
shunsuke-sugita Apr 20, 2026
fa3f022
fix test
shunsuke-sugita Apr 22, 2026
3ccbaf5
Merge branch 'main' into provider-git-fix-fetch-submodul
shunsuke-sugita Apr 22, 2026
b8084ed
Fix git bundle and Celery executor unit tests failing in provider CI
shunsuke-sugita Apr 22, 2026
33def75
Fix git bundle and Celery executor unit tests for provider CI
shunsuke-sugita Apr 22, 2026
ce418f4
Merge branch 'main' into provider-git-fix-fetch-submodul
shunsuke-sugita Apr 22, 2026
76cfd3a
Revert celery executor test changes (keep Git bundle PR scoped)
shunsuke-sugita Apr 22, 2026
996de12
Merge branch 'main' into provider-git-fix-fetch-submodul
shunsuke-sugita Apr 22, 2026
cb599d8
Merge branch 'main' into provider-git-fix-fetch-submodul
shunsuke-sugita May 7, 2026
6b1cc87
undo test_celery_executor fix
shunsuke-sugita May 7, 2026
0b0c77a
Merge branch 'main' into provider-git-fix-fetch-submodul
shunsuke-sugita May 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions providers/git/src/airflow/providers/git/bundles/git.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,10 +287,10 @@ def _has_version(repo: Repo, version: str) -> bool:

def _fetch_bare_repo(self):
refspecs = ["+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"]
cm = nullcontext()
ssh_env_cm = nullcontext()
if self.hook and (cmd := self.hook.env.get("GIT_SSH_COMMAND")):
cm = self.bare_repo.git.custom_environment(GIT_SSH_COMMAND=cmd)
with cm:
ssh_env_cm = self.bare_repo.git.custom_environment(GIT_SSH_COMMAND=cmd)
with ssh_env_cm:
self.bare_repo.remotes.origin.fetch(refspecs)
self.bare_repo.close()

Expand All @@ -300,9 +300,13 @@ def _fetch_bare_repo(self):
reraise=True,
)
def _fetch_submodules(self) -> None:
self._log.info("Initializing and updating submodules", repo_path=self.repo_path)
self.repo.git.submodule("sync", "--recursive")
self.repo.git.submodule("update", "--init", "--recursive", "--jobs", "1")
ssh_env_cm = nullcontext()
if self.hook and (cmd := self.hook.env.get("GIT_SSH_COMMAND")):
ssh_env_cm = self.repo.git.custom_environment(GIT_SSH_COMMAND=cmd)
Comment thread
shunsuke-sugita marked this conversation as resolved.
Outdated
with ssh_env_cm:
self._log.info("Initializing and updating submodules", repo_path=self.repo_path)
self.repo.git.submodule("sync", "--recursive")
self.repo.git.submodule("update", "--init", "--recursive", "--jobs", "1")

def refresh(self) -> None:
if self.version:
Expand Down
60 changes: 60 additions & 0 deletions providers/git/tests/unit/git/bundles/test_git.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -920,6 +920,66 @@ def test_clone_bare_repo_invalid_repository_error_retry_fails(
# Verify Repo was called twice (failed attempt + failed retry)
assert mock_repo_class.call_count == 2

@mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_fetch_submodules_uses_custom_environment_when_git_ssh_command_set(
self, mock_githook_class
):
"""GIT_SSH_COMMAND from hook.env is passed to custom_environment; submodule runs inside that context."""
expected_ssh_cmd = (
"ssh -i /path/key -o IdentitiesOnly=yes -o UserKnownHostsFile=/path/known_hosts"
)
mock_hook = mock_githook_class.return_value
mock_hook.repo_url = "git@github.com:apache/airflow.git"
mock_hook.env = {"GIT_SSH_COMMAND": expected_ssh_cmd}

mock_repo = mock.MagicMock()
ssh_ctx = mock.MagicMock()
mock_repo.git.custom_environment.return_value = ssh_ctx
Comment thread
shunsuke-sugita marked this conversation as resolved.
Outdated

bundle = GitDagBundle(
name="test",
git_conn_id="git_default",
tracking_ref="main",
version="123456",
submodules=True,
)
bundle.repo = mock_repo

bundle._fetch_submodules()

mock_repo.git.custom_environment.assert_called_once_with(GIT_SSH_COMMAND=expected_ssh_cmd)
ssh_ctx.__enter__.assert_called_once()
ssh_ctx.__exit__.assert_called_once()
mock_repo.git.submodule.assert_has_calls(
[mock.call("sync", "--recursive"), mock.call("update", "--init", "--recursive", "--jobs", "1")]
)
assert mock_repo.git.mock_calls[0] == mock.call.custom_environment(GIT_SSH_COMMAND=expected_ssh_cmd)
Comment thread
shunsuke-sugita marked this conversation as resolved.
Outdated
assert mock_repo.git.mock_calls[1] == mock.call.submodule("sync", "--recursive")
Comment thread
shunsuke-sugita marked this conversation as resolved.
Outdated

@mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_fetch_submodules_skips_custom_environment_without_git_ssh_command(self, mock_githook_class):
"""When hook.env has no GIT_SSH_COMMAND, submodule update does not use custom_environment."""
mock_hook = mock_githook_class.return_value
mock_hook.repo_url = "git@github.com:apache/airflow.git"
mock_hook.env = {}

mock_repo = mock.MagicMock()
bundle = GitDagBundle(
Comment thread
shunsuke-sugita marked this conversation as resolved.
Outdated
name="test",
git_conn_id="git_default",
tracking_ref="main",
version="123456",
submodules=True,
)
bundle.repo = mock_repo

bundle._fetch_submodules()

mock_repo.git.custom_environment.assert_not_called()
mock_repo.git.submodule.assert_has_calls(
[mock.call("sync", "--recursive"), mock.call("update", "--init", "--recursive", "--jobs", "1")]
)

@mock.patch("airflow.providers.git.bundles.git.shutil.rmtree")
@mock.patch("airflow.providers.git.bundles.git.os.path.exists")
@mock.patch("airflow.providers.git.bundles.git.GitHook")
Expand Down
Loading