diff --git a/sync/commit.py b/sync/commit.py index 1c4228747..f897c8334 100644 --- a/sync/commit.py +++ b/sync/commit.py @@ -11,7 +11,7 @@ from . import log from .env import Environment from .errors import AbortError -from .lando import git2hg +from .lando import git2hg, hg2git from .repos import cinnabar, cinnabar_map, pygit2_get from typing import Dict @@ -215,6 +215,12 @@ def canonical_rev(self) -> str: return self.cinnabar.git2hg(self.sha1) return self.sha1 + @property + def canonical_rev_git(self) -> str: + if self.cinnabar: + raise ValueError(f"Commit {self.sha1} doesn't have a canonical git SHA1") + return self.sha1 + @property def msg(self) -> bytes: return self.pygit2_commit.raw_message @@ -539,6 +545,16 @@ def bug(self) -> int | None: assert isinstance(bugs[0], int) return bugs[0] + @property + def canonical_rev_git(self) -> str: + if self.cinnabar: + if "gecko-commit-git" not in self.notes: + self.notes["gecko-commit-git"] = hg2git(self.sha1) + sha1 = self.notes["gecko-commit-git"] + assert sha1 is not None + return sha1 + return self.sha1 + def has_wpt_changes(self) -> bool: prefix = env.config["gecko"]["path"]["wpt"] return not self.is_empty(prefix) diff --git a/sync/lando.py b/sync/lando.py index 075663a03..11b31ebb5 100644 --- a/sync/lando.py +++ b/sync/lando.py @@ -1,21 +1,23 @@ -import json -import urllib.request +import requests from .env import Environment -from . import log env = Environment() -logger = log.get_logger(__name__) + +def hg2git(hg_hash: str) -> str: + response = requests.get(env.config["lando"]["api_url"] + "/hg2git/firefox/" + hg_hash) + data = response.json() + assert isinstance(data, dict) + assert isinstance(data["git_hash"], str) + + return data["git_hash"] def git2hg(git_hash: str) -> str: - response = urllib.request.urlopen( - env.config["lando"]["api_url"] + "/git2hg/firefox/" + git_hash - ) # nosec B310 - data = response.read() - map = json.loads(data.decode("utf-8")) - assert isinstance(map, dict) - assert isinstance(map["hg_hash"], str) - - return map["hg_hash"] + response = requests.get(env.config["lando"]["api_url"] + "/git2hg/firefox/" + git_hash) + data = response.json() + assert isinstance(data, dict) + assert isinstance(data["hg_hash"], str) + + return data["hg_hash"] diff --git a/sync/upstream.py b/sync/upstream.py index c02411667..fd8fca2a5 100644 --- a/sync/upstream.py +++ b/sync/upstream.py @@ -326,7 +326,10 @@ def repository(self) -> str: def add_commit(self, gecko_commit: GeckoCommit) -> tuple[Commit | None, bool]: git_work = self.wpt_worktree.get() - metadata = {"gecko-commit": gecko_commit.canonical_rev} + metadata = { + "gecko-commit": gecko_commit.canonical_rev, + "gecko-commit-git": gecko_commit.canonical_rev_git, + } if os.path.exists(os.path.join(git_work.working_dir, gecko_commit.canonical_rev + ".diff")): # If there's already a patch file here then don't try to create a new one diff --git a/test/test_downstream.py b/test/test_downstream.py index d01a670d8..f75be6bc7 100644 --- a/test/test_downstream.py +++ b/test/test_downstream.py @@ -398,7 +398,10 @@ def test_next_try_push_infra_fail_try_rebase( bookmarks="mozilla/central", ) downstream.update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - upstream.gecko_push(git_gecko, git_wpt, "mozilla-central", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + upstream.gecko_push( + git_gecko, git_wpt, "mozilla-central", rev, raise_on_error=True + ) sync.data["affected-tests"] = {"testharness": ["example"]} sync.data["skip"] = False @@ -475,7 +478,10 @@ def test_next_try_push_infra_fail_try_rebase_failed( bookmarks="mozilla/central", ) downstream.update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - upstream.gecko_push(git_gecko, git_wpt, "mozilla-central", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + upstream.gecko_push( + git_gecko, git_wpt, "mozilla-central", rev, raise_on_error=True + ) try_push = sync.next_try_push(try_cls=MockTryCls) with try_push.as_mut(lock): diff --git a/test/test_landing.py b/test/test_landing.py index 616e7983d..c0cda295c 100644 --- a/test/test_landing.py +++ b/test/test_landing.py @@ -337,7 +337,8 @@ def test_landing_reapply( rev = upstream_gecko_commit(test_changes=test_changes, bug=1111, message=b"Add change1 file") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - pushed, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + pushed, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) sync_1 = pushed.pop() # Update central @@ -359,7 +360,8 @@ def test_landing_reapply( rev = upstream_gecko_commit(test_changes=test_changes, bug=1112, message=b"Add change2 file") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - pushed, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + pushed, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) sync_2 = pushed.pop() hg_gecko_upstream.bookmark("mozilla/central", "-r", rev) @@ -393,7 +395,8 @@ def test_landing_reapply( rev = upstream_gecko_commit(test_changes=test_changes, bug=1113, message=b"Add change3 file") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - pushed, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + pushed, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) # Now start a landing with patch.object(trypush.TryCommit, "read_treeherder", autospec=True) as mock_read: @@ -541,7 +544,9 @@ def create_and_upstream_gecko_bug( rev = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Change CONFIG") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) + git_rev = "test_revision" + with patch("sync.commit.hg2git", return_value=git_rev): + upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug) sync = syncs["open"].pop() @@ -550,7 +555,8 @@ def create_and_upstream_gecko_bug( hg_gecko_upstream.bookmark("mozilla/central", "-r", rev) update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - upstream.gecko_push(git_gecko, git_wpt, "mozilla-central", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value=git_rev): + upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) # Set commits PR number so they will be skipped. for commit_item in git_wpt.iter_commits(sync.wpt_commits.head.sha1): diff --git a/test/test_sync.py b/test/test_sync.py index 5438e057d..7d1674e21 100644 --- a/test/test_sync.py +++ b/test/test_sync.py @@ -1,5 +1,6 @@ import git import pytest +from unittest.mock import patch from sync import index, upstream from sync.gitutils import update_repositories from sync.lock import SyncLock @@ -12,7 +13,8 @@ def test_delete(env, git_gecko, git_wpt, upstream_gecko_commit): rev = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Change README") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - _, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + _, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) sync = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug, flat=True).pop() process_name = sync.process_name diff --git a/test/test_upstream.py b/test/test_upstream.py index a662a8b6b..5d66191c9 100644 --- a/test/test_upstream.py +++ b/test/test_upstream.py @@ -15,9 +15,11 @@ def test_create_pr(env, git_gecko, git_wpt, upstream_gecko_commit): update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) initial_data_commits = list(git_gecko.iter_commits(env.config["sync"]["ref"])) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "autoland", rev, raise_on_error=True - ) + git_rev = "test_revision" + with patch("sync.commit.hg2git", return_value=git_rev): + pushed, landed, failed = upstream.gecko_push( + git_gecko, git_wpt, "autoland", rev, raise_on_error=True + ) assert len(pushed) == 1 assert len(landed) == 0 assert len(failed) == 0 @@ -44,6 +46,7 @@ def test_create_pr(env, git_gecko, git_wpt, upstream_gecko_commit): assert wpt_commit.metadata == { "bugzilla-url": "https://bugzilla-dev.allizom.org/show_bug.cgi?id=1234", "gecko-commit": rev, + "gecko-commit-git": git_rev, } assert sync.pr assert "Posting to bug %s" % bug in env.bz.output.getvalue() @@ -57,7 +60,8 @@ def test_create_pr_backout(git_gecko, git_wpt, upstream_gecko_commit, upstream_g rev = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Change README") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug) assert list(syncs.keys()) == ["open"] @@ -93,7 +97,8 @@ def test_create_pr_revert(git_gecko, git_wpt, upstream_gecko_commit, upstream_ge rev = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=message.encode()) update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug) assert list(syncs.keys()) == ["open"] @@ -132,7 +137,8 @@ def test_create_pr_backout_reland( rev = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Change README") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) backout_rev = upstream_gecko_backout(rev, bug) @@ -157,7 +163,8 @@ def test_create_pr_backout_reland( ) update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - upstream.gecko_push(git_gecko, git_wpt, "autoland", relanding_rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value=relanding_rev): + upstream.gecko_push(git_gecko, git_wpt, "autoland", relanding_rev, raise_on_error=True) syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug) assert list(syncs.keys()) == ["open"] @@ -183,7 +190,8 @@ def test_create_partial_backout_reland( ) update_repositories(git_gecko, git_wpt) - upstream.gecko_push(git_gecko, git_wpt, "autoland", rev1, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + upstream.gecko_push(git_gecko, git_wpt, "autoland", rev1, raise_on_error=True) upstream_gecko_backout(rev1, bug) @@ -197,7 +205,8 @@ def test_create_partial_backout_reland( ) update_repositories(git_gecko, git_wpt, wait_gecko_commit=relanding_rev) - upstream.gecko_push(git_gecko, git_wpt, "autoland", relanding_rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value=relanding_rev): + upstream.gecko_push(git_gecko, git_wpt, "autoland", relanding_rev, raise_on_error=True) syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug) assert list(syncs.keys()) == ["open"] @@ -217,9 +226,9 @@ def test_land_pr(env, git_gecko, git_wpt, hg_gecko_upstream, upstream_gecko_comm rev = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Change README") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "autoland", rev, raise_on_error=True - ) + git_rev = "test_revision" + with patch("sync.commit.hg2git", return_value=git_rev): + upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug) assert list(syncs.keys()) == ["open"] @@ -231,9 +240,8 @@ def test_land_pr(env, git_gecko, git_wpt, hg_gecko_upstream, upstream_gecko_comm hg_gecko_upstream.bookmark("mozilla/central", "-r", rev) update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "mozilla-central", rev, raise_on_error=True - ) + with patch("sync.commit.hg2git", return_value=git_rev): + upstream.gecko_push(git_gecko, git_wpt, "mozilla-central", rev, raise_on_error=True) syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug) assert syncs == {"wpt-merged": {sync}} @@ -267,9 +275,9 @@ def test_land_pr_after_status_change( rev = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Change README") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "autoland", rev, raise_on_error=True - ) + with patch("sync.commit.hg2git", return_value="test_revision"): + upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) + syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug) assert list(syncs.keys()) == ["open"] assert len(syncs["open"]) == 1 @@ -312,9 +320,11 @@ def test_no_upstream_downstream( wpt-commits: 0000000000000000000000000000000000000000""", ) update_repositories(git_gecko, git_wpt, wait_gecko_commit=hg_rev) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "autoland", hg_rev, raise_on_error=True - ) + with patch("sync.commit.hg2git", return_value="test_revision"): + pushed, landed, failed = upstream.gecko_push( + git_gecko, git_wpt, "autoland", hg_rev, raise_on_error=True + ) + assert not pushed assert not landed assert not failed @@ -337,9 +347,11 @@ def test_upstream_existing(env, git_gecko, git_wpt, upstream_gecko_commit, upstr upstream_wpt_commit(file_data=test_changes_1) update_repositories(git_gecko, git_wpt, wait_gecko_commit=gecko_rev_2) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "autoland", gecko_rev_2, raise_on_error=True - ) + git_rev = "test_revision" + with patch("sync.commit.hg2git", return_value=git_rev): + pushed, landed, failed = upstream.gecko_push( + git_gecko, git_wpt, "autoland", gecko_rev_2, raise_on_error=True + ) assert len(pushed) == 1 assert len(landed) == 0 assert len(failed) == 0 @@ -360,6 +372,7 @@ def test_upstream_existing(env, git_gecko, git_wpt, upstream_gecko_commit, upstr assert wpt_commit.metadata == { "bugzilla-url": "https://bugzilla-dev.allizom.org/show_bug.cgi?id=1234", "gecko-commit": gecko_rev_2, + "gecko-commit-git": git_rev, } # Now make another push to the same bug and check we handle it correctly @@ -367,9 +380,10 @@ def test_upstream_existing(env, git_gecko, git_wpt, upstream_gecko_commit, upstr test_changes_3 = {"YET_ANOTHER": b"Add more files\n"} gecko_rev_3 = upstream_gecko_commit(test_changes=test_changes_3, bug=bug, message=b"Add more") update_repositories(git_gecko, git_wpt, wait_gecko_commit=gecko_rev_3) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "autoland", gecko_rev_3, raise_on_error=True - ) + with patch("sync.commit.hg2git", return_value=git_rev): + pushed, landed, failed = upstream.gecko_push( + git_gecko, git_wpt, "autoland", gecko_rev_3, raise_on_error=True + ) assert len(sync.gecko_commits) == 3 assert len(sync.wpt_commits) == 2 assert [item.metadata.get("gecko-commit") for item in sync.wpt_commits] == [ @@ -384,9 +398,11 @@ def test_upstream_multi(env, git_gecko, git_wpt, upstream_gecko_commit): rev_0 = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Add README") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev_0) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "autoland", rev_0, raise_on_error=True - ) + git_rev = "test_revision" + with patch("sync.commit.hg2git", return_value=git_rev): + pushed, _, _ = upstream.gecko_push( + git_gecko, git_wpt, "autoland", rev_0, raise_on_error=True + ) assert len(pushed) == 1 sync_0 = pushed.pop() @@ -394,9 +410,10 @@ def test_upstream_multi(env, git_gecko, git_wpt, upstream_gecko_commit): rev_1 = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Add README1") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev_1) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "autoland", rev_1, raise_on_error=True - ) + with patch("sync.commit.hg2git", return_value=git_rev): + pushed, _, _ = upstream.gecko_push( + git_gecko, git_wpt, "autoland", rev_1, raise_on_error=True + ) assert len(pushed) == 1 assert pushed == {sync_0} assert len(sync_0.upstreamed_gecko_commits) == 2 @@ -413,9 +430,10 @@ def test_upstream_multi(env, git_gecko, git_wpt, upstream_gecko_commit): rev_2 = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Add README2") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev_2) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "autoland", rev_2, raise_on_error=True - ) + with patch("sync.commit.hg2git", return_value=git_rev): + pushed, _, _ = upstream.gecko_push( + git_gecko, git_wpt, "autoland", rev_2, raise_on_error=True + ) assert len(pushed) == 1 sync_1 = pushed.pop() @@ -436,9 +454,10 @@ def test_upstream_multi(env, git_gecko, git_wpt, upstream_gecko_commit): rev_3 = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Add README3") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev_3) - pushed, landed, failed = upstream.gecko_push( - git_gecko, git_wpt, "autoland", rev_3, raise_on_error=True - ) + with patch("sync.commit.hg2git", return_value=git_rev): + pushed, landed, failed = upstream.gecko_push( + git_gecko, git_wpt, "autoland", rev_3, raise_on_error=True + ) assert len(pushed) == 1 sync_2 = pushed.pop() assert sync_2.process_name not in (sync_1.process_name, sync_0.process_name) @@ -458,7 +477,8 @@ def test_upstream_reprocess_commits( rev = upstream_gecko_commit(test_changes=test_changes, bug=bug, message=b"Change README") update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev) - pushed, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) + with patch("sync.commit.hg2git", return_value="test_revision"): + pushed, _, _ = upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True) sync = pushed.pop() assert sync.gecko_commits[0].upstream_sync(git_gecko, git_wpt) == sync @@ -521,9 +541,10 @@ def test_pr_commits_merge( upstream_gecko_commit, set_pr_status, ): - sync = setup_repo( - env, git_wpt, git_gecko, hg_gecko_upstream, upstream_gecko_commit, set_pr_status - ) + with patch("sync.commit.hg2git", return_value="test_revision"): + sync = setup_repo( + env, git_wpt, git_gecko, hg_gecko_upstream, upstream_gecko_commit, set_pr_status + ) # Make changes on master git_wpt_upstream.branches.master.checkout() @@ -569,9 +590,10 @@ def test_pr_commits_squash_merge( upstream_gecko_commit, set_pr_status, ): - sync = setup_repo( - env, git_wpt, git_gecko, hg_gecko_upstream, upstream_gecko_commit, set_pr_status - ) + with patch("sync.commit.hg2git", return_value="test_revision"): + sync = setup_repo( + env, git_wpt, git_gecko, hg_gecko_upstream, upstream_gecko_commit, set_pr_status + ) # Make changes on master git_wpt_upstream.branches.master.checkout() @@ -618,9 +640,10 @@ def test_pr_commits_fast_forward( upstream_gecko_commit, set_pr_status, ): - sync = setup_repo( - env, git_wpt, git_gecko, hg_gecko_upstream, upstream_gecko_commit, set_pr_status - ) + with patch("sync.commit.hg2git", return_value="test_revision"): + sync = setup_repo( + env, git_wpt, git_gecko, hg_gecko_upstream, upstream_gecko_commit, set_pr_status + ) base = git_wpt_upstream.head.commit.hexsha