Support integration branch bisection for Fenix (Bug 1857026)

mozregression/fetch_build_info.py

@@ -212,6 +212,21 @@ def _fetch_build_info_from_url(self, url, index, lst):
             with self._fetch_lock:
                 lst.append((index, data))
 
+    def _update_build_info_from_json_pushes(self, date, infos):
+        """
+        If the build info isn't available in text file, we can also try asking the json-pushes
+        API to find the changeset for us using the build timestamp.
+        """
+        build_dt = self.fetch_config.get_nightly_timestamp_from_url(infos["build_url"])
+        try:
+            branch = self.fetch_config.get_nightly_repo(date)
+            jpushes = JsonPushes(branch=branch)
+            push = jpushes.push_by_timestamp(build_dt)[-1]
+            infos["repository"] = jpushes.repo_url
+            infos["changeset"] = push.changeset
+        except Exception:
+            LOG.info(f"Unable to fetch push by timestamp for {build_dt}")
+
     def _get_month_links(self, url):
         with self._lock:
             if url not in self._cache_months:
@@ -223,7 +238,7 @@ def _get_urls(self, date):
         Get the url list of the build folder for a given date.
 
         This methods needs to be thread-safe as it is used in
-        :meth:`NightlyBuildData.get_build_url`.
+        :meth:`find_build_info`.
         """
         LOG.debug("Get URLs for {}".format(date))
         url = self.fetch_config.get_nightly_base_url(date)
@@ -277,6 +292,8 @@ def find_build_info(self, date, fetch_txt_info=True, max_workers=2):
                 infos = sorted(valid_builds, key=lambda b: b[0])[0][1]
                 if fetch_txt_info:
                     self._update_build_info_from_txt(infos)
+                if ("changeset" not in infos) and self.fetch_config.has_json_pushes:
+                    self._update_build_info_from_json_pushes(date, infos)
 
                 build_info = NightlyBuildInfo(
                     self.fetch_config,

mozregression/fetch_configs.py

@@ -130,7 +130,7 @@ def __init__(self, os, bits, processor, arch):
         self.processor = processor
         self.set_arch(arch)
         self.repo = None
-        self.set_build_type("opt")
+        self.set_build_type(self.BUILD_TYPES[0])
         self._used_build_index = 0
 
     @property
@@ -270,7 +270,7 @@ class NightlyConfigMixin(metaclass=ABCMeta):
     A nightly build url is divided in 2 parts here:
 
     1. the base part as returned by :meth:`get_nightly_base_url`
-    2. the final part, which can be found using :meth:`get_nighly_repo_regex`
+    2. the final part, which can be found using :meth:`get_nightly_repo_regex`
 
     The final part contains a repo name, which is returned by
     :meth:`get_nightly_repo`.
@@ -283,6 +283,7 @@ class NightlyConfigMixin(metaclass=ABCMeta):
     nightly_base_repo_name = "firefox"
     nightly_repo = None
     has_build_info = True
+    has_json_pushes = False
 
     def set_base_url(self, url):
         self.archive_base_url = url.rstrip("/")
@@ -339,6 +340,13 @@ def _get_nightly_repo_regex(self, date, repo):
             )
         return r"/%04d-%02d-%02d-[\d-]+%s/$" % (date.year, date.month, date.day, repo)
 
+    def get_nightly_timestamp_from_url(self, url):
+        """
+        Extract the build timestamp from a build url.
+        """
+        matches = re.search(r"/(\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2})-(.*)/", url)
+        return datetime.datetime.strptime(matches.group(1), "%Y-%m-%d-%H-%M-%S")
+
     def can_go_integration(self):
         """
         Indicate if we can bisect integration from this nightly config.
@@ -425,23 +433,22 @@ def get_nightly_repo_regex(self, date):
 
 class FenixNightlyConfigMixin(NightlyConfigMixin):
     nightly_base_repo_name = "fenix"
-    arch_regex_bits = ""
+    has_build_info = False
+    has_json_pushes = True
 
     def _get_nightly_repo(self, date):
-        return "fenix"
+        return "mozilla-central"
 
     def get_nightly_repo_regex(self, date):
-        repo = self.get_nightly_repo(date)
-        repo += self.arch_regex_bits  # e.g., ".*arm64.*".
+        repo = f"{self.nightly_base_repo_name}-[^-]+-android"
+        if self.arch:
+            repo += f"-{self.arch}"
         return self._get_nightly_repo_regex(date, repo)
 
 
 class FocusNightlyConfigMixin(FenixNightlyConfigMixin):
     nightly_base_repo_name = "focus"
 
-    def _get_nightly_repo(self, date):
-        return "focus"
-
 
 class IntegrationConfigMixin(metaclass=ABCMeta):
     """
@@ -554,6 +561,21 @@ def tk_routes(self, push):
         return
 
 
+class FenixIntegrationConfigMixin(IntegrationConfigMixin):
+    tk_name = "fenix"
+
+    def tk_routes(self, push):
+        for build_type in self.build_types:
+            yield "gecko.v2.{}.revision.{}.mobile.{}-{}".format(
+                self.integration_branch,
+                push.changeset,
+                self.tk_name,
+                build_type,
+            )
+            self._inc_used_build()
+        return
+
+
 class ThunderbirdIntegrationConfigMixin(IntegrationConfigMixin):
     default_integration_branch = "comm-central"
 
@@ -626,10 +648,6 @@ class FirefoxConfig(CommonConfig, FirefoxNightlyConfigMixin, FirefoxIntegrationC
         "opt": ("shippable", "pgo"),
     }
 
-    def __init__(self, os, bits, processor, arch):
-        super(FirefoxConfig, self).__init__(os, bits, processor, arch)
-        self.set_build_type("shippable")
-
     def build_regex(self):
         return (
             get_build_regex(
@@ -702,9 +720,14 @@ def available_bits(self):
 
 
 @REGISTRY.register("fenix")
-class FenixConfig(CommonConfig, FenixNightlyConfigMixin):
+class FenixConfig(CommonConfig, FenixNightlyConfigMixin, FenixIntegrationConfigMixin):
+    BUILD_TYPES = ("nightly",)
+    BUILD_TYPE_FALLBACKS = {
+        "nightly": ("nightly-simulation",),
+    }
+
     def build_regex(self):
-        return r"fenix-.+\.apk"
+        return r"(target.{}|fenix-.*)\.apk".format(self.arch)
 
     def available_bits(self):
         return ()
@@ -717,17 +740,6 @@ def available_archs(self):
             "x86_64",
         ]
 
-    def set_arch(self, arch):
-        CommonConfig.set_arch(self, arch)
-        # Map "arch" value to one that can be used in the nightly repo regex lookup.
-        mapping = {
-            "arm64-v8a": "-.+-android-arm64-v8a",
-            "armeabi-v7a": "-.+-android-armeabi-v7a",
-            "x86": "-.+-android-x86",
-            "x86_64": "-.+-android-x86_64",
-        }
-        self.arch_regex_bits = mapping.get(self.arch, "")
-
     def should_use_archive(self):
         return True
 

mozregression/json_pushes.py

@@ -155,3 +155,23 @@ def push(self, changeset, **kwargs):
                     "No pushes available for the date %s on %s." % (changeset, self.branch)
                 )
         return self.pushes(changeset=changeset, **kwargs)[0]
+
+    def push_by_timestamp(self, timestamp):
+        """
+        Returns a list of Push objects for a timestamp.
+
+        This will return at least one Push. In case of error it will raise
+        a MozRegressionError.
+        """
+
+        assert isinstance(timestamp, datetime.datetime)
+
+        # The server interprets these as exclusive ranges, so shift
+        # them each by the minimum time unit.
+        dt = datetime.timedelta(seconds=1)
+        kwargs = {
+            "startdate": str(timestamp - dt),
+            "enddate": str(timestamp + dt),
+        }
+
+        return self.pushes(**kwargs)

tests/unit/test_fetch_configs.py

@@ -278,6 +278,11 @@ def test_get_nightly_repo_regex(self, app_name):
             assert "mozilla-central-android-api-15" in regex
             regex = conf.get_nightly_repo_regex(datetime.date(2017, 8, 30))
             assert "mozilla-central-android-api-16" in regex
+        elif app_name in ["fenix", "focus"]:
+            conf = create_config(app_name, None, None, None, "arm64-v8a")
+            date = datetime.date(2022, 1, 1)
+            regex = conf.get_nightly_repo_regex(date)
+            assert regex == f"/{date.isoformat()}-[\\d-]+{app_name}-[^-]+-android-arm64-v8a/$"
         else:
             conf = create_config(app_name, "linux", 64, None)
             date = datetime.date(2023, 1, 1)
@@ -303,12 +308,12 @@ def setUp(self):
         self.conf = create_config("gve", "linux", 64, None)
 
     def test_fallbacking(self):
-        assert self.conf.build_type == "opt"
-        self.conf._inc_used_build()
         assert self.conf.build_type == "shippable"
-        # Check we wrap
         self.conf._inc_used_build()
         assert self.conf.build_type == "opt"
+        # Check we wrap
+        self.conf._inc_used_build()
+        assert self.conf.build_type == "shippable"
 
 
 class TestGetBuildUrl(unittest.TestCase):
@@ -481,7 +486,7 @@ def test_aarch64_build_types(self):
             None,
             None,
             TIMESTAMP_FENNEC_API_15 - 1,
-            "gecko.v2.mozilla-central.revision.%s.mobile.android-api-11-opt" % CHSET,
+            "gecko.v2.mozilla-central.shippable.revision.%s.mobile.android-api-11-opt" % CHSET,
         ),
         (
             "fennec",
@@ -490,7 +495,7 @@ def test_aarch64_build_types(self):
             None,
             None,
             TIMESTAMP_FENNEC_API_15,
-            "gecko.v2.mozilla-central.revision.%s.mobile.android-api-15-opt" % CHSET,
+            "gecko.v2.mozilla-central.shippable.revision.%s.mobile.android-api-15-opt" % CHSET,
         ),
         (
             "fennec",
@@ -499,7 +504,7 @@ def test_aarch64_build_types(self):
             None,
             None,
             TIMESTAMP_FENNEC_API_16,
-            "gecko.v2.mozilla-central.revision.%s.mobile.android-api-16-opt" % CHSET,
+            "gecko.v2.mozilla-central.shippable.revision.%s.mobile.android-api-16-opt" % CHSET,
         ),
         # thunderbird
         (
@@ -627,6 +632,18 @@ def test_jsshell_aarch64_build_regex():
     assert re.match(conf.build_regex(), "jsshell-win64-x86_64.zip")
 
 
+def test_nightly_timestamp_parse():
+    conf = create_config("fenix", None, None, None, "arm64-v8a")
+    build_url = (
+        "https://archive.mozilla.org/pub/fenix/nightly/2025/12/"
+        "2025-12-01-10-27-59-fenix-147.0a1-android-arm64-v8a/"
+        "fenix-147.0a1.multi.android-arm64-v8a.apk"
+    )
+    expected_dt = datetime.datetime(2025, 12, 1, 10, 27, 59)
+
+    assert conf.get_nightly_timestamp_from_url(build_url) == expected_dt
+
+
 @pytest.mark.parametrize(
     "os,bits,processor,tc_suffix",
     [

tests/unit/test_json_pushes.py

@@ -106,3 +106,22 @@ def test_push_with_date_raise_appropriate_error():
         jpushes.push(date(2015, 1, 1))
 
     assert str(ctx.value) == "No pushes available for the date 2015-01-01 on inbound."
+
+
+def test_push_by_timestamp_builds_url(mocker):
+    timestamp = datetime(2025, 1, 2, 3, 4, 5)
+    pushlog = {"1": {"changesets": ["abc"], "date": 12345}}
+
+    retry_get = mocker.patch("mozregression.json_pushes.retry_get")
+    retry_get.return_value = Mock(json=Mock(return_value=pushlog))
+
+    jpushes = JsonPushes(branch="mozilla-central")
+    pushes = jpushes.push_by_timestamp(timestamp)
+
+    assert pushes[0].push_id == "1"
+    assert pushes[0].changeset == "abc"
+    expected_url = (
+        "https://hg.mozilla.org/mozilla-central/json-pushes?"
+        "enddate=2025-01-02 03:04:06&startdate=2025-01-02 03:04:04"
+    )
+    retry_get.assert_called_once_with(expected_url)