Create new test type aamtest for accessibility API testing

.taskcluster.yml

@@ -57,7 +57,7 @@ tasks:
             owner: ${owner}
             source: ${event.repository.clone_url}
           payload:
-            image: ghcr.io/web-platform-tests/wpt:2
+            image: ghcr.io/web-platform-tests/wpt:3
             maxRunTime: 7200
             artifacts:
               public/results:

lint.ignore

@@ -52,6 +52,7 @@ TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.wbn
 TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.avif
 TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.annexb
 TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.crx
+TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.tlb
 
 ## .gitignore
 W3C-TEST.ORG: .gitignore

tools/ci/tc/tasks/test.yml

@@ -4,7 +4,7 @@ components:
     workerType: ci
     schedulerId: taskcluster-github
     deadline: "24 hours"
-    image: ghcr.io/web-platform-tests/wpt:2
+    image: ghcr.io/web-platform-tests/wpt:3
     maxRunTime: 7200
     artifacts:
       public/results:

tools/docker/Dockerfile

@@ -20,6 +20,9 @@ RUN apt-get -qqy update \
     glib-networking-services \
     gstreamer1.0-plugins-bad \
     gstreamer1.0-gl \
+    libatspi2.0-dev \
+    libcairo2-dev \
+    libgirepository1.0-dev \
     libosmesa6-dev \
     libproxy1-plugin-webkit \
     libvirt-daemon-system \

tools/manifest/item.py

@@ -354,6 +354,22 @@ def to_json(self) -> Tuple[Optional[Text], Dict[Text, Any]]:
         return rv
 
 
+class AccessibilityAPIMappingTest(URLManifestItem):
+    __slots__ = ()
+
+    item_type = "aamtest"
+
+    @property
+    def timeout(self) -> Optional[Text]:
+        return self._extras.get("timeout")
+
+    def to_json(self) -> Tuple[Optional[Text], Dict[Text, Any]]:
+        rv = super().to_json()
+        if self.timeout is not None:
+            rv[-1]["timeout"] = self.timeout
+        return rv
+
+
 class SupportFile(ManifestItem):
     __slots__ = ()
 

tools/manifest/manifest.py

@@ -8,7 +8,8 @@
 
 from . import jsonlib
 from . import vcs
-from .item import (ConformanceCheckerTest,
+from .item import (AccessibilityAPIMappingTest,
+                   ConformanceCheckerTest,
                    CrashTest,
                    ManifestItem,
                    ManualTest,
@@ -46,6 +47,7 @@ class InvalidCacheError(Exception):
                                                 "crashtest": CrashTest,
                                                 "manual": ManualTest,
                                                 "wdspec": WebDriverSpecTest,
+                                                "aamtest": AccessibilityAPIMappingTest,
                                                 "conformancechecker": ConformanceCheckerTest,
                                                 "visual": VisualTest,
                                                 "spec": SpecItem,

tools/manifest/sourcefile.py

@@ -16,7 +16,8 @@
 import html5lib
 
 from . import XMLParser
-from .item import (ConformanceCheckerTest,
+from .item import (AccessibilityAPIMappingTest,
+                   ConformanceCheckerTest,
                    CrashTest,
                    ManifestItem,
                    ManualTest,
@@ -33,7 +34,7 @@
 # because relative import beyond toplevel throws *ImportError*!
 from metadata.webfeatures.schema import WEB_FEATURES_YML_FILENAME  # type: ignore
 
-wd_pattern = "*.py"
+py_pattern = "*.py"
 js_meta_re = re.compile(br"//\s*META:\s*(\w*)=(.*)$")
 python_meta_re = re.compile(br"#\s*META:\s*(\w*)=(.*)$")
 
@@ -395,7 +396,16 @@ def name_is_webdriver(self) -> bool:
                  (rel_path_parts[:2] == ("infrastructure", "webdriver") and
                   len(rel_path_parts) > 2)) and
                 self.filename not in ("__init__.py", "conftest.py") and
-                fnmatch(self.filename, wd_pattern))
+                fnmatch(self.filename, py_pattern))
+
+    @property
+    def name_is_wai_aria_aam(self) -> bool:
+        """Check if the file name matches the conditions for the file to
+        be a WAI-ARIA AAM spec test file"""
+        rel_path_parts = self.rel_path_parts
+        return ((rel_path_parts[0] == "wai-aria-aam" and len(rel_path_parts) > 1) and
+                self.filename not in ("__init__.py", "conftest.py") and
+                fnmatch(self.filename, py_pattern))
 
     @property
     def name_is_reference(self) -> bool:
@@ -474,7 +484,7 @@ def pac_nodes(self) -> List[ElementTree.Element]:
     def script_metadata(self) -> Optional[List[Tuple[Text, Text]]]:
         if self.name_is_worker or self.name_is_multi_global or self.name_is_window or self.name_is_extension:
             regexp = js_meta_re
-        elif self.name_is_webdriver:
+        elif self.name_is_webdriver or self.name_is_wai_aria_aam:
             regexp = python_meta_re
         else:
             return None
@@ -899,6 +909,9 @@ def possible_types(self) -> Set[Text]:
         if self.name_is_webdriver:
             return {WebDriverSpecTest.item_type}
 
+        if self.name_is_wai_aria_aam:
+            return {AccessibilityAPIMappingTest.item_type}
+
         if self.name_is_visual:
             return {VisualTest.item_type}
 
@@ -986,6 +999,16 @@ def manifest_items(self) -> Tuple[Text, List[ManifestItem]]:
                     timeout=self.timeout
                 )]
 
+        elif self.name_is_wai_aria_aam:
+            rv = AccessibilityAPIMappingTest.item_type, [
+                AccessibilityAPIMappingTest(
+                    self.tests_root,
+                    self.rel_path,
+                    self.url_base,
+                    self.rel_url,
+                    timeout=self.timeout
+                )]
+
         elif self.name_is_visual:
             rv = VisualTest.item_type, [
                 VisualTest(

tools/mypy.ini

@@ -42,6 +42,9 @@ follow_imports = silent
 follow_imports = silent
 
 # Ignore missing or untyped libraries.
+[mypy-ApplicationServices.*]
+ignore_missing_imports = True
+
 [mypy-Cocoa.*]
 ignore_missing_imports = True
 
@@ -51,9 +54,15 @@ ignore_missing_imports = True
 [mypy-Quartz.*]
 ignore_missing_imports = True
 
+[mypy-comtypes.*]
+ignore_missing_imports = True
+
 [mypy-dnslib.*]
 ignore_missing_imports = True
 
+[mypy-gi.*]
+ignore_missing_imports = True
+
 [mypy-marionette_driver.*]
 ignore_missing_imports = True
 

tools/wpt/run.py

@@ -904,6 +904,10 @@ def setup_wptrunner(venv, **kwargs):
     if not venv.skip_virtualenv_setup:
         requirements = [os.path.join(wpt_root, "tools", "wptrunner", "requirements.txt")]
         requirements.extend(setup_cls.requirements())
+
+        if "aamtest" in kwargs["test_types"]:
+            requirements.append(os.path.join(wpt_root, "tools", "wptrunner", "requirements_platform_accessibility.txt"))
+
         venv.install_requirements(*requirements)
 
     affected_revish = kwargs.get("affected")

tools/wpt/testfiles.py

@@ -238,7 +238,7 @@ def affected_testfiles(files_changed: Iterable[Text],
     nontests_changed = set(files_changed)
     wpt_manifest = load_manifest(manifest_path, manifest_update)
 
-    test_types = ["crashtest", "print-reftest", "reftest", "testharness", "wdspec"]
+    test_types = ["crashtest", "print-reftest", "reftest", "testharness", "wdspec", "aamtest"]
     support_files = {os.path.join(wpt_root, path)
                      for _, path, _ in wpt_manifest.itertypes("support")}
     wdspec_test_files = {os.path.join(wpt_root, path)

tools/wptrunner/requirements_platform_accessibility.txt

@@ -0,0 +1,8 @@
+comtypes==1.4.11; sys_platform == 'win32'
+PyGObject==3.50.1; sys_platform == 'linux' and python_version >= '3.9'
+PyGObject==3.48.2; sys_platform == 'linux' and python_version < '3.9'
+pyobjc-framework-Accessibility==11.1; sys_platform == 'darwin' and python_version >= '3.9'
+pyobjc-framework-ApplicationServices==11.1; sys_platform == 'darwin' and python_version >= '3.9'
+pyobjc==10.3.2; sys_platform == 'darwin' and python_version < '3.9'
+pyobjc-framework-Accessibility==10.3.2; sys_platform == 'darwin' and python_version < '3.9'
+pyobjc-framework-ApplicationServices==10.3.2; sys_platform == 'darwin' and python_version < '3.9'

tools/wptrunner/wptrunner/browsers/chrome.py

@@ -1,5 +1,6 @@
 # mypy: allow-untyped-defs
 
+import os
 import re
 import time
 
@@ -33,6 +34,7 @@
                               "reftest": "ChromeDriverRefTestExecutor",
                               "print-reftest": "ChromeDriverPrintRefTestExecutor",
                               "wdspec": "WdspecExecutor",
+                              "aamtest": "WdspecExecutor",
                               "crashtest": "ChromeDriverCrashTestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
@@ -222,6 +224,13 @@ def executor_kwargs(logger, test_type, test_environment, run_info_data, subsuite
     if test_type == "wdspec":
         executor_kwargs["binary_args"] = chrome_options["args"]
 
+    if test_type == "aamtest":
+        # Necessary to force chrome to register in AT-SPI2.
+        os.environ["ACCESSIBILITY_ENABLED"] = "1"
+        if "--force-renderer-accessibility" not in chrome_options["args"]:
+            chrome_options["args"].append("--force-renderer-accessibility")
+            executor_kwargs["binary_args"] = chrome_options["args"]
+
     executor_kwargs["capabilities"] = capabilities
 
     return executor_kwargs
@@ -256,11 +265,10 @@ def __init__(self,
         self._require_webdriver_bidi: Optional[bool] = None
 
     def restart_on_test_type_change(self, new_test_type: str, old_test_type: str) -> bool:
-        # Restart the test runner when switch from/to wdspec tests. Wdspec test
-        # is using a different protocol class so a restart is always needed.
-        if "wdspec" in [old_test_type, new_test_type]:
-            return True
-        return False
+        # Restart the test runner when switch from/to wdspec or aamtest tests.
+        # These tests use a different protocol class so a restart is always needed.
+        wdspec_types = {"wdspec", "aamtest"}
+        return old_test_type in wdspec_types or new_test_type in wdspec_types
 
     def create_output_handler(self, cmd: List[str]) -> OutputHandler:
         return ChromeDriverOutputHandler(

tools/wptrunner/wptrunner/browsers/chromium.py

@@ -17,6 +17,7 @@
                               "reftest": "WebDriverRefTestExecutor",
                               "print-reftest": "ChromeDriverPrintRefTestExecutor",
                               "wdspec": "WdspecExecutor",
+                              "aamtest": "WdspecExecutor",
                               "crashtest": "WebDriverCrashtestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",

tools/wptrunner/wptrunner/browsers/edge.py

@@ -19,6 +19,7 @@
                               "reftest": "EdgeDriverRefTestExecutor",
                               "print-reftest": "EdgeDriverPrintRefTestExecutor",
                               "wdspec": "WdspecExecutor",
+                              "aamtest": "WdspecExecutor",
                               "crashtest": "WebDriverCrashtestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",

tools/wptrunner/wptrunner/browsers/firefox.py

@@ -40,12 +40,14 @@
 __wptrunner__ = {"product": "firefox",
                  "check_args": "check_args",
                  "browser": {None: "FirefoxBrowser",
-                             "wdspec": "FirefoxWdSpecBrowser"},
+                             "wdspec": "FirefoxWdSpecBrowser",
+                             "aamtest": "FirefoxWdSpecBrowser"},
                  "executor": {"crashtest": "MarionetteCrashtestExecutor",
                               "testharness": "MarionetteTestharnessExecutor",
                               "reftest": "MarionetteRefTestExecutor",
                               "print-reftest": "MarionettePrintRefTestExecutor",
-                              "wdspec": "MarionetteWdspecExecutor"},
+                              "wdspec": "MarionetteWdspecExecutor",
+                              "aamtest": "MarionetteWdspecExecutor",},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
                  "env_extras": "env_extras",
@@ -71,7 +73,7 @@ def get_timeout_multiplier(test_type, run_info_data, **kwargs):
             return 4 * multiplier
         else:
             return 2 * multiplier
-    elif test_type == "wdspec":
+    elif test_type in ("wdspec", "aamtest"):
         if (run_info_data.get("asan") or
             run_info_data.get("ccov") or
             run_info_data.get("debug")):
@@ -126,7 +128,7 @@ def browser_kwargs(logger, test_type, run_info_data, config, subsuite, **kwargs)
                       "gmp_path": kwargs["gmp_path"] if "gmp_path" in kwargs else None,
                       "debug_test": kwargs["debug_test"]}
 
-    if test_type == "wdspec":
+    if test_type in ("wdspec", "aamtest"):
         browser_kwargs["webdriver_binary"] = kwargs["webdriver_binary"]
         browser_kwargs["webdriver_args"] = kwargs["webdriver_args"].copy()
 
@@ -145,6 +147,15 @@ def browser_kwargs(logger, test_type, run_info_data, config, subsuite, **kwargs)
         browser_kwargs["test_type"] = test_type
         browser_kwargs["timeout_multiplier"] = get_timeout_multiplier(test_type, run_info_data, **kwargs)
 
+    if test_type == "aamtest":
+        # Enable accessibility in the browser.
+        if ('accessibility.force_disabled', '-1') not in browser_kwargs["extra_prefs"]:
+            browser_kwargs["extra_prefs"].append(('accessibility.force_disabled', '-1'))
+        # Cache all attributes immediately for testing.
+        if ('accessibility.enable_all_cache_domains', 'true') not in browser_kwargs["extra_prefs"]:
+            browser_kwargs["extra_prefs"].append(('accessibility.enable_all_cache_domains', 'true'))
+
+
     browser_kwargs["extra_prefs"].extend(subsuite.config.get("prefs", []))
     return browser_kwargs
 
@@ -172,7 +183,8 @@ def executor_kwargs(logger, test_type, test_environment, run_info_data,
             else:
                 cache_screenshots = major_version < 14
         executor_kwargs["cache_screenshots"] = cache_screenshots
-    if test_type == "wdspec":
+
+    if test_type in ("wdspec", "aamtest"):
         options = {"args": []}
         if kwargs["binary"]:
             executor_kwargs["webdriver_args"].extend(["--binary", kwargs["binary"]])
@@ -775,7 +787,7 @@ def _get_default_prefs(self):
         if self.test_type == "print-reftest":
             prefs["print.always_print_silent"] = True
 
-        if self.test_type == "wdspec":
+        if self.test_type in ("wdspec", "aamtest"):
             prefs.update(
                 {
                     "remote.prefs.recommended": True,
@@ -978,6 +990,7 @@ def __init__(self, logger, binary, package_name, prefs_root, webdriver_binary, w
 
         self.env = self.get_env(binary, debug_info, headless, gmp_path, chaos_mode_flags, e10s)
 
+        # Todo: need test type to use "aam" test in profile_creator_cls
         profile_creator = profile_creator_cls(logger,
                                               prefs_root,
                                               config,

tools/wptrunner/wptrunner/browsers/servo.py

@@ -27,6 +27,7 @@
         "reftest": "ServoRefTestExecutor",
         "crashtest": "ServoCrashtestExecutor",
         "wdspec": "WdspecExecutor",
+        "aamtest": "WdspecExecutor",
     },
     "browser_kwargs": "browser_kwargs",
     "executor_kwargs": "executor_kwargs",

tools/wptrunner/wptrunner/executors/base.py

@@ -39,7 +39,7 @@ def executor_kwargs(test_type, test_environment, run_info_data, subsuite, **kwar
         executor_kwargs["screenshot_cache"] = screenshot_cache
         executor_kwargs["reftest_screenshot"] = kwargs["reftest_screenshot"]
 
-    if test_type == "wdspec":
+    if test_type in ("wdspec", "aamtest"):
         executor_kwargs["binary"] = kwargs["binary"]
         executor_kwargs["binary_args"] = kwargs["binary_args"].copy()
         executor_kwargs["webdriver_binary"] = kwargs["webdriver_binary"]

tools/wptrunner/wptrunner/executors/pytestrunner/runner.py

@@ -52,6 +52,7 @@ def run(path, server_config, session_config, timeout=0):
 
             config = session_config.copy()
             config["wptserve"] = server_config.as_dict()
+            config["timeout"] = timeout
 
             with open(config_path, "w") as f:
                 json.dump(config, f)
@@ -144,12 +145,15 @@ def record_error(self, report, message):
         self.record(report.nodeid, "ERROR", message, report.longrepr)
 
     def record_skip(self, report):
-        self.record(
-            report.nodeid,
-            "ERROR",
-            "In-test skip decorators are disallowed, "
-            "please use WPT metadata to ignore tests.",
-        )
+        if "PRECONDITION_FAILED" in report.longrepr[2]:
+            self.record(report.nodeid, "PRECONDITION_FAILED")
+        else:
+            self.record(
+                report.nodeid,
+                "ERROR",
+                "In-test skip decorators are disallowed, "
+                "please use WPT metadata to ignore tests.",
+            )
 
     def record(self, test, status, message=None, stack=None):
         if stack is not None: