testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorselenium.py
author Csoregi Natalia <ncsoregi@mozilla.com>
Thu, 21 Feb 2019 23:10:13 +0200
changeset 460352 749b314d6c2807b4f413b890be1d52e78cbf1b9c
parent 460259 855b3f20c07099106d46f0c17cfe9a03696d440a
child 461100 186ecc82160b10a9a8054bc9107613200a134090
permissions -rw-r--r--
Backed out 95 changesets (bug 1526580, bug 1527281, bug 1527283, bug 1526852, bug 1526596, bug 1526595, bug 1526831, bug 1488208, bug 1526839, bug 1526599, bug 1526598, bug 1526592, bug 1526746, bug 1526728, bug 1526601, bug 1527139, bug 1526606, bug 1526604, bug 1526625, bug 1526700, bug 1526626, bug 1526621, bug 1526627, bug 1526623, bug 1526585, bug 1527455, bug 1526264, bug 1526608, bug 1526644, bug 1527156, bug 1527157, bug 1526643, bug 1527153, bug 1526893, bug 1528824, bug 1526610, bug 1526594, bug 1526581, bug 1526603, bug 1526583, bug 1526827, bug 973341, bug 1526587, bug 1526620, bug 1526270, bug 1526849, bug 1526732, bug 1527044, bug 1526730, bug 1527042, bug 1526718, bug 1526616, bug 1526617, bug 1526619, bug 1526739, bug 1526612, bug 1526860, bug 1527046, bug 1527273, bug 1527149, bug 1527444, bug 1526848, bug 1527278, bug 1527448, bug 1527276, bug 1527143, bug 1526767, bug 1526611, bug 1504391, bug 1516736, bug 1527461) for spidermonkey bustages on streams/readable-streams/brand-checks.any.js. CLOSED TREE Backed out changeset a15983fcecca (bug 1528824) Backed out changeset 2553bea1f659 (bug 1527461) Backed out changeset 83f53c1db6f2 (bug 1527455) Backed out changeset 93eca4ed498e (bug 1527448) Backed out changeset 09ce1b8d9831 (bug 1526700) Backed out changeset 4551719aa859 (bug 1526700) Backed out changeset 5addc6c0ced0 (bug 1527444) Backed out changeset 8de204a8ad00 (bug 1527283) Backed out changeset e8c23fbb5c1a (bug 1527281) Backed out changeset 28d2c5bcd984 (bug 1527278) Backed out changeset 9128147cb4f1 (bug 1527276) Backed out changeset 70750463a84e (bug 1527273) Backed out changeset 11e5c7e27821 (bug 1526732) Backed out changeset cc6673d871d1 (bug 1526732) Backed out changeset d9c437f2f5f7 (bug 1527157) Backed out changeset f5f376dc06dc (bug 1526718) Backed out changeset 4ca9dfb6c9bf (bug 1527156) Backed out changeset a9a12335f516 (bug 1527153) Backed out changeset 3294c6e7ac26 (bug 1527149) Backed out changeset 3624a9aa806e (bug 1527149) Backed out changeset a8f45fe8d265 (bug 1526827) Backed out changeset 4b473920911b (bug 1526746) Backed out changeset 0185b1ebd9d9 (bug 1527143) Backed out changeset de035411df51 (bug 1527139) Backed out changeset 8aaea6655329 (bug 1526893) Backed out changeset e65136aedf28 (bug 1526730) Backed out changeset 5ebf7fe450aa (bug 1526730) Backed out changeset 31c2d8270ec1 (bug 1526643) Backed out changeset 20c1c690147b (bug 1526644) Backed out changeset ad78e25fa8ee (bug 1526644) Backed out changeset 5af6492c193d (bug 1526728) Backed out changeset 6653907d02d6 (bug 1526728) Backed out changeset 742cfa6b6a4f (bug 1527046) Backed out changeset 1774580635e0 (bug 1527046) Backed out changeset e6fa4f764ee3 (bug 1527044) Backed out changeset b9e8c0189449 (bug 1527044) Backed out changeset 88eaa5938f41 (bug 1527042) Backed out changeset 2d76ea713170 (bug 1527042) Backed out changeset 0a4e51667fbf (bug 1526852) Backed out changeset e509572cb5f9 (bug 1526852) Backed out changeset bbed44007e47 (bug 1526767) Backed out changeset 270a208a65fa (bug 1526739) Backed out changeset 5255301763d0 (bug 1526860) Backed out changeset d55f34dc2515 (bug 1526849) Backed out changeset 24c35c053f86 (bug 1526848) Backed out changeset 790dd37ed6d8 (bug 1526831) Backed out changeset be0f5bc2d2c0 (bug 1526839) Backed out changeset 1de44fe83358 (bug 1488208) Backed out changeset a8e5651e3fe2 (bug 1526627) Backed out changeset 9b419751710b (bug 1526627) Backed out changeset 9eeef2cb714f (bug 1526626) Backed out changeset b3909a782c84 (bug 1526625) Backed out changeset edf27a067f59 (bug 1526625) Backed out changeset 508a73466815 (bug 1526623) Backed out changeset 50987fe94dc4 (bug 1516736) Backed out changeset 8d1357f4a0e8 (bug 1516736) Backed out changeset cb06a5a9b130 (bug 1504391) Backed out changeset 0cc2cbc16dda (bug 1526621) Backed out changeset fef6b005a837 (bug 1526620) Backed out changeset 55f8f5e09d25 (bug 1526619) Backed out changeset 1a8d89e21632 (bug 1526617) Backed out changeset 2f27bcbec486 (bug 1526617) Backed out changeset 57f0ec9c5142 (bug 1526616) Backed out changeset dcf8e575dc67 (bug 1526616) Backed out changeset d128a11e3f65 (bug 1526264) Backed out changeset 1eb176f1d2b7 (bug 1526612) Backed out changeset 0cde19c45133 (bug 1526612) Backed out changeset e2346048d1dc (bug 1526611) Backed out changeset 963cd1e41204 (bug 1526610) Backed out changeset 25ce8e1fb425 (bug 1526608) Backed out changeset 6513512dcbb9 (bug 1526608) Backed out changeset 7e79631753f0 (bug 1526606) Backed out changeset 7b74aa781fdb (bug 1526604) Backed out changeset be67fead7966 (bug 1526603) Backed out changeset ce9b01ac64aa (bug 1526603) Backed out changeset d51d6ded843f (bug 1526601) Backed out changeset 6bb3662762ec (bug 1526270) Backed out changeset 7f235380b022 (bug 1526270) Backed out changeset c10b10b9fcc1 (bug 1526599) Backed out changeset 5070af3aaa0b (bug 1526599) Backed out changeset 7348bd11befd (bug 973341) Backed out changeset d2ff85c428aa (bug 1526598) Backed out changeset 7fd619e9d874 (bug 1526596) Backed out changeset a3f2954c055c (bug 1526596) Backed out changeset 9bc8a95c51a5 (bug 1526595) Backed out changeset 4cb5caf1eefa (bug 1526595) Backed out changeset 21f3cd50bec5 (bug 1526594) Backed out changeset 97ef7717938f (bug 1526594) Backed out changeset 81fcc5ce16ec (bug 1526592) Backed out changeset 8475d3df7a35 (bug 1526592) Backed out changeset 4710c9e39a64 (bug 1526587) Backed out changeset fa56083af140 (bug 1526585) Backed out changeset 855b3f20c070 (bug 1526583) Backed out changeset d84c9dfa9d2c (bug 1526581) Backed out changeset 471b198ee5f9 (bug 1526580)

import json
import os
import socket
import threading
import time
import traceback
import urlparse
import uuid

from .base import (CallbackHandler,
                   RefTestExecutor,
                   RefTestImplementation,
                   TestharnessExecutor,
                   extra_timeout,
                   strip_server)
from .protocol import (BaseProtocolPart,
                       TestharnessProtocolPart,
                       Protocol,
                       SelectorProtocolPart,
                       ClickProtocolPart,
                       SendKeysProtocolPart,
                       ActionSequenceProtocolPart,
                       TestDriverProtocolPart)
from ..testrunner import Stop

here = os.path.join(os.path.split(__file__)[0])

webdriver = None
exceptions = None
RemoteConnection = None
Command = None


def do_delayed_imports():
    global webdriver
    global exceptions
    global RemoteConnection
    global Command
    from selenium import webdriver
    from selenium.common import exceptions
    from selenium.webdriver.remote.remote_connection import RemoteConnection
    from selenium.webdriver.remote.command import Command


class SeleniumBaseProtocolPart(BaseProtocolPart):
    def setup(self):
        self.webdriver = self.parent.webdriver

    def execute_script(self, script, async=False):
        method = self.webdriver.execute_async_script if async else self.webdriver.execute_script
        return method(script)

    def set_timeout(self, timeout):
        self.webdriver.set_script_timeout(timeout * 1000)

    @property
    def current_window(self):
        return self.webdriver.current_window_handle

    def set_window(self, handle):
        self.webdriver.switch_to_window(handle)

    def wait(self):
        while True:
            try:
                self.webdriver.execute_async_script("")
            except exceptions.TimeoutException:
                pass
            except (socket.timeout, exceptions.NoSuchWindowException,
                    exceptions.ErrorInResponseException, IOError):
                break
            except Exception as e:
                self.logger.error(traceback.format_exc(e))
                break


class SeleniumTestharnessProtocolPart(TestharnessProtocolPart):
    def setup(self):
        self.webdriver = self.parent.webdriver
        self.runner_handle = None
        with open(os.path.join(here, "runner.js")) as f:
            self.runner_script = f.read()

    def load_runner(self, url_protocol):
        if self.runner_handle:
            self.webdriver.switch_to_window(self.runner_handle)
        url = urlparse.urljoin(self.parent.executor.server_url(url_protocol),
                               "/testharness_runner.html")
        self.logger.debug("Loading %s" % url)
        self.webdriver.get(url)
        self.runner_handle = self.webdriver.current_window_handle
        format_map = {"title": threading.current_thread().name.replace("'", '"')}
        self.parent.base.execute_script(self.runner_script % format_map)

    def close_old_windows(self):
        handles = [item for item in self.webdriver.window_handles if item != self.runner_handle]
        for handle in handles:
            try:
                self.webdriver.switch_to_window(handle)
                self.webdriver.close()
            except exceptions.NoSuchWindowException:
                pass
        self.webdriver.switch_to_window(self.runner_handle)
        return self.runner_handle

    def get_test_window(self, window_id, parent, timeout=5):
        """Find the test window amongst all the open windows.
        This is assumed to be either the named window or the one after the parent in the list of
        window handles

        :param window_id: The DOM name of the Window
        :param parent: The handle of the runner window
        :param timeout: The time in seconds to wait for the window to appear. This is because in
                        some implementations there's a race between calling window.open and the
                        window being added to the list of WebDriver accessible windows."""
        test_window = None
        end_time = time.time() + timeout
        while time.time() < end_time:
            try:
                # Try using the JSON serialization of the WindowProxy object,
                # it's in Level 1 but nothing supports it yet
                win_s = self.webdriver.execute_script("return window['%s'];" % window_id)
                win_obj = json.loads(win_s)
                test_window = win_obj["window-fcc6-11e5-b4f8-330a88ab9d7f"]
            except Exception:
                pass

            if test_window is None:
                after = self.webdriver.window_handles
                if len(after) == 2:
                    test_window = next(iter(set(after) - set([parent])))
                elif after[0] == parent and len(after) > 2:
                    # Hope the first one here is the test window
                    test_window = after[1]

            if test_window is not None:
                assert test_window != parent
                return test_window

            time.sleep(0.1)

        raise Exception("unable to find test window")


class SeleniumSelectorProtocolPart(SelectorProtocolPart):
    def setup(self):
        self.webdriver = self.parent.webdriver

    def elements_by_selector(self, selector):
        return self.webdriver.find_elements_by_css_selector(selector)


class SeleniumClickProtocolPart(ClickProtocolPart):
    def setup(self):
        self.webdriver = self.parent.webdriver

    def element(self, element):
        return element.click()


class SeleniumSendKeysProtocolPart(SendKeysProtocolPart):
    def setup(self):
        self.webdriver = self.parent.webdriver

    def send_keys(self, element, keys):
        return element.send_keys(keys)


class SeleniumActionSequenceProtocolPart(ActionSequenceProtocolPart):
    def setup(self):
        self.webdriver = self.parent.webdriver

    def send_actions(self, actions):
        self.webdriver.execute(Command.W3C_ACTIONS, {"actions": actions})


class SeleniumTestDriverProtocolPart(TestDriverProtocolPart):
    def setup(self):
        self.webdriver = self.parent.webdriver

    def send_message(self, message_type, status, message=None):
        obj = {
            "type": "testdriver-%s" % str(message_type),
            "status": str(status)
        }
        if message:
            obj["message"] = str(message)
        self.webdriver.execute_script("window.postMessage(%s, '*')" % json.dumps(obj))


class SeleniumProtocol(Protocol):
    implements = [SeleniumBaseProtocolPart,
                  SeleniumTestharnessProtocolPart,
                  SeleniumSelectorProtocolPart,
                  SeleniumClickProtocolPart,
                  SeleniumSendKeysProtocolPart,
                  SeleniumTestDriverProtocolPart,
                  SeleniumActionSequenceProtocolPart]

    def __init__(self, executor, browser, capabilities, **kwargs):
        do_delayed_imports()

        super(SeleniumProtocol, self).__init__(executor, browser)
        self.capabilities = capabilities
        self.url = browser.webdriver_url
        self.webdriver = None

    def connect(self):
        """Connect to browser via Selenium's WebDriver implementation."""
        self.logger.debug("Connecting to Selenium on URL: %s" % self.url)

        self.webdriver = webdriver.Remote(command_executor=RemoteConnection(self.url.strip("/"),
                                                                            resolve_ip=False),
                                          desired_capabilities=self.capabilities)

    def after_conect(self):
        pass

    def teardown(self):
        self.logger.debug("Hanging up on Selenium session")
        try:
            self.webdriver.quit()
        except Exception:
            pass
        del self.webdriver

    def is_alive(self):
        try:
            # Get a simple property over the connection
            self.webdriver.current_window_handle
        # TODO what exception?
        except (socket.timeout, exceptions.ErrorInResponseException):
            return False
        return True

    def after_connect(self):
        self.testharness.load_runner(self.executor.last_environment["protocol"])


class SeleniumRun(object):
    def __init__(self, func, protocol, url, timeout):
        self.func = func
        self.result = None
        self.protocol = protocol
        self.url = url
        self.timeout = timeout
        self.result_flag = threading.Event()

    def run(self):
        timeout = self.timeout

        try:
            self.protocol.base.set_timeout((timeout + extra_timeout))
        except exceptions.ErrorInResponseException:
            self.logger.error("Lost WebDriver connection")
            return Stop

        executor = threading.Thread(target=self._run)
        executor.start()

        flag = self.result_flag.wait(timeout + 2 * extra_timeout)
        if self.result is None:
            if flag:
                # flag is True unless we timeout; this *shouldn't* happen, but
                # it can if self._run fails to set self.result due to raising
                self.result = False, ("INTERNAL-ERROR", "self._run didn't set a result")
            else:
                self.result = False, ("EXTERNAL-TIMEOUT", None)

        return self.result

    def _run(self):
        try:
            self.result = True, self.func(self.protocol, self.url, self.timeout)
        except exceptions.TimeoutException:
            self.result = False, ("EXTERNAL-TIMEOUT", None)
        except (socket.timeout, exceptions.ErrorInResponseException):
            self.result = False, ("CRASH", None)
        except Exception as e:
            message = str(getattr(e, "message", ""))
            if message:
                message += "\n"
            message += traceback.format_exc(e)
            self.result = False, ("INTERNAL-ERROR", message)
        finally:
            self.result_flag.set()


class SeleniumTestharnessExecutor(TestharnessExecutor):
    supports_testdriver = True

    def __init__(self, browser, server_config, timeout_multiplier=1,
                 close_after_done=True, capabilities=None, debug_info=None,
                 **kwargs):
        """Selenium-based executor for testharness.js tests"""
        TestharnessExecutor.__init__(self, browser, server_config,
                                     timeout_multiplier=timeout_multiplier,
                                     debug_info=debug_info)
        self.protocol = SeleniumProtocol(self, browser, capabilities)
        with open(os.path.join(here, "testharness_webdriver_resume.js")) as f:
            self.script_resume = f.read()
        self.close_after_done = close_after_done
        self.window_id = str(uuid.uuid4())

    def is_alive(self):
        return self.protocol.is_alive()

    def on_environment_change(self, new_environment):
        if new_environment["protocol"] != self.last_environment["protocol"]:
            self.protocol.testharness.load_runner(new_environment["protocol"])

    def do_test(self, test):
        url = self.test_url(test)

        success, data = SeleniumRun(self.do_testharness,
                                    self.protocol,
                                    url,
                                    test.timeout * self.timeout_multiplier).run()

        if success:
            return self.convert_result(test, data)

        return (test.result_cls(*data), [])

    def do_testharness(self, protocol, url, timeout):
        format_map = {"url": strip_server(url)}

        parent_window = protocol.testharness.close_old_windows()
        # Now start the test harness
        protocol.base.execute_script("window.open('about:blank', '%s', 'noopener')" % self.window_id)
        test_window = protocol.testharness.get_test_window(self.window_id, parent_window,
                                                           timeout=5*self.timeout_multiplier)
        self.protocol.base.set_window(test_window)
        protocol.webdriver.get(url)
        handler = CallbackHandler(self.logger, protocol, test_window)
        while True:
            result = protocol.base.execute_script(
                self.script_resume % format_map, async=True)
            done, rv = handler(result)
            if done:
                break
        return rv


class SeleniumRefTestExecutor(RefTestExecutor):
    def __init__(self, browser, server_config, timeout_multiplier=1,
                 screenshot_cache=None, close_after_done=True,
                 debug_info=None, capabilities=None, **kwargs):
        """Selenium WebDriver-based executor for reftests"""
        RefTestExecutor.__init__(self,
                                 browser,
                                 server_config,
                                 screenshot_cache=screenshot_cache,
                                 timeout_multiplier=timeout_multiplier,
                                 debug_info=debug_info)
        self.protocol = SeleniumProtocol(self, browser,
                                         capabilities=capabilities)
        self.implementation = RefTestImplementation(self)
        self.close_after_done = close_after_done
        self.has_window = False

        with open(os.path.join(here, "reftest-wait_webdriver.js")) as f:
            self.wait_script = f.read()

    def is_alive(self):
        return self.protocol.is_alive()

    def do_test(self, test):
        self.logger.info("Test requires OS-level window focus")

        width_offset, height_offset = self.protocol.webdriver.execute_script(
            """return [window.outerWidth - window.innerWidth,
                       window.outerHeight - window.innerHeight];"""
        )
        self.protocol.webdriver.set_window_size(600 + width_offset, 600 + height_offset)

        result = self.implementation.run_test(test)

        return self.convert_result(test, result)

    def screenshot(self, test, viewport_size, dpi):
        # https://github.com/w3c/wptrunner/issues/166
        assert viewport_size is None
        assert dpi is None

        return SeleniumRun(self._screenshot,
                           self.protocol,
                           self.test_url(test),
                           test.timeout).run()

    def _screenshot(self, protocol, url, timeout):
        webdriver = protocol.webdriver
        webdriver.get(url)

        webdriver.execute_async_script(self.wait_script)

        screenshot = webdriver.get_screenshot_as_base64()

        # strip off the data:img/png, part of the url
        if screenshot.startswith("data:image/png;base64,"):
            screenshot = screenshot.split(",", 1)[1]

        return screenshot