Bug 1540284 [wpt PR 16108] - [Azure Pipelines] Run scheduled runs of Safari stable, a=testonly
authorPhilip Jägenstedt <philip@foolip.org>
Thu, 18 Apr 2019 11:57:52 +0000
changeset 529958 46d4aa839e751424963a9c5caada61074af62a01
parent 529957 76b3bb91416d686f448433a71383df113ef955a9
child 529959 b31d9e60ef5ec0a11416268edd6ce93f595713cb
push id11265
push userffxbld-merge
push dateMon, 13 May 2019 10:53:39 +0000
treeherdermozilla-beta@77e0fe8dbdd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1540284, 16108
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1540284 [wpt PR 16108] - [Azure Pipelines] Run scheduled runs of Safari stable, a=testonly Automatic update from web-platform-tests [Azure Pipelines] Run daily run of Safari stable (#16108) Rather than adding support for getting the version of Safari stable using PlistBuddy in tools/wpt/browser.py, pass in the version with `--browser-version` and make sure that it is not overwritten internally. The next stable Safari release should support `safaridriver --version`. -- wpt-commits: 146c7368173ea5336d39049788d993f93774eca6 wpt-pr: 16108
testing/web-platform/tests/.azure-pipelines.yml
testing/web-platform/tests/tools/ci/azure/install_safari.yml
testing/web-platform/tests/tools/wpt/run.py
--- a/testing/web-platform/tests/.azure-pipelines.yml
+++ b/testing/web-platform/tests/.azure-pipelines.yml
@@ -193,16 +193,50 @@ jobs:
     inputs:
       artifactName: 'edge-results'
   - template: tools/ci/azure/cleanup_win10.yml
 - template: tools/ci/azure/fyi_hook.yml
   parameters:
     dependsOn: results_edge
     artifactName: edge-results
 
+- job: results_safari
+  displayName: 'all tests (Safari)'
+  condition: eq(variables['Build.Reason'], 'Schedule')
+  strategy:
+    parallel: 4 # chosen to make runtime ~2h
+  timeoutInMinutes: 360
+  pool:
+    vmImage: 'macOS-10.13'
+  steps:
+  - template: tools/ci/azure/checkout.yml
+  - template: tools/ci/azure/pip_install.yml
+    parameters:
+      packages: virtualenv
+  - template: tools/ci/azure/install_fonts.yml
+  - template: tools/ci/azure/install_certs.yml
+  - template: tools/ci/azure/install_safari.yml
+    parameters:
+      channel: stable
+  - template: tools/ci/azure/update_hosts.yml
+  - template: tools/ci/azure/update_manifest.yml
+  - script: |
+      # TODO: drop this when `safaridriver --version` is supported.
+      SAFARI_VERSION="$(/usr/libexec/PlistBuddy -c 'Print :CFBundleShortVersionString' /Applications/Safari.app/Contents/Info.plist) ($(/usr/libexec/PlistBuddy -c 'Print :CFBundleVersion' /Applications/Safari.app/Contents/Info.plist))"
+      no_proxy='*' ./wpt run --no-manifest-update --no-restart-on-unexpected --no-fail-on-unexpected --this-chunk=$(System.JobPositionInPhase) --total-chunks=$(System.TotalJobsInPhase) --chunk-type hash --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report_$(System.JobPositionInPhase).json --log-wptscreenshot $(Build.ArtifactStagingDirectory)/wpt_screenshot_$(System.JobPositionInPhase).txt --browser-version "$SAFARI_VERSION" safari
+    displayName: 'Run tests'
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish results'
+    inputs:
+      artifactName: 'safari-results'
+- template: tools/ci/azure/fyi_hook.yml
+  parameters:
+    dependsOn: results_safari
+    artifactName: safari-results
+
 - job: results_safari_preview
   displayName: 'all tests (Safari Technology Preview)'
   condition: |
     or(eq(variables['Build.Reason'], 'Schedule'),
        and(eq(variables['Build.Reason'], 'Manual'), variables['run_all_safari_preview']))
   strategy:
     parallel: 4 # chosen to make runtime ~2h
   timeoutInMinutes: 360
--- a/testing/web-platform/tests/tools/ci/azure/install_safari.yml
+++ b/testing/web-platform/tests/tools/ci/azure/install_safari.yml
@@ -1,9 +1,19 @@
+parameters:
+  channel: preview
+
 steps:
-- script: |
-    # This is equivalent to `Homebrew/homebrew-cask-versions/safari-technology-preview`,
-    # but the raw URL is used to bypass caching.
-    HOMEBREW_NO_AUTO_UPDATE=1 brew cask install https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/safari-technology-preview.rb
-    # https://web-platform-tests.org/running-tests/safari.html
-    sudo "/Applications/Safari Technology Preview.app/Contents/MacOS/safaridriver" --enable
-    defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
-  displayName: 'Install Safari Technology Preview'
+- ${{ if eq(parameters.channel, 'preview') }}:
+  - script: |
+      # This is equivalent to `Homebrew/homebrew-cask-versions/safari-technology-preview`,
+      # but the raw URL is used to bypass caching.
+      HOMEBREW_NO_AUTO_UPDATE=1 brew cask install https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/safari-technology-preview.rb
+      # https://web-platform-tests.org/running-tests/safari.html
+      sudo "/Applications/Safari Technology Preview.app/Contents/MacOS/safaridriver" --enable
+      defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
+    displayName: 'Install Safari Technology Preview'
+- ${{ if eq(parameters.channel, 'stable') }}:
+  - script: |
+      # https://web-platform-tests.org/running-tests/safari.html
+      sudo safaridriver --enable
+      defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
+    displayName: 'Configure Safari'
--- a/testing/web-platform/tests/tools/wpt/run.py
+++ b/testing/web-platform/tests/tools/wpt/run.py
@@ -563,18 +563,22 @@ def setup_wptrunner(venv, prompt=True, i
 
     wptcommandline.check_args(kwargs)
 
     wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner")
 
     if not venv.skip_virtualenv_setup:
         venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
 
-    kwargs['browser_version'] = setup_cls.browser.version(binary=kwargs.get("binary"),
-                                                          webdriver_binary=kwargs.get("webdriver_binary"))
+    # Only update browser_version if it was not given as a command line
+    # argument, so that it can be overridden on the command line.
+    if not kwargs["browser_version"]:
+        kwargs["browser_version"] = setup_cls.browser.version(binary=kwargs.get("binary"),
+                                                              webdriver_binary=kwargs.get("webdriver_binary"))
+
     return kwargs
 
 
 def run(venv, **kwargs):
     setup_logging(kwargs)
 
     # Remove arguments that aren't passed to wptrunner
     prompt = kwargs.pop("prompt", True)