Bug 1540284 [wpt PR 16108] - [Azure Pipelines] Run scheduled runs of Safari stable, a=testonly
☠☠ backed out by 41b4f50eae55 ☠ ☠
authorPhilip Jägenstedt <philip@foolip.org>
Thu, 18 Apr 2019 11:57:52 +0000
changeset 470902 47d97703ac7403a44b4bcf02d79bbaebe8aae260
parent 470901 1814feb1d3d9b61f4bbf198dd969821f2db83e45
child 470903 23a225d2d306b6eaa5d2f6a044dc58eb0ff75b1e
push id35911
push usercbrindusan@mozilla.com
push dateWed, 24 Apr 2019 21:55:25 +0000
treeherdermozilla-central@c7a9affeb604 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1540284, 16108
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1540284 [wpt PR 16108] - [Azure Pipelines] Run scheduled runs of Safari stable, a=testonly Automatic update from web-platform-tests [Azure Pipelines] Run daily run of Safari stable (#16108) Rather than adding support for getting the version of Safari stable using PlistBuddy in tools/wpt/browser.py, pass in the version with `--browser-version` and make sure that it is not overwritten internally. The next stable Safari release should support `safaridriver --version`. -- wpt-commits: 146c7368173ea5336d39049788d993f93774eca6 wpt-pr: 16108
testing/web-platform/tests/.azure-pipelines.yml
testing/web-platform/tests/tools/ci/azure/install_safari.yml
testing/web-platform/tests/tools/wpt/run.py
--- a/testing/web-platform/tests/.azure-pipelines.yml
+++ b/testing/web-platform/tests/.azure-pipelines.yml
@@ -193,16 +193,50 @@ jobs:
     inputs:
       artifactName: 'edge-results'
   - template: tools/ci/azure/cleanup_win10.yml
 - template: tools/ci/azure/fyi_hook.yml
   parameters:
     dependsOn: results_edge
     artifactName: edge-results
 
+- job: results_safari
+  displayName: 'all tests (Safari)'
+  condition: eq(variables['Build.Reason'], 'Schedule')
+  strategy:
+    parallel: 4 # chosen to make runtime ~2h
+  timeoutInMinutes: 360
+  pool:
+    vmImage: 'macOS-10.13'
+  steps:
+  - template: tools/ci/azure/checkout.yml
+  - template: tools/ci/azure/pip_install.yml
+    parameters:
+      packages: virtualenv
+  - template: tools/ci/azure/install_fonts.yml
+  - template: tools/ci/azure/install_certs.yml
+  - template: tools/ci/azure/install_safari.yml
+    parameters:
+      channel: stable
+  - template: tools/ci/azure/update_hosts.yml
+  - template: tools/ci/azure/update_manifest.yml
+  - script: |
+      # TODO: drop this when `safaridriver --version` is supported.
+      SAFARI_VERSION="$(/usr/libexec/PlistBuddy -c 'Print :CFBundleShortVersionString' /Applications/Safari.app/Contents/Info.plist) ($(/usr/libexec/PlistBuddy -c 'Print :CFBundleVersion' /Applications/Safari.app/Contents/Info.plist))"
+      no_proxy='*' ./wpt run --no-manifest-update --no-restart-on-unexpected --no-fail-on-unexpected --this-chunk=$(System.JobPositionInPhase) --total-chunks=$(System.TotalJobsInPhase) --chunk-type hash --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report_$(System.JobPositionInPhase).json --log-wptscreenshot $(Build.ArtifactStagingDirectory)/wpt_screenshot_$(System.JobPositionInPhase).txt --browser-version "$SAFARI_VERSION" safari
+    displayName: 'Run tests'
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish results'
+    inputs:
+      artifactName: 'safari-results'
+- template: tools/ci/azure/fyi_hook.yml
+  parameters:
+    dependsOn: results_safari
+    artifactName: safari-results
+
 - job: results_safari_preview
   displayName: 'all tests (Safari Technology Preview)'
   condition: |
     or(eq(variables['Build.Reason'], 'Schedule'),
        and(eq(variables['Build.Reason'], 'Manual'), variables['run_all_safari_preview']))
   strategy:
     parallel: 4 # chosen to make runtime ~2h
   timeoutInMinutes: 360
--- a/testing/web-platform/tests/tools/ci/azure/install_safari.yml
+++ b/testing/web-platform/tests/tools/ci/azure/install_safari.yml
@@ -1,9 +1,19 @@
+parameters:
+  channel: preview
+
 steps:
-- script: |
-    # This is equivalent to `Homebrew/homebrew-cask-versions/safari-technology-preview`,
-    # but the raw URL is used to bypass caching.
-    HOMEBREW_NO_AUTO_UPDATE=1 brew cask install https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/safari-technology-preview.rb
-    # https://web-platform-tests.org/running-tests/safari.html
-    sudo "/Applications/Safari Technology Preview.app/Contents/MacOS/safaridriver" --enable
-    defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
-  displayName: 'Install Safari Technology Preview'
+- ${{ if eq(parameters.channel, 'preview') }}:
+  - script: |
+      # This is equivalent to `Homebrew/homebrew-cask-versions/safari-technology-preview`,
+      # but the raw URL is used to bypass caching.
+      HOMEBREW_NO_AUTO_UPDATE=1 brew cask install https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/safari-technology-preview.rb
+      # https://web-platform-tests.org/running-tests/safari.html
+      sudo "/Applications/Safari Technology Preview.app/Contents/MacOS/safaridriver" --enable
+      defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
+    displayName: 'Install Safari Technology Preview'
+- ${{ if eq(parameters.channel, 'stable') }}:
+  - script: |
+      # https://web-platform-tests.org/running-tests/safari.html
+      sudo safaridriver --enable
+      defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
+    displayName: 'Configure Safari'
--- a/testing/web-platform/tests/tools/wpt/run.py
+++ b/testing/web-platform/tests/tools/wpt/run.py
@@ -563,18 +563,22 @@ def setup_wptrunner(venv, prompt=True, i
 
     wptcommandline.check_args(kwargs)
 
     wptrunner_path = os.path.join(wpt_root, "tools", "wptrunner")
 
     if not venv.skip_virtualenv_setup:
         venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
 
-    kwargs['browser_version'] = setup_cls.browser.version(binary=kwargs.get("binary"),
-                                                          webdriver_binary=kwargs.get("webdriver_binary"))
+    # Only update browser_version if it was not given as a command line
+    # argument, so that it can be overridden on the command line.
+    if not kwargs["browser_version"]:
+        kwargs["browser_version"] = setup_cls.browser.version(binary=kwargs.get("binary"),
+                                                              webdriver_binary=kwargs.get("webdriver_binary"))
+
     return kwargs
 
 
 def run(venv, **kwargs):
     setup_logging(kwargs)
 
     # Remove arguments that aren't passed to wptrunner
     prompt = kwargs.pop("prompt", True)