testing/web-platform/tests/.azure-pipelines.yml
author Philip Jägenstedt <philip@foolip.org>
Thu, 18 Apr 2019 11:56:44 +0000
changeset 471015 bcdf27a511f57c52d34e19f63c81973a4c580e3f
parent 470938 00de5d349a10b60a20c1ac999fc5c755cf438c36
child 471032 46d4aa839e751424963a9c5caada61074af62a01
permissions -rw-r--r--
Bug 1540252 [wpt PR 16107] - Run full Edge runs every 6 hours, a=testonly Automatic update from web-platform-tests Run full Edge runs every 6 hours (#16107) This will end up in the same build as the Safari TP run. The wpt.fyi integration should ensure that one failing doesn't affect the other, but there is some risk here and it will need to be monitored. -- wpt-commits: b53840ae8e05244f694a32d411251c009c249a26 wpt-pr: 16107

# This is the configuration file for Azure Pipelines, used to run tests on
# macOS. Documentation to help understand this setup:
# https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema
# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/multiple-phases
# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/templates
# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables
# https://docs.microsoft.com/en-us/azure/devops/pipelines/tasks/index
#
# In addition to this configuration file, some setup in the Azure DevOps
# project is required:
#  - The "Build pull requests from forks of this repository" setting must be
#    enabled: https://docs.microsoft.com/en-us/azure/devops/pipelines/repos/github#validate-contributions-from-forks
#  - A scheduled build needs to be set up for one of the epochs/* branches.
#  - Self-hosted agents for Windows 10 are used:
#    - 'Hosted Windows Client' is the latest Windows 10
#    - 'Hosted Windows Client Next' is Windows 10 Insider Preview
#    Documention for the setup of these agents:
#    https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/v2-windows

jobs:
# The affected tests jobs are unconditional for speed, as most PRs have one or
# more affected tests: https://github.com/web-platform-tests/wpt/issues/13936.
- job: affected_safari_preview
  displayName: 'affected tests (Safari Technology Preview)'
  condition: eq(variables['Build.Reason'], 'PullRequest')
  pool:
    vmImage: 'macOS-10.13'
  steps:
  - template: tools/ci/azure/affected_tests.yml
    parameters:
      artifactName: 'safari-preview-affected-tests'
- template: tools/ci/azure/fyi_hook.yml
  parameters:
    dependsOn: affected_safari_preview
    artifactName: safari-preview-affected-tests

- job: affected_without_changes_safari_preview
  displayName: 'affected tests without changes (Safari Technology Preview)'
  condition: eq(variables['Build.Reason'], 'PullRequest')
  pool:
    vmImage: 'macOS-10.13'
  steps:
  - template: tools/ci/azure/affected_tests.yml
    parameters:
      checkoutCommit: 'HEAD^1'
      affectedRange: 'HEAD@{1}'
      artifactName: 'safari-preview-affected-tests-without-changes'
- template: tools/ci/azure/fyi_hook.yml
  parameters:
    dependsOn: affected_without_changes_safari_preview
    artifactName: safari-preview-affected-tests-without-changes

# The decision jobs runs `./wpt test-jobs` to determine which jobs to run,
# and all following jobs wait for it to finish and depend on its output.
- job: decision
  displayName: './wpt test-jobs'
  condition: eq(variables['Build.Reason'], 'PullRequest')
  pool:
    vmImage: 'ubuntu-16.04'
  steps:
  - template: tools/ci/azure/checkout.yml
  - script: |
      ./wpt test-jobs | while read job; do
        echo "$job"
        echo "##vso[task.setvariable variable=$job;isOutput=true]true";
      done
    name: test_jobs
    displayName: 'Run ./wpt test-jobs'

- job: infrastructure_macOS
  displayName: 'infrastructure/ tests (macOS)'
  dependsOn: decision
  condition: dependencies.decision.outputs['test_jobs.wptrunner_infrastructure']
  pool:
    vmImage: 'macOS-10.13'
  steps:
  - template: tools/ci/azure/checkout.yml
  - template: tools/ci/azure/pip_install.yml
    parameters:
      packages: virtualenv
  - template: tools/ci/azure/install_fonts.yml
  - template: tools/ci/azure/install_certs.yml
  - template: tools/ci/azure/install_chrome.yml
  - template: tools/ci/azure/install_firefox.yml
  - template: tools/ci/azure/install_safari.yml
  - template: tools/ci/azure/update_hosts.yml
  - template: tools/ci/azure/update_manifest.yml
  - script: no_proxy='*' ./wpt run --yes --no-manifest-update --manifest MANIFEST.json --metadata infrastructure/metadata/ --log-tbpl $(Build.ArtifactStagingDirectory)/chrome.tbpl.log --log-tbpl-level info --channel dev chrome infrastructure/
    displayName: 'Run tests (Chrome Dev)'
  - script: no_proxy='*' ./wpt run --yes --no-manifest-update --manifest MANIFEST.json --metadata infrastructure/metadata/ --log-tbpl $(Build.ArtifactStagingDirectory)/firefox.tbpl.log --log-tbpl-level info --channel nightly firefox infrastructure/
    displayName: 'Run tests (Firefox Nightly)'
  - script: no_proxy='*' ./wpt run --yes --no-manifest-update --manifest MANIFEST.json --metadata infrastructure/metadata/ --log-tbpl $(Build.ArtifactStagingDirectory)/safari.tbpl.log --log-tbpl-level info --channel preview safari infrastructure/
    displayName: 'Run tests (Safari Technology Preview)'
  - task: PublishBuildArtifacts@1
    displayName: 'Publish results'
    inputs:
      artifactName: 'infrastructure'
    condition: always()

- job: tools_unittest_macOS
  displayName: 'tools/ unittests (macOS)'
  dependsOn: decision
  condition: dependencies.decision.outputs['test_jobs.tools_unittest']
  pool:
    vmImage: 'macOS-10.13'
  steps:
  - template: tools/ci/azure/checkout.yml
  - template: tools/ci/azure/tox_pytest.yml
    parameters:
      directory: tools/
      toxenv: py27

- job: wptrunner_unittest_macOS
  displayName: 'tools/wptrunner/ unittests (macOS)'
  dependsOn: decision
  condition: dependencies.decision.outputs['test_jobs.wptrunner_unittest']
  pool:
    vmImage: 'macOS-10.13'
  steps:
  - template: tools/ci/azure/checkout.yml
  - template: tools/ci/azure/tox_pytest.yml
    parameters:
      directory: tools/wptrunner/

- job: wpt_integration_macOS
  displayName: 'tools/wpt/ tests (macOS)'
  dependsOn: decision
  condition: dependencies.decision.outputs['test_jobs.wpt_integration']
  pool:
    vmImage: 'macOS-10.13'
  steps:
  # full checkout required
  - template: tools/ci/azure/install_chrome.yml
  - template: tools/ci/azure/install_firefox.yml
  - template: tools/ci/azure/update_hosts.yml
  - template: tools/ci/azure/update_manifest.yml
  - template: tools/ci/azure/tox_pytest.yml
    parameters:
      directory: tools/wpt/

- job: infrastructure_win10
  displayName: 'infrastructure/ tests (Windows 10)'
  # This job is only triggered manually until it has been shown to be robust.
  condition: and(eq(variables['Build.Reason'], 'Manual'), variables['run_infrastructure_win10'])
  pool:
    name: 'Hosted Windows Client'
  steps:
  - template: tools/ci/azure/system_info.yml
  - template: tools/ci/azure/checkout.yml
  - template: tools/ci/azure/install_python.yml
  - template: tools/ci/azure/pip_install.yml
    parameters:
      packages: virtualenv
  - template: tools/ci/azure/install_certs.yml
  - template: tools/ci/azure/update_hosts.yml
  - template: tools/ci/azure/update_manifest.yml
  - script: python ./wpt run --yes --no-manifest-update --install-fonts --manifest MANIFEST.json --metadata infrastructure/metadata/ --webdriver-arg=--verbose --log-tbpl $(Build.ArtifactStagingDirectory)/edge.tbpl.log --log-tbpl-level info edge_webdriver infrastructure/
    displayName: 'Run tests (Edge)'
  - task: PublishBuildArtifacts@1
    displayName: 'Publish results'
    inputs:
      artifactName: 'infrastructure'
    condition: always()
  - template: tools/ci/azure/cleanup_win10.yml

# All `./wpt run` tests are run from epochs/* branches on a schedule. See
# documentation at the top of this file for required setup.
- job: results_edge
  displayName: 'all tests (Edge)'
  condition: |
    or(eq(variables['Build.Reason'], 'Schedule'),
       and(eq(variables['Build.Reason'], 'Manual'), variables['run_all_edge']))
  # There are 12 agents in the pool, but use more jobs so that each takes <1h.
  strategy:
    parallel: 20
  timeoutInMinutes: 360
  pool:
    name: 'Hosted Windows Client'
  steps:
  - template: tools/ci/azure/system_info.yml
  - template: tools/ci/azure/checkout.yml
  - template: tools/ci/azure/install_python.yml
  - template: tools/ci/azure/pip_install.yml
    parameters:
      packages: virtualenv
  - template: tools/ci/azure/install_certs.yml
  - template: tools/ci/azure/update_hosts.yml
  - template: tools/ci/azure/update_manifest.yml
  - script: python ./wpt run --no-manifest-update --no-fail-on-unexpected --install-fonts --webdriver-arg=--verbose --test-types reftest testharness --this-chunk $(System.JobPositionInPhase) --total-chunks $(System.TotalJobsInPhase) --chunk-type hash --log-tbpl - --log-tbpl-level info --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report_$(System.JobPositionInPhase).json --log-wptscreenshot $(Build.ArtifactStagingDirectory)/wpt_screenshot_$(System.JobPositionInPhase).txt edge_webdriver
    displayName: 'Run tests'
  - task: PublishBuildArtifacts@1
    displayName: 'Publish results'
    inputs:
      artifactName: 'edge-results'
  - template: tools/ci/azure/cleanup_win10.yml
- template: tools/ci/azure/fyi_hook.yml
  parameters:
    dependsOn: results_edge
    artifactName: edge-results

- job: results_safari_preview
  displayName: 'all tests (Safari Technology Preview)'
  condition: |
    or(eq(variables['Build.Reason'], 'Schedule'),
       and(eq(variables['Build.Reason'], 'Manual'), variables['run_all_safari_preview']))
  strategy:
    parallel: 4 # chosen to make runtime ~2h
  timeoutInMinutes: 360
  pool:
    vmImage: 'macOS-10.13'
  steps:
  - template: tools/ci/azure/checkout.yml
  - template: tools/ci/azure/pip_install.yml
    parameters:
      packages: virtualenv
  - template: tools/ci/azure/install_fonts.yml
  - template: tools/ci/azure/install_certs.yml
  - template: tools/ci/azure/install_safari.yml
  - template: tools/ci/azure/update_hosts.yml
  - template: tools/ci/azure/update_manifest.yml
  - script: no_proxy='*' ./wpt run --no-manifest-update --no-restart-on-unexpected --no-fail-on-unexpected --this-chunk=$(System.JobPositionInPhase) --total-chunks=$(System.TotalJobsInPhase) --chunk-type hash --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report_$(System.JobPositionInPhase).json --log-wptscreenshot $(Build.ArtifactStagingDirectory)/wpt_screenshot_$(System.JobPositionInPhase).txt --channel preview safari
    displayName: 'Run tests'
  - task: PublishBuildArtifacts@1
    displayName: 'Publish results'
    inputs:
      artifactName: 'safari-preview-results'
- template: tools/ci/azure/fyi_hook.yml
  parameters:
    dependsOn: results_safari_preview
    artifactName: safari-preview-results