Bug 1355009 - Harden update tests with better error messages. r=ato, a=test-only
authorHenrik Skupin <mail@hskupin.info>
Tue, 11 Apr 2017 13:42:44 +0200
changeset 393494 431b2f377dc05ec1add04153acd545e2e39f6887
parent 393493 e0f7bd5b58572307e2c45ba3cb41b4f3b804bfac
child 393495 a5069d5ac7328df53705a59d4db33941bc213622
push id7198
push userjlorenzo@mozilla.com
push dateTue, 18 Apr 2017 12:07:49 +0000
treeherdermozilla-beta@d57aa49c3948 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato, test-only
bugs1355009, 1353717
milestone54.0a2
Bug 1355009 - Harden update tests with better error messages. r=ato, a=test-only To ensure better failure messages a refactoring of checks has to be done. It includes the following changes: * No further checks for a follow-up (watershed) update. It's not supported and as such doesn't need assertions (bug 1353717) * Checks for fallback updates have to be made to ensure that an invalidated partial/complete update does not cause an upgrade of Firefox during the restart. MozReview-Commit-ID: CLb0aXoIur2
testing/firefox-ui/harness/firefox_ui_harness/testcases.py
--- a/testing/firefox-ui/harness/firefox_ui_harness/testcases.py
+++ b/testing/firefox-ui/harness/firefox_ui_harness/testcases.py
@@ -131,60 +131,90 @@ class UpdateTestCase(PuppeteerMixin, Mar
             message='Check for updates has been finished.')
 
         return about_window.deck.selected_panel != about_window.deck.no_updates_found
 
     def check_update_applied(self):
         """Check that the update has been applied correctly"""
         self.update_status['build_post'] = self.software_update.build_info
 
-        about_window = self.browser.open_about_window()
-        try:
-            # Bug 604364 - We do not support watershed releases yet.
-            update_available = self.check_for_updates(about_window)
-            self.assertFalse(update_available,
-                             'Additional update found due to watershed release {}'.format(
-                                 self.update_status['build_post']['version']))
+        # Ensure that the target version is the same or higher. No downgrade
+        # should have happened.
+        version_check = self.marionette.execute_script("""
+          Components.utils.import("resource://gre/modules/Services.jsm");
+
+          return Services.vc.compare(arguments[0], arguments[1]);
+        """, script_args=(self.update_status['build_post']['version'],
+                          self.update_status['build_pre']['version']))
+
+        self.assertGreaterEqual(version_check, 0,
+                                'A downgrade from version {} to {} is not allowed'.format(
+                                    self.update_status['build_pre']['version'],
+                                    self.update_status['build_post']['version']))
 
-            # The upgraded version should be identical with the version given by
-            # the update and we shouldn't have run a downgrade
-            check = self.marionette.execute_script("""
-              Components.utils.import("resource://gre/modules/Services.jsm");
+        self.assertNotEqual(self.update_status['build_post']['buildid'],
+                            self.update_status['build_pre']['buildid'],
+                            'The staged update to buildid {} has not been applied'.format(
+                                self.update_status['patch']['buildid']))
 
-              return  Services.vc.compare(arguments[0], arguments[1]);
-            """, script_args=[self.update_status['build_post']['version'],
-                              self.update_status['build_pre']['version']])
+        self.assertEqual(self.update_status['build_post']['buildid'],
+                         self.update_status['patch']['buildid'],
+                         'Unexpected target buildid after applying the patch, {} != {}'.format(
+                             self.update_status['build_post']['buildid'],
+                             self.update_status['patch']['buildid']))
 
-            self.assertGreaterEqual(check, 0,
-                                    'The version of the upgraded build is higher or equal')
+        self.assertEqual(self.update_status['build_post']['locale'],
+                         self.update_status['build_pre']['locale'],
+                         'Unexpected change of the locale from {} to {}'.format(
+                             self.update_status['build_pre']['locale'],
+                             self.update_status['build_post']['locale']))
 
-            # If a target version has been specified, check if it matches the updated build
-            if self.target_version:
-                self.assertEqual(self.update_status['build_post']['version'], self.target_version)
+        self.assertEqual(self.update_status['build_post']['disabled_addons'],
+                         self.update_status['build_pre']['disabled_addons'],
+                         'Application-wide addons have been unexpectedly disabled: {}'.format(
+                             ', '.join(set(self.update_status['build_pre']['locale']) -
+                                       set(self.update_status['build_post']['locale']))
+        ))
 
-            # The post buildid should be identical with the buildid contained in the patch
+        if self.target_version:
+            self.assertEqual(self.update_status['build_post']['version'],
+                             self.target_version,
+                             'Current target version {} does not match expected version {}'.format(
+                                 self.update_status['build_post']['version'], self.target_version))
+
+        if self.target_buildid:
             self.assertEqual(self.update_status['build_post']['buildid'],
-                             self.update_status['patch']['buildid'])
+                             self.target_buildid,
+                             'Current target buildid {} does not match expected buildid {}'.format(
+                                 self.update_status['build_post']['buildid'], self.target_buildid))
 
-            # If a target buildid has been specified, check if it matches the updated build
-            if self.target_buildid:
-                self.assertEqual(self.update_status['build_post']['buildid'], self.target_buildid)
+        self.update_status['success'] = True
 
-            # An upgrade should not change the builds locale
-            self.assertEqual(self.update_status['build_post']['locale'],
-                             self.update_status['build_pre']['locale'])
+    def check_update_not_applied(self):
+        """Check that the update has not been applied due to a forced invalidation of the patch"""
+        build_info = self.software_update.build_info
+
+        # Ensure that the version has not been changed
+        version_check = self.marionette.execute_script("""
+          Components.utils.import("resource://gre/modules/Services.jsm");
 
-            # Check that no application-wide add-ons have been disabled
-            self.assertEqual(self.update_status['build_post']['disabled_addons'],
-                             self.update_status['build_pre']['disabled_addons'])
+          return Services.vc.compare(arguments[0], arguments[1]);
+        """, script_args=(build_info['version'],
+                          self.update_status['build_pre']['version']))
 
-            self.update_status['success'] = True
+        self.assertEqual(version_check, 0,
+                         'An update from version {} to {} has been unexpectedly applied'.format(
+                             self.update_status['build_pre']['version'],
+                             build_info['version']))
 
-        finally:
-            about_window.close()
+        # Check that the build id of the source build and the current build are identical
+        self.assertEqual(build_info['buildid'],
+                         self.update_status['build_pre']['buildid'],
+                         'The build id has been unexpectedly changed from {} to {}'.format(
+                             self.update_status['build_pre']['buildid'], build_info['buildid']))
 
     def download_update(self, window, wait_for_finish=True, timeout=TIMEOUT_UPDATE_DOWNLOAD):
         """ Download the update patch.
 
         :param window: Instance of :class:`AboutWindow` or :class:`UpdateWizardDialog`.
         :param wait_for_finish: If True the function has to wait for the download to be finished.
          Optional, default to `True`.
         :param timeout: How long to wait for the download to finish. Optional, default to 360s.
@@ -280,16 +310,18 @@ class UpdateTestCase(PuppeteerMixin, Mar
         if force_fallback:
             # Set the downloaded update into failed state
             self.software_update.force_fallback()
 
         # Restart Firefox to apply the downloaded update
         self.restart()
 
     def download_and_apply_forced_update(self):
+        self.check_update_not_applied()
+
         # The update wizard dialog opens automatically after the restart but with a short delay
         dialog = Wait(self.marionette, ignored_exceptions=[NoSuchWindowException]).until(
             lambda _: self.puppeteer.windows.switch_to(lambda win: type(win) is UpdateWizardDialog)
         )
 
         # In case of a broken complete update the about window has to be used
         if self.update_status['patch']['is_complete']:
             about_window = None