Bug 1743688: Allow a moz.yaml to skip certain vendoring steps r=jewilde
authorTom Ritter <tom@mozilla.com>
Tue, 04 Jan 2022 20:53:27 +0000
changeset 603661 f3185bb3038fc36f1629a99ea104db4b9745dbfa
parent 603660 4f2c77264d2d82f42b252de82d246445963056f1
child 603662 681981a13d65dcf66cdee9eec2c7d9b990a47038
push id39115
push usermlaza@mozilla.com
push dateWed, 05 Jan 2022 04:21:59 +0000
treeherdermozilla-central@dad86d8d05b4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjewilde
bugs1743688
milestone97.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1743688: Allow a moz.yaml to skip certain vendoring steps r=jewilde Differential Revision: https://phabricator.services.mozilla.com/D129434
python/mozbuild/mozbuild/vendor/moz_yaml.py
python/mozbuild/mozbuild/vendor/vendor_manifest.py
--- a/python/mozbuild/mozbuild/vendor/moz_yaml.py
+++ b/python/mozbuild/mozbuild/vendor/moz_yaml.py
@@ -141,23 +141,34 @@ vendoring:
 
   # Repository URL to vendor from
   # eg. https://github.com/kinetiknz/nestegg
   # Any repository host can be specified here, however initially we'll only
   # support automated vendoring from selected sources.
   url: source url (generally repository clone url)
 
   # Type of hosting for the upstream repository
-  # Valid values are 'gitlab', 'github'
+  # Valid values are 'gitlab', 'github', googlesource
   source-hosting: gitlab
 
   # Base directory of the location where the source files will live in-tree.
   # If omitted, will default to the location the moz.yaml file is in.
   vendor-directory: third_party/directory
 
+  # Allows skipping certain steps of the vendoring process.
+  # Most useful if e.g. vendoring upstream is complicated and should be done by a script
+  # The valid steps that can be skipped are listed below
+  skip-vendoring-steps:
+    - fetch
+    - exclude
+    - update-moz-yaml
+    - update-actions
+    - hg-add
+    - update-moz-build
+
   # List of patch files to apply after vendoring. Applied in the order
   # specified, and alphabetically if globbing is used. Patches must apply
   # cleanly before changes are pushed
   # All patch files are implicitly added to the keep file list.
   # optional
   patches:
     - file
     - path/to/file
@@ -387,16 +398,17 @@ def _schema_1():
             },
             "vendoring": {
                 Required("url"): FqdnUrl(),
                 Required("source-hosting"): All(
                     str,
                     Length(min=1),
                     In(VALID_SOURCE_HOSTS, msg="Unsupported Source Hosting"),
                 ),
+                "skip-vendoring-steps": Unique([str]),
                 "vendor-directory": All(str, Length(min=1)),
                 "patches": Unique([str]),
                 "keep": Unique([str]),
                 "exclude": Unique([str]),
                 "include": Unique([str]),
                 "update-actions": All(
                     UpdateActions(),
                     [
--- a/python/mozbuild/mozbuild/vendor/vendor_manifest.py
+++ b/python/mozbuild/mozbuild/vendor/vendor_manifest.py
@@ -60,46 +60,75 @@ class VendorManifest(MozbuildObject):
                 {"ref_type": ref_type},
                 "Latest upstream {ref_type} matches {ref_type} in-tree. Returning.",
             )
             return
         elif check_for_update:
             print("%s %s" % (ref, timestamp))
             return
 
-        self.fetch_and_unpack(ref)
+        def perform_step(step):
+            return step not in self.manifest["vendoring"].get(
+                "skip-vendoring-steps", []
+            )
 
-        self.log(logging.INFO, "vendor", {}, "Removing unnecessary files.")
-        self.clean_upstream()
+        if perform_step("fetch"):
+            self.fetch_and_unpack(ref)
+        else:
+            self.log(logging.INFO, "vendor", {}, "Skipping fetching upstream source.")
 
-        self.log(logging.INFO, "vendor", {}, "Updating moz.yaml.")
-        self.update_yaml(yaml_file, ref, timestamp)
+        if perform_step("exclude"):
+            self.log(logging.INFO, "vendor", {}, "Removing unnecessary files.")
+            self.clean_upstream()
+        else:
+            self.log(logging.INFO, "vendor", {}, "Skipping removing excluded files.")
 
-        self.log(logging.INFO, "vendor", {}, "Updating files")
-        self.update_files(ref, yaml_file)
+        if perform_step("update-moz-yaml"):
+            self.log(logging.INFO, "vendor", {}, "Updating moz.yaml.")
+            self.update_yaml(yaml_file, ref, timestamp)
+        else:
+            self.log(logging.INFO, "vendor", {}, "Skipping updating the moz.yaml file.")
 
-        self.log(
-            logging.INFO, "vendor", {}, "Registering changes with version control."
-        )
-        self.repository.add_remove_files(
-            self.manifest["vendoring"]["vendor-directory"], os.path.dirname(yaml_file)
-        )
+        if perform_step("update-actions"):
+            self.log(logging.INFO, "vendor", {}, "Updating files")
+            self.update_files(ref, yaml_file)
+        else:
+            self.log(logging.INFO, "vendor", {}, "Skipping running the update actions.")
 
-        self.log(logging.INFO, "vendor", {}, "Updating moz.build files")
-        self.update_moz_build(
-            self.manifest["vendoring"]["vendor-directory"],
-            os.path.dirname(yaml_file),
-            add_to_exports,
-        )
+        if perform_step("hg-add"):
+            self.log(
+                logging.INFO, "vendor", {}, "Registering changes with version control."
+            )
+            self.repository.add_remove_files(
+                self.manifest["vendoring"]["vendor-directory"],
+                os.path.dirname(yaml_file),
+            )
+        else:
+            self.log(
+                logging.INFO,
+                "vendor",
+                {},
+                "Skipping registering changes with version control.",
+            )
+
+        if perform_step("update-moz-build"):
+            self.log(logging.INFO, "vendor", {}, "Updating moz.build files")
+            self.update_moz_build(
+                self.manifest["vendoring"]["vendor-directory"],
+                os.path.dirname(yaml_file),
+                add_to_exports,
+            )
+        else:
+            self.log(logging.INFO, "vendor", {}, "Skipping update of moz.build files")
 
         self.log(
             logging.INFO,
             "done",
             {"revision": revision},
-            "Update to version '{revision}' ready to commit.",
+            "Update to version '{revision}' completed.",
         )
 
     def get_source_host(self):
         if self.manifest["vendoring"]["source-hosting"] == "gitlab":
             from mozbuild.vendor.host_gitlab import GitLabHost
 
             return GitLabHost(self.manifest)
         elif self.manifest["vendoring"]["source-hosting"] == "github":