servo: Merge #17040 - Improve `clean-cargo-cache` (from UK992:clean-cargo-cache); r=jdm
authorUK992 <urbankrajnc92@gmail.com>
Sun, 28 May 2017 22:27:18 -0500
changeset 361132 8be1ec05e745ff5fe17f34cef8696f9f74b47bba
parent 361080 fc4d115e549ae73dc220bdeb486de0bd47bacedc
child 361133 fa4b5a89201b3d27a62580467eff8c89f837d6ce
push id90813
push usercbook@mozilla.com
push dateTue, 30 May 2017 09:53:44 +0000
treeherdermozilla-inbound@b5c3bb245c4e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjdm
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
servo: Merge #17040 - Improve `clean-cargo-cache` (from UK992:clean-cargo-cache); r=jdm It won't delete pack files from `db` directory anymore, which cause https://github.com/servo/servo/issues/16787 and it will remove empty folders now. Source-Repo: https://github.com/servo/servo Source-Revision: fecfd306c5e743a8fe8e558b1448e725e53cde7a
servo/python/servo/bootstrap_commands.py
--- a/servo/python/servo/bootstrap_commands.py
+++ b/servo/python/servo/bootstrap_commands.py
@@ -379,16 +379,18 @@ class MachCommands(CommandBase):
             'crates': {},
             'git': {},
         }
         import toml
         if os.environ.get("CARGO_HOME", "") and custom_path:
             cargo_dir = os.environ.get("CARGO_HOME")
         else:
             cargo_dir = path.join(self.context.topdir, ".cargo")
+        if not os.path.isdir(cargo_dir):
+            return
         cargo_file = open(path.join(self.context.topdir, "Cargo.lock"))
         content = toml.load(cargo_file)
 
         for package in content.get("package", []):
             source = package.get("source", "")
             version = package["version"]
             if source == u"registry+https://github.com/rust-lang/crates.io-index":
                 crate_name = "{}-{}".format(package["name"], version)
@@ -433,22 +435,27 @@ class MachCommands(CommandBase):
         for d in list(set(git_db_list + git_checkout_list)):
             crate_name = d.replace("-{}".format(d.split("-")[-1]), "")
             if not packages["git"].get(crate_name, False):
                 packages["git"][crate_name] = {
                     "current": [],
                     "exist": [],
                 }
             if os.path.isdir(path.join(git_checkout_dir, d)):
-                for d2 in os.listdir(path.join(git_checkout_dir, d)):
+                with cd(path.join(git_checkout_dir, d)):
+                    git_crate_hash = glob.glob('*')
+                if not git_crate_hash or not os.path.isdir(path.join(git_db_dir, d)):
+                    packages["git"][crate_name]["exist"].append(("del", d, ""))
+                    continue
+                for d2 in git_crate_hash:
                     dep_path = path.join(git_checkout_dir, d, d2)
                     if os.path.isdir(dep_path):
                         packages["git"][crate_name]["exist"].append((path.getmtime(dep_path), d, d2))
             elif os.path.isdir(path.join(git_db_dir, d)):
-                packages["git"][crate_name]["exist"].append(("db", d, ""))
+                packages["git"][crate_name]["exist"].append(("del", d, ""))
 
         for d in os.listdir(crates_src_dir):
             crate_name = re.sub(r"\-\d+(\.\d+){1,3}.+", "", d)
             if not packages["crates"].get(crate_name, False):
                 packages["crates"][crate_name] = {
                     "current": [],
                     "exist": [],
                 }
@@ -458,46 +465,40 @@ class MachCommands(CommandBase):
         for packages_type in ["git", "crates"]:
             sorted_packages = sorted(packages[packages_type])
             for crate_name in sorted_packages:
                 crate_count = 0
                 existed_crates = packages[packages_type][crate_name]["exist"]
                 for exist in sorted(existed_crates, reverse=True):
                     current_crate = packages[packages_type][crate_name]["current"]
                     size = 0
-                    exist_name = exist
+                    exist_name = path.join(exist[1], exist[2]) if packages_type == "git" else exist
                     exist_item = exist[2] if packages_type == "git" else exist
                     if exist_item not in current_crate:
                         crate_count += 1
-                        removing_anything = True
-                        if int(crate_count) >= int(keep) or not current_crate:
+                        if int(crate_count) >= int(keep) or not current_crate or \
+                           exist[0] == "del" or exist[2] == "master":
+                            removing_anything = True
                             crate_paths = []
                             if packages_type == "git":
                                 exist_checkout_path = path.join(git_checkout_dir, exist[1])
                                 exist_db_path = path.join(git_db_dir, exist[1])
-                                exist_name = path.join(exist[1], exist[2])
                                 exist_path = path.join(git_checkout_dir, exist_name)
 
-                                if exist[0] == "db":
-                                    crate_paths.append(exist_db_path)
+                                if exist[0] == "del":
+                                    if os.path.isdir(exist_checkout_path):
+                                        crate_paths.append(exist_checkout_path)
+                                    if os.path.isdir(exist_db_path):
+                                        crate_paths.append(exist_db_path)
                                     crate_count += -1
                                 else:
                                     crate_paths.append(exist_path)
 
-                                    # remove crate from checkout if doesn't exist in db directory
-                                    if not os.path.isdir(exist_db_path):
-                                        crate_count += -1
-
-                                    with cd(path.join(exist_path, ".git", "objects", "pack")):
-                                        for pack in glob.glob("*"):
-                                            pack_path = path.join(exist_db_path, "objects", "pack", pack)
-                                            if os.path.exists(pack_path):
-                                                crate_paths.append(pack_path)
-
-                                    if len(os.listdir(exist_checkout_path)) <= 1:
+                                    exist_checkout_list = glob.glob(path.join(exist_checkout_path, '*'))
+                                    if len(exist_checkout_list) <= 1:
                                         crate_paths.append(exist_checkout_path)
                                         if os.path.isdir(exist_db_path):
                                             crate_paths.append(exist_db_path)
                             else:
                                 crate_paths.append(path.join(crates_cache_dir, "{}.crate".format(exist)))
                                 crate_paths.append(path.join(crates_src_dir, exist))
 
                             size = sum(get_size(p) for p in crate_paths) if show_size else 0