servo: Merge #16593 - Mach: Add `mach clean-cargo-cache` command (from UK992:clean-cargo-cache); r=wafflespeanut
authorUK992 <urbankrajnc92@gmail.com>
Mon, 08 May 2017 04:37:21 -0500
changeset 357057 aea57457600e0f41f3aaac138e2b10ca54fdfadc
parent 357056 d309a5a0a165a6d9c6d82da8141df6b711a5a2c2
child 357058 558c843afbd35b6a15d3dbe397048bf0e772f779
push id31781
push userkwierso@gmail.com
push dateMon, 08 May 2017 20:44:15 +0000
treeherdermozilla-central@e0955584782e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerswafflespeanut
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
servo: Merge #16593 - Mach: Add `mach clean-cargo-cache` command (from UK992:clean-cargo-cache); r=wafflespeanut <!-- Please describe your changes on the following line: --> --- <!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: --> - [x] `./mach build -d` does not report any errors - [x] `./mach test-tidy` does not report any errors - [ ] These changes fix #__ (github issue number if applicable). <!-- Either: --> - [ ] There are tests for these changes OR - [ ] These changes do not require tests because _____ <!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.--> <!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. --> Source-Repo: https://github.com/servo/servo Source-Revision: f6bd158fd4287226a881e58020f7dc154fa32532
servo/.travis.yml
servo/python/servo/bootstrap_commands.py
--- a/servo/.travis.yml
+++ b/servo/.travis.yml
@@ -25,16 +25,17 @@ matrix:
          - ./mach test-unit -p style
       cache:
         directories:
           - .cargo
           - .servo
           - $HOME/.ccache
       before_cache:
         - ./mach clean-nightlies --keep 2 --force
+        - ./mach clean-cargo-cache --keep 2 --force
       env: CCACHE=/usr/bin/ccache
       addons:
         apt:
           packages:
             - cmake
             - freeglut3-dev
             - gperf
             - libosmesa6-dev
--- a/servo/python/servo/bootstrap_commands.py
+++ b/servo/python/servo/bootstrap_commands.py
@@ -13,25 +13,26 @@ import base64
 import json
 import os
 import os.path as path
 import re
 import shutil
 import subprocess
 import sys
 import urllib2
+import glob
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 import servo.bootstrap as bootstrap
-from servo.command_base import CommandBase, BIN_SUFFIX
+from servo.command_base import CommandBase, BIN_SUFFIX, cd
 from servo.util import delete, download_bytes, download_file, extract, host_triple
 
 
 @CommandProvider
 class MachCommands(CommandBase):
     @Command('env',
              description='Print environment setup commands',
              category='bootstrap')
@@ -341,8 +342,180 @@ class MachCommands(CommandBase):
                         delete(full_path)
                     else:
                         print("Would remove {}".format(full_path))
         if not removing_anything:
             print("Nothing to remove.")
         elif not force:
             print("Nothing done. "
                   "Run `./mach clean-nightlies -f` to actually remove.")
+
+    @Command('clean-cargo-cache',
+             description='Clean unused Cargo packages',
+             category='bootstrap')
+    @CommandArgument('--force', '-f',
+                     action='store_true',
+                     help='Actually remove stuff')
+    @CommandArgument('--show-size', '-s',
+                     action='store_true',
+                     help='Show packages size')
+    @CommandArgument('--keep',
+                     default='1',
+                     help='Keep up to this many most recent dependencies')
+    @CommandArgument('--custom-path', '-c',
+                     action='store_true',
+                     help='Get Cargo path from CARGO_HOME environment variable')
+    def clean_cargo_cache(self, force=False, show_size=False, keep=None, custom_path=False):
+        def get_size(path):
+            if os.path.isfile(path):
+                return os.path.getsize(path) / (1024 * 1024.0)
+            total_size = 0
+            for dirpath, dirnames, filenames in os.walk(path):
+                for f in filenames:
+                    fp = os.path.join(dirpath, f)
+                    total_size += os.path.getsize(fp)
+            return total_size / (1024 * 1024.0)
+
+        removing_anything = False
+        packages = {
+            'crates': {},
+            'git': {},
+        }
+        import toml
+        if os.environ.get("CARGO_HOME", "") and custom_path:
+            cargo_dir = os.environ.get("CARGO_HOME")
+        else:
+            cargo_dir = path.join(self.context.topdir, ".cargo")
+        cargo_file = open(path.join(self.context.topdir, "Cargo.lock"))
+        content = toml.load(cargo_file)
+
+        for package in content.get("package", []):
+            source = package.get("source", "")
+            version = package["version"]
+            if source == u"registry+https://github.com/rust-lang/crates.io-index":
+                crate_name = "{}-{}".format(package["name"], version)
+                if not packages["crates"].get(crate_name, False):
+                    packages["crates"][package["name"]] = {
+                        "current": [],
+                        "exist": [],
+                    }
+                packages["crates"][package["name"]]["current"].append(crate_name)
+            elif source.startswith("git+"):
+                name = source.split("#")[0].split("/")[-1].replace(".git", "")
+                branch = ""
+                crate_name = "{}-{}".format(package["name"], source.split("#")[1])
+                crate_branch = name.split("?")
+                if len(crate_branch) > 1:
+                    branch = crate_branch[1].replace("branch=", "")
+                    name = crate_branch[0]
+
+                if not packages["git"].get(name, False):
+                    packages["git"][name] = {
+                        "current": [],
+                        "exist": [],
+                    }
+                packages["git"][name]["current"].append(source.split("#")[1][:7])
+                if branch:
+                    packages["git"][name]["current"].append(branch)
+
+        crates_dir = path.join(cargo_dir, "registry")
+        crates_cache_dir = ""
+        crates_src_dir = ""
+        if os.path.isdir(path.join(crates_dir, "cache")):
+            for p in os.listdir(path.join(crates_dir, "cache")):
+                crates_cache_dir = path.join(crates_dir, "cache", p)
+                crates_src_dir = path.join(crates_dir, "src", p)
+
+        git_dir = path.join(cargo_dir, "git")
+        git_db_dir = path.join(git_dir, "db")
+        git_checkout_dir = path.join(git_dir, "checkouts")
+        git_db_list = filter(lambda f: not f.startswith('.'), os.listdir(git_db_dir))
+        git_checkout_list = os.listdir(git_checkout_dir)
+
+        for d in list(set(git_db_list + git_checkout_list)):
+            crate_name = d.replace("-{}".format(d.split("-")[-1]), "")
+            if not packages["git"].get(crate_name, False):
+                packages["git"][crate_name] = {
+                    "current": [],
+                    "exist": [],
+                }
+            if os.path.isdir(path.join(git_checkout_dir, d)):
+                for d2 in os.listdir(path.join(git_checkout_dir, d)):
+                    dep_path = path.join(git_checkout_dir, d, d2)
+                    if os.path.isdir(dep_path):
+                        packages["git"][crate_name]["exist"].append((path.getmtime(dep_path), d, d2))
+            elif os.path.isdir(path.join(git_db_dir, d)):
+                packages["git"][crate_name]["exist"].append(("db", d, ""))
+
+        for d in os.listdir(crates_src_dir):
+            crate_name = re.sub(r"\-\d+(\.\d+){1,3}.+", "", d)
+            if not packages["crates"].get(crate_name, False):
+                packages["crates"][crate_name] = {
+                    "current": [],
+                    "exist": [],
+                }
+            packages["crates"][crate_name]["exist"].append(d)
+
+        total_size = 0
+        for packages_type in ["git", "crates"]:
+            sorted_packages = sorted(packages[packages_type])
+            for crate_name in sorted_packages:
+                crate_count = 0
+                existed_crates = packages[packages_type][crate_name]["exist"]
+                for exist in sorted(existed_crates, reverse=True):
+                    current_crate = packages[packages_type][crate_name]["current"]
+                    size = 0
+                    exist_name = exist
+                    exist_item = exist[2] if packages_type == "git" else exist
+                    if exist_item not in current_crate:
+                        crate_count += 1
+                        removing_anything = True
+                        if int(crate_count) >= int(keep) or not current_crate:
+                            crate_paths = []
+                            if packages_type == "git":
+                                exist_checkout_path = path.join(git_checkout_dir, exist[1])
+                                exist_db_path = path.join(git_db_dir, exist[1])
+                                exist_name = path.join(exist[1], exist[2])
+                                exist_path = path.join(git_checkout_dir, exist_name)
+
+                                if exist[0] == "db":
+                                    crate_paths.append(exist_db_path)
+                                    crate_count += -1
+                                else:
+                                    crate_paths.append(exist_path)
+
+                                    # remove crate from checkout if doesn't exist in db directory
+                                    if not os.path.isdir(exist_db_path):
+                                        crate_count += -1
+
+                                    with cd(path.join(exist_path, ".git", "objects", "pack")):
+                                        for pack in glob.glob("*"):
+                                            pack_path = path.join(exist_db_path, "objects", "pack", pack)
+                                            if os.path.exists(pack_path):
+                                                crate_paths.append(pack_path)
+
+                                    if len(os.listdir(exist_checkout_path)) <= 1:
+                                        crate_paths.append(exist_checkout_path)
+                                        if os.path.isdir(exist_db_path):
+                                            crate_paths.append(exist_db_path)
+                            else:
+                                crate_paths.append(path.join(crates_cache_dir, "{}.crate".format(exist)))
+                                crate_paths.append(path.join(crates_src_dir, exist))
+
+                            size = sum(get_size(p) for p in crate_paths) if show_size else 0
+                            total_size += size
+                            print_msg = (exist_name, " ({}MB)".format(round(size, 2)) if show_size else "", cargo_dir)
+                            if force:
+                                print("Removing `{}`{} package from {}".format(*print_msg))
+                                for crate_path in crate_paths:
+                                    if os.path.exists(crate_path):
+                                        delete(crate_path)
+                            else:
+                                print("Would remove `{}`{} package from {}".format(*print_msg))
+
+        if removing_anything and show_size:
+            print("\nTotal size of {} MB".format(round(total_size, 2)))
+
+        if not removing_anything:
+            print("Nothing to remove.")
+        elif not force:
+            print("\nNothing done. "
+                  "Run `./mach clean-cargo-cache -f` to actually remove.")