Bug 1656740 - Integrate `clangd` in `vscode` for C++ language support. r=froydnj
authorAndi-Bogdan Postelnicu <bpostelnicu@mozilla.com>
Thu, 06 Aug 2020 06:25:17 +0000
changeset 543557 1d35f9fe239eca93ff6c9907b5bb7865cc657467
parent 543556 786545256e8814da7899374a2165f22509c08993
child 543558 ed78a72ce214798f905b8cd41a39e6a45f85c078
push id123532
push userbpostelnicu@mozilla.com
push dateThu, 06 Aug 2020 07:24:12 +0000
treeherderautoland@38dd1ab6680f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs1656740
milestone81.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1656740 - Integrate `clangd` in `vscode` for C++ language support. r=froydnj In order to have a cross platform ide for C++ language support we've added `clangd` extenssion and artifact part of `vscode` suite. To generate the configuration you simply run: `./mach ide vscode `. Differential Revision: https://phabricator.services.mozilla.com/D85416
.vscode/extensions.json
docs/contributing/editor.rst
python/mozbuild/mozbuild/backend/__init__.py
python/mozbuild/mozbuild/backend/clangd.py
python/mozbuild/mozbuild/backend/mach_commands.py
python/mozbuild/mozbuild/compilation/database.py
--- a/.vscode/extensions.json
+++ b/.vscode/extensions.json
@@ -3,18 +3,18 @@
   // for the documentation about the extensions.json format
   "recommendations": [
     // Trim only touched lines.
     "NathanRidley.autotrim",
     // ESLint support.
     "dbaeumer.vscode-eslint",
     // Prettier support.
     "esbenp.prettier-vscode",
-    // C/C++ language support.
-    "ms-vscode.cpptools",
+    // C/C++ language support with clangd
+    "llvm-vs-code-extensions.vscode-clangd",
     // Rust language support.
     "rust-lang.rust",
     // Provides support for rust-analyzer: novel LSP server for the Rust programming language.
     "matklad.rust-analyzer",
     // CSS support for HTML documents.
     "ecmel.vscode-html-css",
     // Web app and extension debugging.
     "firefox-devtools.vscode-firefox-debug",
--- a/docs/contributing/editor.rst
+++ b/docs/contributing/editor.rst
@@ -11,24 +11,31 @@ them.
     This page is a work in progress. Please enhance this page with instructions
     for your favourite editor.
 
 Visual Studio Code
 ------------------
 
 For general information on using VS Code, see their
 `home page <https://code.visualstudio.com/>`__,
-`repo <https://github.com/Microsoft/vscode/>`__ and
-`guide to working with C++ <https://code.visualstudio.com/docs/languages/cpp>`__.
+`repo <https://github.com/Microsoft/vscode/>`__.
+
+For C++ support we offer an out of the box configuration based on
+`clangd <https://clangd.llvm.org>`__. This covers code completion, compile errors,
+go-to-definition and more.
 
-For IntelliSense to work properly, a
-:ref:`compilation database <CompileDB back-end / compileflags>` as described
-below is required. When it is present when you open the mozilla source code
-folder, it will be automatically detected and Visual Studio Code will ask you
-if it should use it, which you should confirm.
+In order to build the configuration for `VS Code` simply run from
+the terminal:
+
+`./mach ide vscode`
+
+If `VS Code` is already open with a previous configuration generated, please make sure to
+restart `VS Code` otherwise the new configuration will not be used, and the `compile_commands.json`
+needed by `clangd` server will not be refreshed. This is a known `bug <https://github.com/clangd/vscode-clangd/issues/42>`__
+in `clangd-vscode` extension
 
 VS Code provides number of extensions for JavaScript, Rust, etc.
 
 Useful preferences
 ~~~~~~~~~~~~~~~~~~
 
 When setting the preference
 
--- a/python/mozbuild/mozbuild/backend/__init__.py
+++ b/python/mozbuild/mozbuild/backend/__init__.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function
 
 backends = {
+    'Clangd': 'mozbuild.backend.clangd',
     'ChromeMap': 'mozbuild.codecoverage.chrome_map',
     'CompileDB': 'mozbuild.compilation.database',
     'CppEclipse': 'mozbuild.backend.cpp_eclipse',
     'FasterMake': 'mozbuild.backend.fastermake',
     'FasterMake+RecursiveMake': None,
     'GnConfigGen': 'mozbuild.gn_processor',
     'GnMozbuildWriter': 'mozbuild.gn_processor',
     'RecursiveMake': 'mozbuild.backend.recursivemake',
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/backend/clangd.py
@@ -0,0 +1,47 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides a backend for `clangd` in order to have support for
+# code completion, compile errors, go-to-definition and more.
+# It is based on `database.py` with the difference that we don't generate
+# an unified `compile_commands.json` but we generate a per file basis `command` in
+# `objdir/clangd/compile_commands.json`
+
+from __future__ import absolute_import, print_function
+
+import os
+
+from mozbuild.compilation.database import CompileDBBackend
+
+import mozpack.path as mozpath
+
+
+class ClangdBackend(CompileDBBackend):
+    """
+    Configuration that generates the backend for clangd, it is used with `clangd`
+    extension for vscode
+    """
+
+    def _init(self):
+        CompileDBBackend._init(self)
+
+    def _build_cmd(self, cmd, filename, unified):
+        cmd = list(cmd)
+
+        cmd.append(filename)
+
+        return cmd
+
+    def _outputfile_path(self):
+        clangd_cc_path = os.path.join(self.environment.topobjdir, "clangd")
+
+        if not os.path.exists(clangd_cc_path):
+            os.mkdir(clangd_cc_path)
+
+        # Output the database (a JSON file) to objdir/clangd/compile_commands.json
+        return mozpath.join(clangd_cc_path, "compile_commands.json")
+
+    def _process_unified_sources(self, obj):
+        for f in list(sorted(obj.files)):
+            self._build_db_line(obj.objdir, obj.relsrcdir, obj.config, f, obj.canonical_suffix)
--- a/python/mozbuild/mozbuild/backend/mach_commands.py
+++ b/python/mozbuild/mozbuild/backend/mach_commands.py
@@ -1,66 +1,296 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
+import logging
 import os
 import subprocess
 
 from mozbuild.base import MachCommandBase
+from mozbuild.build_commands import Build
+
 from mozfile import which
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
+import mozpack.path as mozpath
+
 
 @CommandProvider
 class MachCommands(MachCommandBase):
-    @Command('ide', category='devenv',
-             description='Generate a project and launch an IDE.')
-    @CommandArgument('ide', choices=['eclipse', 'visualstudio'])
-    @CommandArgument('args', nargs=argparse.REMAINDER)
+    @Command("ide", category="devenv", description="Generate a project and launch an IDE.")
+    @CommandArgument("ide", choices=["eclipse", "visualstudio", "vscode"])
+    @CommandArgument("args", nargs=argparse.REMAINDER)
     def eclipse(self, ide, args):
-        if ide == 'eclipse':
-            backend = 'CppEclipse'
-        elif ide == 'visualstudio':
-            backend = 'VisualStudio'
+        if ide == "eclipse":
+            backend = "CppEclipse"
+        elif ide == "visualstudio":
+            backend = "VisualStudio"
+        elif ide == "vscode":
+            backend = "Clangd"
 
-        if ide == 'eclipse' and not which('eclipse'):
-            print('Eclipse CDT 8.4 or later must be installed in your PATH.')
-            print('Download: http://www.eclipse.org/cdt/downloads.php')
+        if ide == "eclipse" and not which("eclipse"):
+            self.log(
+                logging.ERROR,
+                "ide",
+                {},
+                "Eclipse CDT 8.4 or later must be installed in your PATH.",
+            )
+            self.log(
+                logging.ERROR, "ide", {}, "Download: http://www.eclipse.org/cdt/downloads.php"
+            )
             return 1
 
-        # Here we refresh the whole build. 'build export' is sufficient here and is probably more
-        # correct but it's also nice having a single target to get a fully built and indexed
-        # project (gives a easy target to use before go out to lunch).
-        res = self._mach_context.commands.dispatch('build', self._mach_context)
-        if res != 0:
-            return 1
+        if ide == "vscode":
+            # Verify if platform has VSCode installed
+            if not self.found_vscode_path():
+                self.log(logging.ERROR, "ide", {}, "VSCode cannot be found, abording!")
+                return 1
+
+            # Create the Build environment to configure the tree
+            builder = Build(self._mach_context)
+
+            rc = builder.configure()
+            if rc != 0:
+                return rc
+
+            # First install what we can through install manifests.
+            rc = builder._run_make(
+                directory=self.topobjdir, target="pre-export", line_handler=None
+            )
+            if rc != 0:
+                return rc
+
+            # Then build the rest of the build dependencies by running the full
+            # export target, because we can't do anything better.
+            for target in ("export", "pre-compile"):
+                rc = builder._run_make(directory=self.topobjdir, target=target, line_handler=None)
+                if rc != 0:
+                    return rc
+        else:
+            # Here we refresh the whole build. 'build export' is sufficient here and is
+            # probably more correct but it's also nice having a single target to get a fully
+            # built and indexed project (gives a easy target to use before go out to lunch).
+            res = self._mach_context.commands.dispatch("build", self._mach_context)
+            if res != 0:
+                return 1
 
         # Generate or refresh the IDE backend.
         python = self.virtualenv_manager.python_path
-        config_status = os.path.join(self.topobjdir, 'config.status')
-        args = [python, config_status, '--backend=%s' % backend]
+        config_status = os.path.join(self.topobjdir, "config.status")
+        args = [python, config_status, "--backend=%s" % backend]
         res = self._run_command_in_objdir(args=args, pass_thru=True, ensure_exit_code=False)
         if res != 0:
             return 1
 
-        if ide == 'eclipse':
+        if ide == "eclipse":
             eclipse_workspace_dir = self.get_eclipse_workspace_path()
-            subprocess.check_call(['eclipse', '-data', eclipse_workspace_dir])
-        elif ide == 'visualstudio':
+            subprocess.check_call(["eclipse", "-data", eclipse_workspace_dir])
+        elif ide == "visualstudio":
             visual_studio_workspace_dir = self.get_visualstudio_workspace_path()
-            subprocess.check_call(
-                ['explorer.exe', visual_studio_workspace_dir]
-            )
+            subprocess.check_call(["explorer.exe", visual_studio_workspace_dir])
+        elif ide == "vscode":
+            return self.setup_vscode()
 
     def get_eclipse_workspace_path(self):
         from mozbuild.backend.cpp_eclipse import CppEclipseBackend
+
         return CppEclipseBackend.get_workspace_path(self.topsrcdir, self.topobjdir)
 
     def get_visualstudio_workspace_path(self):
-        return os.path.join(self.topobjdir, 'msvc', 'mozilla.sln')
+        return os.path.join(self.topobjdir, "msvc", "mozilla.sln")
+
+    def found_vscode_path(self):
+
+        if "linux" in self.platform[0]:
+            self.vscode_path = "/usr/bin/code"
+        elif "macos" in self.platform[0]:
+            self.vscode_path = "/usr/local/bin/code"
+        elif "win64" in self.platform[0]:
+            from pathlib import Path
+
+            self.vscode_path = mozpath.join(
+                str(Path.home()), "AppData", "Local", "Programs", "Microsoft VS Code", "Code.exe",
+            )
+
+        # Path found
+        if os.path.exists(self.vscode_path):
+            return True
+
+        for _ in range(5):
+            self.vscode_path = input(
+                "Could not find the VSCode binary. Please provide the full path to it:\n"
+            )
+            if os.path.exists(self.vscode_path):
+                return True
+
+        # Path cannot be found
+        return False
+
+    def setup_vscode(self):
+        vscode_settings = mozpath.join(self.topsrcdir, ".vscode", "settings.json")
+
+        clangd_cc_path = mozpath.join(self.topobjdir, "clangd")
+
+        # Verify if the required files are present
+        clang_tools_path = mozpath.join(self._mach_context.state_dir, "clang-tools")
+        clang_tidy_bin = mozpath.join(clang_tools_path, "clang-tidy", "bin")
+
+        clangd_path = mozpath.join(
+            clang_tidy_bin, "clangd" + self.config_environment.substs.get("BIN_SUFFIX", ""),
+        )
+
+        if not os.path.exists(clangd_path):
+            self.log(
+                logging.ERROR, "ide", {}, "Unable to locate clangd in {}.".format(clang_tidy_bin)
+            )
+            rc = self._get_clang_tools(clang_tools_path)
+
+            if rc != 0:
+                return rc
+
+        import multiprocessing
+        import json
+
+        clangd_json = json.loads(
+            """
+        {
+            "clangd.path": "%s",
+            "clangd.arguments": [
+                "--compile-commands-dir",
+                "%s",
+                "-j",
+                "%s",
+                "--limit-results",
+                "0",
+                "--completion-style",
+                "detailed",
+                "--background-index",
+                "--all-scopes-completion",
+                "--log",
+                "error",
+                "--pch-storage",
+                "memory"
+            ]
+        }
+        """
+            % (clangd_path, clangd_cc_path, multiprocessing.cpu_count(),)
+        )
+
+        # Create an empty settings dictionary
+        settings = {}
+
+        # Modify the .vscode/settings.json configuration file
+        if os.path.exists(vscode_settings):
+            # If exists prompt for a configuration change
+            choice = prompt_bool(
+                "Configuration for {settings} must change. "
+                "Do you want to proceed?".format(settings=vscode_settings)
+            )
+            if not choice:
+                return 1
+
+            # Read the original vscode settings
+            with open(vscode_settings) as fh:
+                try:
+                    settings = json.load(fh)
+                    print(
+                        "The following modifications will occur:\nOriginal:\n{orig}\n"
+                        "New:\n{new}".format(
+                            orig=json.dumps(
+                                {
+                                    key: settings[key] if key in settings else ""
+                                    for key in ["clangd.path", "clangd.arguments"]
+                                },
+                                indent=4,
+                            ),
+                            new=json.dumps(clangd_json, indent=4),
+                        )
+                    )
+
+                except ValueError:
+                    # Decoding has failed, work with an empty dict
+                    settings = {}
+
+        # Write our own Configuration
+        settings["clangd.path"] = clangd_json["clangd.path"]
+        settings["clangd.arguments"] = clangd_json["clangd.arguments"]
+
+        with open(vscode_settings, "w") as fh:
+            fh.write(json.dumps(settings, indent=4))
+
+        # Open vscode with new configuration
+        rc = subprocess.call([self.vscode_path, self.topsrcdir])
+
+        if rc != 0:
+            self.log(
+                logging.ERROR,
+                "ide",
+                {},
+                "Unable to open VS Code. Please open VS Code manually and load "
+                "directory: {}".format(self.topsrcdir),
+            )
+            return rc
+
+        return 0
+
+    def _get_clang_tools(self, clang_tools_path):
+
+        import shutil
+
+        if os.path.isdir(clang_tools_path):
+            shutil.rmtree(clang_tools_path)
+
+        # Create base directory where we store clang binary
+        os.mkdir(clang_tools_path)
+
+        from mozbuild.artifact_commands import PackageFrontend
+
+        self._artifact_manager = PackageFrontend(self._mach_context)
+
+        job, _ = self.platform
+
+        if job is None:
+            self.log(
+                logging.ERROR,
+                "ide",
+                {},
+                "The current platform isn't supported. "
+                "Currently only the following platforms are "
+                "supported: win32/win64, linux64 and macosx64.",
+            )
+            return 1
+
+        job += "-clang-tidy"
+
+        # We want to unpack data in the clang-tidy mozbuild folder
+        currentWorkingDir = os.getcwd()
+        os.chdir(clang_tools_path)
+        rc = self._artifact_manager.artifact_toolchain(
+            verbose=False, from_build=[job], no_unpack=False, retry=0
+        )
+        # Change back the cwd
+        os.chdir(currentWorkingDir)
+
+        return rc
+
+
+def prompt_bool(prompt, limit=5):
+    """ Prompts the user with prompt and requires a boolean value. """
+    from distutils.util import strtobool
+
+    for _ in range(limit):
+        try:
+            return strtobool(input(prompt + " [Y/N]\n"))
+        except ValueError:
+            print(
+                "ERROR! Please enter a valid option! Please use any of the following:"
+                " Y, N, True, False, 1, 0"
+            )
+    return False
--- a/python/mozbuild/mozbuild/compilation/database.py
+++ b/python/mozbuild/mozbuild/compilation/database.py
@@ -37,16 +37,25 @@ class CompileDBBackend(CommonBackend):
 
         # The cache for per-directory flags
         self._flags = {}
 
         self._envs = {}
         self._local_flags = defaultdict(dict)
         self._per_source_flags = defaultdict(list)
 
+    def _build_cmd(self, cmd, filename, unified):
+        cmd = list(cmd)
+        if unified is None:
+            cmd.append(filename)
+        else:
+            cmd.append(unified)
+
+        return cmd
+
     def consume_object(self, obj):
         # Those are difficult directories, that will be handled later.
         if obj.relsrcdir in (
                 'build/unix/elfhack',
                 'build/unix/elfhack/inject',
                 'build/clang-plugin',
                 'build/clang-plugin/tests'):
             return True
@@ -81,21 +90,17 @@ class CompileDBBackend(CommonBackend):
 
     def consume_finished(self):
         CommonBackend.consume_finished(self)
 
         db = []
 
         for (directory, filename, unified), cmd in self._db.items():
             env = self._envs[directory]
-            cmd = list(cmd)
-            if unified is None:
-                cmd.append(filename)
-            else:
-                cmd.append(unified)
+            cmd = self._build_cmd(cmd, filename, unified)
             variables = {
                 'DIST': mozpath.join(env.topobjdir, 'dist'),
                 'DEPTH': env.topobjdir,
                 'MOZILLA_DIR': env.topsrcdir,
                 'topsrcdir': env.topsrcdir,
                 'topobjdir': env.topobjdir,
             }
             variables.update(self._local_flags[directory])
@@ -131,21 +136,24 @@ class CompileDBBackend(CommonBackend):
                 c.extend(per_source_flags)
             db.append({
                 'directory': directory,
                 'command': ' '.join(shell_quote(a) for a in c),
                 'file': mozpath.join(directory, filename),
             })
 
         import json
-        # Output the database (a JSON file) to objdir/compile_commands.json
-        outputfile = os.path.join(self.environment.topobjdir, 'compile_commands.json')
+        outputfile = self._outputfile_path()
         with self._write_file(outputfile) as jsonout:
             json.dump(db, jsonout, indent=0)
 
+    def _outputfile_path(self):
+        # Output the database (a JSON file) to objdir/compile_commands.json
+        return os.path.join(self.environment.topobjdir, 'compile_commands.json')
+
     def _process_unified_sources(self, obj):
         if not obj.have_unified_mapping:
             for f in list(sorted(obj.files)):
                 self._build_db_line(obj.objdir, obj.relsrcdir, obj.config, f,
                                     obj.canonical_suffix)
             return
 
         # For unified sources, only include the unified source file.