Bug 1577726 - Move generated directories into a dedicated file (Generated.txt) from ThirdPartyPaths.txt r=ahal
authorSylvestre Ledru <sledru@mozilla.com>
Thu, 05 Sep 2019 14:24:21 +0000
changeset 491859 df7da1b22c2f259242adb42b4ea271b8d7a5a828
parent 491858 c2a2da3de5613508d0d15e8850d8766f6980ef7b
child 491860 b2ae7f641b2dc9829827a4e00b0c1fb035bb9cdb
push id36540
push userapavel@mozilla.com
push dateFri, 06 Sep 2019 03:58:29 +0000
treeherdermozilla-central@3bfc0e27d038 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1577726
milestone71.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1577726 - Move generated directories into a dedicated file (Generated.txt) from ThirdPartyPaths.txt r=ahal Differential Revision: https://phabricator.services.mozilla.com/D44147
build/clang-plugin/ThirdPartyPaths.py
build/clang-plugin/import_mozilla_checks.py
build/clang-plugin/moz.build
python/mozbuild/mozbuild/code-analysis/mach_commands.py
python/mozbuild/mozbuild/controller/building.py
tools/infer/config.yaml
tools/lint/mach_commands.py
tools/rewriting/Generated.txt
tools/rewriting/ThirdPartyPaths.txt
--- a/build/clang-plugin/ThirdPartyPaths.py
+++ b/build/clang-plugin/ThirdPartyPaths.py
@@ -1,27 +1,31 @@
 #!/usr/bin/env python3
 
 import json
 
 
-def generate(output, tpp_txt):
+def generate(output, *input_paths):
     """
     This file generates a ThirdPartyPaths.cpp file from the ThirdPartyPaths.txt
     file in /tools/rewriting, which is used by the Clang Plugin to help identify
     sources which should be ignored.
     """
+    tpp_list = []
+    lines = set()
 
-    tpp_list = []
-    with open(tpp_txt) as f:
-        for line in f.readlines():
-            line = line.strip()
-            if line.endswith('/'):
-                line = line[:-1]
-            tpp_list.append(line)
+    for path in input_paths:
+        with open(path) as f:
+            lines.update(f.readlines())
+
+    for line in lines:
+        line = line.strip()
+        if line.endswith('/'):
+            line = line[:-1]
+        tpp_list.append(line)
     tpp_strings = ',\n  '.join([json.dumps(tpp) for tpp in tpp_list])
 
     output.write("""\
 /* THIS FILE IS GENERATED BY ThirdPartyPaths.py - DO NOT EDIT */
 
 #include <stdint.h>
 
 const char* MOZ_THIRD_PARTY_PATHS[] = {
--- a/build/clang-plugin/import_mozilla_checks.py
+++ b/build/clang-plugin/import_mozilla_checks.py
@@ -91,18 +91,20 @@ def add_item_to_cmake_section(cmake_path
             f.write(line)
 
     f.close()
 
 
 def write_third_party_paths(mozilla_path, module_path):
     tpp_txt = os.path.join(
         mozilla_path, '../../tools/rewriting/ThirdPartyPaths.txt')
+    generated_txt = os.path.join(
+        mozilla_path, '../../tools/rewriting/Generated.txt')
     with open(os.path.join(module_path, 'ThirdPartyPaths.cpp'), 'w') as f:
-        ThirdPartyPaths.generate(f, tpp_txt)
+        ThirdPartyPaths.generate(f, tpp_txt, generated_txt)
 
 
 def do_import(mozilla_path, clang_tidy_path):
     module = 'mozilla'
     module_path = os.path.join(clang_tidy_path, module)
     try:
         os.makedirs(module_path)
     except OSError as e:
--- a/build/clang-plugin/moz.build
+++ b/build/clang-plugin/moz.build
@@ -57,16 +57,17 @@ if CONFIG['ENABLE_MOZSEARCH_PLUGIN']:
         'mozsearch-plugin/StringOperations.cpp',
     ]
 
 GENERATED_FILES += ['ThirdPartyPaths.cpp']
 third_party_paths = GENERATED_FILES['ThirdPartyPaths.cpp']
 third_party_paths.script = "ThirdPartyPaths.py:generate"
 third_party_paths.inputs = [
     '/tools/rewriting/ThirdPartyPaths.txt',
+    '/tools/rewriting/Generated.txt',
 ]
 
 HOST_COMPILE_FLAGS['STL'] = []
 HOST_COMPILE_FLAGS['VISIBILITY'] = []
 
 # libc++ is required to build plugins against clang on OS X.
 if CONFIG['HOST_OS_ARCH'] == 'Darwin':
     HOST_CXXFLAGS += ['-stdlib=libc++']
--- a/python/mozbuild/mozbuild/code-analysis/mach_commands.py
+++ b/python/mozbuild/mozbuild/code-analysis/mach_commands.py
@@ -687,20 +687,21 @@ class StaticAnalysis(MachCommandBase):
             if rc != 0:
                 return rc
         rc = self._get_infer(verbose=verbose)
         if rc != 0:
             self.log(logging.WARNING, 'static-analysis', {},
                      'This command is only available for linux64!')
             return rc
         # which checkers to use, and which folders to exclude
-        all_checkers, third_party_path = self._get_infer_config()
+        all_checkers, third_party_path, generated_path = self._get_infer_config()
         checkers, excludes = self._get_infer_args(
             checks=checks or all_checkers,
-            third_party_path=third_party_path
+            third_party_path=third_party_path,
+            generated_path=generated_path
         )
         rc = rc or self._gradle(['clean'])  # clean so that we can recompile
         # infer capture command
         capture_cmd = [self._infer_path, 'capture'] + excludes + ['--']
         rc = rc or self._gradle([task], infer_args=capture_cmd, verbose=verbose)
         tmp_file, args = self._get_infer_source_args(java_sources)
         # infer analyze command
         analysis_cmd = [self._infer_path, 'analyze', '--keep-going'] +  \
@@ -751,33 +752,35 @@ class StaticAnalysis(MachCommandBase):
         with open(mozpath.join(self.topsrcdir, 'tools',
                                'infer', 'config.yaml')) as f:
             try:
                 config = yaml.safe_load(f)
                 for item in config['infer_checkers']:
                     if item['publish']:
                         checkers.append(item['name'])
                 tp_path = mozpath.join(self.topsrcdir, config['third_party'])
+                generated_path = mozpath.join(self.topsrcdir, config['generated'])
             except Exception:
                 print('Looks like config.yaml is not valid, so we are unable '
                       'to determine default checkers, and which folder to '
                       'exclude, using defaults provided by infer')
-        return checkers, tp_path
+        return checkers, tp_path, generated_path
 
-    def _get_infer_args(self, checks, third_party_path):
+    def _get_infer_args(self, checks, *input_paths):
         '''Return the arguments which include the checkers <checks>, and
         excludes all folder in <third_party_path>.'''
         checkers = ['-a', 'checkers']
         excludes = []
         for checker in checks:
             checkers.append('--' + checker)
-        with open(third_party_path) as f:
-            for line in f:
-                excludes.append('--skip-analysis-in-path')
-                excludes.append(line.strip('\n'))
+        for path in input_paths:
+            with open(path) as f:
+                for line in f:
+                    excludes.append('--skip-analysis-in-path')
+                    excludes.append(line.strip('\n'))
         return checkers, excludes
 
     def _get_clang_tidy_config(self):
         try:
             file_handler = open(mozpath.join(self.topsrcdir, "tools", "clang-tidy", "config.yaml"))
             config = yaml.safe_load(file_handler)
         except Exception:
             self.log(logging.ERROR, 'static-analysis', {},
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1223,20 +1223,26 @@ class BuildDriver(MozbuildObject):
             # until we suppress them for real.
             # TODO remove entries/feature once we stop generating warnings
             # in these directories.
             pathToThirdparty = os.path.join(self.topsrcdir,
                                             "tools",
                                             "rewriting",
                                             "ThirdPartyPaths.txt")
 
+            pathToGenerated = os.path.join(self.topsrcdir,
+                                           "tools",
+                                           "rewriting",
+                                           "Generated.txt")
+
             if os.path.exists(pathToThirdparty):
-                with open(pathToThirdparty) as f:
+                with open(pathToThirdparty) as f, open(pathToGenerated) as g:
                     # Normalize the path (no trailing /)
-                    LOCAL_SUPPRESS_DIRS = tuple(d.rstrip('/') for d in f.read().splitlines())
+                    suppress = f.readlines() + g.readlines()
+                    LOCAL_SUPPRESS_DIRS = tuple(s.strip('/') for s in suppress)
             else:
                 # For application based on gecko like thunderbird
                 LOCAL_SUPPRESS_DIRS = ()
 
             suppressed_by_dir = Counter()
 
             for warning in sorted(monitor.instance_warnings):
                 path = mozpath.normsep(warning['filename'])
--- a/tools/infer/config.yaml
+++ b/tools/infer/config.yaml
@@ -24,8 +24,9 @@ infer_checkers:
     publish: !!bool no
   - name: racerd
     publish: !!bool yes
   # I think this is only for c++, can't trigger these errors in Java
   - name: liveness
     publish: !!bool no
 # Third party files from mozilla-central
 third_party: tools/rewriting/ThirdPartyPaths.txt
+generated: tools/rewriting/Generated.txt
--- a/tools/lint/mach_commands.py
+++ b/tools/lint/mach_commands.py
@@ -17,17 +17,20 @@ from mozbuild.base import (
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 
 here = os.path.abspath(os.path.dirname(__file__))
-THIRD_PARTY_PATHS = os.path.join('tools', 'rewriting', 'ThirdPartyPaths.txt')
+EXCLUSION_FILES = [
+    os.path.join('tools', 'rewriting', 'Generated.txt'),
+    os.path.join('tools', 'rewriting', 'ThirdPartyPaths.txt'),
+]
 GLOBAL_EXCLUDES = [
     'node_modules',
     'tools/lint/test/files',
 ]
 
 
 def setup_argument_parser():
     from mozlint import cli
@@ -37,19 +40,20 @@ def setup_argument_parser():
 def get_global_excludes(topsrcdir):
     # exclude misc paths
     excludes = GLOBAL_EXCLUDES[:]
 
     # exclude top level paths that look like objdirs
     excludes.extend([name for name in os.listdir(topsrcdir)
                      if name.startswith('obj') and os.path.isdir(name)])
 
-    # exclude third party paths
-    with open(os.path.join(topsrcdir, THIRD_PARTY_PATHS), 'r') as fh:
-        excludes.extend([f.strip() for f in fh.readlines()])
+    for path in EXCLUSION_FILES:
+        # exclude third party paths
+        with open(os.path.join(topsrcdir, path), 'r') as fh:
+            excludes.extend([f.strip() for f in fh.readlines()])
 
     return excludes
 
 
 @CommandProvider
 class MachCommands(MachCommandBase):
 
     @Command(
@@ -68,17 +72,18 @@ class MachCommands(MachCommandBase):
             buildargs['topobjdir'] = self.topobjdir
             lintargs.update(buildargs)
         except BuildEnvironmentNotFoundException:
             pass
 
         lintargs.setdefault('root', self.topsrcdir)
         lintargs['exclude'] = get_global_excludes(lintargs['root'])
         cli.SEARCH_PATHS.append(here)
-        parser.GLOBAL_SUPPORT_FILES.append(os.path.join(self.topsrcdir, THIRD_PARTY_PATHS))
+        for path in EXCLUSION_FILES:
+            parser.GLOBAL_SUPPORT_FILES.append(os.path.join(self.topsrcdir, path))
         return cli.run(*runargs, **lintargs)
 
     @Command('eslint', category='devenv',
              description='Run eslint or help configure eslint for optimal development.')
     @CommandArgument('paths', default=None, nargs='*',
                      help="Paths to file or directories to lint, like "
                           "'browser/' Defaults to the "
                           "current directory if not given.")
new file mode 100644
--- /dev/null
+++ b/tools/rewriting/Generated.txt
@@ -0,0 +1,7 @@
+browser/components/newtab/node_modules/
+devtools/client/aboutdebugging/test/jest/node_modules/
+devtools/client/application/test/components/node_modules/
+devtools/client/debugger/node_modules/
+dom/tests/ajax/jquery/
+dom/tests/ajax/mochikit/
+node_modules/
--- a/tools/rewriting/ThirdPartyPaths.txt
+++ b/tools/rewriting/ThirdPartyPaths.txt
@@ -1,18 +1,14 @@
 build/pymake
 browser/components/newtab/logs/
-browser/components/newtab/node_modules/
 browser/components/newtab/vendor/
 browser/components/translation/cld2/
 browser/extensions/screenshots/build/raven.js
-devtools/client/aboutdebugging/test/jest/node_modules/
-devtools/client/application/test/components/node_modules/
-devtools/client/debugger/flow-typed/
-devtools/client/debugger/node_modules/
+devtools/client/debugger/flow-typed/npm/
 devtools/client/shared/build/babel.js
 devtools/client/shared/sourceeditor/codemirror/
 devtools/client/shared/sourceeditor/tern/
 devtools/client/shared/vendor/
 devtools/shared/jsbeautify/
 devtools/shared/heapsnapshot/CoreDump.pb.cc
 devtools/shared/heapsnapshot/CoreDump.pb.h
 dom/canvas/test/webgl-conf/checkout/
@@ -27,18 +23,16 @@ dom/media/platforms/ffmpeg/libav53/
 dom/media/platforms/ffmpeg/libav54/
 dom/media/platforms/ffmpeg/libav55/
 dom/media/webspeech/recognition/endpointer.cc
 dom/media/webspeech/recognition/endpointer.h
 dom/media/webspeech/recognition/energy_endpointer.cc
 dom/media/webspeech/recognition/energy_endpointer.h
 dom/media/webspeech/recognition/energy_endpointer_params.cc
 dom/media/webspeech/recognition/energy_endpointer_params.h
-dom/tests/ajax/jquery/
-dom/tests/ajax/mochikit/
 dom/tests/mochitest/ajax/
 dom/tests/mochitest/dom-level1-core/
 dom/tests/mochitest/dom-level2-core/
 dom/tests/mochitest/dom-level2-html/
 dom/u2f/tests/pkijs/
 dom/webauthn/cbor-cpp/
 dom/webauthn/tests/pkijs/
 editor/libeditor/tests/browserscope/lib/richtext/
@@ -108,17 +102,16 @@ modules/fdlibm/
 modules/freetype2/
 modules/woff2/
 modules/xz-embedded/
 modules/zlib/
 mozglue/misc/decimal/
 netwerk/dns/nsIDNKitInterface.h
 netwerk/sctp/src/
 netwerk/srtp/src/
-node_modules/
 nsprpub/
 other-licenses/
 parser/expat/
 security/nss/
 security/sandbox/chromium/
 security/sandbox/chromium-shim/
 testing/gtest/gmock/
 testing/gtest/gtest/