Bug 1574977: restore `mach pastebin` and port to new service r=nalexander
authorConnor Sheehan <sheehan@mozilla.com>
Thu, 22 Aug 2019 18:11:26 +0000
changeset 489485 2120cea83d4c0d517eb7e12968e2c088a37fc12c
parent 489484 57a5a83f9a6275931ffa5b3f6504e2fea1f91ebf
child 489486 0bab25d1df161291023cbd703eca35974d66e9a9
push id93376
push usercosheehan@mozilla.com
push dateThu, 22 Aug 2019 18:27:43 +0000
treeherderautoland@2120cea83d4c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1574977
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1574977: restore `mach pastebin` and port to new service r=nalexander This commit re-implements `mach pastebin` for use with the new Mozilla pastebin tool (paste.mozilla.org). As with the old implementation, the tool supports passing a file for upload as well as reading content to paste from stdin. `dpaste` (the software that runs `paste.mo`) is supposed to support sniffing the correct syntax highlighter from the given filename, but this feature does not seem to work (both on `paste.mo` nor `dpaste.de`). Instead we have a mapping of known filename extensions to known available highlighters on `paste.mo`. `mach pastebin` will attempt to guess a highlighter from the file extension or the basename of the file if none is available. Differential Revision: https://phabricator.services.mozilla.com/D42902
tools/mach_commands.py
--- a/tools/mach_commands.py
+++ b/tools/mach_commands.py
@@ -111,29 +111,238 @@ class UUIDProvider(object):
                 print('')
         if format in [None, 'cpp', 'c++']:
             u = u.hex
             print('{ 0x%s, 0x%s, 0x%s, \\' % (u[0:8], u[8:12], u[12:16]))
             pairs = tuple(map(lambda n: u[n:n+2], range(16, 32, 2)))
             print(('  { ' + '0x%s, ' * 7 + '0x%s } }') % pairs)
 
 
-def REMOVED(cls):
-    """Command no longer exists!
+MACH_PASTEBIN_DURATIONS = {
+    'onetime': 'onetime',
+    'hour': '3600',
+    'day': '86400',
+    'week': '604800',
+    'month': '2073600',
+}
 
-    This functionality is no longer supported in mach.
-    """
-    return False
+EXTENSION_TO_HIGHLIGHTER = {
+    '.hgrc': 'ini',
+    'Dockerfile': 'docker',
+    'Makefile': 'make',
+    'applescript': 'applescript',
+    'arduino': 'arduino',
+    'bash': 'bash',
+    'bat': 'bat',
+    'c': 'c',
+    'clojure': 'clojure',
+    'cmake': 'cmake',
+    'coffee': 'coffee-script',
+    'console': 'console',
+    'cpp': 'cpp',
+    'cs': 'csharp',
+    'css': 'css',
+    'cu': 'cuda',
+    'cuda': 'cuda',
+    'dart': 'dart',
+    'delphi': 'delphi',
+    'diff': 'diff',
+    'django': 'django',
+    'docker': 'docker',
+    'elixir': 'elixir',
+    'erlang': 'erlang',
+    'go': 'go',
+    'h': 'c',
+    'handlebars': 'handlebars',
+    'haskell': 'haskell',
+    'hs': 'haskell',
+    'html': 'html',
+    'ini': 'ini',
+    'ipy': 'ipythonconsole',
+    'ipynb': 'ipythonconsole',
+    'irc': 'irc',
+    'j2': 'django',
+    'java': 'java',
+    'js': 'js',
+    'json': 'json',
+    'jsx': 'jsx',
+    'kt': 'kotlin',
+    'less': 'less',
+    'lisp': 'common-lisp',
+    'lsp': 'common-lisp',
+    'lua': 'lua',
+    'm': 'objective-c',
+    'make': 'make',
+    'matlab': 'matlab',
+    'md': '_markdown',
+    'nginx': 'nginx',
+    'numpy': 'numpy',
+    'patch': 'diff',
+    'perl': 'perl',
+    'php': 'php',
+    'pm': 'perl',
+    'postgresql': 'postgresql',
+    'py': 'python',
+    'rb': 'rb',
+    'rs': 'rust',
+    'rst': 'rst',
+    'sass': 'sass',
+    'scss': 'scss',
+    'sh': 'bash',
+    'sol': 'sol',
+    'sql': 'sql',
+    'swift': 'swift',
+    'tex': 'tex',
+    'typoscript': 'typoscript',
+    'vim': 'vim',
+    'xml': 'xml',
+    'xslt': 'xslt',
+    'yaml': 'yaml',
+    'yml': 'yaml'
+}
+
+
+def guess_highlighter_from_path(path):
+    '''Return a known highlighter from a given path
+
+    Attempt to select a highlighter by checking the file extension in the mapping
+    of extensions to highlighter. If that fails, attempt to pass the basename of
+    the file. Return `_code` as the default highlighter if that fails.
+    '''
+    import os
+
+    _name, ext = os.path.splitext(path)
+
+    if ext.startswith('.'):
+        ext = ext[1:]
+
+    if ext in EXTENSION_TO_HIGHLIGHTER:
+        return EXTENSION_TO_HIGHLIGHTER[ext]
+
+    basename = os.path.basename(path)
+
+    return EXTENSION_TO_HIGHLIGHTER.get(basename, '_code')
+
+
+PASTEMO_MAX_CONTENT_LENGTH = 250 * 1024 * 1024
+
+PASTEMO_URL = 'https://paste.mozilla.org/api/'
+
+MACH_PASTEBIN_DESCRIPTION = '''
+Command line interface to paste.mozilla.org.
+
+Takes either a filename whose content should be pasted, or reads
+content from standard input. If a highlighter is specified it will
+be used, otherwise the file name will be used to determine an
+appropriate highlighter.
+'''
 
 
 @CommandProvider
 class PastebinProvider(object):
-    @Command('pastebin', category='misc', conditions=[REMOVED])
-    def pastebin(self):
-        pass
+    @Command('pastebin', category='misc',
+             description=MACH_PASTEBIN_DESCRIPTION)
+    @CommandArgument('--list-highlighters', action='store_true',
+                     help='List known highlighters and exit')
+    @CommandArgument('--highlighter', default=None,
+                     help='Syntax highlighting to use for paste')
+    @CommandArgument('--expires', default='week',
+                     choices=sorted(MACH_PASTEBIN_DURATIONS.keys()),
+                     help='Expire paste after given time duration (default: %(default)s)')
+    @CommandArgument('--verbose', action='store_true',
+                     help='Print extra info such as selected syntax highlighter')
+    @CommandArgument('path', nargs='?', default=None,
+                     help='Path to file for upload to paste.mozilla.org')
+    def pastebin(self, list_highlighters, highlighter, expires, verbose, path):
+        import requests
+
+        def verbose_print(*args, **kwargs):
+            '''Print a string if `--verbose` flag is set'''
+            if verbose:
+                print(*args, **kwargs)
+
+        # Show known highlighters and exit.
+        if list_highlighters:
+            lexers = set(EXTENSION_TO_HIGHLIGHTER.values())
+            print('Available lexers:\n'
+                  '    - %s' % '\n    - '.join(sorted(lexers)))
+            return 0
+
+        # Get a correct expiry value.
+        try:
+            verbose_print('Setting expiry from %s' % expires)
+            expires = MACH_PASTEBIN_DURATIONS[expires]
+            verbose_print('Using %s as expiry' % expires)
+        except KeyError:
+            print('%s is not a valid duration.\n'
+                  '(hint: try one of %s)' %
+                  (expires, ', '.join(MACH_PASTEBIN_DURATIONS.keys())))
+            return 1
+
+        data = {
+            'format': 'json',
+            'expires': expires,
+        }
+
+        # Get content to be pasted.
+        if path:
+            verbose_print('Reading content from %s' % path)
+            try:
+                with open(path, 'r') as f:
+                    content = f.read()
+            except IOError:
+                print('ERROR. No such file %s' % path)
+                return 1
+
+            lexer = guess_highlighter_from_path(path)
+            if lexer:
+                data['lexer'] = lexer
+        else:
+            verbose_print('Reading content from stdin')
+            content = sys.stdin.read()
+
+        # Assert the length of content to be posted does not exceed the maximum.
+        content_length = len(content)
+        verbose_print('Checking size of content is okay (%d)' % content_length)
+        if content_length > PASTEMO_MAX_CONTENT_LENGTH:
+            print('Paste content is too large (%d, maximum %d)' %
+                  (content_length, PASTEMO_MAX_CONTENT_LENGTH))
+            return 1
+
+        data['content'] = content
+
+        # Highlight as specified language, overwriting value set from filename.
+        if highlighter:
+            verbose_print('Setting %s as highlighter' % highlighter)
+            data['lexer'] = highlighter
+
+        try:
+            verbose_print('Sending request to %s' % PASTEMO_URL)
+            resp = requests.post(PASTEMO_URL, data=data)
+
+            # Error code should always be 400.
+            # Response content will include a helpful error message,
+            # so print it here (for example, if an invalid highlighter is
+            # provided, it will return a list of valid highlighters).
+            if resp.status_code >= 400:
+                print('Error code %d: %s' % (resp.status_code, resp.content))
+                return 1
+
+            verbose_print('Pasted successfully')
+
+            response_json = resp.json()
+
+            verbose_print('Paste highlighted as %s' % response_json['lexer'])
+            print(response_json['url'])
+
+            return 0
+        except Exception as e:
+            print('ERROR. Paste failed.')
+            print('%s' % e)
+        return 1
 
 
 def mozregression_import():
     # Lazy loading of mozregression.
     # Note that only the mach_interface module should be used from this file.
     try:
         import mozregression.mach_interface
     except ImportError: