Bug 784691 - Upgrade clang to r162360. Use a local copy of tooltool and setup.sh.
authorRafael Ávila de Espíndola <respindola@mozilla.com>
Thu, 23 Aug 2012 12:52:50 -0400
changeset 105221 5853a841672ebbdd4f45e8d05610af903a4353d1
parent 105220 fcf13879b58754bac6f084d0cf340398113bc2f9
child 105222 00dea115746d20840e13d816e8be618964464bf5
push id55
push usershu@rfrn.org
push dateThu, 30 Aug 2012 01:33:09 +0000
bugs784691
milestone17.0a1
Bug 784691 - Upgrade clang to r162360. Use a local copy of tooltool and setup.sh.
build/unix/build-clang/create-manifest.py
build/unix/build-clang/tooltool.py
--- a/build/unix/build-clang/create-manifest.py
+++ b/build/unix/build-clang/create-manifest.py
@@ -1,39 +1,39 @@
 #!/bin/python
 
 import os
+import os.path
 import simplejson
 import sys
 import subprocess
 import urllib
 import glob
 
 def check_run(args):
     r = subprocess.call(args)
     assert r == 0
 
-old_files = glob.glob('*.manifest') + ['tooltool.py', 'setup.sh']
+old_files = glob.glob('*.manifest')
 for f in old_files:
     try:
         os.unlink(f)
     except:
         pass
 
-urllib.urlretrieve('https://raw.github.com/jhford/tooltool/master/tooltool.py',
-                   'tooltool.py')
-urllib.urlretrieve('https://hg.mozilla.org/mozilla-central/raw-file/tip/build/unix/build-clang/setup.sh',
-                   'setup.sh')
+basedir = os.path.split(os.path.realpath(sys.argv[0]))[0]
+tooltool = basedir + '/tooltool.py'
+setup = basedir + '/setup.sh'
 
-check_run(['python', 'tooltool.py', '-m', 'linux32.manifest', 'add',
-           'clang-linux32.tar.bz2', 'setup.sh'])
-check_run(['python', 'tooltool.py', '-m', 'linux64.manifest', 'add',
-           'clang-linux64.tar.bz2', 'setup.sh'])
-check_run(['python', 'tooltool.py', '-m', 'darwin.manifest', 'add',
-           'clang-darwin.tar.bz2', 'setup.sh'])
+check_run(['python', tooltool, '-m', 'linux32.manifest', 'add',
+           'clang-linux32.tar.bz2', setup])
+check_run(['python', tooltool, '-m', 'linux64.manifest', 'add',
+           'clang-linux64.tar.bz2', setup])
+check_run(['python', tooltool, '-m', 'darwin.manifest', 'add',
+           'clang-darwin.tar.bz2', setup])
 
 def key_sort(item):
     item = item[0]
     if item == 'size':
         return 0
     if item == 'digest':
         return 1
     if item == 'algorithm':
new file mode 100644
--- /dev/null
+++ b/build/unix/build-clang/tooltool.py
@@ -0,0 +1,564 @@
+#!/usr/bin/env python
+
+#tooltool is a lookaside cache implemented in Python
+#Copyright (C) 2011 John H. Ford <john@johnford.info>
+#
+#This program is free software; you can redistribute it and/or
+#modify it under the terms of the GNU General Public License
+#as published by the Free Software Foundation version 2
+#
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#GNU General Public License for more details.
+#
+#You should have received a copy of the GNU General Public License
+#along with this program; if not, write to the Free Software
+#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+# An manifest file specifies files in that directory that are stored
+# elsewhere.  This file should only contain file in the directory
+# which the manifest file resides in and it should be called 'manifest.manifest'
+
+__version__ = '1'
+
+import os
+import optparse
+import logging
+import hashlib
+import urllib2
+import ConfigParser
+try:
+    import simplejson as json # I hear simplejson is faster
+except ImportError:
+    import json
+
+log = logging.getLogger(__name__)
+
+class FileRecordJSONEncoderException(Exception): pass
+class InvalidManifest(Exception): pass
+class ExceptionWithFilename(Exception):
+    def __init__(self, filename):
+        Exception.__init__(self)
+        self.filename = filename
+
+class DigestMismatchException(ExceptionWithFilename): pass
+class MissingFileException(ExceptionWithFilename): pass
+
+class FileRecord(object):
+    def __init__(self, filename, size, digest, algorithm):
+        object.__init__(self)
+        self.filename = filename
+        self.size = size
+        self.digest = digest
+        self.algorithm = algorithm
+        log.debug("creating %s 0x%x" % (self.__class__.__name__, id(self)))
+
+    def __eq__(self, other):
+        if self is other:
+            return True
+        if self.filename == other.filename and \
+            self.size == other.size and \
+            self.digest == other.digest and \
+            self.algorithm == other.algorithm:
+            return True
+        else:
+            return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "%s.%s(filename='%s', size='%s', digest='%s', algorithm='%s')" % (__name__,
+                self.__class__.__name__,
+                self.filename, self.size, self.digest, self.algorithm)
+
+    def present(self):
+        # Doesn't check validity
+        return os.path.exists(self.filename)
+
+    def validate_size(self):
+        if self.present():
+            return self.size == os.path.getsize(self.filename)
+        else:
+            log.debug("trying to validate size on a missing file, %s", self.filename)
+            raise MissingFileException(filename=self.filename)
+
+    def validate_digest(self):
+        if self.present():
+            with open(self.filename, 'rb') as f:
+                return self.digest == digest_file(f, self.algorithm)
+        else:
+            log.debug("trying to validate digest on a missing file, %s', self.filename")
+            raise MissingFileException(filename=self.filename)
+
+    def validate(self):
+        if self.validate_size():
+            if self.validate_digest():
+                return True
+        return False
+
+    def describe(self):
+        if self.present() and self.validate():
+            return "'%s' is present and valid" % self.filename
+        elif self.present():
+            return "'%s' is present and invalid" % self.filename
+        else:
+            return "'%s' is absent" % self.filename
+
+
+def create_file_record(filename, algorithm):
+    fo = open(filename, 'rb')
+    stored_filename = os.path.split(filename)[1]
+    fr = FileRecord(stored_filename, os.path.getsize(filename), digest_file(fo, algorithm), algorithm)
+    fo.close()
+    return fr
+
+
+class FileRecordJSONEncoder(json.JSONEncoder):
+    def encode_file_record(self, obj):
+        if not issubclass(type(obj), FileRecord):
+            err = "FileRecordJSONEncoder is only for FileRecord and lists of FileRecords, not %s" % obj.__class__.__name__
+            log.warn(err)
+            raise FileRecordJSONEncoderException(err)
+        else:
+            return {'filename': obj.filename, 'size': obj.size, 'algorithm': obj.algorithm, 'digest': obj.digest}
+
+    def default(self, f):
+        if issubclass(type(f), list):
+            record_list = []
+            for i in f:
+                record_list.append(self.encode_file_record(i))
+            return record_list
+        else:
+            return self.encode_file_record(f)
+
+
+class FileRecordJSONDecoder(json.JSONDecoder):
+    """I help the json module materialize a FileRecord from
+    a JSON file.  I understand FileRecords and lists of
+    FileRecords.  I ignore things that I don't expect for now"""
+    # TODO: make this more explicit in what it's looking for
+    # and error out on unexpected things
+    def process_file_records(self, obj):
+        if isinstance(obj, list):
+            record_list = []
+            for i in obj:
+                record = self.process_file_records(i)
+                if issubclass(type(record), FileRecord):
+                    record_list.append(record)
+            return record_list
+        if isinstance(obj, dict) and \
+           len(obj.keys()) == 4 and \
+           obj.has_key('filename') and \
+           obj.has_key('size') and \
+           obj.has_key('algorithm') and \
+           obj.has_key('digest'):
+            rv = FileRecord(obj['filename'], obj['size'], obj['digest'], obj['algorithm'])
+            log.debug("materialized %s" % rv)
+            return rv
+        return obj
+
+    def decode(self, s):
+        decoded = json.JSONDecoder.decode(self, s)
+        rv = self.process_file_records(decoded)
+        return rv
+
+
+class Manifest(object):
+
+    valid_formats = ('json',)
+
+    def __init__(self, file_records=[]):
+        self.file_records = file_records
+
+    def __eq__(self, other):
+        if self is other:
+            return True
+        if len(self.file_records) != len(other.file_records):
+            log.debug('Manifests differ in number of files')
+            return False
+        #TODO: Lists in a different order should be equal
+        for record in range(0,len(self.file_records)):
+            if self.file_records[record] != other.file_records[record]:
+                log.debug('FileRecords differ, %s vs %s' % (self.file_records[record],
+                                                            other.file_records[record]))
+                return False
+        return True
+
+    def __deepcopy__(self, memo):
+        # This is required for a deep copy
+        return Manifest(self.file_records[:])
+
+    def __copy__(self):
+        return Manifest(self.file_records)
+
+    def copy(self):
+        return Manifest(self.file_records[:])
+
+    def present(self):
+        return all(i.present() for i in self.file_records)
+
+    def validate_sizes(self):
+        return all(i.validate_size() for i in self.file_records)
+
+    def validate_digests(self):
+        return all(i.validate_digest() for i in self.file_records)
+
+    def validate(self):
+        return all(i.validate() for i in self.file_records)
+
+    def sort(self):
+        #TODO: WRITE TESTS
+        self.file_records.sort(key=lambda x: x.size)
+
+    def load(self, data_file, fmt='json'):
+        assert fmt in self.valid_formats
+        if fmt == 'json':
+            try:
+                self.file_records.extend(json.load(data_file, cls=FileRecordJSONDecoder))
+                self.sort()
+            except ValueError:
+                raise InvalidManifest("trying to read invalid manifest file")
+
+    def loads(self, data_string, fmt='json'):
+        assert fmt in self.valid_formats
+        if fmt == 'json':
+            try:
+                self.file_records.extend(json.loads(data_string, cls=FileRecordJSONDecoder))
+                self.sort()
+            except ValueError:
+                raise InvalidManifest("trying to read invalid manifest file")
+
+    def dump(self, output_file, fmt='json'):
+        assert fmt in self.valid_formats
+        self.sort()
+        if fmt == 'json':
+            rv = json.dump(self.file_records, output_file, indent=0, cls=FileRecordJSONEncoder)
+            print >> output_file, ''
+            return rv
+
+    def dumps(self, fmt='json'):
+        assert fmt in self.valid_formats
+        self.sort()
+        if fmt == 'json':
+            return json.dumps(self.file_records, cls=FileRecordJSONEncoder)
+
+
+def digest_file(f, a):
+    """I take a file like object 'f' and return a hex-string containing
+    of the result of the algorithm 'a' applied to 'f'."""
+    h = hashlib.new(a)
+    chunk_size = 1024*10
+    data = f.read(chunk_size)
+    while data:
+        h.update(data)
+        data = f.read(chunk_size)
+    if hasattr(f, 'name'):
+        log.debug('hashed %s with %s to be %s', f.name, a, h.hexdigest())
+    else:
+        log.debug('hashed a file with %s to be %s', a, h.hexdigest())
+    return h.hexdigest()
+
+# TODO: write tests for this function
+def open_manifest(manifest_file):
+    """I know how to take a filename and load it into a Manifest object"""
+    if os.path.exists(manifest_file):
+        manifest = Manifest()
+        with open(manifest_file) as f:
+            manifest.load(f)
+            log.debug("loaded manifest from file '%s'" % manifest_file)
+        return manifest
+    else:
+        log.debug("tried to load absent file '%s' as manifest" % manifest_file)
+        raise InvalidManifest("manifest file '%s' does not exist" % manifest_file)
+
+# TODO: write tests for this function
+def list_manifest(manifest_file):
+    """I know how print all the files in a location"""
+    try:
+        manifest = open_manifest(manifest_file)
+    except InvalidManifest:
+        log.error("failed to load manifest file at '%s'" % manifest_file)
+        return False
+    for f in manifest.file_records:
+        print "%s\t%s\t%s" % ("P" if f.present() else "-",
+                              "V" if f.present() and f.validate() else "-",
+                              f.filename)
+    return True
+
+def validate_manifest(manifest_file):
+    """I validate that all files in a manifest are present and valid but
+    don't fetch or delete them if they aren't"""
+    try:
+        manifest = open_manifest(manifest_file)
+    except InvalidManifest:
+        log.error("failed to load manifest file at '%s'" % manifest_file)
+        return False
+    invalid_files = []
+    absent_files = []
+    for f in manifest.file_records:
+        if not f.present():
+            absent_files.append(f)
+        else:
+            if not f.validate():
+                invalid_files.append(f)
+    if len(invalid_files + absent_files) == 0:
+        return True
+    else:
+        return False
+
+# TODO: write tests for this function
+def add_files(manifest_file, algorithm, filenames):
+    # returns True if all files successfully added, False if not
+    # and doesn't catch library Exceptions.  If any files are already
+    # tracked in the manifest, return will be False because they weren't
+    # added
+    all_files_added = True
+    # Create a old_manifest object to add to
+    if os.path.exists(manifest_file):
+        old_manifest = open_manifest(manifest_file)
+    else:
+        old_manifest = Manifest()
+        log.debug("creating a new manifest file")
+    new_manifest = Manifest() # use a different manifest for the output
+    for filename in filenames:
+        log.debug("adding %s" % filename)
+        path, name = os.path.split(filename)
+        new_fr = create_file_record(filename, algorithm)
+        log.debug("appending a new file record to manifest file")
+        add = True
+        for fr in old_manifest.file_records:
+            log.debug("manifest file has '%s'" % "', ".join([x.filename for x in old_manifest.file_records]))
+            if new_fr == fr and new_fr.validate():
+                # TODO: Decide if this case should really cause a False return
+                log.info("file already in old_manifest file and matches")
+                add = False
+            elif new_fr == fr and not new_fr.validate():
+                log.error("file already in old_manifest file but is invalid")
+                add = False
+            if filename == fr.filename:
+                log.error("manifest already contains file named %s" % filename)
+                add = False
+        if add:
+            new_manifest.file_records.append(new_fr)
+            log.debug("added '%s' to manifest" % filename)
+        else:
+            all_files_added = False
+    with open(manifest_file, 'wb') as output:
+        new_manifest.dump(output, fmt='json')
+    return all_files_added
+
+
+# TODO: write tests for this function
+def fetch_file(base_url, file_record, overwrite=False, grabchunk=1024*4):
+    # A file which is requested to be fetched that exists locally will be hashed.
+    # If the hash matches the requested file's hash, nothing will be done and the
+    # function will return.  If the function is told to overwrite and there is a 
+    # digest mismatch, the exiting file will be overwritten
+    if file_record.present():
+        if file_record.validate():
+            log.info("existing '%s' is valid, not fetching" % file_record.filename)
+            return True
+        if overwrite:
+            log.info("overwriting '%s' as requested" % file_record.filename)
+        else:
+            # All of the following is for a useful error message
+            with open(file_record.filename, 'rb') as f:
+                d = digest_file(f, file_record.algorithm)
+            log.error("digest mismatch between manifest(%s...) and local file(%s...)" % \
+                    (file_record.digest[:8], d[:8]))
+            log.debug("full digests: manifest (%s) local file (%s)" % (file_record.digest, d))
+            # Let's bail!
+            return False
+
+    # Generate the URL for the file on the server side
+    url = "%s/%s/%s" % (base_url, file_record.algorithm, file_record.digest)
+
+    log.debug("fetching from '%s'" % url)
+
+    # TODO: This should be abstracted to make generic retreival protocol handling easy
+    # Well, the file doesn't exist locally.  Lets fetch it.
+    try:
+        f = urllib2.urlopen(url)
+        log.debug("opened %s for reading" % url)
+        with open(file_record.filename, 'wb') as out:
+            k = True
+            size = 0
+            while k:
+                # TODO: print statistics as file transfers happen both for info and to stop
+                # buildbot timeouts
+                indata = f.read(grabchunk)
+                out.write(indata)
+                size += len(indata)
+                if indata == '':
+                    k = False
+            if size != file_record.size:
+                log.error("transfer from %s to %s failed due to a difference of %d bytes" % (url,
+                            file_record.filename, file_record.size - size))
+                return False
+            log.info("fetched %s" % file_record.filename)
+    except (urllib2.URLError, urllib2.HTTPError) as e:
+        log.error("failed to fetch '%s': %s" % (file_record.filename, e),
+                  exc_info=True)
+        return False
+    except IOError:
+        log.error("failed to write to '%s'" % file_record.filename,
+                  exc_info=True)
+        return False
+    return True
+
+
+# TODO: write tests for this function
+def fetch_files(manifest_file, base_url, overwrite, filenames=[]):
+    # Lets load the manifest file
+    try:
+        manifest = open_manifest(manifest_file)
+    except InvalidManifest:
+        log.error("failed to load manifest file at '%s'" % manifest_file)
+        return False
+    # We want to track files that fail to be fetched as well as
+    # files that are fetched
+    failed_files = []
+
+    # Lets go through the manifest and fetch the files that we want
+    fetched_files = []
+    for f in manifest.file_records:
+        if f.filename in filenames or len(filenames) == 0:
+            log.debug("fetching %s" % f.filename)
+            if fetch_file(base_url, f, overwrite):
+                fetched_files.append(f)
+            else:
+                failed_files.append(f.filename)
+        else:
+            log.debug("skipping %s" % f.filename)
+
+    # Even if we get the file, lets ensure that it matches what the
+    # manifest specified
+    for localfile in fetched_files:
+        if not localfile.validate():
+            log.error("'%s'" % localfile.describe())
+
+    # If we failed to fetch or validate a file, we need to fail
+    if len(failed_files) > 0:
+        log.error("The following files failed: '%s'" % "', ".join(failed_files))
+        return False
+    return True
+
+
+# TODO: write tests for this function
+def process_command(options, args):
+    """ I know how to take a list of program arguments and
+    start doing the right thing with them"""
+    cmd = args[0]
+    cmd_args = args[1:]
+    log.debug("processing '%s' command with args '%s'" % (cmd, '", "'.join(cmd_args)))
+    log.debug("using options: %s" % options)
+    if cmd == 'list':
+        return list_manifest(options['manifest'])
+    if cmd == 'validate':
+        return validate_manifest(options['manifest'])
+    elif cmd == 'add':
+        return add_files(options['manifest'], options['algorithm'], cmd_args)
+    elif cmd == 'fetch':
+        if not options.has_key('base_url') or options.get('base_url') is None:
+            log.critical('fetch command requires url option')
+            return False
+        return fetch_files(options['manifest'], options['base_url'], options['overwrite'], cmd_args)
+    else:
+        log.critical('command "%s" is not implemented' % cmd)
+        return False
+
+# fetching api:
+#   http://hostname/algorithm/hash
+#   example: http://people.mozilla.org/sha1/1234567890abcedf
+# This will make it possible to have the server allow clients to
+# use different algorithms than what was uploaded to the server
+
+# TODO: Implement the following features:
+#   -optimization: do small files first, justification is that they are faster
+#    and cause a faster failure if they are invalid
+#   -store permissions
+#   -local renames i.e. call the file one thing on the server and
+#    something different locally
+#   -deal with the cases:
+#     -local data matches file requested with different filename
+#     -two different files with same name, different hash
+#   -?only ever locally to digest as filename, symlink to real name
+#   -?maybe deal with files as a dir of the filename with all files in that dir as the versions of that file
+#      - e.g. ./python-2.6.7.dmg/0123456789abcdef and ./python-2.6.7.dmg/abcdef0123456789
+
+def main():
+    # Set up logging, for now just to the console
+    ch = logging.StreamHandler()
+    cf = logging.Formatter("%(levelname)s - %(message)s")
+    ch.setFormatter(cf)
+
+    # Set up option parsing
+    parser = optparse.OptionParser()
+    # I wish there was a way to say "only allow args to be
+    # sequential and at the end of the argv.
+    # OH! i could step through sys.argv and check for things starting without -/-- before things starting with them
+    parser.add_option('-q', '--quiet', default=False,
+            dest='quiet', action='store_true')
+    parser.add_option('-v', '--verbose', default=False,
+            dest='verbose', action='store_true')
+    parser.add_option('-m', '--manifest', default='manifest.tt',
+            dest='manifest', action='store',
+            help='specify the manifest file to be operated on')
+    parser.add_option('-d', '--algorithm', default='sha512',
+            dest='algorithm', action='store',
+            help='openssl hashing algorithm to use')
+    parser.add_option('-o', '--overwrite', default=False,
+            dest='overwrite', action='store_true',
+            help='if fetching, remote copy will overwrite a local copy that is different. ')
+    parser.add_option('--url', dest='base_url', action='store',
+            help='base url for fetching files')
+    parser.add_option('--ignore-config-files', action='store_true', default=False,
+                     dest='ignore_cfg_files')
+    (options_obj, args) = parser.parse_args()
+    # Dictionaries are easier to work with
+    options = vars(options_obj)
+
+
+    # Use some of the option parser to figure out application
+    # log level
+    if options.get('verbose'):
+        ch.setLevel(logging.DEBUG)
+    elif options.get('quiet'):
+        ch.setLevel(logging.ERROR)
+    else:
+        ch.setLevel(logging.INFO)
+    log.addHandler(ch)
+
+    cfg_file = ConfigParser.SafeConfigParser()
+    if not options.get("ignore_cfg_files"):
+        read_files = cfg_file.read(['/etc/tooltool', os.path.expanduser('~/.tooltool'),
+                   os.path.join(os.getcwd(), '.tooltool')])
+        log.debug("read in the config files '%s'" % '", '.join(read_files))
+    else:
+        log.debug("skipping config files")
+
+    for option in ('base_url', 'algorithm'):
+        if not options.get(option):
+            try:
+                options[option] = cfg_file.get('general', option)
+                log.debug("read '%s' as '%s' from cfg_file" % (option, options[option]))
+            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError) as e:
+                log.debug("%s in config file" % e, exc_info=True)
+
+    if not options.has_key('manifest'):
+        parser.error("no manifest file specified")
+
+    if len(args) < 1:
+        parser.error('You must specify a command')
+    exit(0 if process_command(options, args) else 1)
+
+if __name__ == "__main__":
+    main()
+else:
+    log.addHandler(logging.NullHandler())
+    #log.addHandler(logging.StreamHandler())