Bug 578393 - Create a file with checksums for all builds that we upload, r=ted a=gavin
authorJohn Ford <jhford@mozilla.com>
Fri, 15 Oct 2010 16:09:53 -0700
changeset 55922 ab6d8c5a300a9f1b54f9d86628d6292ad89d1c15
parent 55921 1025790e0c072e738da9d37547537f4ee8ca977a
child 55923 2402ccc1d42833639b140c7ea73a80c4f7a6d35b
push idunknown
push userunknown
push dateunknown
reviewersted, gavin
bugs578393
milestone2.0b8pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 578393 - Create a file with checksums for all builds that we upload, r=ted a=gavin
build/checksums.py
toolkit/mozapps/installer/packager.mk
new file mode 100755
--- /dev/null
+++ b/build/checksums.py
@@ -0,0 +1,187 @@
+#!/usr/bin/python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is a checksum generation utility.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#    John Ford <jhford@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+from optparse import OptionParser
+import logging
+import os
+try:
+    import hashlib
+except:
+    hashlib = None
+
+def digest_file(filename, digest, chunk_size=1024):
+    '''Produce a checksum for the file specified by 'filename'.  'filename'
+    is a string path to a file that is opened and read in this function.  The
+    checksum algorithm is specified by 'digest' and is a valid OpenSSL
+    algorithm.  If the digest used is not valid or Python's hashlib doesn't
+    work, the None object will be returned instead.  The size of blocks
+    that this function will read from the file object it opens based on
+    'filename' can be specified by 'chunk_size', which defaults to 1K'''
+    assert not os.path.isdir(filename), 'this function only works with files'
+    logger = logging.getLogger('checksums.py')
+    if hashlib is not None:
+        logger.debug('Creating new %s object' % digest)
+        h = hashlib.new(digest)
+        f = open(filename)
+        while True:
+            data = f.read(chunk_size)
+            if not data:
+                logger.debug('Finished reading in file')
+                break
+            h.update(data)
+        f.close()
+        hash = h.hexdigest()
+        logger.debug('Hash for %s is %s' % (filename, hash))
+        return hash
+    else:
+        # In this case we could subprocess.Popen and .communicate with
+        # sha1sum or md5sum
+        logger.warn('The python module for hashlib is missing!')
+        return None
+
+
+def process_files(files, output_filename, digest, strip):
+    '''This function takes a list of file names, 'files'.  It will then
+    compute the checksum for each of the files by opening the files.
+    Once each file is read and its checksum is computed, this function
+    will write the information to the file specified by 'output_filename'.
+    The path written in the output file will have anything specified by 'strip'
+    removed from the path.  The output file is closed before returning nothing
+    The algorithm to compute checksums with can be specified by 'digest' 
+    and needs to be a valid OpenSSL algorithm.
+
+    The output file is written in the format:
+        <hash> <algorithm> <filesize> <filepath>
+    Example:
+        d1fa09a<snip>e4220 sha1 14250744 firefox-4.0b6pre.en-US.mac64.dmg
+    '''
+
+    logger = logging.getLogger('checksums.py')
+    if os.path.exists(output_filename):
+        logger.debug('Overwriting existing checksums file "%s"' %
+                     output_filename)
+    else:
+        logger.debug('Creating a new checksums file "%s"' % output_filename)
+    output = open(output_filename, 'w+')
+    for file in files:
+        if os.path.isdir(file):
+            logger.warn('%s is a directory, skipping' % file)
+        else:
+            hash = digest_file(file, digest)
+            if hash is None:
+                logger.warn('Unable to generate a hash for %s. ' +
+                            'Using NOHASH as fallback' % file)
+                hash = 'NOHASH'
+            if file.startswith(strip):
+                short_file = file[len(strip):]
+                short_file = short_file.lstrip('/')
+            else:
+                short_file = file
+            print >>output, '%s %s %s %s' % (hash, digest,
+                                             os.path.getsize(file),
+                                             short_file)
+    output.close()
+
+def setup_logging(level=logging.DEBUG):
+    '''This function sets up the logging module using a speficiable logging
+    module logging level.  The default log level is DEBUG.
+
+    The output is in the format:
+        <level> - <message>
+    Example:
+        DEBUG - Finished reading in file
+'''
+
+    logger = logging.getLogger('checksums.py')
+    logger.setLevel(logging.DEBUG)
+    handler = logging.StreamHandler()
+    handler.setLevel(level)
+    formatter = logging.Formatter("%(levelname)s - %(message)s")
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+def main():
+    '''This is a main function that parses arguments, sets up logging
+    and generates a checksum file'''
+    # Parse command line arguments
+    parser = OptionParser()
+    parser.add_option('-d', '--digest', help='checksum algorithm to use',
+                      action='store', dest='digest', default='sha1')
+    parser.add_option('-o', '--output', help='output file to use',
+                      action='store', dest='outfile', default='checksums')
+    parser.add_option('-v', '--verbose',
+                      help='Be noisy (takes precedence over quiet)',
+                      action='store_true', dest='verbose', default=False)
+    parser.add_option('-q', '--quiet', help='Be quiet', action='store_true',
+                      dest='quiet', default=False)
+    parser.add_option('-s', '--strip',
+                      help='strip this path from the filenames',
+                      dest='strip', default=os.getcwd())
+    options, args = parser.parse_args()
+
+    #Figure out which logging level to use
+    if options.verbose:
+        loglevel = logging.DEBUG
+    elif options.quiet:
+        loglevel = logging.ERROR
+    else:
+        loglevel = logging.INFO
+
+    #Set up logging
+    setup_logging(loglevel)
+    logger = logging.getLogger('checksums.py')
+
+    # Validate the digest type to use
+    try:
+        hashlib.new(options.digest)
+    except ValueError, ve:
+        logger.error('Could not create a "%s" hash object (%s)' %
+                     (options.digest, ve.args[0]))
+        exit(1)
+
+    # Validate the files to checksum
+    files = []
+    for i in args:
+        if os.path.exists(i):
+            files.append(i)
+        else:
+            logger.info('File "%s" was not found on the filesystem' % i)
+    process_files(files, options.outfile, options.digest, options.strip)
+
+if __name__ == '__main__':
+    main()
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -649,28 +649,49 @@ endif
 
 # These are necessary because some of our packages/installers contain spaces
 # in their filenames and GNU Make's $(wildcard) function doesn't properly
 # deal with them.
 empty :=
 space = $(empty) $(empty)
 QUOTED_WILDCARD = $(if $(wildcard $(subst $(space),?,$(1))),"$(1)")
 
-upload:
+# This variable defines which OpenSSL algorithm to use to 
+# generate checksums for files that we upload
+CHECKSUM_ALGORITHM = 'sha512'
+
+# This variable defines where the checksum file will be located
+CHECKSUM_FILE = $(DIST)/$(PKG_PATH)/$(PKG_BASENAME).checksums
+
+UPLOAD_FILES= \
+  $(call QUOTED_WILDCARD,$(DIST)/$(PACKAGE)) \
+  $(call QUOTED_WILDCARD,$(INSTALLER_PACKAGE)) \
+  $(call QUOTED_WILDCARD,$(DIST)/$(COMPLETE_MAR)) \
+  $(call QUOTED_WILDCARD,$(wildcard $(DIST)/$(PARTIAL_MAR))) \
+  $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)) \
+  $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip) \
+  $(call QUOTED_WILDCARD,$(DIST)/$(SDK)) \
+  $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)/$(PKG_BASENAME).txt) \
+  $(if $(UPLOAD_EXTRA_FILES), $(foreach f, $(UPLOAD_EXTRA_FILES), $(wildcard $(DIST)/$(f))))
+
+checksum:
+	@$(PYTHON) $(MOZILLA_DIR)/build/checksums.py \
+		-o $(CHECKSUM_FILE) \
+		-d $(CHECKSUM_ALGORITHM) \
+		-s $(call QUOTED_WILDCARD,$(DIST)) \
+		$(UPLOAD_FILES)
+	@echo "CHECKSUM FILE START"
+	@cat $(CHECKSUM_FILE)
+	@echo "CHECKSUM FILE END"
+
+
+upload: checksum
 	$(PYTHON) $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) \
-		$(call QUOTED_WILDCARD,$(DIST)/$(PACKAGE)) \
-		$(call QUOTED_WILDCARD,$(INSTALLER_PACKAGE)) \
-		$(call QUOTED_WILDCARD,$(DIST)/$(COMPLETE_MAR)) \
-		$(call QUOTED_WILDCARD,$(wildcard $(DIST)/$(PARTIAL_MAR))) \
-		$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)) \
-		$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip) \
-		$(call QUOTED_WILDCARD,$(DIST)/$(SDK)) \
-		$(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)/$(PKG_BASENAME).txt) \
-		$(call QUOTED_WILDCARD,$(DIST)/$(LANGPACK)) \
-		$(if $(UPLOAD_EXTRA_FILES), $(foreach f, $(UPLOAD_EXTRA_FILES), $(wildcard $(DIST)/$(f))))
+		$(UPLOAD_FILES) \
+		$(CHECKSUM_FILE)
 
 ifndef MOZ_PKG_SRCDIR
 MOZ_PKG_SRCDIR = $(topsrcdir)
 endif
 
 CREATE_SOURCE_TAR = $(TAR) -c --owner=0 --group=0 --numeric-owner \
   --mode="go-w" --exclude=".hg*" --exclude="CVS" --exclude=".cvs*" -f