bug 715586: checksums.py should generate sha1 and md5 checksums. r=catlee,ted
authorRail Aliiev <rail@mozilla.com>
Mon, 16 Jan 2012 09:30:59 -0500
changeset 85783 1b89605ede03c96532646cc624fc68a2571eb162
parent 85782 9259ed35120571b57f5bd7778c25b609d82e5cdf
child 85784 b3b8d62e0d925bed5c6722eceaedf9d57cf55b95
child 90844 723b97cb9362ba85be48fdfc25949cfd9fb8b445
push id805
push userakeybl@mozilla.com
push dateWed, 01 Feb 2012 18:17:35 +0000
treeherdermozilla-aurora@6fb3bf232436 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscatlee, ted
bugs715586
milestone12.0a1
bug 715586: checksums.py should generate sha1 and md5 checksums. r=catlee,ted
build/checksums.py
toolkit/mozapps/installer/packager.mk
--- a/build/checksums.py
+++ b/build/checksums.py
@@ -31,16 +31,18 @@
 # use your version of this file under the terms of the MPL, indicate your
 # decision by deleting the provisions above and replace them with the notice
 # and other provisions required by the GPL or the LGPL. If you do not delete
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK *****
 
+from __future__ import with_statement
+
 from optparse import OptionParser
 import logging
 import os
 try:
     import hashlib
 except:
     hashlib = None
 
@@ -52,75 +54,74 @@ def digest_file(filename, digest, chunk_
     work, the None object will be returned instead.  The size of blocks
     that this function will read from the file object it opens based on
     'filename' can be specified by 'chunk_size', which defaults to 1K'''
     assert not os.path.isdir(filename), 'this function only works with files'
     logger = logging.getLogger('checksums.py')
     if hashlib is not None:
         logger.debug('Creating new %s object' % digest)
         h = hashlib.new(digest)
-        f = open(filename, "rb")
-        while True:
-            data = f.read(chunk_size)
-            if not data:
-                logger.debug('Finished reading in file')
-                break
-            h.update(data)
-        f.close()
+        with open(filename, 'rb') as f:
+            while True:
+                data = f.read(chunk_size)
+                if not data:
+                    logger.debug('Finished reading in file')
+                    break
+                h.update(data)
         hash = h.hexdigest()
         logger.debug('Hash for %s is %s' % (filename, hash))
         return hash
     else:
         # In this case we could subprocess.Popen and .communicate with
         # sha1sum or md5sum
         logger.warn('The python module for hashlib is missing!')
         return None
 
 
-def process_files(files, output_filename, digest, strip):
+def process_files(files, output_filename, digests, strip):
     '''This function takes a list of file names, 'files'.  It will then
     compute the checksum for each of the files by opening the files.
     Once each file is read and its checksum is computed, this function
     will write the information to the file specified by 'output_filename'.
     The path written in the output file will have anything specified by 'strip'
     removed from the path.  The output file is closed before returning nothing
-    The algorithm to compute checksums with can be specified by 'digest' 
-    and needs to be a valid OpenSSL algorithm.
+    The algorithm to compute checksums with can be specified by 'digests' 
+    and needs to be a list of valid OpenSSL algorithms.
 
     The output file is written in the format:
         <hash> <algorithm> <filesize> <filepath>
     Example:
         d1fa09a<snip>e4220 sha1 14250744 firefox-4.0b6pre.en-US.mac64.dmg
     '''
 
     logger = logging.getLogger('checksums.py')
     if os.path.exists(output_filename):
         logger.debug('Overwriting existing checksums file "%s"' %
                      output_filename)
     else:
         logger.debug('Creating a new checksums file "%s"' % output_filename)
-    output = open(output_filename, 'w+')
-    for file in files:
-        if os.path.isdir(file):
-            logger.warn('%s is a directory, skipping' % file)
-        else:
-            hash = digest_file(file, digest)
-            if hash is None:
-                logger.warn('Unable to generate a hash for %s. ' +
-                            'Using NOHASH as fallback' % file)
-                hash = 'NOHASH'
-            if file.startswith(strip):
-                short_file = file[len(strip):]
-                short_file = short_file.lstrip('/')
+    with open(output_filename, 'w+') as output:
+        for file in files:
+            if os.path.isdir(file):
+                logger.warn('%s is a directory, skipping' % file)
             else:
-                short_file = file
-            print >>output, '%s %s %s %s' % (hash, digest,
-                                             os.path.getsize(file),
-                                             short_file)
-    output.close()
+                for digest in digests:
+                    hash = digest_file(file, digest)
+                    if hash is None:
+                        logger.warn('Unable to generate a hash for %s. ' +
+                                    'Skipping.' % file)
+                        continue
+                    if file.startswith(strip):
+                        short_file = file[len(strip):]
+                        short_file = short_file.lstrip('/')
+                    else:
+                        short_file = file
+                    print >>output, '%s %s %s %s' % (hash, digest,
+                                                     os.path.getsize(file),
+                                                     short_file)
 
 def setup_logging(level=logging.DEBUG):
     '''This function sets up the logging module using a speficiable logging
     module logging level.  The default log level is DEBUG.
 
     The output is in the format:
         <level> - <message>
     Example:
@@ -136,17 +137,17 @@ def setup_logging(level=logging.DEBUG):
     logger.addHandler(handler)
 
 def main():
     '''This is a main function that parses arguments, sets up logging
     and generates a checksum file'''
     # Parse command line arguments
     parser = OptionParser()
     parser.add_option('-d', '--digest', help='checksum algorithm to use',
-                      action='store', dest='digest', default='sha1')
+                      action='append', dest='digests')
     parser.add_option('-o', '--output', help='output file to use',
                       action='store', dest='outfile', default='checksums')
     parser.add_option('-v', '--verbose',
                       help='Be noisy (takes precedence over quiet)',
                       action='store_true', dest='verbose', default=False)
     parser.add_option('-q', '--quiet', help='Be quiet', action='store_true',
                       dest='quiet', default=False)
     parser.add_option('-s', '--strip',
@@ -162,26 +163,29 @@ def main():
     else:
         loglevel = logging.INFO
 
     #Set up logging
     setup_logging(loglevel)
     logger = logging.getLogger('checksums.py')
 
     # Validate the digest type to use
+    if not options.digests:
+        options.digests = ['sha1']
     try:
-        hashlib.new(options.digest)
+        for digest in options.digests:
+            hashlib.new(digest)
     except ValueError, ve:
         logger.error('Could not create a "%s" hash object (%s)' %
-                     (options.digest, ve.args[0]))
+                     (digest, ve.args[0]))
         exit(1)
 
     # Validate the files to checksum
     files = []
     for i in args:
         if os.path.exists(i):
             files.append(i)
         else:
             logger.info('File "%s" was not found on the filesystem' % i)
-    process_files(files, options.outfile, options.digest, options.strip)
+    process_files(files, options.outfile, options.digests, options.strip)
 
 if __name__ == '__main__':
     main()
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -899,17 +899,17 @@ endif
 # deal with them.
 empty :=
 space = $(empty) $(empty)
 QUOTED_WILDCARD = $(if $(wildcard $(subst $(space),?,$(1))),"$(1)")
 ESCAPE_SPACE = $(subst $(space),\$(space),$(1))
 
 # This variable defines which OpenSSL algorithm to use to 
 # generate checksums for files that we upload
-CHECKSUM_ALGORITHM = 'sha512'
+CHECKSUM_ALGORITHM_PARAM = -d sha512 -d md5 -d sha1
 
 # This variable defines where the checksum file will be located
 CHECKSUM_FILE = "$(DIST)/$(PKG_PATH)/$(CHECKSUMS_FILE_BASENAME).checksums"
 CHECKSUM_FILES = $(CHECKSUM_FILE)
 
 UPLOAD_FILES= \
   $(call QUOTED_WILDCARD,$(DIST)/$(PACKAGE)) \
   $(call QUOTED_WILDCARD,$(INSTALLER_PACKAGE)) \
@@ -936,17 +936,17 @@ UPLOAD_FILES += $(call QUOTED_WILDCARD,$
 UPLOAD_FILES += $(call QUOTED_WILDCARD,$(INSTALLER_PACKAGE).asc)
 endif
 endif
 
 checksum:
 	mkdir -p `dirname $(CHECKSUM_FILE)`
 	@$(PYTHON) $(MOZILLA_DIR)/build/checksums.py \
 		-o $(CHECKSUM_FILE) \
-		-d $(CHECKSUM_ALGORITHM) \
+		$(CHECKSUM_ALGORITHM_PARAM) \
 		-s $(call QUOTED_WILDCARD,$(DIST)) \
 		$(UPLOAD_FILES)
 	@echo "CHECKSUM FILE START"
 	@cat $(CHECKSUM_FILE)
 	@echo "CHECKSUM FILE END"
 	$(SIGN_CHECKSUM_CMD)