Bug 1455143 - Refactor checksumming to occur after upload.py; r=ted
authorGregory Szorc <gps@mozilla.com>
Wed, 18 Apr 2018 16:24:03 -0700
changeset 468158 12cfbcd2ccf467c79bb614e3b99cf4a677ff95a6
parent 468157 1fa5254b9a69c3f96087dbddd116625c74f8a3c3
child 468159 b604acdb1c78b9f3789feb36fa3edcbce2d09748
push id9165
push userasasaki@mozilla.com
push dateThu, 26 Apr 2018 21:04:54 +0000
treeherdermozilla-beta@064c3804de2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1455143
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1455143 - Refactor checksumming to occur after upload.py; r=ted checksums.py is now run after upload.py, as part of the "upload" make target. As part of the refactor, checksums.py now takes as arguments a list of directories containing files to checksum. It will recursively checksum all files in listed directories. This means we no longer have to pass an explicit list of files into checksums.py. Instead, we can pass the artifact directory and automatically checksum everything within. This will allow us to generate files directly into the artifact directory instead of having to run upload.py to copy files there. MozReview-Commit-ID: 6ntnXU2Pp0E
build/checksums.py
toolkit/mozapps/installer/packager.mk
--- a/build/checksums.py
+++ b/build/checksums.py
@@ -7,16 +7,17 @@ from __future__ import with_statement
 
 from optparse import OptionParser
 import hashlib
 import logging
 import os
 
 logger = logging.getLogger('checksums.py')
 
+
 def digest_file(filename, digest, chunk_size=131072):
     '''Produce a checksum for the file specified by 'filename'.  'filename'
     is a string path to a file that is opened and read in this function.  The
     checksum algorithm is specified by 'digest' and is a valid OpenSSL
     algorithm.  If the digest used is not valid or Python's hashlib doesn't
     work, the None object will be returned instead.  The size of blocks
     that this function will read from the file object it opens based on
     'filename' can be specified by 'chunk_size', which defaults to 1K'''
@@ -31,19 +32,19 @@ def digest_file(filename, digest, chunk_
                 logger.debug('Finished reading in file')
                 break
             h.update(data)
     hash = h.hexdigest()
     logger.debug('Hash for %s is %s' % (filename, hash))
     return hash
 
 
-def process_files(files, output_filename, digests, strip):
-    '''This function takes a list of file names, 'files'.  It will then
-    compute the checksum for each of the files by opening the files.
+def process_files(dirs, output_filename, digests):
+    '''This function takes a list of directory names, 'drs'. It will then
+    compute the checksum for each of the files in these by by opening the files.
     Once each file is read and its checksum is computed, this function
     will write the information to the file specified by 'output_filename'.
     The path written in the output file will have anything specified by 'strip'
     removed from the path.  The output file is closed before returning nothing
     The algorithm to compute checksums with can be specified by 'digests'
     and needs to be a list of valid OpenSSL algorithms.
 
     The output file is written in the format:
@@ -53,28 +54,27 @@ def process_files(files, output_filename
     '''
 
     if os.path.exists(output_filename):
         logger.debug('Overwriting existing checksums file "%s"' %
                      output_filename)
     else:
         logger.debug('Creating a new checksums file "%s"' % output_filename)
     with open(output_filename, 'w+') as output:
-        for file in files:
-            for digest in digests:
-                hash = digest_file(file, digest)
+        for d in dirs:
+            for root, dirs, files in os.walk(d):
+                for f in files:
+                    full = os.path.join(root, f)
+                    rel = os.path.relpath(full, d)
 
-                if file.startswith(strip):
-                    short_file = file[len(strip):]
-                    short_file = short_file.lstrip('/')
-                else:
-                    short_file = file
+                    for digest in digests:
+                        hash = digest_file(full, digest)
 
-                output.write('%s %s %s %s\n' % (
-                    hash, digest, os.path.getsize(file), short_file))
+                        output.write('%s %s %s %s\n' % (
+                            hash, digest, os.path.getsize(full), rel))
 
 
 def setup_logging(level=logging.DEBUG):
     '''This function sets up the logging module using a speficiable logging
     module logging level.  The default log level is DEBUG.
 
     The output is in the format:
         <level> - <message>
@@ -100,19 +100,17 @@ def main():
                       action='append', dest='digests')
     parser.add_option('-o', '--output', help='output file to use',
                       action='store', dest='outfile', default='checksums')
     parser.add_option('-v', '--verbose',
                       help='Be noisy (takes precedence over quiet)',
                       action='store_true', dest='verbose', default=False)
     parser.add_option('-q', '--quiet', help='Be quiet', action='store_true',
                       dest='quiet', default=False)
-    parser.add_option('-s', '--strip',
-                      help='strip this path from the filenames',
-                      dest='strip', default=os.getcwd())
+
     options, args = parser.parse_args()
 
     # Figure out which logging level to use
     if options.verbose:
         loglevel = logging.DEBUG
     elif options.quiet:
         loglevel = logging.ERROR
     else:
@@ -120,22 +118,18 @@ def main():
 
     # Set up logging
     setup_logging(loglevel)
 
     # Validate the digest type to use
     if not options.digests:
         options.digests = ['sha1']
 
-    # Validate the files to checksum
-    files = []
     for i in args:
-        if os.path.isdir(i):
-            logger.warn('%s is a directory; ignoring' % i)
-        elif os.path.exists(i):
-            files.append(i)
-        else:
-            logger.info('File "%s" was not found on the filesystem' % i)
-    process_files(files, options.outfile, options.digests, options.strip)
+        if not os.path.isdir(i):
+            logger.error('%s is not a directory' % i)
+            exit(1)
+
+    process_files(args, options.outfile, options.digests)
 
 
 if __name__ == '__main__':
     main()
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -131,33 +131,28 @@ ifeq (bundle,$(MOZ_FS_LAYOUT))
 endif
 	$(NSINSTALL) -D $(DESTDIR)$(installdir)
 	(cd $(DIST)/$(MOZ_PKG_DIR) && $(TAR) --exclude=precomplete $(TAR_CREATE_FLAGS) - .) | \
 	  (cd $(DESTDIR)$(installdir) && tar -xf -)
 	$(NSINSTALL) -D $(DESTDIR)$(bindir)
 	$(RM) -f $(DESTDIR)$(bindir)/$(MOZ_APP_NAME)
 	ln -s $(installdir)/$(MOZ_APP_NAME) $(DESTDIR)$(bindir)
 
-checksum:
+upload:
+	$(PYTHON) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(UPLOAD_FILES)
 	mkdir -p `dirname $(CHECKSUM_FILE)`
 	@$(PYTHON) $(MOZILLA_DIR)/build/checksums.py \
 		-o $(CHECKSUM_FILE) \
 		$(CHECKSUM_ALGORITHM_PARAM) \
-		-s $(call QUOTED_WILDCARD,$(DIST)) \
-		$(UPLOAD_FILES)
+		$(UPLOAD_PATH)
 	@echo 'CHECKSUM FILE START'
 	@cat $(CHECKSUM_FILE)
 	@echo 'CHECKSUM FILE END'
 	$(SIGN_CHECKSUM_CMD)
-
-
-upload: checksum
-	$(PYTHON) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) \
-		$(UPLOAD_FILES) \
-		$(CHECKSUM_FILES)
+	$(PYTHON) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(CHECKSUM_FILES)
 
 # source-package creates a source tarball from the files in MOZ_PKG_SRCDIR,
 # which is either set to a clean checkout or defaults to $topsrcdir
 source-package:
 	@echo 'Generate the sourcestamp file'
 	# Make sure to have repository information available and then generate the
 	# sourcestamp file.
 	$(MAKE) -C $(DEPTH) 'source-repo.h'