bug 1442306 - generate a buildhub.json file during packaging. r=gps
authorTed Mielczarek <ted@mielczarek.org>
Mon, 30 Apr 2018 13:03:17 -0400
changeset 473132 7507516552fe76d0d00d32917d88a0056e731fb5
parent 473131 63abddfab15f0effa13c6019b0889dc58cdb33b6
child 473133 a585d77f897ff1c669a426cd70f1deb6278ae56e
push id1728
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:12:27 +0000
treeherdermozilla-release@c296fde26f5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1442306
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1442306 - generate a buildhub.json file during packaging. r=gps buildhub is a service that stores a list of nightly and release builds and can be queried to find specific builds. Currently it ingests data by scraping info from ftp.mozilla.org. This patch makes it so we generate a buildhub.json during packaging with the data that buildhub wants. There are a few pieces of data that we can't accurately provide from the build system such as the URL to the build, so we provide some stub data there with the expectation that a release engineering process will fill them in later. MozReview-Commit-ID: 266BnZZBFoL
toolkit/mozapps/installer/informulate.py
toolkit/mozapps/installer/package-name.mk
toolkit/mozapps/installer/packager.mk
toolkit/mozapps/installer/upload-files.mk
--- a/toolkit/mozapps/installer/informulate.py
+++ b/toolkit/mozapps/installer/informulate.py
@@ -4,28 +4,34 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # Generate build info files for use by other tools.
 # This script assumes it is being run in a Mozilla CI build.
 
 from __future__ import unicode_literals
 
 from argparse import ArgumentParser
+import datetime
+import buildconfig
 import json
-import buildconfig
+import mozinfo
 import os
 
 
 def main():
     parser = ArgumentParser()
     parser.add_argument('output_json', help='Output JSON file')
+    parser.add_argument('buildhub_json', help='Output buildhub JSON file')
     parser.add_argument('output_txt', help='Output text file')
     # TODO: Move package-name.mk variables into moz.configure.
     parser.add_argument('pkg_platform', help='Package platform identifier')
+    parser.add_argument('--package', help='Path to application package file')
+    parser.add_argument('--installer', help='Path to application installer file')
     args = parser.parse_args()
+    mozinfo.find_and_update_from_json()
 
     important_substitutions = [
         'target_alias', 'target_cpu', 'target_os', 'target_vendor',
         'host_alias', 'host_cpu', 'host_os', 'host_vendor',
         'MOZ_UPDATE_CHANNEL', 'MOZ_APP_VENDOR', 'MOZ_APP_NAME',
         'MOZ_APP_VERSION', 'MOZ_APP_MAXVERSION', 'MOZ_APP_ID',
         'CC', 'CXX', 'AS', 'MOZ_SOURCE_REPO',
     ]
@@ -38,14 +44,58 @@ def main():
         'moz_source_stamp': buildconfig.substs['MOZ_SOURCE_CHANGESET'],
         'moz_pkg_platform': args.pkg_platform,
     })
 
     with open(args.output_json, 'wb') as f:
         json.dump(all_key_value_pairs, f, indent=2, sort_keys=True)
         f.write('\n')
 
+    with open(args.buildhub_json, 'wb') as f:
+        if args.installer and os.path.exists(args.installer):
+            package = args.installer
+        else:
+            package = args.package
+        build_time = datetime.datetime.strptime(build_id, '%Y%m%d%H%M%S')
+        st = os.stat(package)
+        mtime = datetime.datetime.fromtimestamp(st.st_mtime)
+        s = buildconfig.substs
+        record = {
+            'build': {
+                'id': build_id,
+                'date': build_time.isoformat() + 'Z',
+                'as': s['AS'],
+                'cc': s['CC'],
+                'cxx': s['CXX'],
+                'host': s['host_alias'],
+                'target': s['target_alias'],
+            },
+            'source': {
+                'product': s['MOZ_APP_NAME'],
+                'repository': s['MOZ_SOURCE_REPO'],
+                'tree': os.environ['MH_BRANCH'],
+                'revision': s['MOZ_SOURCE_CHANGESET'],
+            },
+            'target': {
+                'platform': args.pkg_platform,
+                'os': mozinfo.info['os'],
+                # This would be easier if the locale was specified at configure time.
+                'locale': os.environ.get('AB_CD', 'en-US'),
+                'version': s['MOZ_APP_VERSION'],
+                'channel': s['MOZ_UPDATE_CHANNEL'],
+            },
+            'download': {
+                # The release pipeline will update these keys.
+                'url': os.path.basename(package),
+                'mimetype': 'application/octet-stream',
+                'date': mtime.isoformat() + 'Z',
+                'size': st.st_size,
+            }
+        }
+        json.dump(record, f, indent=2, sort_keys=True)
+        f.write('\n')
+
     with open(args.output_txt, 'wb') as f:
         f.write('buildID={}\n'.format(build_id))
 
 
 if __name__ == '__main__':
     main()
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -107,16 +107,17 @@ GTEST_PACKAGE = $(PKG_BASENAME).gtest.te
 ifneq (,$(wildcard $(DIST)/bin/application.ini))
 BUILDID = $(shell $(PYTHON) $(MOZILLA_DIR)/config/printconfigsetting.py $(DIST)/bin/application.ini App BuildID)
 else
 BUILDID = $(shell $(PYTHON) $(MOZILLA_DIR)/config/printconfigsetting.py $(DIST)/bin/platform.ini Build BuildID)
 endif
 
 MOZ_SOURCESTAMP_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).txt
 MOZ_BUILDINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).json
+MOZ_BUILDHUB_JSON = $(DIST)/$(PKG_PATH)/buildhub.json
 MOZ_BUILDID_INFO_TXT_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME)_info.txt
 MOZ_MOZINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).mozinfo.json
 MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/$(PKG_BASENAME).test_packages.json
 
 # JavaScript Shell
 ifdef MOZ_SIMPLE_PACKAGE_NAME
 JSSHELL_NAME := $(MOZ_SIMPLE_PACKAGE_NAME).jsshell.zip
 else
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -89,17 +89,20 @@ make-package: FORCE
 ifeq (WINNT,$(OS_ARCH))
 ifeq ($(MOZ_PKG_FORMAT),ZIP)
 	$(MAKE) -C windows ZIP_IN='$(ABS_DIST)/$(PACKAGE)' installer
 endif
 endif
 ifdef MOZ_AUTOMATION
 	cp $(DEPTH)/mozinfo.json $(MOZ_MOZINFO_FILE)
 	$(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/informulate.py \
-		$(MOZ_BUILDINFO_FILE) $(MOZ_BUILDID_INFO_TXT_FILE) $(MOZ_PKG_PLATFORM)
+		$(MOZ_BUILDINFO_FILE) $(MOZ_BUILDHUB_JSON) $(MOZ_BUILDID_INFO_TXT_FILE) \
+		$(MOZ_PKG_PLATFORM) \
+		--package=$(DIST)/$(PACKAGE) \
+		--installer=$(INSTALLER_PACKAGE)
 endif
 	$(TOUCH) $@
 
 GARBAGE += make-package
 
 make-sourcestamp-file::
 	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
 	@echo '$(BUILDID)' > $(MOZ_SOURCESTAMP_FILE)
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -389,16 +389,17 @@ UPLOAD_FILES= \
   $(call QUOTED_WILDCARD,$(DIST)/$(COMPLETE_MAR)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(LANGPACK)) \
   $(call QUOTED_WILDCARD,$(wildcard $(DIST)/$(PARTIAL_MAR))) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZHARNESS_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(MOZ_SOURCESTAMP_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDINFO_FILE)) \
+  $(call QUOTED_WILDCARD,$(MOZ_BUILDHUB_JSON)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDID_INFO_TXT_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_MOZINFO_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_TEST_PACKAGES_FILE)) \
   $(call QUOTED_WILDCARD,$(PKG_JSSHELL)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip) \
   $(call QUOTED_WILDCARD,$(topobjdir)/$(MOZ_BUILD_APP)/installer/windows/instgen/setup.exe) \
   $(call QUOTED_WILDCARD,$(topobjdir)/$(MOZ_BUILD_APP)/installer/windows/instgen/setup-stub.exe) \
   $(call QUOTED_WILDCARD,$(topsrcdir)/toolchains.json) \