Bug 1212942 - Add pushdate routes; r=garndt,jlund
authorMike Shal <mshal@mozilla.com>
Fri, 09 Oct 2015 09:26:34 -0400
changeset 268896 e1bd4c97811fc6c2fb464d8e5ad9180bae22f612
parent 268895 59164fa9fe90766b454a101d02ab77d1db87c20b
child 268897 20035adc5b896636a4e11a259db2271c7bd1ff30
push id29566
push usercbook@mozilla.com
push dateThu, 22 Oct 2015 09:45:32 +0000
treeherderautoland@76bd0c01d72e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgarndt, jlund
bugs1212942
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1212942 - Add pushdate routes; r=garndt,jlund
testing/mozharness/mozharness/mozilla/building/buildbase.py
testing/mozharness/scripts/desktop_l10n.py
testing/mozharness/scripts/mobile_l10n.py
testing/taskcluster/mach_commands.py
testing/taskcluster/routes.json
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1391,22 +1391,27 @@ or run without that action (ie: --no-{ac
                             property_conditions=[]):
         if not self.client_id or not self.access_token:
             self.warning('Skipping S3 file upload: No taskcluster credentials.')
             return
 
         repo = self._query_repo()
         revision = self.query_revision()
         pushinfo = self.vcs_query_pushinfo(repo, revision)
+        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
 
         index = self.config.get('taskcluster_index', 'index.garbage.staging')
         fmt = {
             'index': index,
             'project': self.buildbot_config['properties']['branch'],
             'head_rev': revision,
+            'pushdate': pushdate,
+            'year': pushdate[0:4],
+            'month': pushdate[4:6],
+            'day': pushdate[6:8],
             'build_product': self.config['stage_product'],
             'build_name': self.query_build_name(),
             'build_type': self.query_build_type(),
             'locale': locale,
         }
         fmt.update(self.buildid_to_dict(self.query_buildid()))
         routes = []
         for template in templates:
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -6,16 +6,17 @@
 # ***** END LICENSE BLOCK *****
 """desktop_l10n.py
 
 This script manages Desktop repacks for nightly builds.
 """
 import os
 import re
 import sys
+import time
 import shlex
 import logging
 
 import subprocess
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
@@ -960,31 +961,36 @@ class DesktopSingleLocale(LocalesMixin, 
 
         branch = self.config['branch']
         platform = self.config['platform']
         revision = self._query_revision()
         repo = self.query_l10n_repo()
         if not repo:
             self.fatal("Unable to determine repository for querying the push info.")
         pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool')
+        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
 
         routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'],
                                    'testing/taskcluster/routes.json')
         with open(routes_json) as f:
             contents = json.load(f)
             templates = contents['l10n']
 
         for locale, files in self.upload_files.iteritems():
             self.info("Uploading files to S3 for locale '%s': %s" % (locale, files))
             routes = []
             for template in templates:
                 fmt = {
                     'index': self.config.get('taskcluster_index', 'index.garbage.staging'),
                     'project': branch,
                     'head_rev': revision,
+                    'pushdate': pushdate,
+                    'year': pushdate[0:4],
+                    'month': pushdate[4:6],
+                    'day': pushdate[6:8],
                     'build_product': self.config['stage_product'],
                     'build_name': self.query_build_name(),
                     'build_type': self.query_build_type(),
                     'locale': locale,
                 }
                 fmt.update(self.buildid_to_dict(self._query_buildid()))
                 routes.append(template.format(**fmt))
 
--- a/testing/mozharness/scripts/mobile_l10n.py
+++ b/testing/mozharness/scripts/mobile_l10n.py
@@ -10,16 +10,17 @@ This currently supports nightly and rele
 Android.  This also creates nightly updates.
 """
 
 from copy import deepcopy
 import os
 import re
 import subprocess
 import sys
+import time
 import shlex
 
 try:
     import simplejson as json
     assert json
 except ImportError:
     import json
 
@@ -481,16 +482,17 @@ class MobileSingleLocale(MockMixin, Loca
         locales = self.query_locales()
         make = self.query_exe("make")
         upload_env = self.query_upload_env()
         cwd = dirs['abs_locales_dir']
         branch = self.config['branch']
         revision = self.query_revision()
         repo = self.query_l10n_repo()
         pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool')
+        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
         routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'],
                                    'testing/taskcluster/routes.json')
         with open(routes_json) as f:
             contents = json.load(f)
             templates = contents['l10n']
 
         for locale in locales:
             output = self.get_output_from_command_m(
@@ -501,16 +503,20 @@ class MobileSingleLocale(MockMixin, Loca
             files = shlex.split(output)
             abs_files = [os.path.abspath(os.path.join(cwd, f)) for f in files]
 
             routes = []
             fmt = {
                 'index': self.config.get('taskcluster_index', 'index.garbage.staging'),
                 'project': branch,
                 'head_rev': revision,
+                'pushdate': pushdate,
+                'year': pushdate[0:4],
+                'month': pushdate[4:6],
+                'day': pushdate[6:8],
                 'build_product': self.config['stage_product'],
                 'build_name': self.query_build_name(),
                 'build_type': self.query_build_type(),
                 'locale': locale,
             }
             for template in templates:
                 routes.append(template.format(**fmt))
 
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -6,16 +6,18 @@
 
 from __future__ import absolute_import
 
 from collections import defaultdict
 import os
 import json
 import copy
 import sys
+import time
+from collections import namedtuple
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 
@@ -198,16 +200,54 @@ def remove_caches_from_task(task):
     try:
         caches = task["task"]["payload"]["cache"]
         for cache in caches.keys():
             if cache not in whitelist:
                 caches.pop(cache)
     except KeyError:
         pass
 
+def query_pushinfo(repository, revision):
+    """Query the pushdate and pushid of a repository/revision.
+    This is intended to be used on hg.mozilla.org/mozilla-central and
+    similar. It may or may not work for other hg repositories.
+    """
+    PushInfo = namedtuple('PushInfo', ['pushid', 'pushdate'])
+
+    try:
+        import urllib2
+        url = '%s/json-pushes?changeset=%s' % (repository, revision)
+        sys.stderr.write("Querying URL for pushdate: %s\n" % url)
+        contents = json.load(urllib2.urlopen(url))
+
+        # The contents should be something like:
+        # {
+        #   "28537": {
+        #    "changesets": [
+        #     "1d0a914ae676cc5ed203cdc05c16d8e0c22af7e5",
+        #    ],
+        #    "date": 1428072488,
+        #    "user": "user@mozilla.com"
+        #   }
+        # }
+        #
+        # So we grab the first element ("28537" in this case) and then pull
+        # out the 'date' field.
+        pushid = contents.iterkeys().next()
+        pushdate = contents[pushid]['date']
+        return PushInfo(pushid, pushdate)
+
+    except Exception:
+        sys.stderr.write(
+            "Error querying pushinfo for repository '%s' revision '%s'\n" % (
+                repository, revision,
+            )
+        )
+        return None
+
 @CommandProvider
 class DecisionTask(object):
     @Command('taskcluster-decision', category="ci",
         description="Build a decision task")
     @CommandArgument('--project',
         required=True,
         help='Treeherder project name')
     @CommandArgument('--url',
@@ -328,27 +368,37 @@ class Graph(object):
         job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
 
         jobs = templates.load(job_path, {})
 
         job_graph = parse_commit(message, jobs)
 
         cmdline_interactive = params.get('interactive', False)
 
+        # Default to current time if querying the head rev fails
+        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
+        pushinfo = query_pushinfo(params['head_repository'], params['head_rev'])
+        if pushinfo:
+            pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
+
         # Template parameters used when expanding the graph
         parameters = dict(gaia_info().items() + {
             'index': 'index',
             'project': project,
             'pushlog_id': params.get('pushlog_id', 0),
             'docker_image': docker_image,
             'base_repository': params['base_repository'] or \
                 params['head_repository'],
             'head_repository': params['head_repository'],
             'head_ref': params['head_ref'] or params['head_rev'],
             'head_rev': params['head_rev'],
+            'pushdate': pushdate,
+            'year': pushdate[0:4],
+            'month': pushdate[4:6],
+            'day': pushdate[6:8],
             'owner': params['owner'],
             'from_now': json_time_from_now,
             'now': current_json_time(),
             'revision_hash': params['revision_hash']
         }.items())
 
         treeherder_route = '{}.{}'.format(
             params['project'],
--- a/testing/taskcluster/routes.json
+++ b/testing/taskcluster/routes.json
@@ -1,16 +1,18 @@
 {
     "routes": [
         "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
+        "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.latest.{build_product}.{build_name}-{build_type}"
     ],
     "nightly": [
         "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.latest.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.nightly.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.nightly.latest.{build_product}.{build_name}-{build_type}"
     ],
     "l10n": [
         "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
+        "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
         "{index}.gecko.v2.{project}.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}"
     ]
 }