[PATCH] Bug 1053814 - Update the product details and mozilla.com from FIREFOX_35_0b1_RELEASE
authorSylvestre Ledru <sylvestre@debian.org>
Thu, 25 Sep 2014 11:54:38 +0200
changeset 5249 839656c805b05b1ceba6b459f9e83be6a2903994
parent 5248 ab79aa516e3d7cdc1bb60cc22acb9677a76ea746
child 5250 40df9c7ba90264196d1912cfd4a3d41c79e50b11
push id12
push usersledru@mozilla.com
push dateFri, 05 Dec 2014 17:54:44 +0000
bugs1053814, 100644
[PATCH] Bug 1053814 - Update the product details and mozilla.com from From 3bfeb411a0d5af04ec2eb04cc0cf096528f840c9 Mon Sep 17 00:00:00 2001 stage-tasks.py --- lib/python/release/product_details_update.py | 193 +++++++++++++++++++++++++++ lib/python/util/svn.py | 70 ++++++++++ scripts/release/stage-tasks.py | 31 +++++ 3 files changed, 294 insertions(+) create mode 100644 lib/python/release/product_details_update.py create mode 100644 lib/python/util/svn.py
lib/python/release/product_details_update.py
lib/python/util/svn.py
scripts/release/stage-tasks.py
new file mode 100644
--- /dev/null
+++ b/lib/python/release/product_details_update.py
@@ -0,0 +1,193 @@
+from jinja2 import Environment, FileSystemLoader
+import os
+import re
+import time
+from build.versions import ANY_VERSION_REGEX
+
+
+PRODUCT_VERSION_TEMPLATES = {
+    "fennec": {
+        "aurora": {
+            "version_template": "mobile_alpha_version.php.tpl",
+        },
+        "beta": {
+            "version_template": "mobile_beta_version.php.tpl",
+            "history_info": {
+                "template": "mobileHistory.class.php",
+                "marker": "NEXT_DEVELOPMENT",
+            },
+        },
+        "major": {
+            "version_template": "mobile_latest_version.php.tpl",
+            "history_info": {
+                "template": "mobileHistory.class.php",
+                "marker": "NEXT_MAJOR"
+            },
+        },
+        "minor": {
+            "version_template": "mobile_latest_version.php.tpl",
+            "history_info": {
+                "template": "mobileHistory.class.php",
+                "marker": "NEXT_STABILITY"
+            },
+        },
+    },
+
+    "firefox": {
+        "aurora": {
+            "version_template": "FIREFOX_AURORA.php.tpl",
+        },
+        "beta": {
+            "version_template": "LATEST_FIREFOX_DEVEL_VERSION.php.tpl",
+            "version_template_2": "LATEST_FIREFOX_RELEASED_DEVEL_VERSION.php.tpl",
+            "history_info": {
+                "template": "firefoxHistory.class.php",
+                "marker": "NEXT_DEVELOPMENT",
+            },
+        },
+        "major": {
+            "version_template": "LATEST_FIREFOX_VERSION.php.tpl",
+            "version_template_2": "LATEST_FIREFOX_RELEASED_VERSION.php.tpl",
+            "history_info": {
+                "template": "firefoxHistory.class.php",
+                "marker": "NEXT_MAJOR"
+            },
+        },
+        "minor": {
+            "version_template": "LATEST_FIREFOX_VERSION.php.tpl",
+            "version_template_2": "LATEST_FIREFOX_RELEASED_VERSION.php.tpl",
+            "history_info": {
+                "template": "firefoxHistory.class.php",
+                "marker": "NEXT_STABILITY"
+            },
+        },
+        "esr": {
+            "version_template": "FIREFOX_ESR.php.tpl",
+            "version_template_2": "FIREFOX_ESR_NEXT.php.tpl",
+            "history_info": {
+                "template": "firefoxHistory.class.php",
+                "marker": "NEXT_STABILITY"
+            },
+        },
+    },
+
+    "thunderbird": {
+        # Thundebird is not doing anything about beta
+        "major": {
+            "version_template": "LATEST_THUNDERBIRD_VERSION.php.tpl",
+            "history_info": {
+                "template": "thunderbirdHistory.class.php",
+                "marker": "NEXT_MAJOR"
+            },
+        },
+        "minor": {
+            "version_template": "LATEST_THUNDERBIRD_VERSION.php.tpl",
+            "history_info": {
+                "template": "thunderbirdHistory.class.php",
+                "marker": "NEXT_STABILITY"
+            },
+        },
+    },
+
+}
+
+
+def updateVersionTemplate(targetSVNDirectory, templateFile, version):
+    """
+    Templates are stored in dedicated directory. use jinja to update them
+    """
+    fullPath = os.path.dirname(os.path.abspath(__file__))
+    templatesDir = os.path.join(fullPath, targetSVNDirectory, "templates")
+
+    env = Environment(loader=FileSystemLoader(templatesDir))
+
+    if not os.path.isfile(os.path.join(templatesDir, templateFile)):
+        raise Exception("Could not find templateFile: " + templateFile)
+
+    template = env.get_template(templateFile)
+    output_from_parsed_template = template.render(VERSION=version)
+
+    # Save the results
+    targetFile = os.path.join(fullPath, targetSVNDirectory,
+                              templateFile.replace(".tpl", ""))
+    with open(targetFile, "wb") as fh:
+        fh.write(output_from_parsed_template)
+    fh.close()
+
+
+def addNewVersion(targetSVNDirectory, category, filename, version):
+    """
+    We cannot use a template mechanism for history file
+    because we need to keep the markers
+    Do it by hand
+    """
+    fullPath = os.path.dirname(os.path.abspath(__file__))
+    sourceFile = os.path.join(fullPath, targetSVNDirectory, "history",
+                              filename)
+    f = open(sourceFile, 'r')
+    filedata = f.read()
+    f.close()
+
+    indent = "            "
+    toreplace = "// " + category
+    newVersion = "'" + version + "' => '" + time.strftime("%Y-%m-%d") + "',"
+
+    if newVersion in filedata:
+        print "New version '" + newVersion + "' already existing in the file " + filename
+        return
+
+    newdata = filedata.replace(toreplace, newVersion + "\n" + indent + toreplace)
+    targetFile = os.path.join(targetSVNDirectory, "history", filename)
+    f = open(targetFile, 'w')
+    f.write(newdata)
+    f.close()
+
+
+def getTypeFromVersion(version):
+    """
+    From the version, return the code name of the version
+    This is going to be used to search in the datastructure
+    """
+    m = re.match(ANY_VERSION_REGEX, version)
+    if m and m.groups()[2] == "a":
+        return "aurora"
+    if m and m.groups()[2] == "b":
+        return "beta"
+    if m and m.groups()[3] == "esr":
+        return "esr"
+
+    if m and all(e is None for e in m.groups()[1:]):
+        # only groups()[0] is not None
+        # either major or minor release
+        if m.groups()[0].count(".") > 1:
+            return "minor"
+        else:
+            return "major"
+
+
+def updateProductDetailFiles(targetSVNDirectory, product, version):
+    """
+    Update the various files
+    """
+    type_ = getTypeFromVersion(version)
+
+    info = PRODUCT_VERSION_TEMPLATES.get(product, {}).get(type_)
+    template = info["version_template"]
+
+    history_info = PRODUCT_VERSION_TEMPLATES.get(product, {}).get(type_).get("history_info")
+
+    if template:
+        updateVersionTemplate(targetSVNDirectory, template, version)
+
+    if "version_template_2" in info:
+        # Some variables are stored in two files, update the second file
+        template_2 = info["version_template_2"]
+        if template_2:
+            updateVersionTemplate(targetSVNDirectory, template_2, version)
+
+    if type_ == "esr":
+        # Strip the 'esr' from the version. It is not used in the history
+        version = version.replace("esr", "")
+
+    if history_info:
+        addNewVersion(targetSVNDirectory, history_info["marker"], history_info["template"], version)
new file mode 100644
--- /dev/null
+++ b/lib/python/util/svn.py
@@ -0,0 +1,70 @@
+from util.retry import retry
+from util.commands import run_cmd, get_output
+import os
+import logging
+log = logging.getLogger(__name__)
+
+
+def checkoutSVN(targetSVNDirectory, svnURL):
+    """
+    Checkout the product details SVN
+    """
+    if not os.path.isdir(targetSVNDirectory):
+        cmd = ["svn", "co", svnURL, targetSVNDirectory]
+        run_cmd(cmd)
+
+    # Sanity check
+    svnStatus = get_output(["svn", "status", targetSVNDirectory])
+    if len(svnStatus) != 0:
+        raise Exception("Uncommited changes: " + svnStatus)
+
+
+def exportJSON(targetSVNDirectory):
+    """
+    Export the PHP product details to json
+    """
+    retval = os.getcwd()
+    os.chdir(targetSVNDirectory)
+    run_cmd(["php", "export_json.php"])
+    os.chdir(retval)
+
+
+def doCommitSVN(commitMSG):
+    """
+    Actually do the commit (called with retry)
+    """
+    log.info("svn commit -m " + commitMSG)
+    run_cmd(["svn", "commit", "-m", commitMSG])
+
+
+def getSVNrev(targetSVNDirectory):
+    """
+    Return the svn revision
+    """
+    retval = os.getcwd()
+    os.chdir(targetSVNDirectory)
+    rev = get_output(["svnversion"])
+    os.chdir(retval)
+    return rev
+
+
+def commitSVN(targetSVNDirectory, product, version, fakeCommit=False):
+    """
+    Commit the change in the svn
+    """
+    retval = os.getcwd()
+    os.chdir(targetSVNDirectory)
+    try:
+        if not os.path.isdir(".svn"):
+            raise Exception("Could not find the svn directory")
+        commitMSG = "'" + product + " version " + version + "'"
+        if not fakeCommit:
+            retry(doCommitSVN, args=(commitMSG), attempts=3, sleeptime=0)
+    finally:
+        os.chdir(retval)
+
+
+def updateRev(targetSVNDirectory, newRev):
+    run_cmd(["svn", "propset", "svn:externals", "'product-details -r" + newRev, "http://svn.mozilla.org/libs/product-details", "tags/stage/includes'"])
+    run_cmd(["svn", "propset", "svn:externals", "'product-details -r" + newRev, "http://svn.mozilla.org/libs/product-details", "tags/productions/includes'"])
+    run_cmd(["svn", "up"])
--- a/scripts/release/stage-tasks.py
+++ b/scripts/release/stage-tasks.py
@@ -16,18 +16,22 @@ log = logging.getLogger(__name__)
 site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
 site.addsitedir(path.join(path.dirname(__file__), "../../lib/python/vendor"))
 from release.info import readReleaseConfig, readConfig
 from release.paths import makeCandidatesDir, makeReleasesDir
 from util.hg import update, make_hg_url, mercurial
 from util.commands import run_remote_cmd
 from util.transfer import scp
 from util.retry import retry
+from util.svn import checkoutSVN, exportJSON, commitSVN, getSVNrev, updateRev
+
 import requests
 
+from product_details_update import updateProductDetailFiles
+
 
 DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url('hg.mozilla.org',
                                             'build/buildbot-configs')
 
 REQUIRED_RELEASE_CONFIG = ("productName", "version", "buildNumber",
                            "stage_product")
 
 DEFAULT_RSYNC_EXCLUDES = ['--exclude=*tests*',
@@ -262,16 +266,36 @@ def update_bouncer_alias(tuxedoServerUrl
     # Wrap the real call to hide credentials from retry's logging
     def do_update_bouncer_alias():
         requests.post(url, data=data, auth=auth, config={'danger_mode': True},
                       verify=False)
 
     retry(do_update_bouncer_alias)
 
 
+def updateProductDetail(targetSVNDirectory, targetSVNDirectoryMozilla, productName, version, svnURLpd, svnURLmozilla, fakeCommit=False):
+    """
+    Add a new version to the product details
+    """
+    retry(checkoutSVN, args=(targetSVNDirectory, svnURLpd), attempts=3, sleeptime=0)
+    retry(checkoutSVN, args=(targetSVNDirectoryMozilla, svnURLmozilla), attempts=3, sleeptime=0)
+    # Update the PHP files
+    updateProductDetailFiles(targetSVNDirectory, productName, version)
+    # Export to json
+    exportJSON(targetSVNDirectory)
+    # Commit to svn
+    commitSVN(targetSVNDirectory, productName, version, fakeCommit)
+    # Get the svn revision of the p-d repository
+    svnRev = getSVNrev(targetSVNDirectory)
+    # Update Mozilla.com
+    updateRev(targetSVNDirectoryMozilla, svnRev)
+    # commit Mozilla.com
+    commitSVN(targetSVNDirectoryMozilla, productName, version, fakeCommit)
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
 
     parser.set_defaults(
     )
     parser.add_argument("-r", "--release-config", required=True,
                         help="Release config file location relative to "
                         "buildbot-configs repo root")
@@ -386,8 +410,15 @@ if __name__ == '__main__':
             auth = (credentials["tuxedoUsername"],
                     credentials["tuxedoPassword"])
 
             update_bouncer_aliases(
                 tuxedoServerUrl=releaseConfig["tuxedoServerUrl"],
                 auth=auth,
                 version=version,
                 bouncer_aliases=bouncer_aliases)
+
+    if 'product-details' in args:
+        targetSVNDirectory = "product-details.svn"
+        targetSVNDirectoryMozilla = "mozilla.com.svn"
+        svnURLpd = "svn+ssh://ffxbld@svn.mozilla.org/libs/product-details"
+        svnURLmozilla = "svn+ssh://ffxbld@svn.mozilla.org/projects/mozilla.com"
+        updateProductDetail(targetSVNDirectory, targetSVNDirectoryMozilla, productName, version, svnURLpd, svnURLmozilla)