#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""desktop_l10n.py
This script manages Desktop repacks for nightly builds.
"""
import glob
import os
import shlex
import sys
# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0])) # noqa
from mozharness.base.errors import MakefileErrorList
from mozharness.base.script import BaseScript
from mozharness.base.vcs.vcsbase import VCSMixin
from mozharness.mozilla.automation import AutomationMixin
from mozharness.mozilla.building.buildbase import (
MakeUploadOutputParser,
get_mozconfig_path,
)
from mozharness.mozilla.l10n.locales import LocalesMixin
try:
import simplejson as json
assert json
except ImportError:
import json
# needed by _map
SUCCESS = 0
FAILURE = 1
SUCCESS_STR = "Success"
FAILURE_STR = "Failed"
# DesktopSingleLocale {{{1
class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript):
"""Manages desktop repacks"""
config_options = [
[
[
"--locale",
],
{
"action": "extend",
"dest": "locales",
"type": "string",
"help": "Specify the locale(s) to sign and update. Optionally pass"
" revision separated by colon, en-GB:default.",
},
],
[
[
"--tag-override",
],
{
"action": "store",
"dest": "tag_override",
"type": "string",
"help": "Override the tags set for all repos",
},
],
[
[
"--en-us-installer-url",
],
{
"action": "store",
"dest": "en_us_installer_url",
"type": "string",
"help": "Specify the url of the en-us binary",
},
],
]
def __init__(self, require_config_file=True):
# fxbuild style:
buildscript_kwargs = {
"all_actions": [
"clone-locales",
"list-locales",
"setup",
"repack",
"summary",
],
"config": {
"ignore_locales": ["en-US"],
"locales_dir": "browser/locales",
"log_name": "single_locale",
"hg_l10n_base": "https://hg.mozilla.org/l10n-central",
},
}
LocalesMixin.__init__(self)
BaseScript.__init__(
self,
config_options=self.config_options,
require_config_file=require_config_file,
**buildscript_kwargs
)
self.bootstrap_env = None
self.upload_env = None
self.upload_urls = {}
self.pushdate = None
# upload_files is a dictionary of files to upload, keyed by locale.
self.upload_files = {}
# Helper methods {{{2
def query_bootstrap_env(self):
"""returns the env for repacks"""
if self.bootstrap_env:
return self.bootstrap_env
config = self.config
abs_dirs = self.query_abs_dirs()
bootstrap_env = self.query_env(
partial_env=config.get("bootstrap_env"), replace_dict=abs_dirs
)
bootstrap_env["L10NBASEDIR"] = abs_dirs["abs_l10n_dir"]
if self.query_is_nightly():
# we might set update_channel explicitly
if config.get("update_channel"):
update_channel = config["update_channel"]
else: # Let's just give the generic channel based on branch.
update_channel = "nightly-%s" % (config["branch"],)
if not isinstance(update_channel, bytes):
update_channel = update_channel.encode("utf-8")
bootstrap_env["MOZ_UPDATE_CHANNEL"] = update_channel
self.info(
"Update channel set to: {}".format(bootstrap_env["MOZ_UPDATE_CHANNEL"])
)
self.bootstrap_env = bootstrap_env
return self.bootstrap_env
def _query_upload_env(self):
"""returns the environment used for the upload step"""
if self.upload_env:
return self.upload_env
config = self.config
upload_env = self.query_env(partial_env=config.get("upload_env"))
# check if there are any extra option from the platform configuration
# and append them to the env
if "upload_env_extra" in config:
for extra in config["upload_env_extra"]:
upload_env[extra] = config["upload_env_extra"][extra]
self.upload_env = upload_env
return self.upload_env
def query_l10n_env(self):
l10n_env = self._query_upload_env().copy()
l10n_env.update(self.query_bootstrap_env())
return l10n_env
def _query_make_variable(self, variable, make_args=None):
"""returns the value of make echo-variable-<variable>
it accepts extra make arguements (make_args)
"""
dirs = self.query_abs_dirs()
make_args = make_args or []
target = ["echo-variable-%s" % variable] + make_args
cwd = dirs["abs_locales_dir"]
raw_output = self._get_output_from_make(
target, cwd=cwd, env=self.query_bootstrap_env()
)
# we want to log all the messages from make
output = []
for line in raw_output.split("\n"):
output.append(line.strip())
output = " ".join(output).strip()
self.info("echo-variable-%s: %s" % (variable, output))
return output
def _map(self, func, items):
"""runs func for any item in items, calls the add_failure() for each
error. It assumes that function returns 0 when successful.
returns a two element tuple with (success_count, total_count)"""
success_count = 0
total_count = len(items)
name = func.__name__
for item in items:
result = func(item)
if result == SUCCESS:
# success!
success_count += 1
else:
# func failed...
message = "failure: %s(%s)" % (name, item)
self.add_failure(item, message)
return (success_count, total_count)
# Actions {{{2
def clone_locales(self):
self.pull_locale_source()
def setup(self):
"""setup step"""
self._run_tooltool()
self._copy_mozconfig()
self._mach_configure()
self._run_make_in_config_dir()
self.make_wget_en_US()
self.make_unpack_en_US()
def _run_make_in_config_dir(self):
"""this step creates nsinstall, needed my make_wget_en_US()"""
dirs = self.query_abs_dirs()
config_dir = os.path.join(dirs["abs_obj_dir"], "config")
env = self.query_bootstrap_env()
return self._make(target=["export"], cwd=config_dir, env=env)
def _copy_mozconfig(self):
"""copies the mozconfig file into abs_src_dir/.mozconfig
and logs the content
"""
config = self.config
dirs = self.query_abs_dirs()
src = get_mozconfig_path(self, config, dirs)
dst = os.path.join(dirs["abs_src_dir"], ".mozconfig")
self.copyfile(src, dst)
self.read_from_file(dst, verbose=True)
def _mach(self, target, env, halt_on_failure=True, output_parser=None):
dirs = self.query_abs_dirs()
mach = self._get_mach_executable()
return self.run_command(
mach + target,
halt_on_failure=True,
env=env,
cwd=dirs["abs_src_dir"],
output_parser=None,
)
def _mach_configure(self):
"""calls mach configure"""
env = self.query_bootstrap_env()
target = ["configure"]
return self._mach(target=target, env=env)
def _get_mach_executable(self):
return [sys.executable, "mach"]
def _make(
self,
target,
cwd,
env,
error_list=MakefileErrorList,
halt_on_failure=True,
output_parser=None,
):
"""Runs make. Returns the exit code"""
make = ["make"]
if target:
make = make + target
return self.run_command(
make,
cwd=cwd,
env=env,
error_list=error_list,
halt_on_failure=halt_on_failure,
output_parser=output_parser,
)
def _get_output_from_make(
self, target, cwd, env, halt_on_failure=True, ignore_errors=False
):
"""runs make and returns the output of the command"""
return self.get_output_from_command(
["make"] + target,
cwd=cwd,
env=env,
silent=True,
halt_on_failure=halt_on_failure,
ignore_errors=ignore_errors,
)
def make_unpack_en_US(self):
"""wrapper for make unpack"""
config = self.config
dirs = self.query_abs_dirs()
env = self.query_bootstrap_env()
cwd = os.path.join(dirs["abs_obj_dir"], config["locales_dir"])
return self._make(target=["unpack"], cwd=cwd, env=env)
def make_wget_en_US(self):
"""wrapper for make wget-en-US"""
env = self.query_bootstrap_env()
dirs = self.query_abs_dirs()
cwd = dirs["abs_locales_dir"]
return self._make(target=["wget-en-US"], cwd=cwd, env=env)
def make_upload(self, locale):
"""wrapper for make upload command"""
env = self.query_l10n_env()
dirs = self.query_abs_dirs()
target = ["upload", "AB_CD=%s" % (locale)]
cwd = dirs["abs_locales_dir"]
parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj)
retval = self._make(
target=target, cwd=cwd, env=env, halt_on_failure=False, output_parser=parser
)
if retval == SUCCESS:
self.info("Upload successful (%s)" % locale)
ret = SUCCESS
else:
self.error("failed to upload %s" % locale)
ret = FAILURE
if ret == FAILURE:
# If we failed above, we shouldn't even attempt a SIMPLE_NAME move
# even if we are configured to do so
return ret
# XXX Move the files to a SIMPLE_NAME format until we can enable
# Simple names in the build system
if self.config.get("simple_name_move"):
# Assume an UPLOAD PATH
upload_target = self.config["upload_env"]["UPLOAD_PATH"]
target_path = os.path.join(upload_target, locale)
self.mkdir_p(target_path)
glob_name = "*.%s.*" % locale
matches = (
glob.glob(os.path.join(upload_target, glob_name))
+ glob.glob(os.path.join(upload_target, "update", glob_name))
+ glob.glob(os.path.join(upload_target, "*", "xpi", glob_name))
+ glob.glob(os.path.join(upload_target, "install", "sea", glob_name))
+ glob.glob(os.path.join(upload_target, "setup.exe"))
+ glob.glob(os.path.join(upload_target, "setup-stub.exe"))
)
targets_exts = [
"tar.bz2",
"dmg",
"langpack.xpi",
"checksums",
"zip",
"installer.exe",
"installer-stub.exe",
]
targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts]
targets.extend([(f, f) for f in ("setup.exe", "setup-stub.exe")])
for f in matches:
possible_targets = [
(tail, target_file)
for (tail, target_file) in targets
if f.endswith(tail)
]
if len(possible_targets) == 1:
_, target_file = possible_targets[0]
# Remove from list of available options for this locale
targets.remove(possible_targets[0])
else:
# wasn't valid (or already matched)
raise RuntimeError(
"Unexpected matching file name encountered: %s" % f
)
self.move(os.path.join(f), os.path.join(target_path, target_file))
self.log("Converted uploads for %s to simple names" % locale)
return ret
def set_upload_files(self, locale):
# The tree doesn't have a good way of exporting the list of files
# created during locale generation, but we can grab them by echoing the
# UPLOAD_FILES variable for each locale.
env = self.query_l10n_env()
target = [
"echo-variable-UPLOAD_FILES",
"echo-variable-CHECKSUM_FILES",
"AB_CD=%s" % locale,
]
dirs = self.query_abs_dirs()
cwd = dirs["abs_locales_dir"]
# Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found
# we should ignore stderr as unfortunately it's expected when parsing for values
output = self._get_output_from_make(
target=target, cwd=cwd, env=env, ignore_errors=True
)
self.info('UPLOAD_FILES is "%s"' % output)
files = shlex.split(output)
if not files:
self.error("failed to get upload file list for locale %s" % locale)
return FAILURE
self.upload_files[locale] = [
os.path.abspath(os.path.join(cwd, f)) for f in files
]
return SUCCESS
def make_installers(self, locale):
"""wrapper for make installers-(locale)"""
env = self.query_l10n_env()
env["PYTHONIOENCODING"] = "utf-8"
self._copy_mozconfig()
dirs = self.query_abs_dirs()
cwd = os.path.join(dirs["abs_locales_dir"])
target = [
"installers-%s" % locale,
]
return self._make(target=target, cwd=cwd, env=env, halt_on_failure=False)
def repack_locale(self, locale):
"""wraps the logic for make installers and generating
complete updates."""
# run make installers
if self.make_installers(locale) != SUCCESS:
self.error("make installers-%s failed" % (locale))
return FAILURE
# now try to upload the artifacts
if self.make_upload(locale):
self.error("make upload for locale %s failed!" % (locale))
return FAILURE
# set_upload_files() should be called after make upload, to make sure
# we have all files in place (checksums, etc)
if self.set_upload_files(locale):
self.error("failed to get list of files to upload for locale %s" % locale)
return FAILURE
return SUCCESS
def repack(self):
"""creates the repacks and udpates"""
self._map(self.repack_locale, self.query_locales())
def _run_tooltool(self):
env = self.query_bootstrap_env()
config = self.config
dirs = self.query_abs_dirs()
manifest_src = os.environ.get("TOOLTOOL_MANIFEST")
if not manifest_src:
manifest_src = config.get("tooltool_manifest_src")
if not manifest_src:
return
python = sys.executable
cmd = [
python,
"-u",
os.path.join(dirs["abs_src_dir"], "mach"),
"artifact",
"toolchain",
"-v",
"--retry",
"4",
"--artifact-manifest",
os.path.join(dirs["abs_src_dir"], "toolchains.json"),
]
if manifest_src:
cmd.extend(
[
"--tooltool-manifest",
os.path.join(dirs["abs_src_dir"], manifest_src),
]
)
cache = config["bootstrap_env"].get("TOOLTOOL_CACHE")
if cache:
cmd.extend(["--cache-dir", cache])
self.info(str(cmd))
self.run_command(cmd, cwd=dirs["abs_src_dir"], halt_on_failure=True, env=env)
# main {{{
if __name__ == "__main__":
single_locale = DesktopSingleLocale()
single_locale.run_and_exit()
