python/l10n/mozxchannel/__init__.py
author Mozilla Releng Treescript <release+treescript@mozilla.org>
Tue, 02 Aug 2022 19:04:10 +0000
changeset 625720 335652eb938ddb0101b016b8e29b60feccdd24eb
parent 617754 08f6da5d0d467ff290a22fa9f2eb7ddd15c2acda
child 642650 77ab8bde7ce1be423ce223828d11edf8da2408a5
permissions -rw-r--r--
no bug - Bumping Firefox l10n changesets r=release a=l10n-bump DONTBUILD gn -> 534e108b825ab32b04666f8671e77634f483c6f4 kk -> 3ba121cb043c21c6b2084285b442f9a2aae6f7f8 tg -> 007157fc59cf0908dbf4688152ae324fc44afaff

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this,
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from collections import defaultdict
from dataclasses import dataclass, field
from datetime import datetime
import os
import shutil
from compare_locales import merge
import hglib
from pathlib import Path

from mozpack import path as mozpath
from . import projectconfig
from . import source


def get_default_config(topsrcdir, strings_path):
    assert isinstance(topsrcdir, Path)
    assert isinstance(strings_path, Path)
    return {
        "strings": {
            "path": strings_path,
            "url": "https://hg.mozilla.org/l10n/gecko-strings-quarantine/",
            "heads": {"default": "default"},
            "update_on_pull": True,
            "push_url": "ssh://hg.mozilla.org/l10n/gecko-strings-quarantine/",
        },
        "source": {
            "mozilla-unified": {
                "path": topsrcdir,
                "url": "https://hg.mozilla.org/mozilla-unified/",
                "heads": {
                    # This list of repositories is ordered, starting with the
                    # one with the most recent content (central) to the oldest
                    # (ESR). In case two ESR versions are supported, the oldest
                    # ESR goes last (e.g. esr78 goes after esr91).
                    "central": "mozilla-central",
                    "beta": "releases/mozilla-beta",
                    "release": "releases/mozilla-release",
                    "esr102": "releases/mozilla-esr102",
                },
                "config_files": [
                    "browser/locales/l10n.toml",
                    "mobile/android/locales/l10n.toml",
                ],
            },
            "comm-central": {
                "path": topsrcdir / "comm",
                "post-clobber": True,
                "url": "https://hg.mozilla.org/comm-central/",
                "heads": {
                    # This list of repositories is ordered, starting with the
                    # one with the most recent content (central) to the oldest
                    # (ESR). In case two ESR versions are supported, the oldest
                    # ESR goes last (e.g. esr78 goes after esr91).
                    "comm": "comm-central",
                    "comm-beta": "releases/comm-beta",
                    "comm-esr102": "releases/comm-esr102",
                },
                "config_files": [
                    "comm/calendar/locales/l10n.toml",
                    "comm/mail/locales/l10n.toml",
                    "comm/suite/locales/l10n.toml",
                ],
            },
        },
    }


@dataclass
class TargetRevs:
    target: bytes = None
    revs: list = field(default_factory=list)


@dataclass
class CommitRev:
    repo: str
    rev: bytes

    @property
    def message(self):
        return (
            f"X-Channel-Repo: {self.repo}\n"
            f'X-Channel-Revision: {self.rev.decode("ascii")}'
        )


class CrossChannelCreator:
    def __init__(self, config):
        self.config = config
        self.strings_path = config["strings"]["path"]
        self.message = (
            f"cross-channel content for {datetime.utcnow().strftime('%Y-%m-%d %H:%M')}"
        )

    def create_content(self):
        self.prune_target()
        revs = []
        for repo_name, repo_config in self.config["source"].items():
            with hglib.open(repo_config["path"]) as repo:
                revs.extend(self.create_for_repo(repo, repo_name, repo_config))
        self.commit(revs)
        return 0

    def prune_target(self):
        for leaf in self.config["strings"]["path"].iterdir():
            if leaf.name == ".hg":
                continue
            shutil.rmtree(leaf)

    def create_for_repo(self, repo, repo_name, repo_config):
        print(f"Processing {repo_name} in {repo_config['path']}")
        source_target_revs = defaultdict(TargetRevs)
        revs_for_commit = []
        parse_kwargs = {
            "env": {"l10n_base": str(self.strings_path.parent)},
            "ignore_missing_includes": True,
        }
        for head, head_name in repo_config["heads"].items():
            print(f"Gathering files for {head}")
            rev = repo.log(revrange=head)[0].node
            revs_for_commit.append(CommitRev(head_name, rev))
            p = source.HgTOMLParser(repo, rev)
            project_configs = []
            for config_file in repo_config["config_files"]:
                project_configs.append(p.parse(config_file, **parse_kwargs))
                project_configs[-1].set_locales(["en-US"], deep=True)
            hgfiles = source.HGFiles(repo, rev, project_configs)
            for targetpath, refpath, _, _ in hgfiles:
                source_target_revs[refpath].revs.append(rev)
                source_target_revs[refpath].target = targetpath
        root = repo.root()
        print(f"Writing {repo_name} content to target")
        for refpath, targetrevs in source_target_revs.items():
            local_ref = mozpath.relpath(refpath, root)
            content = self.get_content(local_ref, repo, targetrevs.revs)
            target_dir = mozpath.dirname(targetrevs.target)
            if not os.path.isdir(target_dir):
                os.makedirs(target_dir)
            with open(targetrevs.target, "wb") as fh:
                fh.write(content)
        return revs_for_commit

    def commit(self, revs):
        message = self.message + "\n\n"
        if "TASK_ID" in os.environ:
            message += f"X-Task-ID: {os.environ['TASK_ID']}\n\n"
        message += "\n".join(rev.message for rev in revs)
        with hglib.open(self.strings_path) as repo:
            repo.commit(message=message, addremove=True)

    def get_content(self, local_ref, repo, revs):
        if local_ref.endswith(b".toml"):
            return self.get_config_content(local_ref, repo, revs)
        if len(revs) < 2:
            return repo.cat([b"path:" + local_ref], rev=revs[0])
        contents = [repo.cat([b"path:" + local_ref], rev=rev) for rev in revs]
        try:
            return merge.merge_channels(local_ref.decode("utf-8"), contents)
        except merge.MergeNotSupportedError:
            return contents[0]

    def get_config_content(self, local_ref, repo, revs):
        # We don't support merging toml files
        content = repo.cat([b"path:" + local_ref], rev=revs[0])
        return projectconfig.process_config(content)